diff --git a/packaging/generate-packages-index.sh b/packaging/generate-packages-index.sh index 33d93deba8a..3496f4aa1e9 100755 --- a/packaging/generate-packages-index.sh +++ b/packaging/generate-packages-index.sh @@ -106,7 +106,7 @@ fi BASE_PATH="$(cd "$BASE_PATH" && pwd)" WORK_DIR="$(mktemp -d)" OBJECT_LIST="$WORK_DIR/objects.txt" -TREE_LIST="$WORK_DIR/tree.txt" +TREE_PATHS="$WORK_DIR/catalog-paths.txt" VERSION_ROWS="$WORK_DIR/version-rows.tsv" trap cleanup EXIT @@ -190,6 +190,101 @@ build_object_list() sort -u -o "$OBJECT_LIST" "$OBJECT_LIST" } +filter_package_paths() +{ + local candidates="$WORK_DIR/package-candidates.txt" + local rc=0 + + grep -Ev 'source-|pool|dists' "$OBJECT_LIST" > "$candidates" + rc=$? + if [[ "$rc" -ne 0 && "$rc" -ne 1 ]]; then + echo "ERROR: failed filtering package paths from object list" >&2 + exit 1 + fi + if [[ "$rc" -eq 1 ]]; then + : > "$candidates" + fi + + grep -E '\.(rpm|deb|key|repo|exe|msi|zip|pkg)$' "$candidates" > "$TREE_PATHS" + rc=$? + if [[ "$rc" -ne 0 && "$rc" -ne 1 ]]; then + echo "ERROR: failed selecting package file paths" >&2 + exit 1 + fi + if [[ "$rc" -eq 1 ]]; then + : > "$TREE_PATHS" + fi +} + +absolutize_tree_hrefs() +{ + awk -v base="$BASE_URL" ' + function absolutize(url) { + if (url ~ /^https?:\/\//) { + return url + } + if (url == "/") { + return base "/" + } + if (substr(url, 1, 1) == "/") { + return base url + } + return base "/" url + } + { + line = $0 + out = "" + while (match(line, /href="[^"]+"/)) { + out = out substr(line, 1, RSTART - 1) + href = substr(line, RSTART, RLENGTH) + url = substr(href, 7, length(href) - 7) + out = out "href=\"" absolutize(url) "\"" + line = substr(line, RSTART + RLENGTH) + } + print out line + }' +} + +build_tree_html() +{ + local -a tree_pipe_status=() + + BASE_URL="${BASE_URL%/}" + ( + cd "$WORK_DIR" + tree --noreport --charset utf-8 --fromfile catalog-paths.txt -H "${BASE_URL}/" + ) | sed -e "s|${BASE_URL}/catalog-paths.txt/|${BASE_URL}/|g" \ + -e "s|${BASE_URL}/catalog-paths.txt|${BASE_URL}/|g" \ + -e "s|${BASE_URL}catalog-paths.txt/|${BASE_URL}/|g" \ + -e "s|${BASE_URL}catalog-paths.txt|${BASE_URL}/|g" \ + -e 's|>catalog-paths.txt|>/|g' \ + -e 's|\(https://[^/]*\)//|\1/|g' \ + | absolutize_tree_hrefs \ + | awk '/
/ { exit } { print } END { print "" }' > "$WORK_DIR/tree.html" + tree_pipe_status=("${PIPESTATUS[@]}") + + if [[ "${tree_pipe_status[0]}" -ne 0 ]]; then + echo "ERROR: tree failed to generate package index HTML" >&2 + exit 1 + fi + if [[ "${tree_pipe_status[1]}" -ne 0 ]]; then + echo "ERROR: failed to normalize package index tree links" >&2 + exit 1 + fi + if [[ "${tree_pipe_status[2]}" -ne 0 ]]; then + echo "ERROR: failed to absolutize package index tree links" >&2 + exit 1 + fi + if [[ "${tree_pipe_status[3]}" -ne 0 ]]; then + echo "ERROR: failed to finalize package index tree HTML" >&2 + exit 1 + fi + if [[ ! -s "$WORK_DIR/tree.html" ]]; then + echo "ERROR: package index tree HTML is empty" >&2 + exit 1 + fi +} + render_index_html() { local latest="$1" @@ -207,27 +302,25 @@ if [[ ! -s "$OBJECT_LIST" ]]; then exit 1 fi -grep -Ev 'source-|pool|dists' "$OBJECT_LIST" | \ - grep -E '\.(rpm|deb|key|repo|exe|msi|zip|pkg)$' > "$TREE_LIST" || true +filter_package_paths awk -v emit_mode=filter \ -v min_catalog_major="$MIN_CATALOG_MAJOR" \ -f "$SCRIPT_DIR/build-catalog.awk" \ - "$TREE_LIST" > "$WORK_DIR/tree-filtered.txt" + "$TREE_PATHS" > "$WORK_DIR/tree-filtered.txt" if [[ -s "$WORK_DIR/tree-filtered.txt" ]]; then - mv "$WORK_DIR/tree-filtered.txt" "$TREE_LIST" + mv "$WORK_DIR/tree-filtered.txt" "$TREE_PATHS" else - : > "$TREE_LIST" + : > "$TREE_PATHS" fi -if [[ ! -s "$TREE_LIST" ]]; then +if [[ ! -s "$TREE_PATHS" ]]; then echo "ERROR: no package files found for index.html" >&2 exit 1 fi -tree --noreport --charset utf-8 --fromfile "$TREE_LIST" -H "$BASE_URL" | \ - awk '/
/ { exit } { print } END { print "" }' > "$WORK_DIR/tree.html" +build_tree_html REPO_PATHS="$(printf '%s ' "${LINUX_REPO_PATHS[@]}")" awk -v emit_mode=versions \ diff --git a/packaging/testing/test-packages-index.sh b/packaging/testing/test-packages-index.sh index 0c0da50dff1..d082b8443a2 100755 --- a/packaging/testing/test-packages-index.sh +++ b/packaging/testing/test-packages-index.sh @@ -10,7 +10,8 @@ S3_LISTING_FILE="" cleanup() { rm -rf "$OUTPUT_DIR" "${OUTPUT_DIR_S3:-}" "${OUTPUT_DIR_OVERRIDE:-}" \ - "${OUTPUT_DIR_MIN:-}" "${OUTPUT_DIR_ALL:-}" \ + "${OUTPUT_DIR_MIN:-}" "${OUTPUT_DIR_ALL:-}" "${OUTPUT_DIR_TRAILING:-}" \ + "${OUTPUT_DIR_FILTERED:-}" \ "${EMPTY_BASE_PATH:-}" "${JQ_STRESS_DIR:-}" "$S3_LISTING_FILE" } @@ -142,6 +143,47 @@ assert_contains "$OUTPUT_DIR/index.html" "Latest release: 4.2.7 assert_contains "$OUTPUT_DIR/index.html" "4.2.6" assert_contains "$OUTPUT_DIR/index.html" "fluent-bit-4.2.7-win64.exe" assert_contains "$OUTPUT_DIR/index.html" "https://packages.example.test/versions.json" +assert_contains "$OUTPUT_DIR/index.html" \ + 'href="https://packages.example.test/centos/9/fluent-bit-4.2.7-1.x86_64.rpm"' + +if grep -Eo 'href="[^"]*(catalog-paths\.txt|/tree\.txt/)' "$OUTPUT_DIR/index.html" >/dev/null; then + echo "ERROR: index.html contains broken tree listing paths in href attributes" >&2 + grep -Eo 'href="[^"]*(catalog-paths\.txt|/tree\.txt/)' "$OUTPUT_DIR/index.html" >&2 || true + exit 1 +fi + +OUTPUT_DIR_TRAILING="$(mktemp -d)" +setup_local_fixture "$OUTPUT_DIR_TRAILING" + +BASE_PATH="$OUTPUT_DIR_TRAILING" \ +AWS_S3_REMOTE_DISCOVERY=false \ +BASE_URL=https://packages.example.test/ \ +"$GENERATOR" + +assert_contains "$OUTPUT_DIR_TRAILING/index.html" \ + 'href="https://packages.example.test/centos/9/fluent-bit-4.2.7-1.x86_64.rpm"' + +if grep -Eo 'href="https://[^/]+//[^"]*"' "$OUTPUT_DIR_TRAILING/index.html" >/dev/null; then + echo "ERROR: index.html contains double slashes in href attributes" >&2 + grep -Eo 'href="https://[^/]+//[^"]*"' "$OUTPUT_DIR_TRAILING/index.html" >&2 || true + exit 1 +fi + +if ! jq -e '.base_url == "https://packages.example.test"' "$OUTPUT_DIR_TRAILING/versions.json" >/dev/null; then + echo "ERROR: versions.json base_url was not normalized without trailing slash" >&2 + jq . "$OUTPUT_DIR_TRAILING/versions.json" >&2 || true + exit 1 +fi + +OUTPUT_DIR_FILTERED="$(mktemp -d)" +mkdir -p "$OUTPUT_DIR_FILTERED/centos/9" "$OUTPUT_DIR_FILTERED/windows" +touch \ + "$OUTPUT_DIR_FILTERED/centos/9/fluent-bit-2.1.0-1.x86_64.rpm" \ + "$OUTPUT_DIR_FILTERED/windows/fluent-bit-2.1.0-win64.exe" + +assert_fails_with "ERROR: no package files found for index.html" \ + env BASE_PATH="$OUTPUT_DIR_FILTERED" AWS_S3_REMOTE_DISCOVERY=false \ + BASE_URL=https://packages.example.test "$GENERATOR" if ! jq -e '.latest == "4.2.7" and (.versions | length) == 2' "$OUTPUT_DIR/versions.json" >/dev/null; then echo "ERROR: versions.json did not contain the expected version entries" >&2