From c4e6a37782cf010b41494970e38869189e1ddf89 Mon Sep 17 00:00:00 2001 From: lecaros Date: Wed, 5 Aug 2026 11:27:33 -0400 Subject: [PATCH] packaging: pass the elements to jq by pipes instead of command line Signed-off-by: lecaros --- packaging/build-catalog.awk | 57 ++++++++++++--- packaging/generate-packages-index.sh | 26 ++++++- packaging/testing/test-packages-index.sh | 93 +++++++++++++++++++++++- 3 files changed, 161 insertions(+), 15 deletions(-) diff --git a/packaging/build-catalog.awk b/packaging/build-catalog.awk index b5a388d5f59..2ed28a5f57d 100644 --- a/packaging/build-catalog.awk +++ b/packaging/build-catalog.awk @@ -1,11 +1,41 @@ BEGIN { nrepos = split(repo_paths, repos, " ") + if (min_catalog_major !~ /^[0-9]+$/) { + print "ERROR: min_catalog_major must be a nonnegative integer: " min_catalog_major > "/dev/stderr" + exit 1 + } + min_catalog_major = min_catalog_major + 0 } function is_version(v) { return v ~ /^[0-9]+\.[0-9]+(\.[0-9]+)*$/ } +function version_major(v, parts) { + split(v, parts, ".") + return parts[1] + 0 +} + +function version_in_catalog(v) { + if (!is_version(v)) { + return 0 + } + if (min_catalog_major == 0) { + return 1 + } + return version_major(v) >= min_catalog_major +} + +function path_in_catalog(path, n, name, v, parts) { + n = split(path, parts, "/") + name = parts[n] + v = extract_version(name) + if (v == "") { + return 1 + } + return version_in_catalog(v) +} + function schema_version(name) { if (name !~ /^fluent-bit-schema-[0-9]+\.[0-9]+(\.[0-9]+)*\.json$/) { return "" @@ -49,10 +79,6 @@ function extract_version(name, rest, i, c, following) { return rest } -function version_includes_linux(v) { - return v ~ /^[45]\./ -} - function repo_prefix(path, i, p) { for (i = 1; i <= nrepos; i++) { p = repos[i] @@ -117,7 +143,7 @@ function json_url(path) { } function note_linux(v, path, name, repo, key, label, idx) { - if (!version_includes_linux(v)) { + if (!version_in_catalog(v)) { return } key = v SUBSEP path @@ -212,12 +238,20 @@ function emit_version(v, out, i, repo_key, repo, repos_out, repos_n) { path = $0 sub(/\r$/, "", path) + + if (emit_mode == "filter") { + if (path_in_catalog(path)) { + print path + } + next + } + n = split(path, parts, "/") name = parts[n] if (name ~ /^fluent-bit-schema-[0-9]+\.[0-9]+(\.[0-9]+)*\.json$/) { v = schema_version(name) - if (is_version(v)) { + if (version_in_catalog(v)) { versions[v] = 1 schema[v] = path } @@ -227,7 +261,7 @@ function emit_version(v, out, i, repo_key, repo, repos_out, repos_n) { if (path ~ /^windows\//) { v = extract_version(name) key = windows_key(name, v) - if (key != "") { + if (key != "" && version_in_catalog(v)) { versions[v] = 1 windows[v, key] = path } @@ -236,7 +270,7 @@ function emit_version(v, out, i, repo_key, repo, repos_out, repos_n) { if (path ~ /^macos\//) { v = extract_version(name) - if (v == "") { + if (v == "" || !version_in_catalog(v)) { next } versions[v] = 1 @@ -254,7 +288,7 @@ function emit_version(v, out, i, repo_key, repo, repos_out, repos_n) { next } v = extract_version(name) - if (!is_version(v)) { + if (!version_in_catalog(v)) { next } versions[v] = 1 @@ -264,8 +298,11 @@ function emit_version(v, out, i, repo_key, repo, repos_out, repos_n) { END { mode = emit_mode + if (mode == "filter") { + exit 0 + } for (v in versions) { - if (!is_version(v)) { + if (!version_in_catalog(v)) { continue } if (mode == "versions") { diff --git a/packaging/generate-packages-index.sh b/packaging/generate-packages-index.sh index 2029446346a..33d93deba8a 100755 --- a/packaging/generate-packages-index.sh +++ b/packaging/generate-packages-index.sh @@ -12,6 +12,7 @@ AWS_S3_BUCKET=${AWS_S3_BUCKET:-} AWS_S3_REMOTE_DISCOVERY=${AWS_S3_REMOTE_DISCOVERY:-false} AWS_S3_NO_SIGN_REQUEST=${AWS_S3_NO_SIGN_REQUEST:-true} AWS_S3_ENDPOINT=${AWS_S3_ENDPOINT:-} +MIN_CATALOG_MAJOR=${MIN_CATALOG_MAJOR:-3} WORK_DIR="" OBJECT_LIST="" @@ -57,6 +58,7 @@ Environment variables: AWS_S3_ENDPOINT Optional custom S3 endpoint URL GITHUB_REPO GitHub repository URL for release links DOCS_URL Installation documentation URL + MIN_CATALOG_MAJOR Minimum major version in catalog (default: 3, 0 = all) Example: BASE_PATH=./catalog ./generate-packages-index.sh @@ -96,6 +98,11 @@ if ! command -v tree >/dev/null 2>&1; then exit 1 fi +if [[ ! "$MIN_CATALOG_MAJOR" =~ ^[0-9]+$ ]]; then + echo "ERROR: MIN_CATALOG_MAJOR must be a nonnegative integer: $MIN_CATALOG_MAJOR" >&2 + exit 1 +fi + BASE_PATH="$(cd "$BASE_PATH" && pwd)" WORK_DIR="$(mktemp -d)" OBJECT_LIST="$WORK_DIR/objects.txt" @@ -203,6 +210,17 @@ fi grep -Ev 'source-|pool|dists' "$OBJECT_LIST" | \ grep -E '\.(rpm|deb|key|repo|exe|msi|zip|pkg)$' > "$TREE_LIST" || true +awk -v emit_mode=filter \ + -v min_catalog_major="$MIN_CATALOG_MAJOR" \ + -f "$SCRIPT_DIR/build-catalog.awk" \ + "$TREE_LIST" > "$WORK_DIR/tree-filtered.txt" + +if [[ -s "$WORK_DIR/tree-filtered.txt" ]]; then + mv "$WORK_DIR/tree-filtered.txt" "$TREE_LIST" +else + : > "$TREE_LIST" +fi + if [[ ! -s "$TREE_LIST" ]]; then echo "ERROR: no package files found for index.html" >&2 exit 1 @@ -217,6 +235,7 @@ awk -v emit_mode=versions \ -v github_release="$GITHUB_REPO" \ -v docs_url="$DOCS_URL" \ -v repo_paths="$REPO_PATHS" \ + -v min_catalog_major="$MIN_CATALOG_MAJOR" \ -f "$SCRIPT_DIR/build-catalog.awk" \ "$OBJECT_LIST" | sort -t $'\t' -V -k1,1 > "$VERSION_ROWS" @@ -249,16 +268,15 @@ fi GENERATED_AT="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" -jq -n \ +awk -F '\t' '{ print $2 }' "$VERSION_ROWS" | jq -s \ --arg latest "$LATEST" \ --arg generated_at "$GENERATED_AT" \ --arg base_url "$BASE_URL" \ - --argjson versions "$(awk -F '\t' '{ print $2 }' "$VERSION_ROWS" | jq -s 'reverse')" \ - '{ + 'reverse | { latest: $latest, generated_at: $generated_at, base_url: $base_url, - versions: $versions + versions: . }' > "$BASE_PATH/versions.json" printf '%s\n' "$LATEST" > "$BASE_PATH/latest-version.txt" diff --git a/packaging/testing/test-packages-index.sh b/packaging/testing/test-packages-index.sh index 65337e4d201..0c0da50dff1 100755 --- a/packaging/testing/test-packages-index.sh +++ b/packaging/testing/test-packages-index.sh @@ -10,7 +10,8 @@ S3_LISTING_FILE="" cleanup() { rm -rf "$OUTPUT_DIR" "${OUTPUT_DIR_S3:-}" "${OUTPUT_DIR_OVERRIDE:-}" \ - "${EMPTY_BASE_PATH:-}" "$S3_LISTING_FILE" + "${OUTPUT_DIR_MIN:-}" "${OUTPUT_DIR_ALL:-}" \ + "${EMPTY_BASE_PATH:-}" "${JQ_STRESS_DIR:-}" "$S3_LISTING_FILE" } trap cleanup EXIT @@ -48,6 +49,16 @@ setup_local_fixture() "$root/windows/fluent-bit-4.2.7-win64.zip" } +setup_local_fixture_with_legacy() +{ + local root="$1" + + setup_local_fixture "$root" + touch \ + "$root/centos/9/fluent-bit-2.1.0-1.x86_64.rpm" \ + "$root/windows/fluent-bit-2.1.0-win64.exe" +} + setup_s3_listing_fixture() { S3_LISTING_FILE="$(mktemp)" @@ -237,3 +248,83 @@ if ! jq -e '.latest == "4.2.7"' "$PREVIEW_DIR/versions.json" >/dev/null; then fi echo "packages-index preview script test passed" + +OUTPUT_DIR_MIN="$(mktemp -d)" +setup_local_fixture_with_legacy "$OUTPUT_DIR_MIN" + +BASE_PATH="$OUTPUT_DIR_MIN" \ +AWS_S3_REMOTE_DISCOVERY=false \ +BASE_URL=https://packages.example.test \ +"$GENERATOR" + +if jq -e '.versions[] | select(.version == "2.1.0")' "$OUTPUT_DIR_MIN/versions.json" >/dev/null; then + echo "ERROR: versions.json should exclude 2.1.0 with default MIN_CATALOG_MAJOR" >&2 + exit 1 +fi + +if grep -Fq "fluent-bit-2.1.0" "$OUTPUT_DIR_MIN/index.html"; then + echo "ERROR: index.html should exclude 2.1.0 packages with default MIN_CATALOG_MAJOR" >&2 + exit 1 +fi + +echo "packages-index min catalog major test passed" + +OUTPUT_DIR_ALL="$(mktemp -d)" +setup_local_fixture_with_legacy "$OUTPUT_DIR_ALL" + +BASE_PATH="$OUTPUT_DIR_ALL" \ +AWS_S3_REMOTE_DISCOVERY=false \ +MIN_CATALOG_MAJOR=0 \ +BASE_URL=https://packages.example.test \ +"$GENERATOR" + +if ! jq -e '.versions[] | select(.version == "2.1.0")' "$OUTPUT_DIR_ALL/versions.json" >/dev/null; then + echo "ERROR: versions.json should include 2.1.0 when MIN_CATALOG_MAJOR=0" >&2 + exit 1 +fi + +if ! jq -e '.versions[] | select(.version == "2.1.0") | .artifacts.linux[] | select(.name == "fluent-bit-2.1.0-1.x86_64.rpm")' \ + "$OUTPUT_DIR_ALL/versions.json" >/dev/null; then + echo "ERROR: versions.json missing Linux RPM artifact for 2.1.0 when MIN_CATALOG_MAJOR=0" >&2 + exit 1 +fi + +assert_contains "$OUTPUT_DIR_ALL/index.html" "fluent-bit-2.1.0-win64.exe" + +echo "packages-index disabled min catalog major test passed" + +assert_fails_with "MIN_CATALOG_MAJOR must be a nonnegative integer" \ + env BASE_PATH="$OUTPUT_DIR" AWS_S3_REMOTE_DISCOVERY=false MIN_CATALOG_MAJOR=-1 \ + BASE_URL=https://packages.example.test "$GENERATOR" + +assert_fails_with "MIN_CATALOG_MAJOR must be a nonnegative integer" \ + env BASE_PATH="$OUTPUT_DIR" AWS_S3_REMOTE_DISCOVERY=false MIN_CATALOG_MAJOR=abc \ + BASE_URL=https://packages.example.test "$GENERATOR" + +echo "packages-index min catalog major validation test passed" + +JQ_STRESS_DIR="$(mktemp -d)" +VERSION_ROWS="$JQ_STRESS_DIR/version-rows.tsv" +i=0 +while [[ "$i" -lt 500 ]]; do + i=$((i + 1)) + printf '3.0.%s\t{"version":"3.0.%s"}\n' "$i" "$i" +done > "$VERSION_ROWS" + +awk -F '\t' '{ print $2 }' "$VERSION_ROWS" | jq -s \ + --arg latest "3.0.500" \ + --arg generated_at "2026-01-01T00:00:00Z" \ + --arg base_url "https://packages.example.test" \ + 'reverse | { + latest: $latest, + generated_at: $generated_at, + base_url: $base_url, + versions: . + }' > "$JQ_STRESS_DIR/versions.json" + +if ! jq -e '.latest == "3.0.500" and (.versions | length) == 500' "$JQ_STRESS_DIR/versions.json" >/dev/null; then + echo "ERROR: large versions.json envelope generation failed" >&2 + exit 1 +fi + +echo "packages-index large versions.json test passed"