packaging: pass the elements to jq by pipes instead of command line

Signed-off-by: lecaros <lecaros@chronosphere.io>

Author: lecaros

packaging/build-catalog.awk

@@ -1,11 +1,41 @@
 BEGIN {
     nrepos = split(repo_paths, repos, " ")
+    if (min_catalog_major !~ /^[0-9]+$/) {
+        print "ERROR: min_catalog_major must be a nonnegative integer: " min_catalog_major > "/dev/stderr"
+        exit 1
+    }
+    min_catalog_major = min_catalog_major + 0
 }
 
 function is_version(v) {
     return v ~ /^[0-9]+\.[0-9]+(\.[0-9]+)*$/
 }
 
+function version_major(v,    parts) {
+    split(v, parts, ".")
+    return parts[1] + 0
+}
+
+function version_in_catalog(v) {
+    if (!is_version(v)) {
+        return 0
+    }
+    if (min_catalog_major == 0) {
+        return 1
+    }
+    return version_major(v) >= min_catalog_major
+}
+
+function path_in_catalog(path,    n, name, v, parts) {
+    n = split(path, parts, "/")
+    name = parts[n]
+    v = extract_version(name)
+    if (v == "") {
+        return 1
+    }
+    return version_in_catalog(v)
+}
+
 function schema_version(name) {
     if (name !~ /^fluent-bit-schema-[0-9]+\.[0-9]+(\.[0-9]+)*\.json$/) {
         return ""
@@ -49,10 +79,6 @@ function extract_version(name,    rest, i, c, following) {
     return rest
 }
 
-function version_includes_linux(v) {
-    return v ~ /^[45]\./
-}
-
 function repo_prefix(path,    i, p) {
     for (i = 1; i <= nrepos; i++) {
         p = repos[i]
@@ -117,7 +143,7 @@ function json_url(path) {
 }
 
 function note_linux(v, path, name, repo,    key, label, idx) {
-    if (!version_includes_linux(v)) {
+    if (!version_in_catalog(v)) {
         return
     }
     key = v SUBSEP path
@@ -212,12 +238,20 @@ function emit_version(v,    out, i, repo_key, repo, repos_out, repos_n) {
 
     path = $0
     sub(/\r$/, "", path)
+
+    if (emit_mode == "filter") {
+        if (path_in_catalog(path)) {
+            print path
+        }
+        next
+    }
+
     n = split(path, parts, "/")
     name = parts[n]
 
     if (name ~ /^fluent-bit-schema-[0-9]+\.[0-9]+(\.[0-9]+)*\.json$/) {
         v = schema_version(name)
-        if (is_version(v)) {
+        if (version_in_catalog(v)) {
             versions[v] = 1
             schema[v] = path
         }
@@ -227,7 +261,7 @@ function emit_version(v,    out, i, repo_key, repo, repos_out, repos_n) {
     if (path ~ /^windows\//) {
         v = extract_version(name)
         key = windows_key(name, v)
-        if (key != "") {
+        if (key != "" && version_in_catalog(v)) {
             versions[v] = 1
             windows[v, key] = path
         }
@@ -236,7 +270,7 @@ function emit_version(v,    out, i, repo_key, repo, repos_out, repos_n) {
 
     if (path ~ /^macos\//) {
         v = extract_version(name)
-        if (v == "") {
+        if (v == "" || !version_in_catalog(v)) {
             next
         }
         versions[v] = 1
@@ -254,7 +288,7 @@ function emit_version(v,    out, i, repo_key, repo, repos_out, repos_n) {
             next
         }
         v = extract_version(name)
-        if (!is_version(v)) {
+        if (!version_in_catalog(v)) {
             next
         }
         versions[v] = 1
@@ -264,8 +298,11 @@ function emit_version(v,    out, i, repo_key, repo, repos_out, repos_n) {
 
 END {
     mode = emit_mode
+    if (mode == "filter") {
+        exit 0
+    }
     for (v in versions) {
-        if (!is_version(v)) {
+        if (!version_in_catalog(v)) {
             continue
         }
         if (mode == "versions") {

packaging/generate-packages-index.sh

@@ -12,6 +12,7 @@ AWS_S3_BUCKET=${AWS_S3_BUCKET:-}
 AWS_S3_REMOTE_DISCOVERY=${AWS_S3_REMOTE_DISCOVERY:-false}
 AWS_S3_NO_SIGN_REQUEST=${AWS_S3_NO_SIGN_REQUEST:-true}
 AWS_S3_ENDPOINT=${AWS_S3_ENDPOINT:-}
+MIN_CATALOG_MAJOR=${MIN_CATALOG_MAJOR:-3}
 
 WORK_DIR=""
 OBJECT_LIST=""
@@ -57,6 +58,7 @@ Environment variables:
   AWS_S3_ENDPOINT         Optional custom S3 endpoint URL
   GITHUB_REPO             GitHub repository URL for release links
   DOCS_URL                Installation documentation URL
+  MIN_CATALOG_MAJOR       Minimum major version in catalog (default: 3, 0 = all)
 
 Example:
   BASE_PATH=./catalog ./generate-packages-index.sh
@@ -96,6 +98,11 @@ if ! command -v tree >/dev/null 2>&1; then
     exit 1
 fi
 
+if [[ ! "$MIN_CATALOG_MAJOR" =~ ^[0-9]+$ ]]; then
+    echo "ERROR: MIN_CATALOG_MAJOR must be a nonnegative integer: $MIN_CATALOG_MAJOR" >&2
+    exit 1
+fi
+
 BASE_PATH="$(cd "$BASE_PATH" && pwd)"
 WORK_DIR="$(mktemp -d)"
 OBJECT_LIST="$WORK_DIR/objects.txt"
@@ -203,6 +210,17 @@ fi
 grep -Ev 'source-|pool|dists' "$OBJECT_LIST" | \
     grep -E '\.(rpm|deb|key|repo|exe|msi|zip|pkg)$' > "$TREE_LIST" || true
 
+awk -v emit_mode=filter \
+    -v min_catalog_major="$MIN_CATALOG_MAJOR" \
+    -f "$SCRIPT_DIR/build-catalog.awk" \
+    "$TREE_LIST" > "$WORK_DIR/tree-filtered.txt"
+
+if [[ -s "$WORK_DIR/tree-filtered.txt" ]]; then
+    mv "$WORK_DIR/tree-filtered.txt" "$TREE_LIST"
+else
+    : > "$TREE_LIST"
+fi
+
 if [[ ! -s "$TREE_LIST" ]]; then
     echo "ERROR: no package files found for index.html" >&2
     exit 1
@@ -217,6 +235,7 @@ awk -v emit_mode=versions \
     -v github_release="$GITHUB_REPO" \
     -v docs_url="$DOCS_URL" \
     -v repo_paths="$REPO_PATHS" \
+    -v min_catalog_major="$MIN_CATALOG_MAJOR" \
     -f "$SCRIPT_DIR/build-catalog.awk" \
     "$OBJECT_LIST" | sort -t $'\t' -V -k1,1 > "$VERSION_ROWS"
 
@@ -249,16 +268,15 @@ fi
 
 GENERATED_AT="$(date -u +"%Y-%m-%dT%H:%M:%SZ")"
 
-jq -n \
+awk -F '\t' '{ print $2 }' "$VERSION_ROWS" | jq -s \
     --arg latest "$LATEST" \
     --arg generated_at "$GENERATED_AT" \
     --arg base_url "$BASE_URL" \
-    --argjson versions "$(awk -F '\t' '{ print $2 }' "$VERSION_ROWS" | jq -s 'reverse')" \
-    '{
+    'reverse | {
         latest: $latest,
         generated_at: $generated_at,
         base_url: $base_url,
-        versions: $versions
+        versions: .
     }' > "$BASE_PATH/versions.json"
 
 printf '%s\n' "$LATEST" > "$BASE_PATH/latest-version.txt"

packaging/testing/test-packages-index.sh

@@ -10,7 +10,8 @@ S3_LISTING_FILE=""
 cleanup()
 {
     rm -rf "$OUTPUT_DIR" "${OUTPUT_DIR_S3:-}" "${OUTPUT_DIR_OVERRIDE:-}" \
-        "${EMPTY_BASE_PATH:-}" "$S3_LISTING_FILE"
+        "${OUTPUT_DIR_MIN:-}" "${OUTPUT_DIR_ALL:-}" \
+        "${EMPTY_BASE_PATH:-}" "${JQ_STRESS_DIR:-}" "$S3_LISTING_FILE"
 }
 
 trap cleanup EXIT
@@ -48,6 +49,16 @@ setup_local_fixture()
         "$root/windows/fluent-bit-4.2.7-win64.zip"
 }
 
+setup_local_fixture_with_legacy()
+{
+    local root="$1"
+
+    setup_local_fixture "$root"
+    touch \
+        "$root/centos/9/fluent-bit-2.1.0-1.x86_64.rpm" \
+        "$root/windows/fluent-bit-2.1.0-win64.exe"
+}
+
 setup_s3_listing_fixture()
 {
     S3_LISTING_FILE="$(mktemp)"
@@ -237,3 +248,83 @@ if ! jq -e '.latest == "4.2.7"' "$PREVIEW_DIR/versions.json" >/dev/null; then
 fi
 
 echo "packages-index preview script test passed"
+
+OUTPUT_DIR_MIN="$(mktemp -d)"
+setup_local_fixture_with_legacy "$OUTPUT_DIR_MIN"
+
+BASE_PATH="$OUTPUT_DIR_MIN" \
+AWS_S3_REMOTE_DISCOVERY=false \
+BASE_URL=https://packages.example.test \
+"$GENERATOR"
+
+if jq -e '.versions[] | select(.version == "2.1.0")' "$OUTPUT_DIR_MIN/versions.json" >/dev/null; then
+    echo "ERROR: versions.json should exclude 2.1.0 with default MIN_CATALOG_MAJOR" >&2
+    exit 1
+fi
+
+if grep -Fq "fluent-bit-2.1.0" "$OUTPUT_DIR_MIN/index.html"; then
+    echo "ERROR: index.html should exclude 2.1.0 packages with default MIN_CATALOG_MAJOR" >&2
+    exit 1
+fi
+
+echo "packages-index min catalog major test passed"
+
+OUTPUT_DIR_ALL="$(mktemp -d)"
+setup_local_fixture_with_legacy "$OUTPUT_DIR_ALL"
+
+BASE_PATH="$OUTPUT_DIR_ALL" \
+AWS_S3_REMOTE_DISCOVERY=false \
+MIN_CATALOG_MAJOR=0 \
+BASE_URL=https://packages.example.test \
+"$GENERATOR"
+
+if ! jq -e '.versions[] | select(.version == "2.1.0")' "$OUTPUT_DIR_ALL/versions.json" >/dev/null; then
+    echo "ERROR: versions.json should include 2.1.0 when MIN_CATALOG_MAJOR=0" >&2
+    exit 1
+fi
+
+if ! jq -e '.versions[] | select(.version == "2.1.0") | .artifacts.linux[] | select(.name == "fluent-bit-2.1.0-1.x86_64.rpm")' \
+    "$OUTPUT_DIR_ALL/versions.json" >/dev/null; then
+    echo "ERROR: versions.json missing Linux RPM artifact for 2.1.0 when MIN_CATALOG_MAJOR=0" >&2
+    exit 1
+fi
+
+assert_contains "$OUTPUT_DIR_ALL/index.html" "fluent-bit-2.1.0-win64.exe"
+
+echo "packages-index disabled min catalog major test passed"
+
+assert_fails_with "MIN_CATALOG_MAJOR must be a nonnegative integer" \
+    env BASE_PATH="$OUTPUT_DIR" AWS_S3_REMOTE_DISCOVERY=false MIN_CATALOG_MAJOR=-1 \
+    BASE_URL=https://packages.example.test "$GENERATOR"
+
+assert_fails_with "MIN_CATALOG_MAJOR must be a nonnegative integer" \
+    env BASE_PATH="$OUTPUT_DIR" AWS_S3_REMOTE_DISCOVERY=false MIN_CATALOG_MAJOR=abc \
+    BASE_URL=https://packages.example.test "$GENERATOR"
+
+echo "packages-index min catalog major validation test passed"
+
+JQ_STRESS_DIR="$(mktemp -d)"
+VERSION_ROWS="$JQ_STRESS_DIR/version-rows.tsv"
+i=0
+while [[ "$i" -lt 500 ]]; do
+    i=$((i + 1))
+    printf '3.0.%s\t{"version":"3.0.%s"}\n' "$i" "$i"
+done > "$VERSION_ROWS"
+
+awk -F '\t' '{ print $2 }' "$VERSION_ROWS" | jq -s \
+    --arg latest "3.0.500" \
+    --arg generated_at "2026-01-01T00:00:00Z" \
+    --arg base_url "https://packages.example.test" \
+    'reverse | {
+        latest: $latest,
+        generated_at: $generated_at,
+        base_url: $base_url,
+        versions: .
+    }' > "$JQ_STRESS_DIR/versions.json"
+
+if ! jq -e '.latest == "3.0.500" and (.versions | length) == 500' "$JQ_STRESS_DIR/versions.json" >/dev/null; then
+    echo "ERROR: large versions.json envelope generation failed" >&2
+    exit 1
+fi
+
+echo "packages-index large versions.json test passed"