Merge branch 'develop' into ngates/stats-7707/zoned-statfn-pruning

Author: gatesn

.github/runs-on.yml

@@ -1,6 +1,18 @@
 _extends: .github-private
+
 images:
-  ubuntu24-full-arm64-pre:
+  ubuntu24-full-x64-pre-v2:
+    platform: "linux"
+    arch: "x64"
+    name: "vortex-ci-v2-x64-*"
+    owner: "245040174862"
+  ubuntu24-full-arm64-pre-v2:
     platform: "linux"
     arch: "arm64"
-    ami: "ami-0f1ea4587146704c3"
+    name: "vortex-ci-v2-arm64-*"
+    owner: "245040174862"
+  windows25-full-x64-pre:
+    platform: "windows"
+    arch: "x64"
+    name: "vortex-ci-windows-x64-*"
+    owner: "245040174862"

.github/workflows/codspeed.yml

@@ -76,7 +76,8 @@ jobs:
           - { shard: 3, name: "Standalone kernels", benches: "alp_cuda date_time_parts_cuda dict_cuda fsst_cuda runend_cuda" }
     name: "Benchmark with Codspeed (CUDA Shard #${{ matrix.shard }} - ${{ matrix.name }})"
     timeout-minutes: 30
-    runs-on: runs-on=${{ github.run_id }}/family=g5/image=ubuntu24-gpu-x64/tag=bench-codspeed-cuda-${{ matrix.shard }}
+    runs-on: >-
+      runs-on=${{ github.run_id }}/family=g5/cpu=8/image=ubuntu24-gpu-x64/tag=bench-codspeed-cuda-${{ matrix.shard }}
     steps:
       - uses: runs-on/action@v2
         with:

.github/workflows/cuda.yaml

@@ -150,5 +150,6 @@ jobs:
       - name: Download and run cudf-test-harness
         run: |
           curl -fsSL https://github.com/vortex-data/cudf-test-harness/releases/latest/download/cudf-test-harness-x86_64.tar.gz | tar -xz
-          cd cudf-test-harness-x86_64
-          compute-sanitizer --tool memcheck --error-exitcode 1 ./cudf-test-harness check $GITHUB_WORKSPACE/target/x86_64-unknown-linux-gnu/ci/libvortex_test_e2e_cuda.so
+          $GITHUB_WORKSPACE/target/x86_64-unknown-linux-gnu/ci/cudf_harness_runner \
+            ./cudf-test-harness-x86_64/cudf-test-harness \
+            $GITHUB_WORKSPACE/target/x86_64-unknown-linux-gnu/ci/libvortex_test_e2e_cuda.so

.github/workflows/fuzz-coverage.yml

@@ -40,11 +40,14 @@ jobs:
             rustup component add llvm-tools-preview --toolchain $NIGHTLY_TOOLCHAIN
 
           # Verify llvm-profdata is accessible (cargo-fuzz needs it for coverage merging)
-          LLVM_PROFDATA="$(rustc +$NIGHTLY_TOOLCHAIN --print sysroot)/lib/rustlib/$(rustc +$NIGHTLY_TOOLCHAIN -vV | sed -n 's|host: ||p')/bin/llvm-profdata"
+          RUST_SYSROOT="$(rustc +$NIGHTLY_TOOLCHAIN --print sysroot)"
+          RUST_HOST="$(rustc +$NIGHTLY_TOOLCHAIN --print host-tuple)"
+          LLVM_TOOLS_BIN="${RUST_SYSROOT}/lib/rustlib/${RUST_HOST}/bin"
+          LLVM_PROFDATA="${LLVM_TOOLS_BIN}/llvm-profdata"
           if [ ! -f "$LLVM_PROFDATA" ]; then
             echo "ERROR: llvm-profdata not found at $LLVM_PROFDATA"
             echo "Listing bin directory:"
-            ls -la "$(dirname "$LLVM_PROFDATA")" || echo "Directory does not exist"
+            ls -la "$LLVM_TOOLS_BIN" || echo "Directory does not exist"
             exit 1
           fi
           echo "Found llvm-profdata at $LLVM_PROFDATA"
@@ -84,13 +87,13 @@ jobs:
           COVERAGE_DIR="fuzz/coverage/${{ matrix.fuzz_target }}"
 
           # llvm tools are installed via rustup's llvm-tools component
-          LLVM_TOOLS_BIN="$(rustc +$NIGHTLY_TOOLCHAIN --print sysroot)/lib/rustlib/$(rustc +$NIGHTLY_TOOLCHAIN -vV | sed -n 's|host: ||p')/bin"
-
-          TARGET_TRIPLE=$(rustc +$NIGHTLY_TOOLCHAIN -vV | sed -n 's|host: ||p')
+          RUST_SYSROOT="$(rustc +$NIGHTLY_TOOLCHAIN --print sysroot)"
+          RUST_HOST="$(rustc +$NIGHTLY_TOOLCHAIN --print host-tuple)"
+          LLVM_TOOLS_BIN="${RUST_SYSROOT}/lib/rustlib/${RUST_HOST}/bin"
 
           # cargo-fuzz coverage places the binary at:
           #   target/<triple>/coverage/<triple>/release/<fuzz_target>
-          FUZZ_BIN="target/${TARGET_TRIPLE}/coverage/${TARGET_TRIPLE}/release/${{ matrix.fuzz_target }}"
+          FUZZ_BIN="target/${RUST_HOST}/coverage/${RUST_HOST}/release/${{ matrix.fuzz_target }}"
 
           if [ ! -f "$FUZZ_BIN" ]; then
             echo "ERROR: Could not find fuzz binary at $FUZZ_BIN"

.github/workflows/package.yml

@@ -44,7 +44,7 @@ jobs:
       - name: Install cargo-edit
         uses: taiki-e/cache-cargo-install-action@66c9585ef5ca780ee69399975a5e911f47905995
         with:
-          tool: cargo-edit
+          tool: cargo-edit@0.13.10
 
       - name: Cargo Set Version
         run: |

.github/workflows/publish-dry-runs.yml

@@ -117,7 +117,7 @@ jobs:
       - name: Install cargo-edit
         uses: taiki-e/cache-cargo-install-action@66c9585ef5ca780ee69399975a5e911f47905995
         with:
-          tool: cargo-edit
+          tool: cargo-edit@0.13.10
       - name: Set Version
         run: |
           # This is mostly a dummy version, we don't actually publish anything but it cannot exist in crates.io

.github/workflows/publish.yml

@@ -33,7 +33,7 @@ jobs:
       - name: Install cargo-edit
         uses: taiki-e/cache-cargo-install-action@66c9585ef5ca780ee69399975a5e911f47905995
         with:
-          tool: cargo-edit
+          tool: cargo-edit@0.13.10
 
       - name: Cargo Set Version
         run: |

.github/workflows/rust-instrumented.yml

@@ -47,13 +47,35 @@ jobs:
           sccache: s3
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6
       - uses: ./.github/actions/setup-prebuild
+      - name: Ensure llvm-tools are installed
+        run: |
+          rustup component add llvm-tools || \
+            rustup component add llvm-tools-preview
+
+          # Verify llvm-profdata is accessible (grcov needs it for coverage merging)
+          RUST_SYSROOT="$(rustc --print sysroot)"
+          RUST_HOST="$(rustc --print host-tuple)"
+          LLVM_TOOLS_BIN="${RUST_SYSROOT}/lib/rustlib/${RUST_HOST}/bin"
+          LLVM_PROFDATA="${LLVM_TOOLS_BIN}/llvm-profdata"
+          if [ ! -f "$LLVM_PROFDATA" ]; then
+            echo "ERROR: llvm-profdata not found at $LLVM_PROFDATA"
+            echo "Listing bin directory:"
+            ls -la "$LLVM_TOOLS_BIN" || echo "Directory does not exist"
+            exit 1
+          fi
+          echo "Found llvm-profdata at $LLVM_PROFDATA"
+
       - name: Rust Tests
         if: ${{ matrix.suite == 'tests' }}
         run: |
           cargo nextest run --locked --workspace --all-features --no-fail-fast
       - name: Generate coverage report
         run: |
+          RUST_SYSROOT="$(rustc --print sysroot)"
+          RUST_HOST="$(rustc --print host-tuple)"
+          LLVM_TOOLS_BIN="${RUST_SYSROOT}/lib/rustlib/${RUST_HOST}/bin"
           grcov . --binary-path target/debug/ -s . -t lcov --llvm --ignore-not-existing \
+            --llvm-path "${LLVM_TOOLS_BIN}" \
             --threads $(nproc) \
             --ignore '../*' --ignore '/*' --ignore 'fuzz/*' --ignore 'vortex-bench/*' \
             --ignore 'home/*' --ignore 'xtask/*' --ignore 'target/*' --ignore 'vortex-error/*' \
@@ -62,6 +84,7 @@ jobs:
             --ignore 'vortex-ffi/examples/*' --ignore '*/arbitrary/*' --ignore '*/arbitrary.rs' --ignore 'vortex-cxx/*' \
             --ignore benchmarks/* --ignore 'vortex-test/*' \
             -o ${{ env.GRCOV_OUTPUT_FILE }}
+          test -s ${{ env.GRCOV_OUTPUT_FILE }}
       - name: Codecov
         uses: codecov/codecov-action@e79a6962e0d4c0c17b229090214935d2e33f8354  # v6
         with:

.github/workflows/sql-benchmarks.yml

@@ -252,6 +252,29 @@ on:
                 {"engine": "datafusion", "format": "vortex"}
               ],
               "scale_factor": "1"
+            },
+            {
+              "id": "appian-nvme",
+              "subcommand": "appian",
+              "name": "Appian on NVME",
+              "data_formats": ["parquet", "vortex", "vortex-compact", "duckdb"],
+              "pr_targets": [
+                {"engine": "datafusion", "format": "parquet"},
+                {"engine": "datafusion", "format": "vortex"},
+                {"engine": "duckdb", "format": "parquet"},
+                {"engine": "duckdb", "format": "vortex"},
+                {"engine": "duckdb", "format": "duckdb"}
+              ],
+              "develop_targets": [
+                {"engine": "datafusion", "format": "parquet"},
+                {"engine": "datafusion", "format": "vortex"},
+                {"engine": "datafusion", "format": "vortex-compact"},
+                {"engine": "duckdb", "format": "parquet"},
+                {"engine": "duckdb", "format": "vortex"},
+                {"engine": "duckdb", "format": "vortex-compact"},
+                {"engine": "duckdb", "format": "duckdb"}
+              ],
+              "iterations": "10"
             }
           ]
 
@@ -402,6 +425,18 @@ jobs:
             ${{ matrix.iterations && format('--iterations {0}', matrix.iterations) || '' }} \
             --opt remote-data-dir=${{ matrix.remote_storage }} \
             ${{ matrix.scale_factor && format('--opt scale-factor={0}', matrix.scale_factor) || '' }}
+
+      - name: Capture file sizes
+        if: matrix.remote_storage == null
+        shell: bash
+        run: |
+          uv run --no-project scripts/capture-file-sizes.py \
+            vortex-bench/data \
+            --benchmark ${{ matrix.subcommand }} \
+            --commit ${{ inputs.mode == 'pr' && github.event.pull_request.head.sha || github.sha }} \
+            -o sizes.json
+          cat sizes.json >> results.json
+
       - name: Compare results
         if: inputs.mode == 'pr'
         shell: bash
@@ -435,56 +470,6 @@ jobs:
           # unique benchmark configuration must have a unique comment-tag.
           comment-tag: bench-pr-comment-${{ matrix.id }}
 
-      - name: Compare file sizes
-        if: inputs.mode == 'pr' && matrix.remote_storage == null
-        shell: bash
-        run: |
-          set -Eeu -o pipefail -x
-
-          # Capture HEAD file sizes (vortex formats only)
-          uv run --no-project scripts/capture-file-sizes.py \
-            vortex-bench/data \
-            --benchmark ${{ matrix.subcommand }} \
-            --commit ${{ github.event.pull_request.head.sha }} \
-            -o head-sizes.json
-
-          # Get base commit SHA (same as benchmark comparison)
-          base_commit_sha=$(\
-            curl -L \
-              -H "Accept: application/vnd.github+json" \
-              -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
-              https://api.github.com/repos/vortex-data/vortex/actions/workflows/bench.yml/runs\?branch\=develop\&status\=success\&per_page\=1 \
-            | jq -r '.workflow_runs[].head_sha' \
-          )
-
-          # Download file sizes baseline (per-benchmark file)
-          python3 scripts/s3-download.py s3://vortex-ci-benchmark-results/file-sizes-${{ matrix.id }}.json.gz file-sizes.json.gz --no-sign-request || true
-
-          # Generate comparison report
-          echo '# File Sizes: ${{ matrix.name }}' > sizes-comment.md
-          echo '' >> sizes-comment.md
-
-          if [ -f file-sizes.json.gz ]; then
-            gzip -d -c file-sizes.json.gz | grep $base_commit_sha > base-sizes.json || true
-            if [ -s base-sizes.json ]; then
-              uv run --no-project scripts/compare-file-sizes.py base-sizes.json head-sizes.json \
-                >> sizes-comment.md
-            else
-              echo '_No baseline file sizes found for base commit._' >> sizes-comment.md
-            fi
-          else
-            echo '_No baseline file sizes available yet._' >> sizes-comment.md
-          fi
-
-          cat sizes-comment.md >> $GITHUB_STEP_SUMMARY
-
-      - name: Comment PR with file sizes
-        if: inputs.mode == 'pr' && matrix.remote_storage == null && github.event.pull_request.head.repo.fork == false
-        uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b  # v3
-        with:
-          file-path: sizes-comment.md
-          comment-tag: file-sizes-${{ matrix.id }}
-
       - name: Comment PR on failure
         if: failure() && inputs.mode == 'pr' && github.event.pull_request.head.repo.fork == false
         uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b  # v3
@@ -513,17 +498,6 @@ jobs:
             --benchmark-id "${{ matrix.id }}" \
             --repo-url "${{ github.server_url }}/${{ github.repository }}"
 
-      - name: Upload File Sizes
-        if: inputs.mode == 'develop' && matrix.remote_storage == null
-        shell: bash
-        run: |
-          uv run --no-project scripts/capture-file-sizes.py \
-            vortex-bench/data \
-            --benchmark ${{ matrix.subcommand }} \
-            --commit ${{ github.sha }} \
-            -o sizes.json
-          bash scripts/cat-s3.sh vortex-ci-benchmark-results file-sizes-${{ matrix.id }}.json.gz sizes.json
-
       - name: Alert incident.io
         if: failure() && inputs.mode == 'develop'
         uses: ./.github/actions/alert-incident-io

bench-orchestrator/README.md

@@ -41,7 +41,7 @@ vx-bench run <benchmark> [options]
 
 **Arguments:**
 
-- `benchmark`: Benchmark suite to run (`tpch`, `tpcds`, `clickbench`, `fineweb`, `gh-archive`, `polarsignals`, `public-bi`, `statpopgen`)
+- `benchmark`: Benchmark suite to run (`appian`, `tpch`, `tpcds`, `clickbench`, `fineweb`, `gh-archive`, `polarsignals`, `public-bi`, `statpopgen`)
 
 **Options:**