Merge remote-tracking branch 'origin/develop' into claude/add-transpose-metadata-8iZD0

Signed-off-by: Joe Isaacs <joe.isaacs@live.co.uk>

# Conflicts:
#	encodings/fastlanes/Cargo.toml
#	encodings/sparse/public-api.lock
#	encodings/sparse/src/lib.rs
#	vortex-array/public-api.lock
#	vortex-file/src/lib.rs

Author: joseph-isaacs

.agents/skills/pr-review/SKILL.md

@@ -60,16 +60,16 @@ network permissions.
 
 ## Review Areas
 
-| Area | Focus |
-| --- | --- |
-| Correctness | Length and dtype invariants, nullability, validity masks, offset math, canonicalization, boundary conditions, empty arrays, scalar vs array behavior |
-| Rust soundness | `unsafe` blocks, aliasing, lifetimes, alignment, FFI boundaries, panic safety, ownership of buffers and arrays |
-| Compression and IO | Encoding metadata, statistics, layout evolution, file compatibility, scan projection/filter behavior, async IO edge cases |
-| Performance | Unnecessary copies, lost zero-copy behavior, avoidable allocations, poor cache locality, quadratic loops, excessive dynamic dispatch in hot paths |
-| Error handling | Correct `vortex_err!` and `vortex_bail!` usage, useful messages, no accidental panics on user data |
-| API compatibility | Public API docs, public-api lock updates, feature flags, crate boundaries, Python/Java binding impacts |
-| Tests | Regression coverage, edge cases, parameterized cases with `rstest`, use of `assert_arrays_eq!`, docs doctests when docs change |
-| Verification scope | Avoid requesting or running expensive workspace checks when the PR only changes docs, agent files, symlinks, or metadata |
+| Area               | Focus                                                                                                                                                |
+| ------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------- |
+| Correctness        | Length and dtype invariants, nullability, validity masks, offset math, canonicalization, boundary conditions, empty arrays, scalar vs array behavior |
+| Rust soundness     | `unsafe` blocks, aliasing, lifetimes, alignment, FFI boundaries, panic safety, ownership of buffers and arrays                                       |
+| Compression and IO | Encoding metadata, statistics, layout evolution, file compatibility, scan projection/filter behavior, async IO edge cases                            |
+| Performance        | Unnecessary copies, lost zero-copy behavior, avoidable allocations, poor cache locality, quadratic loops, excessive dynamic dispatch in hot paths    |
+| Error handling     | Correct `vortex_err!` and `vortex_bail!` usage, useful messages, no accidental panics on user data                                                   |
+| API compatibility  | Public API docs, feature flags, crate boundaries, Python/Java binding impacts                                                                        |
+| Tests              | Regression coverage, edge cases, parameterized cases with `rstest`, use of `assert_arrays_eq!`, docs doctests when docs change                       |
+| Verification scope | Avoid requesting or running expensive workspace checks when the PR only changes docs, agent files, symlinks, or metadata                             |
 
 ## Output
 

.github/AGENTS.md

@@ -4,7 +4,6 @@ Nightly is required for:
 - `cargo fmt` (nightly formatting options)
 - `-Z` flags: sanitizers (`-Zsanitizer=address`), miri (`-Zmiri-*`), publish (`-Zpublish-timeout`)
 - `cargo-fuzz` (requires nightly)
-- `public-api` xtask (nightly rustdoc JSON)
 
 Everything else (build, clippy, tests, docs, benchmarks, packaging) should use stable.
 

.github/pull_request_template.md

@@ -39,8 +39,7 @@ Consider whether the change affects users in one of the following ways:
 3. Should some documentation be updated to reflect this change?
 
 If a public API is changed in a breaking manner, make sure to add the
-appropriate label. You can run `./scripts/public-api.sh` locally to see if there
-are any public API changes (and this also runs in our CI).
+appropriate label.
 -->
 
 ## Testing

.github/workflows/bench-pr.yml

@@ -49,7 +49,7 @@ jobs:
 
       - name: Install DuckDB
         run: |
-          wget -qO- https://github.com/duckdb/duckdb/releases/download/v1.5.2/duckdb_cli-linux-amd64.zip | funzip > duckdb
+          wget -qO- https://github.com/duckdb/duckdb/releases/download/v1.5.3/duckdb_cli-linux-amd64.zip | funzip > duckdb
           chmod +x duckdb
           echo "$PWD" >> $GITHUB_PATH
 

.github/workflows/bench.yml

@@ -63,7 +63,7 @@ jobs:
 
       - name: Install DuckDB
         run: |
-          wget -qO- https://github.com/duckdb/duckdb/releases/download/v1.5.2/duckdb_cli-linux-amd64.zip | funzip > duckdb
+          wget -qO- https://github.com/duckdb/duckdb/releases/download/v1.5.3/duckdb_cli-linux-amd64.zip | funzip > duckdb
           chmod +x duckdb
           echo "$PWD" >> $GITHUB_PATH
 
@@ -107,7 +107,6 @@ jobs:
 
       - name: Ingest results to v3 server
         if: vars.V3_INGEST_URL != ''
-        continue-on-error: true
         shell: bash
         env:
           INGEST_BEARER_TOKEN: ${{ secrets.INGEST_BEARER_TOKEN }}

.github/workflows/ci.yml

@@ -162,7 +162,7 @@ jobs:
             target: wasm32-unknown-unknown
             env:
               rustflags: "RUSTFLAGS='-A warnings --cfg getrandom_backend=\"unsupported\"'"
-            args: "--target wasm32-unknown-unknown --exclude vortex --exclude vortex-cuda --exclude vortex-cub --exclude vortex-nvcomp --exclude vortex-datafusion --exclude vortex-duckdb --exclude vortex-tui --exclude vortex-zstd --exclude vortex-test-e2e-cuda --exclude vortex-sqllogictest"
+            args: "--target wasm32-unknown-unknown --exclude vortex --exclude vortex-cuda --exclude vortex-cub --exclude vortex-nvcomp --exclude vortex-datafusion --exclude vortex-duckdb --exclude vortex-tui --exclude vortex-zstd --exclude vortex-test-e2e-cuda --exclude vortex-sqllogictest --exclude vortex-parquet-variant"
     steps:
       - uses: runs-on/action@v2
         if: github.repository == 'vortex-data/vortex'
@@ -255,7 +255,7 @@ jobs:
               await github.rest.checks.update({
                 ...context.repo,
                 check_run_id: Number(url.split('/').pop()),
-                output: { title: `${failed.join(', ')} — failing`, summary: '' },
+                output: { title: `${failed.join(', ')} - failing`, summary: '' },
               });
             }
             core.setFailed(`Lint failed: ${failed.join(', ')}`);
@@ -270,6 +270,7 @@ jobs:
         run: |
           git ls-files vortex-cuda vortex-cxx vortex-duckdb vortex-ffi \
             | grep -E '\.(cpp|hpp|cu|cuh|h)$' \
+            | grep -v 'arrow/reference/arrow_c_device\.h$' \
             | grep -v 'kernels/src/bit_unpack_.*\.cu$' \
             | grep -v 'kernels/src/bit_unpack_.*_lanes\.cuh$' \
             | xargs clang-format --dry-run --Werror --style=file
@@ -293,40 +294,6 @@ jobs:
         run: |
           cargo hack --no-dev-deps --ignore-private clippy --profile ci --no-default-features -- -D warnings
 
-  public-api:
-    name: "Public API lock files"
-    timeout-minutes: 30
-    runs-on: >-
-      ${{ github.repository == 'vortex-data/vortex'
-          && format('runs-on={0}/runner=amd64-xsmall/image=ubuntu24-full-x64-pre-v2/tag=public-api', github.run_id)
-          || 'ubuntu-latest' }}
-    steps:
-      - uses: runs-on/action@v2
-        if: github.repository == 'vortex-data/vortex'
-        with:
-          sccache: s3
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6
-      - uses: ./.github/actions/setup-prebuild
-      - name: Install nightly for public-api
-        run: rustup toolchain install $NIGHTLY_TOOLCHAIN
-      - name: Regenerate public API lock files
-        run: cargo +$NIGHTLY_TOOLCHAIN run --profile ci -p xtask -- public-api
-      - name: Verify lock files are up to date
-        run: |
-          if ! git diff --quiet '**/public-api.lock'; then
-            git diff -U0 '**/public-api.lock'
-            echo ""
-            echo ""
-            echo "Changed files:"
-            git diff --name-only '**/public-api.lock'
-            echo ""
-            echo ""
-            echo "Public API lock files are out of date."
-            echo "Run './scripts/public-api.sh' locally to"
-            echo "regenerate them, then commit the changes."
-            exit 1
-          fi
-
   rust-test-other:
     name: "Rust tests (${{ matrix.os }})"
     timeout-minutes: 30
@@ -369,6 +336,19 @@ jobs:
         if: matrix.os != 'windows-x64'
         run: |
           cargo nextest run --cargo-profile ci --locked --workspace --all-features --no-fail-fast --exclude vortex-bench --exclude xtask --exclude vortex-sqllogictest
+      - name: vortex-bench-server admin snapshot tests (Linux only - network-dependent)
+        # The /api/admin/snapshot tests INSTALL+LOAD the vortex DuckDB
+        # core extension from extensions.duckdb.org on first call. They
+        # are #[ignore]'d by default so `cargo test` works in offline
+        # environments (sandboxed CI, local dev without network). This
+        # step runs them explicitly on the Linux runners, which DO have
+        # outbound network, so the entire backup contract is covered
+        # in CI before merge. macOS/arm64 also exercises them - same
+        # extension, same network. Windows skipped (bench-server is
+        # excluded from Windows test matrix above).
+        if: matrix.os == 'linux-x64' || matrix.os == 'linux-arm64' || matrix.os == 'macos-arm64'
+        run: |
+          cargo nextest run --cargo-profile ci --locked -p vortex-bench-server --test admin --run-ignored only
       - uses: ./.github/actions/check-rebuild
         if: matrix.os != 'windows-x64'
         with:
@@ -412,7 +392,7 @@ jobs:
     continue-on-error: ${{ matrix.checks == 'advisories' }}
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6
-      - uses: EmbarkStudios/cargo-deny-action@6c8f9facfa5047ec02d8485b6bf52b587b7777d1  # v2
+      - uses: EmbarkStudios/cargo-deny-action@a531616d8ce3b9177443e48a1159bc945a099823  # v2
         with:
           command: check ${{ matrix.checks }}
 

.github/workflows/claude-review.yml

@@ -242,7 +242,7 @@ jobs:
 
       - name: Run Claude Code in review-only mode
         id: claude
-        uses: anthropics/claude-code-action@51ea8ea73a139f2a74ff649e3092c25a904aed7e  # v1.0.123
+        uses: anthropics/claude-code-action@787c5a0ce96a9a6cfb050ea0c8f4c05f2447c251  # v1.0.133
         with:
           claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
           # This workflow deliberately uses the built-in token because it only needs

.github/workflows/claude-write.yml

@@ -197,7 +197,7 @@ jobs:
 
       - name: Run Claude Code
         id: claude
-        uses: anthropics/claude-code-action@51ea8ea73a139f2a74ff649e3092c25a904aed7e  # v1.0.123
+        uses: anthropics/claude-code-action@787c5a0ce96a9a6cfb050ea0c8f4c05f2447c251  # v1.0.133
         with:
           claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
           github_token: ${{ steps.app-token.outputs.token }}

.github/workflows/cuda.yaml

@@ -69,7 +69,7 @@ jobs:
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
       - name: Install nextest
-        uses: taiki-e/install-action@7be9fd86bd1707236395105d6e9329dd1511a7e1  # v2
+        uses: taiki-e/install-action@8f531eaecd1898bc3da7d104ad91bee98d1b97bd  # v2
         with:
           tool: nextest
       - name: Rust Tests

.github/workflows/fuzzer-fix-automation.yml

@@ -236,7 +236,7 @@ jobs:
           CRASH_FILE: ${{ steps.extract.outputs.crash_file }}
           CRASH_FILE_PATH: ${{ steps.download.outputs.crash_file_path }}
           ARTIFACT_URL: ${{ steps.extract.outputs.artifact_url }}
-        uses: anthropics/claude-code-action@51ea8ea73a139f2a74ff649e3092c25a904aed7e  # v1
+        uses: anthropics/claude-code-action@787c5a0ce96a9a6cfb050ea0c8f4c05f2447c251  # v1
         with:
           claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
           github_token: ${{ secrets.GITHUB_TOKEN }}