Merge remote-tracking branch 'origin/develop' into ngates/layout27

Signed-off-by: Nicholas Gates <nick@nickgates.com>

# Conflicts:
#	Cargo.lock
#	vortex-file/Cargo.toml

Author: gatesn

.github/actions/check-rebuild/action.yml

@@ -2,8 +2,12 @@ name: "Build and verify rebuild is a no-op"
 description: "Run a cargo command, then re-run it to verify all artifacts are cached"
 inputs:
   command:
-    description: "The cargo build command to run and verify (--message-format json is appended on the verification run)"
+    description: "The cargo build command to run and verify"
     required: true
+  message-format-flag:
+    description: "The flag used to request Cargo compiler-artifact JSON during the verification run"
+    required: false
+    default: "--message-format"
 runs:
   using: "composite"
   steps:
@@ -14,7 +18,7 @@ runs:
       shell: bash
       run: |
         stale=$(${{ inputs.command }} \
-          --message-format json 2>/dev/null \
+          ${{ inputs.message-format-flag }} json 2>/dev/null \
           | jq -r 'select(.reason == "compiler-artifact" and .fresh == false) | .target.name')
         if [ -n "$stale" ]; then
           echo "ERROR: Rebuild recompiled crates that should have been cached:"

.github/pull_request_template.md

@@ -1,55 +1,23 @@
-<!--
-Thank you for submitting a pull request! We appreciate your time and effort.
-
-Please make sure to provide enough information so that we can review your pull
-request. The Summary and Testing sections below contain guidance on what to
-include.
--->
-
-## Summary
+## Rationale for this change
 
 <!--
-If this PR is related to a tracked effort, please link to the relevant issue
-here (e.g., `Closes: #123`). Otherwise, feel free to ignore / delete this.
-
-In this section, please:
-
-1. Explain the rationale for this change.
-2. Summarize the changes included in this PR.
+Why are you proposing this change, and what is its impact? 
+Is it part of a long term effort, or a bigger change?
 
-A general rule of thumb is that larger PRs should have larger summaries. If
-there are a lot of changes, please help us review the code by explaining what
-was changed and why.
-
-If there is an issue or discussion attached, there is no need to duplicate all
-the details, but clarity is always preferred over brevity.
+If this PR is related to a tracked effort or an open issue, please link to the relevant issue.
 -->
 
-Closes: #000
+- Closes: #000
 
+## What changes are included in this PR?
 <!--
-## API Changes
-
-Uncomment this section if there are any user-facing changes.
-
-Consider whether the change affects users in one of the following ways:
-
-1. Breaks public APIs in some way.
-2. Changes the underlying behavior of one of the engine integrations.
-3. Should some documentation be updated to reflect this change?
-
-If a public API is changed in a breaking manner, make sure to add the
-appropriate label.
+No need to duplicate information from the previous section, but if you're touching many
+parts of the code base, its worth explicitly noting the important changes or how they are tested.
 -->
 
-## Testing
+## What APIs are changed? Are there any user-facing changes?
 
 <!--
-Please describe how this change was tested. Here are some common categories for
-testing in Vortex:
-
-1. Verifying existing behavior is maintained.
-2. Verifying new behavior and functionality works correctly.
-3. Serialization compatibility (backwards and forwards) should be maintained or
-   explicitly broken.
+Are there any user-facing changes that might require documentation updates
+Is any public API changed?
 -->

.github/workflows/ci.yml

@@ -154,9 +154,9 @@ jobs:
         run: |
           # Build docs for the whole workspace (aside from vortex-python that is having hard time with the python-docs syntax),
           # including all private docs.
-          cargo doc --profile ci --no-deps --document-private-items --workspace --exclude vortex-python
+          cargo doc --profile ci --no-deps --document-private-items --workspace --exclude vortex-python --exclude vortex-python-cuda
           # nextest doesn't support doc tests, so we run it here
-          cargo test --profile ci --doc --workspace --all-features --exclude vortex-cxx --exclude vortex-jni --exclude vortex-ffi --exclude xtask --no-fail-fast
+          cargo test --profile ci --doc --workspace --all-features --exclude vortex-cxx --exclude vortex-jni --exclude vortex-ffi --exclude xtask --exclude vortex-python-cuda --no-fail-fast
 
   build-rust:
     name: "Rust build (${{matrix.config.name}})"
@@ -187,7 +187,7 @@ jobs:
             target: wasm32-unknown-unknown
             env:
               rustflags: "RUSTFLAGS='-A warnings --cfg getrandom_backend=\"unsupported\"'"
-            args: "--target wasm32-unknown-unknown --exclude vortex --exclude vortex-cuda --exclude vortex-cub --exclude vortex-nvcomp --exclude vortex-datafusion --exclude vortex-duckdb --exclude vortex-tui --exclude vortex-zstd --exclude vortex-test-e2e-cuda --exclude vortex-sqllogictest --exclude vortex-parquet-variant"
+            args: "--target wasm32-unknown-unknown --exclude vortex --exclude vortex-cuda --exclude vortex-cub --exclude vortex-nvcomp --exclude vortex-datafusion --exclude vortex-duckdb --exclude vortex-tui --exclude vortex-zstd --exclude vortex-test-e2e-cuda --exclude vortex-python-cuda --exclude vortex-sqllogictest --exclude vortex-parquet-variant"
     steps:
       - uses: runs-on/action@v2
         if: github.repository == 'vortex-data/vortex'
@@ -355,7 +355,7 @@ jobs:
             --exclude vortex-bench `
             --exclude vortex-python --exclude vortex-duckdb `
             --exclude vortex-fuzz --exclude vortex-cuda --exclude vortex-cuda-ffi `
-            --exclude vortex-nvcomp --exclude vortex-cub --exclude vortex-test-e2e-cuda `
+            --exclude vortex-nvcomp --exclude vortex-cub --exclude vortex-test-e2e-cuda --exclude vortex-python-cuda `
             --exclude duckdb-bench `
             --exclude lance-bench --exclude datafusion-bench --exclude random-access-bench `
             --exclude compress-bench --exclude xtask --exclude vortex-datafusion `
@@ -460,10 +460,11 @@ jobs:
       - name: Run sqllogictest tests
         run: |
           ./vortex-sqllogictest/slt/tpch/generate_data.sh
-          cargo test --profile ci -p vortex-sqllogictest --test sqllogictests
+          cargo nextest run --cargo-profile ci -p vortex-sqllogictest
       - uses: ./.github/actions/check-rebuild
         with:
-          command: "cargo test --profile ci -p vortex-sqllogictest --test sqllogictests --no-run"
+          command: "cargo nextest run --cargo-profile ci -p vortex-sqllogictest --no-run"
+          message-format-flag: "--cargo-message-format"
 
   wasm-integration:
     name: "WASM integration smoke test"

.github/workflows/codspeed.yml

@@ -100,10 +100,6 @@ jobs:
             --bench fsst_cuda \
             --bench runend_cuda \
             --profile bench
-      - name: Verify CUDA kernels were built
-        run: test -f vortex-cuda/kernels/gen/release/dynamic_dispatch.ptx
-      - name: Stage CUDA kernels with Codspeed artifacts
-        run: cp -R vortex-cuda/kernels/gen/release target/codspeed/cuda-kernels
       - name: Upload benchmark executables
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7
         with:
@@ -156,7 +152,6 @@ jobs:
         uses: CodSpeedHQ/action@d872884a306dd4853acf0f584f4b706cf0cc72a2
         env:
           CARGO_MANIFEST_DIR: ${{ github.workspace }}/vortex-cuda
-          VORTEX_CUDA_KERNELS_DIR: ${{ github.workspace }}/target/codspeed/cuda-kernels
         with:
           run: cargo codspeed run $(printf -- '--bench %s ' ${{ matrix.benches }})
           token: ${{ secrets.CODSPEED_TOKEN }}

.github/workflows/cuda.yaml

@@ -38,7 +38,13 @@ jobs:
           filters: |
             cuda:
               - "vortex-cuda/**"
+              - "vortex-python/python/vortex/__init__.py"
+              - "vortex-python/pyproject.toml"
+              - "vortex-python/test/test_cuda.py"
+              - "vortex-python-cuda/**"
               - "vortex-test/**"
+              - "pyproject.toml"
+              - "uv.lock"
               - ".github/workflows/**"
 
   cuda-build-lint:
@@ -62,7 +68,7 @@ jobs:
           command: >-
             cargo build --profile ci --locked --all-features --all-targets
             -p vortex-cuda -p vortex-cuda-ffi -p vortex-cub -p vortex-nvcomp
-            -p gpu-scan-cli -p vortex-test-e2e-cuda
+            -p gpu-scan-cli -p vortex-test-e2e-cuda -p vortex-python-cuda
       - name: Clippy CUDA crates
         run: |
           cargo clippy --profile ci --locked --all-features --all-targets \
@@ -72,6 +78,7 @@ jobs:
             -p vortex-nvcomp \
             -p gpu-scan-cli \
             -p vortex-test-e2e-cuda \
+            -p vortex-python-cuda \
             -- -D warnings
 
   cuda-test:
@@ -95,6 +102,10 @@ jobs:
       - uses: ./.github/actions/setup-rust
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Install uv
+        uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63  # 0.18.6
+        with:
+          sync: false
       - name: Install nextest
         uses: taiki-e/install-action@9e1e5806d4a4822de933115878265be9aaa786d9  # v2
         with:
@@ -112,10 +123,19 @@ jobs:
             -p vortex-cub \
             -p vortex-nvcomp \
             -p vortex-test-e2e-cuda \
+            -p vortex-python-cuda \
             --all-features \
             --no-fail-fast \
             --target x86_64-unknown-linux-gnu \
             --verbose
+      - name: Python CUDA extension checks
+        env:
+          MATURIN_PEP517_ARGS: "--profile ci"
+        run: |
+          # --all-packages installs the shared dev tooling (basedpyright) from the
+          # root `dev` group; --extra cuda adds the vortex-data-cuda extension.
+          uv run --all-packages --extra cuda basedpyright vortex-python vortex-python-cuda
+          uv run --all-packages --extra cuda pytest --benchmark-disable vortex-python/test/test_cuda.py vortex-python-cuda/test
 
   cuda-test-sanitizer:
     needs: [changes]

.github/workflows/musl.yml

@@ -66,7 +66,7 @@ jobs:
         run: |
           cargo nextest run --cargo-profile ci --locked --workspace --no-fail-fast \
             --exclude vortex-cuda --exclude vortex-cub --exclude vortex-nvcomp \
-            --exclude gpu-scan-cli --exclude vortex-test-e2e-cuda \
+            --exclude gpu-scan-cli --exclude vortex-test-e2e-cuda --exclude vortex-python-cuda \
             --exclude vortex-duckdb --exclude duckdb-bench --exclude vortex-sqllogictest \
             --exclude vortex-bench --exclude lance-bench --exclude datafusion-bench --exclude vortex-datafusion \
             --exclude compress-bench --exclude random-access-bench --exclude vortex-bench-server

.github/workflows/package.yml

@@ -50,6 +50,34 @@ jobs:
         run: |
           cargo set-version --workspace ${{ inputs.version }}
 
+      # cargo set-version bumps Cargo.toml crate versions but not the ==0.1.0 pins
+      # between these two wheels in their pyproject.toml. The base and CUDA extension
+      # wheels must require each other at the exact released version (they exchange
+      # Vortex IPC buffers), so rewrite those pins here.
+      - name: Set Python extension version pins
+        env:
+          VERSION: ${{ inputs.version }}
+        run: |
+          python3 - <<'PY'
+          import os
+          import re
+          from pathlib import Path
+
+          version = os.environ["VERSION"]
+          replacements = {
+              Path("vortex-python/pyproject.toml"): "vortex-data-cuda",
+              Path("vortex-python-cuda/pyproject.toml"): "vortex-data",
+          }
+          for path, package in replacements.items():
+              text = path.read_text()
+              text, count = re.subn(
+                  f"{re.escape(package)}==[^\"\\]]+", f"{package}=={version}", text
+              )
+              if count != 1:
+                  raise RuntimeError(f"Expected one {package} version pin in {path}, found {count}")
+              path.write_text(text)
+          PY
+
       - name: Set up Python
         uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6
         # Latest macOS doesn't allow maturin to install stuff into the global Python interpreter

.github/workflows/rust-instrumented.yml

@@ -88,7 +88,7 @@ jobs:
       - name: Rust Tests
         if: ${{ matrix.suite == 'tests' }}
         run: |
-          cargo nextest run --locked --workspace --all-features --no-fail-fast
+          cargo nextest run --locked --workspace --all-features --no-fail-fast --exclude vortex-python-cuda
       - name: Generate coverage report
         run: |
           RUST_SYSROOT="$(rustc --print sysroot)"

.github/workflows/sql-benchmarks.yml

@@ -681,14 +681,9 @@ jobs:
           python3 scripts/s3-download.py s3://vortex-ci-benchmark-results/data.json.gz data.json.gz --no-sign-request
           gzip -d -c data.json.gz > base.json
 
-          benchmark_name="${{ matrix.name }}"
-          if [ "${{ inputs.benchmark_profile }}" != "full" ]; then
-            benchmark_name="$benchmark_name (${{ inputs.benchmark_profile }})"
-          fi
-
-          echo "# Benchmarks: $benchmark_name" > comment.md
+          echo "# Benchmarks: ${{ matrix.name }}" > comment.md
           echo '' >> comment.md
-          uv run --no-project scripts/compare-benchmark-jsons.py base.json results.json "$benchmark_name" \
+          uv run --no-project scripts/compare-benchmark-jsons.py base.json results.json "${{ matrix.name }}" \
             >> comment.md
           cat comment.md >> "$GITHUB_STEP_SUMMARY"
 
@@ -698,9 +693,8 @@ jobs:
         with:
           file-path: comment.md
           # There is exactly one comment per comment-tag. If a comment with this tag already exists,
-          # this action will *update* the comment instead of posting a new comment. Therefore, each
-          # unique benchmark configuration must have a unique comment-tag.
-          comment-tag: bench-pr-comment-${{ matrix.id }}${{ inputs.benchmark_profile == 'base' && '-base' || '' }}
+          # this action will *update* the comment instead of posting a new comment.
+          comment-tag: bench-pr-comment-${{ matrix.id }}
 
       - name: Comment PR on failure
         if: failure() && inputs.mode == 'pr' && github.event.pull_request.head.repo.fork == false
@@ -710,7 +704,7 @@ jobs:
             # 🚨🚨🚨❌❌❌ SQL BENCHMARK FAILED ❌❌❌🚨🚨🚨
 
             Benchmark `${{ matrix.name }}` (${{ inputs.benchmark_profile }}) failed! Check the [workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details.
-          comment-tag: bench-pr-comment-${{ matrix.id }}${{ inputs.benchmark_profile == 'base' && '-base' || '' }}
+          comment-tag: bench-pr-comment-${{ matrix.id }}
 
       - name: Upload Benchmark Results
         if: inputs.mode == 'develop'