fix: Speed up coverage tests and install LLVM tools

Signed-off-by: Alexander Droste <alexander.droste@protonmail.com>

Author: 0ax1

.github/workflows/rust-instrumented.yml

@@ -47,13 +47,20 @@ jobs:
           sccache: s3
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6
       - uses: ./.github/actions/setup-prebuild
+      - name: Install LLVM tools
+        run: |
+          rustup component add llvm-tools || rustup component add llvm-tools-preview
       - name: Rust Tests
         if: ${{ matrix.suite == 'tests' }}
         run: |
           cargo nextest run --locked --workspace --all-features --no-fail-fast
       - name: Generate coverage report
         run: |
+          set -euo pipefail
+          LLVM_TOOLS_BIN="$(rustc --print sysroot)/lib/rustlib/$(rustc -vV | sed -n 's|host: ||p')/bin"
+          test -x "${LLVM_TOOLS_BIN}/llvm-profdata"
           grcov . --binary-path target/debug/ -s . -t lcov --llvm --ignore-not-existing \
+            --llvm-path "${LLVM_TOOLS_BIN}" \
             --threads $(nproc) \
             --ignore '../*' --ignore '/*' --ignore 'fuzz/*' --ignore 'vortex-bench/*' \
             --ignore 'home/*' --ignore 'xtask/*' --ignore 'target/*' --ignore 'vortex-error/*' \
@@ -62,6 +69,7 @@ jobs:
             --ignore 'vortex-ffi/examples/*' --ignore '*/arbitrary/*' --ignore '*/arbitrary.rs' --ignore 'vortex-cxx/*' \
             --ignore benchmarks/* --ignore 'vortex-test/*' \
             -o ${{ env.GRCOV_OUTPUT_FILE }}
+          test -s ${{ env.GRCOV_OUTPUT_FILE }}
       - name: Codecov
         uses: codecov/codecov-action@e79a6962e0d4c0c17b229090214935d2e33f8354  # v6
         with:

benchmarks-website/server/tests/landing.rs

@@ -416,11 +416,11 @@ async fn landing_page_honours_filter_query_params() -> Result<()> {
 /// full history via the explicit `/api/chart/{slug}?n=all` refetch.
 #[tokio::test]
 async fn landing_first_group_shard_caps_commits() -> Result<()> {
-    // 250 commits is comfortably above the 100-commit artifact cap so the
+    // 101 commits is the smallest fixture above the 100-commit artifact cap, so the
     // cap actually kicks in. `seed_long_history` only seeds the Random-Access
     // group; with the canonical group ordering Random Access sorts first.
     let server = Server::start().await?;
-    seed_long_history(&server, 250).await?;
+    seed_long_history(&server, 101).await?;
 
     let client = reqwest::Client::new();
     let body = client.get(server.url("/")).send().await?.text().await?;
@@ -454,7 +454,7 @@ async fn landing_first_group_shard_caps_commits() -> Result<()> {
     assert_eq!(
         commits.len(),
         100,
-        "with 250 seeded commits the shard payload should be exactly the \
+        "with 101 seeded commits the shard payload should be exactly the \
          100-commit cap; got {}",
         commits.len(),
     );

benchmarks-website/server/tests/permalinks.rs

@@ -25,7 +25,7 @@ use self::common::seed_long_history;
 #[tokio::test]
 async fn permalink_pages_default_to_latest_100_and_opt_into_full_history() -> Result<()> {
     let server = Server::start().await?;
-    seed_long_history(&server, 200).await?;
+    seed_long_history(&server, 101).await?;
 
     let chart_slug = pick_chart_slug(&server, |s| s == "Random Access").await?;
     let group_slug = pick_group_slug(&server, |s| s == "Random Access").await?;
@@ -61,7 +61,7 @@ async fn permalink_pages_default_to_latest_100_and_opt_into_full_history() -> Re
             .as_array()
             .context("all commits is array")?
             .len(),
-        200,
+        101,
         "/chart?n=all should inline the full raw history",
     );
 

encodings/fsst/src/tests.rs

@@ -1,23 +1,23 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright the Vortex contributors
 
-use rand::SeedableRng;
-use rand::rngs::StdRng;
-use rand::seq::IndexedRandom;
 use vortex_array::ArrayRef;
 use vortex_array::IntoArray;
 use vortex_array::LEGACY_SESSION;
 use vortex_array::VortexSessionExecute;
 use vortex_array::arrays::VarBinViewArray;
+use vortex_array::arrays::varbin::VarBinArrayExt;
 use vortex_array::arrays::varbin::builder::VarBinBuilder;
 use vortex_array::assert_arrays_eq;
 use vortex_array::assert_nth_scalar;
 use vortex_array::dtype::DType;
 use vortex_array::dtype::Nullability;
+use vortex_array::dtype::PType;
 use vortex_buffer::buffer;
 use vortex_mask::Mask;
 
 use crate::FSST;
+use crate::FSSTArrayExt;
 use crate::fsst_compress;
 use crate::fsst_train_compressor;
 
@@ -111,60 +111,28 @@ fn test_fsst_array_ops() {
     assert_arrays_eq!(fsst_array, canonical_array);
 }
 
-// TODO(someone): ideally CI would run this in release mode as well since debug builds make the
-// allocation and compression loop substantially slower.
-/// Regression for #7833: [`fsst_compress`] must accept inputs whose cumulative compressed
-/// bytes exceed [`i32::MAX`]. Before the fix, [`fsst_compress_iter`] hardcoded
-/// [`VarBinBuilder<i32>`] for the FSST output and panicked in
-/// [`VarBinBuilder::append_value`] once cumulative compressed bytes crossed the boundary.
+/// Regression coverage for #7833.
 ///
-/// The input is built with [`VarBinBuilder<i64>`] so the input itself does not panic, which
-/// confirms the overflow is on the FSST output side. After the fix the test must succeed
-/// with the row count preserved.
-///
-/// Allocates ~2.5 GiB for the input and ~2.5 GiB for the FSST output (~5 GiB total), so it
-/// is gated to CI runs and skipped when `VORTEX_SKIP_SLOW_TESTS` is set. To run it locally:
-///
-/// ```text
-/// CI=1 cargo test --release -p vortex-fsst fsst_compress_offsets
-/// ```
+/// The bug was that [`fsst_compress_iter`] built compressed codes with [`VarBinBuilder<i32>`],
+/// which panicked once cumulative compressed bytes crossed [`i32::MAX`]. The end-to-end
+/// behavior is "large FSST outputs do not overflow", but exercising that literally requires
+/// multi-GiB input. This test checks the implementation detail that currently guarantees that
+/// behavior: compressed code offsets are widened to [`i64`].
 ///
 /// [`fsst_compress_iter`]: crate::compress::fsst_compress_iter
-#[test_with::env(CI)]
-#[test_with::no_env(VORTEX_SKIP_SLOW_TESTS)]
-fn fsst_compress_offsets_overflow_i32() {
-    // High-entropy ASCII strings sliced from a random pool. FSST is a symbol-table
-    // compressor; pseudo-random data with no recurring byte sequences resists compression,
-    // so the compressed output stays close to input size and crosses the i32 boundary.
-    const STRING_LEN: usize = 64 * 1024;
-    const TOTAL_BYTES: usize = (1usize << 31) + (512 << 20); // ~2.5 GiB
-    const N: usize = TOTAL_BYTES / STRING_LEN;
-    const POOL_LEN: usize = 64 * 1024 * 1024;
-
-    // Printable ASCII alphabet so the result is valid UTF-8.
-    const ALPHABET: &[u8; 95] =
-        b" !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~";
-
-    let mut rng = StdRng::seed_from_u64(0xC0DE_C011_B711);
-    let pool: Vec<u8> = (0..POOL_LEN)
-        .map(|_| *ALPHABET.choose(&mut rng).unwrap())
-        .collect();
-
-    println!("building large VarBinArray");
-    let mut builder = VarBinBuilder::<i64>::with_capacity(N);
-    for i in 0..N {
-        let off = i.wrapping_mul(31337) % (POOL_LEN - STRING_LEN);
-        builder.append_value(&pool[off..off + STRING_LEN]);
-    }
+#[test]
+fn fsst_compress_uses_i64_code_offsets() {
+    let mut builder = VarBinBuilder::<i32>::with_capacity(2);
+    builder.append_value(b"hello world");
+    builder.append_value(b"hello vortex");
     let array = builder.finish(DType::Utf8(Nullability::NonNullable));
 
-    println!("training FSST compressor");
     let compressor = fsst_train_compressor(&array);
     let len = array.len();
     let dtype = array.dtype().clone();
     let mut ctx = LEGACY_SESSION.create_execution_ctx();
 
-    println!("compressing to FSST");
     let compressed = fsst_compress(array, len, &dtype, &compressor, &mut ctx);
     assert_eq!(compressed.len(), len);
+    assert_eq!(compressed.codes().offsets().dtype().as_ptype(), PType::I64);
 }