zstdbuffers

Signed-off-by: Mikhail Kot <mikhail@spiraldb.com>

Author: myrrc

benchmarks/datafusion-bench/Cargo.toml

@@ -46,7 +46,7 @@ custom-labels = { workspace = true }
 
 [features]
 cuda = ["dep:vortex-cuda"]
-unstable_encodings = ["vortex/unstable_encodings"]
+unstable_encodings = ["vortex/unstable_encodings", "vortex-bench/unstable_encodings"]
 
 [lints]
 workspace = true

benchmarks/duckdb-bench/Cargo.toml

@@ -27,7 +27,7 @@ vortex-duckdb = { workspace = true }
 
 [features]
 cuda = ["dep:vortex-cuda"]
-unstable_encodings = ["vortex/unstable_encodings"]
+unstable_encodings = ["vortex/unstable_encodings", "vortex-bench/unstable_encodings"]
 
 [lints]
 workspace = true

vortex-bench/src/lib.rs

@@ -234,11 +234,17 @@ pub enum CompactionStrategy {
 impl CompactionStrategy {
     pub fn apply_options(&self, options: VortexWriteOptions) -> VortexWriteOptions {
         match self {
-            CompactionStrategy::Compact => options.with_strategy(
-                WriteStrategyBuilder::default()
-                    .with_btrblocks_builder(BtrBlocksCompressorBuilder::default().with_compact())
-                    .build(),
-            ),
+            CompactionStrategy::Compact => {
+                #[cfg(feature = "unstable_encodings")]
+                let btrblocks = BtrBlocksCompressorBuilder::default().with_compact_buffers();
+                #[cfg(not(feature = "unstable_encodings"))]
+                let btrblocks = BtrBlocksCompressorBuilder::default().with_compact();
+                options.with_strategy(
+                    WriteStrategyBuilder::default()
+                        .with_btrblocks_builder(btrblocks)
+                        .build(),
+                )
+            }
             CompactionStrategy::Default => options,
         }
     }

vortex-btrblocks/src/builder.rs

@@ -161,6 +161,21 @@ impl BtrBlocksCompressorBuilder {
         builder
     }
 
+    /// Same as with_compact() but ZstdBuffers is used instead of Zstd
+    #[cfg(all(feature = "zstd", feature = "unstable_encodings"))]
+    pub fn with_compact_buffers(self) -> Self {
+        let builder = self
+            .with_new_scheme(&string::ZstdBuffersScheme)
+            .with_new_scheme(&binary::ZstdBuffersScheme);
+
+        #[cfg(feature = "pco")]
+        let builder = builder
+            .with_new_scheme(&integer::PcoScheme)
+            .with_new_scheme(&float::PcoScheme);
+
+        builder
+    }
+
     /// Excludes schemes without CUDA kernel support and adds Zstd for string and binary compression.
     ///
     /// With the `unstable_encodings` feature, buffer-level Zstd compression is used which