From b79e50e6bb8a3ffa4621f03795441e4987286084 Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Fri, 3 Apr 2026 17:34:24 +0100 Subject: [PATCH 01/14] Try to implement compare/between for bitpacked arrays Signed-off-by: Adam Gutglick --- .../src/bitpacking/compute/compare.rs | 491 ++++++++++++++++++ .../fastlanes/src/bitpacking/compute/mod.rs | 1 + .../src/bitpacking/vtable/kernels.rs | 4 + 3 files changed, 496 insertions(+) create mode 100644 encodings/fastlanes/src/bitpacking/compute/compare.rs diff --git a/encodings/fastlanes/src/bitpacking/compute/compare.rs b/encodings/fastlanes/src/bitpacking/compute/compare.rs new file mode 100644 index 00000000000..8377c319469 --- /dev/null +++ b/encodings/fastlanes/src/bitpacking/compute/compare.rs @@ -0,0 +1,491 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +use fastlanes::BitPacking; +use fastlanes::BitPackingCompare; +use fastlanes::FastLanesComparable; +use num_traits::AsPrimitive; +use vortex_array::ArrayRef; +use vortex_array::ArrayView; +use vortex_array::ExecutionCtx; +use vortex_array::IntoArray; +use vortex_array::arrays::BoolArray; +use vortex_array::arrays::PrimitiveArray; +use vortex_array::dtype::NativePType; +use vortex_array::dtype::Nullability; +use vortex_array::dtype::UnsignedPType; +use vortex_array::match_each_integer_ptype; +use vortex_array::match_each_unsigned_integer_ptype; +use vortex_array::patches::Patches; +use vortex_array::scalar_fn::fns::between::BetweenKernel; +use vortex_array::scalar_fn::fns::between::BetweenOptions; +use vortex_array::scalar_fn::fns::between::StrictComparison; +use vortex_array::scalar_fn::fns::binary::CompareKernel; +use vortex_array::scalar_fn::fns::operators::CompareOperator; +use vortex_buffer::BitBuffer; +use vortex_buffer::BitBufferMut; +use vortex_error::VortexResult; + +use crate::BitPacked; +use crate::BitPackedArrayExt; + +impl CompareKernel for BitPacked { + fn compare( + lhs: ArrayView<'_, Self>, + rhs: &ArrayRef, + operator: CompareOperator, + ctx: &mut ExecutionCtx, + ) -> VortexResult> { + let Some(constant) = rhs.as_constant() else { + return Ok(None); + }; + + if !constant.dtype().is_int() { + return Ok(None); + } + match_each_integer_ptype!(lhs.dtype().as_ptype(), |T| { + let value = T::try_from(&constant)?; + compare_constant::(lhs, value, rhs.dtype().nullability(), operator, ctx).map(Some) + }) + } +} + +impl BetweenKernel for BitPacked { + fn between( + array: ArrayView<'_, Self>, + lower: &ArrayRef, + upper: &ArrayRef, + options: &BetweenOptions, + ctx: &mut ExecutionCtx, + ) -> VortexResult> { + let (Some(lower), Some(upper)) = (lower.as_constant(), upper.as_constant()) else { + return Ok(None); + }; + + if !lower.dtype().is_int() || !upper.dtype().is_int() { + return Ok(None); + } + + let nullability = + array.dtype().nullability() | lower.dtype().nullability() | upper.dtype().nullability(); + + match_each_integer_ptype!(array.dtype().as_ptype(), |T| { + let lower = T::try_from(&lower)?; + let upper = T::try_from(&upper)?; + between_constant::(array, lower, upper, nullability, options, ctx).map(Some) + }) + } +} + +fn compare_constant( + array: ArrayView<'_, BitPacked>, + value: T, + nullability: Nullability, + operator: CompareOperator, + ctx: &mut ExecutionCtx, +) -> VortexResult +where + T: NativePType + FastLanesComparable, + ::Bitpacked: UnsignedPType + BitPacking + BitPackingCompare, +{ + compare_constant_typed::<::Bitpacked, T>( + array, + value, + nullability, + operator, + ctx, + ) +} + +fn compare_constant_typed( + array: ArrayView<'_, BitPacked>, + value: T, + nullability: Nullability, + operator: CompareOperator, + ctx: &mut ExecutionCtx, +) -> VortexResult +where + T: NativePType + FastLanesComparable, + U: UnsignedPType + BitPacking + BitPackingCompare, +{ + let mut bits = + collect_chunk_masks::(array, |bit_width, packed_chunk, chunk_matches| unsafe { + U::unchecked_unpack_cmp( + bit_width, + packed_chunk, + chunk_matches, + |lhs, rhs| compare_values(lhs, rhs, operator), + value, + ); + }); + + if let Some(patches) = array.patches() { + apply_patch_predicate::(&mut bits, &patches, ctx, |patched| { + compare_values(patched, value, operator) + })?; + } + + Ok(BoolArray::new( + bits.freeze(), + array.validity()?.union_nullability(nullability), + ) + .into_array()) +} + +fn between_constant( + array: ArrayView<'_, BitPacked>, + lower: T, + upper: T, + nullability: Nullability, + options: &BetweenOptions, + ctx: &mut ExecutionCtx, +) -> VortexResult +where + T: NativePType + FastLanesComparable, + ::Bitpacked: UnsignedPType + BitPacking + BitPackingCompare, +{ + between_constant_typed::<::Bitpacked, T>( + array, + lower, + upper, + nullability, + options, + ctx, + ) +} + +fn between_constant_typed( + array: ArrayView<'_, BitPacked>, + lower: T, + upper: T, + nullability: Nullability, + options: &BetweenOptions, + ctx: &mut ExecutionCtx, +) -> VortexResult +where + T: NativePType + FastLanesComparable, + U: UnsignedPType + BitPacking + BitPackingCompare, +{ + let mut bits = collect_chunk_masks::(array, |bit_width, packed_chunk, lower_matches| { + let mut upper_matches = [false; 1024]; + + unsafe { + U::unchecked_unpack_cmp( + bit_width, + packed_chunk, + lower_matches, + |value, lower_bound| lower_matches_bound(lower_bound, value, options.lower_strict), + lower, + ); + U::unchecked_unpack_cmp( + bit_width, + packed_chunk, + &mut upper_matches, + |value, upper_bound| upper_matches_bound(value, upper_bound, options.upper_strict), + upper, + ); + } + + for (lower_match, upper_match) in lower_matches.iter_mut().zip(upper_matches) { + *lower_match &= upper_match; + } + }); + + if let Some(patches) = array.patches() { + apply_patch_predicate::(&mut bits, &patches, ctx, |patched| { + lower_matches_bound(lower, patched, options.lower_strict) + && upper_matches_bound(patched, upper, options.upper_strict) + })?; + } + + Ok(BoolArray::new( + bits.freeze(), + array.validity()?.union_nullability(nullability), + ) + .into_array()) +} + +fn collect_chunk_masks( + array: ArrayView<'_, BitPacked>, + mut fill_chunk: impl FnMut(usize, &[U], &mut [bool; 1024]), +) -> BitBufferMut +where + U: UnsignedPType + BitPacking, +{ + let bit_width = array.bit_width() as usize; + let packed = array.packed_slice::(); + let elems_per_chunk = 128 * bit_width / size_of::(); + let num_chunks = (array.offset() as usize + array.len()).div_ceil(1024); + + let mut remaining = array.len(); + let mut output = BitBufferMut::with_capacity(array.len()); + + for chunk_idx in 0..num_chunks { + let chunk_start = if chunk_idx == 0 { + array.offset() as usize + } else { + 0 + }; + let chunk_len = (1024 - chunk_start).min(remaining); + let chunk_end = chunk_start + chunk_len; + + let packed_chunk = &packed[chunk_idx * elems_per_chunk..][..elems_per_chunk]; + let mut chunk_matches = [false; 1024]; + fill_chunk(bit_width, packed_chunk, &mut chunk_matches); + + let chunk_bits = chunk_matches.into_iter().collect::(); + output.append_buffer(&chunk_bits.slice(chunk_start..chunk_end)); + remaining -= chunk_len; + } + + debug_assert_eq!(remaining, 0); + output +} + +fn apply_patch_predicate( + bits: &mut BitBufferMut, + patches: &Patches, + ctx: &mut ExecutionCtx, + mut predicate: impl FnMut(T) -> bool, +) -> VortexResult<()> +where + T: NativePType, +{ + let indices = patches.indices().clone().execute::(ctx)?; + let values = patches.values().clone().execute::(ctx)?; + let values = values.as_slice::(); + let offset = patches.offset(); + + match_each_unsigned_integer_ptype!(indices.ptype(), |I| { + for (&index, &value) in indices.as_slice::().iter().zip(values) { + let absolute_index: usize = index.as_(); + if absolute_index < offset { + continue; + } + + let bit_index = absolute_index - offset; + if bit_index >= bits.len() { + break; + } + + bits.set_to(bit_index, predicate(value)); + } + Ok(()) + }) +} + +#[inline] +fn compare_values(lhs: T, rhs: T, operator: CompareOperator) -> bool { + match operator { + CompareOperator::Eq => lhs.is_eq(rhs), + CompareOperator::NotEq => !lhs.is_eq(rhs), + CompareOperator::Gt => lhs.is_gt(rhs), + CompareOperator::Gte => lhs.is_ge(rhs), + CompareOperator::Lt => lhs.is_lt(rhs), + CompareOperator::Lte => lhs.is_le(rhs), + } +} + +#[inline] +fn lower_matches_bound(lower: T, value: T, strict: StrictComparison) -> bool { + match strict { + StrictComparison::Strict => lower.is_lt(value), + StrictComparison::NonStrict => lower.is_le(value), + } +} + +#[inline] +fn upper_matches_bound(value: T, upper: T, strict: StrictComparison) -> bool { + match strict { + StrictComparison::Strict => value.is_lt(upper), + StrictComparison::NonStrict => value.is_le(upper), + } +} + +#[cfg(test)] +mod tests { + use vortex_array::Canonical; + use vortex_array::IntoArray; + use vortex_array::LEGACY_SESSION; + use vortex_array::VortexSessionExecute; + use vortex_array::arrays::BoolArray; + use vortex_array::arrays::ConstantArray; + use vortex_array::arrays::PrimitiveArray; + use vortex_array::assert_arrays_eq; + use vortex_array::builtins::ArrayBuiltins; + use vortex_array::scalar_fn::fns::between::BetweenOptions; + use vortex_array::scalar_fn::fns::between::StrictComparison; + use vortex_array::scalar_fn::fns::binary::CompareKernel; + use vortex_array::scalar_fn::fns::operators::CompareOperator; + + use crate::BitPacked; + use crate::bitpack_compress::bitpack_encode; + use crate::bitpacking::array::BitPackedArrayExt; + + fn bp(array: &PrimitiveArray, bit_width: u8) -> crate::BitPackedArray { + bitpack_encode( + array, + bit_width, + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap() + } + + #[test] + fn compare_unsigned_constant() { + let array = bp(&PrimitiveArray::from_iter([1u32, 2, 3, 4, 5]), 3); + let rhs = ConstantArray::new(3u32, array.len()).into_array(); + + let result = ::compare( + array.as_view(), + &rhs, + CompareOperator::Gt, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap() + .unwrap(); + + assert_arrays_eq!( + result, + BoolArray::from_iter([false, false, false, true, true]) + ); + } + + #[test] + fn compare_signed_constant() { + let array = bp(&PrimitiveArray::from_iter([1i32, 2, 3, 4, 5]), 3); + let rhs = ConstantArray::new(2i32, array.len()).into_array(); + + let result = ::compare( + array.as_view(), + &rhs, + CompareOperator::Gte, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap() + .unwrap(); + + assert_arrays_eq!( + result, + BoolArray::from_iter([false, true, true, true, true]) + ); + } + + #[test] + fn compare_with_patches() { + let array = bp(&PrimitiveArray::from_iter(0u32..257), 8); + assert!(array.patches().is_some()); + + let rhs = ConstantArray::new(256u32, array.len()).into_array(); + let result = ::compare( + array.as_view(), + &rhs, + CompareOperator::Eq, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap() + .unwrap(); + + assert_arrays_eq!( + result, + BoolArray::from_indices( + array.len(), + [256usize], + vortex_array::validity::Validity::NonNullable, + ) + ); + } + + #[test] + fn compare_nullable() { + let array = bp( + &PrimitiveArray::from_option_iter([Some(1u16), None, Some(3), Some(4), None]), + 3, + ); + let rhs = ConstantArray::new(3u16, array.len()).into_array(); + + let result = ::compare( + array.as_view(), + &rhs, + CompareOperator::Eq, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap() + .unwrap(); + + assert_arrays_eq!( + result, + BoolArray::from_iter([Some(false), None, Some(true), Some(false), None]) + ); + } + + #[test] + fn binary_compare_pushdown_executes() { + let array = bp(&PrimitiveArray::from_iter([1u32, 2, 3, 4, 5]), 3).into_array(); + let rhs = ConstantArray::new(4u32, array.len()).into_array(); + + let result = array + .binary(rhs, CompareOperator::Lt.into()) + .unwrap() + .execute::(&mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .into_array(); + + assert_arrays_eq!( + result, + BoolArray::from_iter([true, true, true, false, false]) + ); + } + + #[test] + fn between_executes_in_encoded_space() { + let array = bp(&PrimitiveArray::from_iter(0u32..257), 8).into_array(); + let len = array.len(); + + let result = array + .between( + ConstantArray::new(255u32, len).into_array(), + ConstantArray::new(256u32, len).into_array(), + BetweenOptions { + lower_strict: StrictComparison::NonStrict, + upper_strict: StrictComparison::NonStrict, + }, + ) + .unwrap() + .execute::(&mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .into_array(); + + assert_arrays_eq!( + result, + BoolArray::from_indices( + len, + [255usize, 256], + vortex_array::validity::Validity::NonNullable, + ) + ); + } + + #[test] + fn between_strict_upper() { + let array = bp(&PrimitiveArray::from_iter([10i32, 11, 12, 13]), 4).into_array(); + let len = array.len(); + + let result = array + .between( + ConstantArray::new(10i32, len).into_array(), + ConstantArray::new(12i32, len).into_array(), + BetweenOptions { + lower_strict: StrictComparison::NonStrict, + upper_strict: StrictComparison::Strict, + }, + ) + .unwrap() + .execute::(&mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .into_array(); + + assert_arrays_eq!(result, BoolArray::from_iter([true, true, false, false])); + } +} diff --git a/encodings/fastlanes/src/bitpacking/compute/mod.rs b/encodings/fastlanes/src/bitpacking/compute/mod.rs index 2501d952356..169546d311c 100644 --- a/encodings/fastlanes/src/bitpacking/compute/mod.rs +++ b/encodings/fastlanes/src/bitpacking/compute/mod.rs @@ -2,6 +2,7 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors mod cast; +mod compare; mod filter; pub(crate) mod is_constant; mod slice; diff --git a/encodings/fastlanes/src/bitpacking/vtable/kernels.rs b/encodings/fastlanes/src/bitpacking/vtable/kernels.rs index cb020dc2ce9..be790c9cca8 100644 --- a/encodings/fastlanes/src/bitpacking/vtable/kernels.rs +++ b/encodings/fastlanes/src/bitpacking/vtable/kernels.rs @@ -5,12 +5,16 @@ use vortex_array::arrays::dict::TakeExecuteAdaptor; use vortex_array::arrays::filter::FilterExecuteAdaptor; use vortex_array::arrays::slice::SliceExecuteAdaptor; use vortex_array::kernel::ParentKernelSet; +use vortex_array::scalar_fn::fns::between::BetweenExecuteAdaptor; +use vortex_array::scalar_fn::fns::binary::CompareExecuteAdaptor; use vortex_array::scalar_fn::fns::cast::CastExecuteAdaptor; use crate::BitPacked; pub(crate) const PARENT_KERNELS: ParentKernelSet = ParentKernelSet::new(&[ ParentKernelSet::lift(&CastExecuteAdaptor(BitPacked)), + ParentKernelSet::lift(&CompareExecuteAdaptor(BitPacked)), + ParentKernelSet::lift(&BetweenExecuteAdaptor(BitPacked)), ParentKernelSet::lift(&FilterExecuteAdaptor(BitPacked)), ParentKernelSet::lift(&SliceExecuteAdaptor(BitPacked)), ParentKernelSet::lift(&TakeExecuteAdaptor(BitPacked)), From 10c87120fe33064db606d583effcc9c5e191bd8f Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Fri, 3 Apr 2026 18:59:17 +0100 Subject: [PATCH 02/14] Use patched version Signed-off-by: Adam Gutglick --- Cargo.lock | 3 +- Cargo.toml | 3 ++ .../src/bitpacking/compute/compare.rs | 42 ++++++++++--------- 3 files changed, 26 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6f2b7bb670e..ae05b220323 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3146,8 +3146,7 @@ checksum = "f8eb564c5c7423d25c886fb561d1e4ee69f72354d16918afa32c08811f6b6a55" [[package]] name = "fastlanes" version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "414cb755aee48ff7b0907995d2949c68c8c17900970076dff6a808e18e592d71" +source = "git+https://github.com/spiraldb/fastlanes.git?branch=adamg%2Factually-compare-packed-bools#aff6e2d94e0a48c5bfbebb4066607e97bda73b60" dependencies = [ "arrayref", "const_for", diff --git a/Cargo.toml b/Cargo.toml index fb87a953154..0c6b5e9949b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -407,3 +407,6 @@ debug = false debug-assertions = false strip = "debuginfo" incremental = false + +[patch.crates-io] +fastlanes = { git = "https://github.com/spiraldb/fastlanes.git", branch = "adamg/actually-compare-packed-bools" } diff --git a/encodings/fastlanes/src/bitpacking/compute/compare.rs b/encodings/fastlanes/src/bitpacking/compute/compare.rs index 8377c319469..e4efc7c59a1 100644 --- a/encodings/fastlanes/src/bitpacking/compute/compare.rs +++ b/encodings/fastlanes/src/bitpacking/compute/compare.rs @@ -22,8 +22,8 @@ use vortex_array::scalar_fn::fns::between::BetweenOptions; use vortex_array::scalar_fn::fns::between::StrictComparison; use vortex_array::scalar_fn::fns::binary::CompareKernel; use vortex_array::scalar_fn::fns::operators::CompareOperator; -use vortex_buffer::BitBuffer; use vortex_buffer::BitBufferMut; +use vortex_buffer::BufferMut; use vortex_error::VortexResult; use crate::BitPacked; @@ -167,7 +167,7 @@ where U: UnsignedPType + BitPacking + BitPackingCompare, { let mut bits = collect_chunk_masks::(array, |bit_width, packed_chunk, lower_matches| { - let mut upper_matches = [false; 1024]; + let mut upper_matches = [0u64; 16]; unsafe { U::unchecked_unpack_cmp( @@ -207,39 +207,41 @@ where fn collect_chunk_masks( array: ArrayView<'_, BitPacked>, - mut fill_chunk: impl FnMut(usize, &[U], &mut [bool; 1024]), + mut fill_chunk: impl FnMut(usize, &[U], &mut [u64; 16]), ) -> BitBufferMut where U: UnsignedPType + BitPacking, { + if array.is_empty() { + return BitBufferMut::empty(); + } + let bit_width = array.bit_width() as usize; let packed = array.packed_slice::(); let elems_per_chunk = 128 * bit_width / size_of::(); let num_chunks = (array.offset() as usize + array.len()).div_ceil(1024); - - let mut remaining = array.len(); - let mut output = BitBufferMut::with_capacity(array.len()); + let mut output = BufferMut::::with_capacity(num_chunks * 16); for chunk_idx in 0..num_chunks { - let chunk_start = if chunk_idx == 0 { - array.offset() as usize - } else { - 0 - }; - let chunk_len = (1024 - chunk_start).min(remaining); - let chunk_end = chunk_start + chunk_len; - let packed_chunk = &packed[chunk_idx * elems_per_chunk..][..elems_per_chunk]; - let mut chunk_matches = [false; 1024]; + let mut chunk_matches = [0u64; 16]; fill_chunk(bit_width, packed_chunk, &mut chunk_matches); + output.extend_from_slice(&chunk_matches); + } - let chunk_bits = chunk_matches.into_iter().collect::(); - output.append_buffer(&chunk_bits.slice(chunk_start..chunk_end)); - remaining -= chunk_len; + let total_len = num_chunks * 1024; + let mut output = BitBufferMut::from_buffer(output.into_byte_buffer(), 0, total_len); + + if array.offset() == 0 { + output.truncate(array.len()); + return output; } - debug_assert_eq!(remaining, 0); - output + BitBufferMut::copy_from( + &output + .freeze() + .slice(array.offset() as usize..array.offset() as usize + array.len()), + ) } fn apply_patch_predicate( From e1067fd598882ac67527915ca60fd0a54536b3fd Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Fri, 3 Apr 2026 19:46:32 +0100 Subject: [PATCH 03/14] why not Signed-off-by: Adam Gutglick --- .../src/bitpacking/compute/compare.rs | 152 +++++++++++++++--- 1 file changed, 132 insertions(+), 20 deletions(-) diff --git a/encodings/fastlanes/src/bitpacking/compute/compare.rs b/encodings/fastlanes/src/bitpacking/compute/compare.rs index e4efc7c59a1..e78188fc503 100644 --- a/encodings/fastlanes/src/bitpacking/compute/compare.rs +++ b/encodings/fastlanes/src/bitpacking/compute/compare.rs @@ -166,30 +166,44 @@ where T: NativePType + FastLanesComparable, U: UnsignedPType + BitPacking + BitPackingCompare, { - let mut bits = collect_chunk_masks::(array, |bit_width, packed_chunk, lower_matches| { - let mut upper_matches = [0u64; 16]; - - unsafe { - U::unchecked_unpack_cmp( - bit_width, - packed_chunk, - lower_matches, - |value, lower_bound| lower_matches_bound(lower_bound, value, options.lower_strict), + let mut bits = match (options.lower_strict, options.upper_strict) { + (StrictComparison::Strict, StrictComparison::Strict) => { + collect_between_masks::( + array, lower, - ); - U::unchecked_unpack_cmp( - bit_width, - packed_chunk, - &mut upper_matches, - |value, upper_bound| upper_matches_bound(value, upper_bound, options.upper_strict), upper, - ); + NativePType::is_lt, + NativePType::is_lt, + ) } - - for (lower_match, upper_match) in lower_matches.iter_mut().zip(upper_matches) { - *lower_match &= upper_match; + (StrictComparison::Strict, StrictComparison::NonStrict) => { + collect_between_masks::( + array, + lower, + upper, + NativePType::is_lt, + NativePType::is_le, + ) + } + (StrictComparison::NonStrict, StrictComparison::Strict) => { + collect_between_masks::( + array, + lower, + upper, + NativePType::is_le, + NativePType::is_lt, + ) + } + (StrictComparison::NonStrict, StrictComparison::NonStrict) => { + collect_between_masks::( + array, + lower, + upper, + NativePType::is_le, + NativePType::is_le, + ) } - }); + }; if let Some(patches) = array.patches() { apply_patch_predicate::(&mut bits, &patches, ctx, |patched| { @@ -205,6 +219,31 @@ where .into_array()) } +fn collect_between_masks( + array: &BitPackedData, + lower: T, + upper: T, + lower_matches: LF, + upper_matches: UF, +) -> BitBufferMut +where + T: NativePType + FastLanesComparable, + U: UnsignedPType + BitPacking, + LF: Fn(T, T) -> bool + Copy, + UF: Fn(T, T) -> bool + Copy, +{ + collect_unpacked_chunk_masks::(array, |unpacked, chunk_matches| { + fill_between_chunk::( + unpacked, + chunk_matches, + lower, + upper, + lower_matches, + upper_matches, + ); + }) +} + fn collect_chunk_masks( array: ArrayView<'_, BitPacked>, mut fill_chunk: impl FnMut(usize, &[U], &mut [u64; 16]), @@ -244,6 +283,79 @@ where ) } +fn collect_unpacked_chunk_masks( + array: &BitPackedData, + mut fill_chunk: impl FnMut(&[U; 1024], &mut [u64; 16]), +) -> BitBufferMut +where + U: UnsignedPType + BitPacking, +{ + if array.is_empty() { + return BitBufferMut::empty(); + } + + let bit_width = array.bit_width() as usize; + let packed = array.packed_slice::(); + let elems_per_chunk = 128 * bit_width / size_of::(); + let num_chunks = (array.offset() as usize + array.len()).div_ceil(1024); + let mut output = BufferMut::::with_capacity(num_chunks * 16); + + for chunk_idx in 0..num_chunks { + let packed_chunk = &packed[chunk_idx * elems_per_chunk..][..elems_per_chunk]; + let mut unpacked = [U::default(); 1024]; + let mut chunk_matches = [0u64; 16]; + + unsafe { + U::unchecked_unpack(bit_width, packed_chunk, &mut unpacked); + } + + fill_chunk(&unpacked, &mut chunk_matches); + output.extend_from_slice(&chunk_matches); + } + + let total_len = num_chunks * 1024; + let mut output = BitBufferMut::from_buffer(output.into_byte_buffer(), 0, total_len); + + if array.offset() == 0 { + output.truncate(array.len()); + return output; + } + + BitBufferMut::copy_from( + &output + .freeze() + .slice(array.offset() as usize..array.offset() as usize + array.len()), + ) +} + +#[inline] +fn fill_between_chunk( + unpacked: &[U; 1024], + chunk_matches: &mut [u64; 16], + lower: T, + upper: T, + lower_matches: LF, + upper_matches: UF, +) where + T: NativePType + FastLanesComparable, + U: UnsignedPType, + LF: Fn(T, T) -> bool, + UF: Fn(T, T) -> bool, +{ + for (word_idx, word) in chunk_matches.iter_mut().enumerate() { + let start = word_idx * 64; + let mut mask = 0u64; + + for bit_idx in 0..64 { + let value = T::as_unpacked(unpacked[start + bit_idx]); + mask |= + u64::from(lower_matches(lower, value) && upper_matches(value, upper)) << bit_idx; + } + + *word = mask; + } +} + fn apply_patch_predicate( bits: &mut BitBufferMut, patches: &Patches, From 54691bc7cf5edc63c02abcc3448fe7862807df62 Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Fri, 3 Apr 2026 20:11:35 +0100 Subject: [PATCH 04/14] reuse Signed-off-by: Adam Gutglick --- encodings/fastlanes/src/bitpacking/compute/compare.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/encodings/fastlanes/src/bitpacking/compute/compare.rs b/encodings/fastlanes/src/bitpacking/compute/compare.rs index e78188fc503..5caed110e7f 100644 --- a/encodings/fastlanes/src/bitpacking/compute/compare.rs +++ b/encodings/fastlanes/src/bitpacking/compute/compare.rs @@ -299,11 +299,12 @@ where let elems_per_chunk = 128 * bit_width / size_of::(); let num_chunks = (array.offset() as usize + array.len()).div_ceil(1024); let mut output = BufferMut::::with_capacity(num_chunks * 16); + let mut unpacked = [U::default(); 1024]; + let mut chunk_matches = [0u64; 16]; for chunk_idx in 0..num_chunks { let packed_chunk = &packed[chunk_idx * elems_per_chunk..][..elems_per_chunk]; - let mut unpacked = [U::default(); 1024]; - let mut chunk_matches = [0u64; 16]; + chunk_matches.fill(0); unsafe { U::unchecked_unpack(bit_width, packed_chunk, &mut unpacked); From 6e5f4958905c9c238fea7f0eff8f74295fefe150 Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Fri, 3 Apr 2026 20:36:30 +0100 Subject: [PATCH 05/14] public APIs Signed-off-by: Adam Gutglick --- encodings/fastlanes/public-api.lock | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/encodings/fastlanes/public-api.lock b/encodings/fastlanes/public-api.lock index 4f0ce3df18c..bd3f2b88a27 100644 --- a/encodings/fastlanes/public-api.lock +++ b/encodings/fastlanes/public-api.lock @@ -194,6 +194,14 @@ impl vortex_array::scalar_fn::fns::cast::kernel::CastKernel for vortex_fastlanes pub fn vortex_fastlanes::BitPacked::cast(vortex_array::array::view::ArrayView<'_, Self>, &vortex_array::dtype::DType, &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult> +impl vortex_array::scalar_fn::fns::between::kernel::BetweenKernel for vortex_fastlanes::BitPacked + +pub fn vortex_fastlanes::BitPacked::between(array: vortex_array::array::view::ArrayView<'_, Self>, lower: &vortex_array::array::erased::ArrayRef, upper: &vortex_array::array::erased::ArrayRef, options: &vortex_array::scalar_fn::fns::between::BetweenOptions, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult> + +impl vortex_array::scalar_fn::fns::binary::compare::CompareKernel for vortex_fastlanes::BitPacked + +pub fn vortex_fastlanes::BitPacked::compare(lhs: vortex_array::array::view::ArrayView<'_, Self>, rhs: &vortex_array::array::erased::ArrayRef, operator: vortex_array::scalar_fn::fns::operators::CompareOperator, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult> + impl vortex_array::scalar_fn::fns::cast::kernel::CastReduce for vortex_fastlanes::BitPacked pub fn vortex_fastlanes::BitPacked::cast(vortex_array::array::view::ArrayView<'_, Self>, &vortex_array::dtype::DType) -> vortex_error::VortexResult> From 10fe02645baa446fd7fa279b66e7e8274adf9a30 Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Fri, 3 Apr 2026 20:54:27 +0100 Subject: [PATCH 06/14] fix Signed-off-by: Adam Gutglick --- .../src/bitpacking/compute/compare.rs | 60 ++++++++++++------- 1 file changed, 40 insertions(+), 20 deletions(-) diff --git a/encodings/fastlanes/src/bitpacking/compute/compare.rs b/encodings/fastlanes/src/bitpacking/compute/compare.rs index 5caed110e7f..f21369b7e8c 100644 --- a/encodings/fastlanes/src/bitpacking/compute/compare.rs +++ b/encodings/fastlanes/src/bitpacking/compute/compare.rs @@ -28,6 +28,7 @@ use vortex_error::VortexResult; use crate::BitPacked; use crate::BitPackedArrayExt; +use crate::BitPackedData; impl CompareKernel for BitPacked { fn compare( @@ -43,6 +44,7 @@ impl CompareKernel for BitPacked { if !constant.dtype().is_int() { return Ok(None); } + match_each_integer_ptype!(lhs.dtype().as_ptype(), |T| { let value = T::try_from(&constant)?; compare_constant::(lhs, value, rhs.dtype().nullability(), operator, ctx).map(Some) @@ -108,8 +110,11 @@ where T: NativePType + FastLanesComparable, U: UnsignedPType + BitPacking + BitPackingCompare, { - let mut bits = - collect_chunk_masks::(array, |bit_width, packed_chunk, chunk_matches| unsafe { + let mut bits = collect_chunk_masks::( + array.data(), + array.len(), + array.offset(), + |bit_width, packed_chunk, chunk_matches| unsafe { U::unchecked_unpack_cmp( bit_width, packed_chunk, @@ -117,7 +122,8 @@ where |lhs, rhs| compare_values(lhs, rhs, operator), value, ); - }); + }, + ); if let Some(patches) = array.patches() { apply_patch_predicate::(&mut bits, &patches, ctx, |patched| { @@ -169,7 +175,9 @@ where let mut bits = match (options.lower_strict, options.upper_strict) { (StrictComparison::Strict, StrictComparison::Strict) => { collect_between_masks::( - array, + array.data(), + array.len(), + array.offset(), lower, upper, NativePType::is_lt, @@ -178,7 +186,9 @@ where } (StrictComparison::Strict, StrictComparison::NonStrict) => { collect_between_masks::( - array, + array.data(), + array.len(), + array.offset(), lower, upper, NativePType::is_lt, @@ -187,7 +197,9 @@ where } (StrictComparison::NonStrict, StrictComparison::Strict) => { collect_between_masks::( - array, + array.data(), + array.len(), + array.offset(), lower, upper, NativePType::is_le, @@ -196,7 +208,9 @@ where } (StrictComparison::NonStrict, StrictComparison::NonStrict) => { collect_between_masks::( - array, + array.data(), + array.len(), + array.offset(), lower, upper, NativePType::is_le, @@ -221,6 +235,8 @@ where fn collect_between_masks( array: &BitPackedData, + len: usize, + offset: u16, lower: T, upper: T, lower_matches: LF, @@ -232,7 +248,7 @@ where LF: Fn(T, T) -> bool + Copy, UF: Fn(T, T) -> bool + Copy, { - collect_unpacked_chunk_masks::(array, |unpacked, chunk_matches| { + collect_unpacked_chunk_masks::(array, len, offset, |unpacked, chunk_matches| { fill_between_chunk::( unpacked, chunk_matches, @@ -245,20 +261,22 @@ where } fn collect_chunk_masks( - array: ArrayView<'_, BitPacked>, + array: &BitPackedData, + len: usize, + offset: u16, mut fill_chunk: impl FnMut(usize, &[U], &mut [u64; 16]), ) -> BitBufferMut where U: UnsignedPType + BitPacking, { - if array.is_empty() { + if len == 0 { return BitBufferMut::empty(); } let bit_width = array.bit_width() as usize; let packed = array.packed_slice::(); let elems_per_chunk = 128 * bit_width / size_of::(); - let num_chunks = (array.offset() as usize + array.len()).div_ceil(1024); + let num_chunks = (offset as usize + len).div_ceil(1024); let mut output = BufferMut::::with_capacity(num_chunks * 16); for chunk_idx in 0..num_chunks { @@ -271,33 +289,35 @@ where let total_len = num_chunks * 1024; let mut output = BitBufferMut::from_buffer(output.into_byte_buffer(), 0, total_len); - if array.offset() == 0 { - output.truncate(array.len()); + if offset == 0 { + output.truncate(len); return output; } BitBufferMut::copy_from( &output .freeze() - .slice(array.offset() as usize..array.offset() as usize + array.len()), + .slice(offset as usize..offset as usize + len), ) } fn collect_unpacked_chunk_masks( array: &BitPackedData, + len: usize, + offset: u16, mut fill_chunk: impl FnMut(&[U; 1024], &mut [u64; 16]), ) -> BitBufferMut where U: UnsignedPType + BitPacking, { - if array.is_empty() { + if len == 0 { return BitBufferMut::empty(); } let bit_width = array.bit_width() as usize; let packed = array.packed_slice::(); let elems_per_chunk = 128 * bit_width / size_of::(); - let num_chunks = (array.offset() as usize + array.len()).div_ceil(1024); + let num_chunks = (offset as usize + len).div_ceil(1024); let mut output = BufferMut::::with_capacity(num_chunks * 16); let mut unpacked = [U::default(); 1024]; let mut chunk_matches = [0u64; 16]; @@ -317,15 +337,15 @@ where let total_len = num_chunks * 1024; let mut output = BitBufferMut::from_buffer(output.into_byte_buffer(), 0, total_len); - if array.offset() == 0 { - output.truncate(array.len()); + if offset == 0 { + output.truncate(len); return output; } BitBufferMut::copy_from( &output .freeze() - .slice(array.offset() as usize..array.offset() as usize + array.len()), + .slice(offset as usize..offset as usize + len), ) } @@ -434,8 +454,8 @@ mod tests { use vortex_array::scalar_fn::fns::operators::CompareOperator; use crate::BitPacked; + use crate::BitPackedArrayExt; use crate::bitpack_compress::bitpack_encode; - use crate::bitpacking::array::BitPackedArrayExt; fn bp(array: &PrimitiveArray, bit_width: u8) -> crate::BitPackedArray { bitpack_encode( From a0b5917d556949afbcbd0870dac8ba3346f63561 Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Fri, 3 Apr 2026 21:19:59 +0100 Subject: [PATCH 07/14] vroom vroom Signed-off-by: Adam Gutglick --- .../fastlanes/src/bitpacking/array/mod.rs | 3 +- .../src/bitpacking/compute/compare.rs | 42 +++++++++++-------- encodings/fastlanes/src/delta/compute/cast.rs | 14 +++---- encodings/sequence/src/compute/compare.rs | 5 ++- encodings/zstd/src/zstd_buffers.rs | 8 ++-- .../src/expr/transform/match_between.rs | 6 ++- 6 files changed, 44 insertions(+), 34 deletions(-) diff --git a/encodings/fastlanes/src/bitpacking/array/mod.rs b/encodings/fastlanes/src/bitpacking/array/mod.rs index e5c64252fbc..d799680cb85 100644 --- a/encodings/fastlanes/src/bitpacking/array/mod.rs +++ b/encodings/fastlanes/src/bitpacking/array/mod.rs @@ -328,6 +328,7 @@ mod test { use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; use vortex_array::session::ArraySession; + use vortex_array::validity::Validity; use vortex_buffer::Buffer; use vortex_session::VortexSession; @@ -386,7 +387,7 @@ mod test { .unwrap(); assert_arrays_eq!( packed_primitive, - PrimitiveArray::new(values, vortex_array::validity::Validity::NonNullable) + PrimitiveArray::new(values, Validity::NonNullable) ); } } diff --git a/encodings/fastlanes/src/bitpacking/compute/compare.rs b/encodings/fastlanes/src/bitpacking/compute/compare.rs index f21369b7e8c..dae8c16ead0 100644 --- a/encodings/fastlanes/src/bitpacking/compute/compare.rs +++ b/encodings/fastlanes/src/bitpacking/compute/compare.rs @@ -281,9 +281,9 @@ where for chunk_idx in 0..num_chunks { let packed_chunk = &packed[chunk_idx * elems_per_chunk..][..elems_per_chunk]; - let mut chunk_matches = [0u64; 16]; - fill_chunk(bit_width, packed_chunk, &mut chunk_matches); - output.extend_from_slice(&chunk_matches); + append_chunk_matches(&mut output, |chunk_matches| { + fill_chunk(bit_width, packed_chunk, chunk_matches); + }); } let total_len = num_chunks * 1024; @@ -320,18 +320,17 @@ where let num_chunks = (offset as usize + len).div_ceil(1024); let mut output = BufferMut::::with_capacity(num_chunks * 16); let mut unpacked = [U::default(); 1024]; - let mut chunk_matches = [0u64; 16]; for chunk_idx in 0..num_chunks { let packed_chunk = &packed[chunk_idx * elems_per_chunk..][..elems_per_chunk]; - chunk_matches.fill(0); unsafe { U::unchecked_unpack(bit_width, packed_chunk, &mut unpacked); } - fill_chunk(&unpacked, &mut chunk_matches); - output.extend_from_slice(&chunk_matches); + append_chunk_matches(&mut output, |chunk_matches| { + fill_chunk(&unpacked, chunk_matches); + }); } let total_len = num_chunks * 1024; @@ -349,6 +348,22 @@ where ) } +#[inline] +fn append_chunk_matches(output: &mut BufferMut, fill_chunk: impl FnOnce(&mut [u64; 16])) { + let base_len = output.len(); + + let spare = output.spare_capacity_mut(); + debug_assert!(spare.len() >= 16); + let chunk_matches = unsafe { &mut *(spare.as_mut_ptr().cast::<[u64; 16]>()) }; + + fill_chunk(chunk_matches); + + // SAFETY: `fill_chunk` initializes all 16 words before we expose them via `set_len`. + unsafe { + output.set_len(base_len + 16); + } +} + #[inline] fn fill_between_chunk( unpacked: &[U; 1024], @@ -452,6 +467,7 @@ mod tests { use vortex_array::scalar_fn::fns::between::StrictComparison; use vortex_array::scalar_fn::fns::binary::CompareKernel; use vortex_array::scalar_fn::fns::operators::CompareOperator; + use vortex_array::validity::Validity; use crate::BitPacked; use crate::BitPackedArrayExt; @@ -524,11 +540,7 @@ mod tests { assert_arrays_eq!( result, - BoolArray::from_indices( - array.len(), - [256usize], - vortex_array::validity::Validity::NonNullable, - ) + BoolArray::from_indices(array.len(), [256usize], Validity::NonNullable,) ); } @@ -594,11 +606,7 @@ mod tests { assert_arrays_eq!( result, - BoolArray::from_indices( - len, - [255usize, 256], - vortex_array::validity::Validity::NonNullable, - ) + BoolArray::from_indices(len, [255usize, 256], Validity::NonNullable,) ); } diff --git a/encodings/fastlanes/src/delta/compute/cast.rs b/encodings/fastlanes/src/delta/compute/cast.rs index 43a247df9f0..6520937e9b1 100644 --- a/encodings/fastlanes/src/delta/compute/cast.rs +++ b/encodings/fastlanes/src/delta/compute/cast.rs @@ -61,6 +61,7 @@ mod tests { use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; use vortex_array::session::ArraySession; + use vortex_array::validity::Validity; use vortex_buffer::buffer; use vortex_session::VortexSession; @@ -92,10 +93,7 @@ mod tests { fn test_cast_delta_nullable() { // DeltaArray doesn't support nullable arrays - the validity is handled at the DeltaArray level // Create a non-nullable array and then add validity to the DeltaArray - let values = PrimitiveArray::new( - buffer![100u16, 0, 200, 300, 0], - vortex_array::validity::Validity::NonNullable, - ); + let values = PrimitiveArray::new(buffer![100u16, 0, 200, 300, 0], Validity::NonNullable); let array = Delta::try_from_primitive_array(&values, &mut SESSION.create_execution_ctx()).unwrap(); @@ -113,25 +111,25 @@ mod tests { #[case::u8( PrimitiveArray::new( buffer![0u8, 10, 20, 30, 40, 50], - vortex_array::validity::Validity::NonNullable, + Validity::NonNullable, ) )] #[case::u16( PrimitiveArray::new( buffer![0u16, 100, 200, 300, 400, 500], - vortex_array::validity::Validity::NonNullable, + Validity::NonNullable, ) )] #[case::u32( PrimitiveArray::new( buffer![0u32, 1000, 2000, 3000, 4000], - vortex_array::validity::Validity::NonNullable, + Validity::NonNullable, ) )] #[case::u64( PrimitiveArray::new( buffer![0u64, 10000, 20000, 30000], - vortex_array::validity::Validity::NonNullable, + Validity::NonNullable, ) )] fn test_cast_delta_conformance(#[case] primitive: PrimitiveArray) { diff --git a/encodings/sequence/src/compute/compare.rs b/encodings/sequence/src/compute/compare.rs index c0c9d1367d6..7a0a69cea20 100644 --- a/encodings/sequence/src/compute/compare.rs +++ b/encodings/sequence/src/compute/compare.rs @@ -14,6 +14,7 @@ use vortex_array::scalar::PValue; use vortex_array::scalar::Scalar; use vortex_array::scalar_fn::fns::binary::CompareKernel; use vortex_array::scalar_fn::fns::operators::CompareOperator; +use vortex_array::validity::Validity; use vortex_buffer::BitBuffer; use vortex_error::VortexExpect; use vortex_error::VortexResult; @@ -51,8 +52,8 @@ impl CompareKernel for Sequence { let nullability = lhs.dtype().nullability() | rhs.dtype().nullability(); let validity = match nullability { - Nullability::NonNullable => vortex_array::validity::Validity::NonNullable, - Nullability::Nullable => vortex_array::validity::Validity::AllValid, + Nullability::NonNullable => Validity::NonNullable, + Nullability::Nullable => Validity::AllValid, }; if let Ok(set_idx) = set_idx { diff --git a/encodings/zstd/src/zstd_buffers.rs b/encodings/zstd/src/zstd_buffers.rs index c05fe82da06..2ec2f6c1c70 100644 --- a/encodings/zstd/src/zstd_buffers.rs +++ b/encodings/zstd/src/zstd_buffers.rs @@ -25,6 +25,8 @@ use vortex_array::dtype::DType; use vortex_array::scalar::Scalar; use vortex_array::serde::ArrayChildren; use vortex_array::session::ArraySessionExt; +use vortex_array::validity::Validity; +use vortex_array::vtable; use vortex_array::vtable::OperationsVTable; use vortex_array::vtable::VTable; use vortex_array::vtable::ValidityVTable; @@ -472,11 +474,9 @@ impl OperationsVTable for ZstdBuffers { } impl ValidityVTable for ZstdBuffers { - fn validity( - array: ArrayView<'_, ZstdBuffers>, - ) -> VortexResult { + fn validity(array: ArrayView<'_, ZstdBuffers>) -> VortexResult { if !array.dtype().is_nullable() { - return Ok(vortex_array::validity::Validity::NonNullable); + return Ok(Validity::NonNullable); } let inner_array = ZstdBuffers::decompress_and_build_inner( diff --git a/vortex-array/src/expr/transform/match_between.rs b/vortex-array/src/expr/transform/match_between.rs index 56f03dbac4d..5d66e7d1c67 100644 --- a/vortex-array/src/expr/transform/match_between.rs +++ b/vortex-array/src/expr/transform/match_between.rs @@ -1,10 +1,11 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use vortex_error::VortexExpect; + use crate::expr::Expression; use crate::expr::and_collect; use crate::expr::forms::conjuncts; -use crate::expr::lit; use crate::scalar_fn::ScalarFnVTableExt; use crate::scalar_fn::fns::between::Between; use crate::scalar_fn::fns::between::BetweenOptions; @@ -45,7 +46,8 @@ pub fn find_between(expr: Expression) -> Expression { } } - and_collect(rest).unwrap_or_else(|| lit(true)) + debug_assert!(!rest.is_empty()); + and_collect(rest).vortex_expect("find_between must produce at least one conjunct") } fn maybe_match(lhs: &Expression, rhs: &Expression) -> Option { From a0b9c3b80b14a2d5a38ecc8dc038e87c87211c88 Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Fri, 3 Apr 2026 21:44:14 +0100 Subject: [PATCH 08/14] better Signed-off-by: Adam Gutglick --- .../src/bitpacking/compute/compare.rs | 57 ++++++++++++------- 1 file changed, 38 insertions(+), 19 deletions(-) diff --git a/encodings/fastlanes/src/bitpacking/compute/compare.rs b/encodings/fastlanes/src/bitpacking/compute/compare.rs index dae8c16ead0..faba9de0fa8 100644 --- a/encodings/fastlanes/src/bitpacking/compute/compare.rs +++ b/encodings/fastlanes/src/bitpacking/compute/compare.rs @@ -109,26 +109,52 @@ fn compare_constant_typed( where T: NativePType + FastLanesComparable, U: UnsignedPType + BitPacking + BitPackingCompare, +{ + match operator { + CompareOperator::Eq => { + compare_constant_with::(array, value, nullability, ctx, T::is_eq) + } + CompareOperator::NotEq => { + compare_constant_with::(array, value, nullability, ctx, is_ne::) + } + CompareOperator::Gt => { + compare_constant_with::(array, value, nullability, ctx, T::is_gt) + } + CompareOperator::Gte => { + compare_constant_with::(array, value, nullability, ctx, T::is_ge) + } + CompareOperator::Lt => { + compare_constant_with::(array, value, nullability, ctx, T::is_lt) + } + CompareOperator::Lte => { + compare_constant_with::(array, value, nullability, ctx, T::is_le) + } + } +} + +fn compare_constant_with( + array: ArrayView<'_, BitPacked>, + value: T, + nullability: Nullability, + ctx: &mut ExecutionCtx, + compare: C, +) -> VortexResult +where + T: NativePType + FastLanesComparable, + U: UnsignedPType + BitPacking + BitPackingCompare, + C: Fn(T, T) -> bool + Copy, { let mut bits = collect_chunk_masks::( array.data(), array.len(), array.offset(), |bit_width, packed_chunk, chunk_matches| unsafe { - U::unchecked_unpack_cmp( - bit_width, - packed_chunk, - chunk_matches, - |lhs, rhs| compare_values(lhs, rhs, operator), - value, - ); + U::unchecked_unpack_cmp(bit_width, packed_chunk, chunk_matches, compare, value); }, ); if let Some(patches) = array.patches() { - apply_patch_predicate::(&mut bits, &patches, ctx, |patched| { - compare_values(patched, value, operator) - })?; + apply_patch_predicate::(&mut bits, &patches, ctx, |patched| compare(patched, value))?; } Ok(BoolArray::new( @@ -425,15 +451,8 @@ where } #[inline] -fn compare_values(lhs: T, rhs: T, operator: CompareOperator) -> bool { - match operator { - CompareOperator::Eq => lhs.is_eq(rhs), - CompareOperator::NotEq => !lhs.is_eq(rhs), - CompareOperator::Gt => lhs.is_gt(rhs), - CompareOperator::Gte => lhs.is_ge(rhs), - CompareOperator::Lt => lhs.is_lt(rhs), - CompareOperator::Lte => lhs.is_le(rhs), - } +fn is_ne(lhs: T, rhs: T) -> bool { + !lhs.is_eq(rhs) } #[inline] From bc6b5c12fa703c46b0cfd347a5de02ebcec79057 Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Fri, 3 Apr 2026 21:56:49 +0100 Subject: [PATCH 09/14] last thing Signed-off-by: Adam Gutglick --- Cargo.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index ae05b220323..80033d50ec5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3146,7 +3146,7 @@ checksum = "f8eb564c5c7423d25c886fb561d1e4ee69f72354d16918afa32c08811f6b6a55" [[package]] name = "fastlanes" version = "0.5.0" -source = "git+https://github.com/spiraldb/fastlanes.git?branch=adamg%2Factually-compare-packed-bools#aff6e2d94e0a48c5bfbebb4066607e97bda73b60" +source = "git+https://github.com/spiraldb/fastlanes.git?branch=adamg%2Factually-compare-packed-bools#56236df6286acca69d63036d7d8681f615d135ac" dependencies = [ "arrayref", "const_for", From a4f60b0efb511aa2f1dba2d211f002cd7b76548f Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Fri, 3 Apr 2026 23:41:21 +0100 Subject: [PATCH 10/14] another update Signed-off-by: Adam Gutglick --- Cargo.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 80033d50ec5..c21c3b57626 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3146,7 +3146,7 @@ checksum = "f8eb564c5c7423d25c886fb561d1e4ee69f72354d16918afa32c08811f6b6a55" [[package]] name = "fastlanes" version = "0.5.0" -source = "git+https://github.com/spiraldb/fastlanes.git?branch=adamg%2Factually-compare-packed-bools#56236df6286acca69d63036d7d8681f615d135ac" +source = "git+https://github.com/spiraldb/fastlanes.git?branch=adamg%2Factually-compare-packed-bools#109c29f25b66a351bf6cb814335d3dff595680be" dependencies = [ "arrayref", "const_for", From b740548f291905306116de646e2946ac866663bb Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Fri, 3 Apr 2026 23:48:26 +0100 Subject: [PATCH 11/14] update2 Signed-off-by: Adam Gutglick --- Cargo.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index c21c3b57626..80033d50ec5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3146,7 +3146,7 @@ checksum = "f8eb564c5c7423d25c886fb561d1e4ee69f72354d16918afa32c08811f6b6a55" [[package]] name = "fastlanes" version = "0.5.0" -source = "git+https://github.com/spiraldb/fastlanes.git?branch=adamg%2Factually-compare-packed-bools#109c29f25b66a351bf6cb814335d3dff595680be" +source = "git+https://github.com/spiraldb/fastlanes.git?branch=adamg%2Factually-compare-packed-bools#56236df6286acca69d63036d7d8681f615d135ac" dependencies = [ "arrayref", "const_for", From e6e7e9099492cbf1ea2c10caef491f95ba5b4c45 Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Wed, 22 Apr 2026 14:10:37 +0100 Subject: [PATCH 12/14] Fix tests Signed-off-by: Adam Gutglick --- .../src/bitpacking/compute/compare.rs | 79 +++++++++++++------ encodings/zstd/src/zstd_buffers.rs | 1 - 2 files changed, 54 insertions(+), 26 deletions(-) diff --git a/encodings/fastlanes/src/bitpacking/compute/compare.rs b/encodings/fastlanes/src/bitpacking/compute/compare.rs index faba9de0fa8..6e27dff74df 100644 --- a/encodings/fastlanes/src/bitpacking/compute/compare.rs +++ b/encodings/fastlanes/src/bitpacking/compute/compare.rs @@ -492,26 +492,23 @@ mod tests { use crate::BitPackedArrayExt; use crate::bitpack_compress::bitpack_encode; - fn bp(array: &PrimitiveArray, bit_width: u8) -> crate::BitPackedArray { - bitpack_encode( - array, - bit_width, - None, - &mut LEGACY_SESSION.create_execution_ctx(), - ) - .unwrap() - } - #[test] fn compare_unsigned_constant() { - let array = bp(&PrimitiveArray::from_iter([1u32, 2, 3, 4, 5]), 3); + let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let array = bitpack_encode( + &PrimitiveArray::from_iter([1u32, 2, 3, 4, 5]), + 3, + None, + &mut ctx, + ) + .unwrap(); let rhs = ConstantArray::new(3u32, array.len()).into_array(); let result = ::compare( array.as_view(), &rhs, CompareOperator::Gt, - &mut LEGACY_SESSION.create_execution_ctx(), + &mut ctx, ) .unwrap() .unwrap(); @@ -524,14 +521,21 @@ mod tests { #[test] fn compare_signed_constant() { - let array = bp(&PrimitiveArray::from_iter([1i32, 2, 3, 4, 5]), 3); + let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let array = bitpack_encode( + &PrimitiveArray::from_iter([1i32, 2, 3, 4, 5]), + 3, + None, + &mut ctx, + ) + .unwrap(); let rhs = ConstantArray::new(2i32, array.len()).into_array(); let result = ::compare( array.as_view(), &rhs, CompareOperator::Gte, - &mut LEGACY_SESSION.create_execution_ctx(), + &mut ctx, ) .unwrap() .unwrap(); @@ -544,7 +548,9 @@ mod tests { #[test] fn compare_with_patches() { - let array = bp(&PrimitiveArray::from_iter(0u32..257), 8); + let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let array = + bitpack_encode(&PrimitiveArray::from_iter(0u32..257), 8, None, &mut ctx).unwrap(); assert!(array.patches().is_some()); let rhs = ConstantArray::new(256u32, array.len()).into_array(); @@ -552,7 +558,7 @@ mod tests { array.as_view(), &rhs, CompareOperator::Eq, - &mut LEGACY_SESSION.create_execution_ctx(), + &mut ctx, ) .unwrap() .unwrap(); @@ -565,17 +571,21 @@ mod tests { #[test] fn compare_nullable() { - let array = bp( + let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let array = bitpack_encode( &PrimitiveArray::from_option_iter([Some(1u16), None, Some(3), Some(4), None]), 3, - ); + None, + &mut ctx, + ) + .unwrap(); let rhs = ConstantArray::new(3u16, array.len()).into_array(); let result = ::compare( array.as_view(), &rhs, CompareOperator::Eq, - &mut LEGACY_SESSION.create_execution_ctx(), + &mut ctx, ) .unwrap() .unwrap(); @@ -588,13 +598,21 @@ mod tests { #[test] fn binary_compare_pushdown_executes() { - let array = bp(&PrimitiveArray::from_iter([1u32, 2, 3, 4, 5]), 3).into_array(); + let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let array = bitpack_encode( + &PrimitiveArray::from_iter([1u32, 2, 3, 4, 5]), + 3, + None, + &mut ctx, + ) + .unwrap() + .into_array(); let rhs = ConstantArray::new(4u32, array.len()).into_array(); let result = array .binary(rhs, CompareOperator::Lt.into()) .unwrap() - .execute::(&mut LEGACY_SESSION.create_execution_ctx()) + .execute::(&mut ctx) .unwrap() .into_array(); @@ -606,7 +624,10 @@ mod tests { #[test] fn between_executes_in_encoded_space() { - let array = bp(&PrimitiveArray::from_iter(0u32..257), 8).into_array(); + let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let array = bitpack_encode(&PrimitiveArray::from_iter(0u32..257), 8, None, &mut ctx) + .unwrap() + .into_array(); let len = array.len(); let result = array @@ -619,7 +640,7 @@ mod tests { }, ) .unwrap() - .execute::(&mut LEGACY_SESSION.create_execution_ctx()) + .execute::(&mut ctx) .unwrap() .into_array(); @@ -631,7 +652,15 @@ mod tests { #[test] fn between_strict_upper() { - let array = bp(&PrimitiveArray::from_iter([10i32, 11, 12, 13]), 4).into_array(); + let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let array = bitpack_encode( + &PrimitiveArray::from_iter([10i32, 11, 12, 13]), + 4, + None, + &mut ctx, + ) + .unwrap() + .into_array(); let len = array.len(); let result = array @@ -644,7 +673,7 @@ mod tests { }, ) .unwrap() - .execute::(&mut LEGACY_SESSION.create_execution_ctx()) + .execute::(&mut ctx) .unwrap() .into_array(); diff --git a/encodings/zstd/src/zstd_buffers.rs b/encodings/zstd/src/zstd_buffers.rs index 2ec2f6c1c70..a6709d7eac7 100644 --- a/encodings/zstd/src/zstd_buffers.rs +++ b/encodings/zstd/src/zstd_buffers.rs @@ -26,7 +26,6 @@ use vortex_array::scalar::Scalar; use vortex_array::serde::ArrayChildren; use vortex_array::session::ArraySessionExt; use vortex_array::validity::Validity; -use vortex_array::vtable; use vortex_array::vtable::OperationsVTable; use vortex_array::vtable::VTable; use vortex_array::vtable::ValidityVTable; From 526936f3d77f5e01e91769a341debea1fcf6217c Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Fri, 24 Apr 2026 10:00:13 +0100 Subject: [PATCH 13/14] refresh Signed-off-by: Adam Gutglick --- Cargo.lock | 2 +- encodings/fastlanes/public-api.lock | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 80033d50ec5..0fb0661b482 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3146,7 +3146,7 @@ checksum = "f8eb564c5c7423d25c886fb561d1e4ee69f72354d16918afa32c08811f6b6a55" [[package]] name = "fastlanes" version = "0.5.0" -source = "git+https://github.com/spiraldb/fastlanes.git?branch=adamg%2Factually-compare-packed-bools#56236df6286acca69d63036d7d8681f615d135ac" +source = "git+https://github.com/spiraldb/fastlanes.git?branch=adamg%2Factually-compare-packed-bools#9c31524ad93bac690984255f2880e4009e4addf7" dependencies = [ "arrayref", "const_for", diff --git a/encodings/fastlanes/public-api.lock b/encodings/fastlanes/public-api.lock index bd3f2b88a27..7af1f658017 100644 --- a/encodings/fastlanes/public-api.lock +++ b/encodings/fastlanes/public-api.lock @@ -190,17 +190,17 @@ impl vortex_array::arrays::slice::SliceReduce for vortex_fastlanes::BitPacked pub fn vortex_fastlanes::BitPacked::slice(vortex_array::array::view::ArrayView<'_, Self>, core::ops::range::Range) -> vortex_error::VortexResult> -impl vortex_array::scalar_fn::fns::cast::kernel::CastKernel for vortex_fastlanes::BitPacked - -pub fn vortex_fastlanes::BitPacked::cast(vortex_array::array::view::ArrayView<'_, Self>, &vortex_array::dtype::DType, &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult> - impl vortex_array::scalar_fn::fns::between::kernel::BetweenKernel for vortex_fastlanes::BitPacked -pub fn vortex_fastlanes::BitPacked::between(array: vortex_array::array::view::ArrayView<'_, Self>, lower: &vortex_array::array::erased::ArrayRef, upper: &vortex_array::array::erased::ArrayRef, options: &vortex_array::scalar_fn::fns::between::BetweenOptions, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_fastlanes::BitPacked::between(vortex_array::array::view::ArrayView<'_, Self>, &vortex_array::array::erased::ArrayRef, &vortex_array::array::erased::ArrayRef, &vortex_array::scalar_fn::fns::between::BetweenOptions, &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult> impl vortex_array::scalar_fn::fns::binary::compare::CompareKernel for vortex_fastlanes::BitPacked -pub fn vortex_fastlanes::BitPacked::compare(lhs: vortex_array::array::view::ArrayView<'_, Self>, rhs: &vortex_array::array::erased::ArrayRef, operator: vortex_array::scalar_fn::fns::operators::CompareOperator, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_fastlanes::BitPacked::compare(vortex_array::array::view::ArrayView<'_, Self>, &vortex_array::array::erased::ArrayRef, vortex_array::scalar_fn::fns::operators::CompareOperator, &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult> + +impl vortex_array::scalar_fn::fns::cast::kernel::CastKernel for vortex_fastlanes::BitPacked + +pub fn vortex_fastlanes::BitPacked::cast(vortex_array::array::view::ArrayView<'_, Self>, &vortex_array::dtype::DType, &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult> impl vortex_array::scalar_fn::fns::cast::kernel::CastReduce for vortex_fastlanes::BitPacked From 7a3b856852683ad9679539773036ceb3d2babaa0 Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Tue, 19 May 2026 12:05:59 +0100 Subject: [PATCH 14/14] lets try this one maybe? Signed-off-by: Adam Gutglick --- Cargo.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 0fb0661b482..793aee47991 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3146,7 +3146,7 @@ checksum = "f8eb564c5c7423d25c886fb561d1e4ee69f72354d16918afa32c08811f6b6a55" [[package]] name = "fastlanes" version = "0.5.0" -source = "git+https://github.com/spiraldb/fastlanes.git?branch=adamg%2Factually-compare-packed-bools#9c31524ad93bac690984255f2880e4009e4addf7" +source = "git+https://github.com/spiraldb/fastlanes.git?branch=adamg%2Factually-compare-packed-bools#989e7df8fb8015ae9cb0ffde7f31517a00aa5ffb" dependencies = [ "arrayref", "const_for",