initial

myrrc · myrrc · commit f0919240e10d · 2026-05-19T13:54:38.000+01:00
Signed-off-by: Mikhail Kot &lt;mikhail@spiraldb.com&gt;
diff --git a/vortex-duckdb/cpp/include/duckdb_vx/vector.h b/vortex-duckdb/cpp/include/duckdb_vx/vector.h
@@ -17,6 +17,9 @@ DUCKDB_INCLUDES_END
 extern "C" {
 #endif
 
+// Create a vector that slices another vector between a pair of offsets [offset, end];
+duckdb_vector duckdb_vx_vector_slice(duckdb_vector ffi_vector, idx_t offset, idx_t end);
+
 /// Slice the vector to a new dictionary vector, using the current vector's values and
 /// the provided selection vector.
 ///
diff --git a/vortex-duckdb/cpp/vector.cpp b/vortex-duckdb/cpp/vector.cpp
@@ -15,6 +15,12 @@ DUCKDB_INCLUDES_END
 
 using namespace duckdb;
 
+extern "C" duckdb_vector duckdb_vx_vector_slice(duckdb_vector ffi_vector, idx_t offset, idx_t end) {
+    const Vector &vector = *reinterpret_cast<Vector *>(ffi_vector);
+    Vector *const sliced = new Vector(vector, offset, end);
+    return reinterpret_cast<duckdb_vector>(sliced);
+}
+
 extern "C" void duckdb_vx_vector_slice_to_dictionary(duckdb_vector ffi_vector,
                                                      duckdb_selection_vector ffi_sel_vec,
                                                      idx_t selection_vector_length) {
diff --git a/vortex-duckdb/src/convert/mod.rs b/vortex-duckdb/src/convert/mod.rs
@@ -7,6 +7,8 @@ mod scalar;
 mod table_filter;
 mod vector;
 
+#[cfg(test)]
+pub use dtype::FromLogicalType;
 pub use dtype::from_duckdb_table;
 pub use expr::try_from_bound_expression;
 pub use scalar::*;
diff --git a/vortex-duckdb/src/duckdb/vector.rs b/vortex-duckdb/src/duckdb/vector.rs
@@ -62,6 +62,10 @@ impl Vector {
     pub fn with_capacity(logical_type: &LogicalTypeRef, len: usize) -> Self {
         unsafe { Self::own(cpp::duckdb_create_vector(logical_type.as_ptr(), len as _)) }
     }
+
+    pub fn slice(other: &VectorRef, offset: u64, end: u64) -> Self {
+        unsafe { Self::own(cpp::duckdb_vx_vector_slice(other.as_ptr(), offset, end)) }
+    }
 }
 
 impl VectorRef {
@@ -307,6 +311,10 @@ impl VectorRef {
         unsafe { Vector::borrow_mut(cpp::duckdb_array_vector_get_child(self.as_ptr())) }
     }
 
+    pub fn list_vector_get_size(&self) -> u64 {
+        unsafe { cpp::duckdb_list_vector_get_size(self.as_ptr()) }
+    }
+
     pub fn list_vector_set_size(&self, size: u64) -> VortexResult<()> {
         let state = unsafe { cpp::duckdb_list_vector_set_size(self.as_ptr(), size) };
         match state {
diff --git a/vortex-duckdb/src/e2e_test/vortex_scan_test.rs b/vortex-duckdb/src/e2e_test/vortex_scan_test.rs
@@ -982,8 +982,10 @@ fn test_vortex_encodings_roundtrip() {
     assert_eq!(list_entries[4].offset, 10);
 
     // Get child vector and verify actual values
+    let list_child_len = list_vec.list_vector_get_size();
+    assert_eq!(list_child_len, 10);
     let list_child = list_vec.list_vector_get_child();
-    let child_values = list_child.as_slice_with_len::<i32>(10); // 10 total child elements
+    let child_values = list_child.as_slice_with_len::<i32>(list_child_len.as_());
     assert_eq!(child_values, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]);
 
     // Verify fixed-size list column (column 9)
diff --git a/vortex-duckdb/src/exporter/list.rs b/vortex-duckdb/src/exporter/list.rs
@@ -12,8 +12,8 @@ use vortex::array::arrays::list::ListDataParts;
 use vortex::array::match_each_integer_ptype;
 use vortex::array::validity::Validity;
 use vortex::dtype::IntegerPType;
+use vortex::error::VortexExpect;
 use vortex::error::VortexResult;
-use vortex::error::vortex_err;
 use vortex::mask::Mask;
 
 use super::ConversionCache;
@@ -110,40 +110,49 @@ impl<O: IntegerPType> ColumnExporter for ListExporter<O> {
         vector: &mut VectorRef,
         _ctx: &mut ExecutionCtx,
     ) -> VortexResult<()> {
+        if len == 0 {
+            return vector.list_vector_set_size(0);
+        }
+
         // SAFETY: TODO(connor): Pretty sure that `export` needs to be `unsafe`.
         let duckdb_list_views: &mut [cpp::duckdb_list_entry] =
             unsafe { vector.as_slice_mut::<cpp::duckdb_list_entry>(len) };
         debug_assert_eq!(duckdb_list_views.len(), len);
 
-        if len > 0 {
-            let offsets = &self.offsets.as_slice::<O>()[offset..offset + len + 1];
-            debug_assert_eq!(offsets.len(), len + 1);
+        let offsets = &self.offsets.as_slice::<O>()[offset..offset + len + 1];
+        assert_eq!(offsets.len(), len + 1);
 
-            for i in 0..len {
-                let offset = offsets[i]
-                    .to_u64()
-                    .ok_or_else(|| vortex_err!("somehow unable to convert an offset to u64"))?;
-                let length = (offsets[i + 1] - offsets[i])
-                    .to_u64()
-                    .ok_or_else(|| vortex_err!("somehow unable to convert an offset to u64"))?;
+        let offset_start = offsets[0].to_u64().vortex_expect("conversion failed");
+        let offset_end = offsets[len].to_u64().vortex_expect("conversion failed");
+        assert!(offset_end <= self.num_elements as u64);
 
-                debug_assert!(offset + length <= self.num_elements as u64);
-
-                duckdb_list_views[i] = cpp::duckdb_list_entry { offset, length };
-            }
+        for i in 0..len {
+            let offset = offsets[i].to_u64().vortex_expect("conversion failed");
+            let length = (offsets[i + 1] - offsets[i])
+                .to_u64()
+                .vortex_expect("conversion failed");
+            let offset = offset - offset_start;
+            duckdb_list_views[i] = cpp::duckdb_list_entry { offset, length };
         }
 
-        let child = vector.list_vector_get_child_mut();
-        child.reference(&self.duckdb_elements.lock());
+        let sliced = {
+            let elements = &self.duckdb_elements.lock();
+            Vector::slice(elements, offset_start, offset_end)
+        };
 
-        vector.list_vector_set_size(self.num_elements as u64)?;
+        let child_len = offset_end - offset_start;
+        // runend is the case where len < child_len, see test_export_runend_list
+        assert!(len as u64 <= child_len);
 
+        vector.list_vector_get_child_mut().reference(&sliced);
+        vector.list_vector_set_size(child_len)?;
         Ok(())
     }
 }
 
 #[cfg(test)]
 mod tests {
+    use num_traits::AsPrimitive;
     use vortex::array::IntoArray as _;
     use vortex::array::VortexSessionExecute;
     use vortex::array::arrays::VarBinArray;
@@ -153,34 +162,34 @@ mod tests {
     use vortex::dtype::DType;
     use vortex::dtype::PType;
     use vortex::encodings::runend::RunEnd;
-    use vortex::error::VortexExpect;
     use vortex::error::VortexResult;
 
     use super::*;
     use crate::SESSION;
+    use crate::convert::FromLogicalType;
     use crate::duckdb::DataChunk;
     use crate::duckdb::LogicalType;
     use crate::exporter::new_array_exporter;
 
     #[test]
-    fn test_export_empty_list() {
+    fn test_export_empty_list() -> VortexResult<()> {
         let list = ListArray::try_new(
             Buffer::<u32>::empty().into_array(),
             buffer![0u32].into_array(),
             Validity::AllValid,
-        )
-        .vortex_expect("list creation should succeed")
+        )?
         .into_array();
 
-        let list_type = LogicalType::list_type(LogicalType::uint32())
-            .vortex_expect("LogicalTypeRef creation should succeed for test data");
+        let list_type = LogicalType::list_type(LogicalType::uint32())?;
         let mut chunk = DataChunk::new([list_type]);
 
         let mut ctx = SESSION.create_execution_ctx();
-        new_array_exporter(list, &ConversionCache::default(), &mut ctx)
-            .unwrap()
-            .export(0, 0, chunk.get_vector_mut(0), &mut ctx)
-            .unwrap();
+        new_array_exporter(list, &ConversionCache::default(), &mut ctx)?.export(
+            0,
+            0,
+            chunk.get_vector_mut(0),
+            &mut ctx,
+        )?;
         chunk.set_len(0);
 
         assert_eq!(
@@ -189,16 +198,16 @@ mod tests {
 - FLAT UINTEGER[]: 0 = [ ]
 "#
         );
+        Ok(())
     }
 
     #[test]
-    fn test_export_u64_list() {
+    fn test_export_u64_list() -> VortexResult<()> {
         let list = ListArray::try_new(
             buffer![1u64, 2, 3, 4, 5].into_array(),
             buffer![0u8, 1, 2, 3, 4, 5].into_array(),
             Validity::AllValid,
-        )
-        .vortex_expect("list creation should succeed")
+        )?
         .into_array();
         assert_eq!(
             list.dtype(),
@@ -208,15 +217,16 @@ mod tests {
             )
         );
 
-        let list_type = LogicalType::list_type(LogicalType::uint64())
-            .vortex_expect("LogicalTypeRef creation should succeed for test data");
+        let list_type = LogicalType::list_type(LogicalType::uint64())?;
         let mut chunk = DataChunk::new([list_type]);
 
         let mut ctx = SESSION.create_execution_ctx();
-        new_array_exporter(list, &ConversionCache::default(), &mut ctx)
-            .unwrap()
-            .export(0, 5, chunk.get_vector_mut(0), &mut ctx)
-            .unwrap();
+        new_array_exporter(list, &ConversionCache::default(), &mut ctx)?.export(
+            0,
+            5,
+            chunk.get_vector_mut(0),
+            &mut ctx,
+        )?;
         chunk.set_len(5);
 
         assert_eq!(
@@ -225,24 +235,75 @@ mod tests {
 - FLAT UBIGINT[]: 5 = [ [1], [2], [3], [4], [5]]
 "#
         );
+        Ok(())
     }
 
-    // Ensure runend-compressed list is properly flattened
     #[test]
-    fn test_export_list_with_runend_elements() -> VortexResult<()> {
+    fn test_export_u64_list_two_pass() -> VortexResult<()> {
+        // [1], [2, 8], [3], [4], [5]
+        let elements = buffer![1u64, 2, 8, 3, 4, 5].into_array();
+        let offsets = buffer![0u8, 1, 3, 4, 5, 6].into_array();
+        let list = ListArray::try_new(elements, offsets, Validity::AllValid)?.into_array();
+
+        let u64_type = LogicalType::uint64();
+        let list_type = LogicalType::list_type(u64_type)?;
+        let mut chunk = DataChunk::new([list_type]);
+
         let mut ctx = SESSION.create_execution_ctx();
-        let elements = RunEnd::encode(buffer![100u32, 100, 200, 200, 200].into_array(), &mut ctx)?;
+        let exporter = new_array_exporter(list, &ConversionCache::default(), &mut ctx)?;
 
-        let list = ListArray::try_new(
-            elements.into_array(),
-            buffer![0u32, 2, 5].into_array(),
-            Validity::AllValid,
-        )
-        .vortex_expect("list creation should succeed")
-        .into_array();
+        exporter.export(0, 2, chunk.get_vector_mut(0), &mut ctx)?;
+        chunk.set_len(2);
+
+        assert_eq!(
+            format!("{}", String::try_from(&*chunk).unwrap()),
+            r#"Chunk - [1 Columns]
+- FLAT UBIGINT[]: 2 = [ [1], [2, 8]]
+"#
+        );
+
+        let u64_type = LogicalType::uint64();
+        let list_vec = chunk.get_vector(0);
+        let list_child = list_vec.list_vector_get_child();
+        assert_eq!(
+            DType::from_logical_type(&list_child.logical_type(), false.into())?,
+            DType::from_logical_type(&u64_type, false.into())?
+        );
+        let child_len: usize = list_vec.list_vector_get_size().as_();
+        assert_eq!(child_len, 3);
+        let child_values = list_child.as_slice_with_len::<u64>(child_len);
+        assert_eq!(child_values, [1, 2, 8]);
+
+        exporter.export(2, 3, chunk.get_vector_mut(0), &mut ctx)?;
+        chunk.set_len(3);
+
+        assert_eq!(
+            format!("{}", String::try_from(&*chunk).unwrap()),
+            r#"Chunk - [1 Columns]
+- FLAT UBIGINT[]: 3 = [ [3], [4], [5]]
+"#
+        );
+
+        let list_vec = chunk.get_vector(0);
+        let child_len: usize = list_vec.list_vector_get_size().as_();
+        assert_eq!(child_len, 3);
+        let list_child = list_vec.list_vector_get_child();
+        let child_values = list_child.as_slice_with_len::<u64>(child_len);
+        assert_eq!(child_values, [3, 4, 5]);
 
-        let list_type = LogicalType::list_type(LogicalType::uint32())
-            .vortex_expect("LogicalTypeRef creation should succeed for test data");
+        Ok(())
+    }
+
+    #[test]
+    fn test_export_runend_list() -> VortexResult<()> {
+        let mut ctx = SESSION.create_execution_ctx();
+        let elements_buffer = buffer![100f32, 100f32, 200f32, 200f32, 200f32].into_array();
+        let elements = RunEnd::encode(elements_buffer, &mut ctx)?.into_array();
+
+        let list_buffer = buffer![0u32, 2, 5].into_array();
+        let list = ListArray::try_new(elements, list_buffer, Validity::AllValid)?.into_array();
+
+        let list_type = LogicalType::list_type(LogicalType::float32())?;
         let mut chunk = DataChunk::new([list_type]);
 
         new_array_exporter(list, &ConversionCache::default(), &mut ctx)?.export(
@@ -256,15 +317,15 @@ mod tests {
         assert_eq!(
             format!("{}", String::try_from(&*chunk)?),
             r#"Chunk - [1 Columns]
-- FLAT UINTEGER[]: 2 = [ [100, 100], [200, 200, 200]]
+- FLAT FLOAT[]: 2 = [ [100.0, 100.0], [200.0, 200.0, 200.0]]
 "#
         );
 
         Ok(())
     }
 
     #[test]
-    fn test_export_non_empty_list_of_strings() {
+    fn test_export_string_list() -> VortexResult<()> {
         let list = ListArray::try_new(
             <VarBinArray as FromIterator<_>>::from_iter([
                 Some("abc"),
@@ -275,19 +336,19 @@ mod tests {
             .into_array(),
             buffer![0u8, 1, 2, 3, 4].into_array(),
             Validity::from_iter([true, true, false, true]),
-        )
-        .vortex_expect("list creation should succeed")
+        )?
         .into_array();
 
-        let list_type = LogicalType::list_type(LogicalType::varchar())
-            .vortex_expect("LogicalTypeRef creation should succeed for test data");
+        let list_type = LogicalType::list_type(LogicalType::varchar())?;
         let mut chunk = DataChunk::new([list_type]);
 
         let mut ctx = SESSION.create_execution_ctx();
-        new_array_exporter(list, &ConversionCache::default(), &mut ctx)
-            .unwrap()
-            .export(0, 4, chunk.get_vector_mut(0), &mut ctx)
-            .unwrap();
+        new_array_exporter(list, &ConversionCache::default(), &mut ctx)?.export(
+            0,
+            4,
+            chunk.get_vector_mut(0),
+            &mut ctx,
+        )?;
         chunk.set_len(4);
 
         assert_eq!(
@@ -296,5 +357,7 @@ mod tests {
 - FLAT VARCHAR[]: 4 = [ [abc], [def], NULL, [ghi]]
 "#
         );
+
+        Ok(())
     }
 }
diff --git a/vortex-duckdb/src/exporter/list_view.rs b/vortex-duckdb/src/exporter/list_view.rs

Original file line number	Diff line number	Diff line change
`@@ -62,6 +62,10 @@ impl Vector {`
`62`	`62`	`pub fn with_capacity(logical_type: &LogicalTypeRef, len: usize) -> Self {`
`63`	`63`	`unsafe { Self::own(cpp::duckdb_create_vector(logical_type.as_ptr(), len as _)) }`
`64`	`64`	`}`
	`65`	`+`
	`66`	`+ pub fn slice(other: &VectorRef, offset: u64, end: u64) -> Self {`
	`67`	`+ unsafe { Self::own(cpp::duckdb_vx_vector_slice(other.as_ptr(), offset, end)) }`
	`68`	`+ }`
`65`	`69`	`}`
`66`	`70`
`67`	`71`	`impl VectorRef {`
`@@ -307,6 +311,10 @@ impl VectorRef {`
`307`	`311`	`unsafe { Vector::borrow_mut(cpp::duckdb_array_vector_get_child(self.as_ptr())) }`
`308`	`312`	`}`
`309`	`313`
	`314`	`+ pub fn list_vector_get_size(&self) -> u64 {`
	`315`	`+ unsafe { cpp::duckdb_list_vector_get_size(self.as_ptr()) }`
	`316`	`+ }`
	`317`	`+`
`310`	`318`	`pub fn list_vector_set_size(&self, size: u64) -> VortexResult<()> {`
`311`	`319`	`let state = unsafe { cpp::duckdb_list_vector_set_size(self.as_ptr(), size) };`
`312`	`320`	`match state {`