Add Arrow to Vortex conversion to C FFI (#7906)

Accepting arrow is useful to support integration with existing systems

Signed-off-by: Robert Kruszewski <github@robertk.io>

Author: robert3005

vortex-ffi/cbindgen.toml

@@ -22,6 +22,7 @@ header = """
 // #include "nanoarrow/common/inline_types.h"
 // #define USE_OWN_ARROW
 // typedef struct ArrowSchema FFI_ArrowSchema;
+// typedef struct ArrowArray FFI_ArrowArray;
 // typedef struct ArrowArrayStream FFI_ArrowArrayStream;
 // #include "vortex.h"
 //
@@ -57,6 +58,7 @@ struct ArrowArrayStream {
     void* private_data;
 };
 typedef struct ArrowSchema FFI_ArrowSchema;
+typedef struct ArrowArray FFI_ArrowArray;
 typedef struct ArrowArrayStream FFI_ArrowArrayStream;
 #endif
 """

vortex-ffi/cinclude/vortex.h

@@ -14,6 +14,7 @@
 // #include "nanoarrow/common/inline_types.h"
 // #define USE_OWN_ARROW
 // typedef struct ArrowSchema FFI_ArrowSchema;
+// typedef struct ArrowArray FFI_ArrowArray;
 // typedef struct ArrowArrayStream FFI_ArrowArrayStream;
 // #include "vortex.h"
 //
@@ -49,6 +50,7 @@ struct ArrowArrayStream {
     void *private_data;
 };
 typedef struct ArrowSchema FFI_ArrowSchema;
+typedef struct ArrowArray FFI_ArrowArray;
 typedef struct ArrowArrayStream FFI_ArrowArrayStream;
 #endif
 
@@ -787,6 +789,34 @@ const vx_array *vx_array_new_primitive(vx_ptype ptype,
                                        const vx_validity *validity,
                                        vx_error **error);
 
+/**
+ * Create a Vortex array by importing an Arrow array via the Arrow C Data Interface.
+ *
+ * `array` and `schema` together describe a single Arrow array (the standard Arrow C Data
+ * Interface pair, e.g. as produced by exporting a record batch). Both are *consumed*: their
+ * `release` callbacks are invoked by this function and the caller must not use or release them
+ * afterwards.
+ *
+ * `nullable` controls the top-level nullability of the resulting array's dtype. For an Arrow
+ * record batch (which has no top-level validity) pass `false`.
+ *
+ * The imported buffers are referenced zero-copy where possible; the returned array keeps the
+ * Arrow data alive until it is freed with [`vx_array_free`].
+ *
+ * On error, returns NULL and sets `error_out`.
+ *
+ * Example:
+ *
+ * // export an Arrow record batch into (array, schema), then:
+ * vx_error* error = NULL;
+ * const vx_array* vx = vx_array_from_arrow(&array, &schema, false, &error);
+ * // ... push it to a sink or write it ...
+ * vx_array_free(vx);
+ *
+ */
+const vx_array *
+vx_array_from_arrow(FFI_ArrowArray *array, FFI_ArrowSchema *schema, bool nullable, vx_error **error_out);
+
 uint8_t vx_array_get_u8(const vx_array *array, size_t index);
 
 uint8_t vx_array_get_storage_u8(const vx_array *array, size_t index);
@@ -1078,6 +1108,18 @@ const vx_string *vx_dtype_time_zone(const DType *dtype);
  */
 int vx_dtype_to_arrow_schema(const vx_dtype *dtype, FFI_ArrowSchema *schema, vx_error **err);
 
+/**
+ * Create a Vortex dtype from an Arrow C Data Interface schema.
+ *
+ * `schema` must point to a valid `ArrowSchema` describing a struct (record-batch) schema. It is
+ * *consumed*: its `release` callback is invoked by this function and the caller must not use or
+ * release it afterwards. The returned dtype is a non-nullable struct, mirroring how Arrow record
+ * batches map to Vortex arrays.
+ *
+ * On error, returns NULL and sets `err`.
+ */
+const vx_dtype *vx_dtype_from_arrow_schema(FFI_ArrowSchema *schema, vx_error **err);
+
 /**
  * Free an owned [`vx_error`] object.
  */

vortex-ffi/examples/scan_to_arrow.c

@@ -5,6 +5,7 @@
 
 #define USE_OWN_ARROW
 typedef struct ArrowSchema FFI_ArrowSchema;
+typedef struct ArrowArray FFI_ArrowArray;
 typedef struct ArrowArrayStream FFI_ArrowArrayStream;
 #include "vortex.h"
 

vortex-ffi/src/array.rs

@@ -7,6 +7,10 @@ use std::ffi::c_void;
 use std::ptr;
 use std::sync::Arc;
 
+use arrow_array::array::make_array;
+use arrow_array::ffi::FFI_ArrowArray;
+use arrow_array::ffi::FFI_ArrowSchema;
+use arrow_array::ffi::from_ffi;
 use paste::paste;
 use vortex::array::ArrayRef;
 use vortex::array::IntoArray;
@@ -16,6 +20,7 @@ use vortex::array::arrays::NullArray;
 use vortex::array::arrays::PrimitiveArray;
 use vortex::array::arrays::StructArray;
 use vortex::array::arrays::struct_::StructArrayExt;
+use vortex::array::arrow::FromArrowArray;
 use vortex::array::validity::Validity;
 use vortex::buffer::Buffer;
 use vortex::dtype::DType;
@@ -326,6 +331,49 @@ pub extern "C-unwind" fn vx_array_new_primitive(
     }
 }
 
+/// Create a Vortex array by importing an Arrow array via the Arrow C Data Interface.
+///
+/// `array` and `schema` together describe a single Arrow array (the standard Arrow C Data
+/// Interface pair, e.g. as produced by exporting a record batch). Both are *consumed*: their
+/// `release` callbacks are invoked by this function and the caller must not use or release them
+/// afterwards.
+///
+/// `nullable` controls the top-level nullability of the resulting array's dtype. For an Arrow
+/// record batch (which has no top-level validity) pass `false`.
+///
+/// The imported buffers are referenced zero-copy where possible; the returned array keeps the
+/// Arrow data alive until it is freed with [`vx_array_free`].
+///
+/// On error, returns NULL and sets `error_out`.
+///
+/// Example:
+///
+/// // export an Arrow record batch into (array, schema), then:
+/// vx_error* error = NULL;
+/// const vx_array* vx = vx_array_from_arrow(&array, &schema, false, &error);
+/// // ... push it to a sink or write it ...
+/// vx_array_free(vx);
+///
+#[unsafe(no_mangle)]
+pub unsafe extern "C-unwind" fn vx_array_from_arrow(
+    array: *mut FFI_ArrowArray,
+    schema: *mut FFI_ArrowSchema,
+    nullable: bool,
+    error_out: *mut *mut vx_error,
+) -> *const vx_array {
+    try_or_default(error_out, || {
+        vortex_ensure!(!array.is_null(), "null arrow array");
+        vortex_ensure!(!schema.is_null(), "null arrow schema");
+        let ffi_array = unsafe { ptr::replace(array, FFI_ArrowArray::empty()) };
+        let ffi_schema = unsafe { ptr::replace(schema, FFI_ArrowSchema::empty()) };
+        let array_data = unsafe { from_ffi(ffi_array, &ffi_schema) }?;
+        drop(ffi_schema);
+        let arrow_array = make_array(array_data);
+        let vortex_array = ArrayRef::from_arrow(arrow_array.as_ref(), nullable)?;
+        Ok(vx_array::new(Arc::new(vortex_array)))
+    })
+}
+
 macro_rules! ffiarray_get_ptype {
     ($ptype:ident) => {
         paste! {
@@ -809,4 +857,67 @@ mod tests {
         // Note: dtype_ptr is now invalid - this test documents the lifetime pattern
         // In real usage, don't access dtype_ptr after freeing the array
     }
+
+    #[test]
+    #[cfg_attr(miri, ignore)]
+    fn test_from_arrow_roundtrip() {
+        use arrow_array::Array as ArrowArrayTrait;
+        use arrow_array::Int32Array;
+        use arrow_array::RecordBatch;
+        use arrow_array::StringArray;
+        use arrow_array::ffi::to_ffi;
+        use arrow_schema::DataType;
+        use arrow_schema::Field;
+        use arrow_schema::Schema as ArrowSchema;
+
+        let schema = Arc::new(ArrowSchema::new(vec![
+            Field::new("a", DataType::Int32, false),
+            Field::new("b", DataType::Utf8, true),
+        ]));
+        let batch = RecordBatch::try_new(
+            schema,
+            vec![
+                Arc::new(Int32Array::from(vec![1, 2, 3])),
+                Arc::new(StringArray::from(vec![Some("x"), None, Some("z")])),
+            ],
+        )
+        .unwrap();
+
+        let data = ArrowArrayTrait::into_data(arrow_array::StructArray::from(batch));
+        let (mut ffi_array, mut ffi_schema) = to_ffi(&data).unwrap();
+
+        let mut error = ptr::null_mut();
+        let vx = unsafe {
+            vx_array_from_arrow(
+                &raw mut ffi_array,
+                &raw mut ffi_schema,
+                false,
+                &raw mut error,
+            )
+        };
+        assert_no_error(error);
+        assert!(!vx.is_null());
+
+        unsafe {
+            assert!(vx_array_has_dtype(vx, vx_dtype_variant::DTYPE_STRUCT));
+            assert_eq!(vx_array_len(vx), 3);
+            assert!(!vx_array_is_nullable(vx));
+
+            let a = vx_array_get_field(vx, 0, &raw mut error);
+            assert_no_error(error);
+            assert!(vx_array_is_primitive(a, vx_ptype::PTYPE_I32));
+            assert_eq!(vx_array_get_i32(a, 0), 1);
+            assert_eq!(vx_array_get_i32(a, 2), 3);
+            vx_array_free(a);
+
+            let b = vx_array_get_field(vx, 1, &raw mut error);
+            assert_no_error(error);
+            assert!(vx_array_has_dtype(b, vx_dtype_variant::DTYPE_UTF8));
+            assert!(vx_array_element_is_invalid(b, 1, &raw mut error));
+            assert_no_error(error);
+            vx_array_free(b);
+
+            vx_array_free(vx);
+        }
+    }
 }

vortex-ffi/src/dtype.rs

@@ -6,9 +6,12 @@ use std::ptr;
 use std::sync::Arc;
 
 use arrow_array::ffi::FFI_ArrowSchema;
+use arrow_schema::Schema;
 use vortex::dtype::DType;
 use vortex::dtype::DecimalDType;
+use vortex::dtype::arrow::FromArrowType;
 use vortex::error::VortexExpect;
+use vortex::error::vortex_ensure;
 use vortex::error::vortex_panic;
 use vortex::extension::datetime::AnyTemporal;
 use vortex::extension::datetime::Date;
@@ -17,6 +20,7 @@ use vortex::extension::datetime::Timestamp;
 
 use crate::arc_wrapper;
 use crate::error::try_or;
+use crate::error::try_or_default;
 use crate::error::vx_error;
 use crate::ptype::vx_ptype;
 use crate::string::vx_string;
@@ -347,6 +351,28 @@ pub unsafe extern "C-unwind" fn vx_dtype_to_arrow_schema(
     })
 }
 
+/// Create a Vortex dtype from an Arrow C Data Interface schema.
+///
+/// `schema` must point to a valid `ArrowSchema` describing a struct (record-batch) schema. It is
+/// *consumed*: its `release` callback is invoked by this function and the caller must not use or
+/// release it afterwards. The returned dtype is a non-nullable struct, mirroring how Arrow record
+/// batches map to Vortex arrays.
+///
+/// On error, returns NULL and sets `err`.
+#[unsafe(no_mangle)]
+pub unsafe extern "C-unwind" fn vx_dtype_from_arrow_schema(
+    schema: *mut FFI_ArrowSchema,
+    err: *mut *mut vx_error,
+) -> *const vx_dtype {
+    try_or_default(err, || {
+        vortex_ensure!(!schema.is_null(), "null arrow schema");
+        let ffi_schema = unsafe { ptr::replace(schema, FFI_ArrowSchema::empty()) };
+        let arrow_schema = Schema::try_from(&ffi_schema)?;
+        drop(ffi_schema);
+        Ok(vx_dtype::new(Arc::new(DType::from_arrow(&arrow_schema))))
+    })
+}
+
 #[cfg(test)]
 #[expect(clippy::cast_possible_truncation)]
 mod tests {
@@ -744,4 +770,38 @@ mod tests {
             vx_array_free(vx_arr);
         }
     }
+
+    #[test]
+    fn test_dtype_from_arrow_schema() {
+        use arrow_schema::DataType;
+        use arrow_schema::Field;
+
+        let arrow_schema = Schema::new(vec![
+            Field::new("a", DataType::Int64, false),
+            Field::new("b", DataType::Utf8, true),
+        ]);
+        let mut ffi_schema = FFI_ArrowSchema::try_from(&arrow_schema).unwrap();
+
+        let mut error = ptr::null_mut();
+        let dtype = unsafe { vx_dtype_from_arrow_schema(&raw mut ffi_schema, &raw mut error) };
+        assert!(error.is_null());
+        assert!(!dtype.is_null());
+
+        unsafe {
+            assert_eq!(vx_dtype_get_variant(dtype), vx_dtype_variant::DTYPE_STRUCT);
+            assert!(!vx_dtype_is_nullable(dtype));
+            let fields = vx_dtype_struct_dtype(dtype);
+            assert_eq!(vx_struct_fields_nfields(fields), 2);
+            let f0 = vx_struct_fields_field_dtype(fields, 0);
+            assert_eq!(vx_dtype_get_variant(f0), vx_dtype_variant::DTYPE_PRIMITIVE);
+            assert_eq!(vx_dtype_primitive_ptype(f0), vx_ptype::PTYPE_I64);
+            assert!(!vx_dtype_is_nullable(f0));
+            vx_dtype_free(f0);
+            let f1 = vx_struct_fields_field_dtype(fields, 1);
+            assert_eq!(vx_dtype_get_variant(f1), vx_dtype_variant::DTYPE_UTF8);
+            assert!(vx_dtype_is_nullable(f1));
+            vx_dtype_free(f1);
+            vx_dtype_free(dtype);
+        }
+    }
 }

vortex-ffi/test/scan.cpp

@@ -11,6 +11,7 @@
 #include <unistd.h>
 
 using FFI_ArrowArrayStream = ArrowArrayStream;
+using FFI_ArrowArray = ArrowArray;
 using FFI_ArrowSchema = ArrowSchema;
 #define USE_OWN_ARROW 1
 #include <vortex.h>