Deconflict vortex and iceberg arrow deps

arrow/src/main/java/org/apache/iceberg/arrow/vectorized/VectorizedArrowReader.java

@@ -891,6 +891,11 @@ private static boolean isNull(VectorHolder holder, int index) {
   private static BigIntVector allocateBigIntVector(Field field, int valueCount) {
     BigIntVector vector = (BigIntVector) field.createVector(ArrowAllocation.rootAllocator());
     vector.allocateNew(valueCount);
+    ArrowBuf validityBuffer = vector.getValidityBuffer();
+    for (int i = 0; i < valueCount; i += 1) {
+      BitVectorHelper.setBit(validityBuffer, i);
+    }
+
     return vector;
   }
 

build.gradle

@@ -436,17 +436,12 @@ project(':iceberg-data') {
     implementation project(':iceberg-core')
     compileOnly project(':iceberg-parquet')
     compileOnly project(':iceberg-orc')
-    compileOnly project(':iceberg-vortex')
     compileOnly(libs.hadoop3.common) {
       exclude group: 'commons-beanutils'
       exclude group: 'org.apache.avro', module: 'avro'
       exclude group: 'org.slf4j', module: 'slf4j-log4j12'
     }
 
-    implementation(libs.vortex.jni) {
-      exclude group: 'com.google.protobuf', module: 'protobuf-java'
-    }
-
     implementation("${libs.orc.core.get().module}:${libs.versions.orc.get()}:nohive") {
       exclude group: 'org.apache.hadoop'
       exclude group: 'commons-lang'
@@ -473,6 +468,7 @@ project(':iceberg-data') {
     testImplementation project(path: ':iceberg-core', configuration: 'testArtifacts')
     testImplementation project(path: ':iceberg-orc', configuration: 'testArtifacts')
     testImplementation(testFixtures(project(':iceberg-parquet')))
+    testRuntimeOnly project(':iceberg-vortex')
   }
 
   test {
@@ -977,9 +973,11 @@ project(':iceberg-vortex') {
     implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
     implementation project(':iceberg-core')
     implementation project(':iceberg-common')
-    implementation(libs.vortex.jni) {
-      exclude group: 'com.google.protobuf', module: 'protobuf-java'
+    // Compile against the shaded artifact so Vortex APIs use relocated Arrow signatures.
+    compileOnly("${libs.vortex.jni.get().module}:${libs.vortex.jni.get().version}:all") {
+      transitive = false
     }
+    runtimeOnly("${libs.vortex.jni.get().module}:${libs.vortex.jni.get().version}:all")
     // VortexFormatModel exposes Arrow types in its public API, so Arrow must be
     // visible to consumers of iceberg-vortex.
     api(libs.arrow.vector) {
@@ -992,6 +990,12 @@ project(':iceberg-vortex') {
       exclude group: 'io.netty', module: 'netty-common'
       exclude group: 'com.google.code.findbugs', module: 'jsr305'
     }
+    implementation(libs.arrow.memory.netty) {
+      exclude group: 'io.netty', module: 'netty-buffer'
+      exclude group: 'io.netty', module: 'netty-common'
+      exclude group: 'com.google.code.findbugs', module: 'jsr305'
+    }
+    runtimeOnly libs.netty.buffer
 
     annotationProcessor libs.immutables.value
     compileOnly libs.immutables.value

core/src/main/java/org/apache/iceberg/formats/FormatModelRegistry.java

@@ -58,6 +58,7 @@ private FormatModelRegistry() {}
       ImmutableList.of(
           "org.apache.iceberg.data.GenericFormatModels",
           "org.apache.iceberg.arrow.vectorized.ArrowFormatModels",
+          "org.apache.iceberg.vortex.VortexFormatModels",
           "org.apache.iceberg.flink.data.FlinkFormatModels",
           "org.apache.iceberg.spark.source.SparkFormatModels");
 

data/src/main/java/org/apache/iceberg/data/GenericFormatModels.java

@@ -25,12 +25,9 @@
 import org.apache.iceberg.data.orc.GenericOrcWriter;
 import org.apache.iceberg.data.parquet.GenericParquetReaders;
 import org.apache.iceberg.data.parquet.GenericParquetWriter;
-import org.apache.iceberg.data.vortex.GenericVortexReader;
-import org.apache.iceberg.data.vortex.GenericVortexWriter;
 import org.apache.iceberg.formats.FormatModelRegistry;
 import org.apache.iceberg.orc.ORCFormatModel;
 import org.apache.iceberg.parquet.ParquetFormatModel;
-import org.apache.iceberg.vortex.VortexFormatModel;
 
 public class GenericFormatModels {
   public static void register() {
@@ -65,14 +62,6 @@ public static void register() {
                 GenericOrcReader.buildReader(icebergSchema, fileSchema, idToConstant)));
 
     FormatModelRegistry.register(ORCFormatModel.forPositionDeletes());
-
-    FormatModelRegistry.register(
-        VortexFormatModel.create(
-            Record.class,
-            Void.class,
-            (icebergSchema, fileSchema, engineSchema) ->
-                GenericVortexWriter.buildWriter(icebergSchema),
-            (VortexFormatModel.ReaderFunction<Record>) GenericVortexReader::buildReader));
   }
 
   private GenericFormatModels() {}

data/src/test/java/org/apache/iceberg/io/TestFileWriterFactory.java

@@ -287,6 +287,9 @@ public void testPositionDeleteWriter() throws IOException {
 
   @TestTemplate
   public void testPositionDeleteWriterWithRow() throws IOException {
+    assumeThat(fileFormat)
+        .as("Vortex does not support position deletes")
+        .isNotEqualTo(FileFormat.VORTEX);
     FileWriterFactory<T> writerFactory = newWriterFactory(table.schema(), table.schema());
 
     // write a data file

data/src/test/java/org/apache/iceberg/io/TestPartitioningWriters.java

@@ -100,6 +100,9 @@ public void testClusteredDataWriterNoRecords() throws IOException {
 
   @TestTemplate
   public void testClusteredDataWriterMultiplePartitions() throws IOException {
+    assumeThat(fileFormat)
+        .as("Vortex does not yet inject partition values supplied via idToConstant")
+        .isNotEqualTo(FileFormat.VORTEX);
     table.updateSpec().addField(Expressions.ref("data")).commit();
 
     FileWriterFactory<T> writerFactory = newWriterFactory(table.schema());
@@ -549,6 +552,9 @@ public void testFanoutDataWriterNoRecords() throws IOException {
 
   @TestTemplate
   public void testFanoutDataWriterMultiplePartitions() throws IOException {
+    assumeThat(fileFormat)
+        .as("Vortex does not yet inject partition values supplied via idToConstant")
+        .isNotEqualTo(FileFormat.VORTEX);
     table.updateSpec().addField(Expressions.ref("data")).commit();
 
     FileWriterFactory<T> writerFactory = newWriterFactory(table.schema());

data/src/test/java/org/apache/iceberg/io/TestRollingFileWriters.java

@@ -105,6 +105,9 @@ public void testRollingDataWriterNoRecords() throws IOException {
 
   @TestTemplate
   public void testRollingDataWriterSplitData() throws IOException {
+    assumeThat(fileFormat)
+        .as("VortexFileAppender.length() returns 0 until the appender is closed")
+        .isNotEqualTo(FileFormat.VORTEX);
     FileWriterFactory<T> writerFactory = newWriterFactory(table.schema());
     RollingDataWriter<T> writer =
         new RollingDataWriter<>(
@@ -148,6 +151,9 @@ public void testRollingEqualityDeleteWriterNoRecords() throws IOException {
 
   @TestTemplate
   public void testRollingEqualityDeleteWriterSplitDeletes() throws IOException {
+    assumeThat(fileFormat)
+        .as("VortexFileAppender.length() returns 0 until the appender is closed")
+        .isNotEqualTo(FileFormat.VORTEX);
     List<Integer> equalityFieldIds = ImmutableList.of(table.schema().findField("id").fieldId());
     Schema equalityDeleteRowSchema = table.schema().select("id");
     FileWriterFactory<T> writerFactory =

flink/v1.20/flink-runtime/runtime-deps.txt

@@ -1,26 +1,11 @@
 com.fasterxml.jackson.core:jackson-annotations:2.21
 com.fasterxml.jackson.core:jackson-core:2.21
 com.fasterxml.jackson.core:jackson-databind:2.21
-com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.21
 com.github.ben-manes.caffeine:caffeine:2.9
 com.github.luben:zstd-jni:1.5
-com.google.errorprone:error_prone_annotations:2.47
-com.google.flatbuffers:flatbuffers-java:25.2
-com.google.guava:failureaccess:1.0
-com.google.guava:guava:33.6
-com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
-com.google.j2objc:j2objc-annotations:3.1
+com.google.errorprone:error_prone_annotations:2.10
 dev.failsafe:failsafe:3.3
-dev.vortex:vortex-jni:0.72
 io.airlift:aircompressor:2.0
-io.netty:netty-buffer:4.2
-io.netty:netty-common:4.2
-org.apache.arrow:arrow-c-data:19.0
-org.apache.arrow:arrow-format:19.0
-org.apache.arrow:arrow-memory-core:19.0
-org.apache.arrow:arrow-memory-netty-buffer-patch:19.0
-org.apache.arrow:arrow-memory-netty:19.0
-org.apache.arrow:arrow-vector:19.0
 org.apache.avro:avro:1.12
 org.apache.datasketches:datasketches-java:6.2
 org.apache.datasketches:datasketches-memory:3.0
@@ -39,7 +24,6 @@ org.apache.parquet:parquet-jackson:1.17
 org.apache.parquet:parquet-variant:1.17
 org.checkerframework:checker-qual:3.19
 org.eclipse.microprofile.openapi:microprofile-openapi-api:4.1
-org.jspecify:jspecify:1.0
 org.locationtech.jts:jts-core:1.20
 org.projectnessie.nessie:nessie-client:0.107
 org.projectnessie.nessie:nessie-model:0.107

flink/v2.0/flink-runtime/runtime-deps.txt

@@ -1,26 +1,11 @@
 com.fasterxml.jackson.core:jackson-annotations:2.21
 com.fasterxml.jackson.core:jackson-core:2.21
 com.fasterxml.jackson.core:jackson-databind:2.21
-com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.21
 com.github.ben-manes.caffeine:caffeine:2.9
 com.github.luben:zstd-jni:1.5
-com.google.errorprone:error_prone_annotations:2.47
-com.google.flatbuffers:flatbuffers-java:25.2
-com.google.guava:failureaccess:1.0
-com.google.guava:guava:33.6
-com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
-com.google.j2objc:j2objc-annotations:3.1
+com.google.errorprone:error_prone_annotations:2.10
 dev.failsafe:failsafe:3.3
-dev.vortex:vortex-jni:0.72
 io.airlift:aircompressor:2.0
-io.netty:netty-buffer:4.2
-io.netty:netty-common:4.2
-org.apache.arrow:arrow-c-data:19.0
-org.apache.arrow:arrow-format:19.0
-org.apache.arrow:arrow-memory-core:19.0
-org.apache.arrow:arrow-memory-netty-buffer-patch:19.0
-org.apache.arrow:arrow-memory-netty:19.0
-org.apache.arrow:arrow-vector:19.0
 org.apache.avro:avro:1.12
 org.apache.datasketches:datasketches-java:6.2
 org.apache.datasketches:datasketches-memory:3.0
@@ -39,7 +24,6 @@ org.apache.parquet:parquet-jackson:1.17
 org.apache.parquet:parquet-variant:1.17
 org.checkerframework:checker-qual:3.19
 org.eclipse.microprofile.openapi:microprofile-openapi-api:4.1
-org.jspecify:jspecify:1.0
 org.locationtech.jts:jts-core:1.20
 org.projectnessie.nessie:nessie-client:0.107
 org.projectnessie.nessie:nessie-model:0.107

flink/v2.1/flink-runtime/runtime-deps.txt

@@ -1,26 +1,11 @@
 com.fasterxml.jackson.core:jackson-annotations:2.21
 com.fasterxml.jackson.core:jackson-core:2.21
 com.fasterxml.jackson.core:jackson-databind:2.21
-com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.21
 com.github.ben-manes.caffeine:caffeine:2.9
 com.github.luben:zstd-jni:1.5
-com.google.errorprone:error_prone_annotations:2.47
-com.google.flatbuffers:flatbuffers-java:25.2
-com.google.guava:failureaccess:1.0
-com.google.guava:guava:33.6
-com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
-com.google.j2objc:j2objc-annotations:3.1
+com.google.errorprone:error_prone_annotations:2.10
 dev.failsafe:failsafe:3.3
-dev.vortex:vortex-jni:0.72
 io.airlift:aircompressor:2.0
-io.netty:netty-buffer:4.2
-io.netty:netty-common:4.2
-org.apache.arrow:arrow-c-data:19.0
-org.apache.arrow:arrow-format:19.0
-org.apache.arrow:arrow-memory-core:19.0
-org.apache.arrow:arrow-memory-netty-buffer-patch:19.0
-org.apache.arrow:arrow-memory-netty:19.0
-org.apache.arrow:arrow-vector:19.0
 org.apache.avro:avro:1.12
 org.apache.datasketches:datasketches-java:6.2
 org.apache.datasketches:datasketches-memory:3.0
@@ -39,7 +24,6 @@ org.apache.parquet:parquet-jackson:1.17
 org.apache.parquet:parquet-variant:1.17
 org.checkerframework:checker-qual:3.19
 org.eclipse.microprofile.openapi:microprofile-openapi-api:4.1
-org.jspecify:jspecify:1.0
 org.locationtech.jts:jts-core:1.20
 org.projectnessie.nessie:nessie-client:0.107
 org.projectnessie.nessie:nessie-model:0.107

kafka-connect/kafka-connect-runtime/runtime-deps.txt

@@ -54,9 +54,9 @@ com.google.cloud:google-cloud-storage:2.68
 com.google.code.findbugs:jsr305:3.0
 com.google.code.gson:gson:2.13
 com.google.errorprone:error_prone_annotations:2.48
-com.google.flatbuffers:flatbuffers-java:25.2
+com.google.flatbuffers:flatbuffers-java:24.3
 com.google.guava:failureaccess:1.0
-com.google.guava:guava:33.6
+com.google.guava:guava:33.5
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.http-client:google-http-client-apache-v2:2.1
 com.google.http-client:google-http-client-appengine:2.1
@@ -73,13 +73,12 @@ com.microsoft.azure:msal4j-persistence-extension:1.3
 com.microsoft.azure:msal4j:1.23
 com.sun.xml.bind:jaxb-impl:2.2
 commons-cli:commons-cli:1.9
-commons-codec:commons-codec:1.21
+commons-codec:commons-codec:1.19
 commons-io:commons-io:2.20
 commons-logging:commons-logging:1.2
 commons-net:commons-net:3.9
 commons-pool:commons-pool:1.6
 dev.failsafe:failsafe:3.3
-dev.vortex:vortex-jni:0.72
 dnsjava:dnsjava:3.6
 io.airlift:aircompressor:2.0
 io.dropwizard.metrics:metrics-core:3.2
@@ -150,12 +149,11 @@ javax.xml.bind:jaxb-api:2.2
 javax.xml.stream:stax-api:1.0-2
 net.java.dev.jna:jna-platform:5.17
 net.java.dev.jna:jna:5.17
-org.apache.arrow:arrow-c-data:19.0
-org.apache.arrow:arrow-format:19.0
-org.apache.arrow:arrow-memory-core:19.0
-org.apache.arrow:arrow-memory-netty-buffer-patch:19.0
-org.apache.arrow:arrow-memory-netty:19.0
-org.apache.arrow:arrow-vector:19.0
+org.apache.arrow:arrow-format:17.0
+org.apache.arrow:arrow-memory-core:17.0
+org.apache.arrow:arrow-memory-netty-buffer-patch:17.0
+org.apache.arrow:arrow-memory-netty:17.0
+org.apache.arrow:arrow-vector:17.0
 org.apache.avro:avro:1.12
 org.apache.commons:commons-collections4:4.4
 org.apache.commons:commons-compress:1.28

spark/v3.5/spark-runtime/baseline-class-uniqueness.lock

@@ -15,13 +15,3 @@
   - io.netty.buffer.NettyArrowBuf
   - io.netty.buffer.PooledByteBufAllocatorL
   - io.netty.buffer.PooledByteBufAllocatorL$InnerAllocator
-[dev.vortex:vortex-jni, dev.vortex:vortex-jni (classifier=all)]
-  - dev.vortex.api.DataSource
-  - dev.vortex.api.Expression
-  - dev.vortex.api.ImmutableScanOptions
-  - dev.vortex.api.Partition
-  - dev.vortex.api.Scan
-  - dev.vortex.api.Session
-  - dev.vortex.api.VortexWriter
-  - dev.vortex.arrow.ArrowAllocation
-  - dev.vortex.jni.NativeLoader

spark/v3.5/spark/baseline-class-uniqueness.lock

@@ -15,13 +15,3 @@
   - io.netty.buffer.NettyArrowBuf
   - io.netty.buffer.PooledByteBufAllocatorL
   - io.netty.buffer.PooledByteBufAllocatorL$InnerAllocator
-[dev.vortex:vortex-jni, dev.vortex:vortex-jni (classifier=all)]
-  - dev.vortex.api.DataSource
-  - dev.vortex.api.Expression
-  - dev.vortex.api.ImmutableScanOptions
-  - dev.vortex.api.Partition
-  - dev.vortex.api.Scan
-  - dev.vortex.api.Session
-  - dev.vortex.api.VortexWriter
-  - dev.vortex.arrow.ArrowAllocation
-  - dev.vortex.jni.NativeLoader

spark/v4.0/spark-runtime/baseline-class-uniqueness.lock

@@ -15,13 +15,3 @@
   - io.netty.buffer.NettyArrowBuf
   - io.netty.buffer.PooledByteBufAllocatorL
   - io.netty.buffer.PooledByteBufAllocatorL$InnerAllocator
-[dev.vortex:vortex-jni, dev.vortex:vortex-jni (classifier=all)]
-  - dev.vortex.api.DataSource
-  - dev.vortex.api.Expression
-  - dev.vortex.api.ImmutableScanOptions
-  - dev.vortex.api.Partition
-  - dev.vortex.api.Scan
-  - dev.vortex.api.Session
-  - dev.vortex.api.VortexWriter
-  - dev.vortex.arrow.ArrowAllocation
-  - dev.vortex.jni.NativeLoader

spark/v4.0/spark/baseline-class-uniqueness.lock

@@ -15,13 +15,3 @@
   - io.netty.buffer.NettyArrowBuf
   - io.netty.buffer.PooledByteBufAllocatorL
   - io.netty.buffer.PooledByteBufAllocatorL$InnerAllocator
-[dev.vortex:vortex-jni, dev.vortex:vortex-jni (classifier=all)]
-  - dev.vortex.api.DataSource
-  - dev.vortex.api.Expression
-  - dev.vortex.api.ImmutableScanOptions
-  - dev.vortex.api.Partition
-  - dev.vortex.api.Scan
-  - dev.vortex.api.Session
-  - dev.vortex.api.VortexWriter
-  - dev.vortex.arrow.ArrowAllocation
-  - dev.vortex.jni.NativeLoader

spark/v4.1/spark-runtime/baseline-class-uniqueness.lock

@@ -15,13 +15,3 @@
   - io.netty.buffer.NettyArrowBuf
   - io.netty.buffer.PooledByteBufAllocatorL
   - io.netty.buffer.PooledByteBufAllocatorL$InnerAllocator
-[dev.vortex:vortex-jni, dev.vortex:vortex-jni (classifier=all)]
-  - dev.vortex.api.DataSource
-  - dev.vortex.api.Expression
-  - dev.vortex.api.ImmutableScanOptions
-  - dev.vortex.api.Partition
-  - dev.vortex.api.Scan
-  - dev.vortex.api.Session
-  - dev.vortex.api.VortexWriter
-  - dev.vortex.arrow.ArrowAllocation
-  - dev.vortex.jni.NativeLoader

spark/v4.1/spark/baseline-class-uniqueness.lock

@@ -15,13 +15,3 @@
   - io.netty.buffer.NettyArrowBuf
   - io.netty.buffer.PooledByteBufAllocatorL
   - io.netty.buffer.PooledByteBufAllocatorL$InnerAllocator
-[dev.vortex:vortex-jni, dev.vortex:vortex-jni (classifier=all)]
-  - dev.vortex.api.DataSource
-  - dev.vortex.api.Expression
-  - dev.vortex.api.ImmutableScanOptions
-  - dev.vortex.api.Partition
-  - dev.vortex.api.Scan
-  - dev.vortex.api.Session
-  - dev.vortex.api.VortexWriter
-  - dev.vortex.arrow.ArrowAllocation
-  - dev.vortex.jni.NativeLoader