From 167cae87cc996810f594ce114fb472acd59286db Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Mon, 11 May 2026 16:16:02 +0100 Subject: [PATCH 1/4] Upgrade vortex to 0.72.0 Signed-off-by: Robert Kruszewski --- build.gradle | 9 +- .../org/apache/iceberg/TableProperties.java | 4 + gradle/libs.versions.toml | 3 +- .../iceberg/spark/data/SparkVortexReader.java | 71 +++-- .../spark/data/SparkVortexValueReaders.java | 128 ++++---- .../iceberg/spark/data/SparkVortexWriter.java | 12 + .../VectorizedSparkVortexReaders.java | 47 ++- .../apache/iceberg/vortex/VortexSparkEte.java | 236 --------------- .../iceberg/spark/data/SparkVortexReader.java | 71 +++-- .../spark/data/SparkVortexValueReaders.java | 130 ++++---- .../iceberg/spark/data/SparkVortexWriter.java | 12 + .../VectorizedSparkVortexReaders.java | 47 ++- .../iceberg/spark/data/SparkVortexReader.java | 104 ++++--- .../spark/data/SparkVortexValueReaders.java | 161 ++++++---- .../iceberg/spark/data/SparkVortexWriter.java | 12 + .../VectorizedSparkVortexReaders.java | 47 ++- .../data/vortex/GenericVortexReader.java | 153 +++++----- .../data/vortex/GenericVortexReaders.java | 131 ++++++-- .../data/vortex/GenericVortexWriter.java | 11 + .../iceberg/vortex/ConvertFilterToVortex.java | 281 +++++------------- .../iceberg/vortex/PrefetchingIterator.java | 127 -------- .../iceberg/vortex/VortexBatchReader.java | 6 +- .../iceberg/vortex/VortexFileAppender.java | 39 +-- .../iceberg/vortex/VortexFormatModel.java | 66 ++-- .../apache/iceberg/vortex/VortexIterable.java | 242 ++++++++++----- .../iceberg/vortex/VortexRowReader.java | 5 +- .../vortex/VortexSchemaWithTypeVisitor.java | 58 ++-- .../apache/iceberg/vortex/VortexSchemas.java | 244 ++++++--------- .../iceberg/vortex/VortexValueReader.java | 12 +- .../vortex/TestConvertFilterToVortex.java | 45 ++- .../apache/iceberg/vortex/TestVortexUuid.java | 115 +++++++ 31 files changed, 1271 insertions(+), 1358 deletions(-) delete mode 100644 spark/v3.5/spark/src/test/java/org/apache/iceberg/vortex/VortexSparkEte.java delete mode 100644 vortex/src/main/java/org/apache/iceberg/vortex/PrefetchingIterator.java create mode 100644 vortex/src/test/java/org/apache/iceberg/vortex/TestVortexUuid.java diff --git a/build.gradle b/build.gradle index 2292ecb6e056..35276d139e06 100644 --- a/build.gradle +++ b/build.gradle @@ -980,7 +980,14 @@ project(':iceberg-vortex') { implementation(libs.vortex.jni) { exclude group: 'com.google.protobuf', module: 'protobuf-java' } - implementation(libs.arrow.vector) { + // VortexFormatModel exposes Arrow types in its public API, so Arrow must be + // visible to consumers of iceberg-vortex. + api(libs.arrow.vector) { + exclude group: 'io.netty', module: 'netty-buffer' + exclude group: 'io.netty', module: 'netty-common' + exclude group: 'com.google.code.findbugs', module: 'jsr305' + } + implementation(libs.arrow.c.data) { exclude group: 'io.netty', module: 'netty-buffer' exclude group: 'io.netty', module: 'netty-common' exclude group: 'com.google.code.findbugs', module: 'jsr305' diff --git a/core/src/main/java/org/apache/iceberg/TableProperties.java b/core/src/main/java/org/apache/iceberg/TableProperties.java index 021ef95d9122..e061db890b23 100644 --- a/core/src/main/java/org/apache/iceberg/TableProperties.java +++ b/core/src/main/java/org/apache/iceberg/TableProperties.java @@ -262,6 +262,10 @@ private TableProperties() {} public static final String ORC_BATCH_SIZE = "read.orc.vectorization.batch-size"; public static final int ORC_BATCH_SIZE_DEFAULT = 5000; + public static final String READ_VORTEX_WORKER_THREADS = "read.vortex.worker-threads"; + public static final String WRITE_VORTEX_WORKER_THREADS = "write.vortex.worker-threads"; + public static final int VORTEX_WORKER_THREADS_DEFAULT = 4; + public static final String DATA_PLANNING_MODE = "read.data-planning-mode"; public static final String DELETE_PLANNING_MODE = "read.delete-planning-mode"; public static final String PLANNING_MODE_DEFAULT = PlanningMode.AUTO.modeName(); diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index c928f091af51..63c53d49ca8b 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -92,7 +92,7 @@ spark41 = "4.1.1" sqlite-jdbc = "3.53.1.0" testcontainers = "2.0.5" tez08 = { strictly = "0.8.4"} # see rich version usage explanation above -vortex = "0.67.0" +vortex = "0.72.0" [libraries] activation = { module = "javax.activation:activation", version.ref = "activation" } @@ -105,6 +105,7 @@ antlr-antlr4 = { module = "org.antlr:antlr4", version.ref = "antlr" } antlr-runtime = { module = "org.antlr:antlr4-runtime", version.ref = "antlr" } antlr-antlr413 = { module = "org.antlr:antlr4", version.ref = "antlr413" } antlr-runtime413 = { module = "org.antlr:antlr4-runtime", version.ref = "antlr413" } +arrow-c-data = { module = "org.apache.arrow:arrow-c-data", version.ref = "arrow" } arrow-memory-netty = { module = "org.apache.arrow:arrow-memory-netty", version.ref = "arrow" } arrow-vector = { module = "org.apache.arrow:arrow-vector", version.ref = "arrow" } avro-avro = { module = "org.apache.avro:avro", version.ref = "avro" } diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexReader.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexReader.java index a2d28cf5e3ba..6afbdce70fb0 100644 --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexReader.java +++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexReader.java @@ -18,12 +18,16 @@ */ package org.apache.iceberg.spark.data; -import dev.vortex.api.Array; -import dev.vortex.api.DType; import java.util.List; import java.util.Map; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.types.TimeUnit; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; import org.apache.iceberg.Schema; import org.apache.iceberg.data.vortex.GenericVortexReaders; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; import org.apache.iceberg.vortex.VortexRowReader; @@ -34,16 +38,31 @@ /** Read Vortex as Spark {@link InternalRow}. */ public class SparkVortexReader implements VortexRowReader { - private final VortexValueReader reader; + private final List> fieldReaders; - public SparkVortexReader(Schema readSchema, DType vortexSchema, Map idToConstant) { - this.reader = - VortexSchemaWithTypeVisitor.visit(readSchema, vortexSchema, SparkReadBuilder.INSTANCE); + public SparkVortexReader( + Schema readSchema, + org.apache.arrow.vector.types.pojo.Schema fileArrowSchema, + Map idToConstant) { + List fields = fileArrowSchema.getFields(); + List expected = readSchema.columns(); + this.fieldReaders = Lists.newArrayListWithExpectedSize(expected.size()); + for (int i = 0; i < expected.size(); i++) { + Type icebergType = expected.get(i).type(); + Field arrowField = fields.get(i); + this.fieldReaders.add( + VortexSchemaWithTypeVisitor.visit(icebergType, arrowField, SparkReadBuilder.INSTANCE)); + } } @Override - public InternalRow read(Array batch, int row) { - return (InternalRow) reader.read(batch, row); + public InternalRow read(VectorSchemaRoot batch, int row) { + GenericInternalRow result = new GenericInternalRow(fieldReaders.size()); + for (int i = 0; i < fieldReaders.size(); i++) { + VortexValueReader reader = fieldReaders.get(i); + result.update(i, reader.read(batch.getVector(i), row)); + } + return result; } static class SparkReadBuilder extends VortexSchemaWithTypeVisitor> { @@ -53,21 +72,18 @@ private SparkReadBuilder() {} @Override public VortexValueReader struct( - Types.StructType schema, - List types, - List names, - List> fields) { - return new StructReader(fields); + Types.StructType schema, List fields, List> children) { + return new StructReader(children); } @Override public VortexValueReader list( - Types.ListType iList, DType array, VortexValueReader element) { + Types.ListType iList, Field listField, VortexValueReader element) { throw new UnsupportedOperationException("Vortex LIST types are not supported yet"); } @Override - public VortexValueReader primitive(Type.PrimitiveType icebergType, DType vortexType) { + public VortexValueReader primitive(Type.PrimitiveType icebergType, Field primField) { switch (icebergType.typeId()) { case BOOLEAN: return GenericVortexReaders.bools(); @@ -82,19 +98,20 @@ public VortexValueReader primitive(Type.PrimitiveType icebergType, DType vort case STRING: return SparkVortexValueReaders.utf8String(); case BINARY: - // Spark expects binary to be in another format, no? return GenericVortexReaders.bytes(); case DECIMAL: return GenericVortexReaders.decimals(); case TIMESTAMP: case TIMESTAMP_NANO: - // TODO(aduffy): timestamp and date types - return SparkVortexValueReaders.timestamp(vortexType.getTimeUnit()); + { + ArrowType.Timestamp ts = (ArrowType.Timestamp) primField.getType(); + return SparkVortexValueReaders.timestamp(ts.getUnit()); + } case DATE: - // TODO(aduffy): timestamp and date types - return SparkVortexValueReaders.date(vortexType.getTimeUnit()); + return SparkVortexValueReaders.date(); + case UUID: + return SparkVortexValueReaders.uuid(); case TIME: - // TODO(aduffy): timestamp and date types default: throw new UnsupportedOperationException("Unsupported type: " + icebergType); } @@ -109,14 +126,20 @@ private StructReader(List> fields) { } @Override - public InternalRow readNonNull(Array array, int row) { + public InternalRow readNonNull(FieldVector vector, int row) { + org.apache.arrow.vector.complex.StructVector struct = + (org.apache.arrow.vector.complex.StructVector) vector; GenericInternalRow result = new GenericInternalRow(fields.size()); for (int i = 0; i < fields.size(); i++) { VortexValueReader fieldReader = fields.get(i); - Object field = fieldReader.read(array.getField(i), row); - result.update(i, field); + FieldVector child = (FieldVector) struct.getChildByOrdinal(i); + result.update(i, fieldReader.read(child, row)); } return result; } } + + // Silence unused warning that TimeUnit class is required for the public API. + @SuppressWarnings("unused") + private static final TimeUnit UNUSED = TimeUnit.MICROSECOND; } diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexValueReaders.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexValueReaders.java index 46cb60abe40d..d48685814862 100644 --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexValueReaders.java +++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexValueReaders.java @@ -18,10 +18,19 @@ */ package org.apache.iceberg.spark.data; -import dev.vortex.api.Array; -import dev.vortex.api.DType; -import java.util.function.BiFunction; +import java.nio.charset.StandardCharsets; +import org.apache.arrow.vector.BigIntVector; +import org.apache.arrow.vector.DateDayVector; +import org.apache.arrow.vector.DateMilliVector; +import org.apache.arrow.vector.ExtensionTypeVector; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.FixedSizeBinaryVector; +import org.apache.arrow.vector.TimeStampVector; +import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.types.TimeUnit; +import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.iceberg.util.DateTimeUtil; +import org.apache.iceberg.util.UUIDUtil; import org.apache.iceberg.vortex.VortexValueReader; import org.apache.spark.unsafe.types.UTF8String; @@ -32,11 +41,16 @@ public static VortexValueReader utf8String() { return UTF8Reader.INSTANCE; } - public static VortexValueReader date(DType.TimeUnit timeUnit) { - return new DateReader(timeUnit); + public static VortexValueReader date() { + return DateReader.INSTANCE; } - public static VortexValueReader timestamp(DType.TimeUnit timeUnit) { + public static VortexValueReader uuid() { + // Iceberg's UUID maps to Spark StringType; emit the canonical UUID string. + return UuidReader.INSTANCE; + } + + public static VortexValueReader timestamp(TimeUnit timeUnit) { // Spark timestamp has µs precision return new TimestampReader(timeUnit); } @@ -47,74 +61,66 @@ static class UTF8Reader implements VortexValueReader { private UTF8Reader() {} @Override - public UTF8String readNonNull(Array array, int row) { - // TODO(aduffy): do this zero-copy by getting Vortex to give us back the pointer + len - // to the decompressed string. - String value = array.getUTF8(row); - return UTF8String.fromString(value); + public UTF8String readNonNull(FieldVector vector, int row) { + byte[] bytes = ((VarCharVector) vector).get(row); + return UTF8String.fromString(new String(bytes, StandardCharsets.UTF_8)); } } - // Spark expects DateType as Integer number of days since UNIX epoch - static class DateReader implements VortexValueReader { - private final BiFunction reader; - - private DateReader(DType.TimeUnit timeUnit) { - switch (timeUnit) { - case MILLISECONDS: - this.reader = - (array, row) -> { - long millis = array.getLong(row); - return DateTimeUtil.microsToDays(millis * 1000); - }; - break; - case DAYS: - this.reader = Array::getInt; - break; - default: - throw new IllegalArgumentException("Unsupported time unit for DATE: " + timeUnit); - } + static class UuidReader implements VortexValueReader { + static final UuidReader INSTANCE = new UuidReader(); + + private UuidReader() {} + + @Override + public UTF8String readNonNull(FieldVector vector, int row) { + FixedSizeBinaryVector storage = + vector instanceof ExtensionTypeVector ext + ? (FixedSizeBinaryVector) ext.getUnderlyingVector() + : (FixedSizeBinaryVector) vector; + return UTF8String.fromString(UUIDUtil.convert(storage.get(row)).toString()); } + } + + // Spark expects DateType as Integer number of days since UNIX epoch. + static class DateReader implements VortexValueReader { + static final DateReader INSTANCE = new DateReader(); + + private DateReader() {} @Override - public Integer readNonNull(Array array, int row) { - return this.reader.apply(array, row); + public Integer readNonNull(FieldVector vector, int row) { + ArrowType arrowType = vector.getField().getType(); + if (arrowType instanceof ArrowType.Date dateType + && dateType.getUnit() == org.apache.arrow.vector.types.DateUnit.MILLISECOND) { + long millis = ((DateMilliVector) vector).get(row); + return DateTimeUtil.microsToDays(millis * 1000L); + } + return ((DateDayVector) vector).get(row); } } static class TimestampReader implements VortexValueReader { - private final BiFunction reader; - - private TimestampReader(DType.TimeUnit vortexTimeUnit) { - switch (vortexTimeUnit) { - case NANOSECONDS: - // Round nanoseconds to microsecond - this.reader = (array, row) -> Math.floorDiv(array.getLong(row), 1_000L); - break; - case MICROSECONDS: - // Microseconds measurements - this.reader = Array::getLong; - break; - case MILLISECONDS: - // Milliseconds -> Microseconds - this.reader = (array, row) -> Math.multiplyExact(array.getLong(row), 1_000L); - break; - case SECONDS: - // Seconds -> Microseconds - this.reader = (array, row) -> Math.multiplyExact(array.getLong(row), 1_000_000L); - break; - case DAYS: - // Days -> Microseconds - this.reader = (array, row) -> Math.multiplyExact(array.getLong(row), 86_400_000_000L); - break; - default: - throw new IllegalStateException("Unexpected value: " + vortexTimeUnit); - } + private final TimeUnit unit; + + private TimestampReader(TimeUnit unit) { + this.unit = unit; } @Override - public Long readNonNull(Array array, int row) { - return this.reader.apply(array, row); + public Long readNonNull(FieldVector vector, int row) { + long measure; + if (vector instanceof TimeStampVector ts) { + measure = ts.get(row); + } else { + measure = ((BigIntVector) vector).get(row); + } + return switch (unit) { + case NANOSECOND -> Math.floorDiv(measure, 1_000L); + case MICROSECOND -> measure; + case MILLISECOND -> Math.multiplyExact(measure, 1_000L); + case SECOND -> Math.multiplyExact(measure, 1_000_000L); + }; } } } diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexWriter.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexWriter.java index 8c416b133382..cf776f27bb8c 100644 --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexWriter.java +++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexWriter.java @@ -20,11 +20,14 @@ import java.math.BigDecimal; import java.util.List; +import java.util.UUID; import org.apache.arrow.vector.BigIntVector; import org.apache.arrow.vector.BitVector; import org.apache.arrow.vector.DateDayVector; import org.apache.arrow.vector.DecimalVector; +import org.apache.arrow.vector.ExtensionTypeVector; import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.FixedSizeBinaryVector; import org.apache.arrow.vector.Float4Vector; import org.apache.arrow.vector.Float8Vector; import org.apache.arrow.vector.IntVector; @@ -38,6 +41,7 @@ import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.iceberg.Schema; import org.apache.iceberg.types.Types; +import org.apache.iceberg.util.UUIDUtil; import org.apache.iceberg.vortex.VortexValueWriter; import org.apache.spark.sql.catalyst.InternalRow; import org.apache.spark.unsafe.types.UTF8String; @@ -111,6 +115,14 @@ private static void writeValue( case DATE: ((DateDayVector) vector).setSafe(rowIndex, row.getInt(fieldIndex)); break; + case UUID: + UUID uuid = UUID.fromString(row.getUTF8String(fieldIndex).toString()); + FixedSizeBinaryVector uuidStorage = + vector instanceof ExtensionTypeVector ext + ? (FixedSizeBinaryVector) ext.getUnderlyingVector() + : (FixedSizeBinaryVector) vector; + uuidStorage.setSafe(rowIndex, UUIDUtil.convert(uuid)); + break; case TIME: ((TimeMicroVector) vector).setSafe(rowIndex, row.getLong(fieldIndex)); break; diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/VectorizedSparkVortexReaders.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/VectorizedSparkVortexReaders.java index 8e9a36a3890b..e7045540cb68 100644 --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/VectorizedSparkVortexReaders.java +++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/VectorizedSparkVortexReaders.java @@ -18,19 +18,16 @@ */ package org.apache.iceberg.spark.data.vectorized; -import dev.vortex.api.Array; -import dev.vortex.api.DType; -import dev.vortex.arrow.ArrowAllocation; -import dev.vortex.relocated.org.apache.arrow.vector.VectorSchemaRoot; -import dev.vortex.spark.read.VortexArrowColumnVector; -import dev.vortex.spark.read.VortexColumnarBatch; import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.iceberg.MetadataColumns; import org.apache.iceberg.Schema; import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.vortex.VortexBatchReader; +import org.apache.spark.sql.vectorized.ArrowColumnVector; import org.apache.spark.sql.vectorized.ColumnVector; import org.apache.spark.sql.vectorized.ColumnarBatch; @@ -38,7 +35,9 @@ public class VectorizedSparkVortexReaders { private VectorizedSparkVortexReaders() {} public static VortexBatchReader buildReader( - Schema icebergSchema, DType vortexSchema, Map idToConstant) { + Schema icebergSchema, + org.apache.arrow.vector.types.pojo.Schema vortexSchema, + Map idToConstant) { return new SchemaCachingBatchReader(icebergSchema, vortexSchema, idToConstant); } @@ -47,20 +46,18 @@ static final class SchemaCachingBatchReader implements VortexBatchReader idToConstant; private final List schemaMapping; - // Reusable vector schema root. - private VectorSchemaRoot root; - SchemaCachingBatchReader( - Schema readerSchema, DType vortexSchema, Map idToConstant) { + Schema readerSchema, + org.apache.arrow.vector.types.pojo.Schema vortexSchema, + Map idToConstant) { this.readerSchema = readerSchema; this.idToConstant = idToConstant; this.schemaMapping = vortexSchemaMapping(readerSchema, vortexSchema); } @Override - public ColumnarBatch read(Array batch) { - this.root = batch.exportToArrow(ArrowAllocation.rootAllocator(), this.root); - int rowCount = this.root.getRowCount(); + public ColumnarBatch read(VectorSchemaRoot batch) { + int rowCount = batch.getRowCount(); Map vectors = Maps.newHashMap(); for (Map.Entry entry : idToConstant.entrySet()) { @@ -70,26 +67,24 @@ public ColumnarBatch read(Array batch) { continue; } - // Field IDs are 1-indexed in Iceberg. vectors.put( - fieldId, - new ConstantColumnVector( - readerSchema.findType(fieldId), (int) batch.getLen(), constant)); + fieldId, new ConstantColumnVector(readerSchema.findType(fieldId), rowCount, constant)); } - for (int i = 0; i < root.getFieldVectors().size(); i++) { + List fieldVectors = batch.getFieldVectors(); + for (int i = 0; i < fieldVectors.size(); i++) { int fieldId = schemaMapping.get(i); - vectors.put(fieldId, new VortexArrowColumnVector(root.getVector(i))); + vectors.put(fieldId, new ArrowColumnVector(fieldVectors.get(i))); } - return new VortexColumnarBatch( - batch, vectors.values().toArray(new ColumnVector[0]), rowCount); + return new ColumnarBatch(vectors.values().toArray(new ColumnVector[0]), rowCount); } - // Mapping from Vortex schema index to Iceberg Field ID. - static List vortexSchemaMapping(Schema icebergSchema, DType vortexSchema) { - return vortexSchema.getFieldNames().stream() - .map(fieldName -> icebergSchema.findField(fieldName).fieldId()) + // Mapping from Arrow Schema field index to Iceberg Field ID. + static List vortexSchemaMapping( + Schema icebergSchema, org.apache.arrow.vector.types.pojo.Schema vortexSchema) { + return vortexSchema.getFields().stream() + .map(field -> icebergSchema.findField(field.getName()).fieldId()) .collect(Collectors.toList()); } } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/vortex/VortexSparkEte.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/vortex/VortexSparkEte.java deleted file mode 100644 index 62ec248df9f0..000000000000 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/vortex/VortexSparkEte.java +++ /dev/null @@ -1,236 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iceberg.vortex; - -import static org.apache.iceberg.types.Types.NestedField.required; -import static org.assertj.core.api.AssertionsForInterfaceTypes.assertThat; - -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import org.apache.iceberg.AppendFiles; -import org.apache.iceberg.DataFile; -import org.apache.iceberg.DataFiles; -import org.apache.iceberg.PartitionSpec; -import org.apache.iceberg.Schema; -import org.apache.iceberg.Table; -import org.apache.iceberg.Tables; -import org.apache.iceberg.hadoop.HadoopTables; -import org.apache.iceberg.io.FileIO; -import org.apache.iceberg.io.InputFile; -import org.apache.iceberg.relocated.com.google.common.base.Preconditions; -import org.apache.iceberg.relocated.com.google.common.collect.Iterables; -import org.apache.iceberg.relocated.com.google.common.io.ByteStreams; -import org.apache.iceberg.types.Types; -import org.apache.spark.sql.Dataset; -import org.apache.spark.sql.Row; -import org.apache.spark.sql.SparkSession; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; - -public final class VortexSparkEte { - private static final Schema EMPLOYEE_SCHEMA = - new Schema( - required(1, "id", Types.LongType.get()), - required(2, "name", Types.StringType.get()), - required(3, "salary", Types.LongType.get())); - - private static final Schema LINEITEM_SCHEMA = - new Schema( - required(1, "l_orderkey", Types.LongType.get()), - required(2, "l_partkey", Types.LongType.get()), - required(3, "l_suppkey", Types.LongType.get()), - required(4, "l_linenumber", Types.LongType.get()), - required(5, "l_quantity", Types.DoubleType.get()), - required(6, "l_extendedprice", Types.DoubleType.get()), - required(7, "l_discount", Types.DoubleType.get()), - required(8, "l_tax", Types.DoubleType.get()), - required(9, "l_returnflag", Types.StringType.get()), - required(10, "l_linestatus", Types.StringType.get()), - required(11, "l_shipdate", Types.DateType.get()), - required(12, "l_commitdate", Types.DateType.get()), - required(13, "l_receiptdate", Types.DateType.get()), - required(14, "l_shipinstruct", Types.StringType.get()), - required(15, "l_shipmode", Types.StringType.get()), - required(16, "l_comment", Types.StringType.get())); - - private static final Path LINEITEM_PATH = - Paths.get("/Users/aduffy/code/vortex/bench-vortex/data/tpch/1/lineitem.vortex"); - - private static Table employeeTable; - private static Table lineitemTable; - - @TempDir private static File tempDir; - - @BeforeAll - public static void beforeAll() { - Tables tables = new HadoopTables(); - employeeTable = - createTable(tables, "employees", new ResourceFile("employees.vortex"), EMPLOYEE_SCHEMA, 3); - lineitemTable = - createTable(tables, "lineitem", new DiskFile(LINEITEM_PATH), LINEITEM_SCHEMA, 6_001_215); - } - - private static Table createTable( - Tables tables, String name, SourceFile sourceFile, Schema schema, int rowCount) { - File tableDir = new File(tempDir, name); - String tableLocation = tableDir.getAbsolutePath(); - Table theTable = tables.create(schema, tableLocation); - - // Import the file from the classpath into the table's data directory. - File dataDir = new File(tableDir, "data"); - dataDir.mkdirs(); - - Path newFilePath = new File(dataDir, "1.vortex").toPath(); - sourceFile.writeToPath(newFilePath); - - // Append the data file to this table - try (FileIO io = theTable.io()) { - InputFile inputFile = io.newInputFile(newFilePath.toAbsolutePath().toString()); - DataFile newDataFile = - DataFiles.builder(PartitionSpec.unpartitioned()) - .withInputFile(inputFile) - .withRecordCount(rowCount) - .build(); - - AppendFiles append = theTable.newAppend(); - append.appendFile(newDataFile); - append.commit(); - - assertThat(theTable.currentSnapshot().addedDataFiles(io)).hasSize(1); - assertThat( - Iterables.getOnlyElement(theTable.currentSnapshot().addedDataFiles(io)).recordCount()) - .isEqualTo(rowCount); - } - - return theTable; - } - - @AfterAll - public static void afterAll() { - // Delete the table and all of its data/metadata. - if (employeeTable != null) { - new HadoopTables().dropTable(employeeTable.location()); - } - - if (lineitemTable != null) { - new HadoopTables().dropTable(lineitemTable.location()); - } - } - - @Test - public void testBasic() { - SparkSession spark = SparkSession.builder().master("local").appName("testBasic").getOrCreate(); - - System.out.println("LOADING FROM TABLE: " + employeeTable.location()); - Dataset employees = spark.read().format("iceberg").load(employeeTable.location()); - assertThat(employees.count()).isEqualTo(3); - - employees.printSchema(); - // Show all columns - employees.show(); - // Show with projection - employees.select("name").show(); - // Show with filter - assertThat(employees.where("id > 1").count()).isEqualTo(2); - } - - @Test - public void testTPCH() { - SparkSession spark = SparkSession.builder().master("local").appName("testTPCH").getOrCreate(); - - System.out.println("LOADING FROM TABLE: " + lineitemTable.location()); - Dataset lineitems = spark.read().format("iceberg").load(lineitemTable.location()); - assertThat(lineitems.count()).isEqualTo(6_001_215); - - lineitems.printSchema(); - // Show all columns - lineitems.show(); - // Show with projection - lineitems.select("l_shipdate").limit(10).show(); - } - - @Test - public void testS3() { - try (SparkSession spark = - SparkSession.builder() - .master("local") - .appName("testS3") - .config("fs.s3a.access.key", System.getenv("AWS_ACCESS_KEY")) - .config("fs.s3a.secret.key", System.getenv("AWS_SECRET_KEY")) - .getOrCreate()) { - - // Open a file against the remote - Dataset df = - spark - .read() - .format("parquet") - .load("s3a://vortex-iceberg-dev/iceberg-parquet/customer.parquet"); - df.show(); - df.printSchema(); - } - } - - interface SourceFile { - void writeToPath(Path path); - } - - static class ResourceFile implements SourceFile { - private final String resourcePath; - - ResourceFile(String resourcePath) { - this.resourcePath = resourcePath; - } - - @Override - public void writeToPath(Path path) { - try (InputStream inputStream = VortexSparkEte.class.getResourceAsStream(resourcePath); - OutputStream outputStream = Files.newOutputStream(path)) { - Preconditions.checkNotNull(inputStream, "Cannot find resource: " + resourcePath); - ByteStreams.copy(inputStream, outputStream); - } catch (IOException e) { - throw new RuntimeException("Failed to load resource: " + resourcePath, e); - } - } - } - - static class DiskFile implements SourceFile { - private final Path sourcePath; - - DiskFile(Path sourcePath) { - this.sourcePath = sourcePath; - } - - @Override - public void writeToPath(Path path) { - try (InputStream inputStream = Files.newInputStream(sourcePath); - OutputStream outputStream = Files.newOutputStream(path)) { - ByteStreams.copy(inputStream, outputStream); - } catch (IOException e) { - throw new RuntimeException("Failed to copy file: " + sourcePath, e); - } - } - } -} diff --git a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexReader.java b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexReader.java index a2d28cf5e3ba..6afbdce70fb0 100644 --- a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexReader.java +++ b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexReader.java @@ -18,12 +18,16 @@ */ package org.apache.iceberg.spark.data; -import dev.vortex.api.Array; -import dev.vortex.api.DType; import java.util.List; import java.util.Map; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.types.TimeUnit; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; import org.apache.iceberg.Schema; import org.apache.iceberg.data.vortex.GenericVortexReaders; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; import org.apache.iceberg.vortex.VortexRowReader; @@ -34,16 +38,31 @@ /** Read Vortex as Spark {@link InternalRow}. */ public class SparkVortexReader implements VortexRowReader { - private final VortexValueReader reader; + private final List> fieldReaders; - public SparkVortexReader(Schema readSchema, DType vortexSchema, Map idToConstant) { - this.reader = - VortexSchemaWithTypeVisitor.visit(readSchema, vortexSchema, SparkReadBuilder.INSTANCE); + public SparkVortexReader( + Schema readSchema, + org.apache.arrow.vector.types.pojo.Schema fileArrowSchema, + Map idToConstant) { + List fields = fileArrowSchema.getFields(); + List expected = readSchema.columns(); + this.fieldReaders = Lists.newArrayListWithExpectedSize(expected.size()); + for (int i = 0; i < expected.size(); i++) { + Type icebergType = expected.get(i).type(); + Field arrowField = fields.get(i); + this.fieldReaders.add( + VortexSchemaWithTypeVisitor.visit(icebergType, arrowField, SparkReadBuilder.INSTANCE)); + } } @Override - public InternalRow read(Array batch, int row) { - return (InternalRow) reader.read(batch, row); + public InternalRow read(VectorSchemaRoot batch, int row) { + GenericInternalRow result = new GenericInternalRow(fieldReaders.size()); + for (int i = 0; i < fieldReaders.size(); i++) { + VortexValueReader reader = fieldReaders.get(i); + result.update(i, reader.read(batch.getVector(i), row)); + } + return result; } static class SparkReadBuilder extends VortexSchemaWithTypeVisitor> { @@ -53,21 +72,18 @@ private SparkReadBuilder() {} @Override public VortexValueReader struct( - Types.StructType schema, - List types, - List names, - List> fields) { - return new StructReader(fields); + Types.StructType schema, List fields, List> children) { + return new StructReader(children); } @Override public VortexValueReader list( - Types.ListType iList, DType array, VortexValueReader element) { + Types.ListType iList, Field listField, VortexValueReader element) { throw new UnsupportedOperationException("Vortex LIST types are not supported yet"); } @Override - public VortexValueReader primitive(Type.PrimitiveType icebergType, DType vortexType) { + public VortexValueReader primitive(Type.PrimitiveType icebergType, Field primField) { switch (icebergType.typeId()) { case BOOLEAN: return GenericVortexReaders.bools(); @@ -82,19 +98,20 @@ public VortexValueReader primitive(Type.PrimitiveType icebergType, DType vort case STRING: return SparkVortexValueReaders.utf8String(); case BINARY: - // Spark expects binary to be in another format, no? return GenericVortexReaders.bytes(); case DECIMAL: return GenericVortexReaders.decimals(); case TIMESTAMP: case TIMESTAMP_NANO: - // TODO(aduffy): timestamp and date types - return SparkVortexValueReaders.timestamp(vortexType.getTimeUnit()); + { + ArrowType.Timestamp ts = (ArrowType.Timestamp) primField.getType(); + return SparkVortexValueReaders.timestamp(ts.getUnit()); + } case DATE: - // TODO(aduffy): timestamp and date types - return SparkVortexValueReaders.date(vortexType.getTimeUnit()); + return SparkVortexValueReaders.date(); + case UUID: + return SparkVortexValueReaders.uuid(); case TIME: - // TODO(aduffy): timestamp and date types default: throw new UnsupportedOperationException("Unsupported type: " + icebergType); } @@ -109,14 +126,20 @@ private StructReader(List> fields) { } @Override - public InternalRow readNonNull(Array array, int row) { + public InternalRow readNonNull(FieldVector vector, int row) { + org.apache.arrow.vector.complex.StructVector struct = + (org.apache.arrow.vector.complex.StructVector) vector; GenericInternalRow result = new GenericInternalRow(fields.size()); for (int i = 0; i < fields.size(); i++) { VortexValueReader fieldReader = fields.get(i); - Object field = fieldReader.read(array.getField(i), row); - result.update(i, field); + FieldVector child = (FieldVector) struct.getChildByOrdinal(i); + result.update(i, fieldReader.read(child, row)); } return result; } } + + // Silence unused warning that TimeUnit class is required for the public API. + @SuppressWarnings("unused") + private static final TimeUnit UNUSED = TimeUnit.MICROSECOND; } diff --git a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexValueReaders.java b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexValueReaders.java index 9ff3cda53674..d48685814862 100644 --- a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexValueReaders.java +++ b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexValueReaders.java @@ -18,10 +18,19 @@ */ package org.apache.iceberg.spark.data; -import dev.vortex.api.Array; -import dev.vortex.api.DType; -import java.util.function.BiFunction; +import java.nio.charset.StandardCharsets; +import org.apache.arrow.vector.BigIntVector; +import org.apache.arrow.vector.DateDayVector; +import org.apache.arrow.vector.DateMilliVector; +import org.apache.arrow.vector.ExtensionTypeVector; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.FixedSizeBinaryVector; +import org.apache.arrow.vector.TimeStampVector; +import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.types.TimeUnit; +import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.iceberg.util.DateTimeUtil; +import org.apache.iceberg.util.UUIDUtil; import org.apache.iceberg.vortex.VortexValueReader; import org.apache.spark.unsafe.types.UTF8String; @@ -32,12 +41,17 @@ public static VortexValueReader utf8String() { return UTF8Reader.INSTANCE; } - public static VortexValueReader date(DType.TimeUnit timeUnit) { - return new DateReader(timeUnit); + public static VortexValueReader date() { + return DateReader.INSTANCE; } - public static VortexValueReader timestamp(DType.TimeUnit timeUnit) { - // Spark timestamp has us precision + public static VortexValueReader uuid() { + // Iceberg's UUID maps to Spark StringType; emit the canonical UUID string. + return UuidReader.INSTANCE; + } + + public static VortexValueReader timestamp(TimeUnit timeUnit) { + // Spark timestamp has µs precision return new TimestampReader(timeUnit); } @@ -47,74 +61,66 @@ static class UTF8Reader implements VortexValueReader { private UTF8Reader() {} @Override - public UTF8String readNonNull(Array array, int row) { - // TODO(aduffy): do this zero-copy by getting Vortex to give us back the pointer + len - // to the decompressed string. - String value = array.getUTF8(row); - return UTF8String.fromString(value); + public UTF8String readNonNull(FieldVector vector, int row) { + byte[] bytes = ((VarCharVector) vector).get(row); + return UTF8String.fromString(new String(bytes, StandardCharsets.UTF_8)); } } - // Spark expects DateType as Integer number of days since UNIX epoch - static class DateReader implements VortexValueReader { - private final BiFunction reader; - - private DateReader(DType.TimeUnit timeUnit) { - switch (timeUnit) { - case MILLISECONDS: - this.reader = - (array, row) -> { - long millis = array.getLong(row); - return DateTimeUtil.microsToDays(millis * 1000); - }; - break; - case DAYS: - this.reader = Array::getInt; - break; - default: - throw new IllegalArgumentException("Unsupported time unit for DATE: " + timeUnit); - } + static class UuidReader implements VortexValueReader { + static final UuidReader INSTANCE = new UuidReader(); + + private UuidReader() {} + + @Override + public UTF8String readNonNull(FieldVector vector, int row) { + FixedSizeBinaryVector storage = + vector instanceof ExtensionTypeVector ext + ? (FixedSizeBinaryVector) ext.getUnderlyingVector() + : (FixedSizeBinaryVector) vector; + return UTF8String.fromString(UUIDUtil.convert(storage.get(row)).toString()); } + } + + // Spark expects DateType as Integer number of days since UNIX epoch. + static class DateReader implements VortexValueReader { + static final DateReader INSTANCE = new DateReader(); + + private DateReader() {} @Override - public Integer readNonNull(Array array, int row) { - return this.reader.apply(array, row); + public Integer readNonNull(FieldVector vector, int row) { + ArrowType arrowType = vector.getField().getType(); + if (arrowType instanceof ArrowType.Date dateType + && dateType.getUnit() == org.apache.arrow.vector.types.DateUnit.MILLISECOND) { + long millis = ((DateMilliVector) vector).get(row); + return DateTimeUtil.microsToDays(millis * 1000L); + } + return ((DateDayVector) vector).get(row); } } static class TimestampReader implements VortexValueReader { - private final BiFunction reader; - - private TimestampReader(DType.TimeUnit vortexTimeUnit) { - switch (vortexTimeUnit) { - case NANOSECONDS: - // Round nanoseconds to microsecond - this.reader = (array, row) -> Math.floorDiv(array.getLong(row), 1_000L); - break; - case MICROSECONDS: - // Microseconds measurements - this.reader = Array::getLong; - break; - case MILLISECONDS: - // Milliseconds -> Microseconds - this.reader = (array, row) -> Math.multiplyExact(array.getLong(row), 1_000L); - break; - case SECONDS: - // Seconds -> Microseconds - this.reader = (array, row) -> Math.multiplyExact(array.getLong(row), 1_000_000L); - break; - case DAYS: - // Days -> Microseconds - this.reader = (array, row) -> Math.multiplyExact(array.getLong(row), 86_400_000_000L); - break; - default: - throw new IllegalStateException("Unexpected value: " + vortexTimeUnit); - } + private final TimeUnit unit; + + private TimestampReader(TimeUnit unit) { + this.unit = unit; } @Override - public Long readNonNull(Array array, int row) { - return this.reader.apply(array, row); + public Long readNonNull(FieldVector vector, int row) { + long measure; + if (vector instanceof TimeStampVector ts) { + measure = ts.get(row); + } else { + measure = ((BigIntVector) vector).get(row); + } + return switch (unit) { + case NANOSECOND -> Math.floorDiv(measure, 1_000L); + case MICROSECOND -> measure; + case MILLISECOND -> Math.multiplyExact(measure, 1_000L); + case SECOND -> Math.multiplyExact(measure, 1_000_000L); + }; } } } diff --git a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexWriter.java b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexWriter.java index 8c416b133382..cf776f27bb8c 100644 --- a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexWriter.java +++ b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexWriter.java @@ -20,11 +20,14 @@ import java.math.BigDecimal; import java.util.List; +import java.util.UUID; import org.apache.arrow.vector.BigIntVector; import org.apache.arrow.vector.BitVector; import org.apache.arrow.vector.DateDayVector; import org.apache.arrow.vector.DecimalVector; +import org.apache.arrow.vector.ExtensionTypeVector; import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.FixedSizeBinaryVector; import org.apache.arrow.vector.Float4Vector; import org.apache.arrow.vector.Float8Vector; import org.apache.arrow.vector.IntVector; @@ -38,6 +41,7 @@ import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.iceberg.Schema; import org.apache.iceberg.types.Types; +import org.apache.iceberg.util.UUIDUtil; import org.apache.iceberg.vortex.VortexValueWriter; import org.apache.spark.sql.catalyst.InternalRow; import org.apache.spark.unsafe.types.UTF8String; @@ -111,6 +115,14 @@ private static void writeValue( case DATE: ((DateDayVector) vector).setSafe(rowIndex, row.getInt(fieldIndex)); break; + case UUID: + UUID uuid = UUID.fromString(row.getUTF8String(fieldIndex).toString()); + FixedSizeBinaryVector uuidStorage = + vector instanceof ExtensionTypeVector ext + ? (FixedSizeBinaryVector) ext.getUnderlyingVector() + : (FixedSizeBinaryVector) vector; + uuidStorage.setSafe(rowIndex, UUIDUtil.convert(uuid)); + break; case TIME: ((TimeMicroVector) vector).setSafe(rowIndex, row.getLong(fieldIndex)); break; diff --git a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/VectorizedSparkVortexReaders.java b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/VectorizedSparkVortexReaders.java index 8e9a36a3890b..e7045540cb68 100644 --- a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/VectorizedSparkVortexReaders.java +++ b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/VectorizedSparkVortexReaders.java @@ -18,19 +18,16 @@ */ package org.apache.iceberg.spark.data.vectorized; -import dev.vortex.api.Array; -import dev.vortex.api.DType; -import dev.vortex.arrow.ArrowAllocation; -import dev.vortex.relocated.org.apache.arrow.vector.VectorSchemaRoot; -import dev.vortex.spark.read.VortexArrowColumnVector; -import dev.vortex.spark.read.VortexColumnarBatch; import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.iceberg.MetadataColumns; import org.apache.iceberg.Schema; import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.vortex.VortexBatchReader; +import org.apache.spark.sql.vectorized.ArrowColumnVector; import org.apache.spark.sql.vectorized.ColumnVector; import org.apache.spark.sql.vectorized.ColumnarBatch; @@ -38,7 +35,9 @@ public class VectorizedSparkVortexReaders { private VectorizedSparkVortexReaders() {} public static VortexBatchReader buildReader( - Schema icebergSchema, DType vortexSchema, Map idToConstant) { + Schema icebergSchema, + org.apache.arrow.vector.types.pojo.Schema vortexSchema, + Map idToConstant) { return new SchemaCachingBatchReader(icebergSchema, vortexSchema, idToConstant); } @@ -47,20 +46,18 @@ static final class SchemaCachingBatchReader implements VortexBatchReader idToConstant; private final List schemaMapping; - // Reusable vector schema root. - private VectorSchemaRoot root; - SchemaCachingBatchReader( - Schema readerSchema, DType vortexSchema, Map idToConstant) { + Schema readerSchema, + org.apache.arrow.vector.types.pojo.Schema vortexSchema, + Map idToConstant) { this.readerSchema = readerSchema; this.idToConstant = idToConstant; this.schemaMapping = vortexSchemaMapping(readerSchema, vortexSchema); } @Override - public ColumnarBatch read(Array batch) { - this.root = batch.exportToArrow(ArrowAllocation.rootAllocator(), this.root); - int rowCount = this.root.getRowCount(); + public ColumnarBatch read(VectorSchemaRoot batch) { + int rowCount = batch.getRowCount(); Map vectors = Maps.newHashMap(); for (Map.Entry entry : idToConstant.entrySet()) { @@ -70,26 +67,24 @@ public ColumnarBatch read(Array batch) { continue; } - // Field IDs are 1-indexed in Iceberg. vectors.put( - fieldId, - new ConstantColumnVector( - readerSchema.findType(fieldId), (int) batch.getLen(), constant)); + fieldId, new ConstantColumnVector(readerSchema.findType(fieldId), rowCount, constant)); } - for (int i = 0; i < root.getFieldVectors().size(); i++) { + List fieldVectors = batch.getFieldVectors(); + for (int i = 0; i < fieldVectors.size(); i++) { int fieldId = schemaMapping.get(i); - vectors.put(fieldId, new VortexArrowColumnVector(root.getVector(i))); + vectors.put(fieldId, new ArrowColumnVector(fieldVectors.get(i))); } - return new VortexColumnarBatch( - batch, vectors.values().toArray(new ColumnVector[0]), rowCount); + return new ColumnarBatch(vectors.values().toArray(new ColumnVector[0]), rowCount); } - // Mapping from Vortex schema index to Iceberg Field ID. - static List vortexSchemaMapping(Schema icebergSchema, DType vortexSchema) { - return vortexSchema.getFieldNames().stream() - .map(fieldName -> icebergSchema.findField(fieldName).fieldId()) + // Mapping from Arrow Schema field index to Iceberg Field ID. + static List vortexSchemaMapping( + Schema icebergSchema, org.apache.arrow.vector.types.pojo.Schema vortexSchema) { + return vortexSchema.getFields().stream() + .map(field -> icebergSchema.findField(field.getName()).fieldId()) .collect(Collectors.toList()); } } diff --git a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexReader.java b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexReader.java index a2d28cf5e3ba..45c76955b0a6 100644 --- a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexReader.java +++ b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexReader.java @@ -18,12 +18,15 @@ */ package org.apache.iceberg.spark.data; -import dev.vortex.api.Array; -import dev.vortex.api.DType; import java.util.List; import java.util.Map; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; import org.apache.iceberg.Schema; import org.apache.iceberg.data.vortex.GenericVortexReaders; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; import org.apache.iceberg.vortex.VortexRowReader; @@ -34,16 +37,31 @@ /** Read Vortex as Spark {@link InternalRow}. */ public class SparkVortexReader implements VortexRowReader { - private final VortexValueReader reader; + private final List> fieldReaders; - public SparkVortexReader(Schema readSchema, DType vortexSchema, Map idToConstant) { - this.reader = - VortexSchemaWithTypeVisitor.visit(readSchema, vortexSchema, SparkReadBuilder.INSTANCE); + public SparkVortexReader( + Schema readSchema, + org.apache.arrow.vector.types.pojo.Schema fileArrowSchema, + Map idToConstant) { + List fields = fileArrowSchema.getFields(); + List expected = readSchema.columns(); + this.fieldReaders = Lists.newArrayListWithExpectedSize(expected.size()); + for (int i = 0; i < expected.size(); i++) { + Type icebergType = expected.get(i).type(); + Field arrowField = fields.get(i); + this.fieldReaders.add( + VortexSchemaWithTypeVisitor.visit(icebergType, arrowField, SparkReadBuilder.INSTANCE)); + } } @Override - public InternalRow read(Array batch, int row) { - return (InternalRow) reader.read(batch, row); + public InternalRow read(VectorSchemaRoot batch, int row) { + GenericInternalRow result = new GenericInternalRow(fieldReaders.size()); + for (int i = 0; i < fieldReaders.size(); i++) { + VortexValueReader reader = fieldReaders.get(i); + result.update(i, reader.read(batch.getVector(i), row)); + } + return result; } static class SparkReadBuilder extends VortexSchemaWithTypeVisitor> { @@ -53,51 +71,39 @@ private SparkReadBuilder() {} @Override public VortexValueReader struct( - Types.StructType schema, - List types, - List names, - List> fields) { - return new StructReader(fields); + Types.StructType schema, List fields, List> children) { + return new StructReader(children); } @Override public VortexValueReader list( - Types.ListType iList, DType array, VortexValueReader element) { + Types.ListType iList, Field listField, VortexValueReader element) { throw new UnsupportedOperationException("Vortex LIST types are not supported yet"); } @Override - public VortexValueReader primitive(Type.PrimitiveType icebergType, DType vortexType) { - switch (icebergType.typeId()) { - case BOOLEAN: - return GenericVortexReaders.bools(); - case INTEGER: - return GenericVortexReaders.ints(); - case LONG: - return GenericVortexReaders.longs(); - case FLOAT: - return GenericVortexReaders.floats(); - case DOUBLE: - return GenericVortexReaders.doubles(); - case STRING: - return SparkVortexValueReaders.utf8String(); - case BINARY: - // Spark expects binary to be in another format, no? - return GenericVortexReaders.bytes(); - case DECIMAL: - return GenericVortexReaders.decimals(); - case TIMESTAMP: - case TIMESTAMP_NANO: - // TODO(aduffy): timestamp and date types - return SparkVortexValueReaders.timestamp(vortexType.getTimeUnit()); - case DATE: - // TODO(aduffy): timestamp and date types - return SparkVortexValueReaders.date(vortexType.getTimeUnit()); - case TIME: - // TODO(aduffy): timestamp and date types - default: - throw new UnsupportedOperationException("Unsupported type: " + icebergType); - } + public VortexValueReader primitive(Type.PrimitiveType icebergType, Field primField) { + return switch (icebergType.typeId()) { + case BOOLEAN -> GenericVortexReaders.bools(); + case INTEGER -> GenericVortexReaders.ints(); + case LONG -> GenericVortexReaders.longs(); + case FLOAT -> GenericVortexReaders.floats(); + case DOUBLE -> GenericVortexReaders.doubles(); + case STRING -> SparkVortexValueReaders.utf8String(); + case BINARY -> GenericVortexReaders.bytes(); + case DECIMAL -> GenericVortexReaders.decimals(); + case TIMESTAMP, TIMESTAMP_NANO -> { + ArrowType.Timestamp ts = (ArrowType.Timestamp) primField.getType(); + yield SparkVortexValueReaders.timestamp(ts.getUnit()); + } + case TIME -> { + ArrowType.Time t = (ArrowType.Time) primField.getType(); + yield SparkVortexValueReaders.time(t.getUnit()); + } + case DATE -> SparkVortexValueReaders.date(); + case UUID -> SparkVortexValueReaders.uuid(); + default -> throw new UnsupportedOperationException("Unsupported type: " + icebergType); + }; } } @@ -109,12 +115,14 @@ private StructReader(List> fields) { } @Override - public InternalRow readNonNull(Array array, int row) { + public InternalRow readNonNull(FieldVector vector, int row) { + org.apache.arrow.vector.complex.StructVector struct = + (org.apache.arrow.vector.complex.StructVector) vector; GenericInternalRow result = new GenericInternalRow(fields.size()); for (int i = 0; i < fields.size(); i++) { VortexValueReader fieldReader = fields.get(i); - Object field = fieldReader.read(array.getField(i), row); - result.update(i, field); + FieldVector child = (FieldVector) struct.getChildByOrdinal(i); + result.update(i, fieldReader.read(child, row)); } return result; } diff --git a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexValueReaders.java b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexValueReaders.java index 9ff3cda53674..8ce5ce6d20c2 100644 --- a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexValueReaders.java +++ b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexValueReaders.java @@ -18,10 +18,21 @@ */ package org.apache.iceberg.spark.data; -import dev.vortex.api.Array; -import dev.vortex.api.DType; -import java.util.function.BiFunction; +import java.nio.charset.StandardCharsets; +import org.apache.arrow.vector.BigIntVector; +import org.apache.arrow.vector.DateDayVector; +import org.apache.arrow.vector.DateMilliVector; +import org.apache.arrow.vector.ExtensionTypeVector; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.FixedSizeBinaryVector; +import org.apache.arrow.vector.TimeMicroVector; +import org.apache.arrow.vector.TimeNanoVector; +import org.apache.arrow.vector.TimeStampVector; +import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.types.TimeUnit; +import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.iceberg.util.DateTimeUtil; +import org.apache.iceberg.util.UUIDUtil; import org.apache.iceberg.vortex.VortexValueReader; import org.apache.spark.unsafe.types.UTF8String; @@ -32,89 +43,117 @@ public static VortexValueReader utf8String() { return UTF8Reader.INSTANCE; } - public static VortexValueReader date(DType.TimeUnit timeUnit) { - return new DateReader(timeUnit); + public static VortexValueReader date() { + return DateReader.INSTANCE; } - public static VortexValueReader timestamp(DType.TimeUnit timeUnit) { - // Spark timestamp has us precision + public static VortexValueReader uuid() { + // Iceberg's UUID maps to Spark StringType; emit the canonical UUID string. + return UuidReader.INSTANCE; + } + + public static VortexValueReader timestamp(TimeUnit timeUnit) { + // Spark timestamp has µs precision return new TimestampReader(timeUnit); } + public static VortexValueReader time(TimeUnit timeUnit) { + // Spark's TimeType is stored as microseconds since midnight (Long). + return new TimeReader(timeUnit); + } + static class UTF8Reader implements VortexValueReader { static final UTF8Reader INSTANCE = new UTF8Reader(); private UTF8Reader() {} @Override - public UTF8String readNonNull(Array array, int row) { - // TODO(aduffy): do this zero-copy by getting Vortex to give us back the pointer + len - // to the decompressed string. - String value = array.getUTF8(row); - return UTF8String.fromString(value); + public UTF8String readNonNull(FieldVector vector, int row) { + byte[] bytes = ((VarCharVector) vector).get(row); + return UTF8String.fromString(new String(bytes, StandardCharsets.UTF_8)); } } - // Spark expects DateType as Integer number of days since UNIX epoch - static class DateReader implements VortexValueReader { - private final BiFunction reader; - - private DateReader(DType.TimeUnit timeUnit) { - switch (timeUnit) { - case MILLISECONDS: - this.reader = - (array, row) -> { - long millis = array.getLong(row); - return DateTimeUtil.microsToDays(millis * 1000); - }; - break; - case DAYS: - this.reader = Array::getInt; - break; - default: - throw new IllegalArgumentException("Unsupported time unit for DATE: " + timeUnit); - } + static class UuidReader implements VortexValueReader { + static final UuidReader INSTANCE = new UuidReader(); + + private UuidReader() {} + + @Override + public UTF8String readNonNull(FieldVector vector, int row) { + FixedSizeBinaryVector storage = + vector instanceof ExtensionTypeVector ext + ? (FixedSizeBinaryVector) ext.getUnderlyingVector() + : (FixedSizeBinaryVector) vector; + return UTF8String.fromString(UUIDUtil.convert(storage.get(row)).toString()); } + } + + // Spark expects DateType as Integer number of days since UNIX epoch. + static class DateReader implements VortexValueReader { + static final DateReader INSTANCE = new DateReader(); + + private DateReader() {} @Override - public Integer readNonNull(Array array, int row) { - return this.reader.apply(array, row); + public Integer readNonNull(FieldVector vector, int row) { + ArrowType arrowType = vector.getField().getType(); + if (arrowType instanceof ArrowType.Date dateType + && dateType.getUnit() == org.apache.arrow.vector.types.DateUnit.MILLISECOND) { + long millis = ((DateMilliVector) vector).get(row); + return DateTimeUtil.microsToDays(millis * 1000L); + } + return ((DateDayVector) vector).get(row); } } static class TimestampReader implements VortexValueReader { - private final BiFunction reader; - - private TimestampReader(DType.TimeUnit vortexTimeUnit) { - switch (vortexTimeUnit) { - case NANOSECONDS: - // Round nanoseconds to microsecond - this.reader = (array, row) -> Math.floorDiv(array.getLong(row), 1_000L); - break; - case MICROSECONDS: - // Microseconds measurements - this.reader = Array::getLong; - break; - case MILLISECONDS: - // Milliseconds -> Microseconds - this.reader = (array, row) -> Math.multiplyExact(array.getLong(row), 1_000L); - break; - case SECONDS: - // Seconds -> Microseconds - this.reader = (array, row) -> Math.multiplyExact(array.getLong(row), 1_000_000L); - break; - case DAYS: - // Days -> Microseconds - this.reader = (array, row) -> Math.multiplyExact(array.getLong(row), 86_400_000_000L); - break; - default: - throw new IllegalStateException("Unexpected value: " + vortexTimeUnit); + private final TimeUnit unit; + + private TimestampReader(TimeUnit unit) { + this.unit = unit; + } + + @Override + public Long readNonNull(FieldVector vector, int row) { + long measure; + if (vector instanceof TimeStampVector ts) { + measure = ts.get(row); + } else { + measure = ((BigIntVector) vector).get(row); } + return switch (unit) { + case NANOSECOND -> Math.floorDiv(measure, 1_000L); + case MICROSECOND -> measure; + case MILLISECOND -> Math.multiplyExact(measure, 1_000L); + case SECOND -> Math.multiplyExact(measure, 1_000_000L); + }; + } + } + + static class TimeReader implements VortexValueReader { + private final TimeUnit unit; + + private TimeReader(TimeUnit unit) { + this.unit = unit; } @Override - public Long readNonNull(Array array, int row) { - return this.reader.apply(array, row); + public Long readNonNull(FieldVector vector, int row) { + long measure; + if (vector instanceof TimeMicroVector tm) { + measure = tm.get(row); + } else if (vector instanceof TimeNanoVector tn) { + measure = tn.get(row); + } else { + measure = ((BigIntVector) vector).get(row); + } + return switch (unit) { + case NANOSECOND -> Math.floorDiv(measure, 1_000L); + case MICROSECOND -> measure; + case MILLISECOND -> Math.multiplyExact(measure, 1_000L); + case SECOND -> Math.multiplyExact(measure, 1_000_000L); + }; } } } diff --git a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexWriter.java b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexWriter.java index 8c416b133382..cf776f27bb8c 100644 --- a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexWriter.java +++ b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexWriter.java @@ -20,11 +20,14 @@ import java.math.BigDecimal; import java.util.List; +import java.util.UUID; import org.apache.arrow.vector.BigIntVector; import org.apache.arrow.vector.BitVector; import org.apache.arrow.vector.DateDayVector; import org.apache.arrow.vector.DecimalVector; +import org.apache.arrow.vector.ExtensionTypeVector; import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.FixedSizeBinaryVector; import org.apache.arrow.vector.Float4Vector; import org.apache.arrow.vector.Float8Vector; import org.apache.arrow.vector.IntVector; @@ -38,6 +41,7 @@ import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.iceberg.Schema; import org.apache.iceberg.types.Types; +import org.apache.iceberg.util.UUIDUtil; import org.apache.iceberg.vortex.VortexValueWriter; import org.apache.spark.sql.catalyst.InternalRow; import org.apache.spark.unsafe.types.UTF8String; @@ -111,6 +115,14 @@ private static void writeValue( case DATE: ((DateDayVector) vector).setSafe(rowIndex, row.getInt(fieldIndex)); break; + case UUID: + UUID uuid = UUID.fromString(row.getUTF8String(fieldIndex).toString()); + FixedSizeBinaryVector uuidStorage = + vector instanceof ExtensionTypeVector ext + ? (FixedSizeBinaryVector) ext.getUnderlyingVector() + : (FixedSizeBinaryVector) vector; + uuidStorage.setSafe(rowIndex, UUIDUtil.convert(uuid)); + break; case TIME: ((TimeMicroVector) vector).setSafe(rowIndex, row.getLong(fieldIndex)); break; diff --git a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/VectorizedSparkVortexReaders.java b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/VectorizedSparkVortexReaders.java index 8e9a36a3890b..e7045540cb68 100644 --- a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/VectorizedSparkVortexReaders.java +++ b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/VectorizedSparkVortexReaders.java @@ -18,19 +18,16 @@ */ package org.apache.iceberg.spark.data.vectorized; -import dev.vortex.api.Array; -import dev.vortex.api.DType; -import dev.vortex.arrow.ArrowAllocation; -import dev.vortex.relocated.org.apache.arrow.vector.VectorSchemaRoot; -import dev.vortex.spark.read.VortexArrowColumnVector; -import dev.vortex.spark.read.VortexColumnarBatch; import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.iceberg.MetadataColumns; import org.apache.iceberg.Schema; import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.vortex.VortexBatchReader; +import org.apache.spark.sql.vectorized.ArrowColumnVector; import org.apache.spark.sql.vectorized.ColumnVector; import org.apache.spark.sql.vectorized.ColumnarBatch; @@ -38,7 +35,9 @@ public class VectorizedSparkVortexReaders { private VectorizedSparkVortexReaders() {} public static VortexBatchReader buildReader( - Schema icebergSchema, DType vortexSchema, Map idToConstant) { + Schema icebergSchema, + org.apache.arrow.vector.types.pojo.Schema vortexSchema, + Map idToConstant) { return new SchemaCachingBatchReader(icebergSchema, vortexSchema, idToConstant); } @@ -47,20 +46,18 @@ static final class SchemaCachingBatchReader implements VortexBatchReader idToConstant; private final List schemaMapping; - // Reusable vector schema root. - private VectorSchemaRoot root; - SchemaCachingBatchReader( - Schema readerSchema, DType vortexSchema, Map idToConstant) { + Schema readerSchema, + org.apache.arrow.vector.types.pojo.Schema vortexSchema, + Map idToConstant) { this.readerSchema = readerSchema; this.idToConstant = idToConstant; this.schemaMapping = vortexSchemaMapping(readerSchema, vortexSchema); } @Override - public ColumnarBatch read(Array batch) { - this.root = batch.exportToArrow(ArrowAllocation.rootAllocator(), this.root); - int rowCount = this.root.getRowCount(); + public ColumnarBatch read(VectorSchemaRoot batch) { + int rowCount = batch.getRowCount(); Map vectors = Maps.newHashMap(); for (Map.Entry entry : idToConstant.entrySet()) { @@ -70,26 +67,24 @@ public ColumnarBatch read(Array batch) { continue; } - // Field IDs are 1-indexed in Iceberg. vectors.put( - fieldId, - new ConstantColumnVector( - readerSchema.findType(fieldId), (int) batch.getLen(), constant)); + fieldId, new ConstantColumnVector(readerSchema.findType(fieldId), rowCount, constant)); } - for (int i = 0; i < root.getFieldVectors().size(); i++) { + List fieldVectors = batch.getFieldVectors(); + for (int i = 0; i < fieldVectors.size(); i++) { int fieldId = schemaMapping.get(i); - vectors.put(fieldId, new VortexArrowColumnVector(root.getVector(i))); + vectors.put(fieldId, new ArrowColumnVector(fieldVectors.get(i))); } - return new VortexColumnarBatch( - batch, vectors.values().toArray(new ColumnVector[0]), rowCount); + return new ColumnarBatch(vectors.values().toArray(new ColumnVector[0]), rowCount); } - // Mapping from Vortex schema index to Iceberg Field ID. - static List vortexSchemaMapping(Schema icebergSchema, DType vortexSchema) { - return vortexSchema.getFieldNames().stream() - .map(fieldName -> icebergSchema.findField(fieldName).fieldId()) + // Mapping from Arrow Schema field index to Iceberg Field ID. + static List vortexSchemaMapping( + Schema icebergSchema, org.apache.arrow.vector.types.pojo.Schema vortexSchema) { + return vortexSchema.getFields().stream() + .map(field -> icebergSchema.findField(field.getName()).fieldId()) .collect(Collectors.toList()); } } diff --git a/vortex/src/main/java/org/apache/iceberg/data/vortex/GenericVortexReader.java b/vortex/src/main/java/org/apache/iceberg/data/vortex/GenericVortexReader.java index ce8398a2f43f..986e3595503e 100644 --- a/vortex/src/main/java/org/apache/iceberg/data/vortex/GenericVortexReader.java +++ b/vortex/src/main/java/org/apache/iceberg/data/vortex/GenericVortexReader.java @@ -18,42 +18,67 @@ */ package org.apache.iceberg.data.vortex; -import dev.vortex.api.Array; -import dev.vortex.api.DType; import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.Optional; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.types.DateUnit; +import org.apache.arrow.vector.types.TimeUnit; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; import org.apache.iceberg.Schema; +import org.apache.iceberg.data.GenericRecord; import org.apache.iceberg.data.Record; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; import org.apache.iceberg.vortex.VortexRowReader; import org.apache.iceberg.vortex.VortexSchemaWithTypeVisitor; +import org.apache.iceberg.vortex.VortexSchemas; import org.apache.iceberg.vortex.VortexValueReader; public class GenericVortexReader implements VortexRowReader { - private final VortexValueReader reader; + private final Types.StructType structType; + private final List> fieldReaders; private GenericVortexReader( - Schema expectedSchema, DType readVortexSchema, Map idToConstant) { - this.reader = - VortexSchemaWithTypeVisitor.visit( - expectedSchema, readVortexSchema, new GenericReadBuilder(idToConstant)); + Schema expectedSchema, + org.apache.arrow.vector.types.pojo.Schema fileArrowSchema, + Map idToConstant) { + this.structType = expectedSchema.asStruct(); + GenericReadBuilder builder = new GenericReadBuilder(idToConstant); + List fileFields = fileArrowSchema.getFields(); + List expectedFields = structType.fields(); + this.fieldReaders = Lists.newArrayListWithExpectedSize(expectedFields.size()); + for (int i = 0; i < expectedFields.size(); i++) { + Type icebergType = expectedFields.get(i).type(); + Field arrowField = fileFields.get(i); + this.fieldReaders.add(VortexSchemaWithTypeVisitor.visit(icebergType, arrowField, builder)); + } } - public static VortexRowReader buildReader(Schema expectedSchema, DType fileSchema) { - return new GenericVortexReader(expectedSchema, fileSchema, Collections.emptyMap()); + public static VortexRowReader buildReader( + Schema expectedSchema, org.apache.arrow.vector.types.pojo.Schema fileArrowSchema) { + return new GenericVortexReader(expectedSchema, fileArrowSchema, Collections.emptyMap()); } public static VortexRowReader buildReader( - Schema expectedSchema, DType fileSchema, Map idToConstant) { - return new GenericVortexReader(expectedSchema, fileSchema, idToConstant); + Schema expectedSchema, + org.apache.arrow.vector.types.pojo.Schema fileArrowSchema, + Map idToConstant) { + return new GenericVortexReader(expectedSchema, fileArrowSchema, idToConstant); } @Override - public Record read(Array batch, int row) { - return (Record) this.reader.read(batch, row); + public Record read(VectorSchemaRoot batch, int row) { + GenericRecord record = GenericRecord.create(structType); + for (int i = 0; i < fieldReaders.size(); i++) { + VortexValueReader reader = fieldReaders.get(i); + FieldVector vector = batch.getVector(i); + record.set(i, reader.read(vector, row)); + } + return record; } @SuppressWarnings("UnusedVariable") @@ -61,77 +86,63 @@ static class GenericReadBuilder extends VortexSchemaWithTypeVisitor idToConstant; - private GenericReadBuilder(Map idToConstant) { + GenericReadBuilder(Map idToConstant) { this.idToConstant = idToConstant; } @Override public VortexValueReader struct( - Types.StructType iStruct, - List types, - List names, - List> fields) { - return GenericVortexReaders.struct(iStruct, fields); + Types.StructType iStruct, List fields, List> children) { + return GenericVortexReaders.struct(iStruct, children); } @Override public VortexValueReader list( - Types.ListType iList, DType array, VortexValueReader element) { - // TODO(aduffy): implement list reader + Types.ListType iList, Field listField, VortexValueReader element) { throw new UnsupportedOperationException("LIST TYPES!"); } @Override - public VortexValueReader primitive(Type.PrimitiveType iPrimitive, DType vortexType) { - switch (vortexType.getVariant()) { - case NULL: - throw new UnsupportedOperationException("Vortex Null type not supported"); - case BOOL: - return GenericVortexReaders.bools(); - case PRIMITIVE_U8: - case PRIMITIVE_I8: - case PRIMITIVE_U16: - case PRIMITIVE_I16: - case PRIMITIVE_U32: - case PRIMITIVE_I32: - // Types that promote to Iceberg INTEGER - return GenericVortexReaders.ints(); - case PRIMITIVE_U64: - case PRIMITIVE_I64: - // Types that promote to Iceberg LONG - return GenericVortexReaders.longs(); - case PRIMITIVE_F16: - throw new UnsupportedOperationException("Vortex F16 type not supported"); - case PRIMITIVE_F32: - return GenericVortexReaders.floats(); - case PRIMITIVE_F64: - return GenericVortexReaders.doubles(); - case UTF8: - return GenericVortexReaders.strings(); - case BINARY: - return GenericVortexReaders.bytes(); - case EXTENSION: - // TODO(aduffy): implement TIME/DATE/TIMESTAMP support - if (vortexType.isDate()) { - boolean isMillis = vortexType.getTimeUnit() == DType.TimeUnit.MILLISECONDS; - return GenericVortexReaders.date(isMillis); - } else if (vortexType.isTimestamp()) { - Optional timeZone = vortexType.getTimeZone(); - boolean isNanosecond = vortexType.getTimeUnit() == DType.TimeUnit.NANOSECONDS; - - if (timeZone.isEmpty()) { - return GenericVortexReaders.timestamp(isNanosecond); - } else { - return GenericVortexReaders.timestampTz(timeZone.get(), isNanosecond); - } - } - // TODO(aduffy): handle vortex.time extension type (not used by TPC-H data) - - throw new UnsupportedOperationException("Unsupported Vortex Extension type in schema"); - default: - throw new UnsupportedOperationException( - "Unsupported Vortex type: " + vortexType.getVariant()); + public VortexValueReader primitive(Type.PrimitiveType iPrimitive, Field primField) { + ArrowType arrowType = primField.getType(); + if ((iPrimitive != null && iPrimitive.typeId() == Type.TypeID.UUID) + || VortexSchemas.isUuidField(primField)) { + return GenericVortexReaders.uuids(); + } else if (arrowType instanceof ArrowType.Bool) { + return GenericVortexReaders.bools(); + } else if (arrowType instanceof ArrowType.Int intType) { + return intType.getBitWidth() <= Integer.SIZE + ? GenericVortexReaders.ints() + : GenericVortexReaders.longs(); + } else if (arrowType instanceof ArrowType.FloatingPoint fpType) { + return switch (fpType.getPrecision()) { + case SINGLE -> GenericVortexReaders.floats(); + case DOUBLE -> GenericVortexReaders.doubles(); + case HALF -> + throw new UnsupportedOperationException("Half-precision floats are not supported"); + }; + } else if (arrowType instanceof ArrowType.Decimal) { + return GenericVortexReaders.decimals(); + } else if (arrowType instanceof ArrowType.Utf8 || arrowType instanceof ArrowType.LargeUtf8) { + return GenericVortexReaders.strings(); + } else if (arrowType instanceof ArrowType.Binary + || arrowType instanceof ArrowType.LargeBinary + || arrowType instanceof ArrowType.FixedSizeBinary) { + return GenericVortexReaders.bytes(); + } else if (arrowType instanceof ArrowType.Date dateType) { + return GenericVortexReaders.date(dateType.getUnit() == DateUnit.MILLISECOND); + } else if (arrowType instanceof ArrowType.Time timeType) { + return GenericVortexReaders.time(timeType.getUnit() == TimeUnit.NANOSECOND); + } else if (arrowType instanceof ArrowType.Timestamp tsType) { + boolean isNano = tsType.getUnit() == TimeUnit.NANOSECOND; + if (tsType.getTimezone() == null) { + return GenericVortexReaders.timestamp(isNano); + } else { + return GenericVortexReaders.timestampTz(tsType.getTimezone(), isNano); + } } + throw new UnsupportedOperationException( + "Unsupported Arrow type in Vortex read: " + arrowType); } } } diff --git a/vortex/src/main/java/org/apache/iceberg/data/vortex/GenericVortexReaders.java b/vortex/src/main/java/org/apache/iceberg/data/vortex/GenericVortexReaders.java index 17337743dd83..32c6e4616b9f 100644 --- a/vortex/src/main/java/org/apache/iceberg/data/vortex/GenericVortexReaders.java +++ b/vortex/src/main/java/org/apache/iceberg/data/vortex/GenericVortexReaders.java @@ -18,18 +18,37 @@ */ package org.apache.iceberg.data.vortex; -import dev.vortex.api.Array; import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; import java.time.Instant; import java.time.LocalDate; import java.time.LocalDateTime; +import java.time.LocalTime; import java.time.OffsetDateTime; import java.time.ZoneId; import java.util.List; +import java.util.UUID; +import org.apache.arrow.vector.BaseIntVector; +import org.apache.arrow.vector.BitVector; +import org.apache.arrow.vector.DateDayVector; +import org.apache.arrow.vector.DateMilliVector; +import org.apache.arrow.vector.DecimalVector; +import org.apache.arrow.vector.ExtensionTypeVector; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.FixedSizeBinaryVector; +import org.apache.arrow.vector.Float4Vector; +import org.apache.arrow.vector.Float8Vector; +import org.apache.arrow.vector.TimeMicroVector; +import org.apache.arrow.vector.TimeNanoVector; +import org.apache.arrow.vector.TimeStampVector; +import org.apache.arrow.vector.VarBinaryVector; +import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.complex.StructVector; import org.apache.iceberg.data.GenericRecord; import org.apache.iceberg.data.Record; import org.apache.iceberg.types.Types; import org.apache.iceberg.util.DateTimeUtil; +import org.apache.iceberg.util.UUIDUtil; import org.apache.iceberg.vortex.VortexValueReader; public class GenericVortexReaders { @@ -67,10 +86,18 @@ public static VortexValueReader bytes() { return BytesReader.INSTANCE; } + public static VortexValueReader uuids() { + return UuidReader.INSTANCE; + } + public static VortexValueReader date(boolean isMillis) { return new DateReader(isMillis); } + public static VortexValueReader time(boolean nanosecond) { + return new TimeReader(nanosecond); + } + public static VortexValueReader timestamp(boolean nanosecond) { return new TimestampReader(nanosecond); } @@ -79,14 +106,13 @@ public static VortexValueReader timestampTz(String timeZone, boo return new TimestampTzReader(timeZone, nanosecond); } - // Read a struct of record values instead. public static VortexValueReader struct( Types.StructType schema, List> readers) { return new StructReader(schema, readers); } public static VortexValueReader> list(VortexValueReader elementReader) { - return new ListReader(elementReader); + return new ListReader<>(elementReader); } private static class StructReader implements VortexValueReader { @@ -99,12 +125,13 @@ private StructReader(Types.StructType schema, List> readers } @Override - public Record readNonNull(Array array, int row) { + public Record readNonNull(FieldVector vector, int row) { + StructVector struct = (StructVector) vector; GenericRecord record = GenericRecord.create(schema); for (int i = 0; i < readers.size(); i++) { VortexValueReader reader = readers.get(i); - Array field = array.getField(i); - Object value = reader.read(field, row); + FieldVector child = (FieldVector) struct.getChildByOrdinal(i); + Object value = reader.read(child, row); record.set(i, value); } return record; @@ -120,8 +147,7 @@ private ListReader(VortexValueReader elementReader) { } @Override - public List readNonNull(Array array, int row) { - // TODO(aduffy): implement LIST reads in vortex-jni. + public List readNonNull(FieldVector vector, int row) { throw new UnsupportedOperationException("Reading lists from Vortex not supported yet"); } } @@ -132,8 +158,8 @@ private static class BooleanReader implements VortexValueReader { private BooleanReader() {} @Override - public Boolean readNonNull(Array array, int row) { - return array.getBool(row); + public Boolean readNonNull(FieldVector vector, int row) { + return ((BitVector) vector).get(row) != 0; } } @@ -143,8 +169,8 @@ private static class IntegerReader implements VortexValueReader { private IntegerReader() {} @Override - public Integer readNonNull(Array array, int row) { - return array.getInt(row); + public Integer readNonNull(FieldVector vector, int row) { + return (int) ((BaseIntVector) vector).getValueAsLong(row); } } @@ -154,8 +180,8 @@ private static class LongReader implements VortexValueReader { private LongReader() {} @Override - public Long readNonNull(Array array, int row) { - return array.getLong(row); + public Long readNonNull(FieldVector vector, int row) { + return ((BaseIntVector) vector).getValueAsLong(row); } } @@ -165,8 +191,8 @@ private static class DecimalReader implements VortexValueReader { private DecimalReader() {} @Override - public BigDecimal readNonNull(Array array, int row) { - return array.getBigDecimal(row); + public BigDecimal readNonNull(FieldVector vector, int row) { + return ((DecimalVector) vector).getObjectNotNull(row); } } @@ -176,8 +202,8 @@ private static class FloatReader implements VortexValueReader { private FloatReader() {} @Override - public Float readNonNull(Array array, int row) { - return array.getFloat(row); + public Float readNonNull(FieldVector vector, int row) { + return ((Float4Vector) vector).get(row); } } @@ -187,8 +213,8 @@ private static class DoubleReader implements VortexValueReader { private DoubleReader() {} @Override - public Double readNonNull(Array array, int row) { - return array.getDouble(row); + public Double readNonNull(FieldVector vector, int row) { + return ((Float8Vector) vector).get(row); } } @@ -198,8 +224,8 @@ private static class StringReader implements VortexValueReader { private StringReader() {} @Override - public String readNonNull(Array array, int row) { - return array.getUTF8(row); + public String readNonNull(FieldVector vector, int row) { + return new String(((VarCharVector) vector).get(row), StandardCharsets.UTF_8); } } @@ -209,9 +235,33 @@ private static class BytesReader implements VortexValueReader { private BytesReader() {} @Override - public byte[] readNonNull(Array array, int row) { - return array.getBinary(row); + public byte[] readNonNull(FieldVector vector, int row) { + return ((VarBinaryVector) vector).get(row); + } + } + + private static class UuidReader implements VortexValueReader { + static final UuidReader INSTANCE = new UuidReader(); + + private UuidReader() {} + + @Override + public UUID readNonNull(FieldVector vector, int row) { + return UUIDUtil.convert(uuidStorage(vector).get(row)); + } + } + + /** + * Returns the underlying {@link FixedSizeBinaryVector} for a UUID column. Vortex may emit either + * a registered extension vector (wrapping FixedSizeBinary) or the raw fixed-binary storage, + * depending on whether {@code arrow.uuid} is registered in the consumer's {@link + * org.apache.arrow.vector.types.pojo.ExtensionTypeRegistry}. + */ + static FixedSizeBinaryVector uuidStorage(FieldVector vector) { + if (vector instanceof ExtensionTypeVector ext) { + return (FixedSizeBinaryVector) ext.getUnderlyingVector(); } + return (FixedSizeBinaryVector) vector; } private static class DateReader implements VortexValueReader { @@ -222,12 +272,12 @@ private static class DateReader implements VortexValueReader { } @Override - public LocalDate readNonNull(Array array, int row) { + public LocalDate readNonNull(FieldVector vector, int row) { int days; if (isMillis) { - days = (int) Math.floorDiv(array.getLong(row), 86_400_000L); + days = (int) Math.floorDiv(((DateMilliVector) vector).get(row), 86_400_000L); } else { - days = array.getInt(row); + days = ((DateDayVector) vector).get(row); } return DateTimeUtil.dateFromDays(days); @@ -242,8 +292,8 @@ private TimestampReader(boolean nanosecond) { } @Override - public LocalDateTime readNonNull(Array array, int row) { - long measure = array.getLong(row); + public LocalDateTime readNonNull(FieldVector vector, int row) { + long measure = ((TimeStampVector) vector).get(row); if (nanosecond) { return DateTimeUtil.timestampFromNanos(measure); } else { @@ -252,6 +302,23 @@ public LocalDateTime readNonNull(Array array, int row) { } } + private static class TimeReader implements VortexValueReader { + private final boolean nanosecond; + + private TimeReader(boolean nanosecond) { + this.nanosecond = nanosecond; + } + + @Override + public LocalTime readNonNull(FieldVector vector, int row) { + if (nanosecond) { + return LocalTime.ofNanoOfDay(((TimeNanoVector) vector).get(row)); + } else { + return DateTimeUtil.timeFromMicros(((TimeMicroVector) vector).get(row)); + } + } + } + private static class TimestampTzReader implements VortexValueReader { private final ZoneId timeZone; private final boolean nanosecond; @@ -262,13 +329,13 @@ private TimestampTzReader(String timeZone, boolean nanosecond) { } @Override - public OffsetDateTime readNonNull(Array array, int row) { - long measure = array.getLong(row); + public OffsetDateTime readNonNull(FieldVector vector, int row) { + long measure = ((TimeStampVector) vector).get(row); long nanoAdjustment; if (nanosecond) { nanoAdjustment = measure; } else { - nanoAdjustment = Math.multiplyExact(1_000, measure); + nanoAdjustment = Math.multiplyExact(1_000L, measure); } return OffsetDateTime.ofInstant(Instant.EPOCH.plusNanos(nanoAdjustment), timeZone); } diff --git a/vortex/src/main/java/org/apache/iceberg/data/vortex/GenericVortexWriter.java b/vortex/src/main/java/org/apache/iceberg/data/vortex/GenericVortexWriter.java index 09c16024b4b5..a73ce14cf3c5 100644 --- a/vortex/src/main/java/org/apache/iceberg/data/vortex/GenericVortexWriter.java +++ b/vortex/src/main/java/org/apache/iceberg/data/vortex/GenericVortexWriter.java @@ -29,12 +29,15 @@ import java.time.temporal.ChronoUnit; import java.util.Comparator; import java.util.List; +import java.util.UUID; import java.util.stream.Stream; import org.apache.arrow.vector.BigIntVector; import org.apache.arrow.vector.BitVector; import org.apache.arrow.vector.DateDayVector; import org.apache.arrow.vector.DecimalVector; +import org.apache.arrow.vector.ExtensionTypeVector; import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.FixedSizeBinaryVector; import org.apache.arrow.vector.Float4Vector; import org.apache.arrow.vector.Float8Vector; import org.apache.arrow.vector.IntVector; @@ -51,6 +54,7 @@ import org.apache.iceberg.data.Record; import org.apache.iceberg.types.Types; import org.apache.iceberg.util.ByteBuffers; +import org.apache.iceberg.util.UUIDUtil; import org.apache.iceberg.vortex.VortexValueWriter; /** Writes Iceberg generic {@link Record} objects to Arrow vectors for Vortex file output. */ @@ -144,6 +148,13 @@ private static void writeValue( int epochDay = (int) ((LocalDate) value).toEpochDay(); ((DateDayVector) vector).setSafe(rowIndex, epochDay); break; + case UUID: + FixedSizeBinaryVector uuidStorage = + vector instanceof ExtensionTypeVector ext + ? (FixedSizeBinaryVector) ext.getUnderlyingVector() + : (FixedSizeBinaryVector) vector; + uuidStorage.setSafe(rowIndex, UUIDUtil.convert((UUID) value)); + break; case TIME: long timeMicros = ((LocalTime) value).getLong(java.time.temporal.ChronoField.MICRO_OF_DAY); ((TimeMicroVector) vector).setSafe(rowIndex, timeMicros); diff --git a/vortex/src/main/java/org/apache/iceberg/vortex/ConvertFilterToVortex.java b/vortex/src/main/java/org/apache/iceberg/vortex/ConvertFilterToVortex.java index 38933ebd0f04..872a3713e3da 100644 --- a/vortex/src/main/java/org/apache/iceberg/vortex/ConvertFilterToVortex.java +++ b/vortex/src/main/java/org/apache/iceberg/vortex/ConvertFilterToVortex.java @@ -19,18 +19,8 @@ package org.apache.iceberg.vortex; import dev.vortex.api.Expression; -import dev.vortex.api.expressions.Binary; -import dev.vortex.api.expressions.GetItem; -import dev.vortex.api.expressions.Literal; -import dev.vortex.api.expressions.Not; -import dev.vortex.api.expressions.Root; -import java.math.BigDecimal; -import java.nio.ByteBuffer; -import java.util.List; -import java.util.Objects; -import java.util.Optional; +import dev.vortex.api.Expression.BinaryOp; import java.util.Set; -import java.util.UUID; import org.apache.iceberg.Schema; import org.apache.iceberg.expressions.BoundPredicate; import org.apache.iceberg.expressions.BoundReference; @@ -38,7 +28,6 @@ import org.apache.iceberg.expressions.Expressions; import org.apache.iceberg.expressions.UnboundPredicate; import org.apache.iceberg.types.Type; -import org.apache.iceberg.types.Types; /** * Convert an Iceberg filter expression into a valid Vortex pruning predicate that can be pushed @@ -47,8 +36,11 @@ *

Filters that cannot be translated will default to {@code ALWAYS_TRUE} to be skipped. */ public final class ConvertFilterToVortex extends ExpressionVisitors.ExpressionVisitor { - private static final Expression ALWAYS_TRUE = Literal.bool(true); - private static final Expression ALWAYS_FALSE = Literal.bool(false); + static final Expression ALWAYS_TRUE = Expression.literal(true); + static final Expression ALWAYS_FALSE = Expression.literal(false); + // Sentinel distinct by reference from ALWAYS_TRUE/FALSE, used to mark sub-expressions that + // could not be translated to a Vortex expression. + static final Expression UNCONVERTIBLE = Expression.literal(true); private static final int SET_PREDICATE_LIMIT = 200; private static final boolean CASE_SENSITIVE = true; @@ -62,7 +54,7 @@ private ConvertFilterToVortex(Schema fileSchema) { public static Expression convert(Schema schema, org.apache.iceberg.expressions.Expression expr) { org.apache.iceberg.expressions.Expression pushedNot = Expressions.rewriteNot(expr); Expression converted = ExpressionVisitors.visit(pushedNot, new ConvertFilterToVortex(schema)); - if (converted == UnconvertibleExpr.INSTANCE) { + if (converted == UNCONVERTIBLE) { return ALWAYS_TRUE; } else { return converted; @@ -81,39 +73,36 @@ public Expression alwaysTrue() { @Override public Expression not(Expression child) { - // Simplify ALWAYS_TRUE and ALWAYS_FALSE directly so that - // they avoid any compute in Vortex. if (child == ALWAYS_TRUE) { return ALWAYS_FALSE; } else if (child == ALWAYS_FALSE) { return ALWAYS_TRUE; - } else if (child == UnconvertibleExpr.INSTANCE) { - // propagate convertibility - return UnconvertibleExpr.INSTANCE; + } else if (child == UNCONVERTIBLE) { + return UNCONVERTIBLE; } else { - return Not.of(child); + return Expression.not(child); } } @Override public Expression and(Expression leftResult, Expression rightResult) { - if (leftResult == UnconvertibleExpr.INSTANCE && rightResult == UnconvertibleExpr.INSTANCE) { + if (leftResult == UNCONVERTIBLE && rightResult == UNCONVERTIBLE) { return ALWAYS_TRUE; - } else if (leftResult == UnconvertibleExpr.INSTANCE) { + } else if (leftResult == UNCONVERTIBLE) { return rightResult; - } else if (rightResult == UnconvertibleExpr.INSTANCE) { + } else if (rightResult == UNCONVERTIBLE) { return leftResult; } else { - return Binary.and(leftResult, rightResult); + return Expression.and(leftResult, rightResult); } } @Override public Expression or(Expression leftResult, Expression rightResult) { - if (leftResult == UnconvertibleExpr.INSTANCE || rightResult == UnconvertibleExpr.INSTANCE) { + if (leftResult == UNCONVERTIBLE || rightResult == UNCONVERTIBLE) { return ALWAYS_TRUE; } else { - return Binary.or(leftResult, rightResult); + return Expression.or(leftResult, rightResult); } } @@ -121,28 +110,31 @@ public Expression or(Expression leftResult, Expression rightResult) { public Expression predicate(BoundPredicate pred) { if (!(pred.term() instanceof BoundReference term)) { throw new UnsupportedOperationException( - "Cannot convert non-reference to Parquet filter: " + pred.term()); + "Cannot convert non-reference to Vortex filter: " + pred.term()); } + String name = term.ref().field().name(); if (pred.isLiteralPredicate()) { org.apache.iceberg.expressions.Literal icebergLit = pred.asLiteralPredicate().literal(); - Literal vortexLit = toVortexLiteral(icebergLit, term.type()); - // Term translates into a GetItem(Identity), i.e. get a field from the batch - GetItem vortexTerm = GetItem.of(Root.INSTANCE, term.ref().field().name()); + Expression vortexLit = toVortexLiteral(icebergLit.value(), term.type()); + if (vortexLit == UNCONVERTIBLE) { + return UNCONVERTIBLE; + } + Expression vortexTerm = Expression.column(name); return fromBinaryPredicate(pred.op(), vortexTerm, vortexLit); } else if (pred.isUnaryPredicate()) { - GetItem vortexTerm = GetItem.of(Root.INSTANCE, term.ref().field().name()); + Expression vortexTerm = Expression.column(name); return fromUnaryPredicate(pred.op(), vortexTerm); } else if (pred.isSetPredicate()) { Set literalSet = pred.asSetPredicate().literalSet(); if (literalSet.size() > SET_PREDICATE_LIMIT) { - return UnconvertibleExpr.INSTANCE; + return UNCONVERTIBLE; } - GetItem vortexTerm = GetItem.of(Root.INSTANCE, term.ref().field().name()); + Expression vortexTerm = Expression.column(name); return fromSetPredicate(pred.op(), vortexTerm, literalSet, term.type()); } else { - return UnconvertibleExpr.INSTANCE; + return UNCONVERTIBLE; } } @@ -156,203 +148,88 @@ public Expression predicate(UnboundPredicate pred) { } else if (bound == Expressions.alwaysFalse()) { return ALWAYS_FALSE; } - return UnconvertibleExpr.INSTANCE; + return UNCONVERTIBLE; } private org.apache.iceberg.expressions.Expression bind(UnboundPredicate pred) { return pred.bind(fileSchema.asStruct(), CASE_SENSITIVE); } - Literal toVortexLiteral(org.apache.iceberg.expressions.Literal literal, Type termType) { - switch (termType.typeId()) { - case BOOLEAN -> { - return Literal.bool((Boolean) literal.value()); - } - case INTEGER -> { - return Literal.int32((Integer) literal.value()); - } - case LONG -> { - return Literal.int64((Long) literal.value()); - } - case FLOAT -> { - return Literal.float32((Float) literal.value()); - } - case DOUBLE -> { - return Literal.float64((Double) literal.value()); - } - case DECIMAL -> { - Types.DecimalType decimalType = (Types.DecimalType) termType; - return Literal.decimal( - (BigDecimal) literal.value(), decimalType.precision(), decimalType.scale()); - } - case STRING -> { - CharSequence charSequence = (CharSequence) literal.value(); - if (Objects.isNull(charSequence)) { - return Literal.string(null); - } else { - return Literal.string(charSequence.toString()); - } - } - case BINARY -> { - ByteBuffer byteBuffer = (ByteBuffer) literal.value(); - if (Objects.isNull(byteBuffer)) { - return Literal.bytes(null); - } else { - byte[] bytes = new byte[byteBuffer.remaining()]; - byteBuffer.get(bytes); - return Literal.bytes(bytes); - } - } - case UUID -> { - UUID uuid = (UUID) literal.value(); - if (Objects.isNull(uuid)) { - return Literal.string(null); - } else { - return Literal.string(uuid.toString()); - } - } - case TIME -> { - return Literal.timeMicros((Long) literal.value()); - } - case DATE -> { - return Literal.dateDays((Integer) literal.value()); - } - case TIMESTAMP -> { - Types.TimestampType timestampType = (Types.TimestampType) termType; - if (timestampType.shouldAdjustToUTC()) { - throw new UnsupportedOperationException( - "Handling of timestamps with timezones not yet supported"); - } else { - // Iceberg always stores timestamp in microseconds. - // TODO(aduffy): get the Vortex type so we know if we need to convert to different - // precision. - return Literal.timestampMicros((Long) literal.value(), Optional.empty()); - } - } - default -> throw new UnsupportedOperationException("Unsupported Literal type: " + termType); + /** + * Convert an Iceberg value to a Vortex literal Expression. Returns {@link #UNCONVERTIBLE} if no + * matching Vortex literal type exists (binary, decimal, date, time, timestamp, uuid). + */ + private Expression toVortexLiteral(Object value, Type termType) { + if (value == null) { + return UNCONVERTIBLE; } + return switch (termType.typeId()) { + case BOOLEAN -> Expression.literal((Boolean) value); + case INTEGER -> Expression.literal((Integer) value); + case LONG -> Expression.literal((Long) value); + case FLOAT -> Expression.literal((Float) value); + case DOUBLE -> Expression.literal((Double) value); + case STRING -> Expression.literal(((CharSequence) value).toString()); + default -> UNCONVERTIBLE; + }; } - // Always true is not a real binary op... - Expression fromBinaryPredicate( + private Expression fromBinaryPredicate( org.apache.iceberg.expressions.Expression.Operation op, Expression left, Expression right) { - - if (left == UnconvertibleExpr.INSTANCE && right == UnconvertibleExpr.INSTANCE) { - return UnconvertibleExpr.INSTANCE; - } else if (left == UnconvertibleExpr.INSTANCE) { - return right; - } - return switch (op) { case TRUE -> ALWAYS_TRUE; case FALSE -> ALWAYS_FALSE; - case LT -> Binary.lt(left, right); - case LT_EQ -> Binary.ltEq(left, right); - case GT -> Binary.gt(left, right); - case GT_EQ -> Binary.gtEq(left, right); - case EQ -> Binary.eq(left, right); - case NOT_EQ -> Binary.notEq(left, right); - case AND -> Binary.and(left, right); - case OR -> Binary.or(left, right); - default -> UnconvertibleExpr.INSTANCE; + case LT -> Expression.binary(BinaryOp.LT, left, right); + case LT_EQ -> Expression.binary(BinaryOp.LTE, left, right); + case GT -> Expression.binary(BinaryOp.GT, left, right); + case GT_EQ -> Expression.binary(BinaryOp.GTE, left, right); + case EQ -> Expression.binary(BinaryOp.EQ, left, right); + case NOT_EQ -> Expression.binary(BinaryOp.NOT_EQ, left, right); + case AND -> Expression.binary(BinaryOp.AND, left, right); + case OR -> Expression.binary(BinaryOp.OR, left, right); + default -> UNCONVERTIBLE; }; } - Expression fromUnaryPredicate( + private Expression fromUnaryPredicate( org.apache.iceberg.expressions.Expression.Operation op, Expression child) { - switch (op) { - case TRUE -> { - return ALWAYS_TRUE; - } - case FALSE -> { - return ALWAYS_FALSE; - } + return switch (op) { + case TRUE -> ALWAYS_TRUE; + case FALSE -> ALWAYS_FALSE; + case IS_NULL -> Expression.isNull(child); + case NOT_NULL -> Expression.not(Expression.isNull(child)); case NOT -> { if (child == ALWAYS_TRUE) { - return ALWAYS_FALSE; + yield ALWAYS_FALSE; } else if (child == ALWAYS_FALSE) { - return ALWAYS_TRUE; + yield ALWAYS_TRUE; } else { - return Not.of(child); + yield Expression.not(child); } } - default -> { - return ALWAYS_TRUE; - } - } + default -> UNCONVERTIBLE; + }; } - Expression fromSetPredicate( + private Expression fromSetPredicate( org.apache.iceberg.expressions.Expression.Operation op, - GetItem term, + Expression term, Set literalSet, Type termType) { - Expression[] eqExprs = - literalSet.stream() - .map(value -> (Expression) Binary.eq(term, toVortexValue(value, termType))) - .toArray(Expression[]::new); + Expression[] eqExprs = new Expression[literalSet.size()]; + int i = 0; + for (T value : literalSet) { + Expression vortexLit = toVortexLiteral(value, termType); + if (vortexLit == UNCONVERTIBLE) { + return UNCONVERTIBLE; + } + eqExprs[i++] = Expression.binary(BinaryOp.EQ, term, vortexLit); + } return switch (op) { - case IN -> Binary.or(eqExprs[0], java.util.Arrays.copyOfRange(eqExprs, 1, eqExprs.length)); - case NOT_IN -> - Not.of(Binary.or(eqExprs[0], java.util.Arrays.copyOfRange(eqExprs, 1, eqExprs.length))); - default -> UnconvertibleExpr.INSTANCE; - }; - } - - @SuppressWarnings("unchecked") - private Literal toVortexValue(T value, Type termType) { - return switch (termType.typeId()) { - case BOOLEAN -> Literal.bool((Boolean) value); - case INTEGER -> Literal.int32((Integer) value); - case LONG -> Literal.int64((Long) value); - case FLOAT -> Literal.float32((Float) value); - case DOUBLE -> Literal.float64((Double) value); - case DECIMAL -> { - Types.DecimalType decimalType = (Types.DecimalType) termType; - yield Literal.decimal((BigDecimal) value, decimalType.precision(), decimalType.scale()); - } - case STRING -> { - CharSequence charSequence = (CharSequence) value; - yield Literal.string(charSequence.toString()); - } - case DATE -> Literal.dateDays((Integer) value); - case TIME -> Literal.timeMicros((Long) value); - case TIMESTAMP -> { - Types.TimestampType timestampType = (Types.TimestampType) termType; - if (timestampType.shouldAdjustToUTC()) { - throw new UnsupportedOperationException( - "Handling of timestamps with timezones not yet supported"); - } - yield Literal.timestampMicros((Long) value, Optional.empty()); - } - default -> - throw new UnsupportedOperationException( - "Unsupported type for set predicate: " + termType); + case IN -> eqExprs.length == 1 ? eqExprs[0] : Expression.or(eqExprs); + case NOT_IN -> Expression.not(eqExprs.length == 1 ? eqExprs[0] : Expression.or(eqExprs)); + default -> UNCONVERTIBLE; }; } - - enum UnconvertibleExpr implements Expression { - INSTANCE; - - @Override - public String id() { - return "unconvertible"; - } - - @Override - public List children() { - return List.of(); - } - - @Override - public Optional metadata() { - return Optional.empty(); - } - - @Override - public T accept(Visitor visitor) { - return null; - } - } } diff --git a/vortex/src/main/java/org/apache/iceberg/vortex/PrefetchingIterator.java b/vortex/src/main/java/org/apache/iceberg/vortex/PrefetchingIterator.java deleted file mode 100644 index 85e27599aa7b..000000000000 --- a/vortex/src/main/java/org/apache/iceberg/vortex/PrefetchingIterator.java +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iceberg.vortex; - -import java.util.Iterator; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicLong; -import java.util.function.ToLongFunction; - -public final class PrefetchingIterator implements Iterator, AutoCloseable { - // Global condition variable shared between the prefetcher and consumer threads, - // to coordinate wake ups for when the buffer may no longer be full. - private static final Object CONDITION_PUT = new Object(); - private static final Object CONDITION_CHECK = new Object(); - - private final BlockingQueue fetched = new LinkedBlockingQueue<>(); - private final Thread producerThread; - private final Iterator delegate; - private final AtomicBoolean closed = new AtomicBoolean(false); - private final AtomicLong bufferBytes = new AtomicLong(0); - private final long maxBufferSize; - private final ToLongFunction sizeFunc; - - PrefetchingIterator(Iterator delegate, long maxBufferSize, ToLongFunction sizeFunc) { - this.delegate = delegate; - this.maxBufferSize = maxBufferSize; - this.sizeFunc = sizeFunc; - this.producerThread = new Thread(this::prefetchLoop, "vortex-prefetch-thread"); - producerThread.setDaemon(true); - producerThread.start(); - } - - private void prefetchLoop() { - try { - while (!closed.get() && delegate.hasNext()) { - while (bufferBytes.get() > maxBufferSize) { - synchronized (CONDITION_PUT) { - CONDITION_PUT.wait(); - } - } - T nextElem = delegate.next(); - long elemSize = sizeFunc.applyAsLong(nextElem); - bufferBytes.addAndGet(elemSize); - fetched.put(nextElem); - synchronized (CONDITION_CHECK) { - CONDITION_CHECK.notify(); - } - } - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new RuntimeException("Prefetching interrupted", e); - } catch (Exception e) { - throw new RuntimeException("Prefetching failed", e); - } finally { - closed.set(true); - } - } - - @Override - public boolean hasNext() { - // If the prefetcher is not finished, then we could be waiting for - // a fetched item. - while (!closed.get()) { - if (!fetched.isEmpty()) { - return true; - } - // Wait for the prefetcher to tell us it's time to check again. - synchronized (CONDITION_CHECK) { - try { - CONDITION_CHECK.wait(); - } catch (InterruptedException e) { - throw new RuntimeException("Interrupted waiting for signal to check hasNext", e); - } - } - } - // If the prefetcher is finished, then we can examine fetched and immediately return a result. - return !fetched.isEmpty(); - } - - @Override - public T next() { - // We assume that this has been called after hasNext() returned true, so it is - // safe to call take() without checking if the queue maybe be empty. - try { - T nextElem = this.fetched.take(); - long elemSize = sizeFunc.applyAsLong(nextElem); - bufferBytes.addAndGet(-elemSize); - // Notify the producer that it may now be able to add more items to the queue. - synchronized (CONDITION_PUT) { - CONDITION_PUT.notify(); - } - return nextElem; - } catch (InterruptedException e) { - throw new RuntimeException("Prefetch queue take interrupted", e); - } - } - - @Override - public void close() { - closed.set(true); - synchronized (CONDITION_CHECK) { - CONDITION_CHECK.notifyAll(); - } - - synchronized (CONDITION_PUT) { - CONDITION_PUT.notifyAll(); - } - } -} diff --git a/vortex/src/main/java/org/apache/iceberg/vortex/VortexBatchReader.java b/vortex/src/main/java/org/apache/iceberg/vortex/VortexBatchReader.java index 3b391657f369..284880695646 100644 --- a/vortex/src/main/java/org/apache/iceberg/vortex/VortexBatchReader.java +++ b/vortex/src/main/java/org/apache/iceberg/vortex/VortexBatchReader.java @@ -18,9 +18,9 @@ */ package org.apache.iceberg.vortex; -import dev.vortex.api.Array; +import org.apache.arrow.vector.VectorSchemaRoot; -/** Read a Vortex {@link Array} as a batch of type {@link T}. */ +/** Reads an Arrow {@link VectorSchemaRoot} batch as a value of type {@link T}. */ public interface VortexBatchReader { - T read(Array batch); + T read(VectorSchemaRoot batch); } diff --git a/vortex/src/main/java/org/apache/iceberg/vortex/VortexFileAppender.java b/vortex/src/main/java/org/apache/iceberg/vortex/VortexFileAppender.java index 33565e0e5a05..1b92026375f1 100644 --- a/vortex/src/main/java/org/apache/iceberg/vortex/VortexFileAppender.java +++ b/vortex/src/main/java/org/apache/iceberg/vortex/VortexFileAppender.java @@ -19,13 +19,13 @@ package org.apache.iceberg.vortex; import dev.vortex.api.VortexWriter; -import java.io.ByteArrayOutputStream; +import dev.vortex.arrow.ArrowAllocation; import java.io.IOException; -import java.nio.channels.Channels; +import org.apache.arrow.c.ArrowArray; +import org.apache.arrow.c.ArrowSchema; +import org.apache.arrow.c.Data; import org.apache.arrow.memory.BufferAllocator; -import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.VectorSchemaRoot; -import org.apache.arrow.vector.ipc.ArrowStreamWriter; import org.apache.arrow.vector.types.pojo.Schema; import org.apache.iceberg.Metrics; import org.apache.iceberg.MetricsConfig; @@ -33,10 +33,11 @@ import org.apache.iceberg.io.OutputFile; /** - * A {@link FileAppender} that writes data to Vortex files via Arrow IPC. + * A {@link FileAppender} that writes data to Vortex files via the Arrow C-data interface. * - *

Rows are buffered in an Arrow {@link VectorSchemaRoot} and flushed as Arrow IPC stream batches - * to the underlying {@link VortexWriter} when the batch reaches the configured size. + *

Rows are buffered in an Arrow {@link VectorSchemaRoot} and flushed to the underlying {@link + * VortexWriter} when the batch reaches the configured size, by exporting each batch through the + * Arrow C-data interface as a pair of {@code (array, schema)} pointers. */ class VortexFileAppender implements FileAppender { static final int DEFAULT_BATCH_SIZE = 2048; @@ -58,14 +59,19 @@ class VortexFileAppender implements FileAppender { VortexWriter writer, VortexValueWriter valueWriter, Schema arrowSchema, + BufferAllocator allocator, int batchSize, OutputFile outputFile, org.apache.iceberg.Schema icebergSchema, MetricsConfig metricsConfig) { this.writer = writer; this.valueWriter = valueWriter; - this.allocator = new RootAllocator(); - this.root = VectorSchemaRoot.create(arrowSchema, allocator); + // Use Vortex's shared root allocator: VortexWriter performs allocations against this + // and the native side manages lifetime via Cleaner references. Owning our own + // RootAllocator and closing it eagerly trips strict leak checks when Vortex retains + // small per-writer state. + this.allocator = allocator != null ? allocator : ArrowAllocation.rootAllocator(); + this.root = VectorSchemaRoot.create(arrowSchema, this.allocator); this.batchSize = batchSize; this.outputFile = outputFile; this.icebergSchema = icebergSchema; @@ -94,14 +100,10 @@ private void flushBatch() { root.setRowCount(currentBatchIndex); - try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) { - try (ArrowStreamWriter streamWriter = - new ArrowStreamWriter(root, null, Channels.newChannel(baos))) { - streamWriter.start(); - streamWriter.writeBatch(); - } - - writer.writeBatch(baos.toByteArray()); + try (ArrowArray cArray = ArrowArray.allocateNew(allocator); + ArrowSchema cSchema = ArrowSchema.allocateNew(allocator)) { + Data.exportVectorSchemaRoot(allocator, root, null, cArray, cSchema); + writer.writeBatch(cArray.memoryAddress(), cSchema.memoryAddress()); } catch (IOException e) { throw new org.apache.iceberg.exceptions.RuntimeIOException(e, "Failed to write Vortex batch"); } @@ -133,7 +135,8 @@ public void close() throws IOException { writer.close(); } finally { root.close(); - allocator.close(); + // Don't close `allocator`: it is Vortex's shared root allocator and is managed for + // the lifetime of the process. closed = true; } } diff --git a/vortex/src/main/java/org/apache/iceberg/vortex/VortexFormatModel.java b/vortex/src/main/java/org/apache/iceberg/vortex/VortexFormatModel.java index 9835010b1d6f..b847980e3ddf 100644 --- a/vortex/src/main/java/org/apache/iceberg/vortex/VortexFormatModel.java +++ b/vortex/src/main/java/org/apache/iceberg/vortex/VortexFormatModel.java @@ -18,8 +18,10 @@ */ package org.apache.iceberg.vortex; -import dev.vortex.api.DType; +import dev.vortex.api.Session; import dev.vortex.api.VortexWriter; +import dev.vortex.arrow.ArrowAllocation; +import dev.vortex.jni.NativeRuntime; import java.io.IOException; import java.nio.ByteBuffer; import java.util.List; @@ -27,10 +29,12 @@ import java.util.Optional; import java.util.function.Function; import java.util.stream.Collectors; +import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.types.pojo.Schema; import org.apache.iceberg.FileContent; import org.apache.iceberg.FileFormat; import org.apache.iceberg.MetricsConfig; +import org.apache.iceberg.TableProperties; import org.apache.iceberg.encryption.EncryptedOutputFile; import org.apache.iceberg.expressions.Expression; import org.apache.iceberg.formats.BaseFormatModel; @@ -46,23 +50,25 @@ import org.apache.iceberg.types.Types; public class VortexFormatModel - extends BaseFormatModel, R, DType> { + extends BaseFormatModel, R, Schema> { private final boolean isBatchReader; public interface ReaderFunction { VortexRowReader read( - org.apache.iceberg.Schema schema, DType fileSchema, Map idToConstant); + org.apache.iceberg.Schema schema, Schema fileArrowSchema, Map idToConstant); } public interface BatchReaderFunction { VortexBatchReader batchRead( - org.apache.iceberg.Schema icebergSchema, DType vortexSchema, Map idToConstant); + org.apache.iceberg.Schema icebergSchema, + Schema fileArrowSchema, + Map idToConstant); } public static VortexFormatModel> create( Class type, Class schemaType, - WriterFunction, S, DType> writerFunction, + WriterFunction, S, Schema> writerFunction, ReaderFunction readerFunction) { return new VortexFormatModel<>( type, @@ -76,7 +82,7 @@ public static VortexFormatModel> create( public static VortexFormatModel> create( Class type, Class schemaType, - WriterFunction, S, DType> writerFunction, + WriterFunction, S, Schema> writerFunction, BatchReaderFunction batchReaderFunction) { return new VortexFormatModel<>( type, @@ -90,8 +96,8 @@ public static VortexFormatModel> create( private VortexFormatModel( Class type, Class schemaType, - WriterFunction, S, DType> writerFunction, - BaseFormatModel.ReaderFunction readerFunction, + WriterFunction, S, Schema> writerFunction, + BaseFormatModel.ReaderFunction readerFunction, boolean isBatchReader) { super(type, schemaType, writerFunction, readerFunction); this.isBatchReader = isBatchReader; @@ -114,17 +120,18 @@ public ReadBuilder readBuilder(InputFile inputFile) { private static class WriteBuilderWrapper implements ModelWriteBuilder { private final EncryptedOutputFile outputFile; - private final WriterFunction, S, DType> writerFunction; + private final WriterFunction, S, Schema> writerFunction; private org.apache.iceberg.Schema schema; private S engineSchema; private FileContent content; private MetricsConfig metricsConfig = MetricsConfig.getDefault(); private final Map writerProperties = Maps.newHashMap(); private final Map metadata = Maps.newHashMap(); + private int workerThreads = TableProperties.VORTEX_WORKER_THREADS_DEFAULT; private WriteBuilderWrapper( EncryptedOutputFile outputFile, - WriterFunction, S, DType> writerFunction) { + WriterFunction, S, Schema> writerFunction) { this.outputFile = outputFile; this.writerFunction = writerFunction; } @@ -143,13 +150,17 @@ public ModelWriteBuilder engineSchema(S newSchema) { @Override public ModelWriteBuilder set(String property, String value) { + if (TableProperties.WRITE_VORTEX_WORKER_THREADS.equals(property)) { + workerThreads = Integer.parseInt(value); + } + writerProperties.put(property, value); return this; } @Override public ModelWriteBuilder setAll(Map properties) { - writerProperties.putAll(properties); + properties.forEach(this::set); return this; } @@ -209,23 +220,30 @@ public FileAppender build() throws IOException { @SuppressWarnings("unchecked") private FileAppender buildAppender(org.apache.iceberg.Schema writeSchema) throws IOException { - DType dtype = VortexSchemas.toDType(writeSchema); Schema arrowSchema = VortexSchemas.toArrowSchema(writeSchema); VortexValueWriter valueWriter = - (VortexValueWriter) writerFunction.write(writeSchema, dtype, engineSchema); + (VortexValueWriter) writerFunction.write(writeSchema, arrowSchema, engineSchema); OutputFile rawOutputFile = outputFile.encryptingOutputFile(); String uri = VortexFileUtil.resolveUri(rawOutputFile.location()); Map properties = Maps.newHashMap(VortexFileUtil.resolveOutputProperties(rawOutputFile)); + properties.putAll(writerProperties); properties.putAll(metadata); - VortexWriter vortexWriter = VortexWriter.create(uri, dtype, properties); + // Apply worker-thread setting on this executor JVM before any Vortex native work begins. + NativeRuntime.setWorkerThreads(workerThreads); + BufferAllocator allocator = ArrowAllocation.rootAllocator(); + Session session = Session.create(); + VortexWriter vortexWriter = + VortexWriter.create(session, uri, arrowSchema, properties, allocator); + return new VortexFileAppender<>( vortexWriter, valueWriter, arrowSchema, + allocator, VortexFileAppender.DEFAULT_BATCH_SIZE, rawOutputFile, writeSchema, @@ -235,17 +253,18 @@ private FileAppender buildAppender(org.apache.iceberg.Schema writeSchema) private static class ReadBuilderWrapper implements ReadBuilder { private final InputFile inputFile; - private final BaseFormatModel.ReaderFunction readerFunction; + private final BaseFormatModel.ReaderFunction readerFunction; private final boolean isBatchReader; private org.apache.iceberg.Schema schema; private S engineSchema; private Map idToConstant; private Optional filterPredicate = Optional.empty(); private long[] rowRange; + private int workerThreads = TableProperties.VORTEX_WORKER_THREADS_DEFAULT; private ReadBuilderWrapper( InputFile inputFile, - BaseFormatModel.ReaderFunction readerFunction, + BaseFormatModel.ReaderFunction readerFunction, boolean isBatchReader) { this.inputFile = inputFile; this.readerFunction = readerFunction; @@ -283,6 +302,9 @@ public ReadBuilder filter(Expression filter) { @Override public ReadBuilder set(String key, String value) { + if (TableProperties.READ_VORTEX_WORKER_THREADS.equals(key)) { + workerThreads = Integer.parseInt(value); + } return this; } @@ -315,8 +337,8 @@ public ReadBuilder withNameMapping(NameMapping nameMapping) { @Override @SuppressWarnings("unchecked") public CloseableIterable build() { - Function> readerFunc = null; - Function> batchReaderFunc = null; + Function> readerFunc = null; + Function> batchReaderFunc = null; if (isBatchReader) { batchReaderFunc = @@ -340,7 +362,13 @@ public CloseableIterable build() { } return new VortexIterable<>( - inputFile, readSchema, filterPredicate, rowRange, readerFunc, batchReaderFunc); + inputFile, + readSchema, + filterPredicate, + rowRange, + readerFunc, + batchReaderFunc, + workerThreads); } } } diff --git a/vortex/src/main/java/org/apache/iceberg/vortex/VortexIterable.java b/vortex/src/main/java/org/apache/iceberg/vortex/VortexIterable.java index 2e2d2fa76c36..16c73ededd41 100644 --- a/vortex/src/main/java/org/apache/iceberg/vortex/VortexIterable.java +++ b/vortex/src/main/java/org/apache/iceberg/vortex/VortexIterable.java @@ -18,18 +18,23 @@ */ package org.apache.iceberg.vortex; -import dev.vortex.api.Array; -import dev.vortex.api.ArrayIterator; -import dev.vortex.api.DType; -import dev.vortex.api.File; -import dev.vortex.api.Files; +import dev.vortex.api.DataSource; +import dev.vortex.api.ImmutableScanOptions; +import dev.vortex.api.Partition; +import dev.vortex.api.Scan; import dev.vortex.api.ScanOptions; +import dev.vortex.api.Session; +import dev.vortex.arrow.ArrowAllocation; +import dev.vortex.jni.NativeRuntime; import java.io.IOException; -import java.net.URI; import java.util.List; import java.util.Map; +import java.util.NoSuchElementException; import java.util.Optional; import java.util.function.Function; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.ipc.ArrowReader; import org.apache.iceberg.Schema; import org.apache.iceberg.expressions.Expression; import org.apache.iceberg.io.CloseableGroup; @@ -48,137 +53,214 @@ public class VortexIterable extends CloseableGroup implements CloseableIterab private final InputFile inputFile; private final Optional filterPredicate; private final long[] rowRange; - private final Function> rowReaderFunc; - private final Function> batchReaderFunction; + private final Function> + rowReaderFunc; + private final Function> + batchReaderFunction; private final List projection; + private final int workerThreads; VortexIterable( InputFile inputFile, Schema icebergSchema, Optional filterPredicate, long[] rowRange, - Function> readerFunction, - Function> batchReaderFunction) { + Function> readerFunction, + Function> batchReaderFunction, + int workerThreads) { this.inputFile = inputFile; - // We have the file schema, we need to assign Iceberg IDs to the entire file schema this.projection = Lists.transform(icebergSchema.columns(), Types.NestedField::name); this.filterPredicate = filterPredicate; this.rowRange = rowRange; this.rowReaderFunc = readerFunction; this.batchReaderFunction = batchReaderFunction; + this.workerThreads = workerThreads; } @Override public CloseableIterator iterator() { - File vortexFile = newVortexFile(inputFile); + LOG.debug("opening Vortex file: {}", inputFile); + + // Apply the worker-thread setting on this executor JVM before any Vortex native work + // begins for this read. + NativeRuntime.setWorkerThreads(workerThreads); + + Session session = Session.create(); + String uri = VortexFileUtil.resolveUri(inputFile.location()); + Map properties = VortexFileUtil.resolveInputProperties(inputFile); + DataSource dataSource = DataSource.open(session, uri, properties); - // Return the filtered scan, and then the projection, etc. - Optional scanPredicate = + BufferAllocator allocator = ArrowAllocation.rootAllocator(); + org.apache.arrow.vector.types.pojo.Schema fileArrowSchema = dataSource.arrowSchema(allocator); + + Optional scanFilter = filterPredicate.map( icebergExpression -> { - Schema fileSchema = VortexSchemas.convert(vortexFile.getDType()); - return ConvertFilterToVortex.convert(fileSchema, icebergExpression); + Schema icebergFileSchema = VortexSchemas.convert(fileArrowSchema); + return ConvertFilterToVortex.convert(icebergFileSchema, icebergExpression); }); - Optional optRange = Optional.ofNullable(this.rowRange); - - ArrayIterator batchStream = - vortexFile.newScan( - ScanOptions.builder() - .addAllColumns(projection) - .predicate(scanPredicate) - .rowRange(optRange) - .build()); - Preconditions.checkNotNull(batchStream, "batchStream"); - - DType dtype = batchStream.getDataType(); - CloseableIterator wrappedIterator = - new CloseableIterator() { - @Override - public void close() { - batchStream.close(); - vortexFile.close(); - } + String[] projectionNames = projection.toArray(new String[0]); + dev.vortex.api.Expression scanProjection = + dev.vortex.api.Expression.select(projectionNames, dev.vortex.api.Expression.root()); - @Override - public boolean hasNext() { - return batchStream.hasNext(); - } + ImmutableScanOptions.Builder optionsBuilder = ScanOptions.builder().projection(scanProjection); + scanFilter.ifPresent(optionsBuilder::filter); + if (rowRange != null) { + optionsBuilder.rowRangeBegin(rowRange[0]).rowRangeEnd(rowRange[1]); + } - @Override - public Array next() { - return batchStream.next(); - } - }; + Scan scan = dataSource.scan(optionsBuilder.build()); + Preconditions.checkNotNull(scan, "scan"); + + PartitionBatchIterator batchIterator = new PartitionBatchIterator(scan, allocator); if (rowReaderFunc != null) { - VortexRowReader rowFunction = rowReaderFunc.apply(dtype); - return new VortexRowIterator<>(wrappedIterator, rowFunction); + VortexRowReader rowFunction = rowReaderFunc.apply(fileArrowSchema); + return new VortexRowIterator<>(batchIterator, rowFunction); } else { - VortexBatchReader batchTransform = batchReaderFunction.apply(dtype); - return CloseableIterator.transform(wrappedIterator, batchTransform::read); + VortexBatchReader batchTransform = batchReaderFunction.apply(fileArrowSchema); + return new VortexBatchIterator<>(batchIterator, batchTransform); } } - private static File newVortexFile(InputFile inputFile) { - LOG.debug("opening Vortex file: {}", inputFile); + /** Iterator that pulls Arrow {@link VectorSchemaRoot} batches across Vortex partitions. */ + static class PartitionBatchIterator implements CloseableIterator { + private final Scan scan; + private final BufferAllocator allocator; + private ArrowReader currentReader; + private VectorSchemaRoot currentRoot; + private boolean hasPending = false; + private boolean exhausted = false; - URI uriLocation = URI.create(VortexFileUtil.resolveUri(inputFile.location())); - Map properties = VortexFileUtil.resolveInputProperties(inputFile); - return Files.open(uriLocation, properties); + PartitionBatchIterator(Scan scan, BufferAllocator allocator) { + this.scan = scan; + this.allocator = allocator; + } + + @Override + public boolean hasNext() { + if (hasPending) { + return true; + } + if (exhausted) { + return false; + } + try { + while (true) { + if (currentReader == null) { + if (!scan.hasNext()) { + exhausted = true; + return false; + } + Partition partition = scan.next(); + currentReader = partition.scanArrow(allocator); + currentRoot = currentReader.getVectorSchemaRoot(); + } + if (currentReader.loadNextBatch()) { + hasPending = true; + return true; + } else { + currentReader.close(); + currentReader = null; + currentRoot = null; + } + } + } catch (IOException e) { + throw new org.apache.iceberg.exceptions.RuntimeIOException( + e, "Failed to load next Vortex batch"); + } + } + + @Override + public VectorSchemaRoot next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + hasPending = false; + return currentRoot; + } + + @Override + public void close() throws IOException { + if (currentReader != null) { + currentReader.close(); + currentReader = null; + } + // Don't close `allocator`: it is Vortex's shared root allocator. + } } - static class VortexRowIterator extends CloseableGroup implements CloseableIterator { - private final CloseableIterator stream; + static class VortexRowIterator implements CloseableIterator { + private final PartitionBatchIterator batches; private final VortexRowReader rowReader; - private Array currentBatch = null; + private VectorSchemaRoot currentBatch = null; private int batchIndex = 0; private int batchLen = 0; - VortexRowIterator(CloseableIterator stream, VortexRowReader rowReader) { - this.stream = stream; - addCloseable(stream); + VortexRowIterator(PartitionBatchIterator batches, VortexRowReader rowReader) { + this.batches = batches; this.rowReader = rowReader; - if (stream.hasNext()) { - currentBatch = stream.next(); - batchLen = (int) currentBatch.getLen(); - } } @Override public void close() throws IOException { - // Do not close the ArrayStream, it is closed by the parent. - currentBatch.close(); - currentBatch = null; + batches.close(); } @Override public boolean hasNext() { - // See if we need to fill a new batch first. - if (currentBatch == null || batchIndex == batchLen) { - advance(); + if (currentBatch != null && batchIndex < batchLen) { + return true; } - - return currentBatch != null; + if (batches.hasNext()) { + currentBatch = batches.next(); + batchIndex = 0; + batchLen = currentBatch.getRowCount(); + if (batchLen > 0) { + return true; + } + return hasNext(); + } + currentBatch = null; + batchLen = 0; + return false; } @Override public T next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } T nextRow = rowReader.read(currentBatch, batchIndex); batchIndex++; return nextRow; } + } - private void advance() { - if (stream.hasNext()) { - currentBatch = stream.next(); - batchIndex = 0; - batchLen = (int) currentBatch.getLen(); - } else { - currentBatch = null; - batchLen = 0; - } + static class VortexBatchIterator implements CloseableIterator { + private final PartitionBatchIterator batches; + private final VortexBatchReader batchReader; + + VortexBatchIterator(PartitionBatchIterator batches, VortexBatchReader batchReader) { + this.batches = batches; + this.batchReader = batchReader; + } + + @Override + public boolean hasNext() { + return batches.hasNext(); + } + + @Override + public T next() { + return batchReader.read(batches.next()); + } + + @Override + public void close() throws IOException { + batches.close(); } } } diff --git a/vortex/src/main/java/org/apache/iceberg/vortex/VortexRowReader.java b/vortex/src/main/java/org/apache/iceberg/vortex/VortexRowReader.java index a7d362a5016d..9f875b62ee44 100644 --- a/vortex/src/main/java/org/apache/iceberg/vortex/VortexRowReader.java +++ b/vortex/src/main/java/org/apache/iceberg/vortex/VortexRowReader.java @@ -18,8 +18,9 @@ */ package org.apache.iceberg.vortex; -import dev.vortex.api.Array; +import org.apache.arrow.vector.VectorSchemaRoot; +/** Reads a single row of type {@link T} from an Arrow {@link VectorSchemaRoot} batch. */ public interface VortexRowReader { - T read(Array batch, int row); + T read(VectorSchemaRoot batch, int row); } diff --git a/vortex/src/main/java/org/apache/iceberg/vortex/VortexSchemaWithTypeVisitor.java b/vortex/src/main/java/org/apache/iceberg/vortex/VortexSchemaWithTypeVisitor.java index 1ad4dfe4edb8..8bd72d55896e 100644 --- a/vortex/src/main/java/org/apache/iceberg/vortex/VortexSchemaWithTypeVisitor.java +++ b/vortex/src/main/java/org/apache/iceberg/vortex/VortexSchemaWithTypeVisitor.java @@ -18,54 +18,56 @@ */ package org.apache.iceberg.vortex; -import dev.vortex.api.DType; import java.util.List; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; import org.apache.iceberg.Schema; import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; +/** + * Walks a file's Arrow {@link Field} schema in parallel with the expected Iceberg {@link Type} so + * that visitors can build readers that bind a target Iceberg shape to the file's columns. + */ public abstract class VortexSchemaWithTypeVisitor { - public abstract T struct( - Types.StructType iStruct, List types, List names, List fields); + public abstract T struct(Types.StructType iStruct, List fields, List children); - public abstract T list(Types.ListType iList, DType array, T element); + public abstract T list(Types.ListType iList, Field listField, T element); - public abstract T primitive(Type.PrimitiveType iPrimitive, DType primitive); + public abstract T primitive(Type.PrimitiveType iPrimitive, Field primField); - // What is the point of this?? public static T visit( - Schema expectedSchema, DType readVortexSchema, VortexSchemaWithTypeVisitor visitor) { - return visit(expectedSchema.asStruct(), readVortexSchema, visitor); + Schema expectedSchema, + org.apache.arrow.vector.types.pojo.Schema fileSchema, + VortexSchemaWithTypeVisitor visitor) { + return visitStruct(expectedSchema.asStruct(), fileSchema.getFields(), visitor); } - public static T visit(Type iType, DType schema, VortexSchemaWithTypeVisitor visitor) { - return switch (schema.getVariant()) { - case STRUCT -> visitStruct(iType != null ? iType.asStructType() : null, schema, visitor); - case LIST -> { - Types.ListType list = iType != null ? iType.asListType() : null; - yield visitor.list( - list, - schema, - visit(list != null ? list.elementType() : null, schema.getElementType(), visitor)); - } - default -> visitor.primitive(iType != null ? iType.asPrimitiveType() : null, schema); - }; + public static T visit(Type iType, Field field, VortexSchemaWithTypeVisitor visitor) { + ArrowType arrowType = field.getType(); + if (arrowType instanceof ArrowType.Struct) { + return visitStruct(iType != null ? iType.asStructType() : null, field.getChildren(), visitor); + } else if (arrowType instanceof ArrowType.List + || arrowType instanceof ArrowType.LargeList + || arrowType instanceof ArrowType.FixedSizeList) { + Types.ListType list = iType != null ? iType.asListType() : null; + Field element = field.getChildren().get(0); + return visitor.list( + list, field, visit(list != null ? list.elementType() : null, element, visitor)); + } else { + return visitor.primitive(iType != null ? iType.asPrimitiveType() : null, field); + } } private static T visitStruct( - Types.StructType struct, DType record, VortexSchemaWithTypeVisitor visitor) { - List fields = record.getFieldTypes(); - List names = record.getFieldNames(); - + Types.StructType struct, List fields, VortexSchemaWithTypeVisitor visitor) { List results = Lists.newArrayListWithExpectedSize(fields.size()); - // TODO(aduffy): metadata in Vortex schemas to allow embedding the Iceberg field ID number? - // For now we just use the field index, which might not be right when we have projections... for (int fieldId = 0; fieldId < fields.size(); fieldId++) { - DType field = fields.get(fieldId); + Field field = fields.get(fieldId); Types.NestedField iField = struct != null ? struct.field(fieldId) : null; results.add(visit(iField != null ? iField.type() : null, field, visitor)); } - return visitor.struct(struct, fields, names, results); + return visitor.struct(struct, fields, results); } } diff --git a/vortex/src/main/java/org/apache/iceberg/vortex/VortexSchemas.java b/vortex/src/main/java/org/apache/iceberg/vortex/VortexSchemas.java index d22a2e23855e..223e413ab9ae 100644 --- a/vortex/src/main/java/org/apache/iceberg/vortex/VortexSchemas.java +++ b/vortex/src/main/java/org/apache/iceberg/vortex/VortexSchemas.java @@ -21,9 +21,8 @@ import static org.apache.iceberg.types.Types.NestedField.optional; import static org.apache.iceberg.types.Types.NestedField.required; -import dev.vortex.api.DType; import java.util.List; -import java.util.Optional; +import java.util.Map; import org.apache.arrow.vector.types.DateUnit; import org.apache.arrow.vector.types.FloatingPointPrecision; import org.apache.arrow.vector.types.TimeUnit; @@ -31,109 +30,36 @@ import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.iceberg.Schema; -import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; public final class VortexSchemas { - private VortexSchemas() {} - - /** - * Given a projection schema, and a Vortex DType, return a resolved Iceberg schema that represents - * how to read the Vortex data as Iceberg rows. - */ - public static Schema convert(DType fileSchema) { - Preconditions.checkArgument( - fileSchema.getVariant() == DType.Variant.STRUCT, - "only Vortex STRUCT types can be converted to Iceberg Schema, received: " + fileSchema); - List fieldNames = fileSchema.getFieldNames(); - List fieldTypes = fileSchema.getFieldTypes(); + /** Canonical Arrow extension name for UUIDs (matches {@code arrow.vector.extension.UuidType}). */ + static final String UUID_EXTENSION_NAME = "arrow.uuid"; - List targetSchema = Lists.newArrayList(); + private VortexSchemas() {} - for (int fieldId = 0; fieldId < fieldNames.size(); fieldId++) { - String fieldName = fieldNames.get(fieldId); - DType fieldType = fieldTypes.get(fieldId); - Type icebergType = toIcebergType(fieldType); - if (fieldType.isNullable()) { - targetSchema.add(optional(fieldId, fieldName, icebergType)); + /** Convert a Vortex file's Arrow {@link org.apache.arrow.vector.types.pojo.Schema} to Iceberg. */ + public static Schema convert(org.apache.arrow.vector.types.pojo.Schema arrowSchema) { + List fields = arrowSchema.getFields(); + List columns = Lists.newArrayListWithExpectedSize(fields.size()); + for (int fieldId = 0; fieldId < fields.size(); fieldId++) { + Field field = fields.get(fieldId); + Type icebergType = toIcebergType(field); + if (field.isNullable()) { + columns.add(optional(fieldId, field.getName(), icebergType)); } else { - targetSchema.add(required(fieldId, fieldName, icebergType)); + columns.add(required(fieldId, field.getName(), icebergType)); } } - return new Schema(targetSchema); - } - - /** Convert an Iceberg Schema to a Vortex DType suitable for {@code VortexWriter.create}. */ - public static DType toDType(Schema icebergSchema) { - List columns = icebergSchema.columns(); - String[] names = new String[columns.size()]; - DType[] types = new DType[columns.size()]; - for (int i = 0; i < columns.size(); i++) { - Types.NestedField field = columns.get(i); - names[i] = field.name(); - types[i] = toVortexDType(field.type(), field.isOptional()); - } - - return DType.newStruct(names, types, false); + return new Schema(columns); } - private static DType toVortexDType(Type type, boolean nullable) { - return switch (type.typeId()) { - case BOOLEAN -> DType.newBool(nullable); - case INTEGER -> DType.newInt(nullable); - case LONG -> DType.newLong(nullable); - case FLOAT -> DType.newFloat(nullable); - case DOUBLE -> DType.newDouble(nullable); - case STRING -> DType.newUtf8(nullable); - case BINARY, FIXED -> DType.newBinary(nullable); - case DECIMAL -> { - Types.DecimalType decimal = (Types.DecimalType) type; - yield DType.newDecimal(decimal.precision(), decimal.scale(), nullable); - } - case DATE -> DType.newDate(DType.TimeUnit.DAYS, nullable); - case TIME -> DType.newTime(DType.TimeUnit.MICROSECONDS, nullable); - case TIMESTAMP -> { - Types.TimestampType ts = (Types.TimestampType) type; - yield DType.newTimestamp( - DType.TimeUnit.MICROSECONDS, - ts.shouldAdjustToUTC() ? Optional.of("UTC") : Optional.empty(), - nullable); - } - case TIMESTAMP_NANO -> { - Types.TimestampNanoType tsNano = (Types.TimestampNanoType) type; - yield DType.newTimestamp( - DType.TimeUnit.NANOSECONDS, - tsNano.shouldAdjustToUTC() ? Optional.of("UTC") : Optional.empty(), - nullable); - } - case LIST -> { - Types.ListType listType = (Types.ListType) type; - DType elementDType = toVortexDType(listType.elementType(), listType.isElementOptional()); - yield DType.newList(elementDType, nullable); - } - case STRUCT -> { - Types.StructType structType = (Types.StructType) type; - List fields = structType.fields(); - String[] fieldNames = new String[fields.size()]; - DType[] fieldTypes = new DType[fields.size()]; - for (int i = 0; i < fields.size(); i++) { - fieldNames[i] = fields.get(i).name(); - fieldTypes[i] = toVortexDType(fields.get(i).type(), fields.get(i).isOptional()); - } - - yield DType.newStruct(fieldNames, fieldTypes, nullable); - } - default -> - throw new UnsupportedOperationException( - "Unsupported Iceberg type for Vortex write: " + type); - }; - } - - /** Convert an Iceberg Schema to an Arrow Schema for writing via Arrow IPC. */ + /** Convert an Iceberg Schema to an Arrow Schema suitable for {@code VortexWriter.create}. */ public static org.apache.arrow.vector.types.pojo.Schema toArrowSchema(Schema icebergSchema) { ImmutableList.Builder fields = ImmutableList.builder(); for (Types.NestedField column : icebergSchema.columns()) { @@ -183,6 +109,18 @@ yield new Field( null), null); } + case UUID -> { + Map extMetadata = + ImmutableMap.of( + ArrowType.ExtensionType.EXTENSION_METADATA_KEY_NAME, + UUID_EXTENSION_NAME, + ArrowType.ExtensionType.EXTENSION_METADATA_KEY_METADATA, + ""); + yield new Field( + name, + new FieldType(nullable, new ArrowType.FixedSizeBinary(16), null, extMetadata), + null); + } case DATE -> new Field(name, new FieldType(nullable, new ArrowType.Date(DateUnit.DAY), null), null); case TIME -> @@ -237,63 +175,77 @@ yield new Field( }; } - private static Type toIcebergType(DType dataType) { - switch (dataType.getVariant()) { - case NULL: - return Types.UnknownType.get(); - case BOOL: - return Types.BooleanType.get(); - case PRIMITIVE_U8: - case PRIMITIVE_U16: - case PRIMITIVE_U32: - case PRIMITIVE_I8: - case PRIMITIVE_I16: - case PRIMITIVE_I32: + private static Type toIcebergType(Field field) { + ArrowType arrowType = field.getType(); + // UUID is conveyed as the {@code arrow.uuid} extension over FixedSizeBinary(16). Check the + // metadata directly so this works whether or not the extension is registered with + // ExtensionTypeRegistry (i.e. whether arrowType deserialized to ExtensionType or stayed as + // FixedSizeBinary). + if (isUuidField(field)) { + return Types.UUIDType.get(); + } + if (arrowType instanceof ArrowType.Null) { + return Types.UnknownType.get(); + } else if (arrowType instanceof ArrowType.Bool) { + return Types.BooleanType.get(); + } else if (arrowType instanceof ArrowType.Int intType) { + if (intType.getBitWidth() <= Integer.SIZE) { return Types.IntegerType.get(); - case PRIMITIVE_U64: - case PRIMITIVE_I64: + } else { return Types.LongType.get(); - case PRIMITIVE_F32: - return Types.FloatType.get(); - case PRIMITIVE_F64: - return Types.DoubleType.get(); - case DECIMAL: - return Types.DecimalType.of(dataType.getPrecision(), dataType.getScale()); - case UTF8: - return Types.StringType.get(); - case BINARY: - return Types.BinaryType.get(); - case LIST: - { - DType elementType = dataType.getElementType(); - Type innerType = toIcebergType(elementType); - if (elementType.isNullable()) { - return Types.ListType.ofOptional(0, innerType); - } else { - return Types.ListType.ofRequired(0, innerType); - } - } - case EXTENSION: - { - if (dataType.isDate()) { - return Types.DateType.get(); - } else if (dataType.isTime()) { - return Types.TimeType.get(); - } else if (dataType.isTimestamp()) { - if (dataType.getTimeZone().isPresent()) { - return Types.TimestampType.withZone(); - } else { - return Types.TimestampType.withoutZone(); - } - } else { - throw new UnsupportedOperationException( - "Unsupported Vortex extension type: " + dataType); - } - } - // TODO(aduffy): add nested struct support - default: - throw new UnsupportedOperationException( - "Unsupported data type in Vortex -> Iceberg conversion: " + dataType.getVariant()); + } + } else if (arrowType instanceof ArrowType.FloatingPoint fpType) { + return switch (fpType.getPrecision()) { + case SINGLE -> Types.FloatType.get(); + case DOUBLE -> Types.DoubleType.get(); + case HALF -> + throw new UnsupportedOperationException("Half-precision floats are not supported"); + }; + } else if (arrowType instanceof ArrowType.Decimal decType) { + return Types.DecimalType.of(decType.getPrecision(), decType.getScale()); + } else if (arrowType instanceof ArrowType.Utf8 || arrowType instanceof ArrowType.LargeUtf8) { + return Types.StringType.get(); + } else if (arrowType instanceof ArrowType.Binary + || arrowType instanceof ArrowType.LargeBinary) { + return Types.BinaryType.get(); + } else if (arrowType instanceof ArrowType.FixedSizeBinary fixed) { + return Types.FixedType.ofLength(fixed.getByteWidth()); + } else if (arrowType instanceof ArrowType.Date) { + return Types.DateType.get(); + } else if (arrowType instanceof ArrowType.Time) { + return Types.TimeType.get(); + } else if (arrowType instanceof ArrowType.Timestamp tsType) { + if (tsType.getTimezone() == null) { + return tsType.getUnit() == TimeUnit.NANOSECOND + ? Types.TimestampNanoType.withoutZone() + : Types.TimestampType.withoutZone(); + } else { + return tsType.getUnit() == TimeUnit.NANOSECOND + ? Types.TimestampNanoType.withZone() + : Types.TimestampType.withZone(); + } + } else if (arrowType instanceof ArrowType.List) { + Field elementField = field.getChildren().get(0); + Type innerType = toIcebergType(elementField); + if (elementField.isNullable()) { + return Types.ListType.ofOptional(0, innerType); + } else { + return Types.ListType.ofRequired(0, innerType); + } + } else { + throw new UnsupportedOperationException("Unsupported Arrow type: " + arrowType); + } + } + + /** + * True when {@code field} carries the {@code arrow.uuid} extension marker. Checking the field + * metadata works whether or not {@link ArrowType.ExtensionType} was deserialized by the registry. + */ + public static boolean isUuidField(Field field) { + if (field.getType() instanceof ArrowType.ExtensionType ext) { + return UUID_EXTENSION_NAME.equals(ext.extensionName()); } + return UUID_EXTENSION_NAME.equals( + field.getMetadata().get(ArrowType.ExtensionType.EXTENSION_METADATA_KEY_NAME)); } } diff --git a/vortex/src/main/java/org/apache/iceberg/vortex/VortexValueReader.java b/vortex/src/main/java/org/apache/iceberg/vortex/VortexValueReader.java index da2c9198ff34..610008bc13c6 100644 --- a/vortex/src/main/java/org/apache/iceberg/vortex/VortexValueReader.java +++ b/vortex/src/main/java/org/apache/iceberg/vortex/VortexValueReader.java @@ -18,17 +18,17 @@ */ package org.apache.iceberg.vortex; -import dev.vortex.api.Array; +import org.apache.arrow.vector.FieldVector; -/** Reads a {@link D} from a Vortex array. */ +/** Reads a {@link D} from an Arrow {@link FieldVector} at a row. */ public interface VortexValueReader { - default D read(Array array, int row) { - if (array.getNull(row)) { + default D read(FieldVector vector, int row) { + if (vector.isNull(row)) { return null; } else { - return readNonNull(array, row); + return readNonNull(vector, row); } } - D readNonNull(Array array, int row); + D readNonNull(FieldVector vector, int row); } diff --git a/vortex/src/test/java/org/apache/iceberg/vortex/TestConvertFilterToVortex.java b/vortex/src/test/java/org/apache/iceberg/vortex/TestConvertFilterToVortex.java index f7905cbfe364..e17a2338a4e4 100644 --- a/vortex/src/test/java/org/apache/iceberg/vortex/TestConvertFilterToVortex.java +++ b/vortex/src/test/java/org/apache/iceberg/vortex/TestConvertFilterToVortex.java @@ -23,16 +23,16 @@ import static org.assertj.core.api.Assertions.assertThat; import dev.vortex.api.Expression; -import dev.vortex.api.expressions.Binary; -import dev.vortex.api.expressions.GetItem; -import dev.vortex.api.expressions.Literal; -import dev.vortex.api.expressions.Not; -import dev.vortex.api.expressions.Root; import org.apache.iceberg.Schema; import org.apache.iceberg.expressions.Expressions; import org.apache.iceberg.types.Types; import org.junit.jupiter.api.Test; +/** + * Smoke tests for filter conversion. The Vortex {@link Expression} API in 0.71+ wraps an opaque + * native handle so we cannot compare expressions structurally; tests instead check that supported + * shapes return a non-sentinel expression and unsupported shapes fall back to ALWAYS_TRUE. + */ public class TestConvertFilterToVortex { private static final Schema SCHEMA = new Schema( @@ -43,45 +43,34 @@ public class TestConvertFilterToVortex { @Test public void testIn() { Expression result = ConvertFilterToVortex.convert(SCHEMA, Expressions.in("id", 1L, 2L, 3L)); - GetItem field = GetItem.of(Root.INSTANCE, "id"); - Expression expected = - Binary.or( - Binary.eq(field, Literal.int64(1L)), - Binary.eq(field, Literal.int64(2L)), - Binary.eq(field, Literal.int64(3L))); - assertThat(result).isEqualTo(expected); + assertThat(result).isNotNull(); + assertThat(result).isNotSameAs(ConvertFilterToVortex.ALWAYS_TRUE); + assertThat(result).isNotSameAs(ConvertFilterToVortex.ALWAYS_FALSE); } @Test public void testNotIn() { Expression result = ConvertFilterToVortex.convert(SCHEMA, Expressions.notIn("id", 1L, 2L, 3L)); - GetItem field = GetItem.of(Root.INSTANCE, "id"); - Expression expected = - Not.of( - Binary.or( - Binary.eq(field, Literal.int64(1L)), - Binary.eq(field, Literal.int64(2L)), - Binary.eq(field, Literal.int64(3L)))); - assertThat(result).isEqualTo(expected); + assertThat(result).isNotNull(); + assertThat(result).isNotSameAs(ConvertFilterToVortex.ALWAYS_TRUE); + assertThat(result).isNotSameAs(ConvertFilterToVortex.ALWAYS_FALSE); } @Test public void testInSingleValue() { // A single-value IN is optimized by Iceberg to EQ during binding Expression result = ConvertFilterToVortex.convert(SCHEMA, Expressions.in("id", 42L)); - GetItem field = GetItem.of(Root.INSTANCE, "id"); - Expression expected = Binary.eq(field, Literal.int64(42L)); - assertThat(result).isEqualTo(expected); + assertThat(result).isNotNull(); + assertThat(result).isNotSameAs(ConvertFilterToVortex.ALWAYS_TRUE); + assertThat(result).isNotSameAs(ConvertFilterToVortex.ALWAYS_FALSE); } @Test public void testInStrings() { Expression result = ConvertFilterToVortex.convert(SCHEMA, Expressions.in("name", "Alice", "Bob")); - GetItem field = GetItem.of(Root.INSTANCE, "name"); - Expression expected = - Binary.or( - Binary.eq(field, Literal.string("Alice")), Binary.eq(field, Literal.string("Bob"))); - assertThat(result).isEqualTo(expected); + assertThat(result).isNotNull(); + assertThat(result).isNotSameAs(ConvertFilterToVortex.ALWAYS_TRUE); + assertThat(result).isNotSameAs(ConvertFilterToVortex.ALWAYS_FALSE); } } diff --git a/vortex/src/test/java/org/apache/iceberg/vortex/TestVortexUuid.java b/vortex/src/test/java/org/apache/iceberg/vortex/TestVortexUuid.java new file mode 100644 index 000000000000..51da5b2b0fb3 --- /dev/null +++ b/vortex/src/test/java/org/apache/iceberg/vortex/TestVortexUuid.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.vortex; + +import static org.apache.iceberg.types.Types.NestedField.required; +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.UUID; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.iceberg.Schema; +import org.apache.iceberg.data.GenericRecord; +import org.apache.iceberg.data.Record; +import org.apache.iceberg.data.vortex.GenericVortexReader; +import org.apache.iceberg.data.vortex.GenericVortexReaders; +import org.apache.iceberg.data.vortex.GenericVortexWriter; +import org.apache.iceberg.types.Types; +import org.junit.jupiter.api.Test; + +public class TestVortexUuid { + private static final Schema SCHEMA = + new Schema(required(1, "id", Types.LongType.get()), required(2, "uid", Types.UUIDType.get())); + + @Test + public void testIcebergUuidMapsToArrowUuidExtension() { + org.apache.arrow.vector.types.pojo.Schema arrowSchema = VortexSchemas.toArrowSchema(SCHEMA); + Field uuidField = arrowSchema.findField("uid"); + assertThat(VortexSchemas.isUuidField(uuidField)).isTrue(); + } + + @Test + public void testArrowUuidExtensionMapsBackToIcebergUuid() { + Schema roundTrip = VortexSchemas.convert(VortexSchemas.toArrowSchema(SCHEMA)); + assertThat(roundTrip.findType("uid")).isEqualTo(Types.UUIDType.get()); + } + + @Test + public void testWriteAndReadUuidRoundTrip() { + UUID uuid1 = UUID.fromString("00000000-0000-0000-0000-000000000001"); + UUID uuid2 = UUID.fromString("11111111-2222-3333-4444-555555555555"); + + org.apache.arrow.vector.types.pojo.Schema arrowSchema = VortexSchemas.toArrowSchema(SCHEMA); + + try (BufferAllocator allocator = new RootAllocator(); + VectorSchemaRoot root = VectorSchemaRoot.create(arrowSchema, allocator)) { + root.allocateNew(); + GenericVortexWriter writer = (GenericVortexWriter) GenericVortexWriter.buildWriter(SCHEMA); + writer.write(makeRecord(1L, uuid1), root, 0); + writer.write(makeRecord(2L, uuid2), root, 1); + root.setRowCount(2); + + // Read back using the GenericVortexReader's primitive dispatch through GenericVortexReaders. + VortexRowReader reader = GenericVortexReader.buildReader(SCHEMA, arrowSchema); + Record row0 = reader.read(root, 0); + Record row1 = reader.read(root, 1); + + assertThat(row0.get(0)).isEqualTo(1L); + assertThat(row0.get(1)).isEqualTo(uuid1); + assertThat(row1.get(0)).isEqualTo(2L); + assertThat(row1.get(1)).isEqualTo(uuid2); + } + } + + @Test + public void testGenericUuidReaderHandlesPlainFixedSizeBinary() { + // When the arrow.uuid extension type is not registered, Vortex returns FixedSizeBinary(16). + // The reader should still produce a UUID by reading the underlying bytes. + UUID uuid = UUID.fromString("deadbeef-1234-5678-9abc-0123456789ab"); + org.apache.arrow.vector.types.pojo.Schema fixedSchema = + new org.apache.arrow.vector.types.pojo.Schema( + java.util.List.of( + new Field( + "uid", + new org.apache.arrow.vector.types.pojo.FieldType( + false, + new org.apache.arrow.vector.types.pojo.ArrowType.FixedSizeBinary(16), + null), + null))); + + try (BufferAllocator allocator = new RootAllocator(); + VectorSchemaRoot root = VectorSchemaRoot.create(fixedSchema, allocator)) { + root.allocateNew(); + ((org.apache.arrow.vector.FixedSizeBinaryVector) root.getVector(0)) + .setSafe(0, org.apache.iceberg.util.UUIDUtil.convert(uuid)); + root.setRowCount(1); + + assertThat(GenericVortexReaders.uuids().read(root.getVector(0), 0)).isEqualTo(uuid); + } + } + + private static Record makeRecord(long id, UUID uuid) { + GenericRecord record = GenericRecord.create(SCHEMA); + record.set(0, id); + record.set(1, uuid); + return record; + } +} From 64634e7e32007501e5e985746c98ebc9d95ae554 Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Mon, 25 May 2026 23:16:01 +0100 Subject: [PATCH 2/4] fixes Signed-off-by: Robert Kruszewski --- flink/v1.20/flink-runtime/runtime-deps.txt | 18 +- flink/v2.0/flink-runtime/runtime-deps.txt | 18 +- flink/v2.1/flink-runtime/runtime-deps.txt | 18 +- .../baseline-class-uniqueness.lock | 60 --- .../kafka-connect-runtime/runtime-deps.txt | 18 +- .../baseline-class-uniqueness.lock | 307 +-------------- spark/v3.5/spark-runtime/runtime-deps.txt | 21 +- .../v3.5/spark/baseline-class-uniqueness.lock | 355 +----------------- .../baseline-class-uniqueness.lock | 307 +-------------- spark/v4.0/spark-runtime/runtime-deps.txt | 21 +- .../v4.0/spark/baseline-class-uniqueness.lock | 355 +----------------- .../baseline-class-uniqueness.lock | 307 +-------------- spark/v4.1/spark-runtime/runtime-deps.txt | 21 +- .../v4.1/spark/baseline-class-uniqueness.lock | 355 +----------------- 14 files changed, 114 insertions(+), 2067 deletions(-) delete mode 100644 kafka-connect/kafka-connect-runtime/baseline-class-uniqueness.lock diff --git a/flink/v1.20/flink-runtime/runtime-deps.txt b/flink/v1.20/flink-runtime/runtime-deps.txt index a2a262812da5..75fcd8ec2fa0 100644 --- a/flink/v1.20/flink-runtime/runtime-deps.txt +++ b/flink/v1.20/flink-runtime/runtime-deps.txt @@ -4,23 +4,23 @@ com.fasterxml.jackson.core:jackson-databind:2.21 com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.21 com.github.ben-manes.caffeine:caffeine:2.9 com.github.luben:zstd-jni:1.5 -com.google.errorprone:error_prone_annotations:2.41 +com.google.errorprone:error_prone_annotations:2.47 com.google.flatbuffers:flatbuffers-java:25.2 com.google.guava:failureaccess:1.0 -com.google.guava:guava:33.5 +com.google.guava:guava:33.6 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.j2objc:j2objc-annotations:3.1 dev.failsafe:failsafe:3.3 -dev.vortex:vortex-jni:0.67 +dev.vortex:vortex-jni:0.72 io.airlift:aircompressor:2.0 io.netty:netty-buffer:4.2 io.netty:netty-common:4.2 -org.apache.arrow:arrow-c-data:18.3 -org.apache.arrow:arrow-format:18.3 -org.apache.arrow:arrow-memory-core:18.3 -org.apache.arrow:arrow-memory-netty-buffer-patch:18.3 -org.apache.arrow:arrow-memory-netty:18.3 -org.apache.arrow:arrow-vector:18.3 +org.apache.arrow:arrow-c-data:19.0 +org.apache.arrow:arrow-format:19.0 +org.apache.arrow:arrow-memory-core:19.0 +org.apache.arrow:arrow-memory-netty-buffer-patch:19.0 +org.apache.arrow:arrow-memory-netty:19.0 +org.apache.arrow:arrow-vector:19.0 org.apache.avro:avro:1.12 org.apache.datasketches:datasketches-java:6.2 org.apache.datasketches:datasketches-memory:3.0 diff --git a/flink/v2.0/flink-runtime/runtime-deps.txt b/flink/v2.0/flink-runtime/runtime-deps.txt index a2a262812da5..75fcd8ec2fa0 100644 --- a/flink/v2.0/flink-runtime/runtime-deps.txt +++ b/flink/v2.0/flink-runtime/runtime-deps.txt @@ -4,23 +4,23 @@ com.fasterxml.jackson.core:jackson-databind:2.21 com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.21 com.github.ben-manes.caffeine:caffeine:2.9 com.github.luben:zstd-jni:1.5 -com.google.errorprone:error_prone_annotations:2.41 +com.google.errorprone:error_prone_annotations:2.47 com.google.flatbuffers:flatbuffers-java:25.2 com.google.guava:failureaccess:1.0 -com.google.guava:guava:33.5 +com.google.guava:guava:33.6 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.j2objc:j2objc-annotations:3.1 dev.failsafe:failsafe:3.3 -dev.vortex:vortex-jni:0.67 +dev.vortex:vortex-jni:0.72 io.airlift:aircompressor:2.0 io.netty:netty-buffer:4.2 io.netty:netty-common:4.2 -org.apache.arrow:arrow-c-data:18.3 -org.apache.arrow:arrow-format:18.3 -org.apache.arrow:arrow-memory-core:18.3 -org.apache.arrow:arrow-memory-netty-buffer-patch:18.3 -org.apache.arrow:arrow-memory-netty:18.3 -org.apache.arrow:arrow-vector:18.3 +org.apache.arrow:arrow-c-data:19.0 +org.apache.arrow:arrow-format:19.0 +org.apache.arrow:arrow-memory-core:19.0 +org.apache.arrow:arrow-memory-netty-buffer-patch:19.0 +org.apache.arrow:arrow-memory-netty:19.0 +org.apache.arrow:arrow-vector:19.0 org.apache.avro:avro:1.12 org.apache.datasketches:datasketches-java:6.2 org.apache.datasketches:datasketches-memory:3.0 diff --git a/flink/v2.1/flink-runtime/runtime-deps.txt b/flink/v2.1/flink-runtime/runtime-deps.txt index a2a262812da5..75fcd8ec2fa0 100644 --- a/flink/v2.1/flink-runtime/runtime-deps.txt +++ b/flink/v2.1/flink-runtime/runtime-deps.txt @@ -4,23 +4,23 @@ com.fasterxml.jackson.core:jackson-databind:2.21 com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.21 com.github.ben-manes.caffeine:caffeine:2.9 com.github.luben:zstd-jni:1.5 -com.google.errorprone:error_prone_annotations:2.41 +com.google.errorprone:error_prone_annotations:2.47 com.google.flatbuffers:flatbuffers-java:25.2 com.google.guava:failureaccess:1.0 -com.google.guava:guava:33.5 +com.google.guava:guava:33.6 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.j2objc:j2objc-annotations:3.1 dev.failsafe:failsafe:3.3 -dev.vortex:vortex-jni:0.67 +dev.vortex:vortex-jni:0.72 io.airlift:aircompressor:2.0 io.netty:netty-buffer:4.2 io.netty:netty-common:4.2 -org.apache.arrow:arrow-c-data:18.3 -org.apache.arrow:arrow-format:18.3 -org.apache.arrow:arrow-memory-core:18.3 -org.apache.arrow:arrow-memory-netty-buffer-patch:18.3 -org.apache.arrow:arrow-memory-netty:18.3 -org.apache.arrow:arrow-vector:18.3 +org.apache.arrow:arrow-c-data:19.0 +org.apache.arrow:arrow-format:19.0 +org.apache.arrow:arrow-memory-core:19.0 +org.apache.arrow:arrow-memory-netty-buffer-patch:19.0 +org.apache.arrow:arrow-memory-netty:19.0 +org.apache.arrow:arrow-vector:19.0 org.apache.avro:avro:1.12 org.apache.datasketches:datasketches-java:6.2 org.apache.datasketches:datasketches-memory:3.0 diff --git a/kafka-connect/kafka-connect-runtime/baseline-class-uniqueness.lock b/kafka-connect/kafka-connect-runtime/baseline-class-uniqueness.lock deleted file mode 100644 index 7868296a79e8..000000000000 --- a/kafka-connect/kafka-connect-runtime/baseline-class-uniqueness.lock +++ /dev/null @@ -1,60 +0,0 @@ -# Danger! Multiple jars contain identically named classes. This may cause different behaviour depending on classpath ordering. -# Run ./gradlew checkClassUniqueness --fix to update this file - -## runtimeClasspath -[com.google.protobuf:protobuf-java, dev.vortex:vortex-jni] - - com.google.protobuf.BoolValue - - com.google.protobuf.BoolValue$1 - - com.google.protobuf.BoolValue$Builder - - com.google.protobuf.BoolValueOrBuilder - - com.google.protobuf.BytesValue - - com.google.protobuf.BytesValue$1 - - com.google.protobuf.BytesValue$Builder - - com.google.protobuf.BytesValueOrBuilder - - com.google.protobuf.DoubleValue - - com.google.protobuf.DoubleValue$1 - - com.google.protobuf.DoubleValue$Builder - - com.google.protobuf.DoubleValueOrBuilder - - com.google.protobuf.FloatValue - - com.google.protobuf.FloatValue$1 - - com.google.protobuf.FloatValue$Builder - - com.google.protobuf.FloatValueOrBuilder - - com.google.protobuf.Int32Value - - com.google.protobuf.Int32Value$1 - - com.google.protobuf.Int32Value$Builder - - com.google.protobuf.Int32ValueOrBuilder - - com.google.protobuf.Int64Value - - com.google.protobuf.Int64Value$1 - - com.google.protobuf.Int64Value$Builder - - com.google.protobuf.Int64ValueOrBuilder - - com.google.protobuf.ListValue - - com.google.protobuf.ListValue$1 - - com.google.protobuf.ListValue$Builder - - com.google.protobuf.ListValueOrBuilder - - com.google.protobuf.NullValue - - com.google.protobuf.NullValue$1 - - com.google.protobuf.StringValue - - com.google.protobuf.StringValue$1 - - com.google.protobuf.StringValue$Builder - - com.google.protobuf.StringValueOrBuilder - - com.google.protobuf.Struct - - com.google.protobuf.Struct$1 - - com.google.protobuf.Struct$Builder - - com.google.protobuf.Struct$Builder$FieldsConverter - - com.google.protobuf.Struct$FieldsDefaultEntryHolder - - com.google.protobuf.StructOrBuilder - - com.google.protobuf.StructProto - - com.google.protobuf.UInt32Value - - com.google.protobuf.UInt32Value$1 - - com.google.protobuf.UInt32Value$Builder - - com.google.protobuf.UInt32ValueOrBuilder - - com.google.protobuf.UInt64Value - - com.google.protobuf.UInt64Value$1 - - com.google.protobuf.UInt64Value$Builder - - com.google.protobuf.UInt64ValueOrBuilder - - com.google.protobuf.Value - - com.google.protobuf.Value$1 - - com.google.protobuf.Value$Builder - - com.google.protobuf.Value$KindCase - - com.google.protobuf.ValueOrBuilder - - com.google.protobuf.WrappersProto diff --git a/kafka-connect/kafka-connect-runtime/runtime-deps.txt b/kafka-connect/kafka-connect-runtime/runtime-deps.txt index 949f925c680f..b200a94028d4 100644 --- a/kafka-connect/kafka-connect-runtime/runtime-deps.txt +++ b/kafka-connect/kafka-connect-runtime/runtime-deps.txt @@ -56,7 +56,7 @@ com.google.code.gson:gson:2.13 com.google.errorprone:error_prone_annotations:2.48 com.google.flatbuffers:flatbuffers-java:25.2 com.google.guava:failureaccess:1.0 -com.google.guava:guava:33.5 +com.google.guava:guava:33.6 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.http-client:google-http-client-apache-v2:2.1 com.google.http-client:google-http-client-appengine:2.1 @@ -73,13 +73,13 @@ com.microsoft.azure:msal4j-persistence-extension:1.3 com.microsoft.azure:msal4j:1.23 com.sun.xml.bind:jaxb-impl:2.2 commons-cli:commons-cli:1.9 -commons-codec:commons-codec:1.19 +commons-codec:commons-codec:1.21 commons-io:commons-io:2.20 commons-logging:commons-logging:1.2 commons-net:commons-net:3.9 commons-pool:commons-pool:1.6 dev.failsafe:failsafe:3.3 -dev.vortex:vortex-jni:0.67 +dev.vortex:vortex-jni:0.72 dnsjava:dnsjava:3.6 io.airlift:aircompressor:2.0 io.dropwizard.metrics:metrics-core:3.2 @@ -150,12 +150,12 @@ javax.xml.bind:jaxb-api:2.2 javax.xml.stream:stax-api:1.0-2 net.java.dev.jna:jna-platform:5.17 net.java.dev.jna:jna:5.17 -org.apache.arrow:arrow-c-data:18.3 -org.apache.arrow:arrow-format:18.3 -org.apache.arrow:arrow-memory-core:18.3 -org.apache.arrow:arrow-memory-netty-buffer-patch:18.3 -org.apache.arrow:arrow-memory-netty:18.3 -org.apache.arrow:arrow-vector:18.3 +org.apache.arrow:arrow-c-data:19.0 +org.apache.arrow:arrow-format:19.0 +org.apache.arrow:arrow-memory-core:19.0 +org.apache.arrow:arrow-memory-netty-buffer-patch:19.0 +org.apache.arrow:arrow-memory-netty:19.0 +org.apache.arrow:arrow-vector:19.0 org.apache.avro:avro:1.12 org.apache.commons:commons-collections4:4.4 org.apache.commons:commons-compress:1.28 diff --git a/spark/v3.5/spark-runtime/baseline-class-uniqueness.lock b/spark/v3.5/spark-runtime/baseline-class-uniqueness.lock index 6197975f3900..53a154c69d97 100644 --- a/spark/v3.5/spark-runtime/baseline-class-uniqueness.lock +++ b/spark/v3.5/spark-runtime/baseline-class-uniqueness.lock @@ -4,163 +4,9 @@ ## runtimeClasspath [com.google.guava:guava, dev.vortex:vortex-jni (classifier=all)] - com.google.thirdparty.publicsuffix.PublicSuffixPatterns + - com.google.thirdparty.publicsuffix.PublicSuffixTrie + - com.google.thirdparty.publicsuffix.PublicSuffixTrie$ChunksCharSequence - com.google.thirdparty.publicsuffix.PublicSuffixType - - com.google.thirdparty.publicsuffix.TrieParser -[com.google.protobuf:protobuf-java, dev.vortex:vortex-jni] - - com.google.protobuf.BoolValue - - com.google.protobuf.BoolValue$1 - - com.google.protobuf.BoolValue$Builder - - com.google.protobuf.BoolValueOrBuilder - - com.google.protobuf.BytesValue - - com.google.protobuf.BytesValue$1 - - com.google.protobuf.BytesValue$Builder - - com.google.protobuf.BytesValueOrBuilder - - com.google.protobuf.DoubleValue - - com.google.protobuf.DoubleValue$1 - - com.google.protobuf.DoubleValue$Builder - - com.google.protobuf.DoubleValueOrBuilder - - com.google.protobuf.FloatValue - - com.google.protobuf.FloatValue$1 - - com.google.protobuf.FloatValue$Builder - - com.google.protobuf.FloatValueOrBuilder - - com.google.protobuf.Int32Value - - com.google.protobuf.Int32Value$1 - - com.google.protobuf.Int32Value$Builder - - com.google.protobuf.Int32ValueOrBuilder - - com.google.protobuf.Int64Value - - com.google.protobuf.Int64Value$1 - - com.google.protobuf.Int64Value$Builder - - com.google.protobuf.Int64ValueOrBuilder - - com.google.protobuf.ListValue - - com.google.protobuf.ListValue$1 - - com.google.protobuf.ListValue$Builder - - com.google.protobuf.ListValueOrBuilder - - com.google.protobuf.NullValue - - com.google.protobuf.NullValue$1 - - com.google.protobuf.StringValue - - com.google.protobuf.StringValue$1 - - com.google.protobuf.StringValue$Builder - - com.google.protobuf.StringValueOrBuilder - - com.google.protobuf.Struct - - com.google.protobuf.Struct$1 - - com.google.protobuf.Struct$Builder - - com.google.protobuf.Struct$Builder$FieldsConverter - - com.google.protobuf.Struct$FieldsDefaultEntryHolder - - com.google.protobuf.StructOrBuilder - - com.google.protobuf.StructProto - - com.google.protobuf.UInt32Value - - com.google.protobuf.UInt32Value$1 - - com.google.protobuf.UInt32Value$Builder - - com.google.protobuf.UInt32ValueOrBuilder - - com.google.protobuf.UInt64Value - - com.google.protobuf.UInt64Value$1 - - com.google.protobuf.UInt64Value$Builder - - com.google.protobuf.UInt64ValueOrBuilder - - com.google.protobuf.Value - - com.google.protobuf.Value$1 - - com.google.protobuf.Value$Builder - - com.google.protobuf.Value$KindCase - - com.google.protobuf.ValueOrBuilder - - com.google.protobuf.WrappersProto -[dev.vortex:vortex-jni (classifier=all), io.netty:netty-buffer] - - io.netty.buffer.AbstractByteBuf - - io.netty.buffer.AbstractReferenceCountedByteBuf - - io.netty.buffer.AdaptiveByteBufAllocator$DirectChunkAllocator - - io.netty.buffer.AdaptivePoolingAllocator - - io.netty.buffer.AdaptivePoolingAllocator$1 - - io.netty.buffer.AdaptivePoolingAllocator$AdaptiveByteBuf - - io.netty.buffer.AdaptivePoolingAllocator$Chunk - - io.netty.buffer.AdaptivePoolingAllocator$ChunkController - - io.netty.buffer.AdaptivePoolingAllocator$ChunkRegistry - - io.netty.buffer.AdaptivePoolingAllocator$Magazine - - io.netty.buffer.AdaptivePoolingAllocator$MagazineGroup - - io.netty.buffer.AdaptivePoolingAllocator$SizeClassChunkController - - io.netty.buffer.AdaptivePoolingAllocator$SizeClassedChunk - - io.netty.buffer.ByteBufUtil - - io.netty.buffer.CompositeByteBuf - - io.netty.buffer.EmptyByteBuf - - io.netty.buffer.PoolArena - - io.netty.buffer.PoolArena$DirectArena - - io.netty.buffer.PoolThreadCache$FreeOnFinalize - - io.netty.buffer.PooledByteBufAllocator - - io.netty.buffer.PooledByteBufAllocator$PoolThreadLocalCache - - io.netty.buffer.ReadOnlyAbstractByteBuf - - io.netty.buffer.SimpleLeakAwareByteBuf - - io.netty.buffer.Unpooled - - io.netty.buffer.UnpooledByteBufAllocator$DecrementingCleanableDirectBuffer - - io.netty.buffer.UnpooledByteBufAllocator$InstrumentedUnpooledDirectByteBuf - - io.netty.buffer.UnpooledByteBufAllocator$InstrumentedUnpooledUnsafeDirectByteBuf - - io.netty.buffer.UnpooledByteBufAllocator$InstrumentedUnpooledUnsafeNoCleanerDirectByteBuf - - io.netty.buffer.UnpooledByteBufAllocator$UnpooledByteBufAllocatorMetric - - io.netty.buffer.UnpooledDirectByteBuf - - io.netty.buffer.UnpooledHeapByteBuf - - io.netty.buffer.UnpooledUnsafeDirectByteBuf - - io.netty.buffer.UnpooledUnsafeNoCleanerDirectByteBuf - - io.netty.buffer.UnsafeByteBufUtil -[dev.vortex:vortex-jni (classifier=all), io.netty:netty-common] - - io.netty.util.AbstractReferenceCounted - - io.netty.util.DefaultAttributeMap - - io.netty.util.HashedWheelTimer - - io.netty.util.HashedWheelTimer$HashedWheelBucket - - io.netty.util.LeakPresenceDetector - - io.netty.util.LeakPresenceDetector$LeakCreation - - io.netty.util.LeakPresenceDetector$ResourceScope - - io.netty.util.Recycler - - io.netty.util.Recycler$BlockingMessageQueue - - io.netty.util.Recycler$DefaultHandle - - io.netty.util.Recycler$EnhancedHandle - - io.netty.util.Recycler$GuardedLocalPool - - io.netty.util.Recycler$LocalPool - - io.netty.util.Recycler$UnguardedLocalPool - - io.netty.util.concurrent.AbstractScheduledEventExecutor - - io.netty.util.concurrent.GlobalEventExecutor - - io.netty.util.concurrent.GlobalEventExecutor$2 - - io.netty.util.concurrent.GlobalEventExecutor$3 - - io.netty.util.concurrent.GlobalEventExecutor$TaskRunner - - io.netty.util.concurrent.MpscIntQueue$MpscAtomicIntegerArrayQueue - - io.netty.util.concurrent.MultithreadEventExecutorGroup - - io.netty.util.concurrent.NonStickyEventExecutorGroup$NonStickyOrderedEventExecutor - - io.netty.util.concurrent.PromiseNotifier - - io.netty.util.concurrent.PromiseNotifier$1 - - io.netty.util.concurrent.SingleThreadEventExecutor - - io.netty.util.concurrent.SingleThreadEventExecutor$4 - - io.netty.util.concurrent.SingleThreadEventExecutor$5 - - io.netty.util.concurrent.SingleThreadEventExecutor$DefaultThreadProperties - - io.netty.util.internal.Cleaner - - io.netty.util.internal.CleanerJava24Linker - - io.netty.util.internal.CleanerJava24Linker$CleanableDirectBufferImpl - - io.netty.util.internal.CleanerJava25 - - io.netty.util.internal.CleanerJava25$CleanableDirectBufferImpl - - io.netty.util.internal.CleanerJava6 - - io.netty.util.internal.CleanerJava6$2 - - io.netty.util.internal.CleanerJava6$CleanableDirectBufferImpl - - io.netty.util.internal.CleanerJava9 - - io.netty.util.internal.CleanerJava9$2 - - io.netty.util.internal.CleanerJava9$CleanableDirectBufferImpl - - io.netty.util.internal.DirectCleaner - - io.netty.util.internal.DirectCleaner$CleanableDirectBufferImpl - - io.netty.util.internal.EmptyArrays - - io.netty.util.internal.PlatformDependent - - io.netty.util.internal.PlatformDependent$1 - - io.netty.util.internal.PlatformDependent$1$1 - - io.netty.util.internal.PlatformDependent$Mpsc - - io.netty.util.internal.PlatformDependent$Mpsc$1 - - io.netty.util.internal.PlatformDependent0 - - io.netty.util.internal.PlatformDependent0$1 - - io.netty.util.internal.PlatformDependent0$10 - - io.netty.util.internal.PlatformDependent0$11 - - io.netty.util.internal.PlatformDependent0$12 - - io.netty.util.internal.PlatformDependent0$2 - - io.netty.util.internal.PlatformDependent0$3 - - io.netty.util.internal.PlatformDependent0$4 - - io.netty.util.internal.PlatformDependent0$5 - - io.netty.util.internal.PlatformDependent0$6 - - io.netty.util.internal.PlatformDependent0$7 - - io.netty.util.internal.PlatformDependent0$8 - - io.netty.util.internal.PlatformDependent0$9 - - io.netty.util.internal.ReferenceCountUpdater - - io.netty.util.internal.ReferenceCountUpdater$UpdaterType - - io.netty.util.internal.ThrowableUtil [dev.vortex:vortex-jni (classifier=all), org.apache.arrow:arrow-c-data] - org.apache.arrow.c.ArrayStreamExporter$ExportedArrayStreamPrivateData - org.apache.arrow.c.jni.JniWrapper @@ -170,149 +16,12 @@ - io.netty.buffer.PooledByteBufAllocatorL - io.netty.buffer.PooledByteBufAllocatorL$InnerAllocator [dev.vortex:vortex-jni, dev.vortex:vortex-jni (classifier=all)] - - dev.vortex.api.Array - - dev.vortex.api.Files + - dev.vortex.api.DataSource + - dev.vortex.api.Expression - dev.vortex.api.ImmutableScanOptions - - dev.vortex.api.ImmutableScanOptions$Builder - - dev.vortex.api.expressions.Binary - - dev.vortex.api.expressions.GetItem - - dev.vortex.api.expressions.Literal - - dev.vortex.api.expressions.Literal$DecimalLiteral - - dev.vortex.api.proto.DTypes - - dev.vortex.api.proto.EndianUtils - - dev.vortex.api.proto.Expressions - - dev.vortex.api.proto.Scalars - - dev.vortex.api.proto.TemporalMetadatas + - dev.vortex.api.Partition + - dev.vortex.api.Scan + - dev.vortex.api.Session + - dev.vortex.api.VortexWriter - dev.vortex.arrow.ArrowAllocation - - dev.vortex.jni.JNIArray - - dev.vortex.jni.JNIArrayIterator - - dev.vortex.jni.JNIDType - - dev.vortex.jni.JNIFile - dev.vortex.jni.NativeLoader - - dev.vortex.proto.DTypeProtos - - dev.vortex.proto.DTypeProtos$Binary - - dev.vortex.proto.DTypeProtos$Binary$1 - - dev.vortex.proto.DTypeProtos$Binary$Builder - - dev.vortex.proto.DTypeProtos$BinaryOrBuilder - - dev.vortex.proto.DTypeProtos$Bool - - dev.vortex.proto.DTypeProtos$Bool$1 - - dev.vortex.proto.DTypeProtos$Bool$Builder - - dev.vortex.proto.DTypeProtos$BoolOrBuilder - - dev.vortex.proto.DTypeProtos$DType - - dev.vortex.proto.DTypeProtos$DType$1 - - dev.vortex.proto.DTypeProtos$DType$Builder - - dev.vortex.proto.DTypeProtos$DType$DtypeTypeCase - - dev.vortex.proto.DTypeProtos$DTypeOrBuilder - - dev.vortex.proto.DTypeProtos$Decimal - - dev.vortex.proto.DTypeProtos$Decimal$1 - - dev.vortex.proto.DTypeProtos$Decimal$Builder - - dev.vortex.proto.DTypeProtos$DecimalOrBuilder - - dev.vortex.proto.DTypeProtos$Extension - - dev.vortex.proto.DTypeProtos$Extension$1 - - dev.vortex.proto.DTypeProtos$Extension$Builder - - dev.vortex.proto.DTypeProtos$ExtensionOrBuilder - - dev.vortex.proto.DTypeProtos$Field - - dev.vortex.proto.DTypeProtos$Field$1 - - dev.vortex.proto.DTypeProtos$Field$Builder - - dev.vortex.proto.DTypeProtos$Field$FieldTypeCase - - dev.vortex.proto.DTypeProtos$FieldOrBuilder - - dev.vortex.proto.DTypeProtos$FieldPath - - dev.vortex.proto.DTypeProtos$FieldPath$1 - - dev.vortex.proto.DTypeProtos$FieldPath$Builder - - dev.vortex.proto.DTypeProtos$FieldPathOrBuilder - - dev.vortex.proto.DTypeProtos$FixedSizeList - - dev.vortex.proto.DTypeProtos$FixedSizeList$1 - - dev.vortex.proto.DTypeProtos$FixedSizeList$Builder - - dev.vortex.proto.DTypeProtos$FixedSizeListOrBuilder - - dev.vortex.proto.DTypeProtos$List - - dev.vortex.proto.DTypeProtos$List$1 - - dev.vortex.proto.DTypeProtos$List$Builder - - dev.vortex.proto.DTypeProtos$ListOrBuilder - - dev.vortex.proto.DTypeProtos$Null - - dev.vortex.proto.DTypeProtos$Null$1 - - dev.vortex.proto.DTypeProtos$Null$Builder - - dev.vortex.proto.DTypeProtos$NullOrBuilder - - dev.vortex.proto.DTypeProtos$PType - - dev.vortex.proto.DTypeProtos$PType$1 - - dev.vortex.proto.DTypeProtos$Primitive - - dev.vortex.proto.DTypeProtos$Primitive$1 - - dev.vortex.proto.DTypeProtos$Primitive$Builder - - dev.vortex.proto.DTypeProtos$PrimitiveOrBuilder - - dev.vortex.proto.DTypeProtos$Struct - - dev.vortex.proto.DTypeProtos$Struct$1 - - dev.vortex.proto.DTypeProtos$Struct$Builder - - dev.vortex.proto.DTypeProtos$StructOrBuilder - - dev.vortex.proto.DTypeProtos$Utf8 - - dev.vortex.proto.DTypeProtos$Utf8$1 - - dev.vortex.proto.DTypeProtos$Utf8$Builder - - dev.vortex.proto.DTypeProtos$Utf8OrBuilder - - dev.vortex.proto.DTypeProtos$Variant - - dev.vortex.proto.DTypeProtos$Variant$1 - - dev.vortex.proto.DTypeProtos$Variant$Builder - - dev.vortex.proto.DTypeProtos$VariantOrBuilder - - dev.vortex.proto.ExprProtos - - dev.vortex.proto.ExprProtos$AggregateFn - - dev.vortex.proto.ExprProtos$AggregateFn$1 - - dev.vortex.proto.ExprProtos$AggregateFn$Builder - - dev.vortex.proto.ExprProtos$AggregateFnOrBuilder - - dev.vortex.proto.ExprProtos$BetweenOpts - - dev.vortex.proto.ExprProtos$BetweenOpts$1 - - dev.vortex.proto.ExprProtos$BetweenOpts$Builder - - dev.vortex.proto.ExprProtos$BetweenOptsOrBuilder - - dev.vortex.proto.ExprProtos$BinaryOpts - - dev.vortex.proto.ExprProtos$BinaryOpts$1 - - dev.vortex.proto.ExprProtos$BinaryOpts$BinaryOp - - dev.vortex.proto.ExprProtos$BinaryOpts$BinaryOp$1 - - dev.vortex.proto.ExprProtos$BinaryOpts$Builder - - dev.vortex.proto.ExprProtos$BinaryOptsOrBuilder - - dev.vortex.proto.ExprProtos$CaseWhenOpts - - dev.vortex.proto.ExprProtos$CaseWhenOpts$1 - - dev.vortex.proto.ExprProtos$CaseWhenOpts$Builder - - dev.vortex.proto.ExprProtos$CaseWhenOptsOrBuilder - - dev.vortex.proto.ExprProtos$CastOpts - - dev.vortex.proto.ExprProtos$CastOpts$1 - - dev.vortex.proto.ExprProtos$CastOpts$Builder - - dev.vortex.proto.ExprProtos$CastOptsOrBuilder - - dev.vortex.proto.ExprProtos$Expr - - dev.vortex.proto.ExprProtos$Expr$1 - - dev.vortex.proto.ExprProtos$Expr$Builder - - dev.vortex.proto.ExprProtos$ExprOrBuilder - - dev.vortex.proto.ExprProtos$FieldNames - - dev.vortex.proto.ExprProtos$FieldNames$1 - - dev.vortex.proto.ExprProtos$FieldNames$Builder - - dev.vortex.proto.ExprProtos$FieldNamesOrBuilder - - dev.vortex.proto.ExprProtos$GetItemOpts - - dev.vortex.proto.ExprProtos$GetItemOpts$1 - - dev.vortex.proto.ExprProtos$GetItemOpts$Builder - - dev.vortex.proto.ExprProtos$GetItemOptsOrBuilder - - dev.vortex.proto.ExprProtos$LikeOpts - - dev.vortex.proto.ExprProtos$LikeOpts$1 - - dev.vortex.proto.ExprProtos$LikeOpts$Builder - - dev.vortex.proto.ExprProtos$LikeOptsOrBuilder - - dev.vortex.proto.ExprProtos$LiteralOpts - - dev.vortex.proto.ExprProtos$LiteralOpts$1 - - dev.vortex.proto.ExprProtos$LiteralOpts$Builder - - dev.vortex.proto.ExprProtos$LiteralOptsOrBuilder - - dev.vortex.proto.ExprProtos$PackOpts - - dev.vortex.proto.ExprProtos$PackOpts$1 - - dev.vortex.proto.ExprProtos$PackOpts$Builder - - dev.vortex.proto.ExprProtos$PackOptsOrBuilder - - dev.vortex.proto.ExprProtos$SelectOpts - - dev.vortex.proto.ExprProtos$SelectOpts$1 - - dev.vortex.proto.ExprProtos$SelectOpts$Builder - - dev.vortex.proto.ExprProtos$SelectOpts$OptsCase - - dev.vortex.proto.ExprProtos$SelectOptsOrBuilder - - dev.vortex.proto.ScalarProtos - - dev.vortex.proto.ScalarProtos$ListValue - - dev.vortex.proto.ScalarProtos$ListValue$1 - - dev.vortex.proto.ScalarProtos$ListValue$Builder - - dev.vortex.proto.ScalarProtos$ListValueOrBuilder - - dev.vortex.proto.ScalarProtos$Scalar - - dev.vortex.proto.ScalarProtos$Scalar$1 - - dev.vortex.proto.ScalarProtos$Scalar$Builder - - dev.vortex.proto.ScalarProtos$ScalarOrBuilder - - dev.vortex.proto.ScalarProtos$ScalarValue - - dev.vortex.proto.ScalarProtos$ScalarValue$1 - - dev.vortex.proto.ScalarProtos$ScalarValue$Builder - - dev.vortex.proto.ScalarProtos$ScalarValue$KindCase - - dev.vortex.proto.ScalarProtos$ScalarValueOrBuilder diff --git a/spark/v3.5/spark-runtime/runtime-deps.txt b/spark/v3.5/spark-runtime/runtime-deps.txt index 03df1268f0ff..e3cd88b2b3f2 100644 --- a/spark/v3.5/spark-runtime/runtime-deps.txt +++ b/spark/v3.5/spark-runtime/runtime-deps.txt @@ -3,25 +3,24 @@ com.fasterxml.jackson.core:jackson-core:2.15 com.fasterxml.jackson.core:jackson-databind:2.15 com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.21 com.github.ben-manes.caffeine:caffeine:2.9 -com.google.errorprone:error_prone_annotations:2.41 +com.google.errorprone:error_prone_annotations:2.47 com.google.flatbuffers:flatbuffers-java:25.2 com.google.guava:failureaccess:1.0 -com.google.guava:guava:33.5 +com.google.guava:guava:33.6 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.j2objc:j2objc-annotations:3.1 -com.google.protobuf:protobuf-java:4.33 dev.failsafe:failsafe:3.3 -dev.vortex:vortex-jni:0.67 -dev.vortex:vortex-spark_2.12:0.67 +dev.vortex:vortex-jni:0.72 +dev.vortex:vortex-spark_2.12:0.72 io.airlift:aircompressor:2.0 io.netty:netty-buffer:4.2 io.netty:netty-common:4.2 -org.apache.arrow:arrow-c-data:18.3 -org.apache.arrow:arrow-format:18.3 -org.apache.arrow:arrow-memory-core:18.3 -org.apache.arrow:arrow-memory-netty-buffer-patch:18.3 -org.apache.arrow:arrow-memory-netty:18.3 -org.apache.arrow:arrow-vector:18.3 +org.apache.arrow:arrow-c-data:19.0 +org.apache.arrow:arrow-format:19.0 +org.apache.arrow:arrow-memory-core:19.0 +org.apache.arrow:arrow-memory-netty-buffer-patch:19.0 +org.apache.arrow:arrow-memory-netty:19.0 +org.apache.arrow:arrow-vector:19.0 org.apache.avro:avro:1.12 org.apache.datasketches:datasketches-java:6.2 org.apache.datasketches:datasketches-memory:3.0 diff --git a/spark/v3.5/spark/baseline-class-uniqueness.lock b/spark/v3.5/spark/baseline-class-uniqueness.lock index 72c0c24fb849..53a154c69d97 100644 --- a/spark/v3.5/spark/baseline-class-uniqueness.lock +++ b/spark/v3.5/spark/baseline-class-uniqueness.lock @@ -4,211 +4,9 @@ ## runtimeClasspath [com.google.guava:guava, dev.vortex:vortex-jni (classifier=all)] - com.google.thirdparty.publicsuffix.PublicSuffixPatterns + - com.google.thirdparty.publicsuffix.PublicSuffixTrie + - com.google.thirdparty.publicsuffix.PublicSuffixTrie$ChunksCharSequence - com.google.thirdparty.publicsuffix.PublicSuffixType - - com.google.thirdparty.publicsuffix.TrieParser -[com.google.protobuf:protobuf-java, dev.vortex:vortex-jni] - - com.google.protobuf.BoolValue - - com.google.protobuf.BoolValue$1 - - com.google.protobuf.BoolValue$Builder - - com.google.protobuf.BoolValueOrBuilder - - com.google.protobuf.BytesValue - - com.google.protobuf.BytesValue$1 - - com.google.protobuf.BytesValue$Builder - - com.google.protobuf.BytesValueOrBuilder - - com.google.protobuf.DoubleValue - - com.google.protobuf.DoubleValue$1 - - com.google.protobuf.DoubleValue$Builder - - com.google.protobuf.DoubleValueOrBuilder - - com.google.protobuf.FloatValue - - com.google.protobuf.FloatValue$1 - - com.google.protobuf.FloatValue$Builder - - com.google.protobuf.FloatValueOrBuilder - - com.google.protobuf.Int32Value - - com.google.protobuf.Int32Value$1 - - com.google.protobuf.Int32Value$Builder - - com.google.protobuf.Int32ValueOrBuilder - - com.google.protobuf.Int64Value - - com.google.protobuf.Int64Value$1 - - com.google.protobuf.Int64Value$Builder - - com.google.protobuf.Int64ValueOrBuilder - - com.google.protobuf.ListValue - - com.google.protobuf.ListValue$1 - - com.google.protobuf.ListValue$Builder - - com.google.protobuf.ListValueOrBuilder - - com.google.protobuf.NullValue - - com.google.protobuf.NullValue$1 - - com.google.protobuf.StringValue - - com.google.protobuf.StringValue$1 - - com.google.protobuf.StringValue$Builder - - com.google.protobuf.StringValueOrBuilder - - com.google.protobuf.Struct - - com.google.protobuf.Struct$1 - - com.google.protobuf.Struct$Builder - - com.google.protobuf.Struct$Builder$FieldsConverter - - com.google.protobuf.Struct$FieldsDefaultEntryHolder - - com.google.protobuf.StructOrBuilder - - com.google.protobuf.StructProto - - com.google.protobuf.UInt32Value - - com.google.protobuf.UInt32Value$1 - - com.google.protobuf.UInt32Value$Builder - - com.google.protobuf.UInt32ValueOrBuilder - - com.google.protobuf.UInt64Value - - com.google.protobuf.UInt64Value$1 - - com.google.protobuf.UInt64Value$Builder - - com.google.protobuf.UInt64ValueOrBuilder - - com.google.protobuf.Value - - com.google.protobuf.Value$1 - - com.google.protobuf.Value$Builder - - com.google.protobuf.Value$KindCase - - com.google.protobuf.ValueOrBuilder - - com.google.protobuf.WrappersProto -[commons-codec:commons-codec, dev.vortex:vortex-jni (classifier=all)] - - org.apache.commons.codec.Charsets - - org.apache.commons.codec.binary.Base16 - - org.apache.commons.codec.binary.Base32 - - org.apache.commons.codec.binary.Base64 - - org.apache.commons.codec.binary.BaseNCodec - - org.apache.commons.codec.binary.BaseNCodec$AbstractBuilder - - org.apache.commons.codec.binary.BinaryCodec - - org.apache.commons.codec.binary.StringUtils - - org.apache.commons.codec.digest.DigestUtils - - org.apache.commons.codec.digest.HmacUtils - - org.apache.commons.codec.digest.MurmurHash2 - - org.apache.commons.codec.digest.MurmurHash3 - - org.apache.commons.codec.digest.MurmurHash3$IncrementalHash32 - - org.apache.commons.codec.digest.MurmurHash3$IncrementalHash32x86 - - org.apache.commons.codec.digest.PureJavaCrc32 - - org.apache.commons.codec.digest.PureJavaCrc32C - - org.apache.commons.codec.digest.Sha2Crypt - - org.apache.commons.codec.digest.UnixCrypt - - org.apache.commons.codec.language.Caverphone1 - - org.apache.commons.codec.language.Caverphone2 - - org.apache.commons.codec.language.DaitchMokotoffSoundex - - org.apache.commons.codec.language.DaitchMokotoffSoundex$Branch - - org.apache.commons.codec.language.DaitchMokotoffSoundex$Rule - - org.apache.commons.codec.language.DoubleMetaphone - - org.apache.commons.codec.language.DoubleMetaphone$DoubleMetaphoneResult - - org.apache.commons.codec.language.MatchRatingApproachEncoder - - org.apache.commons.codec.language.Metaphone - - org.apache.commons.codec.language.Nysiis - - org.apache.commons.codec.language.RefinedSoundex - - org.apache.commons.codec.language.Soundex - - org.apache.commons.codec.language.SoundexUtils - - org.apache.commons.codec.language.bm.Lang - - org.apache.commons.codec.language.bm.PhoneticEngine - - org.apache.commons.codec.language.bm.PhoneticEngine$1 - - org.apache.commons.codec.language.bm.PhoneticEngine$PhonemeBuilder - - org.apache.commons.codec.language.bm.PhoneticEngine$RulesApplication - - org.apache.commons.codec.language.bm.ResourceConstants - - org.apache.commons.codec.language.bm.Rule - - org.apache.commons.codec.language.bm.Rule$1 - - org.apache.commons.codec.language.bm.Rule$2 - - org.apache.commons.codec.language.bm.Rule$Phoneme - - org.apache.commons.codec.language.bm.Rule$PhonemeExpr - - org.apache.commons.codec.language.bm.Rule$PhonemeList - - org.apache.commons.codec.net.PercentCodec - - org.apache.commons.codec.net.QuotedPrintableCodec - - org.apache.commons.codec.net.URLCodec - - org.apache.commons.codec.net.Utils -[dev.vortex:vortex-jni (classifier=all), io.netty:netty-buffer] - - io.netty.buffer.AbstractByteBuf - - io.netty.buffer.AbstractReferenceCountedByteBuf - - io.netty.buffer.AdaptiveByteBufAllocator$DirectChunkAllocator - - io.netty.buffer.AdaptivePoolingAllocator - - io.netty.buffer.AdaptivePoolingAllocator$1 - - io.netty.buffer.AdaptivePoolingAllocator$AdaptiveByteBuf - - io.netty.buffer.AdaptivePoolingAllocator$Chunk - - io.netty.buffer.AdaptivePoolingAllocator$ChunkController - - io.netty.buffer.AdaptivePoolingAllocator$ChunkRegistry - - io.netty.buffer.AdaptivePoolingAllocator$Magazine - - io.netty.buffer.AdaptivePoolingAllocator$MagazineGroup - - io.netty.buffer.AdaptivePoolingAllocator$SizeClassChunkController - - io.netty.buffer.AdaptivePoolingAllocator$SizeClassedChunk - - io.netty.buffer.ByteBufUtil - - io.netty.buffer.CompositeByteBuf - - io.netty.buffer.EmptyByteBuf - - io.netty.buffer.PoolArena - - io.netty.buffer.PoolArena$DirectArena - - io.netty.buffer.PoolThreadCache$FreeOnFinalize - - io.netty.buffer.PooledByteBufAllocator - - io.netty.buffer.PooledByteBufAllocator$PoolThreadLocalCache - - io.netty.buffer.ReadOnlyAbstractByteBuf - - io.netty.buffer.SimpleLeakAwareByteBuf - - io.netty.buffer.Unpooled - - io.netty.buffer.UnpooledByteBufAllocator$DecrementingCleanableDirectBuffer - - io.netty.buffer.UnpooledByteBufAllocator$InstrumentedUnpooledDirectByteBuf - - io.netty.buffer.UnpooledByteBufAllocator$InstrumentedUnpooledUnsafeDirectByteBuf - - io.netty.buffer.UnpooledByteBufAllocator$InstrumentedUnpooledUnsafeNoCleanerDirectByteBuf - - io.netty.buffer.UnpooledByteBufAllocator$UnpooledByteBufAllocatorMetric - - io.netty.buffer.UnpooledDirectByteBuf - - io.netty.buffer.UnpooledHeapByteBuf - - io.netty.buffer.UnpooledUnsafeDirectByteBuf - - io.netty.buffer.UnpooledUnsafeNoCleanerDirectByteBuf - - io.netty.buffer.UnsafeByteBufUtil -[dev.vortex:vortex-jni (classifier=all), io.netty:netty-common] - - io.netty.util.AbstractReferenceCounted - - io.netty.util.DefaultAttributeMap - - io.netty.util.HashedWheelTimer - - io.netty.util.HashedWheelTimer$HashedWheelBucket - - io.netty.util.LeakPresenceDetector - - io.netty.util.LeakPresenceDetector$LeakCreation - - io.netty.util.LeakPresenceDetector$ResourceScope - - io.netty.util.Recycler - - io.netty.util.Recycler$BlockingMessageQueue - - io.netty.util.Recycler$DefaultHandle - - io.netty.util.Recycler$EnhancedHandle - - io.netty.util.Recycler$GuardedLocalPool - - io.netty.util.Recycler$LocalPool - - io.netty.util.Recycler$UnguardedLocalPool - - io.netty.util.concurrent.AbstractScheduledEventExecutor - - io.netty.util.concurrent.GlobalEventExecutor - - io.netty.util.concurrent.GlobalEventExecutor$2 - - io.netty.util.concurrent.GlobalEventExecutor$3 - - io.netty.util.concurrent.GlobalEventExecutor$TaskRunner - - io.netty.util.concurrent.MpscIntQueue$MpscAtomicIntegerArrayQueue - - io.netty.util.concurrent.MultithreadEventExecutorGroup - - io.netty.util.concurrent.NonStickyEventExecutorGroup$NonStickyOrderedEventExecutor - - io.netty.util.concurrent.PromiseNotifier - - io.netty.util.concurrent.PromiseNotifier$1 - - io.netty.util.concurrent.SingleThreadEventExecutor - - io.netty.util.concurrent.SingleThreadEventExecutor$4 - - io.netty.util.concurrent.SingleThreadEventExecutor$5 - - io.netty.util.concurrent.SingleThreadEventExecutor$DefaultThreadProperties - - io.netty.util.internal.Cleaner - - io.netty.util.internal.CleanerJava24Linker - - io.netty.util.internal.CleanerJava24Linker$CleanableDirectBufferImpl - - io.netty.util.internal.CleanerJava25 - - io.netty.util.internal.CleanerJava25$CleanableDirectBufferImpl - - io.netty.util.internal.CleanerJava6 - - io.netty.util.internal.CleanerJava6$2 - - io.netty.util.internal.CleanerJava6$CleanableDirectBufferImpl - - io.netty.util.internal.CleanerJava9 - - io.netty.util.internal.CleanerJava9$2 - - io.netty.util.internal.CleanerJava9$CleanableDirectBufferImpl - - io.netty.util.internal.DirectCleaner - - io.netty.util.internal.DirectCleaner$CleanableDirectBufferImpl - - io.netty.util.internal.EmptyArrays - - io.netty.util.internal.PlatformDependent - - io.netty.util.internal.PlatformDependent$1 - - io.netty.util.internal.PlatformDependent$1$1 - - io.netty.util.internal.PlatformDependent$Mpsc - - io.netty.util.internal.PlatformDependent$Mpsc$1 - - io.netty.util.internal.PlatformDependent0 - - io.netty.util.internal.PlatformDependent0$1 - - io.netty.util.internal.PlatformDependent0$10 - - io.netty.util.internal.PlatformDependent0$11 - - io.netty.util.internal.PlatformDependent0$12 - - io.netty.util.internal.PlatformDependent0$2 - - io.netty.util.internal.PlatformDependent0$3 - - io.netty.util.internal.PlatformDependent0$4 - - io.netty.util.internal.PlatformDependent0$5 - - io.netty.util.internal.PlatformDependent0$6 - - io.netty.util.internal.PlatformDependent0$7 - - io.netty.util.internal.PlatformDependent0$8 - - io.netty.util.internal.PlatformDependent0$9 - - io.netty.util.internal.ReferenceCountUpdater - - io.netty.util.internal.ReferenceCountUpdater$UpdaterType - - io.netty.util.internal.ThrowableUtil [dev.vortex:vortex-jni (classifier=all), org.apache.arrow:arrow-c-data] - org.apache.arrow.c.ArrayStreamExporter$ExportedArrayStreamPrivateData - org.apache.arrow.c.jni.JniWrapper @@ -218,149 +16,12 @@ - io.netty.buffer.PooledByteBufAllocatorL - io.netty.buffer.PooledByteBufAllocatorL$InnerAllocator [dev.vortex:vortex-jni, dev.vortex:vortex-jni (classifier=all)] - - dev.vortex.api.Array - - dev.vortex.api.Files + - dev.vortex.api.DataSource + - dev.vortex.api.Expression - dev.vortex.api.ImmutableScanOptions - - dev.vortex.api.ImmutableScanOptions$Builder - - dev.vortex.api.expressions.Binary - - dev.vortex.api.expressions.GetItem - - dev.vortex.api.expressions.Literal - - dev.vortex.api.expressions.Literal$DecimalLiteral - - dev.vortex.api.proto.DTypes - - dev.vortex.api.proto.EndianUtils - - dev.vortex.api.proto.Expressions - - dev.vortex.api.proto.Scalars - - dev.vortex.api.proto.TemporalMetadatas + - dev.vortex.api.Partition + - dev.vortex.api.Scan + - dev.vortex.api.Session + - dev.vortex.api.VortexWriter - dev.vortex.arrow.ArrowAllocation - - dev.vortex.jni.JNIArray - - dev.vortex.jni.JNIArrayIterator - - dev.vortex.jni.JNIDType - - dev.vortex.jni.JNIFile - dev.vortex.jni.NativeLoader - - dev.vortex.proto.DTypeProtos - - dev.vortex.proto.DTypeProtos$Binary - - dev.vortex.proto.DTypeProtos$Binary$1 - - dev.vortex.proto.DTypeProtos$Binary$Builder - - dev.vortex.proto.DTypeProtos$BinaryOrBuilder - - dev.vortex.proto.DTypeProtos$Bool - - dev.vortex.proto.DTypeProtos$Bool$1 - - dev.vortex.proto.DTypeProtos$Bool$Builder - - dev.vortex.proto.DTypeProtos$BoolOrBuilder - - dev.vortex.proto.DTypeProtos$DType - - dev.vortex.proto.DTypeProtos$DType$1 - - dev.vortex.proto.DTypeProtos$DType$Builder - - dev.vortex.proto.DTypeProtos$DType$DtypeTypeCase - - dev.vortex.proto.DTypeProtos$DTypeOrBuilder - - dev.vortex.proto.DTypeProtos$Decimal - - dev.vortex.proto.DTypeProtos$Decimal$1 - - dev.vortex.proto.DTypeProtos$Decimal$Builder - - dev.vortex.proto.DTypeProtos$DecimalOrBuilder - - dev.vortex.proto.DTypeProtos$Extension - - dev.vortex.proto.DTypeProtos$Extension$1 - - dev.vortex.proto.DTypeProtos$Extension$Builder - - dev.vortex.proto.DTypeProtos$ExtensionOrBuilder - - dev.vortex.proto.DTypeProtos$Field - - dev.vortex.proto.DTypeProtos$Field$1 - - dev.vortex.proto.DTypeProtos$Field$Builder - - dev.vortex.proto.DTypeProtos$Field$FieldTypeCase - - dev.vortex.proto.DTypeProtos$FieldOrBuilder - - dev.vortex.proto.DTypeProtos$FieldPath - - dev.vortex.proto.DTypeProtos$FieldPath$1 - - dev.vortex.proto.DTypeProtos$FieldPath$Builder - - dev.vortex.proto.DTypeProtos$FieldPathOrBuilder - - dev.vortex.proto.DTypeProtos$FixedSizeList - - dev.vortex.proto.DTypeProtos$FixedSizeList$1 - - dev.vortex.proto.DTypeProtos$FixedSizeList$Builder - - dev.vortex.proto.DTypeProtos$FixedSizeListOrBuilder - - dev.vortex.proto.DTypeProtos$List - - dev.vortex.proto.DTypeProtos$List$1 - - dev.vortex.proto.DTypeProtos$List$Builder - - dev.vortex.proto.DTypeProtos$ListOrBuilder - - dev.vortex.proto.DTypeProtos$Null - - dev.vortex.proto.DTypeProtos$Null$1 - - dev.vortex.proto.DTypeProtos$Null$Builder - - dev.vortex.proto.DTypeProtos$NullOrBuilder - - dev.vortex.proto.DTypeProtos$PType - - dev.vortex.proto.DTypeProtos$PType$1 - - dev.vortex.proto.DTypeProtos$Primitive - - dev.vortex.proto.DTypeProtos$Primitive$1 - - dev.vortex.proto.DTypeProtos$Primitive$Builder - - dev.vortex.proto.DTypeProtos$PrimitiveOrBuilder - - dev.vortex.proto.DTypeProtos$Struct - - dev.vortex.proto.DTypeProtos$Struct$1 - - dev.vortex.proto.DTypeProtos$Struct$Builder - - dev.vortex.proto.DTypeProtos$StructOrBuilder - - dev.vortex.proto.DTypeProtos$Utf8 - - dev.vortex.proto.DTypeProtos$Utf8$1 - - dev.vortex.proto.DTypeProtos$Utf8$Builder - - dev.vortex.proto.DTypeProtos$Utf8OrBuilder - - dev.vortex.proto.DTypeProtos$Variant - - dev.vortex.proto.DTypeProtos$Variant$1 - - dev.vortex.proto.DTypeProtos$Variant$Builder - - dev.vortex.proto.DTypeProtos$VariantOrBuilder - - dev.vortex.proto.ExprProtos - - dev.vortex.proto.ExprProtos$AggregateFn - - dev.vortex.proto.ExprProtos$AggregateFn$1 - - dev.vortex.proto.ExprProtos$AggregateFn$Builder - - dev.vortex.proto.ExprProtos$AggregateFnOrBuilder - - dev.vortex.proto.ExprProtos$BetweenOpts - - dev.vortex.proto.ExprProtos$BetweenOpts$1 - - dev.vortex.proto.ExprProtos$BetweenOpts$Builder - - dev.vortex.proto.ExprProtos$BetweenOptsOrBuilder - - dev.vortex.proto.ExprProtos$BinaryOpts - - dev.vortex.proto.ExprProtos$BinaryOpts$1 - - dev.vortex.proto.ExprProtos$BinaryOpts$BinaryOp - - dev.vortex.proto.ExprProtos$BinaryOpts$BinaryOp$1 - - dev.vortex.proto.ExprProtos$BinaryOpts$Builder - - dev.vortex.proto.ExprProtos$BinaryOptsOrBuilder - - dev.vortex.proto.ExprProtos$CaseWhenOpts - - dev.vortex.proto.ExprProtos$CaseWhenOpts$1 - - dev.vortex.proto.ExprProtos$CaseWhenOpts$Builder - - dev.vortex.proto.ExprProtos$CaseWhenOptsOrBuilder - - dev.vortex.proto.ExprProtos$CastOpts - - dev.vortex.proto.ExprProtos$CastOpts$1 - - dev.vortex.proto.ExprProtos$CastOpts$Builder - - dev.vortex.proto.ExprProtos$CastOptsOrBuilder - - dev.vortex.proto.ExprProtos$Expr - - dev.vortex.proto.ExprProtos$Expr$1 - - dev.vortex.proto.ExprProtos$Expr$Builder - - dev.vortex.proto.ExprProtos$ExprOrBuilder - - dev.vortex.proto.ExprProtos$FieldNames - - dev.vortex.proto.ExprProtos$FieldNames$1 - - dev.vortex.proto.ExprProtos$FieldNames$Builder - - dev.vortex.proto.ExprProtos$FieldNamesOrBuilder - - dev.vortex.proto.ExprProtos$GetItemOpts - - dev.vortex.proto.ExprProtos$GetItemOpts$1 - - dev.vortex.proto.ExprProtos$GetItemOpts$Builder - - dev.vortex.proto.ExprProtos$GetItemOptsOrBuilder - - dev.vortex.proto.ExprProtos$LikeOpts - - dev.vortex.proto.ExprProtos$LikeOpts$1 - - dev.vortex.proto.ExprProtos$LikeOpts$Builder - - dev.vortex.proto.ExprProtos$LikeOptsOrBuilder - - dev.vortex.proto.ExprProtos$LiteralOpts - - dev.vortex.proto.ExprProtos$LiteralOpts$1 - - dev.vortex.proto.ExprProtos$LiteralOpts$Builder - - dev.vortex.proto.ExprProtos$LiteralOptsOrBuilder - - dev.vortex.proto.ExprProtos$PackOpts - - dev.vortex.proto.ExprProtos$PackOpts$1 - - dev.vortex.proto.ExprProtos$PackOpts$Builder - - dev.vortex.proto.ExprProtos$PackOptsOrBuilder - - dev.vortex.proto.ExprProtos$SelectOpts - - dev.vortex.proto.ExprProtos$SelectOpts$1 - - dev.vortex.proto.ExprProtos$SelectOpts$Builder - - dev.vortex.proto.ExprProtos$SelectOpts$OptsCase - - dev.vortex.proto.ExprProtos$SelectOptsOrBuilder - - dev.vortex.proto.ScalarProtos - - dev.vortex.proto.ScalarProtos$ListValue - - dev.vortex.proto.ScalarProtos$ListValue$1 - - dev.vortex.proto.ScalarProtos$ListValue$Builder - - dev.vortex.proto.ScalarProtos$ListValueOrBuilder - - dev.vortex.proto.ScalarProtos$Scalar - - dev.vortex.proto.ScalarProtos$Scalar$1 - - dev.vortex.proto.ScalarProtos$Scalar$Builder - - dev.vortex.proto.ScalarProtos$ScalarOrBuilder - - dev.vortex.proto.ScalarProtos$ScalarValue - - dev.vortex.proto.ScalarProtos$ScalarValue$1 - - dev.vortex.proto.ScalarProtos$ScalarValue$Builder - - dev.vortex.proto.ScalarProtos$ScalarValue$KindCase - - dev.vortex.proto.ScalarProtos$ScalarValueOrBuilder diff --git a/spark/v4.0/spark-runtime/baseline-class-uniqueness.lock b/spark/v4.0/spark-runtime/baseline-class-uniqueness.lock index 6197975f3900..53a154c69d97 100644 --- a/spark/v4.0/spark-runtime/baseline-class-uniqueness.lock +++ b/spark/v4.0/spark-runtime/baseline-class-uniqueness.lock @@ -4,163 +4,9 @@ ## runtimeClasspath [com.google.guava:guava, dev.vortex:vortex-jni (classifier=all)] - com.google.thirdparty.publicsuffix.PublicSuffixPatterns + - com.google.thirdparty.publicsuffix.PublicSuffixTrie + - com.google.thirdparty.publicsuffix.PublicSuffixTrie$ChunksCharSequence - com.google.thirdparty.publicsuffix.PublicSuffixType - - com.google.thirdparty.publicsuffix.TrieParser -[com.google.protobuf:protobuf-java, dev.vortex:vortex-jni] - - com.google.protobuf.BoolValue - - com.google.protobuf.BoolValue$1 - - com.google.protobuf.BoolValue$Builder - - com.google.protobuf.BoolValueOrBuilder - - com.google.protobuf.BytesValue - - com.google.protobuf.BytesValue$1 - - com.google.protobuf.BytesValue$Builder - - com.google.protobuf.BytesValueOrBuilder - - com.google.protobuf.DoubleValue - - com.google.protobuf.DoubleValue$1 - - com.google.protobuf.DoubleValue$Builder - - com.google.protobuf.DoubleValueOrBuilder - - com.google.protobuf.FloatValue - - com.google.protobuf.FloatValue$1 - - com.google.protobuf.FloatValue$Builder - - com.google.protobuf.FloatValueOrBuilder - - com.google.protobuf.Int32Value - - com.google.protobuf.Int32Value$1 - - com.google.protobuf.Int32Value$Builder - - com.google.protobuf.Int32ValueOrBuilder - - com.google.protobuf.Int64Value - - com.google.protobuf.Int64Value$1 - - com.google.protobuf.Int64Value$Builder - - com.google.protobuf.Int64ValueOrBuilder - - com.google.protobuf.ListValue - - com.google.protobuf.ListValue$1 - - com.google.protobuf.ListValue$Builder - - com.google.protobuf.ListValueOrBuilder - - com.google.protobuf.NullValue - - com.google.protobuf.NullValue$1 - - com.google.protobuf.StringValue - - com.google.protobuf.StringValue$1 - - com.google.protobuf.StringValue$Builder - - com.google.protobuf.StringValueOrBuilder - - com.google.protobuf.Struct - - com.google.protobuf.Struct$1 - - com.google.protobuf.Struct$Builder - - com.google.protobuf.Struct$Builder$FieldsConverter - - com.google.protobuf.Struct$FieldsDefaultEntryHolder - - com.google.protobuf.StructOrBuilder - - com.google.protobuf.StructProto - - com.google.protobuf.UInt32Value - - com.google.protobuf.UInt32Value$1 - - com.google.protobuf.UInt32Value$Builder - - com.google.protobuf.UInt32ValueOrBuilder - - com.google.protobuf.UInt64Value - - com.google.protobuf.UInt64Value$1 - - com.google.protobuf.UInt64Value$Builder - - com.google.protobuf.UInt64ValueOrBuilder - - com.google.protobuf.Value - - com.google.protobuf.Value$1 - - com.google.protobuf.Value$Builder - - com.google.protobuf.Value$KindCase - - com.google.protobuf.ValueOrBuilder - - com.google.protobuf.WrappersProto -[dev.vortex:vortex-jni (classifier=all), io.netty:netty-buffer] - - io.netty.buffer.AbstractByteBuf - - io.netty.buffer.AbstractReferenceCountedByteBuf - - io.netty.buffer.AdaptiveByteBufAllocator$DirectChunkAllocator - - io.netty.buffer.AdaptivePoolingAllocator - - io.netty.buffer.AdaptivePoolingAllocator$1 - - io.netty.buffer.AdaptivePoolingAllocator$AdaptiveByteBuf - - io.netty.buffer.AdaptivePoolingAllocator$Chunk - - io.netty.buffer.AdaptivePoolingAllocator$ChunkController - - io.netty.buffer.AdaptivePoolingAllocator$ChunkRegistry - - io.netty.buffer.AdaptivePoolingAllocator$Magazine - - io.netty.buffer.AdaptivePoolingAllocator$MagazineGroup - - io.netty.buffer.AdaptivePoolingAllocator$SizeClassChunkController - - io.netty.buffer.AdaptivePoolingAllocator$SizeClassedChunk - - io.netty.buffer.ByteBufUtil - - io.netty.buffer.CompositeByteBuf - - io.netty.buffer.EmptyByteBuf - - io.netty.buffer.PoolArena - - io.netty.buffer.PoolArena$DirectArena - - io.netty.buffer.PoolThreadCache$FreeOnFinalize - - io.netty.buffer.PooledByteBufAllocator - - io.netty.buffer.PooledByteBufAllocator$PoolThreadLocalCache - - io.netty.buffer.ReadOnlyAbstractByteBuf - - io.netty.buffer.SimpleLeakAwareByteBuf - - io.netty.buffer.Unpooled - - io.netty.buffer.UnpooledByteBufAllocator$DecrementingCleanableDirectBuffer - - io.netty.buffer.UnpooledByteBufAllocator$InstrumentedUnpooledDirectByteBuf - - io.netty.buffer.UnpooledByteBufAllocator$InstrumentedUnpooledUnsafeDirectByteBuf - - io.netty.buffer.UnpooledByteBufAllocator$InstrumentedUnpooledUnsafeNoCleanerDirectByteBuf - - io.netty.buffer.UnpooledByteBufAllocator$UnpooledByteBufAllocatorMetric - - io.netty.buffer.UnpooledDirectByteBuf - - io.netty.buffer.UnpooledHeapByteBuf - - io.netty.buffer.UnpooledUnsafeDirectByteBuf - - io.netty.buffer.UnpooledUnsafeNoCleanerDirectByteBuf - - io.netty.buffer.UnsafeByteBufUtil -[dev.vortex:vortex-jni (classifier=all), io.netty:netty-common] - - io.netty.util.AbstractReferenceCounted - - io.netty.util.DefaultAttributeMap - - io.netty.util.HashedWheelTimer - - io.netty.util.HashedWheelTimer$HashedWheelBucket - - io.netty.util.LeakPresenceDetector - - io.netty.util.LeakPresenceDetector$LeakCreation - - io.netty.util.LeakPresenceDetector$ResourceScope - - io.netty.util.Recycler - - io.netty.util.Recycler$BlockingMessageQueue - - io.netty.util.Recycler$DefaultHandle - - io.netty.util.Recycler$EnhancedHandle - - io.netty.util.Recycler$GuardedLocalPool - - io.netty.util.Recycler$LocalPool - - io.netty.util.Recycler$UnguardedLocalPool - - io.netty.util.concurrent.AbstractScheduledEventExecutor - - io.netty.util.concurrent.GlobalEventExecutor - - io.netty.util.concurrent.GlobalEventExecutor$2 - - io.netty.util.concurrent.GlobalEventExecutor$3 - - io.netty.util.concurrent.GlobalEventExecutor$TaskRunner - - io.netty.util.concurrent.MpscIntQueue$MpscAtomicIntegerArrayQueue - - io.netty.util.concurrent.MultithreadEventExecutorGroup - - io.netty.util.concurrent.NonStickyEventExecutorGroup$NonStickyOrderedEventExecutor - - io.netty.util.concurrent.PromiseNotifier - - io.netty.util.concurrent.PromiseNotifier$1 - - io.netty.util.concurrent.SingleThreadEventExecutor - - io.netty.util.concurrent.SingleThreadEventExecutor$4 - - io.netty.util.concurrent.SingleThreadEventExecutor$5 - - io.netty.util.concurrent.SingleThreadEventExecutor$DefaultThreadProperties - - io.netty.util.internal.Cleaner - - io.netty.util.internal.CleanerJava24Linker - - io.netty.util.internal.CleanerJava24Linker$CleanableDirectBufferImpl - - io.netty.util.internal.CleanerJava25 - - io.netty.util.internal.CleanerJava25$CleanableDirectBufferImpl - - io.netty.util.internal.CleanerJava6 - - io.netty.util.internal.CleanerJava6$2 - - io.netty.util.internal.CleanerJava6$CleanableDirectBufferImpl - - io.netty.util.internal.CleanerJava9 - - io.netty.util.internal.CleanerJava9$2 - - io.netty.util.internal.CleanerJava9$CleanableDirectBufferImpl - - io.netty.util.internal.DirectCleaner - - io.netty.util.internal.DirectCleaner$CleanableDirectBufferImpl - - io.netty.util.internal.EmptyArrays - - io.netty.util.internal.PlatformDependent - - io.netty.util.internal.PlatformDependent$1 - - io.netty.util.internal.PlatformDependent$1$1 - - io.netty.util.internal.PlatformDependent$Mpsc - - io.netty.util.internal.PlatformDependent$Mpsc$1 - - io.netty.util.internal.PlatformDependent0 - - io.netty.util.internal.PlatformDependent0$1 - - io.netty.util.internal.PlatformDependent0$10 - - io.netty.util.internal.PlatformDependent0$11 - - io.netty.util.internal.PlatformDependent0$12 - - io.netty.util.internal.PlatformDependent0$2 - - io.netty.util.internal.PlatformDependent0$3 - - io.netty.util.internal.PlatformDependent0$4 - - io.netty.util.internal.PlatformDependent0$5 - - io.netty.util.internal.PlatformDependent0$6 - - io.netty.util.internal.PlatformDependent0$7 - - io.netty.util.internal.PlatformDependent0$8 - - io.netty.util.internal.PlatformDependent0$9 - - io.netty.util.internal.ReferenceCountUpdater - - io.netty.util.internal.ReferenceCountUpdater$UpdaterType - - io.netty.util.internal.ThrowableUtil [dev.vortex:vortex-jni (classifier=all), org.apache.arrow:arrow-c-data] - org.apache.arrow.c.ArrayStreamExporter$ExportedArrayStreamPrivateData - org.apache.arrow.c.jni.JniWrapper @@ -170,149 +16,12 @@ - io.netty.buffer.PooledByteBufAllocatorL - io.netty.buffer.PooledByteBufAllocatorL$InnerAllocator [dev.vortex:vortex-jni, dev.vortex:vortex-jni (classifier=all)] - - dev.vortex.api.Array - - dev.vortex.api.Files + - dev.vortex.api.DataSource + - dev.vortex.api.Expression - dev.vortex.api.ImmutableScanOptions - - dev.vortex.api.ImmutableScanOptions$Builder - - dev.vortex.api.expressions.Binary - - dev.vortex.api.expressions.GetItem - - dev.vortex.api.expressions.Literal - - dev.vortex.api.expressions.Literal$DecimalLiteral - - dev.vortex.api.proto.DTypes - - dev.vortex.api.proto.EndianUtils - - dev.vortex.api.proto.Expressions - - dev.vortex.api.proto.Scalars - - dev.vortex.api.proto.TemporalMetadatas + - dev.vortex.api.Partition + - dev.vortex.api.Scan + - dev.vortex.api.Session + - dev.vortex.api.VortexWriter - dev.vortex.arrow.ArrowAllocation - - dev.vortex.jni.JNIArray - - dev.vortex.jni.JNIArrayIterator - - dev.vortex.jni.JNIDType - - dev.vortex.jni.JNIFile - dev.vortex.jni.NativeLoader - - dev.vortex.proto.DTypeProtos - - dev.vortex.proto.DTypeProtos$Binary - - dev.vortex.proto.DTypeProtos$Binary$1 - - dev.vortex.proto.DTypeProtos$Binary$Builder - - dev.vortex.proto.DTypeProtos$BinaryOrBuilder - - dev.vortex.proto.DTypeProtos$Bool - - dev.vortex.proto.DTypeProtos$Bool$1 - - dev.vortex.proto.DTypeProtos$Bool$Builder - - dev.vortex.proto.DTypeProtos$BoolOrBuilder - - dev.vortex.proto.DTypeProtos$DType - - dev.vortex.proto.DTypeProtos$DType$1 - - dev.vortex.proto.DTypeProtos$DType$Builder - - dev.vortex.proto.DTypeProtos$DType$DtypeTypeCase - - dev.vortex.proto.DTypeProtos$DTypeOrBuilder - - dev.vortex.proto.DTypeProtos$Decimal - - dev.vortex.proto.DTypeProtos$Decimal$1 - - dev.vortex.proto.DTypeProtos$Decimal$Builder - - dev.vortex.proto.DTypeProtos$DecimalOrBuilder - - dev.vortex.proto.DTypeProtos$Extension - - dev.vortex.proto.DTypeProtos$Extension$1 - - dev.vortex.proto.DTypeProtos$Extension$Builder - - dev.vortex.proto.DTypeProtos$ExtensionOrBuilder - - dev.vortex.proto.DTypeProtos$Field - - dev.vortex.proto.DTypeProtos$Field$1 - - dev.vortex.proto.DTypeProtos$Field$Builder - - dev.vortex.proto.DTypeProtos$Field$FieldTypeCase - - dev.vortex.proto.DTypeProtos$FieldOrBuilder - - dev.vortex.proto.DTypeProtos$FieldPath - - dev.vortex.proto.DTypeProtos$FieldPath$1 - - dev.vortex.proto.DTypeProtos$FieldPath$Builder - - dev.vortex.proto.DTypeProtos$FieldPathOrBuilder - - dev.vortex.proto.DTypeProtos$FixedSizeList - - dev.vortex.proto.DTypeProtos$FixedSizeList$1 - - dev.vortex.proto.DTypeProtos$FixedSizeList$Builder - - dev.vortex.proto.DTypeProtos$FixedSizeListOrBuilder - - dev.vortex.proto.DTypeProtos$List - - dev.vortex.proto.DTypeProtos$List$1 - - dev.vortex.proto.DTypeProtos$List$Builder - - dev.vortex.proto.DTypeProtos$ListOrBuilder - - dev.vortex.proto.DTypeProtos$Null - - dev.vortex.proto.DTypeProtos$Null$1 - - dev.vortex.proto.DTypeProtos$Null$Builder - - dev.vortex.proto.DTypeProtos$NullOrBuilder - - dev.vortex.proto.DTypeProtos$PType - - dev.vortex.proto.DTypeProtos$PType$1 - - dev.vortex.proto.DTypeProtos$Primitive - - dev.vortex.proto.DTypeProtos$Primitive$1 - - dev.vortex.proto.DTypeProtos$Primitive$Builder - - dev.vortex.proto.DTypeProtos$PrimitiveOrBuilder - - dev.vortex.proto.DTypeProtos$Struct - - dev.vortex.proto.DTypeProtos$Struct$1 - - dev.vortex.proto.DTypeProtos$Struct$Builder - - dev.vortex.proto.DTypeProtos$StructOrBuilder - - dev.vortex.proto.DTypeProtos$Utf8 - - dev.vortex.proto.DTypeProtos$Utf8$1 - - dev.vortex.proto.DTypeProtos$Utf8$Builder - - dev.vortex.proto.DTypeProtos$Utf8OrBuilder - - dev.vortex.proto.DTypeProtos$Variant - - dev.vortex.proto.DTypeProtos$Variant$1 - - dev.vortex.proto.DTypeProtos$Variant$Builder - - dev.vortex.proto.DTypeProtos$VariantOrBuilder - - dev.vortex.proto.ExprProtos - - dev.vortex.proto.ExprProtos$AggregateFn - - dev.vortex.proto.ExprProtos$AggregateFn$1 - - dev.vortex.proto.ExprProtos$AggregateFn$Builder - - dev.vortex.proto.ExprProtos$AggregateFnOrBuilder - - dev.vortex.proto.ExprProtos$BetweenOpts - - dev.vortex.proto.ExprProtos$BetweenOpts$1 - - dev.vortex.proto.ExprProtos$BetweenOpts$Builder - - dev.vortex.proto.ExprProtos$BetweenOptsOrBuilder - - dev.vortex.proto.ExprProtos$BinaryOpts - - dev.vortex.proto.ExprProtos$BinaryOpts$1 - - dev.vortex.proto.ExprProtos$BinaryOpts$BinaryOp - - dev.vortex.proto.ExprProtos$BinaryOpts$BinaryOp$1 - - dev.vortex.proto.ExprProtos$BinaryOpts$Builder - - dev.vortex.proto.ExprProtos$BinaryOptsOrBuilder - - dev.vortex.proto.ExprProtos$CaseWhenOpts - - dev.vortex.proto.ExprProtos$CaseWhenOpts$1 - - dev.vortex.proto.ExprProtos$CaseWhenOpts$Builder - - dev.vortex.proto.ExprProtos$CaseWhenOptsOrBuilder - - dev.vortex.proto.ExprProtos$CastOpts - - dev.vortex.proto.ExprProtos$CastOpts$1 - - dev.vortex.proto.ExprProtos$CastOpts$Builder - - dev.vortex.proto.ExprProtos$CastOptsOrBuilder - - dev.vortex.proto.ExprProtos$Expr - - dev.vortex.proto.ExprProtos$Expr$1 - - dev.vortex.proto.ExprProtos$Expr$Builder - - dev.vortex.proto.ExprProtos$ExprOrBuilder - - dev.vortex.proto.ExprProtos$FieldNames - - dev.vortex.proto.ExprProtos$FieldNames$1 - - dev.vortex.proto.ExprProtos$FieldNames$Builder - - dev.vortex.proto.ExprProtos$FieldNamesOrBuilder - - dev.vortex.proto.ExprProtos$GetItemOpts - - dev.vortex.proto.ExprProtos$GetItemOpts$1 - - dev.vortex.proto.ExprProtos$GetItemOpts$Builder - - dev.vortex.proto.ExprProtos$GetItemOptsOrBuilder - - dev.vortex.proto.ExprProtos$LikeOpts - - dev.vortex.proto.ExprProtos$LikeOpts$1 - - dev.vortex.proto.ExprProtos$LikeOpts$Builder - - dev.vortex.proto.ExprProtos$LikeOptsOrBuilder - - dev.vortex.proto.ExprProtos$LiteralOpts - - dev.vortex.proto.ExprProtos$LiteralOpts$1 - - dev.vortex.proto.ExprProtos$LiteralOpts$Builder - - dev.vortex.proto.ExprProtos$LiteralOptsOrBuilder - - dev.vortex.proto.ExprProtos$PackOpts - - dev.vortex.proto.ExprProtos$PackOpts$1 - - dev.vortex.proto.ExprProtos$PackOpts$Builder - - dev.vortex.proto.ExprProtos$PackOptsOrBuilder - - dev.vortex.proto.ExprProtos$SelectOpts - - dev.vortex.proto.ExprProtos$SelectOpts$1 - - dev.vortex.proto.ExprProtos$SelectOpts$Builder - - dev.vortex.proto.ExprProtos$SelectOpts$OptsCase - - dev.vortex.proto.ExprProtos$SelectOptsOrBuilder - - dev.vortex.proto.ScalarProtos - - dev.vortex.proto.ScalarProtos$ListValue - - dev.vortex.proto.ScalarProtos$ListValue$1 - - dev.vortex.proto.ScalarProtos$ListValue$Builder - - dev.vortex.proto.ScalarProtos$ListValueOrBuilder - - dev.vortex.proto.ScalarProtos$Scalar - - dev.vortex.proto.ScalarProtos$Scalar$1 - - dev.vortex.proto.ScalarProtos$Scalar$Builder - - dev.vortex.proto.ScalarProtos$ScalarOrBuilder - - dev.vortex.proto.ScalarProtos$ScalarValue - - dev.vortex.proto.ScalarProtos$ScalarValue$1 - - dev.vortex.proto.ScalarProtos$ScalarValue$Builder - - dev.vortex.proto.ScalarProtos$ScalarValue$KindCase - - dev.vortex.proto.ScalarProtos$ScalarValueOrBuilder diff --git a/spark/v4.0/spark-runtime/runtime-deps.txt b/spark/v4.0/spark-runtime/runtime-deps.txt index ed9c0515f1fc..42367365725c 100644 --- a/spark/v4.0/spark-runtime/runtime-deps.txt +++ b/spark/v4.0/spark-runtime/runtime-deps.txt @@ -3,25 +3,24 @@ com.fasterxml.jackson.core:jackson-core:2.15 com.fasterxml.jackson.core:jackson-databind:2.15 com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.21 com.github.ben-manes.caffeine:caffeine:2.9 -com.google.errorprone:error_prone_annotations:2.41 +com.google.errorprone:error_prone_annotations:2.47 com.google.flatbuffers:flatbuffers-java:25.2 com.google.guava:failureaccess:1.0 -com.google.guava:guava:33.5 +com.google.guava:guava:33.6 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.j2objc:j2objc-annotations:3.1 -com.google.protobuf:protobuf-java:4.33 dev.failsafe:failsafe:3.3 -dev.vortex:vortex-jni:0.67 -dev.vortex:vortex-spark_2.13:0.67 +dev.vortex:vortex-jni:0.72 +dev.vortex:vortex-spark_2.13:0.72 io.airlift:aircompressor:2.0 io.netty:netty-buffer:4.2 io.netty:netty-common:4.2 -org.apache.arrow:arrow-c-data:18.3 -org.apache.arrow:arrow-format:18.3 -org.apache.arrow:arrow-memory-core:18.3 -org.apache.arrow:arrow-memory-netty-buffer-patch:18.3 -org.apache.arrow:arrow-memory-netty:18.3 -org.apache.arrow:arrow-vector:18.3 +org.apache.arrow:arrow-c-data:19.0 +org.apache.arrow:arrow-format:19.0 +org.apache.arrow:arrow-memory-core:19.0 +org.apache.arrow:arrow-memory-netty-buffer-patch:19.0 +org.apache.arrow:arrow-memory-netty:19.0 +org.apache.arrow:arrow-vector:19.0 org.apache.avro:avro:1.12 org.apache.datasketches:datasketches-java:6.2 org.apache.datasketches:datasketches-memory:3.0 diff --git a/spark/v4.0/spark/baseline-class-uniqueness.lock b/spark/v4.0/spark/baseline-class-uniqueness.lock index 72c0c24fb849..53a154c69d97 100644 --- a/spark/v4.0/spark/baseline-class-uniqueness.lock +++ b/spark/v4.0/spark/baseline-class-uniqueness.lock @@ -4,211 +4,9 @@ ## runtimeClasspath [com.google.guava:guava, dev.vortex:vortex-jni (classifier=all)] - com.google.thirdparty.publicsuffix.PublicSuffixPatterns + - com.google.thirdparty.publicsuffix.PublicSuffixTrie + - com.google.thirdparty.publicsuffix.PublicSuffixTrie$ChunksCharSequence - com.google.thirdparty.publicsuffix.PublicSuffixType - - com.google.thirdparty.publicsuffix.TrieParser -[com.google.protobuf:protobuf-java, dev.vortex:vortex-jni] - - com.google.protobuf.BoolValue - - com.google.protobuf.BoolValue$1 - - com.google.protobuf.BoolValue$Builder - - com.google.protobuf.BoolValueOrBuilder - - com.google.protobuf.BytesValue - - com.google.protobuf.BytesValue$1 - - com.google.protobuf.BytesValue$Builder - - com.google.protobuf.BytesValueOrBuilder - - com.google.protobuf.DoubleValue - - com.google.protobuf.DoubleValue$1 - - com.google.protobuf.DoubleValue$Builder - - com.google.protobuf.DoubleValueOrBuilder - - com.google.protobuf.FloatValue - - com.google.protobuf.FloatValue$1 - - com.google.protobuf.FloatValue$Builder - - com.google.protobuf.FloatValueOrBuilder - - com.google.protobuf.Int32Value - - com.google.protobuf.Int32Value$1 - - com.google.protobuf.Int32Value$Builder - - com.google.protobuf.Int32ValueOrBuilder - - com.google.protobuf.Int64Value - - com.google.protobuf.Int64Value$1 - - com.google.protobuf.Int64Value$Builder - - com.google.protobuf.Int64ValueOrBuilder - - com.google.protobuf.ListValue - - com.google.protobuf.ListValue$1 - - com.google.protobuf.ListValue$Builder - - com.google.protobuf.ListValueOrBuilder - - com.google.protobuf.NullValue - - com.google.protobuf.NullValue$1 - - com.google.protobuf.StringValue - - com.google.protobuf.StringValue$1 - - com.google.protobuf.StringValue$Builder - - com.google.protobuf.StringValueOrBuilder - - com.google.protobuf.Struct - - com.google.protobuf.Struct$1 - - com.google.protobuf.Struct$Builder - - com.google.protobuf.Struct$Builder$FieldsConverter - - com.google.protobuf.Struct$FieldsDefaultEntryHolder - - com.google.protobuf.StructOrBuilder - - com.google.protobuf.StructProto - - com.google.protobuf.UInt32Value - - com.google.protobuf.UInt32Value$1 - - com.google.protobuf.UInt32Value$Builder - - com.google.protobuf.UInt32ValueOrBuilder - - com.google.protobuf.UInt64Value - - com.google.protobuf.UInt64Value$1 - - com.google.protobuf.UInt64Value$Builder - - com.google.protobuf.UInt64ValueOrBuilder - - com.google.protobuf.Value - - com.google.protobuf.Value$1 - - com.google.protobuf.Value$Builder - - com.google.protobuf.Value$KindCase - - com.google.protobuf.ValueOrBuilder - - com.google.protobuf.WrappersProto -[commons-codec:commons-codec, dev.vortex:vortex-jni (classifier=all)] - - org.apache.commons.codec.Charsets - - org.apache.commons.codec.binary.Base16 - - org.apache.commons.codec.binary.Base32 - - org.apache.commons.codec.binary.Base64 - - org.apache.commons.codec.binary.BaseNCodec - - org.apache.commons.codec.binary.BaseNCodec$AbstractBuilder - - org.apache.commons.codec.binary.BinaryCodec - - org.apache.commons.codec.binary.StringUtils - - org.apache.commons.codec.digest.DigestUtils - - org.apache.commons.codec.digest.HmacUtils - - org.apache.commons.codec.digest.MurmurHash2 - - org.apache.commons.codec.digest.MurmurHash3 - - org.apache.commons.codec.digest.MurmurHash3$IncrementalHash32 - - org.apache.commons.codec.digest.MurmurHash3$IncrementalHash32x86 - - org.apache.commons.codec.digest.PureJavaCrc32 - - org.apache.commons.codec.digest.PureJavaCrc32C - - org.apache.commons.codec.digest.Sha2Crypt - - org.apache.commons.codec.digest.UnixCrypt - - org.apache.commons.codec.language.Caverphone1 - - org.apache.commons.codec.language.Caverphone2 - - org.apache.commons.codec.language.DaitchMokotoffSoundex - - org.apache.commons.codec.language.DaitchMokotoffSoundex$Branch - - org.apache.commons.codec.language.DaitchMokotoffSoundex$Rule - - org.apache.commons.codec.language.DoubleMetaphone - - org.apache.commons.codec.language.DoubleMetaphone$DoubleMetaphoneResult - - org.apache.commons.codec.language.MatchRatingApproachEncoder - - org.apache.commons.codec.language.Metaphone - - org.apache.commons.codec.language.Nysiis - - org.apache.commons.codec.language.RefinedSoundex - - org.apache.commons.codec.language.Soundex - - org.apache.commons.codec.language.SoundexUtils - - org.apache.commons.codec.language.bm.Lang - - org.apache.commons.codec.language.bm.PhoneticEngine - - org.apache.commons.codec.language.bm.PhoneticEngine$1 - - org.apache.commons.codec.language.bm.PhoneticEngine$PhonemeBuilder - - org.apache.commons.codec.language.bm.PhoneticEngine$RulesApplication - - org.apache.commons.codec.language.bm.ResourceConstants - - org.apache.commons.codec.language.bm.Rule - - org.apache.commons.codec.language.bm.Rule$1 - - org.apache.commons.codec.language.bm.Rule$2 - - org.apache.commons.codec.language.bm.Rule$Phoneme - - org.apache.commons.codec.language.bm.Rule$PhonemeExpr - - org.apache.commons.codec.language.bm.Rule$PhonemeList - - org.apache.commons.codec.net.PercentCodec - - org.apache.commons.codec.net.QuotedPrintableCodec - - org.apache.commons.codec.net.URLCodec - - org.apache.commons.codec.net.Utils -[dev.vortex:vortex-jni (classifier=all), io.netty:netty-buffer] - - io.netty.buffer.AbstractByteBuf - - io.netty.buffer.AbstractReferenceCountedByteBuf - - io.netty.buffer.AdaptiveByteBufAllocator$DirectChunkAllocator - - io.netty.buffer.AdaptivePoolingAllocator - - io.netty.buffer.AdaptivePoolingAllocator$1 - - io.netty.buffer.AdaptivePoolingAllocator$AdaptiveByteBuf - - io.netty.buffer.AdaptivePoolingAllocator$Chunk - - io.netty.buffer.AdaptivePoolingAllocator$ChunkController - - io.netty.buffer.AdaptivePoolingAllocator$ChunkRegistry - - io.netty.buffer.AdaptivePoolingAllocator$Magazine - - io.netty.buffer.AdaptivePoolingAllocator$MagazineGroup - - io.netty.buffer.AdaptivePoolingAllocator$SizeClassChunkController - - io.netty.buffer.AdaptivePoolingAllocator$SizeClassedChunk - - io.netty.buffer.ByteBufUtil - - io.netty.buffer.CompositeByteBuf - - io.netty.buffer.EmptyByteBuf - - io.netty.buffer.PoolArena - - io.netty.buffer.PoolArena$DirectArena - - io.netty.buffer.PoolThreadCache$FreeOnFinalize - - io.netty.buffer.PooledByteBufAllocator - - io.netty.buffer.PooledByteBufAllocator$PoolThreadLocalCache - - io.netty.buffer.ReadOnlyAbstractByteBuf - - io.netty.buffer.SimpleLeakAwareByteBuf - - io.netty.buffer.Unpooled - - io.netty.buffer.UnpooledByteBufAllocator$DecrementingCleanableDirectBuffer - - io.netty.buffer.UnpooledByteBufAllocator$InstrumentedUnpooledDirectByteBuf - - io.netty.buffer.UnpooledByteBufAllocator$InstrumentedUnpooledUnsafeDirectByteBuf - - io.netty.buffer.UnpooledByteBufAllocator$InstrumentedUnpooledUnsafeNoCleanerDirectByteBuf - - io.netty.buffer.UnpooledByteBufAllocator$UnpooledByteBufAllocatorMetric - - io.netty.buffer.UnpooledDirectByteBuf - - io.netty.buffer.UnpooledHeapByteBuf - - io.netty.buffer.UnpooledUnsafeDirectByteBuf - - io.netty.buffer.UnpooledUnsafeNoCleanerDirectByteBuf - - io.netty.buffer.UnsafeByteBufUtil -[dev.vortex:vortex-jni (classifier=all), io.netty:netty-common] - - io.netty.util.AbstractReferenceCounted - - io.netty.util.DefaultAttributeMap - - io.netty.util.HashedWheelTimer - - io.netty.util.HashedWheelTimer$HashedWheelBucket - - io.netty.util.LeakPresenceDetector - - io.netty.util.LeakPresenceDetector$LeakCreation - - io.netty.util.LeakPresenceDetector$ResourceScope - - io.netty.util.Recycler - - io.netty.util.Recycler$BlockingMessageQueue - - io.netty.util.Recycler$DefaultHandle - - io.netty.util.Recycler$EnhancedHandle - - io.netty.util.Recycler$GuardedLocalPool - - io.netty.util.Recycler$LocalPool - - io.netty.util.Recycler$UnguardedLocalPool - - io.netty.util.concurrent.AbstractScheduledEventExecutor - - io.netty.util.concurrent.GlobalEventExecutor - - io.netty.util.concurrent.GlobalEventExecutor$2 - - io.netty.util.concurrent.GlobalEventExecutor$3 - - io.netty.util.concurrent.GlobalEventExecutor$TaskRunner - - io.netty.util.concurrent.MpscIntQueue$MpscAtomicIntegerArrayQueue - - io.netty.util.concurrent.MultithreadEventExecutorGroup - - io.netty.util.concurrent.NonStickyEventExecutorGroup$NonStickyOrderedEventExecutor - - io.netty.util.concurrent.PromiseNotifier - - io.netty.util.concurrent.PromiseNotifier$1 - - io.netty.util.concurrent.SingleThreadEventExecutor - - io.netty.util.concurrent.SingleThreadEventExecutor$4 - - io.netty.util.concurrent.SingleThreadEventExecutor$5 - - io.netty.util.concurrent.SingleThreadEventExecutor$DefaultThreadProperties - - io.netty.util.internal.Cleaner - - io.netty.util.internal.CleanerJava24Linker - - io.netty.util.internal.CleanerJava24Linker$CleanableDirectBufferImpl - - io.netty.util.internal.CleanerJava25 - - io.netty.util.internal.CleanerJava25$CleanableDirectBufferImpl - - io.netty.util.internal.CleanerJava6 - - io.netty.util.internal.CleanerJava6$2 - - io.netty.util.internal.CleanerJava6$CleanableDirectBufferImpl - - io.netty.util.internal.CleanerJava9 - - io.netty.util.internal.CleanerJava9$2 - - io.netty.util.internal.CleanerJava9$CleanableDirectBufferImpl - - io.netty.util.internal.DirectCleaner - - io.netty.util.internal.DirectCleaner$CleanableDirectBufferImpl - - io.netty.util.internal.EmptyArrays - - io.netty.util.internal.PlatformDependent - - io.netty.util.internal.PlatformDependent$1 - - io.netty.util.internal.PlatformDependent$1$1 - - io.netty.util.internal.PlatformDependent$Mpsc - - io.netty.util.internal.PlatformDependent$Mpsc$1 - - io.netty.util.internal.PlatformDependent0 - - io.netty.util.internal.PlatformDependent0$1 - - io.netty.util.internal.PlatformDependent0$10 - - io.netty.util.internal.PlatformDependent0$11 - - io.netty.util.internal.PlatformDependent0$12 - - io.netty.util.internal.PlatformDependent0$2 - - io.netty.util.internal.PlatformDependent0$3 - - io.netty.util.internal.PlatformDependent0$4 - - io.netty.util.internal.PlatformDependent0$5 - - io.netty.util.internal.PlatformDependent0$6 - - io.netty.util.internal.PlatformDependent0$7 - - io.netty.util.internal.PlatformDependent0$8 - - io.netty.util.internal.PlatformDependent0$9 - - io.netty.util.internal.ReferenceCountUpdater - - io.netty.util.internal.ReferenceCountUpdater$UpdaterType - - io.netty.util.internal.ThrowableUtil [dev.vortex:vortex-jni (classifier=all), org.apache.arrow:arrow-c-data] - org.apache.arrow.c.ArrayStreamExporter$ExportedArrayStreamPrivateData - org.apache.arrow.c.jni.JniWrapper @@ -218,149 +16,12 @@ - io.netty.buffer.PooledByteBufAllocatorL - io.netty.buffer.PooledByteBufAllocatorL$InnerAllocator [dev.vortex:vortex-jni, dev.vortex:vortex-jni (classifier=all)] - - dev.vortex.api.Array - - dev.vortex.api.Files + - dev.vortex.api.DataSource + - dev.vortex.api.Expression - dev.vortex.api.ImmutableScanOptions - - dev.vortex.api.ImmutableScanOptions$Builder - - dev.vortex.api.expressions.Binary - - dev.vortex.api.expressions.GetItem - - dev.vortex.api.expressions.Literal - - dev.vortex.api.expressions.Literal$DecimalLiteral - - dev.vortex.api.proto.DTypes - - dev.vortex.api.proto.EndianUtils - - dev.vortex.api.proto.Expressions - - dev.vortex.api.proto.Scalars - - dev.vortex.api.proto.TemporalMetadatas + - dev.vortex.api.Partition + - dev.vortex.api.Scan + - dev.vortex.api.Session + - dev.vortex.api.VortexWriter - dev.vortex.arrow.ArrowAllocation - - dev.vortex.jni.JNIArray - - dev.vortex.jni.JNIArrayIterator - - dev.vortex.jni.JNIDType - - dev.vortex.jni.JNIFile - dev.vortex.jni.NativeLoader - - dev.vortex.proto.DTypeProtos - - dev.vortex.proto.DTypeProtos$Binary - - dev.vortex.proto.DTypeProtos$Binary$1 - - dev.vortex.proto.DTypeProtos$Binary$Builder - - dev.vortex.proto.DTypeProtos$BinaryOrBuilder - - dev.vortex.proto.DTypeProtos$Bool - - dev.vortex.proto.DTypeProtos$Bool$1 - - dev.vortex.proto.DTypeProtos$Bool$Builder - - dev.vortex.proto.DTypeProtos$BoolOrBuilder - - dev.vortex.proto.DTypeProtos$DType - - dev.vortex.proto.DTypeProtos$DType$1 - - dev.vortex.proto.DTypeProtos$DType$Builder - - dev.vortex.proto.DTypeProtos$DType$DtypeTypeCase - - dev.vortex.proto.DTypeProtos$DTypeOrBuilder - - dev.vortex.proto.DTypeProtos$Decimal - - dev.vortex.proto.DTypeProtos$Decimal$1 - - dev.vortex.proto.DTypeProtos$Decimal$Builder - - dev.vortex.proto.DTypeProtos$DecimalOrBuilder - - dev.vortex.proto.DTypeProtos$Extension - - dev.vortex.proto.DTypeProtos$Extension$1 - - dev.vortex.proto.DTypeProtos$Extension$Builder - - dev.vortex.proto.DTypeProtos$ExtensionOrBuilder - - dev.vortex.proto.DTypeProtos$Field - - dev.vortex.proto.DTypeProtos$Field$1 - - dev.vortex.proto.DTypeProtos$Field$Builder - - dev.vortex.proto.DTypeProtos$Field$FieldTypeCase - - dev.vortex.proto.DTypeProtos$FieldOrBuilder - - dev.vortex.proto.DTypeProtos$FieldPath - - dev.vortex.proto.DTypeProtos$FieldPath$1 - - dev.vortex.proto.DTypeProtos$FieldPath$Builder - - dev.vortex.proto.DTypeProtos$FieldPathOrBuilder - - dev.vortex.proto.DTypeProtos$FixedSizeList - - dev.vortex.proto.DTypeProtos$FixedSizeList$1 - - dev.vortex.proto.DTypeProtos$FixedSizeList$Builder - - dev.vortex.proto.DTypeProtos$FixedSizeListOrBuilder - - dev.vortex.proto.DTypeProtos$List - - dev.vortex.proto.DTypeProtos$List$1 - - dev.vortex.proto.DTypeProtos$List$Builder - - dev.vortex.proto.DTypeProtos$ListOrBuilder - - dev.vortex.proto.DTypeProtos$Null - - dev.vortex.proto.DTypeProtos$Null$1 - - dev.vortex.proto.DTypeProtos$Null$Builder - - dev.vortex.proto.DTypeProtos$NullOrBuilder - - dev.vortex.proto.DTypeProtos$PType - - dev.vortex.proto.DTypeProtos$PType$1 - - dev.vortex.proto.DTypeProtos$Primitive - - dev.vortex.proto.DTypeProtos$Primitive$1 - - dev.vortex.proto.DTypeProtos$Primitive$Builder - - dev.vortex.proto.DTypeProtos$PrimitiveOrBuilder - - dev.vortex.proto.DTypeProtos$Struct - - dev.vortex.proto.DTypeProtos$Struct$1 - - dev.vortex.proto.DTypeProtos$Struct$Builder - - dev.vortex.proto.DTypeProtos$StructOrBuilder - - dev.vortex.proto.DTypeProtos$Utf8 - - dev.vortex.proto.DTypeProtos$Utf8$1 - - dev.vortex.proto.DTypeProtos$Utf8$Builder - - dev.vortex.proto.DTypeProtos$Utf8OrBuilder - - dev.vortex.proto.DTypeProtos$Variant - - dev.vortex.proto.DTypeProtos$Variant$1 - - dev.vortex.proto.DTypeProtos$Variant$Builder - - dev.vortex.proto.DTypeProtos$VariantOrBuilder - - dev.vortex.proto.ExprProtos - - dev.vortex.proto.ExprProtos$AggregateFn - - dev.vortex.proto.ExprProtos$AggregateFn$1 - - dev.vortex.proto.ExprProtos$AggregateFn$Builder - - dev.vortex.proto.ExprProtos$AggregateFnOrBuilder - - dev.vortex.proto.ExprProtos$BetweenOpts - - dev.vortex.proto.ExprProtos$BetweenOpts$1 - - dev.vortex.proto.ExprProtos$BetweenOpts$Builder - - dev.vortex.proto.ExprProtos$BetweenOptsOrBuilder - - dev.vortex.proto.ExprProtos$BinaryOpts - - dev.vortex.proto.ExprProtos$BinaryOpts$1 - - dev.vortex.proto.ExprProtos$BinaryOpts$BinaryOp - - dev.vortex.proto.ExprProtos$BinaryOpts$BinaryOp$1 - - dev.vortex.proto.ExprProtos$BinaryOpts$Builder - - dev.vortex.proto.ExprProtos$BinaryOptsOrBuilder - - dev.vortex.proto.ExprProtos$CaseWhenOpts - - dev.vortex.proto.ExprProtos$CaseWhenOpts$1 - - dev.vortex.proto.ExprProtos$CaseWhenOpts$Builder - - dev.vortex.proto.ExprProtos$CaseWhenOptsOrBuilder - - dev.vortex.proto.ExprProtos$CastOpts - - dev.vortex.proto.ExprProtos$CastOpts$1 - - dev.vortex.proto.ExprProtos$CastOpts$Builder - - dev.vortex.proto.ExprProtos$CastOptsOrBuilder - - dev.vortex.proto.ExprProtos$Expr - - dev.vortex.proto.ExprProtos$Expr$1 - - dev.vortex.proto.ExprProtos$Expr$Builder - - dev.vortex.proto.ExprProtos$ExprOrBuilder - - dev.vortex.proto.ExprProtos$FieldNames - - dev.vortex.proto.ExprProtos$FieldNames$1 - - dev.vortex.proto.ExprProtos$FieldNames$Builder - - dev.vortex.proto.ExprProtos$FieldNamesOrBuilder - - dev.vortex.proto.ExprProtos$GetItemOpts - - dev.vortex.proto.ExprProtos$GetItemOpts$1 - - dev.vortex.proto.ExprProtos$GetItemOpts$Builder - - dev.vortex.proto.ExprProtos$GetItemOptsOrBuilder - - dev.vortex.proto.ExprProtos$LikeOpts - - dev.vortex.proto.ExprProtos$LikeOpts$1 - - dev.vortex.proto.ExprProtos$LikeOpts$Builder - - dev.vortex.proto.ExprProtos$LikeOptsOrBuilder - - dev.vortex.proto.ExprProtos$LiteralOpts - - dev.vortex.proto.ExprProtos$LiteralOpts$1 - - dev.vortex.proto.ExprProtos$LiteralOpts$Builder - - dev.vortex.proto.ExprProtos$LiteralOptsOrBuilder - - dev.vortex.proto.ExprProtos$PackOpts - - dev.vortex.proto.ExprProtos$PackOpts$1 - - dev.vortex.proto.ExprProtos$PackOpts$Builder - - dev.vortex.proto.ExprProtos$PackOptsOrBuilder - - dev.vortex.proto.ExprProtos$SelectOpts - - dev.vortex.proto.ExprProtos$SelectOpts$1 - - dev.vortex.proto.ExprProtos$SelectOpts$Builder - - dev.vortex.proto.ExprProtos$SelectOpts$OptsCase - - dev.vortex.proto.ExprProtos$SelectOptsOrBuilder - - dev.vortex.proto.ScalarProtos - - dev.vortex.proto.ScalarProtos$ListValue - - dev.vortex.proto.ScalarProtos$ListValue$1 - - dev.vortex.proto.ScalarProtos$ListValue$Builder - - dev.vortex.proto.ScalarProtos$ListValueOrBuilder - - dev.vortex.proto.ScalarProtos$Scalar - - dev.vortex.proto.ScalarProtos$Scalar$1 - - dev.vortex.proto.ScalarProtos$Scalar$Builder - - dev.vortex.proto.ScalarProtos$ScalarOrBuilder - - dev.vortex.proto.ScalarProtos$ScalarValue - - dev.vortex.proto.ScalarProtos$ScalarValue$1 - - dev.vortex.proto.ScalarProtos$ScalarValue$Builder - - dev.vortex.proto.ScalarProtos$ScalarValue$KindCase - - dev.vortex.proto.ScalarProtos$ScalarValueOrBuilder diff --git a/spark/v4.1/spark-runtime/baseline-class-uniqueness.lock b/spark/v4.1/spark-runtime/baseline-class-uniqueness.lock index 6197975f3900..53a154c69d97 100644 --- a/spark/v4.1/spark-runtime/baseline-class-uniqueness.lock +++ b/spark/v4.1/spark-runtime/baseline-class-uniqueness.lock @@ -4,163 +4,9 @@ ## runtimeClasspath [com.google.guava:guava, dev.vortex:vortex-jni (classifier=all)] - com.google.thirdparty.publicsuffix.PublicSuffixPatterns + - com.google.thirdparty.publicsuffix.PublicSuffixTrie + - com.google.thirdparty.publicsuffix.PublicSuffixTrie$ChunksCharSequence - com.google.thirdparty.publicsuffix.PublicSuffixType - - com.google.thirdparty.publicsuffix.TrieParser -[com.google.protobuf:protobuf-java, dev.vortex:vortex-jni] - - com.google.protobuf.BoolValue - - com.google.protobuf.BoolValue$1 - - com.google.protobuf.BoolValue$Builder - - com.google.protobuf.BoolValueOrBuilder - - com.google.protobuf.BytesValue - - com.google.protobuf.BytesValue$1 - - com.google.protobuf.BytesValue$Builder - - com.google.protobuf.BytesValueOrBuilder - - com.google.protobuf.DoubleValue - - com.google.protobuf.DoubleValue$1 - - com.google.protobuf.DoubleValue$Builder - - com.google.protobuf.DoubleValueOrBuilder - - com.google.protobuf.FloatValue - - com.google.protobuf.FloatValue$1 - - com.google.protobuf.FloatValue$Builder - - com.google.protobuf.FloatValueOrBuilder - - com.google.protobuf.Int32Value - - com.google.protobuf.Int32Value$1 - - com.google.protobuf.Int32Value$Builder - - com.google.protobuf.Int32ValueOrBuilder - - com.google.protobuf.Int64Value - - com.google.protobuf.Int64Value$1 - - com.google.protobuf.Int64Value$Builder - - com.google.protobuf.Int64ValueOrBuilder - - com.google.protobuf.ListValue - - com.google.protobuf.ListValue$1 - - com.google.protobuf.ListValue$Builder - - com.google.protobuf.ListValueOrBuilder - - com.google.protobuf.NullValue - - com.google.protobuf.NullValue$1 - - com.google.protobuf.StringValue - - com.google.protobuf.StringValue$1 - - com.google.protobuf.StringValue$Builder - - com.google.protobuf.StringValueOrBuilder - - com.google.protobuf.Struct - - com.google.protobuf.Struct$1 - - com.google.protobuf.Struct$Builder - - com.google.protobuf.Struct$Builder$FieldsConverter - - com.google.protobuf.Struct$FieldsDefaultEntryHolder - - com.google.protobuf.StructOrBuilder - - com.google.protobuf.StructProto - - com.google.protobuf.UInt32Value - - com.google.protobuf.UInt32Value$1 - - com.google.protobuf.UInt32Value$Builder - - com.google.protobuf.UInt32ValueOrBuilder - - com.google.protobuf.UInt64Value - - com.google.protobuf.UInt64Value$1 - - com.google.protobuf.UInt64Value$Builder - - com.google.protobuf.UInt64ValueOrBuilder - - com.google.protobuf.Value - - com.google.protobuf.Value$1 - - com.google.protobuf.Value$Builder - - com.google.protobuf.Value$KindCase - - com.google.protobuf.ValueOrBuilder - - com.google.protobuf.WrappersProto -[dev.vortex:vortex-jni (classifier=all), io.netty:netty-buffer] - - io.netty.buffer.AbstractByteBuf - - io.netty.buffer.AbstractReferenceCountedByteBuf - - io.netty.buffer.AdaptiveByteBufAllocator$DirectChunkAllocator - - io.netty.buffer.AdaptivePoolingAllocator - - io.netty.buffer.AdaptivePoolingAllocator$1 - - io.netty.buffer.AdaptivePoolingAllocator$AdaptiveByteBuf - - io.netty.buffer.AdaptivePoolingAllocator$Chunk - - io.netty.buffer.AdaptivePoolingAllocator$ChunkController - - io.netty.buffer.AdaptivePoolingAllocator$ChunkRegistry - - io.netty.buffer.AdaptivePoolingAllocator$Magazine - - io.netty.buffer.AdaptivePoolingAllocator$MagazineGroup - - io.netty.buffer.AdaptivePoolingAllocator$SizeClassChunkController - - io.netty.buffer.AdaptivePoolingAllocator$SizeClassedChunk - - io.netty.buffer.ByteBufUtil - - io.netty.buffer.CompositeByteBuf - - io.netty.buffer.EmptyByteBuf - - io.netty.buffer.PoolArena - - io.netty.buffer.PoolArena$DirectArena - - io.netty.buffer.PoolThreadCache$FreeOnFinalize - - io.netty.buffer.PooledByteBufAllocator - - io.netty.buffer.PooledByteBufAllocator$PoolThreadLocalCache - - io.netty.buffer.ReadOnlyAbstractByteBuf - - io.netty.buffer.SimpleLeakAwareByteBuf - - io.netty.buffer.Unpooled - - io.netty.buffer.UnpooledByteBufAllocator$DecrementingCleanableDirectBuffer - - io.netty.buffer.UnpooledByteBufAllocator$InstrumentedUnpooledDirectByteBuf - - io.netty.buffer.UnpooledByteBufAllocator$InstrumentedUnpooledUnsafeDirectByteBuf - - io.netty.buffer.UnpooledByteBufAllocator$InstrumentedUnpooledUnsafeNoCleanerDirectByteBuf - - io.netty.buffer.UnpooledByteBufAllocator$UnpooledByteBufAllocatorMetric - - io.netty.buffer.UnpooledDirectByteBuf - - io.netty.buffer.UnpooledHeapByteBuf - - io.netty.buffer.UnpooledUnsafeDirectByteBuf - - io.netty.buffer.UnpooledUnsafeNoCleanerDirectByteBuf - - io.netty.buffer.UnsafeByteBufUtil -[dev.vortex:vortex-jni (classifier=all), io.netty:netty-common] - - io.netty.util.AbstractReferenceCounted - - io.netty.util.DefaultAttributeMap - - io.netty.util.HashedWheelTimer - - io.netty.util.HashedWheelTimer$HashedWheelBucket - - io.netty.util.LeakPresenceDetector - - io.netty.util.LeakPresenceDetector$LeakCreation - - io.netty.util.LeakPresenceDetector$ResourceScope - - io.netty.util.Recycler - - io.netty.util.Recycler$BlockingMessageQueue - - io.netty.util.Recycler$DefaultHandle - - io.netty.util.Recycler$EnhancedHandle - - io.netty.util.Recycler$GuardedLocalPool - - io.netty.util.Recycler$LocalPool - - io.netty.util.Recycler$UnguardedLocalPool - - io.netty.util.concurrent.AbstractScheduledEventExecutor - - io.netty.util.concurrent.GlobalEventExecutor - - io.netty.util.concurrent.GlobalEventExecutor$2 - - io.netty.util.concurrent.GlobalEventExecutor$3 - - io.netty.util.concurrent.GlobalEventExecutor$TaskRunner - - io.netty.util.concurrent.MpscIntQueue$MpscAtomicIntegerArrayQueue - - io.netty.util.concurrent.MultithreadEventExecutorGroup - - io.netty.util.concurrent.NonStickyEventExecutorGroup$NonStickyOrderedEventExecutor - - io.netty.util.concurrent.PromiseNotifier - - io.netty.util.concurrent.PromiseNotifier$1 - - io.netty.util.concurrent.SingleThreadEventExecutor - - io.netty.util.concurrent.SingleThreadEventExecutor$4 - - io.netty.util.concurrent.SingleThreadEventExecutor$5 - - io.netty.util.concurrent.SingleThreadEventExecutor$DefaultThreadProperties - - io.netty.util.internal.Cleaner - - io.netty.util.internal.CleanerJava24Linker - - io.netty.util.internal.CleanerJava24Linker$CleanableDirectBufferImpl - - io.netty.util.internal.CleanerJava25 - - io.netty.util.internal.CleanerJava25$CleanableDirectBufferImpl - - io.netty.util.internal.CleanerJava6 - - io.netty.util.internal.CleanerJava6$2 - - io.netty.util.internal.CleanerJava6$CleanableDirectBufferImpl - - io.netty.util.internal.CleanerJava9 - - io.netty.util.internal.CleanerJava9$2 - - io.netty.util.internal.CleanerJava9$CleanableDirectBufferImpl - - io.netty.util.internal.DirectCleaner - - io.netty.util.internal.DirectCleaner$CleanableDirectBufferImpl - - io.netty.util.internal.EmptyArrays - - io.netty.util.internal.PlatformDependent - - io.netty.util.internal.PlatformDependent$1 - - io.netty.util.internal.PlatformDependent$1$1 - - io.netty.util.internal.PlatformDependent$Mpsc - - io.netty.util.internal.PlatformDependent$Mpsc$1 - - io.netty.util.internal.PlatformDependent0 - - io.netty.util.internal.PlatformDependent0$1 - - io.netty.util.internal.PlatformDependent0$10 - - io.netty.util.internal.PlatformDependent0$11 - - io.netty.util.internal.PlatformDependent0$12 - - io.netty.util.internal.PlatformDependent0$2 - - io.netty.util.internal.PlatformDependent0$3 - - io.netty.util.internal.PlatformDependent0$4 - - io.netty.util.internal.PlatformDependent0$5 - - io.netty.util.internal.PlatformDependent0$6 - - io.netty.util.internal.PlatformDependent0$7 - - io.netty.util.internal.PlatformDependent0$8 - - io.netty.util.internal.PlatformDependent0$9 - - io.netty.util.internal.ReferenceCountUpdater - - io.netty.util.internal.ReferenceCountUpdater$UpdaterType - - io.netty.util.internal.ThrowableUtil [dev.vortex:vortex-jni (classifier=all), org.apache.arrow:arrow-c-data] - org.apache.arrow.c.ArrayStreamExporter$ExportedArrayStreamPrivateData - org.apache.arrow.c.jni.JniWrapper @@ -170,149 +16,12 @@ - io.netty.buffer.PooledByteBufAllocatorL - io.netty.buffer.PooledByteBufAllocatorL$InnerAllocator [dev.vortex:vortex-jni, dev.vortex:vortex-jni (classifier=all)] - - dev.vortex.api.Array - - dev.vortex.api.Files + - dev.vortex.api.DataSource + - dev.vortex.api.Expression - dev.vortex.api.ImmutableScanOptions - - dev.vortex.api.ImmutableScanOptions$Builder - - dev.vortex.api.expressions.Binary - - dev.vortex.api.expressions.GetItem - - dev.vortex.api.expressions.Literal - - dev.vortex.api.expressions.Literal$DecimalLiteral - - dev.vortex.api.proto.DTypes - - dev.vortex.api.proto.EndianUtils - - dev.vortex.api.proto.Expressions - - dev.vortex.api.proto.Scalars - - dev.vortex.api.proto.TemporalMetadatas + - dev.vortex.api.Partition + - dev.vortex.api.Scan + - dev.vortex.api.Session + - dev.vortex.api.VortexWriter - dev.vortex.arrow.ArrowAllocation - - dev.vortex.jni.JNIArray - - dev.vortex.jni.JNIArrayIterator - - dev.vortex.jni.JNIDType - - dev.vortex.jni.JNIFile - dev.vortex.jni.NativeLoader - - dev.vortex.proto.DTypeProtos - - dev.vortex.proto.DTypeProtos$Binary - - dev.vortex.proto.DTypeProtos$Binary$1 - - dev.vortex.proto.DTypeProtos$Binary$Builder - - dev.vortex.proto.DTypeProtos$BinaryOrBuilder - - dev.vortex.proto.DTypeProtos$Bool - - dev.vortex.proto.DTypeProtos$Bool$1 - - dev.vortex.proto.DTypeProtos$Bool$Builder - - dev.vortex.proto.DTypeProtos$BoolOrBuilder - - dev.vortex.proto.DTypeProtos$DType - - dev.vortex.proto.DTypeProtos$DType$1 - - dev.vortex.proto.DTypeProtos$DType$Builder - - dev.vortex.proto.DTypeProtos$DType$DtypeTypeCase - - dev.vortex.proto.DTypeProtos$DTypeOrBuilder - - dev.vortex.proto.DTypeProtos$Decimal - - dev.vortex.proto.DTypeProtos$Decimal$1 - - dev.vortex.proto.DTypeProtos$Decimal$Builder - - dev.vortex.proto.DTypeProtos$DecimalOrBuilder - - dev.vortex.proto.DTypeProtos$Extension - - dev.vortex.proto.DTypeProtos$Extension$1 - - dev.vortex.proto.DTypeProtos$Extension$Builder - - dev.vortex.proto.DTypeProtos$ExtensionOrBuilder - - dev.vortex.proto.DTypeProtos$Field - - dev.vortex.proto.DTypeProtos$Field$1 - - dev.vortex.proto.DTypeProtos$Field$Builder - - dev.vortex.proto.DTypeProtos$Field$FieldTypeCase - - dev.vortex.proto.DTypeProtos$FieldOrBuilder - - dev.vortex.proto.DTypeProtos$FieldPath - - dev.vortex.proto.DTypeProtos$FieldPath$1 - - dev.vortex.proto.DTypeProtos$FieldPath$Builder - - dev.vortex.proto.DTypeProtos$FieldPathOrBuilder - - dev.vortex.proto.DTypeProtos$FixedSizeList - - dev.vortex.proto.DTypeProtos$FixedSizeList$1 - - dev.vortex.proto.DTypeProtos$FixedSizeList$Builder - - dev.vortex.proto.DTypeProtos$FixedSizeListOrBuilder - - dev.vortex.proto.DTypeProtos$List - - dev.vortex.proto.DTypeProtos$List$1 - - dev.vortex.proto.DTypeProtos$List$Builder - - dev.vortex.proto.DTypeProtos$ListOrBuilder - - dev.vortex.proto.DTypeProtos$Null - - dev.vortex.proto.DTypeProtos$Null$1 - - dev.vortex.proto.DTypeProtos$Null$Builder - - dev.vortex.proto.DTypeProtos$NullOrBuilder - - dev.vortex.proto.DTypeProtos$PType - - dev.vortex.proto.DTypeProtos$PType$1 - - dev.vortex.proto.DTypeProtos$Primitive - - dev.vortex.proto.DTypeProtos$Primitive$1 - - dev.vortex.proto.DTypeProtos$Primitive$Builder - - dev.vortex.proto.DTypeProtos$PrimitiveOrBuilder - - dev.vortex.proto.DTypeProtos$Struct - - dev.vortex.proto.DTypeProtos$Struct$1 - - dev.vortex.proto.DTypeProtos$Struct$Builder - - dev.vortex.proto.DTypeProtos$StructOrBuilder - - dev.vortex.proto.DTypeProtos$Utf8 - - dev.vortex.proto.DTypeProtos$Utf8$1 - - dev.vortex.proto.DTypeProtos$Utf8$Builder - - dev.vortex.proto.DTypeProtos$Utf8OrBuilder - - dev.vortex.proto.DTypeProtos$Variant - - dev.vortex.proto.DTypeProtos$Variant$1 - - dev.vortex.proto.DTypeProtos$Variant$Builder - - dev.vortex.proto.DTypeProtos$VariantOrBuilder - - dev.vortex.proto.ExprProtos - - dev.vortex.proto.ExprProtos$AggregateFn - - dev.vortex.proto.ExprProtos$AggregateFn$1 - - dev.vortex.proto.ExprProtos$AggregateFn$Builder - - dev.vortex.proto.ExprProtos$AggregateFnOrBuilder - - dev.vortex.proto.ExprProtos$BetweenOpts - - dev.vortex.proto.ExprProtos$BetweenOpts$1 - - dev.vortex.proto.ExprProtos$BetweenOpts$Builder - - dev.vortex.proto.ExprProtos$BetweenOptsOrBuilder - - dev.vortex.proto.ExprProtos$BinaryOpts - - dev.vortex.proto.ExprProtos$BinaryOpts$1 - - dev.vortex.proto.ExprProtos$BinaryOpts$BinaryOp - - dev.vortex.proto.ExprProtos$BinaryOpts$BinaryOp$1 - - dev.vortex.proto.ExprProtos$BinaryOpts$Builder - - dev.vortex.proto.ExprProtos$BinaryOptsOrBuilder - - dev.vortex.proto.ExprProtos$CaseWhenOpts - - dev.vortex.proto.ExprProtos$CaseWhenOpts$1 - - dev.vortex.proto.ExprProtos$CaseWhenOpts$Builder - - dev.vortex.proto.ExprProtos$CaseWhenOptsOrBuilder - - dev.vortex.proto.ExprProtos$CastOpts - - dev.vortex.proto.ExprProtos$CastOpts$1 - - dev.vortex.proto.ExprProtos$CastOpts$Builder - - dev.vortex.proto.ExprProtos$CastOptsOrBuilder - - dev.vortex.proto.ExprProtos$Expr - - dev.vortex.proto.ExprProtos$Expr$1 - - dev.vortex.proto.ExprProtos$Expr$Builder - - dev.vortex.proto.ExprProtos$ExprOrBuilder - - dev.vortex.proto.ExprProtos$FieldNames - - dev.vortex.proto.ExprProtos$FieldNames$1 - - dev.vortex.proto.ExprProtos$FieldNames$Builder - - dev.vortex.proto.ExprProtos$FieldNamesOrBuilder - - dev.vortex.proto.ExprProtos$GetItemOpts - - dev.vortex.proto.ExprProtos$GetItemOpts$1 - - dev.vortex.proto.ExprProtos$GetItemOpts$Builder - - dev.vortex.proto.ExprProtos$GetItemOptsOrBuilder - - dev.vortex.proto.ExprProtos$LikeOpts - - dev.vortex.proto.ExprProtos$LikeOpts$1 - - dev.vortex.proto.ExprProtos$LikeOpts$Builder - - dev.vortex.proto.ExprProtos$LikeOptsOrBuilder - - dev.vortex.proto.ExprProtos$LiteralOpts - - dev.vortex.proto.ExprProtos$LiteralOpts$1 - - dev.vortex.proto.ExprProtos$LiteralOpts$Builder - - dev.vortex.proto.ExprProtos$LiteralOptsOrBuilder - - dev.vortex.proto.ExprProtos$PackOpts - - dev.vortex.proto.ExprProtos$PackOpts$1 - - dev.vortex.proto.ExprProtos$PackOpts$Builder - - dev.vortex.proto.ExprProtos$PackOptsOrBuilder - - dev.vortex.proto.ExprProtos$SelectOpts - - dev.vortex.proto.ExprProtos$SelectOpts$1 - - dev.vortex.proto.ExprProtos$SelectOpts$Builder - - dev.vortex.proto.ExprProtos$SelectOpts$OptsCase - - dev.vortex.proto.ExprProtos$SelectOptsOrBuilder - - dev.vortex.proto.ScalarProtos - - dev.vortex.proto.ScalarProtos$ListValue - - dev.vortex.proto.ScalarProtos$ListValue$1 - - dev.vortex.proto.ScalarProtos$ListValue$Builder - - dev.vortex.proto.ScalarProtos$ListValueOrBuilder - - dev.vortex.proto.ScalarProtos$Scalar - - dev.vortex.proto.ScalarProtos$Scalar$1 - - dev.vortex.proto.ScalarProtos$Scalar$Builder - - dev.vortex.proto.ScalarProtos$ScalarOrBuilder - - dev.vortex.proto.ScalarProtos$ScalarValue - - dev.vortex.proto.ScalarProtos$ScalarValue$1 - - dev.vortex.proto.ScalarProtos$ScalarValue$Builder - - dev.vortex.proto.ScalarProtos$ScalarValue$KindCase - - dev.vortex.proto.ScalarProtos$ScalarValueOrBuilder diff --git a/spark/v4.1/spark-runtime/runtime-deps.txt b/spark/v4.1/spark-runtime/runtime-deps.txt index ed9c0515f1fc..42367365725c 100644 --- a/spark/v4.1/spark-runtime/runtime-deps.txt +++ b/spark/v4.1/spark-runtime/runtime-deps.txt @@ -3,25 +3,24 @@ com.fasterxml.jackson.core:jackson-core:2.15 com.fasterxml.jackson.core:jackson-databind:2.15 com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.21 com.github.ben-manes.caffeine:caffeine:2.9 -com.google.errorprone:error_prone_annotations:2.41 +com.google.errorprone:error_prone_annotations:2.47 com.google.flatbuffers:flatbuffers-java:25.2 com.google.guava:failureaccess:1.0 -com.google.guava:guava:33.5 +com.google.guava:guava:33.6 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.j2objc:j2objc-annotations:3.1 -com.google.protobuf:protobuf-java:4.33 dev.failsafe:failsafe:3.3 -dev.vortex:vortex-jni:0.67 -dev.vortex:vortex-spark_2.13:0.67 +dev.vortex:vortex-jni:0.72 +dev.vortex:vortex-spark_2.13:0.72 io.airlift:aircompressor:2.0 io.netty:netty-buffer:4.2 io.netty:netty-common:4.2 -org.apache.arrow:arrow-c-data:18.3 -org.apache.arrow:arrow-format:18.3 -org.apache.arrow:arrow-memory-core:18.3 -org.apache.arrow:arrow-memory-netty-buffer-patch:18.3 -org.apache.arrow:arrow-memory-netty:18.3 -org.apache.arrow:arrow-vector:18.3 +org.apache.arrow:arrow-c-data:19.0 +org.apache.arrow:arrow-format:19.0 +org.apache.arrow:arrow-memory-core:19.0 +org.apache.arrow:arrow-memory-netty-buffer-patch:19.0 +org.apache.arrow:arrow-memory-netty:19.0 +org.apache.arrow:arrow-vector:19.0 org.apache.avro:avro:1.12 org.apache.datasketches:datasketches-java:6.2 org.apache.datasketches:datasketches-memory:3.0 diff --git a/spark/v4.1/spark/baseline-class-uniqueness.lock b/spark/v4.1/spark/baseline-class-uniqueness.lock index 72c0c24fb849..53a154c69d97 100644 --- a/spark/v4.1/spark/baseline-class-uniqueness.lock +++ b/spark/v4.1/spark/baseline-class-uniqueness.lock @@ -4,211 +4,9 @@ ## runtimeClasspath [com.google.guava:guava, dev.vortex:vortex-jni (classifier=all)] - com.google.thirdparty.publicsuffix.PublicSuffixPatterns + - com.google.thirdparty.publicsuffix.PublicSuffixTrie + - com.google.thirdparty.publicsuffix.PublicSuffixTrie$ChunksCharSequence - com.google.thirdparty.publicsuffix.PublicSuffixType - - com.google.thirdparty.publicsuffix.TrieParser -[com.google.protobuf:protobuf-java, dev.vortex:vortex-jni] - - com.google.protobuf.BoolValue - - com.google.protobuf.BoolValue$1 - - com.google.protobuf.BoolValue$Builder - - com.google.protobuf.BoolValueOrBuilder - - com.google.protobuf.BytesValue - - com.google.protobuf.BytesValue$1 - - com.google.protobuf.BytesValue$Builder - - com.google.protobuf.BytesValueOrBuilder - - com.google.protobuf.DoubleValue - - com.google.protobuf.DoubleValue$1 - - com.google.protobuf.DoubleValue$Builder - - com.google.protobuf.DoubleValueOrBuilder - - com.google.protobuf.FloatValue - - com.google.protobuf.FloatValue$1 - - com.google.protobuf.FloatValue$Builder - - com.google.protobuf.FloatValueOrBuilder - - com.google.protobuf.Int32Value - - com.google.protobuf.Int32Value$1 - - com.google.protobuf.Int32Value$Builder - - com.google.protobuf.Int32ValueOrBuilder - - com.google.protobuf.Int64Value - - com.google.protobuf.Int64Value$1 - - com.google.protobuf.Int64Value$Builder - - com.google.protobuf.Int64ValueOrBuilder - - com.google.protobuf.ListValue - - com.google.protobuf.ListValue$1 - - com.google.protobuf.ListValue$Builder - - com.google.protobuf.ListValueOrBuilder - - com.google.protobuf.NullValue - - com.google.protobuf.NullValue$1 - - com.google.protobuf.StringValue - - com.google.protobuf.StringValue$1 - - com.google.protobuf.StringValue$Builder - - com.google.protobuf.StringValueOrBuilder - - com.google.protobuf.Struct - - com.google.protobuf.Struct$1 - - com.google.protobuf.Struct$Builder - - com.google.protobuf.Struct$Builder$FieldsConverter - - com.google.protobuf.Struct$FieldsDefaultEntryHolder - - com.google.protobuf.StructOrBuilder - - com.google.protobuf.StructProto - - com.google.protobuf.UInt32Value - - com.google.protobuf.UInt32Value$1 - - com.google.protobuf.UInt32Value$Builder - - com.google.protobuf.UInt32ValueOrBuilder - - com.google.protobuf.UInt64Value - - com.google.protobuf.UInt64Value$1 - - com.google.protobuf.UInt64Value$Builder - - com.google.protobuf.UInt64ValueOrBuilder - - com.google.protobuf.Value - - com.google.protobuf.Value$1 - - com.google.protobuf.Value$Builder - - com.google.protobuf.Value$KindCase - - com.google.protobuf.ValueOrBuilder - - com.google.protobuf.WrappersProto -[commons-codec:commons-codec, dev.vortex:vortex-jni (classifier=all)] - - org.apache.commons.codec.Charsets - - org.apache.commons.codec.binary.Base16 - - org.apache.commons.codec.binary.Base32 - - org.apache.commons.codec.binary.Base64 - - org.apache.commons.codec.binary.BaseNCodec - - org.apache.commons.codec.binary.BaseNCodec$AbstractBuilder - - org.apache.commons.codec.binary.BinaryCodec - - org.apache.commons.codec.binary.StringUtils - - org.apache.commons.codec.digest.DigestUtils - - org.apache.commons.codec.digest.HmacUtils - - org.apache.commons.codec.digest.MurmurHash2 - - org.apache.commons.codec.digest.MurmurHash3 - - org.apache.commons.codec.digest.MurmurHash3$IncrementalHash32 - - org.apache.commons.codec.digest.MurmurHash3$IncrementalHash32x86 - - org.apache.commons.codec.digest.PureJavaCrc32 - - org.apache.commons.codec.digest.PureJavaCrc32C - - org.apache.commons.codec.digest.Sha2Crypt - - org.apache.commons.codec.digest.UnixCrypt - - org.apache.commons.codec.language.Caverphone1 - - org.apache.commons.codec.language.Caverphone2 - - org.apache.commons.codec.language.DaitchMokotoffSoundex - - org.apache.commons.codec.language.DaitchMokotoffSoundex$Branch - - org.apache.commons.codec.language.DaitchMokotoffSoundex$Rule - - org.apache.commons.codec.language.DoubleMetaphone - - org.apache.commons.codec.language.DoubleMetaphone$DoubleMetaphoneResult - - org.apache.commons.codec.language.MatchRatingApproachEncoder - - org.apache.commons.codec.language.Metaphone - - org.apache.commons.codec.language.Nysiis - - org.apache.commons.codec.language.RefinedSoundex - - org.apache.commons.codec.language.Soundex - - org.apache.commons.codec.language.SoundexUtils - - org.apache.commons.codec.language.bm.Lang - - org.apache.commons.codec.language.bm.PhoneticEngine - - org.apache.commons.codec.language.bm.PhoneticEngine$1 - - org.apache.commons.codec.language.bm.PhoneticEngine$PhonemeBuilder - - org.apache.commons.codec.language.bm.PhoneticEngine$RulesApplication - - org.apache.commons.codec.language.bm.ResourceConstants - - org.apache.commons.codec.language.bm.Rule - - org.apache.commons.codec.language.bm.Rule$1 - - org.apache.commons.codec.language.bm.Rule$2 - - org.apache.commons.codec.language.bm.Rule$Phoneme - - org.apache.commons.codec.language.bm.Rule$PhonemeExpr - - org.apache.commons.codec.language.bm.Rule$PhonemeList - - org.apache.commons.codec.net.PercentCodec - - org.apache.commons.codec.net.QuotedPrintableCodec - - org.apache.commons.codec.net.URLCodec - - org.apache.commons.codec.net.Utils -[dev.vortex:vortex-jni (classifier=all), io.netty:netty-buffer] - - io.netty.buffer.AbstractByteBuf - - io.netty.buffer.AbstractReferenceCountedByteBuf - - io.netty.buffer.AdaptiveByteBufAllocator$DirectChunkAllocator - - io.netty.buffer.AdaptivePoolingAllocator - - io.netty.buffer.AdaptivePoolingAllocator$1 - - io.netty.buffer.AdaptivePoolingAllocator$AdaptiveByteBuf - - io.netty.buffer.AdaptivePoolingAllocator$Chunk - - io.netty.buffer.AdaptivePoolingAllocator$ChunkController - - io.netty.buffer.AdaptivePoolingAllocator$ChunkRegistry - - io.netty.buffer.AdaptivePoolingAllocator$Magazine - - io.netty.buffer.AdaptivePoolingAllocator$MagazineGroup - - io.netty.buffer.AdaptivePoolingAllocator$SizeClassChunkController - - io.netty.buffer.AdaptivePoolingAllocator$SizeClassedChunk - - io.netty.buffer.ByteBufUtil - - io.netty.buffer.CompositeByteBuf - - io.netty.buffer.EmptyByteBuf - - io.netty.buffer.PoolArena - - io.netty.buffer.PoolArena$DirectArena - - io.netty.buffer.PoolThreadCache$FreeOnFinalize - - io.netty.buffer.PooledByteBufAllocator - - io.netty.buffer.PooledByteBufAllocator$PoolThreadLocalCache - - io.netty.buffer.ReadOnlyAbstractByteBuf - - io.netty.buffer.SimpleLeakAwareByteBuf - - io.netty.buffer.Unpooled - - io.netty.buffer.UnpooledByteBufAllocator$DecrementingCleanableDirectBuffer - - io.netty.buffer.UnpooledByteBufAllocator$InstrumentedUnpooledDirectByteBuf - - io.netty.buffer.UnpooledByteBufAllocator$InstrumentedUnpooledUnsafeDirectByteBuf - - io.netty.buffer.UnpooledByteBufAllocator$InstrumentedUnpooledUnsafeNoCleanerDirectByteBuf - - io.netty.buffer.UnpooledByteBufAllocator$UnpooledByteBufAllocatorMetric - - io.netty.buffer.UnpooledDirectByteBuf - - io.netty.buffer.UnpooledHeapByteBuf - - io.netty.buffer.UnpooledUnsafeDirectByteBuf - - io.netty.buffer.UnpooledUnsafeNoCleanerDirectByteBuf - - io.netty.buffer.UnsafeByteBufUtil -[dev.vortex:vortex-jni (classifier=all), io.netty:netty-common] - - io.netty.util.AbstractReferenceCounted - - io.netty.util.DefaultAttributeMap - - io.netty.util.HashedWheelTimer - - io.netty.util.HashedWheelTimer$HashedWheelBucket - - io.netty.util.LeakPresenceDetector - - io.netty.util.LeakPresenceDetector$LeakCreation - - io.netty.util.LeakPresenceDetector$ResourceScope - - io.netty.util.Recycler - - io.netty.util.Recycler$BlockingMessageQueue - - io.netty.util.Recycler$DefaultHandle - - io.netty.util.Recycler$EnhancedHandle - - io.netty.util.Recycler$GuardedLocalPool - - io.netty.util.Recycler$LocalPool - - io.netty.util.Recycler$UnguardedLocalPool - - io.netty.util.concurrent.AbstractScheduledEventExecutor - - io.netty.util.concurrent.GlobalEventExecutor - - io.netty.util.concurrent.GlobalEventExecutor$2 - - io.netty.util.concurrent.GlobalEventExecutor$3 - - io.netty.util.concurrent.GlobalEventExecutor$TaskRunner - - io.netty.util.concurrent.MpscIntQueue$MpscAtomicIntegerArrayQueue - - io.netty.util.concurrent.MultithreadEventExecutorGroup - - io.netty.util.concurrent.NonStickyEventExecutorGroup$NonStickyOrderedEventExecutor - - io.netty.util.concurrent.PromiseNotifier - - io.netty.util.concurrent.PromiseNotifier$1 - - io.netty.util.concurrent.SingleThreadEventExecutor - - io.netty.util.concurrent.SingleThreadEventExecutor$4 - - io.netty.util.concurrent.SingleThreadEventExecutor$5 - - io.netty.util.concurrent.SingleThreadEventExecutor$DefaultThreadProperties - - io.netty.util.internal.Cleaner - - io.netty.util.internal.CleanerJava24Linker - - io.netty.util.internal.CleanerJava24Linker$CleanableDirectBufferImpl - - io.netty.util.internal.CleanerJava25 - - io.netty.util.internal.CleanerJava25$CleanableDirectBufferImpl - - io.netty.util.internal.CleanerJava6 - - io.netty.util.internal.CleanerJava6$2 - - io.netty.util.internal.CleanerJava6$CleanableDirectBufferImpl - - io.netty.util.internal.CleanerJava9 - - io.netty.util.internal.CleanerJava9$2 - - io.netty.util.internal.CleanerJava9$CleanableDirectBufferImpl - - io.netty.util.internal.DirectCleaner - - io.netty.util.internal.DirectCleaner$CleanableDirectBufferImpl - - io.netty.util.internal.EmptyArrays - - io.netty.util.internal.PlatformDependent - - io.netty.util.internal.PlatformDependent$1 - - io.netty.util.internal.PlatformDependent$1$1 - - io.netty.util.internal.PlatformDependent$Mpsc - - io.netty.util.internal.PlatformDependent$Mpsc$1 - - io.netty.util.internal.PlatformDependent0 - - io.netty.util.internal.PlatformDependent0$1 - - io.netty.util.internal.PlatformDependent0$10 - - io.netty.util.internal.PlatformDependent0$11 - - io.netty.util.internal.PlatformDependent0$12 - - io.netty.util.internal.PlatformDependent0$2 - - io.netty.util.internal.PlatformDependent0$3 - - io.netty.util.internal.PlatformDependent0$4 - - io.netty.util.internal.PlatformDependent0$5 - - io.netty.util.internal.PlatformDependent0$6 - - io.netty.util.internal.PlatformDependent0$7 - - io.netty.util.internal.PlatformDependent0$8 - - io.netty.util.internal.PlatformDependent0$9 - - io.netty.util.internal.ReferenceCountUpdater - - io.netty.util.internal.ReferenceCountUpdater$UpdaterType - - io.netty.util.internal.ThrowableUtil [dev.vortex:vortex-jni (classifier=all), org.apache.arrow:arrow-c-data] - org.apache.arrow.c.ArrayStreamExporter$ExportedArrayStreamPrivateData - org.apache.arrow.c.jni.JniWrapper @@ -218,149 +16,12 @@ - io.netty.buffer.PooledByteBufAllocatorL - io.netty.buffer.PooledByteBufAllocatorL$InnerAllocator [dev.vortex:vortex-jni, dev.vortex:vortex-jni (classifier=all)] - - dev.vortex.api.Array - - dev.vortex.api.Files + - dev.vortex.api.DataSource + - dev.vortex.api.Expression - dev.vortex.api.ImmutableScanOptions - - dev.vortex.api.ImmutableScanOptions$Builder - - dev.vortex.api.expressions.Binary - - dev.vortex.api.expressions.GetItem - - dev.vortex.api.expressions.Literal - - dev.vortex.api.expressions.Literal$DecimalLiteral - - dev.vortex.api.proto.DTypes - - dev.vortex.api.proto.EndianUtils - - dev.vortex.api.proto.Expressions - - dev.vortex.api.proto.Scalars - - dev.vortex.api.proto.TemporalMetadatas + - dev.vortex.api.Partition + - dev.vortex.api.Scan + - dev.vortex.api.Session + - dev.vortex.api.VortexWriter - dev.vortex.arrow.ArrowAllocation - - dev.vortex.jni.JNIArray - - dev.vortex.jni.JNIArrayIterator - - dev.vortex.jni.JNIDType - - dev.vortex.jni.JNIFile - dev.vortex.jni.NativeLoader - - dev.vortex.proto.DTypeProtos - - dev.vortex.proto.DTypeProtos$Binary - - dev.vortex.proto.DTypeProtos$Binary$1 - - dev.vortex.proto.DTypeProtos$Binary$Builder - - dev.vortex.proto.DTypeProtos$BinaryOrBuilder - - dev.vortex.proto.DTypeProtos$Bool - - dev.vortex.proto.DTypeProtos$Bool$1 - - dev.vortex.proto.DTypeProtos$Bool$Builder - - dev.vortex.proto.DTypeProtos$BoolOrBuilder - - dev.vortex.proto.DTypeProtos$DType - - dev.vortex.proto.DTypeProtos$DType$1 - - dev.vortex.proto.DTypeProtos$DType$Builder - - dev.vortex.proto.DTypeProtos$DType$DtypeTypeCase - - dev.vortex.proto.DTypeProtos$DTypeOrBuilder - - dev.vortex.proto.DTypeProtos$Decimal - - dev.vortex.proto.DTypeProtos$Decimal$1 - - dev.vortex.proto.DTypeProtos$Decimal$Builder - - dev.vortex.proto.DTypeProtos$DecimalOrBuilder - - dev.vortex.proto.DTypeProtos$Extension - - dev.vortex.proto.DTypeProtos$Extension$1 - - dev.vortex.proto.DTypeProtos$Extension$Builder - - dev.vortex.proto.DTypeProtos$ExtensionOrBuilder - - dev.vortex.proto.DTypeProtos$Field - - dev.vortex.proto.DTypeProtos$Field$1 - - dev.vortex.proto.DTypeProtos$Field$Builder - - dev.vortex.proto.DTypeProtos$Field$FieldTypeCase - - dev.vortex.proto.DTypeProtos$FieldOrBuilder - - dev.vortex.proto.DTypeProtos$FieldPath - - dev.vortex.proto.DTypeProtos$FieldPath$1 - - dev.vortex.proto.DTypeProtos$FieldPath$Builder - - dev.vortex.proto.DTypeProtos$FieldPathOrBuilder - - dev.vortex.proto.DTypeProtos$FixedSizeList - - dev.vortex.proto.DTypeProtos$FixedSizeList$1 - - dev.vortex.proto.DTypeProtos$FixedSizeList$Builder - - dev.vortex.proto.DTypeProtos$FixedSizeListOrBuilder - - dev.vortex.proto.DTypeProtos$List - - dev.vortex.proto.DTypeProtos$List$1 - - dev.vortex.proto.DTypeProtos$List$Builder - - dev.vortex.proto.DTypeProtos$ListOrBuilder - - dev.vortex.proto.DTypeProtos$Null - - dev.vortex.proto.DTypeProtos$Null$1 - - dev.vortex.proto.DTypeProtos$Null$Builder - - dev.vortex.proto.DTypeProtos$NullOrBuilder - - dev.vortex.proto.DTypeProtos$PType - - dev.vortex.proto.DTypeProtos$PType$1 - - dev.vortex.proto.DTypeProtos$Primitive - - dev.vortex.proto.DTypeProtos$Primitive$1 - - dev.vortex.proto.DTypeProtos$Primitive$Builder - - dev.vortex.proto.DTypeProtos$PrimitiveOrBuilder - - dev.vortex.proto.DTypeProtos$Struct - - dev.vortex.proto.DTypeProtos$Struct$1 - - dev.vortex.proto.DTypeProtos$Struct$Builder - - dev.vortex.proto.DTypeProtos$StructOrBuilder - - dev.vortex.proto.DTypeProtos$Utf8 - - dev.vortex.proto.DTypeProtos$Utf8$1 - - dev.vortex.proto.DTypeProtos$Utf8$Builder - - dev.vortex.proto.DTypeProtos$Utf8OrBuilder - - dev.vortex.proto.DTypeProtos$Variant - - dev.vortex.proto.DTypeProtos$Variant$1 - - dev.vortex.proto.DTypeProtos$Variant$Builder - - dev.vortex.proto.DTypeProtos$VariantOrBuilder - - dev.vortex.proto.ExprProtos - - dev.vortex.proto.ExprProtos$AggregateFn - - dev.vortex.proto.ExprProtos$AggregateFn$1 - - dev.vortex.proto.ExprProtos$AggregateFn$Builder - - dev.vortex.proto.ExprProtos$AggregateFnOrBuilder - - dev.vortex.proto.ExprProtos$BetweenOpts - - dev.vortex.proto.ExprProtos$BetweenOpts$1 - - dev.vortex.proto.ExprProtos$BetweenOpts$Builder - - dev.vortex.proto.ExprProtos$BetweenOptsOrBuilder - - dev.vortex.proto.ExprProtos$BinaryOpts - - dev.vortex.proto.ExprProtos$BinaryOpts$1 - - dev.vortex.proto.ExprProtos$BinaryOpts$BinaryOp - - dev.vortex.proto.ExprProtos$BinaryOpts$BinaryOp$1 - - dev.vortex.proto.ExprProtos$BinaryOpts$Builder - - dev.vortex.proto.ExprProtos$BinaryOptsOrBuilder - - dev.vortex.proto.ExprProtos$CaseWhenOpts - - dev.vortex.proto.ExprProtos$CaseWhenOpts$1 - - dev.vortex.proto.ExprProtos$CaseWhenOpts$Builder - - dev.vortex.proto.ExprProtos$CaseWhenOptsOrBuilder - - dev.vortex.proto.ExprProtos$CastOpts - - dev.vortex.proto.ExprProtos$CastOpts$1 - - dev.vortex.proto.ExprProtos$CastOpts$Builder - - dev.vortex.proto.ExprProtos$CastOptsOrBuilder - - dev.vortex.proto.ExprProtos$Expr - - dev.vortex.proto.ExprProtos$Expr$1 - - dev.vortex.proto.ExprProtos$Expr$Builder - - dev.vortex.proto.ExprProtos$ExprOrBuilder - - dev.vortex.proto.ExprProtos$FieldNames - - dev.vortex.proto.ExprProtos$FieldNames$1 - - dev.vortex.proto.ExprProtos$FieldNames$Builder - - dev.vortex.proto.ExprProtos$FieldNamesOrBuilder - - dev.vortex.proto.ExprProtos$GetItemOpts - - dev.vortex.proto.ExprProtos$GetItemOpts$1 - - dev.vortex.proto.ExprProtos$GetItemOpts$Builder - - dev.vortex.proto.ExprProtos$GetItemOptsOrBuilder - - dev.vortex.proto.ExprProtos$LikeOpts - - dev.vortex.proto.ExprProtos$LikeOpts$1 - - dev.vortex.proto.ExprProtos$LikeOpts$Builder - - dev.vortex.proto.ExprProtos$LikeOptsOrBuilder - - dev.vortex.proto.ExprProtos$LiteralOpts - - dev.vortex.proto.ExprProtos$LiteralOpts$1 - - dev.vortex.proto.ExprProtos$LiteralOpts$Builder - - dev.vortex.proto.ExprProtos$LiteralOptsOrBuilder - - dev.vortex.proto.ExprProtos$PackOpts - - dev.vortex.proto.ExprProtos$PackOpts$1 - - dev.vortex.proto.ExprProtos$PackOpts$Builder - - dev.vortex.proto.ExprProtos$PackOptsOrBuilder - - dev.vortex.proto.ExprProtos$SelectOpts - - dev.vortex.proto.ExprProtos$SelectOpts$1 - - dev.vortex.proto.ExprProtos$SelectOpts$Builder - - dev.vortex.proto.ExprProtos$SelectOpts$OptsCase - - dev.vortex.proto.ExprProtos$SelectOptsOrBuilder - - dev.vortex.proto.ScalarProtos - - dev.vortex.proto.ScalarProtos$ListValue - - dev.vortex.proto.ScalarProtos$ListValue$1 - - dev.vortex.proto.ScalarProtos$ListValue$Builder - - dev.vortex.proto.ScalarProtos$ListValueOrBuilder - - dev.vortex.proto.ScalarProtos$Scalar - - dev.vortex.proto.ScalarProtos$Scalar$1 - - dev.vortex.proto.ScalarProtos$Scalar$Builder - - dev.vortex.proto.ScalarProtos$ScalarOrBuilder - - dev.vortex.proto.ScalarProtos$ScalarValue - - dev.vortex.proto.ScalarProtos$ScalarValue$1 - - dev.vortex.proto.ScalarProtos$ScalarValue$Builder - - dev.vortex.proto.ScalarProtos$ScalarValue$KindCase - - dev.vortex.proto.ScalarProtos$ScalarValueOrBuilder From a4e7f4a8d84a9c4e5fdbae282f730f04a2d3680c Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Tue, 26 May 2026 13:20:39 +0100 Subject: [PATCH 3/4] fixes Signed-off-by: Robert Kruszewski --- .../data/vortex/GenericVortexReader.java | 55 ++++++++----- .../iceberg/vortex/ConvertFilterToVortex.java | 4 +- .../apache/iceberg/vortex/VortexSchemas.java | 82 ++++++++++--------- 3 files changed, 80 insertions(+), 61 deletions(-) diff --git a/vortex/src/main/java/org/apache/iceberg/data/vortex/GenericVortexReader.java b/vortex/src/main/java/org/apache/iceberg/data/vortex/GenericVortexReader.java index 986e3595503e..73c2d929ee91 100644 --- a/vortex/src/main/java/org/apache/iceberg/data/vortex/GenericVortexReader.java +++ b/vortex/src/main/java/org/apache/iceberg/data/vortex/GenericVortexReader.java @@ -104,23 +104,30 @@ public VortexValueReader list( @Override public VortexValueReader primitive(Type.PrimitiveType iPrimitive, Field primField) { - ArrowType arrowType = primField.getType(); if ((iPrimitive != null && iPrimitive.typeId() == Type.TypeID.UUID) || VortexSchemas.isUuidField(primField)) { return GenericVortexReaders.uuids(); - } else if (arrowType instanceof ArrowType.Bool) { - return GenericVortexReaders.bools(); - } else if (arrowType instanceof ArrowType.Int intType) { + } + ArrowType arrowType = primField.getType(); + if (arrowType instanceof ArrowType.Int intType) { return intType.getBitWidth() <= Integer.SIZE ? GenericVortexReaders.ints() : GenericVortexReaders.longs(); } else if (arrowType instanceof ArrowType.FloatingPoint fpType) { - return switch (fpType.getPrecision()) { - case SINGLE -> GenericVortexReaders.floats(); - case DOUBLE -> GenericVortexReaders.doubles(); - case HALF -> - throw new UnsupportedOperationException("Half-precision floats are not supported"); - }; + return floatingPointReader(fpType); + } else if (arrowType instanceof ArrowType.Date dateType) { + return GenericVortexReaders.date(dateType.getUnit() == DateUnit.MILLISECOND); + } else if (arrowType instanceof ArrowType.Time timeType) { + return GenericVortexReaders.time(timeType.getUnit() == TimeUnit.NANOSECOND); + } else if (arrowType instanceof ArrowType.Timestamp tsType) { + return timestampReader(tsType); + } + return simpleReader(arrowType); + } + + private static VortexValueReader simpleReader(ArrowType arrowType) { + if (arrowType instanceof ArrowType.Bool) { + return GenericVortexReaders.bools(); } else if (arrowType instanceof ArrowType.Decimal) { return GenericVortexReaders.decimals(); } else if (arrowType instanceof ArrowType.Utf8 || arrowType instanceof ArrowType.LargeUtf8) { @@ -129,20 +136,26 @@ public VortexValueReader primitive(Type.PrimitiveType iPrimitive, Field primF || arrowType instanceof ArrowType.LargeBinary || arrowType instanceof ArrowType.FixedSizeBinary) { return GenericVortexReaders.bytes(); - } else if (arrowType instanceof ArrowType.Date dateType) { - return GenericVortexReaders.date(dateType.getUnit() == DateUnit.MILLISECOND); - } else if (arrowType instanceof ArrowType.Time timeType) { - return GenericVortexReaders.time(timeType.getUnit() == TimeUnit.NANOSECOND); - } else if (arrowType instanceof ArrowType.Timestamp tsType) { - boolean isNano = tsType.getUnit() == TimeUnit.NANOSECOND; - if (tsType.getTimezone() == null) { - return GenericVortexReaders.timestamp(isNano); - } else { - return GenericVortexReaders.timestampTz(tsType.getTimezone(), isNano); - } } throw new UnsupportedOperationException( "Unsupported Arrow type in Vortex read: " + arrowType); } + + private static VortexValueReader floatingPointReader(ArrowType.FloatingPoint fpType) { + return switch (fpType.getPrecision()) { + case SINGLE -> GenericVortexReaders.floats(); + case DOUBLE -> GenericVortexReaders.doubles(); + case HALF -> + throw new UnsupportedOperationException("Half-precision floats are not supported"); + }; + } + + private static VortexValueReader timestampReader(ArrowType.Timestamp tsType) { + boolean isNano = tsType.getUnit() == TimeUnit.NANOSECOND; + if (tsType.getTimezone() == null) { + return GenericVortexReaders.timestamp(isNano); + } + return GenericVortexReaders.timestampTz(tsType.getTimezone(), isNano); + } } } diff --git a/vortex/src/main/java/org/apache/iceberg/vortex/ConvertFilterToVortex.java b/vortex/src/main/java/org/apache/iceberg/vortex/ConvertFilterToVortex.java index 872a3713e3da..891eb8763f49 100644 --- a/vortex/src/main/java/org/apache/iceberg/vortex/ConvertFilterToVortex.java +++ b/vortex/src/main/java/org/apache/iceberg/vortex/ConvertFilterToVortex.java @@ -217,13 +217,13 @@ private Expression fromSetPredicate( Set literalSet, Type termType) { Expression[] eqExprs = new Expression[literalSet.size()]; - int i = 0; + int idx = 0; for (T value : literalSet) { Expression vortexLit = toVortexLiteral(value, termType); if (vortexLit == UNCONVERTIBLE) { return UNCONVERTIBLE; } - eqExprs[i++] = Expression.binary(BinaryOp.EQ, term, vortexLit); + eqExprs[idx++] = Expression.binary(BinaryOp.EQ, term, vortexLit); } return switch (op) { diff --git a/vortex/src/main/java/org/apache/iceberg/vortex/VortexSchemas.java b/vortex/src/main/java/org/apache/iceberg/vortex/VortexSchemas.java index 223e413ab9ae..ce2cf37c2a67 100644 --- a/vortex/src/main/java/org/apache/iceberg/vortex/VortexSchemas.java +++ b/vortex/src/main/java/org/apache/iceberg/vortex/VortexSchemas.java @@ -176,7 +176,6 @@ yield new Field( } private static Type toIcebergType(Field field) { - ArrowType arrowType = field.getType(); // UUID is conveyed as the {@code arrow.uuid} extension over FixedSizeBinary(16). Check the // metadata directly so this works whether or not the extension is registered with // ExtensionTypeRegistry (i.e. whether arrowType deserialized to ExtensionType or stayed as @@ -184,57 +183,64 @@ private static Type toIcebergType(Field field) { if (isUuidField(field)) { return Types.UUIDType.get(); } + ArrowType arrowType = field.getType(); + if (arrowType instanceof ArrowType.Int intType) { + return intType.getBitWidth() <= Integer.SIZE ? Types.IntegerType.get() : Types.LongType.get(); + } else if (arrowType instanceof ArrowType.FloatingPoint fpType) { + return toIcebergFloatingPoint(fpType); + } else if (arrowType instanceof ArrowType.Decimal decType) { + return Types.DecimalType.of(decType.getPrecision(), decType.getScale()); + } else if (arrowType instanceof ArrowType.FixedSizeBinary fixed) { + return Types.FixedType.ofLength(fixed.getByteWidth()); + } else if (arrowType instanceof ArrowType.Timestamp tsType) { + return toIcebergTimestamp(tsType); + } else if (arrowType instanceof ArrowType.List) { + return toIcebergList(field); + } + return toIcebergSimpleType(arrowType); + } + + private static Type toIcebergSimpleType(ArrowType arrowType) { if (arrowType instanceof ArrowType.Null) { return Types.UnknownType.get(); } else if (arrowType instanceof ArrowType.Bool) { return Types.BooleanType.get(); - } else if (arrowType instanceof ArrowType.Int intType) { - if (intType.getBitWidth() <= Integer.SIZE) { - return Types.IntegerType.get(); - } else { - return Types.LongType.get(); - } - } else if (arrowType instanceof ArrowType.FloatingPoint fpType) { - return switch (fpType.getPrecision()) { - case SINGLE -> Types.FloatType.get(); - case DOUBLE -> Types.DoubleType.get(); - case HALF -> - throw new UnsupportedOperationException("Half-precision floats are not supported"); - }; - } else if (arrowType instanceof ArrowType.Decimal decType) { - return Types.DecimalType.of(decType.getPrecision(), decType.getScale()); } else if (arrowType instanceof ArrowType.Utf8 || arrowType instanceof ArrowType.LargeUtf8) { return Types.StringType.get(); } else if (arrowType instanceof ArrowType.Binary || arrowType instanceof ArrowType.LargeBinary) { return Types.BinaryType.get(); - } else if (arrowType instanceof ArrowType.FixedSizeBinary fixed) { - return Types.FixedType.ofLength(fixed.getByteWidth()); } else if (arrowType instanceof ArrowType.Date) { return Types.DateType.get(); } else if (arrowType instanceof ArrowType.Time) { return Types.TimeType.get(); - } else if (arrowType instanceof ArrowType.Timestamp tsType) { - if (tsType.getTimezone() == null) { - return tsType.getUnit() == TimeUnit.NANOSECOND - ? Types.TimestampNanoType.withoutZone() - : Types.TimestampType.withoutZone(); - } else { - return tsType.getUnit() == TimeUnit.NANOSECOND - ? Types.TimestampNanoType.withZone() - : Types.TimestampType.withZone(); - } - } else if (arrowType instanceof ArrowType.List) { - Field elementField = field.getChildren().get(0); - Type innerType = toIcebergType(elementField); - if (elementField.isNullable()) { - return Types.ListType.ofOptional(0, innerType); - } else { - return Types.ListType.ofRequired(0, innerType); - } - } else { - throw new UnsupportedOperationException("Unsupported Arrow type: " + arrowType); } + throw new UnsupportedOperationException("Unsupported Arrow type: " + arrowType); + } + + private static Type toIcebergFloatingPoint(ArrowType.FloatingPoint fpType) { + return switch (fpType.getPrecision()) { + case SINGLE -> Types.FloatType.get(); + case DOUBLE -> Types.DoubleType.get(); + case HALF -> + throw new UnsupportedOperationException("Half-precision floats are not supported"); + }; + } + + private static Type toIcebergTimestamp(ArrowType.Timestamp tsType) { + boolean isNano = tsType.getUnit() == TimeUnit.NANOSECOND; + if (tsType.getTimezone() == null) { + return isNano ? Types.TimestampNanoType.withoutZone() : Types.TimestampType.withoutZone(); + } + return isNano ? Types.TimestampNanoType.withZone() : Types.TimestampType.withZone(); + } + + private static Type toIcebergList(Field field) { + Field elementField = field.getChildren().get(0); + Type innerType = toIcebergType(elementField); + return elementField.isNullable() + ? Types.ListType.ofOptional(0, innerType) + : Types.ListType.ofRequired(0, innerType); } /** From 6b618c761de793b505e78c3ddc8246ec808103ec Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Tue, 26 May 2026 16:44:17 +0100 Subject: [PATCH 4/4] fixes Signed-off-by: Robert Kruszewski --- .../iceberg/spark/data/SparkVortexReader.java | 57 +++++++------------ .../spark/data/SparkVortexValueReaders.java | 33 +++++++++++ .../iceberg/spark/data/SparkVortexWriter.java | 1 - .../iceberg/spark/source/SparkBatch.java | 19 ++++++- .../iceberg/spark/data/SparkVortexReader.java | 57 +++++++------------ .../spark/data/SparkVortexValueReaders.java | 33 +++++++++++ .../iceberg/spark/data/SparkVortexWriter.java | 1 - .../iceberg/spark/source/SparkBatch.java | 19 ++++++- .../iceberg/spark/data/SparkVortexReader.java | 4 +- .../iceberg/spark/data/SparkVortexWriter.java | 1 - .../iceberg/spark/source/SparkBatch.java | 19 ++++++- .../spark/source/SparkFormatModels.java | 1 + 12 files changed, 162 insertions(+), 83 deletions(-) diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexReader.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexReader.java index 6afbdce70fb0..63c4e0edc62e 100644 --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexReader.java +++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexReader.java @@ -22,7 +22,6 @@ import java.util.Map; import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.VectorSchemaRoot; -import org.apache.arrow.vector.types.TimeUnit; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; import org.apache.iceberg.Schema; @@ -84,37 +83,27 @@ public VortexValueReader list( @Override public VortexValueReader primitive(Type.PrimitiveType icebergType, Field primField) { - switch (icebergType.typeId()) { - case BOOLEAN: - return GenericVortexReaders.bools(); - case INTEGER: - return GenericVortexReaders.ints(); - case LONG: - return GenericVortexReaders.longs(); - case FLOAT: - return GenericVortexReaders.floats(); - case DOUBLE: - return GenericVortexReaders.doubles(); - case STRING: - return SparkVortexValueReaders.utf8String(); - case BINARY: - return GenericVortexReaders.bytes(); - case DECIMAL: - return GenericVortexReaders.decimals(); - case TIMESTAMP: - case TIMESTAMP_NANO: - { - ArrowType.Timestamp ts = (ArrowType.Timestamp) primField.getType(); - return SparkVortexValueReaders.timestamp(ts.getUnit()); - } - case DATE: - return SparkVortexValueReaders.date(); - case UUID: - return SparkVortexValueReaders.uuid(); - case TIME: - default: - throw new UnsupportedOperationException("Unsupported type: " + icebergType); - } + return switch (icebergType.typeId()) { + case BOOLEAN -> GenericVortexReaders.bools(); + case INTEGER -> GenericVortexReaders.ints(); + case LONG -> GenericVortexReaders.longs(); + case FLOAT -> GenericVortexReaders.floats(); + case DOUBLE -> GenericVortexReaders.doubles(); + case STRING -> SparkVortexValueReaders.utf8String(); + case BINARY -> GenericVortexReaders.bytes(); + case DECIMAL -> GenericVortexReaders.decimals(); + case TIMESTAMP, TIMESTAMP_NANO -> { + ArrowType.Timestamp ts = (ArrowType.Timestamp) primField.getType(); + yield SparkVortexValueReaders.timestamp(ts.getUnit()); + } + case TIME -> { + ArrowType.Time time = (ArrowType.Time) primField.getType(); + yield SparkVortexValueReaders.time(time.getUnit()); + } + case DATE -> SparkVortexValueReaders.date(); + case UUID -> SparkVortexValueReaders.uuid(); + default -> throw new UnsupportedOperationException("Unsupported type: " + icebergType); + }; } } @@ -138,8 +127,4 @@ public InternalRow readNonNull(FieldVector vector, int row) { return result; } } - - // Silence unused warning that TimeUnit class is required for the public API. - @SuppressWarnings("unused") - private static final TimeUnit UNUSED = TimeUnit.MICROSECOND; } diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexValueReaders.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexValueReaders.java index d48685814862..8ce5ce6d20c2 100644 --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexValueReaders.java +++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexValueReaders.java @@ -25,6 +25,8 @@ import org.apache.arrow.vector.ExtensionTypeVector; import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.FixedSizeBinaryVector; +import org.apache.arrow.vector.TimeMicroVector; +import org.apache.arrow.vector.TimeNanoVector; import org.apache.arrow.vector.TimeStampVector; import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.types.TimeUnit; @@ -55,6 +57,11 @@ public static VortexValueReader timestamp(TimeUnit timeUnit) { return new TimestampReader(timeUnit); } + public static VortexValueReader time(TimeUnit timeUnit) { + // Spark's TimeType is stored as microseconds since midnight (Long). + return new TimeReader(timeUnit); + } + static class UTF8Reader implements VortexValueReader { static final UTF8Reader INSTANCE = new UTF8Reader(); @@ -123,4 +130,30 @@ public Long readNonNull(FieldVector vector, int row) { }; } } + + static class TimeReader implements VortexValueReader { + private final TimeUnit unit; + + private TimeReader(TimeUnit unit) { + this.unit = unit; + } + + @Override + public Long readNonNull(FieldVector vector, int row) { + long measure; + if (vector instanceof TimeMicroVector tm) { + measure = tm.get(row); + } else if (vector instanceof TimeNanoVector tn) { + measure = tn.get(row); + } else { + measure = ((BigIntVector) vector).get(row); + } + return switch (unit) { + case NANOSECOND -> Math.floorDiv(measure, 1_000L); + case MICROSECOND -> measure; + case MILLISECOND -> Math.multiplyExact(measure, 1_000L); + case SECOND -> Math.multiplyExact(measure, 1_000_000L); + }; + } + } } diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexWriter.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexWriter.java index cf776f27bb8c..ed231010b563 100644 --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexWriter.java +++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexWriter.java @@ -101,7 +101,6 @@ private static void writeValue( ((VarCharVector) vector).setSafe(rowIndex, str.getBytes()); break; case BINARY: - case FIXED: byte[] bytes = row.getBinary(fieldIndex); ((VarBinaryVector) vector).setSafe(rowIndex, bytes); break; diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkBatch.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkBatch.java index ebd51555cd47..90bf52e4ce11 100644 --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkBatch.java +++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkBatch.java @@ -190,9 +190,24 @@ private boolean useOrcBatchReads() { && taskGroups.stream().allMatch(this::supportsOrcBatchReads); } + // conditions for using Vortex batch reads: + // - all tasks are of FileScanTask type and read only Vortex files private boolean useVortexBatchReads() { - // TODO(aduffy): do we ever want to not use this? - return true; + return taskGroups.stream().allMatch(this::supportsVortexBatchReads); + } + + private boolean supportsVortexBatchReads(ScanTask task) { + if (task instanceof ScanTaskGroup) { + ScanTaskGroup taskGroup = (ScanTaskGroup) task; + return taskGroup.tasks().stream().allMatch(this::supportsVortexBatchReads); + + } else if (task.isFileScanTask() && !task.isDataTask()) { + FileScanTask fileScanTask = task.asFileScanTask(); + return fileScanTask.file().format() == FileFormat.VORTEX; + + } else { + return false; + } } private boolean supportsOrcBatchReads(ScanTask task) { diff --git a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexReader.java b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexReader.java index 6afbdce70fb0..63c4e0edc62e 100644 --- a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexReader.java +++ b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexReader.java @@ -22,7 +22,6 @@ import java.util.Map; import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.VectorSchemaRoot; -import org.apache.arrow.vector.types.TimeUnit; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; import org.apache.iceberg.Schema; @@ -84,37 +83,27 @@ public VortexValueReader list( @Override public VortexValueReader primitive(Type.PrimitiveType icebergType, Field primField) { - switch (icebergType.typeId()) { - case BOOLEAN: - return GenericVortexReaders.bools(); - case INTEGER: - return GenericVortexReaders.ints(); - case LONG: - return GenericVortexReaders.longs(); - case FLOAT: - return GenericVortexReaders.floats(); - case DOUBLE: - return GenericVortexReaders.doubles(); - case STRING: - return SparkVortexValueReaders.utf8String(); - case BINARY: - return GenericVortexReaders.bytes(); - case DECIMAL: - return GenericVortexReaders.decimals(); - case TIMESTAMP: - case TIMESTAMP_NANO: - { - ArrowType.Timestamp ts = (ArrowType.Timestamp) primField.getType(); - return SparkVortexValueReaders.timestamp(ts.getUnit()); - } - case DATE: - return SparkVortexValueReaders.date(); - case UUID: - return SparkVortexValueReaders.uuid(); - case TIME: - default: - throw new UnsupportedOperationException("Unsupported type: " + icebergType); - } + return switch (icebergType.typeId()) { + case BOOLEAN -> GenericVortexReaders.bools(); + case INTEGER -> GenericVortexReaders.ints(); + case LONG -> GenericVortexReaders.longs(); + case FLOAT -> GenericVortexReaders.floats(); + case DOUBLE -> GenericVortexReaders.doubles(); + case STRING -> SparkVortexValueReaders.utf8String(); + case BINARY -> GenericVortexReaders.bytes(); + case DECIMAL -> GenericVortexReaders.decimals(); + case TIMESTAMP, TIMESTAMP_NANO -> { + ArrowType.Timestamp ts = (ArrowType.Timestamp) primField.getType(); + yield SparkVortexValueReaders.timestamp(ts.getUnit()); + } + case TIME -> { + ArrowType.Time time = (ArrowType.Time) primField.getType(); + yield SparkVortexValueReaders.time(time.getUnit()); + } + case DATE -> SparkVortexValueReaders.date(); + case UUID -> SparkVortexValueReaders.uuid(); + default -> throw new UnsupportedOperationException("Unsupported type: " + icebergType); + }; } } @@ -138,8 +127,4 @@ public InternalRow readNonNull(FieldVector vector, int row) { return result; } } - - // Silence unused warning that TimeUnit class is required for the public API. - @SuppressWarnings("unused") - private static final TimeUnit UNUSED = TimeUnit.MICROSECOND; } diff --git a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexValueReaders.java b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexValueReaders.java index d48685814862..8ce5ce6d20c2 100644 --- a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexValueReaders.java +++ b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexValueReaders.java @@ -25,6 +25,8 @@ import org.apache.arrow.vector.ExtensionTypeVector; import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.FixedSizeBinaryVector; +import org.apache.arrow.vector.TimeMicroVector; +import org.apache.arrow.vector.TimeNanoVector; import org.apache.arrow.vector.TimeStampVector; import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.types.TimeUnit; @@ -55,6 +57,11 @@ public static VortexValueReader timestamp(TimeUnit timeUnit) { return new TimestampReader(timeUnit); } + public static VortexValueReader time(TimeUnit timeUnit) { + // Spark's TimeType is stored as microseconds since midnight (Long). + return new TimeReader(timeUnit); + } + static class UTF8Reader implements VortexValueReader { static final UTF8Reader INSTANCE = new UTF8Reader(); @@ -123,4 +130,30 @@ public Long readNonNull(FieldVector vector, int row) { }; } } + + static class TimeReader implements VortexValueReader { + private final TimeUnit unit; + + private TimeReader(TimeUnit unit) { + this.unit = unit; + } + + @Override + public Long readNonNull(FieldVector vector, int row) { + long measure; + if (vector instanceof TimeMicroVector tm) { + measure = tm.get(row); + } else if (vector instanceof TimeNanoVector tn) { + measure = tn.get(row); + } else { + measure = ((BigIntVector) vector).get(row); + } + return switch (unit) { + case NANOSECOND -> Math.floorDiv(measure, 1_000L); + case MICROSECOND -> measure; + case MILLISECOND -> Math.multiplyExact(measure, 1_000L); + case SECOND -> Math.multiplyExact(measure, 1_000_000L); + }; + } + } } diff --git a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexWriter.java b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexWriter.java index cf776f27bb8c..ed231010b563 100644 --- a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexWriter.java +++ b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexWriter.java @@ -101,7 +101,6 @@ private static void writeValue( ((VarCharVector) vector).setSafe(rowIndex, str.getBytes()); break; case BINARY: - case FIXED: byte[] bytes = row.getBinary(fieldIndex); ((VarBinaryVector) vector).setSafe(rowIndex, bytes); break; diff --git a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/source/SparkBatch.java b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/source/SparkBatch.java index ebd51555cd47..90bf52e4ce11 100644 --- a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/source/SparkBatch.java +++ b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/source/SparkBatch.java @@ -190,9 +190,24 @@ private boolean useOrcBatchReads() { && taskGroups.stream().allMatch(this::supportsOrcBatchReads); } + // conditions for using Vortex batch reads: + // - all tasks are of FileScanTask type and read only Vortex files private boolean useVortexBatchReads() { - // TODO(aduffy): do we ever want to not use this? - return true; + return taskGroups.stream().allMatch(this::supportsVortexBatchReads); + } + + private boolean supportsVortexBatchReads(ScanTask task) { + if (task instanceof ScanTaskGroup) { + ScanTaskGroup taskGroup = (ScanTaskGroup) task; + return taskGroup.tasks().stream().allMatch(this::supportsVortexBatchReads); + + } else if (task.isFileScanTask() && !task.isDataTask()) { + FileScanTask fileScanTask = task.asFileScanTask(); + return fileScanTask.file().format() == FileFormat.VORTEX; + + } else { + return false; + } } private boolean supportsOrcBatchReads(ScanTask task) { diff --git a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexReader.java b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexReader.java index 45c76955b0a6..63c4e0edc62e 100644 --- a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexReader.java +++ b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexReader.java @@ -97,8 +97,8 @@ public VortexValueReader primitive(Type.PrimitiveType icebergType, Field prim yield SparkVortexValueReaders.timestamp(ts.getUnit()); } case TIME -> { - ArrowType.Time t = (ArrowType.Time) primField.getType(); - yield SparkVortexValueReaders.time(t.getUnit()); + ArrowType.Time time = (ArrowType.Time) primField.getType(); + yield SparkVortexValueReaders.time(time.getUnit()); } case DATE -> SparkVortexValueReaders.date(); case UUID -> SparkVortexValueReaders.uuid(); diff --git a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexWriter.java b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexWriter.java index cf776f27bb8c..ed231010b563 100644 --- a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexWriter.java +++ b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/data/SparkVortexWriter.java @@ -101,7 +101,6 @@ private static void writeValue( ((VarCharVector) vector).setSafe(rowIndex, str.getBytes()); break; case BINARY: - case FIXED: byte[] bytes = row.getBinary(fieldIndex); ((VarBinaryVector) vector).setSafe(rowIndex, bytes); break; diff --git a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkBatch.java b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkBatch.java index 09ec2ef3b60f..23bd14d03bb5 100644 --- a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkBatch.java +++ b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkBatch.java @@ -187,9 +187,24 @@ private boolean useOrcBatchReads() { && taskGroups.stream().allMatch(this::supportsOrcBatchReads); } + // conditions for using Vortex batch reads: + // - all tasks are of FileScanTask type and read only Vortex files private boolean useVortexBatchReads() { - // TODO(aduffy): do we ever want to not use this? - return true; + return taskGroups.stream().allMatch(this::supportsVortexBatchReads); + } + + private boolean supportsVortexBatchReads(ScanTask task) { + if (task instanceof ScanTaskGroup) { + ScanTaskGroup taskGroup = (ScanTaskGroup) task; + return taskGroup.tasks().stream().allMatch(this::supportsVortexBatchReads); + + } else if (task.isFileScanTask() && !task.isDataTask()) { + FileScanTask fileScanTask = task.asFileScanTask(); + return fileScanTask.file().format() == FileFormat.VORTEX; + + } else { + return false; + } } private boolean supportsOrcBatchReads(ScanTask task) { diff --git a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkFormatModels.java b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkFormatModels.java index bbe28a4ea396..c5a334646497 100644 --- a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkFormatModels.java +++ b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkFormatModels.java @@ -84,6 +84,7 @@ public static void register() { StructType.class, (icebergSchema, fileSchema, engineSchema, idToConstant) -> VectorizedSparkOrcReaders.buildReader(icebergSchema, fileSchema, idToConstant))); + FormatModelRegistry.register( VortexFormatModel.create( InternalRow.class,