From 832bb6185ee9e4db831f2fe600f281ca4eb9b9b8 Mon Sep 17 00:00:00 2001 From: Jason Gustafson <12502538+hachikuji@users.noreply.github.com> Date: Mon, 31 Mar 2025 15:12:10 -0700 Subject: [PATCH 01/29] Initial implementation of RS3 window table --- .../db/CassandraWindowFlushManager.java | 2 +- .../kafka/internal/db/FlushManager.java | 5 + .../internal/db/MongoWindowFlushManager.java | 2 +- .../kafka/internal/db/RemoteWindowTable.java | 2 +- .../db/SegmentedWindowFlushManager.java | 113 +++++++++ .../kafka/internal/db/WindowFlushManager.java | 101 +------- .../internal/db/rs3/PssTablePartitioner.java | 10 +- .../internal/db/rs3/RS3KVFlushManager.java | 156 +++--------- .../kafka/internal/db/rs3/RS3KVTable.java | 41 +--- .../internal/db/rs3/RS3TableFactory.java | 40 +++- .../db/rs3/RS3WindowFlushManager.java | 177 ++++++++++++++ .../kafka/internal/db/rs3/RS3WindowTable.java | 226 ++++++++++++++++++ .../internal/db/rs3/RS3WindowedKeySerde.java | 33 +++ .../kafka/internal/db/rs3/RS3Writer.java | 144 +++++++++++ .../internal/db/rs3/client/LssMetadata.java | 36 +++ .../internal/db/rs3/client/RS3ClientUtil.java | 75 ++++++ ...tions.java => RemoteWindowOperations.java} | 51 +++- .../stores/ResponsiveWindowStore.java | 2 +- .../internal/db/rs3/RS3WindowTableTest.java | 91 +++++++ .../db/rs3/client/RS3ClientUtilTest.java | 87 +++++++ .../kafka/internal/db/TTDWindowTable.java | 4 +- 21 files changed, 1120 insertions(+), 278 deletions(-) create mode 100644 kafka-client/src/main/java/dev/responsive/kafka/internal/db/SegmentedWindowFlushManager.java create mode 100644 kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowFlushManager.java create mode 100644 kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowTable.java create mode 100644 kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowedKeySerde.java create mode 100644 kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3Writer.java create mode 100644 kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/LssMetadata.java create mode 100644 kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/RS3ClientUtil.java rename kafka-client/src/main/java/dev/responsive/kafka/internal/stores/{SegmentedOperations.java => RemoteWindowOperations.java} (87%) create mode 100644 kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/RS3WindowTableTest.java create mode 100644 kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/RS3ClientUtilTest.java diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/CassandraWindowFlushManager.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/CassandraWindowFlushManager.java index 2c103b17d..b73d8a17c 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/CassandraWindowFlushManager.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/CassandraWindowFlushManager.java @@ -22,7 +22,7 @@ import org.apache.kafka.common.utils.LogContext; import org.slf4j.Logger; -public class CassandraWindowFlushManager extends WindowFlushManager { +public class CassandraWindowFlushManager extends SegmentedWindowFlushManager { private final String logPrefix; private final Logger log; diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/FlushManager.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/FlushManager.java index cf6764daf..ad20976c6 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/FlushManager.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/FlushManager.java @@ -15,6 +15,11 @@ import dev.responsive.kafka.internal.db.partitioning.TablePartitioner; import dev.responsive.kafka.internal.stores.RemoteWriteResult; +/** + * + * @param Key type + * @param

Table partition type + */ public interface FlushManager { String tableName(); diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/MongoWindowFlushManager.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/MongoWindowFlushManager.java index 5f1b34c03..133c95403 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/MongoWindowFlushManager.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/MongoWindowFlushManager.java @@ -28,7 +28,7 @@ import org.apache.kafka.common.utils.LogContext; import org.slf4j.Logger; -public class MongoWindowFlushManager extends WindowFlushManager { +public class MongoWindowFlushManager extends SegmentedWindowFlushManager { private final String logPrefix; private final Logger log; diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/RemoteWindowTable.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/RemoteWindowTable.java index b6c69fcf6..a0d7488f3 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/RemoteWindowTable.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/RemoteWindowTable.java @@ -25,7 +25,7 @@ public interface RemoteWindowTable extends RemoteTable { * @return a {@link WindowFlushManager} that gives the callee access * to run statements on {@code table} */ - WindowFlushManager init( + WindowFlushManager init( final int kafkaPartition ); diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/SegmentedWindowFlushManager.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/SegmentedWindowFlushManager.java new file mode 100644 index 000000000..195ab38af --- /dev/null +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/SegmentedWindowFlushManager.java @@ -0,0 +1,113 @@ +/* + * Copyright 2024 Responsive Computing, Inc. + * + * This source code is licensed under the Responsive Business Source License Agreement v1.0 + * available at: + * + * https://www.responsive.dev/legal/responsive-bsl-10 + * + * This software requires a valid Commercial License Key for production use. Trial and commercial + * licenses can be obtained at https://www.responsive.dev + */ + +package dev.responsive.kafka.internal.db; + +import dev.responsive.kafka.internal.db.partitioning.Segmenter; +import dev.responsive.kafka.internal.db.partitioning.Segmenter.SegmentPartition; +import dev.responsive.kafka.internal.stores.RemoteWriteResult; +import dev.responsive.kafka.internal.utils.PendingFlushSegmentMetadata; +import dev.responsive.kafka.internal.utils.WindowedKey; + +public abstract class SegmentedWindowFlushManager implements WindowFlushManager { + + private final int kafkaPartition; + private final Segmenter segmenter; + private final PendingFlushSegmentMetadata pendingFlushSegmentMetadata; + + public SegmentedWindowFlushManager( + final String tableName, + final int kafkaPartition, + final Segmenter segmenter, + final long streamTime + ) { + this.kafkaPartition = kafkaPartition; + this.segmenter = segmenter; + this.pendingFlushSegmentMetadata = + new PendingFlushSegmentMetadata(tableName, kafkaPartition, streamTime); + } + + @Override + public long streamTime() { + return pendingFlushSegmentMetadata.batchStreamTime(); + } + + @Override + public void writeAdded(final WindowedKey key) { + pendingFlushSegmentMetadata.updateStreamTime(key.windowStartMs); + } + + @Override + public RemoteWriteResult preFlush() { + final var pendingRoll = pendingFlushSegmentMetadata.prepareRoll(segmenter); + + for (final long segmentStartTimestamp : pendingRoll.segmentsToCreate()) { + final var createResult = + createSegment(new SegmentPartition(kafkaPartition, segmentStartTimestamp)); + if (!createResult.wasApplied()) { + return createResult; + } + } + + return RemoteWriteResult.success(null); + } + + @Override + public RemoteWriteResult postFlush(final long consumedOffset) { + + final var metadataResult = updateOffsetAndStreamTime( + consumedOffset, + pendingFlushSegmentMetadata.batchStreamTime() + ); + if (!metadataResult.wasApplied()) { + return metadataResult; + } + + for (final long segmentStartTimestamp : pendingFlushSegmentMetadata.segmentRoll() + .segmentsToExpire()) { + final var deleteResult = + deleteSegment(new SegmentPartition(kafkaPartition, segmentStartTimestamp)); + if (!deleteResult.wasApplied()) { + return deleteResult; + } + } + + pendingFlushSegmentMetadata.finalizeRoll(); + return RemoteWriteResult.success(null); + } + + /** + * Persist the latest consumed offset and stream-time corresponding to the batch that was just + * flushed to the remote table + */ + protected abstract RemoteWriteResult updateOffsetAndStreamTime( + final long consumedOffset, + final long streamTime + ); + + /** + * "Create" the passed-in segment by executing whatever preparations are needed to + * support writes to this segment. Assumed to be completed synchronously + */ + protected abstract RemoteWriteResult createSegment( + final SegmentPartition partition + ); + + /** + * "Delete" the passed-in expired segment by executing whatever cleanup is needed to + * release the resources held by this segment and reclaim the storage it previously held + */ + protected abstract RemoteWriteResult deleteSegment( + final SegmentPartition partition + ); + +} diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/WindowFlushManager.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/WindowFlushManager.java index 31219b38f..ee9e2fd54 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/WindowFlushManager.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/WindowFlushManager.java @@ -1,5 +1,5 @@ /* - * Copyright 2024 Responsive Computing, Inc. + * Copyright 2025 Responsive Computing, Inc. * * This source code is licensed under the Responsive Business Source License Agreement v1.0 * available at: @@ -12,101 +12,14 @@ package dev.responsive.kafka.internal.db; -import dev.responsive.kafka.internal.db.partitioning.Segmenter; -import dev.responsive.kafka.internal.db.partitioning.Segmenter.SegmentPartition; -import dev.responsive.kafka.internal.stores.RemoteWriteResult; -import dev.responsive.kafka.internal.utils.PendingFlushSegmentMetadata; import dev.responsive.kafka.internal.utils.WindowedKey; -public abstract class WindowFlushManager implements FlushManager { - - private final int kafkaPartition; - private final Segmenter segmenter; - private final PendingFlushSegmentMetadata pendingFlushSegmentMetadata; - - public WindowFlushManager( - final String tableName, - final int kafkaPartition, - final Segmenter segmenter, - final long streamTime - ) { - this.kafkaPartition = kafkaPartition; - this.segmenter = segmenter; - this.pendingFlushSegmentMetadata = - new PendingFlushSegmentMetadata(tableName, kafkaPartition, streamTime); - } - - public long streamTime() { - return pendingFlushSegmentMetadata.batchStreamTime(); - } - - @Override - public void writeAdded(final WindowedKey key) { - pendingFlushSegmentMetadata.updateStreamTime(key.windowStartMs); - } - - @Override - public RemoteWriteResult preFlush() { - final var pendingRoll = pendingFlushSegmentMetadata.prepareRoll(segmenter); - - for (final long segmentStartTimestamp : pendingRoll.segmentsToCreate()) { - final var createResult = - createSegment(new SegmentPartition(kafkaPartition, segmentStartTimestamp)); - if (!createResult.wasApplied()) { - return createResult; - } - } - - return RemoteWriteResult.success(null); - } - - @Override - public RemoteWriteResult postFlush(final long consumedOffset) { - - final var metadataResult = updateOffsetAndStreamTime( - consumedOffset, - pendingFlushSegmentMetadata.batchStreamTime() - ); - if (!metadataResult.wasApplied()) { - return metadataResult; - } - - for (final long segmentStartTimestamp : pendingFlushSegmentMetadata.segmentRoll() - .segmentsToExpire()) { - final var deleteResult = - deleteSegment(new SegmentPartition(kafkaPartition, segmentStartTimestamp)); - if (!deleteResult.wasApplied()) { - return deleteResult; - } - } - - pendingFlushSegmentMetadata.finalizeRoll(); - return RemoteWriteResult.success(null); - } - - /** - * Persist the latest consumed offset and stream-time corresponding to the batch that was just - * flushed to the remote table - */ - protected abstract RemoteWriteResult updateOffsetAndStreamTime( - final long consumedOffset, - final long streamTime - ); - - /** - * "Create" the passed-in segment by executing whatever preparations are needed to - * support writes to this segment. Assumed to be completed synchronously - */ - protected abstract RemoteWriteResult createSegment( - final SegmentPartition partition - ); +/** + * + * @param

Table partition type + */ +public interface WindowFlushManager

extends FlushManager { - /** - * "Delete" the passed-in expired segment by executing whatever cleanup is needed to - * release the resources held by this segment and reclaim the storage it previously held - */ - protected abstract RemoteWriteResult deleteSegment( - final SegmentPartition partition - ); + long streamTime(); } diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/PssTablePartitioner.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/PssTablePartitioner.java index 241d4e492..2ce1cc619 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/PssTablePartitioner.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/PssTablePartitioner.java @@ -15,18 +15,20 @@ import dev.responsive.kafka.internal.db.partitioning.TablePartitioner; import dev.responsive.kafka.internal.db.rs3.client.LssId; import java.util.Objects; -import org.apache.kafka.common.utils.Bytes; -public class PssTablePartitioner implements TablePartitioner { +public abstract class PssTablePartitioner implements TablePartitioner { private final PssPartitioner pssPartitioner; public PssTablePartitioner(final PssPartitioner pssPartitioner) { this.pssPartitioner = Objects.requireNonNull(pssPartitioner); } + public abstract byte[] serialize(K key); + @Override - public Integer tablePartition(int kafkaPartition, Bytes key) { - return pssPartitioner.pss(key.get(), new LssId(kafkaPartition)); + public Integer tablePartition(int kafkaPartition, K key) { + final var serializedKey = serialize(key); + return pssPartitioner.pss(serializedKey, new LssId(kafkaPartition)); } @Override diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3KVFlushManager.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3KVFlushManager.java index aea6a20e6..d9fa81fe3 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3KVFlushManager.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3KVFlushManager.java @@ -17,33 +17,23 @@ import dev.responsive.kafka.internal.db.partitioning.TablePartitioner; import dev.responsive.kafka.internal.db.rs3.client.LssId; import dev.responsive.kafka.internal.db.rs3.client.RS3Client; -import dev.responsive.kafka.internal.db.rs3.client.RS3TransientException; -import dev.responsive.kafka.internal.db.rs3.client.StreamSender; -import dev.responsive.kafka.internal.db.rs3.client.StreamSenderMessageReceiver; import dev.responsive.kafka.internal.db.rs3.client.WalEntry; import dev.responsive.kafka.internal.stores.RemoteWriteResult; -import java.util.ArrayList; import java.util.HashMap; -import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.UUID; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionException; import java.util.concurrent.CompletionStage; -import java.util.function.Consumer; import org.apache.kafka.common.utils.Bytes; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; class RS3KVFlushManager extends KVFlushManager { - private static final Logger LOG = LoggerFactory.getLogger(RS3KVFlushManager.class); - private final UUID storeId; private final RS3Client rs3Client; private final LssId lssId; private final RS3KVTable table; - private final HashMap> writtenOffsets; + private final Map> writtenOffsets; private final int kafkaPartition; private final PssPartitioner pssPartitioner; private final HashMap writers = new HashMap<>(); @@ -53,7 +43,7 @@ public RS3KVFlushManager( final RS3Client rs3Client, final LssId lssId, final RS3KVTable table, - final HashMap> writtenOffsets, + final Map> writtenOffsets, final int kafkaPartition, final PssPartitioner pssPartitioner ) { @@ -73,7 +63,12 @@ public String tableName() { @Override public TablePartitioner partitioner() { - return new PssTablePartitioner(pssPartitioner); + return new PssTablePartitioner<>(pssPartitioner) { + @Override + public byte[] serialize(final Bytes key) { + return key.get(); + } + }; } @Override @@ -162,136 +157,43 @@ public CompletionStage> flush() { } } - private static final class RS3StreamFactory { - private final UUID storeId; - private final RS3Client rs3Client; - private final int pssId; - private final LssId lssId; - private final long endOffset; - private final Optional expectedWrittenOffset; - - private RS3StreamFactory( - final UUID storeId, - final RS3Client rs3Client, - final int pssId, - final LssId lssId, - final long endOffset, - final Optional expectedWrittenOffset - ) { - this.storeId = storeId; - this.rs3Client = rs3Client; - this.pssId = pssId; - this.lssId = lssId; - this.endOffset = endOffset; - this.expectedWrittenOffset = expectedWrittenOffset; - } - - StreamSenderMessageReceiver> writeWalSegmentAsync() { - return rs3Client.writeWalSegmentAsync( - storeId, - lssId, - pssId, - expectedWrittenOffset, - endOffset - ); - } - - Optional writeWalSegmentSync(List entries) { - return rs3Client.writeWalSegment( - storeId, - lssId, - pssId, - expectedWrittenOffset, - endOffset, - entries - ); - } - - } - - private static final class RS3KVWriter implements RemoteWriter { - private final RS3StreamFactory streamFactory; + private static final class RS3KVWriter extends RS3Writer { private final RS3KVTable table; - private final int kafkaPartition; - private final List retryBuffer = new ArrayList<>(); - private final StreamSenderMessageReceiver> sendRecv; private RS3KVWriter( final UUID storeId, final RS3Client rs3Client, - final RS3KVTable table, + final RS3KVTable rs3Table, final int pssId, final LssId lssId, final long endOffset, final Optional expectedWrittenOffset, final int kafkaPartition ) { - this.table = Objects.requireNonNull(table); - this.streamFactory = new RS3StreamFactory( - storeId, - rs3Client, - pssId, - lssId, - endOffset, - expectedWrittenOffset - ); - this.kafkaPartition = kafkaPartition; - this.sendRecv = streamFactory.writeWalSegmentAsync(); - } - - long endOffset() { - return streamFactory.endOffset; + super(storeId, rs3Client, pssId, lssId, endOffset, expectedWrittenOffset, kafkaPartition); + this.table = rs3Table; } @Override - public void insert(final Bytes key, final byte[] value, final long timestampMs) { - maybeSendNext(table.insert(kafkaPartition, key, value, timestampMs)); - } - - @Override - public void delete(final Bytes key) { - maybeSendNext(table.delete(kafkaPartition, key)); - } - - private void maybeSendNext(WalEntry entry) { - retryBuffer.add(entry); - ifActiveStream(sender -> sender.sendNext(entry)); - } - - private void ifActiveStream(Consumer> streamConsumer) { - if (sendRecv.isActive()) { - try { - streamConsumer.accept(sendRecv.sender()); - } catch (final RS3TransientException e) { - // Retry the stream in flush() - } - } + protected WalEntry createInsert( + final Bytes key, + final byte[] value, + final long timestampMs + ) { + return table.insert( + kafkaPartition(), + key, + value, + timestampMs + ); } @Override - public CompletionStage> flush() { - ifActiveStream(StreamSender::finish); - - return sendRecv.completion().handle((result, throwable) -> { - Optional flushedOffset = result; - - var cause = throwable; - if (throwable instanceof CompletionException) { - cause = throwable.getCause(); - } - - if (cause instanceof RS3TransientException) { - flushedOffset = streamFactory.writeWalSegmentSync(retryBuffer); - } else if (cause instanceof RuntimeException) { - throw (RuntimeException) throwable; - } else if (cause != null) { - throw new RuntimeException(throwable); - } - - LOG.debug("last flushed offset for pss/lss {}/{} is {}", - streamFactory.pssId, streamFactory.lssId, flushedOffset); - return RemoteWriteResult.success(kafkaPartition); - }); + protected WalEntry createDelete(final Bytes key) { + return table.delete( + kafkaPartition(), + key + ); } } diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3KVTable.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3KVTable.java index a0c59b562..909b489b7 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3KVTable.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3KVTable.java @@ -15,17 +15,16 @@ import dev.responsive.kafka.internal.db.KVFlushManager; import dev.responsive.kafka.internal.db.RemoteKVTable; import dev.responsive.kafka.internal.db.rs3.client.LssId; +import dev.responsive.kafka.internal.db.rs3.client.LssMetadata; import dev.responsive.kafka.internal.db.rs3.client.MeteredRS3Client; import dev.responsive.kafka.internal.db.rs3.client.Put; import dev.responsive.kafka.internal.db.rs3.client.RS3Client; +import dev.responsive.kafka.internal.db.rs3.client.RS3ClientUtil; import dev.responsive.kafka.internal.db.rs3.client.WalEntry; import dev.responsive.kafka.internal.metrics.ResponsiveMetrics; import dev.responsive.kafka.internal.stores.ResponsiveStoreRegistration; -import java.util.HashMap; import java.util.Objects; -import java.util.Optional; import java.util.UUID; -import java.util.stream.Collectors; import org.apache.kafka.common.serialization.Serializer; import org.apache.kafka.common.utils.Bytes; import org.apache.kafka.streams.state.KeyValueIterator; @@ -38,6 +37,7 @@ public class RS3KVTable implements RemoteKVTable { private final String name; private final UUID storeId; private final RS3Client rs3Client; + private final RS3ClientUtil rs3ClientUtil; private final PssPartitioner pssPartitioner; private LssId lssId; private Long fetchOffset = ResponsiveStoreRegistration.NO_COMMITTED_OFFSET; @@ -58,6 +58,7 @@ public RS3KVTable( Objects.requireNonNull(responsiveMetrics), Objects.requireNonNull(scopeBuilder) ); + this.rs3ClientUtil = new RS3ClientUtil(storeId, rs3Client, pssPartitioner); this.pssPartitioner = Objects.requireNonNull(pssPartitioner); } @@ -74,40 +75,14 @@ public KVFlushManager init(final int kafkaPartition) { this.lssId = new LssId(kafkaPartition); - // TODO: we should write an empty segment periodically to any PSS that we haven't - // written to to bump the written offset - final HashMap> lastWrittenOffset = new HashMap<>(); - for (final int pss : pssPartitioner.pssForLss(this.lssId)) { - final var offsets = rs3Client.getCurrentOffsets(storeId, lssId, pss); - lastWrittenOffset.put(pss, offsets.writtenOffset()); - } - final var fetchOffsetOrMinusOne = lastWrittenOffset.values().stream() - .map(v -> v.orElse(-1L)) - .min(Long::compare) - .orElse(-1L); - if (fetchOffsetOrMinusOne == -1) { - this.fetchOffset = ResponsiveStoreRegistration.NO_COMMITTED_OFFSET; - } else { - this.fetchOffset = fetchOffsetOrMinusOne; - } - - final var writtenOffsetsStr = lastWrittenOffset.entrySet().stream() - .map(e -> String.format("%s -> %s", - e.getKey(), - e.getValue().map(Object::toString).orElse("none"))) - .collect(Collectors.joining(",")); - LOG.info("restore rs3 kv table from offset {} for {}. recorded written offsets: {}", - fetchOffset, - kafkaPartition, - writtenOffsetsStr - ); - - flushManager = new RS3KVFlushManager( + LssMetadata lssMetadata = rs3ClientUtil.fetchLssMetadata(lssId); + this.fetchOffset = lssMetadata.lastWrittenOffset(); + this.flushManager = new RS3KVFlushManager( storeId, rs3Client, lssId, this, - lastWrittenOffset, + lssMetadata.writtenOffsets(), kafkaPartition, pssPartitioner ); diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3TableFactory.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3TableFactory.java index d1802e130..f6e702511 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3TableFactory.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3TableFactory.java @@ -14,6 +14,7 @@ import dev.responsive.kafka.api.config.ResponsiveConfig; import dev.responsive.kafka.internal.db.RemoteKVTable; +import dev.responsive.kafka.internal.db.RemoteWindowTable; import dev.responsive.kafka.internal.db.rs3.client.WalEntry; import dev.responsive.kafka.internal.db.rs3.client.grpc.GrpcRS3Client; import dev.responsive.kafka.internal.metrics.ResponsiveMetrics; @@ -34,15 +35,9 @@ public RemoteKVTable kvTable( final ResponsiveMetrics responsiveMetrics, final ResponsiveMetrics.MetricScopeBuilder scopeBuilder ) { - Map storeIdMapping = config.getMap( - ResponsiveConfig.RS3_LOGICAL_STORE_MAPPING_CONFIG); - final String storeIdHex = storeIdMapping.get(name); - if (storeIdHex == null) { - throw new ConfigException("Failed to find store ID mapping for table " + name); - } - final UUID storeId = UUID.fromString(storeIdHex); - final PssPartitioner pssPartitioner = new PssDirectPartitioner(); + final var storeId = lookupStoreId(config, name); + final var pssPartitioner = new PssDirectPartitioner(); final var rs3Client = connector.connect(); return new RS3KVTable( name, @@ -54,6 +49,35 @@ public RemoteKVTable kvTable( ); } + public RemoteWindowTable windowTable( + final String name, + final ResponsiveConfig config, + final ResponsiveMetrics responsiveMetrics, + final ResponsiveMetrics.MetricScopeBuilder scopeBuilder + ) { + final var storeId = lookupStoreId(config, name); + final var pssPartitioner = new PssDirectPartitioner(); + final var rs3Client = connector.connect(); + return new RS3WindowTable( + name, + storeId, + rs3Client, + pssPartitioner, + responsiveMetrics, + scopeBuilder + ); + } + + private UUID lookupStoreId(ResponsiveConfig config, String name) { + Map storeIdMapping = config.getMap( + ResponsiveConfig.RS3_LOGICAL_STORE_MAPPING_CONFIG); + final String storeIdHex = storeIdMapping.get(name); + if (storeIdHex == null) { + throw new ConfigException("Failed to find store ID mapping for table " + name); + } + return UUID.fromString(storeIdHex); + } + public void close() { } } diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowFlushManager.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowFlushManager.java new file mode 100644 index 000000000..a84bccd8b --- /dev/null +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowFlushManager.java @@ -0,0 +1,177 @@ +/* + * Copyright 2025 Responsive Computing, Inc. + * + * This source code is licensed under the Responsive Business Source License Agreement v1.0 + * available at: + * + * https://www.responsive.dev/legal/responsive-bsl-10 + * + * This software requires a valid Commercial License Key for production use. Trial and commercial + * licenses can be obtained at https://www.responsive.dev + */ + +package dev.responsive.kafka.internal.db.rs3; + +import dev.responsive.kafka.internal.db.RemoteWriter; +import dev.responsive.kafka.internal.db.WindowFlushManager; +import dev.responsive.kafka.internal.db.partitioning.TablePartitioner; +import dev.responsive.kafka.internal.db.rs3.client.LssId; +import dev.responsive.kafka.internal.db.rs3.client.RS3Client; +import dev.responsive.kafka.internal.db.rs3.client.WalEntry; +import dev.responsive.kafka.internal.stores.RemoteWriteResult; +import dev.responsive.kafka.internal.utils.WindowedKey; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; + +class RS3WindowFlushManager implements WindowFlushManager { + private final UUID storeId; + private final RS3Client rs3Client; + private final LssId lssId; + private final RS3WindowTable table; + private final int kafkaPartition; + private final PssPartitioner pssPartitioner; + private final RS3WindowedKeySerde keySerde; + private final Map> writtenOffsets; + + private long streamTime; + private final Map writers = new HashMap<>(); + + public RS3WindowFlushManager( + final UUID storeId, + final RS3Client rs3Client, + final LssId lssId, + final RS3WindowTable table, + final int kafkaPartition, + final PssPartitioner pssPartitioner, + final RS3WindowedKeySerde keySerde, + final Map> writtenOffsets, + final long initialStreamTime + ) { + this.storeId = storeId; + this.rs3Client = rs3Client; + this.lssId = lssId; + this.table = table; + this.kafkaPartition = kafkaPartition; + this.pssPartitioner = pssPartitioner; + this.keySerde = keySerde; + this.writtenOffsets = writtenOffsets; + this.streamTime = initialStreamTime; + } + + @Override + public String tableName() { + return table.name(); + } + + @Override + public TablePartitioner partitioner() { + return new PssTablePartitioner<>(pssPartitioner) { + @Override + public byte[] serialize(final WindowedKey key) { + return keySerde.serialize(key); + } + }; + } + + @Override + public RemoteWriter createWriter( + final Integer pssId, + final long consumedOffset + ) { + final var writer = new Rs3WindowWriter( + storeId, + rs3Client, + table, + pssId, + lssId, + consumedOffset, + writtenOffsets.get(pssId), + kafkaPartition + ); + writers.put(pssId, writer); + return writer; + } + + @Override + public void writeAdded(final WindowedKey key) { + streamTime = Long.max(streamTime, key.windowStartMs); + } + + @Override + public RemoteWriteResult preFlush() { + return RemoteWriteResult.success(null); + } + + @Override + public RemoteWriteResult postFlush(final long consumedOffset) { + for (final var entry : writers.entrySet()) { + writtenOffsets.put(entry.getKey(), Optional.of(entry.getValue().endOffset())); + } + writers.clear(); + return RemoteWriteResult.success(null); + } + + @Override + public String failedFlushInfo( + final long batchOffset, + final Integer failedTablePartition + ) { + return String.format(""); + } + + @Override + public String logPrefix() { + return tableName() + ".rs3.flushmanager"; + } + + @Override + public long streamTime() { + return 0; + } + + public Optional writtenOffset(final int pssId) { + return writtenOffsets.get(pssId); + } + + private static class Rs3WindowWriter extends RS3Writer { + private final RS3WindowTable table; + + protected Rs3WindowWriter( + final UUID storeId, + final RS3Client rs3Client, + final RS3WindowTable table, + final int pssId, + final LssId lssId, + final long endOffset, + final Optional expectedWrittenOffset, + final int kafkaPartition + ) { + super(storeId, rs3Client, pssId, lssId, endOffset, expectedWrittenOffset, kafkaPartition); + this.table = table; + } + + @Override + protected WalEntry createInsert( + final WindowedKey key, + final byte[] value, + final long timestampMs + ) { + return table.insert( + kafkaPartition(), + key, + value, + timestampMs + ); + } + + @Override + protected WalEntry createDelete(final WindowedKey key) { + return table.delete( + kafkaPartition(), + key + ); + } + } +} diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowTable.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowTable.java new file mode 100644 index 000000000..2b55b9fbb --- /dev/null +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowTable.java @@ -0,0 +1,226 @@ +package dev.responsive.kafka.internal.db.rs3; + +import dev.responsive.kafka.internal.db.RemoteWindowTable; +import dev.responsive.kafka.internal.db.WindowFlushManager; +import dev.responsive.kafka.internal.db.rs3.client.LssId; +import dev.responsive.kafka.internal.db.rs3.client.LssMetadata; +import dev.responsive.kafka.internal.db.rs3.client.MeteredRS3Client; +import dev.responsive.kafka.internal.db.rs3.client.Put; +import dev.responsive.kafka.internal.db.rs3.client.RS3Client; +import dev.responsive.kafka.internal.db.rs3.client.RS3ClientUtil; +import dev.responsive.kafka.internal.db.rs3.client.WalEntry; +import dev.responsive.kafka.internal.metrics.ResponsiveMetrics; +import dev.responsive.kafka.internal.utils.WindowedKey; +import java.util.Objects; +import java.util.UUID; +import org.apache.kafka.common.utils.Bytes; +import org.apache.kafka.streams.state.KeyValueIterator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class RS3WindowTable implements RemoteWindowTable { + private static final Logger LOG = LoggerFactory.getLogger(RS3WindowTable.class); + + private final String name; + private final UUID storeId; + private final RS3Client rs3Client; + private final RS3ClientUtil rs3ClientUtil; + private final PssPartitioner pssPartitioner; + private final RS3WindowedKeySerde keySerde = new RS3WindowedKeySerde(); + + // Initialized in `init()` + private LssId lssId; + private Long fetchOffset; + private RS3WindowFlushManager flushManager; + + public RS3WindowTable( + final String name, + final UUID storeId, + final RS3Client rs3Client, + final PssPartitioner pssPartitioner, + final ResponsiveMetrics responsiveMetrics, + final ResponsiveMetrics.MetricScopeBuilder scopeBuilder + ) { + this( + name, + storeId, + new MeteredRS3Client( + Objects.requireNonNull(rs3Client), + Objects.requireNonNull(responsiveMetrics), + Objects.requireNonNull(scopeBuilder) + ), + pssPartitioner + ); + } + + public RS3WindowTable( + final String name, + final UUID storeId, + final RS3Client rs3Client, + final PssPartitioner pssPartitioner + ) { + this.name = Objects.requireNonNull(name); + this.storeId = Objects.requireNonNull(storeId); + this.rs3Client = Objects.requireNonNull(rs3Client); // TODO: Use metered client + this.rs3ClientUtil = new RS3ClientUtil(storeId, rs3Client, pssPartitioner); + this.pssPartitioner = Objects.requireNonNull(pssPartitioner); + } + + @Override + public WindowFlushManager init(final int kafkaPartition) { + if (flushManager != null) { + LOG.error("already initialized for store {}:{}", name, kafkaPartition); + throw new IllegalStateException(String.format( + "already initialized for store %s:%d", + name, + kafkaPartition + )); + } + + this.lssId = new LssId(kafkaPartition); + LssMetadata lssMetadata = rs3ClientUtil.fetchLssMetadata(lssId); + this.fetchOffset = lssMetadata.lastWrittenOffset(); + + long initialStreamTime = -1; // TODO: Initialize from RS3 metadata? + this.flushManager = new RS3WindowFlushManager( + storeId, + rs3Client, + lssId, + this, + kafkaPartition, + pssPartitioner, + keySerde, + lssMetadata.writtenOffsets(), + initialStreamTime + ); + return flushManager; + } + + private void throwIfPartitionNotInitialized(int kafkaPartition) { + if (flushManager == null) { + throw new IllegalStateException(String.format( + "Cannot complete operation on store %s is not yet initialized", + name + )); + } else if (lssId.id() != kafkaPartition) { + throw new IllegalStateException(String.format( + "Cannot complete operation on store %s for kafka partition %d since the store " + + "was initialized for a separate partition %d", + name, + kafkaPartition, + lssId.id() + )); + } + } + + @Override + public byte[] fetch( + final int kafkaPartition, + final Bytes key, + final long windowStart + ) { + throwIfPartitionNotInitialized(kafkaPartition); + final int pssId = pssPartitioner.pss(key.get(), this.lssId); + final var windowKey = new WindowedKey(key, windowStart); + final var windowKeyBytes = keySerde.serialize(windowKey); + + return rs3Client.get( + storeId, + lssId, + pssId, + flushManager.writtenOffset(pssId), + windowKeyBytes + ).orElse(null); + } + + @Override + public KeyValueIterator fetch( + final int kafkaPartition, + final Bytes key, + final long timeFrom, + final long timeTo + ) { + throwIfPartitionNotInitialized(kafkaPartition); + throw new UnsupportedOperationException(); + } + + @Override + public KeyValueIterator backFetch( + final int kafkaPartition, + final Bytes key, + final long timeFrom, + final long timeTo + ) { + throwIfPartitionNotInitialized(kafkaPartition); + throw new UnsupportedOperationException(); + } + + @Override + public KeyValueIterator fetchRange( + final int kafkaPartition, + final Bytes fromKey, + final Bytes toKey, + final long timeFrom, + final long timeTo + ) { + throwIfPartitionNotInitialized(kafkaPartition); + throw new UnsupportedOperationException(); + } + + @Override + public KeyValueIterator backFetchRange( + final int kafkaPartition, + final Bytes fromKey, + final Bytes toKey, + final long timeFrom, + final long timeTo + ) { + throwIfPartitionNotInitialized(kafkaPartition); + throw new UnsupportedOperationException(); + } + + @Override + public KeyValueIterator fetchAll( + final int kafkaPartition, + final long timeFrom, + final long timeTo + ) { + throwIfPartitionNotInitialized(kafkaPartition); + throw new UnsupportedOperationException(); + } + + @Override + public KeyValueIterator backFetchAll( + final int kafkaPartition, + final long timeFrom, + final long timeTo + ) { + throwIfPartitionNotInitialized(kafkaPartition); + throw new UnsupportedOperationException(); + } + + @Override + public String name() { + return name; + } + + @Override + public WalEntry insert( + final int kafkaPartition, + final WindowedKey key, + final byte[] value, + final long timestampMs + ) { + return new Put(keySerde.serialize(key), value); + } + + @Override + public WalEntry delete(final int kafkaPartition, final WindowedKey key) { + return new Put(keySerde.serialize(key), null); + } + + @Override + public long fetchOffset(final int kafkaPartition) { + return fetchOffset; + } +} diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowedKeySerde.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowedKeySerde.java new file mode 100644 index 000000000..21546a4ee --- /dev/null +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowedKeySerde.java @@ -0,0 +1,33 @@ +package dev.responsive.kafka.internal.db.rs3; + +import dev.responsive.kafka.internal.utils.WindowedKey; +import java.nio.ByteBuffer; +import org.apache.kafka.common.utils.Bytes; + +/** + * 1: key-timestamp + * 2. prefix-timestamp-suffix + * 3. + * + * range scans not possible with any kind of hash + */ +public class RS3WindowedKeySerde { + + public byte[] serialize(WindowedKey key) { + byte[] result = new byte[key.key.get().length + 8]; + final ByteBuffer buffer = ByteBuffer.wrap(result); + buffer.put(key.key.get()); + buffer.putLong(key.windowStartMs); + return result; + } + + public WindowedKey deserialize(byte[] bytes) { + final var buffer = ByteBuffer.wrap(bytes); + final var keyLength = bytes.length - 8; + final var keyBytes = new byte[keyLength]; + buffer.get(keyBytes); + final var windowStartMs = buffer.getLong(); + return new WindowedKey(Bytes.wrap(keyBytes), windowStartMs); + } + +} diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3Writer.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3Writer.java new file mode 100644 index 000000000..f8fa34a42 --- /dev/null +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3Writer.java @@ -0,0 +1,144 @@ +package dev.responsive.kafka.internal.db.rs3; + +import dev.responsive.kafka.internal.db.RemoteWriter; +import dev.responsive.kafka.internal.db.rs3.client.LssId; +import dev.responsive.kafka.internal.db.rs3.client.RS3Client; +import dev.responsive.kafka.internal.db.rs3.client.RS3TransientException; +import dev.responsive.kafka.internal.db.rs3.client.StreamSender; +import dev.responsive.kafka.internal.db.rs3.client.StreamSenderMessageReceiver; +import dev.responsive.kafka.internal.db.rs3.client.WalEntry; +import dev.responsive.kafka.internal.stores.RemoteWriteResult; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.CompletionException; +import java.util.concurrent.CompletionStage; +import java.util.function.Consumer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public abstract class RS3Writer implements RemoteWriter { + private static final Logger LOG = LoggerFactory.getLogger(RS3Writer.class); + + private final UUID storeId; + private final RS3Client rs3Client; + private final int pssId; + private final LssId lssId; + private final long endOffset; + private final Optional expectedWrittenOffset; + private final int kafkaPartition; + private final List retryBuffer = new ArrayList<>(); + private final StreamSenderMessageReceiver> sendRecv; + + protected RS3Writer( + final UUID storeId, + final RS3Client rs3Client, + final int pssId, + final LssId lssId, + final long endOffset, + final Optional expectedWrittenOffset, + final int kafkaPartition + ) { + this.storeId = storeId; + this.rs3Client = rs3Client; + this.pssId = pssId; + this.lssId = lssId; + this.endOffset = endOffset; + this.expectedWrittenOffset = expectedWrittenOffset; + this.kafkaPartition = kafkaPartition; + this.sendRecv = writeWalSegmentAsync(); + } + + protected abstract WalEntry createInsert( + K key, + byte[] value, + long timestampMs + ); + + protected abstract WalEntry createDelete( + K key + ); + + public long endOffset() { + return endOffset; + } + + public int kafkaPartition() { + return kafkaPartition; + } + + @Override + public void insert(final K key, final byte[] value, final long timestampMs) { + final var insertEntry = createInsert(key, value, timestampMs); + maybeSendNext(insertEntry); + } + + @Override + public void delete(final K key) { + final var deleteEntry = createDelete(key); + maybeSendNext(deleteEntry); + } + + private void maybeSendNext(WalEntry entry) { + retryBuffer.add(entry); + ifActiveStream(sender -> sender.sendNext(entry)); + } + + private void ifActiveStream(Consumer> streamConsumer) { + if (sendRecv.isActive()) { + try { + streamConsumer.accept(sendRecv.sender()); + } catch (final RS3TransientException e) { + // Retry the stream in flush() + } + } + } + + @Override + public CompletionStage> flush() { + ifActiveStream(StreamSender::finish); + + return sendRecv.completion().handle((result, throwable) -> { + Optional flushedOffset = result; + + var cause = throwable; + if (throwable instanceof CompletionException) { + cause = throwable.getCause(); + } + + if (cause instanceof RS3TransientException) { + flushedOffset = writeWalSegmentSync(retryBuffer); + } else if (cause instanceof RuntimeException) { + throw (RuntimeException) throwable; + } else if (cause != null) { + throw new RuntimeException(throwable); + } + + LOG.debug("last flushed offset for pss/lss {}/{} is {}", pssId, lssId, flushedOffset); + return RemoteWriteResult.success(kafkaPartition); + }); + } + + StreamSenderMessageReceiver> writeWalSegmentAsync() { + return rs3Client.writeWalSegmentAsync( + storeId, + lssId, + pssId, + expectedWrittenOffset, + endOffset + ); + } + + Optional writeWalSegmentSync(List entries) { + return rs3Client.writeWalSegment( + storeId, + lssId, + pssId, + expectedWrittenOffset, + endOffset, + entries + ); + } + +} diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/LssMetadata.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/LssMetadata.java new file mode 100644 index 000000000..c1d7c8145 --- /dev/null +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/LssMetadata.java @@ -0,0 +1,36 @@ +package dev.responsive.kafka.internal.db.rs3.client; + +import java.util.Map; +import java.util.Optional; + +public class LssMetadata { + private final long lastWrittenOffset; + private final Map> writtenOffsets; + + public LssMetadata(long lastWrittenOffset, Map> writtenOffsets) { + this.lastWrittenOffset = lastWrittenOffset; + this.writtenOffsets = writtenOffsets; + } + + /** + * Get the last written offset for the LSS. + * + * @return The last written offset or + * {@link dev.responsive.kafka.internal.stores.ResponsiveStoreRegistration#NO_COMMITTED_OFFSET} + * if there is none + */ + public long lastWrittenOffset() { + return lastWrittenOffset; + } + + /** + * Get the last written offset for each PSS mapped to the LSS. It may return + * `Optional.empty()` if the PSS has no written offsets yet. + * + * @return a map of the last written offsets for each PSS ID + */ + public Map> writtenOffsets() { + return writtenOffsets; + } + +} diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/RS3ClientUtil.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/RS3ClientUtil.java new file mode 100644 index 000000000..11ecefb3b --- /dev/null +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/RS3ClientUtil.java @@ -0,0 +1,75 @@ +/* + * Copyright 2025 Responsive Computing, Inc. + * + * This source code is licensed under the Responsive Business Source License Agreement v1.0 + * available at: + * + * https://www.responsive.dev/legal/responsive-bsl-10 + * + * This software requires a valid Commercial License Key for production use. Trial and commercial + * licenses can be obtained at https://www.responsive.dev + */ + +package dev.responsive.kafka.internal.db.rs3.client; + +import dev.responsive.kafka.internal.db.rs3.PssPartitioner; +import dev.responsive.kafka.internal.stores.ResponsiveStoreRegistration; +import java.util.HashMap; +import java.util.Optional; +import java.util.UUID; +import java.util.stream.Collectors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class RS3ClientUtil { + private static final Logger LOG = LoggerFactory.getLogger(RS3ClientUtil.class); + + private final UUID storeId; + private final PssPartitioner partitioner; + private final RS3Client client; + + public RS3ClientUtil( + final UUID storeId, + final RS3Client client, + final PssPartitioner partitioner + ) { + this.storeId = storeId; + this.partitioner = partitioner; + this.client = client; + } + + public LssMetadata fetchLssMetadata(LssId lssId) { + // TODO: we should write an empty segment periodically to any PSS that we haven't + // written to to bump the written offset + final HashMap> writtenOffsets = new HashMap<>(); + var lastWrittenOffset = ResponsiveStoreRegistration.NO_COMMITTED_OFFSET; + for (final int pss : partitioner.pssForLss(lssId)) { + final var offsets = client.getCurrentOffsets(storeId, lssId, pss); + final var writtenOffset = offsets.writtenOffset(); + + if (writtenOffset.isPresent()) { + final var offset = writtenOffset.get(); + if (offset > lastWrittenOffset) { + lastWrittenOffset = writtenOffset.get(); + } + } + + writtenOffsets.put(pss, offsets.writtenOffset()); + } + + final var writtenOffsetsStr = writtenOffsets.entrySet().stream() + .map(e -> String.format("%s -> %s", + e.getKey(), + e.getValue().map(Object::toString).orElse("none"))) + .collect(Collectors.joining(",")); + LOG.info("Restore RS3 table from offset {} for {}. recorded written offsets: {}", + lastWrittenOffset, + lssId, + writtenOffsetsStr + ); + + return new LssMetadata(lastWrittenOffset, writtenOffsets); + } + + +} diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/stores/SegmentedOperations.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/stores/RemoteWindowOperations.java similarity index 87% rename from kafka-client/src/main/java/dev/responsive/kafka/internal/stores/SegmentedOperations.java rename to kafka-client/src/main/java/dev/responsive/kafka/internal/stores/RemoteWindowOperations.java index b2ee42f6c..a5110dd34 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/stores/SegmentedOperations.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/stores/RemoteWindowOperations.java @@ -13,6 +13,7 @@ package dev.responsive.kafka.internal.stores; import static dev.responsive.kafka.api.config.ResponsiveConfig.STORAGE_BACKEND_TYPE_CONFIG; +import static dev.responsive.kafka.internal.config.InternalSessionConfigs.loadMetrics; import static dev.responsive.kafka.internal.config.InternalSessionConfigs.loadSessionClients; import static dev.responsive.kafka.internal.config.InternalSessionConfigs.loadStoreRegistry; import static dev.responsive.kafka.internal.stores.ResponsiveStoreRegistration.NO_COMMITTED_OFFSET; @@ -30,12 +31,14 @@ import dev.responsive.kafka.internal.db.WindowedKeySpec; import dev.responsive.kafka.internal.db.mongo.ResponsiveMongoClient; import dev.responsive.kafka.internal.db.partitioning.WindowSegmentPartitioner; +import dev.responsive.kafka.internal.metrics.ResponsiveMetrics; import dev.responsive.kafka.internal.metrics.ResponsiveRestoreListener; import dev.responsive.kafka.internal.utils.Iterators; import dev.responsive.kafka.internal.utils.Result; import dev.responsive.kafka.internal.utils.SessionClients; import dev.responsive.kafka.internal.utils.StoreUtil; import dev.responsive.kafka.internal.utils.TableName; +import dev.responsive.kafka.internal.utils.Utils; import dev.responsive.kafka.internal.utils.WindowedKey; import java.util.Collection; import java.util.Map; @@ -53,7 +56,7 @@ import org.apache.kafka.streams.state.WindowStoreIterator; import org.slf4j.Logger; -public class SegmentedOperations implements WindowOperations { +public class RemoteWindowOperations implements WindowOperations { private final Logger log; @@ -70,7 +73,7 @@ public class SegmentedOperations implements WindowOperations { private final long initialStreamTime; - public static SegmentedOperations create( + public static RemoteWindowOperations create( final TableName name, final StateStoreContext storeContext, final ResponsiveWindowParams params, @@ -81,7 +84,7 @@ public static SegmentedOperations create( final var log = new LogContext( String.format("window-store [%s] ", name.kafkaName()) - ).logger(SegmentedOperations.class); + ).logger(RemoteWindowOperations.class); final var context = asInternalProcessorContext(storeContext); final SessionClients sessionClients = loadSessionClients(appConfigs); @@ -106,6 +109,22 @@ public static SegmentedOperations create( case MONGO_DB: table = createMongo(params, sessionClients, partitioner, responsiveConfig); break; + case RS3: + final var responsiveMetrics = loadMetrics(appConfigs); + final var scopeBuilder = responsiveMetrics.storeLevelMetricScopeBuilder( + Utils.extractThreadIdFromThreadName(Thread.currentThread().getName()), + changelog, + params.name().tableName() + ); + + table = createRs3( + params, + sessionClients, + responsiveConfig, + responsiveMetrics, + scopeBuilder + ); + break; case NONE: log.error("Must configure a storage backend type using the config {}", STORAGE_BACKEND_TYPE_CONFIG); @@ -115,7 +134,7 @@ public static SegmentedOperations create( throw new IllegalStateException("Unrecognized value: " + sessionClients.storageBackend()); } - final WindowFlushManager flushManager = table.init(changelog.partition()); + final WindowFlushManager flushManager = table.init(changelog.partition()); log.info("Remote table {} is available for querying.", name.tableName()); @@ -152,7 +171,7 @@ public static SegmentedOperations create( ); storeRegistry.registerStore(registration); - return new SegmentedOperations( + return new RemoteWindowOperations( log, context, params, @@ -213,8 +232,28 @@ private static RemoteWindowTable createMongo( } } + private static RemoteWindowTable createRs3( + final ResponsiveWindowParams params, + final SessionClients sessionClients, + final ResponsiveConfig config, + final ResponsiveMetrics responsiveMetrics, + final ResponsiveMetrics.MetricScopeBuilder scopeBuilder + ) { + if (params.schemaType() == SchemaTypes.WindowSchema.STREAM || params.retainDuplicates()) { + throw new UnsupportedOperationException("Duplicate retention is not yet supported in RS3"); + } + + // TODO: Pass through retention period once we have support for TTL + return sessionClients.rs3TableFactory().windowTable( + params.name().tableName(), + config, + responsiveMetrics, + scopeBuilder + ); + } + @SuppressWarnings("rawtypes") - public SegmentedOperations( + public RemoteWindowOperations( final Logger log, final InternalProcessorContext context, final ResponsiveWindowParams params, diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/stores/ResponsiveWindowStore.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/stores/ResponsiveWindowStore.java index 5a78dc426..3e159e5e2 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/stores/ResponsiveWindowStore.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/stores/ResponsiveWindowStore.java @@ -109,7 +109,7 @@ public void init(final StateStoreContext storeContext, final StateStore root) { throw new IllegalStateException("Store " + name() + " was opened as a standby"); } - windowOperations = SegmentedOperations.create( + windowOperations = RemoteWindowOperations.create( name, storeContext, params, diff --git a/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/RS3WindowTableTest.java b/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/RS3WindowTableTest.java new file mode 100644 index 000000000..c8fdae1f5 --- /dev/null +++ b/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/RS3WindowTableTest.java @@ -0,0 +1,91 @@ +package dev.responsive.kafka.internal.db.rs3; + +import static java.util.Collections.singletonList; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.nullValue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.when; + +import dev.responsive.kafka.internal.db.rs3.client.CurrentOffsets; +import dev.responsive.kafka.internal.db.rs3.client.LssId; +import dev.responsive.kafka.internal.db.rs3.client.RS3Client; +import java.nio.charset.StandardCharsets; +import java.util.Optional; +import java.util.UUID; +import org.apache.kafka.common.utils.Bytes; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class RS3WindowTableTest { + + @Mock + private RS3Client client; + + @Mock + private PssPartitioner partitioner; + + @Test + public void shouldReturnValueFromFetchIfKeyFound() { + // Given: + final var storeId = UUID.randomUUID(); + final var kafkaPartition = 5; + final var lssId = new LssId(kafkaPartition); + final var table = new RS3WindowTable("table", storeId, client, partitioner); + final var key = Bytes.wrap("foo".getBytes(StandardCharsets.UTF_8)); + final var timestamp = 300L; + + // When: + final var pssId = 1; + final byte[] value = "bar".getBytes(StandardCharsets.UTF_8); + when(partitioner.pssForLss(lssId)).thenReturn(singletonList(pssId)); + when(partitioner.pss(any(), eq(lssId))).thenReturn(pssId); + when(client.getCurrentOffsets(storeId, lssId, pssId)) + .thenReturn(new CurrentOffsets(Optional.of(5L), Optional.of(5L))); + when(client.get( + eq(storeId), + eq(lssId), + eq(pssId), + eq(Optional.of(5L)), + any()) + ).thenReturn(Optional.of(value)); + + // Then: + table.init(kafkaPartition); + assertThat(table.fetch(kafkaPartition, key, timestamp), is(value)); + } + + @Test + public void shouldReturnNullFromFetchIfKeyNotFound() { + // Given: + final var storeId = UUID.randomUUID(); + final var kafkaPartition = 5; + final var lssId = new LssId(kafkaPartition); + final var table = new RS3WindowTable("table", storeId, client, partitioner); + final var key = Bytes.wrap("foo".getBytes(StandardCharsets.UTF_8)); + final var timestamp = 300L; + + // When: + final var pssId = 1; + when(partitioner.pssForLss(lssId)).thenReturn(singletonList(pssId)); + when(partitioner.pss(any(), eq(lssId))).thenReturn(pssId); + when(client.getCurrentOffsets(storeId, lssId, pssId)) + .thenReturn(new CurrentOffsets(Optional.of(5L), Optional.of(5L))); + when(client.get( + eq(storeId), + eq(lssId), + eq(pssId), + eq(Optional.of(5L)), + any()) + ).thenReturn(Optional.empty()); + + // Then: + table.init(kafkaPartition); + assertThat(table.fetch(kafkaPartition, key, timestamp), is(nullValue())); + } + +} \ No newline at end of file diff --git a/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/RS3ClientUtilTest.java b/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/RS3ClientUtilTest.java new file mode 100644 index 000000000..35636d77a --- /dev/null +++ b/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/RS3ClientUtilTest.java @@ -0,0 +1,87 @@ +/* + * Copyright 2025 Responsive Computing, Inc. + * + * This source code is licensed under the Responsive Business Source License Agreement v1.0 + * available at: + * + * https://www.responsive.dev/legal/responsive-bsl-10 + * + * This software requires a valid Commercial License Key for production use. Trial and commercial + * licenses can be obtained at https://www.responsive.dev + */ + +package dev.responsive.kafka.internal.db.rs3.client; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; +import static org.mockito.Mockito.when; + +import dev.responsive.kafka.internal.db.rs3.PssPartitioner; +import dev.responsive.kafka.internal.stores.ResponsiveStoreRegistration; +import java.util.Arrays; +import java.util.Optional; +import java.util.UUID; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class RS3ClientUtilTest { + + @Mock + private RS3Client client; + + @Mock + private PssPartitioner partitioner; + + + @Test + public void shouldReturnLastWrittenOffset() { + // Given: + final var storeId = UUID.randomUUID(); + final var lssId = new LssId(0); + when(partitioner.pssForLss(lssId)) + .thenReturn(Arrays.asList(0, 1, 2)); + + // When: + when(client.getCurrentOffsets(storeId, lssId, 0)) + .thenReturn(new CurrentOffsets(Optional.of(15L), Optional.of(10L))); + when(client.getCurrentOffsets(storeId, lssId, 1)) + .thenReturn(new CurrentOffsets(Optional.of(20L), Optional.of(5L))); + when(client.getCurrentOffsets(storeId, lssId, 2)) + .thenReturn(new CurrentOffsets(Optional.empty(), Optional.empty())); + + // Then: + final var clientUtil = new RS3ClientUtil(storeId, client, partitioner); + final var lssMetadata = clientUtil.fetchLssMetadata(lssId); + assertThat(lssMetadata.lastWrittenOffset(), is(20L)); + assertThat(lssMetadata.writtenOffsets().get(0), is(Optional.of(15L))); + assertThat(lssMetadata.writtenOffsets().get(1), is(Optional.of(20L))); + assertThat(lssMetadata.writtenOffsets().get(2), is(Optional.empty())); + } + + @Test + public void shouldReturnMinusOneForLastWrittenOffset() { + // Given: + final var storeId = UUID.randomUUID(); + final var lssId = new LssId(0); + when(partitioner.pssForLss(lssId)) + .thenReturn(Arrays.asList(0, 1)); + + // When: + when(client.getCurrentOffsets(storeId, lssId, 0)) + .thenReturn(new CurrentOffsets(Optional.empty(), Optional.empty())); + when(client.getCurrentOffsets(storeId, lssId, 1)) + .thenReturn(new CurrentOffsets(Optional.empty(), Optional.empty())); + + // Then: + final var clientUtil = new RS3ClientUtil(storeId, client, partitioner); + final var lssMetadata = clientUtil.fetchLssMetadata(lssId); + assertThat( + lssMetadata.lastWrittenOffset(), + is(ResponsiveStoreRegistration.NO_COMMITTED_OFFSET) + ); + } + +} \ No newline at end of file diff --git a/responsive-test-utils/src/main/java/dev/responsive/kafka/internal/db/TTDWindowTable.java b/responsive-test-utils/src/main/java/dev/responsive/kafka/internal/db/TTDWindowTable.java index ebe052224..a36c14e21 100644 --- a/responsive-test-utils/src/main/java/dev/responsive/kafka/internal/db/TTDWindowTable.java +++ b/responsive-test-utils/src/main/java/dev/responsive/kafka/internal/db/TTDWindowTable.java @@ -56,7 +56,7 @@ public String name() { } @Override - public WindowFlushManager init(final int kafkaPartition) { + public SegmentedWindowFlushManager init(final int kafkaPartition) { return new TTDWindowFlushManager(this, kafkaPartition, partitioner); } @@ -154,7 +154,7 @@ public long count() { return 0; } - private static class TTDWindowFlushManager extends WindowFlushManager { + private static class TTDWindowFlushManager extends SegmentedWindowFlushManager { private final String logPrefix; private final TTDWindowTable table; From d50d841bc0b5ee6bb014764a3e70ae0d6f3aa4e6 Mon Sep 17 00:00:00 2001 From: Jason Gustafson <12502538+hachikuji@users.noreply.github.com> Date: Thu, 3 Apr 2025 11:54:58 -0700 Subject: [PATCH 02/29] Remove unneeded comment --- .../kafka/internal/db/rs3/RS3WindowedKeySerde.java | 7 ------- 1 file changed, 7 deletions(-) diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowedKeySerde.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowedKeySerde.java index 21546a4ee..ba802acf5 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowedKeySerde.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowedKeySerde.java @@ -4,13 +4,6 @@ import java.nio.ByteBuffer; import org.apache.kafka.common.utils.Bytes; -/** - * 1: key-timestamp - * 2. prefix-timestamp-suffix - * 3. - * - * range scans not possible with any kind of hash - */ public class RS3WindowedKeySerde { public byte[] serialize(WindowedKey key) { From db3e73d503519b248a1b068480f2e33375185ce7 Mon Sep 17 00:00:00 2001 From: Jason Gustafson <12502538+hachikuji@users.noreply.github.com> Date: Thu, 3 Apr 2025 13:09:09 -0700 Subject: [PATCH 03/29] Test cases for serde --- kafka-client/build.gradle.kts | 1 + .../db/rs3/RS3WindowedKeySerdeTest.java | 23 +++++++++++++++++++ 2 files changed, 24 insertions(+) create mode 100644 kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/RS3WindowedKeySerdeTest.java diff --git a/kafka-client/build.gradle.kts b/kafka-client/build.gradle.kts index 124d550e9..148cc5af8 100644 --- a/kafka-client/build.gradle.kts +++ b/kafka-client/build.gradle.kts @@ -154,4 +154,5 @@ dependencies { testImplementation("io.grpc:grpc-inprocess:${libs.versions.grpc.orNull}") testImplementation("software.amazon.awssdk:kms:2.20.0") testImplementation("software.amazon.awssdk:sso:2.20.0") + testImplementation("net.jqwik:jqwik:1.9.2") } diff --git a/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/RS3WindowedKeySerdeTest.java b/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/RS3WindowedKeySerdeTest.java new file mode 100644 index 000000000..6b0b2fbff --- /dev/null +++ b/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/RS3WindowedKeySerdeTest.java @@ -0,0 +1,23 @@ +package dev.responsive.kafka.internal.db.rs3; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; + +import dev.responsive.kafka.internal.utils.WindowedKey; +import net.jqwik.api.ForAll; +import net.jqwik.api.Property; + +class RS3WindowedKeySerdeTest { + + @Property + public void shouldSerializeAndDeserialize( + @ForAll byte[] key, + @ForAll long timestamp + ) { + final var windowKey = new WindowedKey(key, timestamp); + final var serde = new RS3WindowedKeySerde(); + final var serialized = serde.serialize(windowKey); + assertThat(serde.deserialize(serialized), is(windowKey)); + } + +} \ No newline at end of file From 68e1718e5bde50ff4791bbaf08b09264303973ab Mon Sep 17 00:00:00 2001 From: Jason Gustafson <12502538+hachikuji@users.noreply.github.com> Date: Thu, 3 Apr 2025 16:35:35 -0700 Subject: [PATCH 04/29] Update kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowFlushManager.java Co-authored-by: A. Sophie Blee-Goldman --- .../responsive/kafka/internal/db/rs3/RS3WindowFlushManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowFlushManager.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowFlushManager.java index a84bccd8b..245f3db5a 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowFlushManager.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowFlushManager.java @@ -128,7 +128,7 @@ public String logPrefix() { @Override public long streamTime() { - return 0; + return streamTime; } public Optional writtenOffset(final int pssId) { From 8d47b4d5098f0ae8fb1ad90d9317418040331c98 Mon Sep 17 00:00:00 2001 From: Jason Gustafson <12502538+hachikuji@users.noreply.github.com> Date: Thu, 3 Apr 2025 19:15:04 -0700 Subject: [PATCH 05/29] use more final variables --- .../kafka/internal/db/rs3/RS3WindowedKeySerde.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowedKeySerde.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowedKeySerde.java index ba802acf5..565876711 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowedKeySerde.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowedKeySerde.java @@ -6,15 +6,15 @@ public class RS3WindowedKeySerde { - public byte[] serialize(WindowedKey key) { - byte[] result = new byte[key.key.get().length + 8]; - final ByteBuffer buffer = ByteBuffer.wrap(result); + public byte[] serialize(final WindowedKey key) { + final var result = new byte[key.key.get().length + 8]; + final var buffer = ByteBuffer.wrap(result); buffer.put(key.key.get()); buffer.putLong(key.windowStartMs); return result; } - public WindowedKey deserialize(byte[] bytes) { + public WindowedKey deserialize(final byte[] bytes) { final var buffer = ByteBuffer.wrap(bytes); final var keyLength = bytes.length - 8; final var keyBytes = new byte[keyLength]; From 2770de1787e1f5388f4d475d6f59c1706af48a2b Mon Sep 17 00:00:00 2001 From: Jason Gustafson <12502538+hachikuji@users.noreply.github.com> Date: Thu, 3 Apr 2025 19:25:39 -0700 Subject: [PATCH 06/29] review comments --- kafka-client/.jqwik-database | Bin 0 -> 4 bytes .../kafka/internal/db/rs3/RS3KVFlushManager.java | 5 +++-- .../kafka/internal/db/rs3/RS3TableFactory.java | 9 ++++++--- .../internal/db/rs3/RS3WindowFlushManager.java | 4 +++- .../kafka/internal/db/rs3/RS3WindowTable.java | 2 +- 5 files changed, 13 insertions(+), 7 deletions(-) create mode 100644 kafka-client/.jqwik-database diff --git a/kafka-client/.jqwik-database b/kafka-client/.jqwik-database new file mode 100644 index 0000000000000000000000000000000000000000..711006c3d3b5c6d50049e3f48311f3dbe372803d GIT binary patch literal 4 LcmZ4UmVp%j1%Lsc literal 0 HcmV?d00001 diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3KVFlushManager.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3KVFlushManager.java index d9fa81fe3..9ef93fa29 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3KVFlushManager.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3KVFlushManager.java @@ -127,8 +127,9 @@ public RemoteWriteResult updateOffset(final long consumedOffset) { @Override public String failedFlushInfo(final long batchOffset, final Integer failedTablePartition) { - // TODO: fill me in with info about last written offsets - return ""; + return String.format(">", + batchOffset, table.fetchOffset(kafkaPartition), + storeId, lssId.id()); } @Override diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3TableFactory.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3TableFactory.java index f6e702511..ea814ee1e 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3TableFactory.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3TableFactory.java @@ -68,10 +68,13 @@ public RemoteWindowTable windowTable( ); } - private UUID lookupStoreId(ResponsiveConfig config, String name) { - Map storeIdMapping = config.getMap( + private UUID lookupStoreId( + final ResponsiveConfig config, + final String name + ) { + final var storeIdMapping = config.getMap( ResponsiveConfig.RS3_LOGICAL_STORE_MAPPING_CONFIG); - final String storeIdHex = storeIdMapping.get(name); + final var storeIdHex = storeIdMapping.get(name); if (storeIdHex == null) { throw new ConfigException("Failed to find store ID mapping for table " + name); } diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowFlushManager.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowFlushManager.java index 245f3db5a..69833ffe2 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowFlushManager.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowFlushManager.java @@ -118,7 +118,9 @@ public String failedFlushInfo( final long batchOffset, final Integer failedTablePartition ) { - return String.format(""); + return String.format(">", + batchOffset, table.fetchOffset(kafkaPartition), + storeId, lssId.id()); } @Override diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowTable.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowTable.java index 2b55b9fbb..880b2e76e 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowTable.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowTable.java @@ -81,7 +81,7 @@ public WindowFlushManager init(final int kafkaPartition) { LssMetadata lssMetadata = rs3ClientUtil.fetchLssMetadata(lssId); this.fetchOffset = lssMetadata.lastWrittenOffset(); - long initialStreamTime = -1; // TODO: Initialize from RS3 metadata? + final var initialStreamTime = -1L; // TODO: Initialize from RS3 metadata? this.flushManager = new RS3WindowFlushManager( storeId, rs3Client, From 0cb4b5ec92132107fdd558cd67f958ff7b84af9d Mon Sep 17 00:00:00 2001 From: Jason Gustafson <12502538+hachikuji@users.noreply.github.com> Date: Thu, 3 Apr 2025 19:28:44 -0700 Subject: [PATCH 07/29] Remove TODO --- .../dev/responsive/kafka/internal/db/rs3/RS3WindowTable.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowTable.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowTable.java index 880b2e76e..8afe3dbdd 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowTable.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowTable.java @@ -53,7 +53,8 @@ public RS3WindowTable( ); } - public RS3WindowTable( + // Visible for testing + RS3WindowTable( final String name, final UUID storeId, final RS3Client rs3Client, @@ -61,7 +62,7 @@ public RS3WindowTable( ) { this.name = Objects.requireNonNull(name); this.storeId = Objects.requireNonNull(storeId); - this.rs3Client = Objects.requireNonNull(rs3Client); // TODO: Use metered client + this.rs3Client = Objects.requireNonNull(rs3Client); this.rs3ClientUtil = new RS3ClientUtil(storeId, rs3Client, pssPartitioner); this.pssPartitioner = Objects.requireNonNull(pssPartitioner); } From d47f03a3672416c41a3a0983619575e44b6e1fdc Mon Sep 17 00:00:00 2001 From: Jason Gustafson <12502538+hachikuji@users.noreply.github.com> Date: Thu, 3 Apr 2025 19:29:36 -0700 Subject: [PATCH 08/29] Update kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowTable.java Co-authored-by: A. Sophie Blee-Goldman --- .../dev/responsive/kafka/internal/db/rs3/RS3WindowTable.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowTable.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowTable.java index 8afe3dbdd..eb8136008 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowTable.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowTable.java @@ -97,7 +97,7 @@ public WindowFlushManager init(final int kafkaPartition) { return flushManager; } - private void throwIfPartitionNotInitialized(int kafkaPartition) { + private void throwIfPartitionNotInitialized(final int kafkaPartition) { if (flushManager == null) { throw new IllegalStateException(String.format( "Cannot complete operation on store %s is not yet initialized", From 44523086e4d6a3cee1d9a8db1ce2599fde373a21 Mon Sep 17 00:00:00 2001 From: Jason Gustafson <12502538+hachikuji@users.noreply.github.com> Date: Thu, 3 Apr 2025 19:30:26 -0700 Subject: [PATCH 09/29] reduce visibility on RS3Writer --- .../java/dev/responsive/kafka/internal/db/rs3/RS3Writer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3Writer.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3Writer.java index f8fa34a42..af0b6d142 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3Writer.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3Writer.java @@ -18,7 +18,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public abstract class RS3Writer implements RemoteWriter { +abstract class RS3Writer implements RemoteWriter { private static final Logger LOG = LoggerFactory.getLogger(RS3Writer.class); private final UUID storeId; From a1e79166c42179a665afd8c18a360efa21211644 Mon Sep 17 00:00:00 2001 From: Jason Gustafson <12502538+hachikuji@users.noreply.github.com> Date: Thu, 3 Apr 2025 19:33:22 -0700 Subject: [PATCH 10/29] Add more info to fail message --- .../responsive/kafka/internal/db/rs3/RS3KVFlushManager.java | 4 ++-- .../kafka/internal/db/rs3/RS3WindowFlushManager.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3KVFlushManager.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3KVFlushManager.java index 9ef93fa29..f209aef10 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3KVFlushManager.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3KVFlushManager.java @@ -127,8 +127,8 @@ public RemoteWriteResult updateOffset(final long consumedOffset) { @Override public String failedFlushInfo(final long batchOffset, final Integer failedTablePartition) { - return String.format(">", - batchOffset, table.fetchOffset(kafkaPartition), + return String.format(">", + failedTablePartition, batchOffset, table.fetchOffset(kafkaPartition), storeId, lssId.id()); } diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowFlushManager.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowFlushManager.java index 69833ffe2..5165f8232 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowFlushManager.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowFlushManager.java @@ -118,8 +118,8 @@ public String failedFlushInfo( final long batchOffset, final Integer failedTablePartition ) { - return String.format(">", - batchOffset, table.fetchOffset(kafkaPartition), + return String.format(">", + failedTablePartition, batchOffset, table.fetchOffset(kafkaPartition), storeId, lssId.id()); } From 2a6e2472dca036833de39631716bad8be50713e9 Mon Sep 17 00:00:00 2001 From: Jason Gustafson <12502538+hachikuji@users.noreply.github.com> Date: Fri, 4 Apr 2025 10:30:50 -0700 Subject: [PATCH 11/29] Add basic writer test for window table --- .../internal/db/rs3/RS3KVFlushManager.java | 3 +- .../internal/db/rs3/RS3TableFactory.java | 1 - .../db/rs3/RS3WindowFlushManager.java | 3 +- .../kafka/internal/db/rs3/RS3Writer.java | 2 +- .../internal/db/rs3/RS3WindowTableTest.java | 58 +++++++++++++++++++ 5 files changed, 63 insertions(+), 4 deletions(-) diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3KVFlushManager.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3KVFlushManager.java index f209aef10..b323abf6f 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3KVFlushManager.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3KVFlushManager.java @@ -127,7 +127,8 @@ public RemoteWriteResult updateOffset(final long consumedOffset) { @Override public String failedFlushInfo(final long batchOffset, final Integer failedTablePartition) { - return String.format(">", + return String.format(">", failedTablePartition, batchOffset, table.fetchOffset(kafkaPartition), storeId, lssId.id()); } diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3TableFactory.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3TableFactory.java index ea814ee1e..f16c4231e 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3TableFactory.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3TableFactory.java @@ -18,7 +18,6 @@ import dev.responsive.kafka.internal.db.rs3.client.WalEntry; import dev.responsive.kafka.internal.db.rs3.client.grpc.GrpcRS3Client; import dev.responsive.kafka.internal.metrics.ResponsiveMetrics; -import java.util.Map; import java.util.UUID; import org.apache.kafka.common.config.ConfigException; diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowFlushManager.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowFlushManager.java index 5165f8232..4ecb8014f 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowFlushManager.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowFlushManager.java @@ -118,7 +118,8 @@ public String failedFlushInfo( final long batchOffset, final Integer failedTablePartition ) { - return String.format(">", + return String.format(">", failedTablePartition, batchOffset, table.fetchOffset(kafkaPartition), storeId, lssId.id()); } diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3Writer.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3Writer.java index af0b6d142..ddc215ed2 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3Writer.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3Writer.java @@ -116,7 +116,7 @@ public CompletionStage> flush() { } LOG.debug("last flushed offset for pss/lss {}/{} is {}", pssId, lssId, flushedOffset); - return RemoteWriteResult.success(kafkaPartition); + return RemoteWriteResult.success(pssId); }); } diff --git a/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/RS3WindowTableTest.java b/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/RS3WindowTableTest.java index c8fdae1f5..360b4a4dd 100644 --- a/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/RS3WindowTableTest.java +++ b/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/RS3WindowTableTest.java @@ -11,9 +11,14 @@ import dev.responsive.kafka.internal.db.rs3.client.CurrentOffsets; import dev.responsive.kafka.internal.db.rs3.client.LssId; import dev.responsive.kafka.internal.db.rs3.client.RS3Client; +import dev.responsive.kafka.internal.db.rs3.client.StreamSender; +import dev.responsive.kafka.internal.db.rs3.client.StreamSenderMessageReceiver; +import dev.responsive.kafka.internal.db.rs3.client.WalEntry; +import dev.responsive.kafka.internal.utils.WindowedKey; import java.nio.charset.StandardCharsets; import java.util.Optional; import java.util.UUID; +import java.util.concurrent.CompletableFuture; import org.apache.kafka.common.utils.Bytes; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -29,6 +34,12 @@ class RS3WindowTableTest { @Mock private PssPartitioner partitioner; + @Mock + private StreamSenderMessageReceiver> sendRecv; + + @Mock + private StreamSender streamSender; + @Test public void shouldReturnValueFromFetchIfKeyFound() { // Given: @@ -88,4 +99,51 @@ public void shouldReturnNullFromFetchIfKeyNotFound() { assertThat(table.fetch(kafkaPartition, key, timestamp), is(nullValue())); } + @Test + public void shouldWriteWindowedKeyValues() throws Exception { + // Given: + final var storeId = UUID.randomUUID(); + final var kafkaPartition = 5; + final var lssId = new LssId(kafkaPartition); + final var table = new RS3WindowTable("table", storeId, client, partitioner); + final var pssId = 0; + final var consumedOffset = 4L; + + // When: + final var sendRecvCompletion = new CompletableFuture>(); + when(sendRecv.completion()).thenReturn(sendRecvCompletion); + when(sendRecv.sender()).thenReturn(streamSender); + when(sendRecv.isActive()).thenAnswer(invocation -> !sendRecvCompletion.isDone()); + when(partitioner.pssForLss(lssId)).thenReturn(singletonList(pssId)); + when(client.getCurrentOffsets(storeId, lssId, pssId)) + .thenReturn(new CurrentOffsets(Optional.empty(), Optional.empty())); + when(client.writeWalSegmentAsync(storeId, lssId, pssId, Optional.empty(), consumedOffset)) + .thenReturn(sendRecv); + + // Then: + final var flushManager = table.init(kafkaPartition); + final var writer = flushManager.createWriter(pssId, consumedOffset); + writer.insert( + new WindowedKey(utf8Bytes("super"), 0L), + utf8Bytes("mario"), + 5L + ); + writer.insert( + new WindowedKey(utf8Bytes("super"), 10L), + utf8Bytes("mario"), + 15L + ); + sendRecvCompletion.complete(Optional.of(consumedOffset)); + + final var completion = writer.flush(); + assertThat(completion.toCompletableFuture().isDone(), is(true)); + final var result = completion.toCompletableFuture().get(); + assertThat(result.wasApplied(), is(true)); + assertThat(result.tablePartition(), is(pssId)); + } + + private static byte[] utf8Bytes(String s) { + return s.getBytes(StandardCharsets.UTF_8); + } + } \ No newline at end of file From 7c8fdd1f02cb07ac12e02a406cd9f44eca6c885f Mon Sep 17 00:00:00 2001 From: Jason Gustafson <12502538+hachikuji@users.noreply.github.com> Date: Fri, 4 Apr 2025 18:53:57 -0700 Subject: [PATCH 12/29] Fix breakage from merge --- .../responsive/kafka/internal/db/rs3/RS3KVFlushManager.java | 2 +- .../kafka/internal/db/rs3/RS3WindowFlushManager.java | 2 +- .../dev/responsive/kafka/internal/db/rs3/RS3WindowTable.java | 4 +++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3KVFlushManager.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3KVFlushManager.java index b323abf6f..c504d397e 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3KVFlushManager.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3KVFlushManager.java @@ -129,7 +129,7 @@ public RemoteWriteResult updateOffset(final long consumedOffset) { public String failedFlushInfo(final long batchOffset, final Integer failedTablePartition) { return String.format(">", - failedTablePartition, batchOffset, table.fetchOffset(kafkaPartition), + failedTablePartition, batchOffset, table.lastWrittenOffset(kafkaPartition), storeId, lssId.id()); } diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowFlushManager.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowFlushManager.java index 4ecb8014f..e3e8e63e0 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowFlushManager.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowFlushManager.java @@ -120,7 +120,7 @@ public String failedFlushInfo( ) { return String.format(">", - failedTablePartition, batchOffset, table.fetchOffset(kafkaPartition), + failedTablePartition, batchOffset, table.lastWrittenOffset(kafkaPartition), storeId, lssId.id()); } diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowTable.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowTable.java index eb8136008..ec0026182 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowTable.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowTable.java @@ -221,7 +221,9 @@ public WalEntry delete(final int kafkaPartition, final WindowedKey key) { } @Override - public long fetchOffset(final int kafkaPartition) { + public long lastWrittenOffset(final int kafkaPartition) { return fetchOffset; } } + + From e46e96f47fb2980c19db0d71a15fc73efe0de64f Mon Sep 17 00:00:00 2001 From: Jason Gustafson <12502538+hachikuji@users.noreply.github.com> Date: Tue, 8 Apr 2025 12:29:06 -0700 Subject: [PATCH 13/29] Add range implementations for window table --- .../kafka/internal/db/rs3/RS3KVTable.java | 12 +- .../db/rs3/RS3WindowFlushManager.java | 6 +- .../kafka/internal/db/rs3/RS3WindowTable.java | 141 ++++++++++++++-- .../internal/db/rs3/RS3WindowedKeySerde.java | 26 --- .../kafka/internal/db/rs3/client/Delete.java | 50 ++++++ .../db/rs3/client/MeteredRS3Client.java | 54 +++++- .../kafka/internal/db/rs3/client/Put.java | 13 +- .../internal/db/rs3/client/RS3Client.java | 24 ++- .../kafka/internal/db/rs3/client/Range.java | 52 +++--- .../internal/db/rs3/client/RangeBound.java | 63 ++++--- .../internal/db/rs3/client/WalEntry.java | 11 ++ .../db/rs3/client/WindowedDelete.java | 57 +++++++ .../internal/db/rs3/client/WindowedPut.java | 65 ++++++++ .../rs3/client/grpc/GrpcKeyValueIterator.java | 71 +++++--- .../db/rs3/client/grpc/GrpcRS3Client.java | 157 ++++++++++++++---- .../db/rs3/client/grpc/GrpcRangeKeyCodec.java | 54 ++++++ .../client/grpc/GrpcRangeRequestProxy.java | 4 +- .../db/rs3/client/grpc/WalEntryPutWriter.java | 42 +++++ .../db/rs3/RS3KVTableIntegrationTest.java | 2 +- .../db/rs3/RS3WindowedKeySerdeTest.java | 23 --- .../client/grpc/GrpcKeyValueIteratorTest.java | 30 ++-- .../grpc/GrpcRS3ClientEndToEndTest.java | 8 +- .../db/rs3/client/grpc/GrpcRS3ClientTest.java | 58 +++++-- .../db/rs3/client/grpc/GrpsRs3TestUtil.java | 11 +- 24 files changed, 796 insertions(+), 238 deletions(-) delete mode 100644 kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowedKeySerde.java create mode 100644 kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/Delete.java create mode 100644 kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/WindowedDelete.java create mode 100644 kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/WindowedPut.java create mode 100644 kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRangeKeyCodec.java create mode 100644 kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/WalEntryPutWriter.java delete mode 100644 kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/RS3WindowedKeySerdeTest.java diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3KVTable.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3KVTable.java index 9ad04fe3b..b2ed35708 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3KVTable.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3KVTable.java @@ -14,6 +14,7 @@ import dev.responsive.kafka.internal.db.KVFlushManager; import dev.responsive.kafka.internal.db.RemoteKVTable; +import dev.responsive.kafka.internal.db.rs3.client.Delete; import dev.responsive.kafka.internal.db.rs3.client.LssId; import dev.responsive.kafka.internal.db.rs3.client.LssMetadata; import dev.responsive.kafka.internal.db.rs3.client.MeteredRS3Client; @@ -101,7 +102,7 @@ public byte[] get(final int kafkaPartition, final Bytes key, final long minValid lssId, pssId, flushManager.writtenOffset(pssId), - key.get() + key ).orElse(null); } @@ -112,8 +113,8 @@ public KeyValueIterator range( final Bytes to, final long streamTimeMs ) { - final RangeBound fromBound = RangeBound.inclusive(from.get()); - final RangeBound toBound = RangeBound.exclusive(to.get()); + final RangeBound fromBound = RangeBound.inclusive(from); + final RangeBound toBound = RangeBound.exclusive(to); final List> pssIters = new ArrayList<>(); for (int pssId : pssPartitioner.pssForLss(this.lssId)) { @@ -174,10 +175,7 @@ public WalEntry insert( @Override public WalEntry delete(final int kafkaPartition, final Bytes key) { - return new Put( - key.get(), - null - ); + return new Delete(key.get()); } @Override diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowFlushManager.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowFlushManager.java index e3e8e63e0..e66391e76 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowFlushManager.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowFlushManager.java @@ -32,7 +32,6 @@ class RS3WindowFlushManager implements WindowFlushManager { private final RS3WindowTable table; private final int kafkaPartition; private final PssPartitioner pssPartitioner; - private final RS3WindowedKeySerde keySerde; private final Map> writtenOffsets; private long streamTime; @@ -45,7 +44,6 @@ public RS3WindowFlushManager( final RS3WindowTable table, final int kafkaPartition, final PssPartitioner pssPartitioner, - final RS3WindowedKeySerde keySerde, final Map> writtenOffsets, final long initialStreamTime ) { @@ -55,7 +53,6 @@ public RS3WindowFlushManager( this.table = table; this.kafkaPartition = kafkaPartition; this.pssPartitioner = pssPartitioner; - this.keySerde = keySerde; this.writtenOffsets = writtenOffsets; this.streamTime = initialStreamTime; } @@ -70,7 +67,8 @@ public TablePartitioner partitioner() { return new PssTablePartitioner<>(pssPartitioner) { @Override public byte[] serialize(final WindowedKey key) { - return keySerde.serialize(key); + // TODO: Get rid of this + return key.key.get(); } }; } diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowTable.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowTable.java index ec0026182..efe1695ca 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowTable.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowTable.java @@ -1,3 +1,15 @@ +/* + * Copyright 2025 Responsive Computing, Inc. + * + * This source code is licensed under the Responsive Business Source License Agreement v1.0 + * available at: + * + * https://www.responsive.dev/legal/responsive-bsl-10 + * + * This software requires a valid Commercial License Key for production use. Trial and commercial + * licenses can be obtained at https://www.responsive.dev + */ + package dev.responsive.kafka.internal.db.rs3; import dev.responsive.kafka.internal.db.RemoteWindowTable; @@ -5,15 +17,22 @@ import dev.responsive.kafka.internal.db.rs3.client.LssId; import dev.responsive.kafka.internal.db.rs3.client.LssMetadata; import dev.responsive.kafka.internal.db.rs3.client.MeteredRS3Client; -import dev.responsive.kafka.internal.db.rs3.client.Put; import dev.responsive.kafka.internal.db.rs3.client.RS3Client; import dev.responsive.kafka.internal.db.rs3.client.RS3ClientUtil; +import dev.responsive.kafka.internal.db.rs3.client.Range; +import dev.responsive.kafka.internal.db.rs3.client.RangeBound; import dev.responsive.kafka.internal.db.rs3.client.WalEntry; +import dev.responsive.kafka.internal.db.rs3.client.WindowedDelete; +import dev.responsive.kafka.internal.db.rs3.client.WindowedPut; import dev.responsive.kafka.internal.metrics.ResponsiveMetrics; +import dev.responsive.kafka.internal.utils.MergeKeyValueIterator; import dev.responsive.kafka.internal.utils.WindowedKey; +import java.util.ArrayList; +import java.util.List; import java.util.Objects; import java.util.UUID; import org.apache.kafka.common.utils.Bytes; +import org.apache.kafka.streams.KeyValue; import org.apache.kafka.streams.state.KeyValueIterator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -26,7 +45,6 @@ public class RS3WindowTable implements RemoteWindowTable { private final RS3Client rs3Client; private final RS3ClientUtil rs3ClientUtil; private final PssPartitioner pssPartitioner; - private final RS3WindowedKeySerde keySerde = new RS3WindowedKeySerde(); // Initialized in `init()` private LssId lssId; @@ -90,7 +108,6 @@ public WindowFlushManager init(final int kafkaPartition) { this, kafkaPartition, pssPartitioner, - keySerde, lssMetadata.writtenOffsets(), initialStreamTime ); @@ -123,14 +140,12 @@ public byte[] fetch( throwIfPartitionNotInitialized(kafkaPartition); final int pssId = pssPartitioner.pss(key.get(), this.lssId); final var windowKey = new WindowedKey(key, windowStart); - final var windowKeyBytes = keySerde.serialize(windowKey); - - return rs3Client.get( + return rs3Client.windowedGet( storeId, lssId, pssId, flushManager.writtenOffset(pssId), - windowKeyBytes + windowKey ).orElse(null); } @@ -142,7 +157,17 @@ public KeyValueIterator fetch( final long timeTo ) { throwIfPartitionNotInitialized(kafkaPartition); - throw new UnsupportedOperationException(); + final int pssId = pssPartitioner.pss(key.get(), this.lssId); + final var windowKeyFrom = RangeBound.inclusive(new WindowedKey(key, timeFrom)); + final var windowKeyTo = RangeBound.exclusive(new WindowedKey(key, timeTo)); + return rs3Client.windowedRange( + storeId, + lssId, + pssId, + flushManager.writtenOffset(pssId), + windowKeyFrom, + windowKeyTo + ); } @Override @@ -165,7 +190,20 @@ public KeyValueIterator fetchRange( final long timeTo ) { throwIfPartitionNotInitialized(kafkaPartition); - throw new UnsupportedOperationException(); + final List> pssIters = new ArrayList<>(); + final var windowKeyFrom = RangeBound.inclusive(new WindowedKey(fromKey, timeFrom)); + final var windowKeyTo = RangeBound.exclusive(new WindowedKey(toKey, timeTo)); + for (int pssId : pssPartitioner.pssForLss(this.lssId)) { + pssIters.add(rs3Client.windowedRange( + storeId, + lssId, + pssId, + flushManager.writtenOffset(pssId), + windowKeyFrom, + windowKeyTo + )); + } + return new MergeKeyValueIterator<>(pssIters); } @Override @@ -187,7 +225,29 @@ public KeyValueIterator fetchAll( final long timeTo ) { throwIfPartitionNotInitialized(kafkaPartition); - throw new UnsupportedOperationException(); + final List> pssIters = new ArrayList<>(); + + // TODO: the types break down here. We want to tell the server that we are + // interested in all keys within a given time range, but the schema does + // not support a partially specified bound. For now, we fetch everything and + // filter here. + final var timeRange = new Range<>( + RangeBound.inclusive(timeFrom), + RangeBound.exclusive(timeTo) + ); + + for (int pssId : pssPartitioner.pssForLss(this.lssId)) { + final var rangeIter = rs3Client.windowedRange( + storeId, + lssId, + pssId, + flushManager.writtenOffset(pssId), + RangeBound.unbounded(), + RangeBound.unbounded() + ); + pssIters.add(new TimeRangeFilter(timeRange, rangeIter)); + } + return new MergeKeyValueIterator<>(pssIters); } @Override @@ -212,18 +272,73 @@ public WalEntry insert( final byte[] value, final long timestampMs ) { - return new Put(keySerde.serialize(key), value); + return new WindowedPut( + key.key.get(), + value, + timestampMs, + key.windowStartMs + ); } @Override public WalEntry delete(final int kafkaPartition, final WindowedKey key) { - return new Put(keySerde.serialize(key), null); + return new WindowedDelete( + key.key.get(), + key.windowStartMs + ); } @Override public long lastWrittenOffset(final int kafkaPartition) { return fetchOffset; } -} + private static class TimeRangeFilter implements KeyValueIterator { + private final Range timeRange; + private final KeyValueIterator delegate; + + private TimeRangeFilter( + final Range timeRange, + final KeyValueIterator delegate + ) { + this.timeRange = timeRange; + this.delegate = delegate; + } + + @Override + public void close() { + delegate.close(); + } + + @Override + public WindowedKey peekNextKey() { + skipToNextKeyInRange(); + return delegate.peekNextKey(); + } + private void skipToNextKeyInRange() { + while (true) { + final var nextKey = delegate.peekNextKey(); + if (nextKey == null) { + break; + } else if (timeRange.contains(nextKey.windowStartMs)) { + return; + } else { + delegate.next(); + } + } + } + + @Override + public boolean hasNext() { + return peekNextKey() != null; + } + + @Override + public KeyValue next() { + skipToNextKeyInRange(); + return delegate.next(); + } + } + +} diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowedKeySerde.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowedKeySerde.java deleted file mode 100644 index 565876711..000000000 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowedKeySerde.java +++ /dev/null @@ -1,26 +0,0 @@ -package dev.responsive.kafka.internal.db.rs3; - -import dev.responsive.kafka.internal.utils.WindowedKey; -import java.nio.ByteBuffer; -import org.apache.kafka.common.utils.Bytes; - -public class RS3WindowedKeySerde { - - public byte[] serialize(final WindowedKey key) { - final var result = new byte[key.key.get().length + 8]; - final var buffer = ByteBuffer.wrap(result); - buffer.put(key.key.get()); - buffer.putLong(key.windowStartMs); - return result; - } - - public WindowedKey deserialize(final byte[] bytes) { - final var buffer = ByteBuffer.wrap(bytes); - final var keyLength = bytes.length - 8; - final var keyBytes = new byte[keyLength]; - buffer.get(keyBytes); - final var windowStartMs = buffer.getLong(); - return new WindowedKey(Bytes.wrap(keyBytes), windowStartMs); - } - -} diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/Delete.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/Delete.java new file mode 100644 index 000000000..10500e936 --- /dev/null +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/Delete.java @@ -0,0 +1,50 @@ +/* + * Copyright 2025 Responsive Computing, Inc. + * + * This source code is licensed under the Responsive Business Source License Agreement v1.0 + * available at: + * + * https://www.responsive.dev/legal/responsive-bsl-10 + * + * This software requires a valid Commercial License Key for production use. Trial and commercial + * licenses can be obtained at https://www.responsive.dev + */ + +package dev.responsive.kafka.internal.db.rs3.client; + +import java.util.Arrays; +import java.util.Objects; + +public class Delete extends WalEntry { + private final byte[] key; + + public Delete(final byte[] key) { + this.key = Objects.requireNonNull(key); + } + + public byte[] key() { + return key; + } + + @Override + public void visit(final Visitor visitor) { + visitor.visit(this); + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final Delete delete = (Delete) o; + return Objects.deepEquals(key, delete.key); + } + + @Override + public int hashCode() { + return Arrays.hashCode(key); + } +} diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/MeteredRS3Client.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/MeteredRS3Client.java index c2755b595..b09c7e8f2 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/MeteredRS3Client.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/MeteredRS3Client.java @@ -1,6 +1,7 @@ package dev.responsive.kafka.internal.db.rs3.client; import dev.responsive.kafka.internal.metrics.ResponsiveMetrics; +import dev.responsive.kafka.internal.utils.WindowedKey; import java.time.Duration; import java.time.Instant; import java.util.List; @@ -84,10 +85,16 @@ public Optional get( final LssId lssId, final int pssId, final Optional expectedWrittenOffset, - final byte[] key + final Bytes key ) { final Instant start = Instant.now(); - final Optional result = delegate.get(storeId, lssId, pssId, expectedWrittenOffset, key); + final Optional result = delegate.get( + storeId, + lssId, + pssId, + expectedWrittenOffset, + key + ); getSensor.record(Duration.between(start, Instant.now()).toNanos()); return result; } @@ -98,8 +105,8 @@ public KeyValueIterator range( final LssId lssId, final int pssId, final Optional expectedWrittenOffset, - final RangeBound from, - final RangeBound to + final RangeBound from, + final RangeBound to ) { return delegate.range( storeId, @@ -111,6 +118,45 @@ public KeyValueIterator range( ); } + @Override + public Optional windowedGet( + final UUID storeId, + final LssId lssId, + final int pssId, + final Optional expectedWrittenOffset, + final WindowedKey key + ) { + final Instant start = Instant.now(); + final Optional result = delegate.windowedGet( + storeId, + lssId, + pssId, + expectedWrittenOffset, + key + ); + getSensor.record(Duration.between(start, Instant.now()).toNanos()); + return result; + } + + @Override + public KeyValueIterator windowedRange( + final UUID storeId, + final LssId lssId, + final int pssId, + final Optional expectedWrittenOffset, + final RangeBound from, + final RangeBound to + ) { + return delegate.windowedRange( + storeId, + lssId, + pssId, + expectedWrittenOffset, + from, + to + ); + } + @Override public List listStores() { final Instant start = Instant.now(); diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/Put.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/Put.java index 4bfd8709c..3fe45944f 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/Put.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/Put.java @@ -14,7 +14,6 @@ import java.util.Arrays; import java.util.Objects; -import java.util.Optional; public class Put extends WalEntry { private final byte[] key; @@ -22,15 +21,20 @@ public class Put extends WalEntry { public Put(final byte[] key, final byte[] value) { this.key = Objects.requireNonNull(key); - this.value = value; + this.value = Objects.requireNonNull(value); } public byte[] key() { return key; } - public Optional value() { - return Optional.ofNullable(value); + public byte[] value() { + return value; + } + + @Override + public void visit(final Visitor visitor) { + visitor.visit(this); } @Override @@ -49,4 +53,5 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(Arrays.hashCode(key), Arrays.hashCode(value)); } + } diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/RS3Client.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/RS3Client.java index 4aa1586a1..328e89872 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/RS3Client.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/RS3Client.java @@ -12,6 +12,7 @@ package dev.responsive.kafka.internal.db.rs3.client; +import dev.responsive.kafka.internal.utils.WindowedKey; import java.util.List; import java.util.Optional; import java.util.UUID; @@ -43,7 +44,15 @@ Optional get( LssId lssId, int pssId, Optional expectedWrittenOffset, - byte[] key + Bytes key + ); + + Optional windowedGet( + UUID storeId, + LssId lssId, + int pssId, + Optional expectedWrittenOffset, + WindowedKey key ); KeyValueIterator range( @@ -51,8 +60,17 @@ KeyValueIterator range( LssId lssId, int pssId, Optional expectedWrittenOffset, - RangeBound from, - RangeBound to + RangeBound from, + RangeBound to + ); + + KeyValueIterator windowedRange( + UUID storeId, + LssId lssId, + int pssId, + Optional expectedWrittenOffset, + RangeBound from, + RangeBound to ); List listStores(); diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/Range.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/Range.java index 2f0b21191..306ca51e4 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/Range.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/Range.java @@ -1,61 +1,71 @@ -package dev.responsive.kafka.internal.db.rs3.client; +/* + * Copyright 2025 Responsive Computing, Inc. + * + * This source code is licensed under the Responsive Business Source License Agreement v1.0 + * available at: + * + * https://www.responsive.dev/legal/responsive-bsl-10 + * + * This software requires a valid Commercial License Key for production use. Trial and commercial + * licenses can be obtained at https://www.responsive.dev + */ -import java.util.Arrays; +package dev.responsive.kafka.internal.db.rs3.client; -public class Range { - private final RangeBound start; - private final RangeBound end; +public class Range> { + private final RangeBound start; + private final RangeBound end; - public Range(RangeBound start, RangeBound end) { + public Range(RangeBound start, RangeBound end) { this.start = start; this.end = end; } - public RangeBound start() { + public RangeBound start() { return start; } - public RangeBound end() { + public RangeBound end() { return end; } - public boolean contains(byte[] key) { + public boolean contains(K key) { return greaterThanStartBound(key) && lessThanEndBound(key); } - public boolean greaterThanStartBound(byte[] key) { + public boolean greaterThanStartBound(K key) { return start.map(new RangeBound.Mapper<>() { @Override - public Boolean map(final RangeBound.InclusiveBound b) { - return Arrays.compare(b.key(), key) <= 0; + public Boolean map(final RangeBound.InclusiveBound b) { + return b.key().compareTo(key) <= 0; } @Override - public Boolean map(final RangeBound.ExclusiveBound b) { - return Arrays.compare(b.key(), key) < 0; + public Boolean map(final RangeBound.ExclusiveBound b) { + return b.key().compareTo(key) < 0; } @Override - public Boolean map(final RangeBound.Unbounded b) { + public Boolean map(final RangeBound.Unbounded b) { return true; } }); } - public boolean lessThanEndBound(byte[] key) { + public boolean lessThanEndBound(K key) { return end.map(new RangeBound.Mapper<>() { @Override - public Boolean map(final RangeBound.InclusiveBound b) { - return Arrays.compare(b.key(), key) >= 0; + public Boolean map(final RangeBound.InclusiveBound b) { + return b.key().compareTo(key) >= 0; } @Override - public Boolean map(final RangeBound.ExclusiveBound b) { - return Arrays.compare(b.key(), key) > 0; + public Boolean map(final RangeBound.ExclusiveBound b) { + return b.key().compareTo(key) > 0; } @Override - public Boolean map(final RangeBound.Unbounded b) { + public Boolean map(final RangeBound.Unbounded b) { return true; } }); diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/RangeBound.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/RangeBound.java index fc8b6a3e0..ae81e71fb 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/RangeBound.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/RangeBound.java @@ -12,38 +12,37 @@ package dev.responsive.kafka.internal.db.rs3.client; -import java.util.Arrays; import java.util.Objects; -public interface RangeBound { +public interface RangeBound { - T map(Mapper mapper); + T map(Mapper mapper); - static Unbounded unbounded() { - return Unbounded.INSTANCE; + static Unbounded unbounded() { + return new Unbounded<>(); } - static InclusiveBound inclusive(byte[] key) { - return new InclusiveBound(key); + static InclusiveBound inclusive(K key) { + return new InclusiveBound<>(key); } - static ExclusiveBound exclusive(byte[] key) { - return new ExclusiveBound(key); + static ExclusiveBound exclusive(K key) { + return new ExclusiveBound<>(key); } - class InclusiveBound implements RangeBound { - private final byte[] key; + class InclusiveBound implements RangeBound { + private final K key; - public InclusiveBound(final byte[] key) { + public InclusiveBound(final K key) { this.key = key; } - public byte[] key() { + public K key() { return key; } @Override - public T map(final Mapper mapper) { + public T map(final Mapper mapper) { return mapper.map(this); } @@ -55,29 +54,29 @@ public boolean equals(final Object o) { if (o == null || getClass() != o.getClass()) { return false; } - final InclusiveBound that = (InclusiveBound) o; - return Objects.deepEquals(key, that.key); + final InclusiveBound that = (InclusiveBound) o; + return Objects.equals(key, that.key); } @Override public int hashCode() { - return Arrays.hashCode(key); + return Objects.hashCode(key); } } - class ExclusiveBound implements RangeBound { - private final byte[] key; + class ExclusiveBound implements RangeBound { + private final K key; - public ExclusiveBound(final byte[] key) { + public ExclusiveBound(final K key) { this.key = key; } - public byte[] key() { + public K key() { return key; } @Override - public T map(final Mapper mapper) { + public T map(final Mapper mapper) { return mapper.map(this); } @@ -89,33 +88,31 @@ public boolean equals(final Object o) { if (o == null || getClass() != o.getClass()) { return false; } - final ExclusiveBound that = (ExclusiveBound) o; - return Objects.deepEquals(key, that.key); + final ExclusiveBound that = (ExclusiveBound) o; + return Objects.equals(key, that.key); } @Override public int hashCode() { - return Arrays.hashCode(key); + return Objects.hashCode(key); } } - class Unbounded implements RangeBound { - private static final Unbounded INSTANCE = new Unbounded(); - + class Unbounded implements RangeBound { private Unbounded() {} @Override - public T map(final Mapper mapper) { + public T map(final Mapper mapper) { return mapper.map(this); } } - interface Mapper { - T map(InclusiveBound b); + interface Mapper { + T map(InclusiveBound b); - T map(ExclusiveBound b); + T map(ExclusiveBound b); - T map(Unbounded b); + T map(Unbounded b); } } diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/WalEntry.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/WalEntry.java index 9ab0a8e8b..48a9229f3 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/WalEntry.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/WalEntry.java @@ -13,4 +13,15 @@ package dev.responsive.kafka.internal.db.rs3.client; public abstract class WalEntry { + public abstract void visit(Visitor visitor); + + public interface Visitor { + void visit(Put put); + + void visit(Delete delete); + + void visit(WindowedDelete windowedDelete); + + void visit(WindowedPut windowedPut); + } } diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/WindowedDelete.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/WindowedDelete.java new file mode 100644 index 000000000..4d4fa4758 --- /dev/null +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/WindowedDelete.java @@ -0,0 +1,57 @@ +/* + * Copyright 2025 Responsive Computing, Inc. + * + * This source code is licensed under the Responsive Business Source License Agreement v1.0 + * available at: + * + * https://www.responsive.dev/legal/responsive-bsl-10 + * + * This software requires a valid Commercial License Key for production use. Trial and commercial + * licenses can be obtained at https://www.responsive.dev + */ + +package dev.responsive.kafka.internal.db.rs3.client; + +import java.util.Objects; + +public class WindowedDelete extends Delete { + private final long windowTimestamp; + + public WindowedDelete( + final byte[] key, + final long windowTimestamp + ) { + super(key); + this.windowTimestamp = windowTimestamp; + } + + public long windowTimestamp() { + return windowTimestamp; + } + + @Override + public void visit(final Visitor visitor) { + visitor.visit(this); + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + final WindowedDelete that = (WindowedDelete) o; + return windowTimestamp == that.windowTimestamp; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), windowTimestamp); + } + +} diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/WindowedPut.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/WindowedPut.java new file mode 100644 index 000000000..b732222bc --- /dev/null +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/WindowedPut.java @@ -0,0 +1,65 @@ +/* + * Copyright 2025 Responsive Computing, Inc. + * + * This source code is licensed under the Responsive Business Source License Agreement v1.0 + * available at: + * + * https://www.responsive.dev/legal/responsive-bsl-10 + * + * This software requires a valid Commercial License Key for production use. Trial and commercial + * licenses can be obtained at https://www.responsive.dev + */ + +package dev.responsive.kafka.internal.db.rs3.client; + +import java.util.Objects; + +public class WindowedPut extends Put { + private final long timestamp; + private final long windowTimestamp; + + public WindowedPut( + final byte[] key, + final byte[] value, + final long timestamp, + final long windowTimestamp + ) { + super(key, value); + this.timestamp = timestamp; + this.windowTimestamp = windowTimestamp; + } + + public long timestamp() { + return timestamp; + } + + public long windowTimestamp() { + return windowTimestamp; + } + + @Override + public void visit(final Visitor visitor) { + visitor.visit(this); + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + final WindowedPut that = (WindowedPut) o; + return timestamp == that.timestamp && windowTimestamp == that.windowTimestamp; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), timestamp, windowTimestamp); + } + +} diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcKeyValueIterator.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcKeyValueIterator.java index ccc2d65bc..f21430340 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcKeyValueIterator.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcKeyValueIterator.java @@ -12,10 +12,10 @@ package dev.responsive.kafka.internal.db.rs3.client.grpc; -import com.google.protobuf.ByteString; import dev.responsive.kafka.internal.db.rs3.client.RS3Exception; import dev.responsive.kafka.internal.db.rs3.client.RS3TransientException; import dev.responsive.kafka.internal.db.rs3.client.RangeBound; +import dev.responsive.kafka.internal.utils.WindowedKey; import dev.responsive.rs3.Rs3; import io.grpc.stub.StreamObserver; import java.util.NoSuchElementException; @@ -31,24 +31,50 @@ * Internal iterator implementation which supports retries using RS3's asynchronous * Range API. */ -public class GrpcKeyValueIterator implements KeyValueIterator { +public class GrpcKeyValueIterator implements KeyValueIterator { private static final Logger LOG = LoggerFactory.getLogger(GrpcKeyValueIterator.class); - private final GrpcRangeRequestProxy requestProxy; + private final GrpcRangeRequestProxy requestProxy; + private final GrpcRangeKeyCodec keyCodec; private final GrpcMessageQueue queue; - private RangeBound startBound; + private RangeBound startBound; private RangeResultObserver resultObserver; public GrpcKeyValueIterator( - RangeBound initialStartBound, - GrpcRangeRequestProxy requestProxy + RangeBound initialStartBound, + GrpcRangeRequestProxy requestProxy, + GrpcRangeKeyCodec keyCodec ) { this.requestProxy = requestProxy; + this.keyCodec = keyCodec; this.queue = new GrpcMessageQueue<>(); this.startBound = initialStartBound; sendRangeRequest(); } + static GrpcKeyValueIterator standard( + RangeBound initialStartBound, + GrpcRangeRequestProxy requestProxy + ) { + return new GrpcKeyValueIterator<>( + initialStartBound, + requestProxy, + GrpcRangeKeyCodec.STANDARD_CODEC + ); + } + + static GrpcKeyValueIterator windowed( + RangeBound initialStartBound, + GrpcRangeRequestProxy requestProxy + ) { + + return new GrpcKeyValueIterator<>( + initialStartBound, + requestProxy, + GrpcRangeKeyCodec.WINDOW_CODEC + ); + } + private void sendRangeRequest() { // Note that backoff on retry is handled internally by the request proxy resultObserver = new RangeResultObserver(); @@ -61,12 +87,12 @@ public boolean hasNext() { } @Override - public KeyValue next() { + public KeyValue next() { final var nextKeyValue = peekNextKeyValue(); if (nextKeyValue.isPresent()) { queue.poll(); final var keyValue = nextKeyValue.get(); - this.startBound = RangeBound.exclusive(keyValue.key.get()); + this.startBound = RangeBound.exclusive(keyValue.key); return keyValue; } else { throw new NoSuchElementException(); @@ -80,7 +106,7 @@ public void close() { } } - Optional> peekNextKeyValue() { + Optional> peekNextKeyValue() { while (true) { try { final var message = queue.peek(); @@ -96,29 +122,29 @@ Optional> peekNextKeyValue() { } } - private Optional> tryUnwrapKeyValue(final Message message) { + private Optional> tryUnwrapKeyValue(final Message message) { return message.map(new Mapper<>() { @Override - public Optional> map(final EndOfStream endOfStream) { + public Optional> map(final EndOfStream endOfStream) { return Optional.empty(); } @Override - public Optional> map(final StreamError error) { + public Optional> map(final StreamError error) { throw GrpcRs3Util.wrapThrowable(error.exception); } @Override - public Optional> map(final Result result) { - final var key = Bytes.wrap(result.key.toByteArray()); - final var value = result.value.toByteArray(); + public Optional> map(final Result result) { + final var key = keyCodec.decodeRangeResult(result.keyValue); + final var value = result.keyValue.getValue().toByteArray(); return Optional.of(new KeyValue<>(key, value)); } }); } @Override - public Bytes peekNextKey() { + public K peekNextKey() { return peekNextKeyValue() .map(bytesKeyValue -> bytesKeyValue.key) .orElse(null); @@ -134,8 +160,7 @@ public void onNext(final Rs3.RangeResult rangeResult) { } else if (rangeResult.getType() == Rs3.RangeResult.Type.END_OF_STREAM) { queue.put(new EndOfStream()); } else { - final var result = rangeResult.getResult(); - queue.put(new Result(result.getKey(), result.getValue())); + queue.put(new Result(rangeResult.getResult())); } } @@ -172,12 +197,12 @@ public T map(final Mapper mapper) { } private static class Result implements Message { - private final ByteString key; - private final ByteString value; + private final Rs3.KeyValue keyValue; - private Result(final ByteString key, final ByteString value) { - this.key = key; - this.value = value; + private Result( + final Rs3.KeyValue keyValue + ) { + this.keyValue = keyValue; } @Override diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRS3Client.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRS3Client.java index a1ebc2523..456c325b1 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRS3Client.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRS3Client.java @@ -21,7 +21,6 @@ import dev.responsive.kafka.api.config.ResponsiveConfig; import dev.responsive.kafka.internal.db.rs3.client.CurrentOffsets; import dev.responsive.kafka.internal.db.rs3.client.LssId; -import dev.responsive.kafka.internal.db.rs3.client.Put; import dev.responsive.kafka.internal.db.rs3.client.RS3Client; import dev.responsive.kafka.internal.db.rs3.client.RS3TimeoutException; import dev.responsive.kafka.internal.db.rs3.client.RS3TransientException; @@ -29,6 +28,7 @@ import dev.responsive.kafka.internal.db.rs3.client.Store; import dev.responsive.kafka.internal.db.rs3.client.StreamSenderMessageReceiver; import dev.responsive.kafka.internal.db.rs3.client.WalEntry; +import dev.responsive.kafka.internal.utils.WindowedKey; import dev.responsive.rs3.RS3Grpc; import dev.responsive.rs3.Rs3; import io.grpc.stub.StreamObserver; @@ -224,20 +224,65 @@ public KeyValueIterator range( final LssId lssId, final int pssId, final Optional expectedWrittenOffset, - RangeBound from, - RangeBound to + RangeBound from, + RangeBound to + ) { + return sendRange( + storeId, + lssId, + pssId, + expectedWrittenOffset, + from, + to, + GrpcRangeKeyCodec.STANDARD_CODEC + ); + } + + @Override + public KeyValueIterator windowedRange( + final UUID storeId, + final LssId lssId, + final int pssId, + final Optional expectedWrittenOffset, + final RangeBound from, + final RangeBound to + ) { + return sendRange( + storeId, + lssId, + pssId, + expectedWrittenOffset, + from, + to, + GrpcRangeKeyCodec.WINDOW_CODEC + ); + } + + private KeyValueIterator sendRange( + final UUID storeId, + final LssId lssId, + final int pssId, + final Optional expectedWrittenOffset, + final RangeBound from, + final RangeBound to, + final GrpcRangeKeyCodec keyCodec ) { final var requestBuilder = Rs3.RangeRequest.newBuilder() .setStoreId(uuidToUuidProto(storeId)) .setLssId(lssIdProto(lssId)) .setPssId(pssId) - .setTo(protoBound(to)); + .setTo(protoRangeBound(to, keyCodec)); expectedWrittenOffset.ifPresent(requestBuilder::setExpectedWrittenOffset); final Supplier rangeDescription = () -> "Range(storeId=" + storeId + ", lssId=" + lssId + ", pssId=" + pssId + ")"; final var asyncStub = stubs.stubs(storeId, pssId).asyncStub(); - final var rangeProxy = new RangeProxy(requestBuilder, asyncStub, rangeDescription); - return new GrpcKeyValueIterator(from, rangeProxy); + final var rangeProxy = new RangeProxy<>( + requestBuilder, + asyncStub, + keyCodec, + rangeDescription + ); + return new GrpcKeyValueIterator<>(from, rangeProxy, keyCodec); } @Override @@ -246,17 +291,53 @@ public Optional get( final LssId lssId, final int pssId, final Optional expectedWrittenOffset, - final byte[] key + final Bytes key ) { final var requestBuilder = Rs3.GetRequest.newBuilder() - .setStoreId(uuidToUuidProto(storeId)) + .setKey(ByteString.copyFrom(key.get())); + return sendGet( + requestBuilder, + storeId, + lssId, + pssId, + expectedWrittenOffset + ); + } + + @Override + public Optional windowedGet( + final UUID storeId, + final LssId lssId, + final int pssId, + final Optional expectedWrittenOffset, + final WindowedKey key + ) { + final var requestBuilder = Rs3.GetRequest.newBuilder() + .setKey(ByteString.copyFrom(key.key.get())) + .setWindowTimestamp(key.windowStartMs); + return sendGet( + requestBuilder, + storeId, + lssId, + pssId, + expectedWrittenOffset + ); + } + + private Optional sendGet( + final Rs3.GetRequest.Builder requestBuilder, + final UUID storeId, + final LssId lssId, + final int pssId, + final Optional expectedWrittenOffset + ) { + requestBuilder.setStoreId(uuidToUuidProto(storeId)) .setLssId(lssIdProto(lssId)) - .setPssId(pssId) - .setKey(ByteString.copyFrom(key)); + .setPssId(pssId); expectedWrittenOffset.ifPresent(requestBuilder::setExpectedWrittenOffset); + final var request = requestBuilder.build(); final RS3Grpc.RS3BlockingStub stub = stubs.stubs(storeId, pssId).syncStub(); - final Rs3.GetResult result = withRetry( () -> stub.get(request), () -> "Get(storeId=" + storeId + ", lssId=" + lssId + ", pssId=" + pssId + ")" @@ -289,16 +370,10 @@ private void addWalEntryToSegment( final WalEntry entry, final Rs3.WriteWALSegmentRequest.Builder builder ) { - if (entry instanceof Put) { - final var put = (Put) entry; - final var putBuilder = Rs3.WriteWALSegmentRequest.Put.newBuilder() - .setKey(ByteString.copyFrom(put.key())); - if (put.value().isPresent()) { - putBuilder.setValue(ByteString.copyFrom(put.value().get())); - } - putBuilder.setTtl(Rs3.Ttl.newBuilder().setTtlType(Rs3.Ttl.TtlType.DEFAULT).build()); - builder.setPut(putBuilder.build()); - } + final var putBuilder = Rs3.WriteWALSegmentRequest.Put.newBuilder(); + final var writer = new WalEntryPutWriter(putBuilder); + entry.visit(writer); + builder.setPut(putBuilder); } private void checkField(final Supplier check, final String field) { @@ -307,26 +382,29 @@ private void checkField(final Supplier check, final String field) { } } - private Rs3.Bound protoBound(RangeBound bound) { + private static Rs3.Bound protoRangeBound( + RangeBound bound, + GrpcRangeKeyCodec keyCodec + ) { return bound.map(new RangeBound.Mapper<>() { @Override - public Rs3.Bound map(final RangeBound.InclusiveBound b) { - return Rs3.Bound.newBuilder() - .setType(Rs3.Bound.Type.INCLUSIVE) - .setKey(ByteString.copyFrom(b.key())) - .build(); + public Rs3.Bound map(final RangeBound.InclusiveBound b) { + final var builder = Rs3.Bound.newBuilder() + .setType(Rs3.Bound.Type.INCLUSIVE); + keyCodec.encodeRangeBound(b.key(), builder); + return builder.build(); } @Override - public Rs3.Bound map(final RangeBound.ExclusiveBound b) { - return Rs3.Bound.newBuilder() - .setType(Rs3.Bound.Type.EXCLUSIVE) - .setKey(ByteString.copyFrom(b.key())) - .build(); + public Rs3.Bound map(final RangeBound.ExclusiveBound b) { + final var builder = Rs3.Bound.newBuilder() + .setType(Rs3.Bound.Type.EXCLUSIVE); + keyCodec.encodeRangeBound(b.key(), builder); + return builder.build(); } @Override - public Rs3.Bound map(final RangeBound.Unbounded b) { + public Rs3.Bound map(final RangeBound.Unbounded b) { return Rs3.Bound.newBuilder() .setType(Rs3.Bound.Type.UNBOUNDED) .build(); @@ -334,26 +412,33 @@ public Rs3.Bound map(final RangeBound.Unbounded b) { }); } - private class RangeProxy implements GrpcRangeRequestProxy { + private class RangeProxy implements GrpcRangeRequestProxy { private final Rs3.RangeRequest.Builder requestBuilder; private final RS3Grpc.RS3Stub stub; private final Supplier opDescription; + private final GrpcRangeKeyCodec keyCodec; private int attempts = 0; private long deadlineMs = Long.MAX_VALUE; // Set upon the first retry private RangeProxy( final Rs3.RangeRequest.Builder requestBuilder, final RS3Grpc.RS3Stub stub, + final GrpcRangeKeyCodec keyCodec, final Supplier opDescription ) { this.requestBuilder = requestBuilder; this.stub = stub; + this.keyCodec = keyCodec; this.opDescription = opDescription; } @Override - public void send(final RangeBound start, final StreamObserver resultObserver) { - requestBuilder.setFrom(protoBound(start)); + public void send( + final RangeBound start, + final StreamObserver resultObserver + ) { + final var protoRange = protoRangeBound(start, keyCodec); + requestBuilder.setFrom(protoRange); while (true) { try { diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRangeKeyCodec.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRangeKeyCodec.java new file mode 100644 index 000000000..0c211068a --- /dev/null +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRangeKeyCodec.java @@ -0,0 +1,54 @@ +/* + * Copyright 2025 Responsive Computing, Inc. + * + * This source code is licensed under the Responsive Business Source License Agreement v1.0 + * available at: + * + * https://www.responsive.dev/legal/responsive-bsl-10 + * + * This software requires a valid Commercial License Key for production use. Trial and commercial + * licenses can be obtained at https://www.responsive.dev + */ + +package dev.responsive.kafka.internal.db.rs3.client.grpc; + +import com.google.protobuf.ByteString; +import dev.responsive.kafka.internal.utils.WindowedKey; +import dev.responsive.rs3.Rs3; +import org.apache.kafka.common.utils.Bytes; + +public interface GrpcRangeKeyCodec { + WindowedKeyCodec WINDOW_CODEC = new WindowedKeyCodec(); + StandardKeyCodec STANDARD_CODEC = new StandardKeyCodec(); + + K decodeRangeResult(Rs3.KeyValue keyValue); + + void encodeRangeBound(K key, Rs3.Bound.Builder builder); + + class WindowedKeyCodec implements GrpcRangeKeyCodec { + @Override + public WindowedKey decodeRangeResult(final Rs3.KeyValue keyValue) { + final var key = Bytes.wrap(keyValue.getKey().toByteArray()); + final var windowTimestamp = keyValue.getWindowTimestamp(); + return new WindowedKey(key, windowTimestamp); + } + + @Override + public void encodeRangeBound(final WindowedKey key, final Rs3.Bound.Builder builder) { + builder.setKey(ByteString.copyFrom(key.key.get())); + builder.setWindowTimestamp(key.windowStartMs); + } + } + + class StandardKeyCodec implements GrpcRangeKeyCodec { + @Override + public Bytes decodeRangeResult(final Rs3.KeyValue keyValue) { + return Bytes.wrap(keyValue.getKey().toByteArray()); + } + + @Override + public void encodeRangeBound(final Bytes key, final Rs3.Bound.Builder builder) { + builder.setKey(ByteString.copyFrom(key.get())); + } + } +} diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRangeRequestProxy.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRangeRequestProxy.java index 0b3c41aaf..0592cc4d6 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRangeRequestProxy.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRangeRequestProxy.java @@ -24,7 +24,7 @@ * updated. If the observer encounters an error, then it can retry with the updated * start bound. */ -public interface GrpcRangeRequestProxy { +public interface GrpcRangeRequestProxy { /** * Send a range request with an updated start bound. The results will be passed * through to result observer. If a transient error is encountered through the @@ -33,5 +33,5 @@ public interface GrpcRangeRequestProxy { * @param start The updated start bound based on key-values seen with `resultObserver` * @param resultObserver An observer for key-value results and the end of stream marker */ - void send(RangeBound start, StreamObserver resultObserver); + void send(RangeBound start, StreamObserver resultObserver); } diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/WalEntryPutWriter.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/WalEntryPutWriter.java new file mode 100644 index 000000000..082e76ee0 --- /dev/null +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/WalEntryPutWriter.java @@ -0,0 +1,42 @@ +package dev.responsive.kafka.internal.db.rs3.client.grpc; + +import com.google.protobuf.ByteString; +import dev.responsive.kafka.internal.db.rs3.client.Delete; +import dev.responsive.kafka.internal.db.rs3.client.Put; +import dev.responsive.kafka.internal.db.rs3.client.WalEntry; +import dev.responsive.kafka.internal.db.rs3.client.WindowedDelete; +import dev.responsive.kafka.internal.db.rs3.client.WindowedPut; +import dev.responsive.rs3.Rs3; + +public class WalEntryPutWriter implements WalEntry.Visitor { + private final Rs3.WriteWALSegmentRequest.Put.Builder builder; + + public WalEntryPutWriter(final Rs3.WriteWALSegmentRequest.Put.Builder builder) { + this.builder = builder; + } + + @Override + public void visit(final Put put) { + builder.setKey(ByteString.copyFrom(put.key())); + builder.setValue(ByteString.copyFrom(put.value())); + builder.setTtl(Rs3.Ttl.newBuilder().setTtlType(Rs3.Ttl.TtlType.DEFAULT).build()); + } + + @Override + public void visit(final Delete delete) { + builder.setKey(ByteString.copyFrom(delete.key())); + builder.setTtl(Rs3.Ttl.newBuilder().setTtlType(Rs3.Ttl.TtlType.DEFAULT).build()); + } + + @Override + public void visit(final WindowedDelete windowedDelete) { + visit((Delete) windowedDelete); + builder.setWindowTimestamp(windowedDelete.windowTimestamp()); + } + + @Override + public void visit(final WindowedPut windowedPut) { + visit((Put) windowedPut); + builder.setWindowTimestamp(windowedPut.windowTimestamp()); + } +} diff --git a/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/RS3KVTableIntegrationTest.java b/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/RS3KVTableIntegrationTest.java index 9f6f28434..f8460508c 100644 --- a/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/RS3KVTableIntegrationTest.java +++ b/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/RS3KVTableIntegrationTest.java @@ -152,7 +152,7 @@ public void shouldWriteToStore() throws InterruptedException, ExecutionException new LssId(PARTITION_ID), pss, Optional.of(10L), - key.get() + key ); assertThat(result.get(), is("bar".getBytes())); } diff --git a/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/RS3WindowedKeySerdeTest.java b/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/RS3WindowedKeySerdeTest.java deleted file mode 100644 index 6b0b2fbff..000000000 --- a/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/RS3WindowedKeySerdeTest.java +++ /dev/null @@ -1,23 +0,0 @@ -package dev.responsive.kafka.internal.db.rs3; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.is; - -import dev.responsive.kafka.internal.utils.WindowedKey; -import net.jqwik.api.ForAll; -import net.jqwik.api.Property; - -class RS3WindowedKeySerdeTest { - - @Property - public void shouldSerializeAndDeserialize( - @ForAll byte[] key, - @ForAll long timestamp - ) { - final var windowKey = new WindowedKey(key, timestamp); - final var serde = new RS3WindowedKeySerde(); - final var serialized = serde.serialize(windowKey); - assertThat(serde.deserialize(serialized), is(windowKey)); - } - -} \ No newline at end of file diff --git a/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcKeyValueIteratorTest.java b/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcKeyValueIteratorTest.java index e2ec20da3..972c2c63a 100644 --- a/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcKeyValueIteratorTest.java +++ b/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcKeyValueIteratorTest.java @@ -28,6 +28,7 @@ import io.grpc.StatusRuntimeException; import io.grpc.stub.StreamObserver; import java.nio.charset.StandardCharsets; +import org.apache.kafka.common.utils.Bytes; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; @@ -38,12 +39,12 @@ class GrpcKeyValueIteratorTest { @Mock - private GrpcRangeRequestProxy requestProxy; + private GrpcRangeRequestProxy requestProxy; @Test @SuppressWarnings("unchecked") public void shouldIterateKeyValueResults() { - final var startBound = RangeBound.inclusive("a".getBytes(StandardCharsets.UTF_8)); + final var startBound = RangeBound.inclusive(Bytes.wrap("a".getBytes(StandardCharsets.UTF_8))); Mockito.doAnswer(invocation -> { StreamObserver observer = invocation.getArgument(1, StreamObserver.class); observer.onNext(newKeyValueResult("a")); @@ -54,7 +55,7 @@ public void shouldIterateKeyValueResults() { return null; }).when(requestProxy).send(eq(startBound), any()); - try (final var iter = new GrpcKeyValueIterator(startBound, requestProxy)) { + try (final var iter = GrpcKeyValueIterator.standard(startBound, requestProxy)) { assertNextKey(iter, "a"); assertNextKey(iter, "b"); assertNextKey(iter, "c"); @@ -65,7 +66,8 @@ public void shouldIterateKeyValueResults() { @Test @SuppressWarnings("unchecked") public void shouldRetryRangeRequestAfterTransientFailure() { - final var startBound = RangeBound.inclusive("a".getBytes(StandardCharsets.UTF_8)); + final var startBound = RangeBound.inclusive( + Bytes.wrap("a".getBytes(StandardCharsets.UTF_8))); Mockito.doAnswer(invocation -> { StreamObserver observer = invocation.getArgument(1, StreamObserver.class); observer.onNext(newKeyValueResult("a")); @@ -73,7 +75,8 @@ public void shouldRetryRangeRequestAfterTransientFailure() { return null; }).when(requestProxy).send(eq(startBound), any()); - final var retryStartBound = RangeBound.exclusive("a".getBytes(StandardCharsets.UTF_8)); + final var retryStartBound = RangeBound.exclusive( + Bytes.wrap("a".getBytes(StandardCharsets.UTF_8))); Mockito.doAnswer(invocation -> { StreamObserver observer = invocation.getArgument(1, StreamObserver.class); observer.onNext(newKeyValueResult("b")); @@ -83,7 +86,7 @@ public void shouldRetryRangeRequestAfterTransientFailure() { return null; }).when(requestProxy).send(eq(retryStartBound), any()); - try (final var iter = new GrpcKeyValueIterator(startBound, requestProxy)) { + try (final var iter = GrpcKeyValueIterator.standard(startBound, requestProxy)) { assertNextKey(iter, "a"); assertNextKey(iter, "b"); assertNextKey(iter, "c"); @@ -94,7 +97,7 @@ public void shouldRetryRangeRequestAfterTransientFailure() { @Test @SuppressWarnings("unchecked") public void shouldRetryAfterUnexpectedStreamCompletion() { - final var startBound = RangeBound.inclusive("a".getBytes(StandardCharsets.UTF_8)); + final var startBound = RangeBound.inclusive(Bytes.wrap("a".getBytes(StandardCharsets.UTF_8))); Mockito.doAnswer(invocation -> { StreamObserver observer = invocation.getArgument(1, StreamObserver.class); observer.onNext(newKeyValueResult("a")); @@ -102,7 +105,8 @@ public void shouldRetryAfterUnexpectedStreamCompletion() { return null; }).when(requestProxy).send(eq(startBound), any()); - final var retryStartBound = RangeBound.exclusive("a".getBytes(StandardCharsets.UTF_8)); + final var retryStartBound = RangeBound.exclusive( + Bytes.wrap("a".getBytes(StandardCharsets.UTF_8))); Mockito.doAnswer(invocation -> { StreamObserver observer = invocation.getArgument(1, StreamObserver.class); observer.onNext(newKeyValueResult("b")); @@ -112,7 +116,7 @@ public void shouldRetryAfterUnexpectedStreamCompletion() { return null; }).when(requestProxy).send(eq(retryStartBound), any()); - try (final var iter = new GrpcKeyValueIterator(startBound, requestProxy)) { + try (final var iter = GrpcKeyValueIterator.standard(startBound, requestProxy)) { assertNextKey(iter, "a"); assertNextKey(iter, "b"); assertNextKey(iter, "c"); @@ -123,7 +127,8 @@ public void shouldRetryAfterUnexpectedStreamCompletion() { @Test @SuppressWarnings("unchecked") public void shouldPropagateUnexpectedFailures() { - final var startBound = RangeBound.inclusive("a".getBytes(StandardCharsets.UTF_8)); + final var startBound = RangeBound.inclusive( + Bytes.wrap("a".getBytes(StandardCharsets.UTF_8))); Mockito.doAnswer(invocation -> { StreamObserver observer = invocation.getArgument(1, StreamObserver.class); observer.onNext(newKeyValueResult("a")); @@ -131,7 +136,7 @@ public void shouldPropagateUnexpectedFailures() { return null; }).when(requestProxy).send(eq(startBound), any()); - try (final var iter = new GrpcKeyValueIterator(startBound, requestProxy)) { + try (final var iter = GrpcKeyValueIterator.standard(startBound, requestProxy)) { assertNextKey(iter, "a"); final var rs3Exception = assertThrows(RS3Exception.class, iter::next); assertThat(rs3Exception.getCause(), instanceOf(StatusRuntimeException.class)); @@ -140,7 +145,7 @@ public void shouldPropagateUnexpectedFailures() { } } - private void assertNextKey(GrpcKeyValueIterator iter, String key) { + private void assertNextKey(GrpcKeyValueIterator iter, String key) { assertThat(iter.hasNext(), is(true)); final var keyValue = iter.next(); final var keyBytes = keyValue.key.get(); @@ -148,5 +153,4 @@ private void assertNextKey(GrpcKeyValueIterator iter, String key) { assertThat(keyString, is(key)); } - } \ No newline at end of file diff --git a/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRS3ClientEndToEndTest.java b/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRS3ClientEndToEndTest.java index ddf14fda1..1305d3f16 100644 --- a/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRS3ClientEndToEndTest.java +++ b/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRS3ClientEndToEndTest.java @@ -100,7 +100,7 @@ public void shouldPutAndGet() { LSS_ID, PSS_ID, Optional.of(5L), - key + Bytes.wrap(key) ); assertThat(getResult.isPresent(), is(true)); final var resultValue = getResult.get(); @@ -164,8 +164,8 @@ public void shouldScanKeyValuesInBoundedRange() { LSS_ID, PSS_ID, Optional.of(10L), - RangeBound.inclusive("b".getBytes(StandardCharsets.UTF_8)), - RangeBound.exclusive("e".getBytes(StandardCharsets.UTF_8)) + RangeBound.inclusive(Bytes.wrap("b".getBytes(StandardCharsets.UTF_8))), + RangeBound.exclusive(Bytes.wrap("e".getBytes(StandardCharsets.UTF_8))) ); assertNext(iter, "b", "bar"); @@ -353,7 +353,7 @@ public void range( final var range = GrpsRs3TestUtil.newRangeFromProto(req); for (final var keyValueEntry : table.entrySet()) { - if (!range.contains(keyValueEntry.getKey().get())) { + if (!range.contains(keyValueEntry.getKey())) { continue; } diff --git a/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRS3ClientTest.java b/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRS3ClientTest.java index 0ebce797c..76c1cdd11 100644 --- a/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRS3ClientTest.java +++ b/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRS3ClientTest.java @@ -52,6 +52,7 @@ import java.util.UUID; import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicInteger; +import org.apache.kafka.common.utils.Bytes; import org.apache.kafka.common.utils.MockTime; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -640,10 +641,16 @@ public void shouldGetWithExpectedWrittenOffset() { ); // when: - final var result = client.get(STORE_ID, LSS_ID, PSS_ID, Optional.of(123L), "foo".getBytes()); + final var result = client.get( + STORE_ID, + LSS_ID, + PSS_ID, + Optional.of(123L), + Bytes.wrap("foo".getBytes()) + ); // then: - assertThat(result.get(), is("bar".getBytes())); + assertThat(result.get(), is(Bytes.wrap("bar".getBytes()))); verify(stub).get(Rs3.GetRequest.newBuilder() .setLssId(lssIdProto(LSS_ID)) .setPssId(PSS_ID) @@ -667,7 +674,13 @@ public void shouldGet() { ); // when: - final var result = client.get(STORE_ID, LSS_ID, PSS_ID, Optional.empty(), "foo".getBytes()); + final var result = client.get( + STORE_ID, + LSS_ID, + PSS_ID, + Optional.empty(), + Bytes.wrap("foo".getBytes()) + ); // then: assertThat(result.get(), is("bar".getBytes())); @@ -688,7 +701,13 @@ public void shouldHandleNegativeGet() { ); // when: - final var result = client.get(STORE_ID, LSS_ID, PSS_ID, Optional.of(123L), "foo".getBytes()); + final var result = client.get( + STORE_ID, + LSS_ID, + PSS_ID, + Optional.of(123L), + Bytes.wrap("foo".getBytes()) + ); // then: assertThat(result.isEmpty(), is(true)); @@ -702,7 +721,13 @@ public void shouldRetryGet() { .thenReturn(Rs3.GetResult.newBuilder().build()); // when: - final var result = client.get(STORE_ID, LSS_ID, PSS_ID, Optional.of(123L), "foo".getBytes()); + final var result = client.get( + STORE_ID, + LSS_ID, + PSS_ID, + Optional.of(123L), + Bytes.wrap("foo".getBytes()) + ); // then: assertThat(result.isEmpty(), is(true)); @@ -715,10 +740,13 @@ public void shouldPropagateUnexpectedExceptionsFromGet() { .thenThrow(new StatusRuntimeException(Status.UNKNOWN)); // when: - final RS3Exception exception = assertThrows( - RS3Exception.class, - () -> client.get(STORE_ID, LSS_ID, PSS_ID, Optional.of(123L), "foo".getBytes()) - ); + final RS3Exception exception = assertThrows(RS3Exception.class, () -> client.get( + STORE_ID, + LSS_ID, + PSS_ID, + Optional.of(123L), + Bytes.wrap("foo".getBytes()) + )); // then: assertThat(exception.getCause(), instanceOf(StatusRuntimeException.class)); @@ -738,7 +766,7 @@ public void shouldTimeoutGet() { LSS_ID, PSS_ID, Optional.of(123L), - "foo".getBytes() + Bytes.wrap("foo".getBytes()) )); // then: @@ -985,12 +1013,10 @@ private StreamObserver verifyWalSegmentResultObserver private Rs3.WriteWALSegmentRequest.Put putProto(final Put put) { final var builder = Rs3.WriteWALSegmentRequest.Put.newBuilder() .setKey(ByteString.copyFrom(put.key())); - if (put.value().isPresent()) { - builder.setValue(ByteString.copyFrom(put.value().get())); - builder.setTtl(Rs3.Ttl.newBuilder() - .setTtlType(Rs3.Ttl.TtlType.DEFAULT) - .build()); - } + builder.setValue(ByteString.copyFrom(put.value())); + builder.setTtl(Rs3.Ttl.newBuilder() + .setTtlType(Rs3.Ttl.TtlType.DEFAULT) + .build()); return builder.build(); } diff --git a/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpsRs3TestUtil.java b/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpsRs3TestUtil.java index 3d8730ffa..1d1f5a37f 100644 --- a/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpsRs3TestUtil.java +++ b/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpsRs3TestUtil.java @@ -16,6 +16,7 @@ import dev.responsive.kafka.internal.db.rs3.client.Range; import dev.responsive.kafka.internal.db.rs3.client.RangeBound; import dev.responsive.rs3.Rs3; +import org.apache.kafka.common.utils.Bytes; public class GrpsRs3TestUtil { @@ -37,18 +38,18 @@ public static Rs3.RangeResult newEndOfStreamResult() { .build(); } - public static Range newRangeFromProto(Rs3.RangeRequest req) { + public static Range newRangeFromProto(Rs3.RangeRequest req) { final var startBound = newRangeBoundFromProto(req.getFrom()); final var endBound = newRangeBoundFromProto(req.getTo()); - return new Range(startBound, endBound); + return new Range<>(startBound, endBound); } - private static RangeBound newRangeBoundFromProto(Rs3.Bound bound) { + private static RangeBound newRangeBoundFromProto(Rs3.Bound bound) { switch (bound.getType()) { case EXCLUSIVE: - return RangeBound.exclusive(bound.getKey().toByteArray()); + return RangeBound.exclusive(Bytes.wrap(bound.getKey().toByteArray())); case INCLUSIVE: - return RangeBound.inclusive(bound.getKey().toByteArray()); + return RangeBound.inclusive(Bytes.wrap(bound.getKey().toByteArray())); case UNBOUNDED: return RangeBound.unbounded(); default: From b385f5e31284e6c2925e5654250ae76703eb8f11 Mon Sep 17 00:00:00 2001 From: Jason Gustafson <12502538+hachikuji@users.noreply.github.com> Date: Tue, 8 Apr 2025 14:12:22 -0700 Subject: [PATCH 14/29] Update rs3 proto --- controller-api/src/main/external-protos/opentelemetry-proto | 2 +- kafka-client/src/main/external-protos/rs3 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/controller-api/src/main/external-protos/opentelemetry-proto b/controller-api/src/main/external-protos/opentelemetry-proto index b41217bad..3ee8e6aab 160000 --- a/controller-api/src/main/external-protos/opentelemetry-proto +++ b/controller-api/src/main/external-protos/opentelemetry-proto @@ -1 +1 @@ -Subproject commit b41217bad1fb49b65a0442650c496abfadfc2216 +Subproject commit 3ee8e6aab7b5fcf2d183f40be55c0723dcc88ce8 diff --git a/kafka-client/src/main/external-protos/rs3 b/kafka-client/src/main/external-protos/rs3 index eab53946a..68e2a7c0e 160000 --- a/kafka-client/src/main/external-protos/rs3 +++ b/kafka-client/src/main/external-protos/rs3 @@ -1 +1 @@ -Subproject commit eab53946afd1c8e6ef704435338854df3d949e7f +Subproject commit 68e2a7c0e1a42e3914f5e0934de899dd79e35e3f From 03bdf57d83e385e919a92dfdb38cdee36d6651f3 Mon Sep 17 00:00:00 2001 From: Jason Gustafson <12502538+hachikuji@users.noreply.github.com> Date: Tue, 8 Apr 2025 14:51:39 -0700 Subject: [PATCH 15/29] Fix failing tests --- .../internal/db/rs3/RS3WindowTableTest.java | 26 +++---- .../db/rs3/client/grpc/GrpcRS3ClientTest.java | 75 ++++++++++++++++++- 2 files changed, 87 insertions(+), 14 deletions(-) diff --git a/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/RS3WindowTableTest.java b/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/RS3WindowTableTest.java index 360b4a4dd..045e6f631 100644 --- a/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/RS3WindowTableTest.java +++ b/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/RS3WindowTableTest.java @@ -57,13 +57,13 @@ public void shouldReturnValueFromFetchIfKeyFound() { when(partitioner.pss(any(), eq(lssId))).thenReturn(pssId); when(client.getCurrentOffsets(storeId, lssId, pssId)) .thenReturn(new CurrentOffsets(Optional.of(5L), Optional.of(5L))); - when(client.get( - eq(storeId), - eq(lssId), - eq(pssId), - eq(Optional.of(5L)), - any()) - ).thenReturn(Optional.of(value)); + when(client.windowedGet( + storeId, + lssId, + pssId, + Optional.of(5L), + new WindowedKey(key, timestamp) + )).thenReturn(Optional.of(value)); // Then: table.init(kafkaPartition); @@ -86,12 +86,12 @@ public void shouldReturnNullFromFetchIfKeyNotFound() { when(partitioner.pss(any(), eq(lssId))).thenReturn(pssId); when(client.getCurrentOffsets(storeId, lssId, pssId)) .thenReturn(new CurrentOffsets(Optional.of(5L), Optional.of(5L))); - when(client.get( - eq(storeId), - eq(lssId), - eq(pssId), - eq(Optional.of(5L)), - any()) + when(client.windowedGet( + storeId, + lssId, + pssId, + Optional.of(5L), + new WindowedKey(key, timestamp)) ).thenReturn(Optional.empty()); // Then: diff --git a/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRS3ClientTest.java b/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRS3ClientTest.java index 76c1cdd11..8c867a86e 100644 --- a/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRS3ClientTest.java +++ b/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRS3ClientTest.java @@ -38,6 +38,7 @@ import dev.responsive.kafka.internal.db.rs3.client.RS3TimeoutException; import dev.responsive.kafka.internal.db.rs3.client.RangeBound; import dev.responsive.kafka.internal.db.rs3.client.WalEntry; +import dev.responsive.kafka.internal.utils.WindowedKey; import dev.responsive.rs3.RS3Grpc; import dev.responsive.rs3.Rs3; import dev.responsive.rs3.Rs3.ListStoresResult; @@ -650,7 +651,7 @@ public void shouldGetWithExpectedWrittenOffset() { ); // then: - assertThat(result.get(), is(Bytes.wrap("bar".getBytes()))); + assertThat(result.get(), is("bar".getBytes())); verify(stub).get(Rs3.GetRequest.newBuilder() .setLssId(lssIdProto(LSS_ID)) .setPssId(PSS_ID) @@ -774,6 +775,78 @@ public void shouldTimeoutGet() { assertThat(endTimeMs - startTimeMs, is(retryTimeoutMs)); } + @Test + public void shouldWindowedGet() { + final var windowTimestamp = 500L; + final var key = "foo".getBytes(); + + // given: + when(stub.get(any())) + .thenReturn(Rs3.GetResult.newBuilder().setResult( + Rs3.KeyValue.newBuilder() + .setKey(ByteString.copyFromUtf8("foo")) + .setValue(ByteString.copyFromUtf8("bar")) + .setWindowTimestamp(windowTimestamp) + ).build()); + + // when: + final var result = client.windowedGet( + STORE_ID, + LSS_ID, + PSS_ID, + Optional.empty(), + new WindowedKey(Bytes.wrap(key), windowTimestamp) + ); + + // then: + assertThat(result.get(), is("bar".getBytes())); + verify(stub).get(Rs3.GetRequest.newBuilder() + .setLssId(lssIdProto(LSS_ID)) + .setPssId(PSS_ID) + .setStoreId(uuidToUuidProto(STORE_ID)) + .setKey(ByteString.copyFromUtf8("foo")) + .setWindowTimestamp(windowTimestamp) + .build() + ); + } + + @Test + public void shouldRetryWindowedGet() { + final var windowTimestamp = 500L; + final var key = "foo".getBytes(); + + // given: + when(stub.get(any())) + .thenThrow(new StatusRuntimeException(Status.UNAVAILABLE)) + .thenReturn(Rs3.GetResult.newBuilder().setResult( + Rs3.KeyValue.newBuilder() + .setKey(ByteString.copyFromUtf8("foo")) + .setValue(ByteString.copyFromUtf8("bar")) + .setWindowTimestamp(windowTimestamp) + ).build()); + + // when: + final var result = client.windowedGet( + STORE_ID, + LSS_ID, + PSS_ID, + Optional.empty(), + new WindowedKey(Bytes.wrap(key), windowTimestamp) + ); + + // then: + assertThat(result.get(), is("bar".getBytes())); + verify(stub, times(2)) + .get(Rs3.GetRequest.newBuilder() + .setLssId(lssIdProto(LSS_ID)) + .setPssId(PSS_ID) + .setStoreId(uuidToUuidProto(STORE_ID)) + .setKey(ByteString.copyFromUtf8("foo")) + .setWindowTimestamp(windowTimestamp) + .build() + ); + } + @Test @SuppressWarnings("unchecked") public void shouldRetryRangeRequest() { From 3e7ab01fec4962b549833ba87770849362b40675 Mon Sep 17 00:00:00 2001 From: Jason Gustafson <12502538+hachikuji@users.noreply.github.com> Date: Tue, 15 Apr 2025 10:27:41 -0700 Subject: [PATCH 16/29] Reset opentelemetry to main branch --- controller-api/src/main/external-protos/opentelemetry-proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/controller-api/src/main/external-protos/opentelemetry-proto b/controller-api/src/main/external-protos/opentelemetry-proto index 3ee8e6aab..35c97806f 160000 --- a/controller-api/src/main/external-protos/opentelemetry-proto +++ b/controller-api/src/main/external-protos/opentelemetry-proto @@ -1 +1 @@ -Subproject commit 3ee8e6aab7b5fcf2d183f40be55c0723dcc88ce8 +Subproject commit 35c97806f233c17680f9a00461310b17e0085dd8 From c158a388532b4d1d27a15679a81819bfbd430751 Mon Sep 17 00:00:00 2001 From: Jason Gustafson <12502538+hachikuji@users.noreply.github.com> Date: Wed, 23 Apr 2025 11:34:01 -0700 Subject: [PATCH 17/29] Fix breakage wip --- .../kafka/internal/db/rs3/RS3WindowTable.java | 22 ++++++++++--------- .../internal/db/rs3/client/RS3Client.java | 3 +-- .../db/rs3/client/grpc/GrpcRS3Client.java | 22 ++++++++++--------- .../db/rs3/client/grpc/GrpcRs3Util.java | 8 +++++++ 4 files changed, 33 insertions(+), 22 deletions(-) diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowTable.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowTable.java index efe1695ca..8120fad90 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowTable.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowTable.java @@ -158,15 +158,16 @@ public KeyValueIterator fetch( ) { throwIfPartitionNotInitialized(kafkaPartition); final int pssId = pssPartitioner.pss(key.get(), this.lssId); - final var windowKeyFrom = RangeBound.inclusive(new WindowedKey(key, timeFrom)); - final var windowKeyTo = RangeBound.exclusive(new WindowedKey(key, timeTo)); + final var windowRange = new Range<>( + RangeBound.inclusive(new WindowedKey(key, timeFrom)), + RangeBound.exclusive(new WindowedKey(key, timeTo)) + ); return rs3Client.windowedRange( storeId, lssId, pssId, flushManager.writtenOffset(pssId), - windowKeyFrom, - windowKeyTo + windowRange ); } @@ -191,16 +192,18 @@ public KeyValueIterator fetchRange( ) { throwIfPartitionNotInitialized(kafkaPartition); final List> pssIters = new ArrayList<>(); - final var windowKeyFrom = RangeBound.inclusive(new WindowedKey(fromKey, timeFrom)); - final var windowKeyTo = RangeBound.exclusive(new WindowedKey(toKey, timeTo)); + final var windowRange = new Range<>( + RangeBound.inclusive(new WindowedKey(fromKey, timeFrom)), + RangeBound.exclusive(new WindowedKey(toKey, timeTo)) + ); + for (int pssId : pssPartitioner.pssForLss(this.lssId)) { pssIters.add(rs3Client.windowedRange( storeId, lssId, pssId, flushManager.writtenOffset(pssId), - windowKeyFrom, - windowKeyTo + windowRange )); } return new MergeKeyValueIterator<>(pssIters); @@ -242,8 +245,7 @@ public KeyValueIterator fetchAll( lssId, pssId, flushManager.writtenOffset(pssId), - RangeBound.unbounded(), - RangeBound.unbounded() + Range.unbounded() ); pssIters.add(new TimeRangeFilter(timeRange, rangeIter)); } diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/RS3Client.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/RS3Client.java index d73a02592..5400a3977 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/RS3Client.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/RS3Client.java @@ -70,8 +70,7 @@ KeyValueIterator windowedRange( LssId lssId, int pssId, Optional expectedWrittenOffset, - RangeBound from, - RangeBound to + Range range ); List listStores(); diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRS3Client.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRS3Client.java index d11fe30a9..c2b21ab25 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRS3Client.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRS3Client.java @@ -15,6 +15,7 @@ import static dev.responsive.kafka.internal.db.rs3.client.grpc.GrpcRs3Util.basicDeleteProto; import static dev.responsive.kafka.internal.db.rs3.client.grpc.GrpcRs3Util.basicKeyProto; import static dev.responsive.kafka.internal.db.rs3.client.grpc.GrpcRs3Util.basicPutProto; +import static dev.responsive.kafka.internal.db.rs3.client.grpc.GrpcRs3Util.windowKeyProto; import static dev.responsive.kafka.internal.utils.Utils.lssIdProto; import static dev.responsive.kafka.internal.utils.Utils.uuidFromProto; import static dev.responsive.kafka.internal.utils.Utils.uuidToProto; @@ -249,25 +250,24 @@ public KeyValueIterator windowedRange( final LssId lssId, final int pssId, final Optional expectedWrittenOffset, - final RangeBound from, - final RangeBound to + final Range range ) { return sendRange( storeId, lssId, pssId, expectedWrittenOffset, - new Range<>(from, to), + range, GrpcRangeKeyCodec.WINDOW_CODEC ); } - private KeyValueIterator sendRange( + private > KeyValueIterator sendRange( final UUID storeId, final LssId lssId, final int pssId, final Optional expectedWrittenOffset, - final Range range, + final Range range, final GrpcRangeKeyCodec keyCodec ) { final var requestBuilder = Rs3.RangeRequest.newBuilder() @@ -296,8 +296,10 @@ public Optional get( final Optional expectedWrittenOffset, final Bytes key ) { + final var keyProto = Rs3.Key.newBuilder() + .setBasicKey(basicKeyProto(key.get())); final var requestBuilder = Rs3.GetRequest.newBuilder() - .setKey(ByteString.copyFrom(key.get())); + .setKey(keyProto); return sendGet( requestBuilder, storeId, @@ -315,9 +317,10 @@ public Optional windowedGet( final Optional expectedWrittenOffset, final WindowedKey key ) { + final var keyProto = Rs3.Key.newBuilder() + .setWindowKey(windowKeyProto(key)); final var requestBuilder = Rs3.GetRequest.newBuilder() - .setKey(ByteString.copyFrom(key.key.get())) - .setWindowTimestamp(key.windowStartMs); + .setKey(keyProto); return sendGet( requestBuilder, storeId, @@ -337,7 +340,6 @@ private Optional sendGet( requestBuilder.setStoreId(uuidToProto(storeId)) .setLssId(lssIdProto(lssId)) .setPssId(pssId); - .setKey(Rs3.Key.newBuilder().setBasicKey(basicKeyProto(key))); expectedWrittenOffset.ifPresent(requestBuilder::setExpectedWrittenOffset); final var request = requestBuilder.build(); @@ -442,7 +444,7 @@ public Rs3.BasicBound map(final RangeBound.Unbounded b) { }); } - private class RangeProxy implements GrpcRangeRequestProxy { + private class RangeProxy> implements GrpcRangeRequestProxy { private final Rs3.RangeRequest.Builder requestBuilder; private final RS3Grpc.RS3Stub stub; private final Supplier opDescription; diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRs3Util.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRs3Util.java index 135538e27..af72c1252 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRs3Util.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRs3Util.java @@ -18,6 +18,7 @@ import dev.responsive.kafka.internal.db.rs3.client.Put; import dev.responsive.kafka.internal.db.rs3.client.RS3Exception; import dev.responsive.kafka.internal.db.rs3.client.RS3TransientException; +import dev.responsive.kafka.internal.utils.WindowedKey; import dev.responsive.rs3.Rs3; import io.grpc.Status; import io.grpc.StatusException; @@ -68,6 +69,13 @@ public static Rs3.BasicKey basicKeyProto(final byte[] key) { .build(); } + public static Rs3.WindowKey windowKeyProto(final WindowedKey key) { + return Rs3.WindowKey.newBuilder() + .setKey(ByteString.copyFrom(key.key.get())) + .setWindowTimestamp(key.windowStartMs) + .build(); + } + public static Rs3.BasicKeyValue basicKeyValueProto( final byte[] key, final byte[] value From b07c812d6f3c34cf212dbc784fb37ca5c8b9769b Mon Sep 17 00:00:00 2001 From: Jason Gustafson <12502538+hachikuji@users.noreply.github.com> Date: Thu, 24 Apr 2025 14:31:45 -0700 Subject: [PATCH 18/29] Closer to compiling --- .../kafka/internal/db/rs3/RS3KVTable.java | 2 +- .../internal/db/rs3/RS3TableFactory.java | 30 +++-- .../db/rs3/client/MeteredRS3Client.java | 8 +- .../internal/db/rs3/client/RS3Client.java | 2 +- .../kafka/internal/db/rs3/client/Range.java | 4 +- .../rs3/client/grpc/GrpcKeyValueIterator.java | 2 +- .../db/rs3/client/grpc/GrpcRS3Client.java | 45 +------ .../db/rs3/client/grpc/GrpcRangeKeyCodec.java | 114 +++++++++++++++--- .../client/grpc/GrpcRangeRequestProxy.java | 2 +- .../stores/RemoteWindowOperations.java | 17 ++- .../stores/ResponsiveWindowStore.java | 4 +- 11 files changed, 145 insertions(+), 85 deletions(-) diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3KVTable.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3KVTable.java index e2d347ccf..f6e6c7e62 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3KVTable.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3KVTable.java @@ -114,7 +114,7 @@ public KeyValueIterator range( final Bytes to, final long streamTimeMs ) { - final var range = new Range(RangeBound.inclusive(from.get()), RangeBound.exclusive(to.get())); + final var range = new Range<>(RangeBound.inclusive(from), RangeBound.exclusive(to)); final List> pssIters = new ArrayList<>(); for (int pssId : pssPartitioner.pssForLss(this.lssId)) { diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3TableFactory.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3TableFactory.java index 8775933dd..04e599643 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3TableFactory.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3TableFactory.java @@ -12,6 +12,7 @@ package dev.responsive.kafka.internal.db.rs3; +import dev.responsive.kafka.api.config.ResponsiveConfig; import dev.responsive.kafka.internal.db.RemoteKVTable; import dev.responsive.kafka.internal.db.RemoteWindowTable; import dev.responsive.kafka.internal.db.rs3.client.CreateStoreTypes; @@ -51,7 +52,11 @@ public RemoteKVTable kvTable( final var rs3Client = connector.connect(); final UUID storeId = createdStores.computeIfAbsent(storeName, n -> createStore( - storeName, ttlResolver, computeNumKafkaPartitions.get(), rs3Client + storeName, + CreateStoreTypes.StoreType.WINDOW, + ttlResolver, + computeNumKafkaPartitions.get(), + rs3Client )); final PssPartitioner pssPartitioner = new PssDirectPartitioner(); @@ -66,16 +71,24 @@ public RemoteKVTable kvTable( } public RemoteWindowTable windowTable( - final String name, - final ResponsiveConfig config, + final String storeName, + final Optional> ttlResolver, final ResponsiveMetrics responsiveMetrics, - final ResponsiveMetrics.MetricScopeBuilder scopeBuilder + final ResponsiveMetrics.MetricScopeBuilder scopeBuilder, + final Supplier computeNumKafkaPartitions ) { - final var storeId = lookupStoreId(config, name); - final var pssPartitioner = new PssDirectPartitioner(); final var rs3Client = connector.connect(); + final UUID storeId = createdStores.computeIfAbsent(storeName, n -> createStore( + storeName, + CreateStoreTypes.StoreType.BASIC, + ttlResolver, + computeNumKafkaPartitions.get(), + rs3Client + )); + + final var pssPartitioner = new PssDirectPartitioner(); return new RS3WindowTable( - name, + storeName, storeId, rs3Client, pssPartitioner, @@ -86,6 +99,7 @@ public RemoteWindowTable windowTable( public static UUID createStore( final String storeName, + final CreateStoreTypes.StoreType storeType, final Optional> ttlResolver, final int numKafkaPartitions, final RS3Client rs3Client @@ -98,7 +112,7 @@ public static UUID createStore( final var options = new CreateStoreOptions( numKafkaPartitions, - CreateStoreTypes.StoreType.BASIC, + storeType, ttlResolver.isPresent() ? Optional.of(ClockType.WALL_CLOCK) : Optional.empty(), defaultTtl, Optional.empty() diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/MeteredRS3Client.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/MeteredRS3Client.java index a6cbba85c..f5c1bd006 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/MeteredRS3Client.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/MeteredRS3Client.java @@ -107,7 +107,7 @@ public KeyValueIterator range( final LssId lssId, final int pssId, final Optional expectedWrittenOffset, - final Range range + final Range range ) { return delegate.range( storeId, @@ -144,16 +144,14 @@ public KeyValueIterator windowedRange( final LssId lssId, final int pssId, final Optional expectedWrittenOffset, - final RangeBound from, - final RangeBound to + final Range range ) { return delegate.windowedRange( storeId, lssId, pssId, expectedWrittenOffset, - from, - to + range ); } diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/RS3Client.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/RS3Client.java index 5400a3977..96abbadb1 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/RS3Client.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/RS3Client.java @@ -62,7 +62,7 @@ KeyValueIterator range( LssId lssId, int pssId, Optional expectedWrittenOffset, - Range range + Range range ); KeyValueIterator windowedRange( diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/Range.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/Range.java index c2610d142..ad3894253 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/Range.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/Range.java @@ -74,7 +74,7 @@ public Boolean map(final RangeBound.Unbounded b) { } public static > Range unbounded() { - return new Range<>(RangeBound.unbounded(), RangeBound.unbounded()); + return new Range(RangeBound.unbounded(), RangeBound.unbounded()); } @Override @@ -85,7 +85,7 @@ public boolean equals(final Object o) { if (o == null || getClass() != o.getClass()) { return false; } - final Range range = (Range) o; + final Range range = (Range) o; return Objects.equals(start, range.start) && Objects.equals(end, range.end); } diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcKeyValueIterator.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcKeyValueIterator.java index de1bada4a..b151403bb 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcKeyValueIterator.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcKeyValueIterator.java @@ -138,7 +138,7 @@ public Optional> map(final StreamError error) { @Override public Optional> map(final Result result) { - final var key = keyCodec.decodeRangeResult(result.keyValue); + final var key = keyCodec.decodeKeyValue(result.keyValue); final var value = result.keyValue.getValue().toByteArray(); return Optional.of(new KeyValue<>(key, value)); } diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRS3Client.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRS3Client.java index c2b21ab25..39498b2c6 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRS3Client.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRS3Client.java @@ -12,27 +12,22 @@ package dev.responsive.kafka.internal.db.rs3.client.grpc; -import static dev.responsive.kafka.internal.db.rs3.client.grpc.GrpcRs3Util.basicDeleteProto; import static dev.responsive.kafka.internal.db.rs3.client.grpc.GrpcRs3Util.basicKeyProto; -import static dev.responsive.kafka.internal.db.rs3.client.grpc.GrpcRs3Util.basicPutProto; import static dev.responsive.kafka.internal.db.rs3.client.grpc.GrpcRs3Util.windowKeyProto; import static dev.responsive.kafka.internal.utils.Utils.lssIdProto; import static dev.responsive.kafka.internal.utils.Utils.uuidFromProto; import static dev.responsive.kafka.internal.utils.Utils.uuidToProto; import com.google.common.annotations.VisibleForTesting; -import com.google.protobuf.ByteString; import dev.responsive.kafka.api.config.ResponsiveConfig; import dev.responsive.kafka.internal.db.rs3.client.CreateStoreTypes.CreateStoreOptions; import dev.responsive.kafka.internal.db.rs3.client.CreateStoreTypes.CreateStoreResult; import dev.responsive.kafka.internal.db.rs3.client.CurrentOffsets; -import dev.responsive.kafka.internal.db.rs3.client.Delete; import dev.responsive.kafka.internal.db.rs3.client.LssId; import dev.responsive.kafka.internal.db.rs3.client.RS3Client; import dev.responsive.kafka.internal.db.rs3.client.RS3TimeoutException; import dev.responsive.kafka.internal.db.rs3.client.RS3TransientException; import dev.responsive.kafka.internal.db.rs3.client.Range; -import dev.responsive.kafka.internal.db.rs3.client.RangeBound; import dev.responsive.kafka.internal.db.rs3.client.Store; import dev.responsive.kafka.internal.db.rs3.client.StreamSenderMessageReceiver; import dev.responsive.kafka.internal.db.rs3.client.WalEntry; @@ -232,7 +227,7 @@ public KeyValueIterator range( final LssId lssId, final int pssId, final Optional expectedWrittenOffset, - Range range + Range range ) { return sendRange( storeId, @@ -408,42 +403,6 @@ private void checkField(final Supplier check, final String field) { } } - private Rs3.Range protoRange(Range range) { - final var protoRange = Rs3.BasicRange.newBuilder() - .setFrom(protoBound(range.start())) - .setTo(protoBound(range.end())); - return Rs3.Range.newBuilder() - .setBasicRange(protoRange) - .build(); - } - - private Rs3.BasicBound protoBound(RangeBound bound) { - return bound.map(new RangeBound.Mapper<>() { - @Override - public Rs3.BasicBound map(final RangeBound.InclusiveBound b) { - return Rs3.BasicBound.newBuilder() - .setType(Rs3.BoundType.INCLUSIVE) - .setKey(basicKeyProto(b.key())) - .build(); - } - - @Override - public Rs3.BasicBound map(final RangeBound.ExclusiveBound b) { - return Rs3.BasicBound.newBuilder() - .setType(Rs3.BoundType.EXCLUSIVE) - .setKey(basicKeyProto(b.key())) - .build(); - } - - @Override - public Rs3.BasicBound map(final RangeBound.Unbounded b) { - return Rs3.BasicBound.newBuilder() - .setType(Rs3.BoundType.UNBOUNDED) - .build(); - } - }); - } - private class RangeProxy> implements GrpcRangeRequestProxy { private final Rs3.RangeRequest.Builder requestBuilder; private final RS3Grpc.RS3Stub stub; @@ -469,7 +428,7 @@ public void send( final Range range, final StreamObserver resultObserver ) { - requestBuilder.setRange(protoRange(range)); + requestBuilder.setRange(keyCodec.encodeRange(range)); while (true) { try { diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRangeKeyCodec.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRangeKeyCodec.java index 0c211068a..ddb4457c3 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRangeKeyCodec.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRangeKeyCodec.java @@ -13,42 +13,124 @@ package dev.responsive.kafka.internal.db.rs3.client.grpc; import com.google.protobuf.ByteString; +import dev.responsive.kafka.internal.db.rs3.client.Range; +import dev.responsive.kafka.internal.db.rs3.client.RangeBound; import dev.responsive.kafka.internal.utils.WindowedKey; import dev.responsive.rs3.Rs3; import org.apache.kafka.common.utils.Bytes; +import org.apache.kafka.streams.KeyValue; -public interface GrpcRangeKeyCodec { +public interface GrpcRangeKeyCodec> { WindowedKeyCodec WINDOW_CODEC = new WindowedKeyCodec(); - StandardKeyCodec STANDARD_CODEC = new StandardKeyCodec(); + BasicKeyCodec STANDARD_CODEC = new BasicKeyCodec(); - K decodeRangeResult(Rs3.KeyValue keyValue); + KeyValue decodeKeyValue(Rs3.KeyValue keyValue); - void encodeRangeBound(K key, Rs3.Bound.Builder builder); + Rs3.Range encodeRange(Range range); class WindowedKeyCodec implements GrpcRangeKeyCodec { @Override - public WindowedKey decodeRangeResult(final Rs3.KeyValue keyValue) { - final var key = Bytes.wrap(keyValue.getKey().toByteArray()); - final var windowTimestamp = keyValue.getWindowTimestamp(); - return new WindowedKey(key, windowTimestamp); + public KeyValue decodeKeyValue(final Rs3.KeyValue keyValue) { + final var kvProto = keyValue.getWindowKv(); + final var keyProto = kvProto.getKey(); + + final var key = new WindowedKey( + Bytes.wrap(keyProto.getKey().toByteArray()), + keyProto.getWindowTimestamp() + ); + final var value = kvProto.getValue().toByteArray(); + return new KeyValue<>(key, value); } @Override - public void encodeRangeBound(final WindowedKey key, final Rs3.Bound.Builder builder) { - builder.setKey(ByteString.copyFrom(key.key.get())); - builder.setWindowTimestamp(key.windowStartMs); + public Rs3.Range encodeRange(final Range range) { + final var rangeProto = Rs3.WindowRange.newBuilder(); + rangeProto.setFrom(boundProto(range.start())); + rangeProto.setTo(boundProto(range.end())); + return Rs3.Range.newBuilder() + .setWindowRange(rangeProto) + .build(); + } + + private Rs3.WindowBound boundProto(RangeBound bound) { + final var boundProto = Rs3.WindowBound.newBuilder(); + return bound.map(new RangeBound.Mapper<>() { + @Override + public Rs3.WindowBound map(final RangeBound.InclusiveBound b) { + final var key = Rs3.WindowKey.newBuilder() + .setWindowTimestamp(b.key().windowStartMs) + .setKey(ByteString.copyFrom(b.key().key.get())); + return boundProto.setType(Rs3.BoundType.INCLUSIVE) + .setKey(key) + .build(); + } + + @Override + public Rs3.WindowBound map(final RangeBound.ExclusiveBound b) { + final var key = Rs3.WindowKey.newBuilder() + .setWindowTimestamp(b.key().windowStartMs) + .setKey(ByteString.copyFrom(b.key().key.get())); + return boundProto.setType(Rs3.BoundType.EXCLUSIVE) + .setKey(key) + .build(); + } + + @Override + public Rs3.WindowBound map(final RangeBound.Unbounded b) { + return boundProto.setType(Rs3.BoundType.UNBOUNDED) + .build(); + } + }); } } - class StandardKeyCodec implements GrpcRangeKeyCodec { + class BasicKeyCodec implements GrpcRangeKeyCodec { @Override - public Bytes decodeRangeResult(final Rs3.KeyValue keyValue) { - return Bytes.wrap(keyValue.getKey().toByteArray()); + public KeyValue decodeKeyValue(final Rs3.KeyValue keyValue) { + final var kvProto = keyValue.getBasicKv(); + final var key = Bytes.wrap(kvProto.getKey().getKey().toByteArray()); + final var value = kvProto.getValue().toByteArray(); + return new KeyValue<>(key, value); } @Override - public void encodeRangeBound(final Bytes key, final Rs3.Bound.Builder builder) { - builder.setKey(ByteString.copyFrom(key.get())); + public Rs3.Range encodeRange(final Range range) { + final var rangeProto = Rs3.BasicRange.newBuilder(); + rangeProto.setFrom(boundProto(range.start())); + rangeProto.setTo(boundProto(range.end())); + return Rs3.Range.newBuilder() + .setBasicRange(rangeProto) + .build(); + } + + private Rs3.BasicBound boundProto(RangeBound bound) { + final var boundProto = Rs3.BasicBound.newBuilder(); + return bound.map(new RangeBound.Mapper<>() { + @Override + public Rs3.BasicBound map(final RangeBound.InclusiveBound b) { + final var key = Rs3.BasicKey.newBuilder() + .setKey(ByteString.copyFrom(b.key().get())); + return boundProto.setType(Rs3.BoundType.INCLUSIVE) + .setKey(key) + .build(); + } + + @Override + public Rs3.BasicBound map(final RangeBound.ExclusiveBound b) { + final var key = Rs3.BasicKey.newBuilder() + .setKey(ByteString.copyFrom(b.key().get())); + return boundProto.setType(Rs3.BoundType.INCLUSIVE) + .setKey(key) + .build(); + } + + @Override + public Rs3.BasicBound map(final RangeBound.Unbounded b) { + return boundProto.setType(Rs3.BoundType.UNBOUNDED) + .build(); + } + }); } } + } diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRangeRequestProxy.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRangeRequestProxy.java index 9ee9ccb82..e7ef3f66f 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRangeRequestProxy.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRangeRequestProxy.java @@ -24,7 +24,7 @@ * updated. If the observer encounters an error, then it can retry with the updated * start bound. */ -public interface GrpcRangeRequestProxy { +public interface GrpcRangeRequestProxy> { /** * Send a range request with an updated start bound. The results will be passed * through to result observer. If a transient error is encountered through the diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/stores/RemoteWindowOperations.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/stores/RemoteWindowOperations.java index a17722f8a..02d30b0ad 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/stores/RemoteWindowOperations.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/stores/RemoteWindowOperations.java @@ -17,6 +17,7 @@ import static dev.responsive.kafka.internal.config.InternalSessionConfigs.loadSessionClients; import static dev.responsive.kafka.internal.config.InternalSessionConfigs.loadStoreRegistry; import static dev.responsive.kafka.internal.stores.ResponsiveStoreRegistration.NO_COMMITTED_OFFSET; +import static dev.responsive.kafka.internal.utils.StoreUtil.numPartitionsForKafkaTopic; import static dev.responsive.kafka.internal.utils.StoreUtil.streamThreadId; import static org.apache.kafka.streams.processor.internals.ProcessorContextUtils.asInternalProcessorContext; import static org.apache.kafka.streams.processor.internals.ProcessorContextUtils.changelogFor; @@ -42,6 +43,7 @@ import dev.responsive.kafka.internal.utils.WindowedKey; import java.util.Collection; import java.util.Map; +import java.util.Optional; import java.util.OptionalLong; import java.util.concurrent.TimeoutException; import java.util.function.Predicate; @@ -79,7 +81,8 @@ public static RemoteWindowOperations create( final ResponsiveWindowParams params, final Map appConfigs, final ResponsiveConfig responsiveConfig, - final Predicate withinRetention + final Predicate withinRetention, + final Optional> ttlResolver ) throws InterruptedException, TimeoutException { final var log = new LogContext( @@ -120,7 +123,8 @@ public static RemoteWindowOperations create( table = createRs3( params, sessionClients, - responsiveConfig, + changelog.topic(), + ttlResolver, responsiveMetrics, scopeBuilder ); @@ -235,7 +239,8 @@ private static RemoteWindowTable createMongo( private static RemoteWindowTable createRs3( final ResponsiveWindowParams params, final SessionClients sessionClients, - final ResponsiveConfig config, + final String changelogTopicName, + final Optional> ttlResolver, final ResponsiveMetrics responsiveMetrics, final ResponsiveMetrics.MetricScopeBuilder scopeBuilder ) { @@ -243,12 +248,12 @@ private static RemoteWindowTable createRs3( throw new UnsupportedOperationException("Duplicate retention is not yet supported in RS3"); } - // TODO: Pass through retention period once we have support for TTL return sessionClients.rs3TableFactory().windowTable( params.name().tableName(), - config, + ttlResolver, responsiveMetrics, - scopeBuilder + scopeBuilder, + () -> numPartitionsForKafkaTopic(sessionClients.admin(), changelogTopicName) ); } diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/stores/ResponsiveWindowStore.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/stores/ResponsiveWindowStore.java index 2f0883e92..335ac93d2 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/stores/ResponsiveWindowStore.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/stores/ResponsiveWindowStore.java @@ -25,6 +25,7 @@ import dev.responsive.kafka.internal.utils.Iterators; import dev.responsive.kafka.internal.utils.TableName; import java.util.Collection; +import java.util.Optional; import java.util.concurrent.TimeoutException; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.common.utils.Bytes; @@ -102,7 +103,8 @@ public void init(final StateStoreContext storeContext, final StateStore root) { params, appConfigs, config, - window -> window.windowStartMs >= minValidTimestamp() + window -> window.windowStartMs >= minValidTimestamp(), + Optional.empty() // FIXME: Provide TTLResolver ); numBloomFilterWindows = params.retainDuplicates() From a9d7615fc2b14fca22431432adf03eba54ae5798 Mon Sep 17 00:00:00 2001 From: Jason Gustafson <12502538+hachikuji@users.noreply.github.com> Date: Thu, 24 Apr 2025 15:44:00 -0700 Subject: [PATCH 19/29] compiling again --- .../rs3/client/grpc/GrpcKeyValueIterator.java | 4 +- .../db/rs3/client/grpc/GrpcRS3Client.java | 33 ++++++------ .../db/rs3/client/grpc/GrpcRs3Util.java | 19 +++++-- .../db/rs3/client/grpc/WalEntryPutWriter.java | 43 ++++++++++++---- .../grpc/GrpcRS3ClientEndToEndTest.java | 6 +-- .../db/rs3/client/grpc/GrpcRS3ClientTest.java | 50 +++++++++---------- 6 files changed, 90 insertions(+), 65 deletions(-) diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcKeyValueIterator.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcKeyValueIterator.java index b151403bb..d7de8c706 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcKeyValueIterator.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcKeyValueIterator.java @@ -138,9 +138,7 @@ public Optional> map(final StreamError error) { @Override public Optional> map(final Result result) { - final var key = keyCodec.decodeKeyValue(result.keyValue); - final var value = result.keyValue.getValue().toByteArray(); - return Optional.of(new KeyValue<>(key, value)); + return Optional.of(keyCodec.decodeKeyValue(result.keyValue)); } }); } diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRS3Client.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRS3Client.java index 39498b2c6..b49cc3961 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRS3Client.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRS3Client.java @@ -156,7 +156,7 @@ public StreamSenderMessageReceiver> writeWalSegmentAsyn .setPssId(pssId) .setEndOffset(endOffset) .setExpectedWrittenOffset(expectedWrittenOffset.orElse(WAL_OFFSET_NONE)); - addWalEntryToSegment(entry, entryBuilder); + entry.visit(new WalEntryPutWriter(entryBuilder)); return entryBuilder.build(); }, streamObserver @@ -295,13 +295,18 @@ public Optional get( .setBasicKey(basicKeyProto(key.get())); final var requestBuilder = Rs3.GetRequest.newBuilder() .setKey(keyProto); - return sendGet( + final var kvOpt = sendGet( requestBuilder, storeId, lssId, pssId, expectedWrittenOffset ); + return kvOpt.map(kv -> { + checkField(kv::hasBasicKv, "value"); + final var value = kv.getBasicKv().getValue().getValue(); + return value.toByteArray(); + }); } @Override @@ -316,16 +321,21 @@ public Optional windowedGet( .setWindowKey(windowKeyProto(key)); final var requestBuilder = Rs3.GetRequest.newBuilder() .setKey(keyProto); - return sendGet( + final var kvOpt = sendGet( requestBuilder, storeId, lssId, pssId, expectedWrittenOffset ); + return kvOpt.map(kv -> { + checkField(kv::hasWindowKv, "value"); + final var value = kv.getWindowKv().getValue().getValue(); + return value.toByteArray(); + }); } - private Optional sendGet( + private Optional sendGet( final Rs3.GetRequest.Builder requestBuilder, final UUID storeId, final LssId lssId, @@ -346,10 +356,7 @@ private Optional sendGet( if (!result.hasResult()) { return Optional.empty(); } - final Rs3.KeyValue keyValue = result.getResult(); - checkField(keyValue::hasBasicKv, "value"); - final var value = keyValue.getBasicKv().getValue().getValue(); - return Optional.of(value.toByteArray()); + return Optional.of(result.getResult()); } @Override @@ -387,16 +394,6 @@ public CreateStoreResult createStore( return new CreateStoreResult(uuidFromProto(result.getStoreId()), result.getPssIdsList()); } - private void addWalEntryToSegment( - final WalEntry entry, - final Rs3.WriteWALSegmentRequest.Builder builder - ) { - final var putBuilder = Rs3.WriteWALSegmentRequest.Put.newBuilder(); - final var writer = new WalEntryPutWriter(putBuilder); - entry.visit(writer); - builder.setPut(putBuilder); - } - private void checkField(final Supplier check, final String field) { if (!check.get()) { throw new RuntimeException("rs3 resp proto missing field " + field); diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRs3Util.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRs3Util.java index af72c1252..1eed3b6bf 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRs3Util.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRs3Util.java @@ -80,14 +80,25 @@ public static Rs3.BasicKeyValue basicKeyValueProto( final byte[] key, final byte[] value ) { - final var keyBldr = Rs3.BasicKey.newBuilder(); - keyBldr.setKey(ByteString.copyFrom(key)); - + final var keyProto = basicKeyProto(key); final var valueBldr = Rs3.BasicValue.newBuilder(); valueBldr.setValue(ByteString.copyFrom(value)); return Rs3.BasicKeyValue.newBuilder() - .setKey(keyBldr) + .setKey(keyProto) + .setValue(valueBldr) + .build(); + } + + public static Rs3.WindowKeyValue windowKeyValueProto( + final WindowedKey key, + final byte[] value + ) { + final var keyProto = windowKeyProto(key); + final var valueBldr = Rs3.WindowValue.newBuilder(); + valueBldr.setValue(ByteString.copyFrom(value)); + return Rs3.WindowKeyValue.newBuilder() + .setKey(keyProto) .setValue(valueBldr) .build(); } diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/WalEntryPutWriter.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/WalEntryPutWriter.java index 082e76ee0..6bf05c277 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/WalEntryPutWriter.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/WalEntryPutWriter.java @@ -9,34 +9,55 @@ import dev.responsive.rs3.Rs3; public class WalEntryPutWriter implements WalEntry.Visitor { - private final Rs3.WriteWALSegmentRequest.Put.Builder builder; + private final Rs3.WriteWALSegmentRequest.Builder builder; - public WalEntryPutWriter(final Rs3.WriteWALSegmentRequest.Put.Builder builder) { + public WalEntryPutWriter(final Rs3.WriteWALSegmentRequest.Builder builder) { this.builder = builder; } @Override public void visit(final Put put) { - builder.setKey(ByteString.copyFrom(put.key())); - builder.setValue(ByteString.copyFrom(put.value())); - builder.setTtl(Rs3.Ttl.newBuilder().setTtlType(Rs3.Ttl.TtlType.DEFAULT).build()); + final var kvProto = Rs3.BasicKeyValue.newBuilder() + .setKey(Rs3.BasicKey.newBuilder().setKey(ByteString.copyFrom(put.key()))) + .setValue(Rs3.BasicValue.newBuilder().setValue(ByteString.copyFrom(put.value()))); + final var putProto = Rs3.WriteWALSegmentRequest.Put.newBuilder() + .setKv(Rs3.KeyValue.newBuilder().setBasicKv(kvProto)) + .setTtl(Rs3.Ttl.newBuilder().setTtlType(Rs3.Ttl.TtlType.DEFAULT)); + builder.setPut(putProto); } @Override public void visit(final Delete delete) { - builder.setKey(ByteString.copyFrom(delete.key())); - builder.setTtl(Rs3.Ttl.newBuilder().setTtlType(Rs3.Ttl.TtlType.DEFAULT).build()); + final var keyProto = Rs3.BasicKey.newBuilder() + .setKey(ByteString.copyFrom(delete.key())); + final var deleteProto = Rs3.WriteWALSegmentRequest.Delete.newBuilder() + .setKey(Rs3.Key.newBuilder().setBasicKey(keyProto)); + builder.setDelete(deleteProto); } @Override public void visit(final WindowedDelete windowedDelete) { - visit((Delete) windowedDelete); - builder.setWindowTimestamp(windowedDelete.windowTimestamp()); + final var keyProto = Rs3.WindowKey.newBuilder() + .setKey(ByteString.copyFrom(windowedDelete.key())) + .setWindowTimestamp(windowedDelete.windowTimestamp()); + final var deleteProto = Rs3.WriteWALSegmentRequest.Delete.newBuilder() + .setKey(Rs3.Key.newBuilder().setWindowKey(keyProto)); + builder.setDelete(deleteProto); } @Override public void visit(final WindowedPut windowedPut) { - visit((Put) windowedPut); - builder.setWindowTimestamp(windowedPut.windowTimestamp()); + final var keyProto = Rs3.WindowKey.newBuilder() + .setKey(ByteString.copyFrom(windowedPut.key())) + .setWindowTimestamp(windowedPut.windowTimestamp()); + final var valueProto = Rs3.WindowValue.newBuilder() + .setValue(ByteString.copyFrom(windowedPut.value())); + final var kvProto = Rs3.WindowKeyValue.newBuilder() + .setKey(keyProto) + .setValue(valueProto); + final var putProto = Rs3.WriteWALSegmentRequest.Put.newBuilder() + .setKv(Rs3.KeyValue.newBuilder().setWindowKv(kvProto)) + .setTtl(Rs3.Ttl.newBuilder().setTtlType(Rs3.Ttl.TtlType.DEFAULT)); + builder.setPut(putProto); } } diff --git a/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRS3ClientEndToEndTest.java b/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRS3ClientEndToEndTest.java index cbe20ec39..2f1b14293 100644 --- a/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRS3ClientEndToEndTest.java +++ b/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRS3ClientEndToEndTest.java @@ -121,7 +121,7 @@ public void shouldScanAllKeyValues() { LSS_ID, PSS_ID, Optional.of(5L), - new Range( + new Range( RangeBound.unbounded(), RangeBound.unbounded() ) @@ -189,7 +189,7 @@ public void shouldRetryRangeWithNetworkInterruption() { Mockito.doAnswer(invocation -> { @SuppressWarnings("unchecked") - final var call = (ClientCall) + final var call = (ClientCall) invocation.callRealMethod(); final var callSpy = Mockito.spy(call); Mockito.doThrow(new StatusRuntimeException(Status.UNAVAILABLE)) @@ -205,7 +205,7 @@ public void shouldRetryRangeWithNetworkInterruption() { LSS_ID, PSS_ID, Optional.of(5L), - new Range( + new Range( RangeBound.unbounded(), RangeBound.unbounded() ) diff --git a/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRS3ClientTest.java b/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRS3ClientTest.java index f5fd0d746..f1afc31a8 100644 --- a/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRS3ClientTest.java +++ b/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRS3ClientTest.java @@ -33,6 +33,7 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import com.google.protobuf.ByteString; import dev.responsive.kafka.internal.db.rs3.client.CreateStoreTypes; import dev.responsive.kafka.internal.db.rs3.client.CreateStoreTypes.ClockType; import dev.responsive.kafka.internal.db.rs3.client.CreateStoreTypes.CreateStoreOptions; @@ -796,13 +797,15 @@ public void shouldWindowedGet() { final var key = "foo".getBytes(); // given: - when(stub.get(any())) - .thenReturn(Rs3.GetResult.newBuilder().setResult( - Rs3.KeyValue.newBuilder() - .setKey(ByteString.copyFromUtf8("foo")) - .setValue(ByteString.copyFromUtf8("bar")) - .setWindowTimestamp(windowTimestamp) - ).build()); + + final var kvProto = GrpcRs3Util.windowKeyValueProto( + new WindowedKey("foo".getBytes(StandardCharsets.UTF_8), windowTimestamp), + "bar".getBytes(StandardCharsets.UTF_8) + ); + when(stub.get(any())).thenReturn( + Rs3.GetResult.newBuilder() + .setResult(Rs3.KeyValue.newBuilder().setWindowKv(kvProto)) + .build()); // when: final var result = client.windowedGet( @@ -815,12 +818,14 @@ public void shouldWindowedGet() { // then: assertThat(result.get(), is("bar".getBytes())); + final var keyProto = GrpcRs3Util.windowKeyProto( + new WindowedKey("foo".getBytes(StandardCharsets.UTF_8), windowTimestamp) + ); verify(stub).get(Rs3.GetRequest.newBuilder() .setLssId(lssIdProto(LSS_ID)) .setPssId(PSS_ID) - .setStoreId(uuidToUuidProto(STORE_ID)) - .setKey(ByteString.copyFromUtf8("foo")) - .setWindowTimestamp(windowTimestamp) + .setStoreId(uuidToProto(STORE_ID)) + .setKey(Rs3.Key.newBuilder().setWindowKey(keyProto)) .build() ); } @@ -835,9 +840,10 @@ public void shouldRetryWindowedGet() { .thenThrow(new StatusRuntimeException(Status.UNAVAILABLE)) .thenReturn(Rs3.GetResult.newBuilder().setResult( Rs3.KeyValue.newBuilder() - .setKey(ByteString.copyFromUtf8("foo")) - .setValue(ByteString.copyFromUtf8("bar")) - .setWindowTimestamp(windowTimestamp) + .setWindowKv(GrpcRs3Util.windowKeyValueProto( + new WindowedKey("foo".getBytes(StandardCharsets.UTF_8), windowTimestamp), + "bar".getBytes(StandardCharsets.UTF_8) + )) ).build()); // when: @@ -851,13 +857,15 @@ public void shouldRetryWindowedGet() { // then: assertThat(result.get(), is("bar".getBytes())); + final var keyProto = GrpcRs3Util.windowKeyProto( + new WindowedKey("foo".getBytes(StandardCharsets.UTF_8), windowTimestamp) + ); verify(stub, times(2)) .get(Rs3.GetRequest.newBuilder() .setLssId(lssIdProto(LSS_ID)) .setPssId(PSS_ID) - .setStoreId(uuidToUuidProto(STORE_ID)) - .setKey(ByteString.copyFromUtf8("foo")) - .setWindowTimestamp(windowTimestamp) + .setStoreId(uuidToProto(STORE_ID)) + .setKey(Rs3.Key.newBuilder().setWindowKey(keyProto)) .build() ); } @@ -1227,16 +1235,6 @@ private StreamObserver verifyWalSegmentResultObserver return writeWALSegmentResultObserverCaptor.getValue(); } - private Rs3.WriteWALSegmentRequest.Put putProto(final Put put) { - final var builder = Rs3.WriteWALSegmentRequest.Put.newBuilder() - .setKey(ByteString.copyFrom(put.key())); - builder.setValue(ByteString.copyFrom(put.value())); - builder.setTtl(Rs3.Ttl.newBuilder() - .setTtlType(Rs3.Ttl.TtlType.DEFAULT) - .build()); - return builder.build(); - } - public static class TestException extends RuntimeException { private static final long serialVersionUID = 0L; } From 173b798d36d82ab0640a6c5127141d6496553f57 Mon Sep 17 00:00:00 2001 From: Jason Gustafson <12502538+hachikuji@users.noreply.github.com> Date: Thu, 24 Apr 2025 15:48:02 -0700 Subject: [PATCH 20/29] Fix checkstyle --- .../dev/responsive/kafka/internal/db/rs3/RS3TableFactory.java | 1 - .../kafka/internal/db/rs3/client/grpc/GrpcRS3ClientTest.java | 1 - 2 files changed, 2 deletions(-) diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3TableFactory.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3TableFactory.java index 04e599643..b5c8d3ba2 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3TableFactory.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3TableFactory.java @@ -12,7 +12,6 @@ package dev.responsive.kafka.internal.db.rs3; -import dev.responsive.kafka.api.config.ResponsiveConfig; import dev.responsive.kafka.internal.db.RemoteKVTable; import dev.responsive.kafka.internal.db.RemoteWindowTable; import dev.responsive.kafka.internal.db.rs3.client.CreateStoreTypes; diff --git a/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRS3ClientTest.java b/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRS3ClientTest.java index f1afc31a8..aa3f3d868 100644 --- a/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRS3ClientTest.java +++ b/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRS3ClientTest.java @@ -33,7 +33,6 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import com.google.protobuf.ByteString; import dev.responsive.kafka.internal.db.rs3.client.CreateStoreTypes; import dev.responsive.kafka.internal.db.rs3.client.CreateStoreTypes.ClockType; import dev.responsive.kafka.internal.db.rs3.client.CreateStoreTypes.CreateStoreOptions; From cfa3e71ac74c645796ce408bd86d4088d96db609 Mon Sep 17 00:00:00 2001 From: Jason Gustafson <12502538+hachikuji@users.noreply.github.com> Date: Fri, 25 Apr 2025 09:42:54 -0700 Subject: [PATCH 21/29] Pass through default ttl/use stream time clock --- .../ResponsiveWindowBytesStoreSupplier.java | 4 +-- .../api/stores/ResponsiveWindowParams.java | 2 +- .../internal/db/rs3/RS3TableFactory.java | 29 +++++++++++-------- .../stores/RemoteWindowOperations.java | 14 ++++----- .../stores/ResponsiveWindowStore.java | 6 ++-- 5 files changed, 28 insertions(+), 27 deletions(-) diff --git a/kafka-client/src/main/java/dev/responsive/kafka/api/stores/ResponsiveWindowBytesStoreSupplier.java b/kafka-client/src/main/java/dev/responsive/kafka/api/stores/ResponsiveWindowBytesStoreSupplier.java index 0335092fc..fdfa924c4 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/api/stores/ResponsiveWindowBytesStoreSupplier.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/api/stores/ResponsiveWindowBytesStoreSupplier.java @@ -27,7 +27,7 @@ public class ResponsiveWindowBytesStoreSupplier implements WindowBytesStoreSuppl public ResponsiveWindowBytesStoreSupplier(final ResponsiveWindowParams params) { this.params = params; - this.segmentIntervalMs = computeSegmentInterval(params.retentionPeriod(), params.numSegments()); + this.segmentIntervalMs = computeSegmentInterval(params.retentionPeriodMs(), params.numSegments()); } @Override @@ -57,7 +57,7 @@ public boolean retainDuplicates() { @Override public long retentionPeriod() { - return params.retentionPeriod(); + return params.retentionPeriodMs(); } @Override diff --git a/kafka-client/src/main/java/dev/responsive/kafka/api/stores/ResponsiveWindowParams.java b/kafka-client/src/main/java/dev/responsive/kafka/api/stores/ResponsiveWindowParams.java index 825df67c5..d009ba3ca 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/api/stores/ResponsiveWindowParams.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/api/stores/ResponsiveWindowParams.java @@ -106,7 +106,7 @@ public long windowSize() { return windowSizeMs; } - public long retentionPeriod() { + public long retentionPeriodMs() { return retentionPeriodMs; } diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3TableFactory.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3TableFactory.java index b5c8d3ba2..15abcf487 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3TableFactory.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3TableFactory.java @@ -22,6 +22,7 @@ import dev.responsive.kafka.internal.db.rs3.client.grpc.GrpcRS3Client; import dev.responsive.kafka.internal.metrics.ResponsiveMetrics; import dev.responsive.kafka.internal.stores.TtlResolver; +import java.time.Duration; import java.util.Map; import java.util.Optional; import java.util.UUID; @@ -48,12 +49,20 @@ public RemoteKVTable kvTable( final ResponsiveMetrics.MetricScopeBuilder scopeBuilder, final Supplier computeNumKafkaPartitions ) { + final Optional defaultTtl = ttlResolver.isPresent() && ttlResolver.get().defaultTtl().isFinite() + ? Optional.of(ttlResolver.get().defaultTtl().duration()) + : Optional.empty(); + final Optional clockType = ttlResolver.isPresent() + ? Optional.of(ClockType.WALL_CLOCK) + : Optional.empty(); + final var rs3Client = connector.connect(); final UUID storeId = createdStores.computeIfAbsent(storeName, n -> createStore( storeName, CreateStoreTypes.StoreType.WINDOW, - ttlResolver, + clockType, + defaultTtl, computeNumKafkaPartitions.get(), rs3Client )); @@ -71,7 +80,7 @@ public RemoteKVTable kvTable( public RemoteWindowTable windowTable( final String storeName, - final Optional> ttlResolver, + final Duration defaultTtl, final ResponsiveMetrics responsiveMetrics, final ResponsiveMetrics.MetricScopeBuilder scopeBuilder, final Supplier computeNumKafkaPartitions @@ -80,7 +89,8 @@ public RemoteWindowTable windowTable( final UUID storeId = createdStores.computeIfAbsent(storeName, n -> createStore( storeName, CreateStoreTypes.StoreType.BASIC, - ttlResolver, + Optional.of(ClockType.STREAM_TIME), + Optional.of(defaultTtl), computeNumKafkaPartitions.get(), rs3Client )); @@ -99,21 +109,16 @@ public RemoteWindowTable windowTable( public static UUID createStore( final String storeName, final CreateStoreTypes.StoreType storeType, - final Optional> ttlResolver, + final Optional clockType, + final Optional defaultTtl, final int numKafkaPartitions, final RS3Client rs3Client ) { - - final Optional defaultTtl = - ttlResolver.isPresent() && ttlResolver.get().defaultTtl().isFinite() - ? Optional.of(ttlResolver.get().defaultTtl().duration().toMillis()) - : Optional.empty(); - final var options = new CreateStoreOptions( numKafkaPartitions, storeType, - ttlResolver.isPresent() ? Optional.of(ClockType.WALL_CLOCK) : Optional.empty(), - defaultTtl, + clockType, + defaultTtl.map(Duration::toMillis), Optional.empty() ); diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/stores/RemoteWindowOperations.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/stores/RemoteWindowOperations.java index 02d30b0ad..54cc40110 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/stores/RemoteWindowOperations.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/stores/RemoteWindowOperations.java @@ -41,9 +41,9 @@ import dev.responsive.kafka.internal.utils.TableName; import dev.responsive.kafka.internal.utils.Utils; import dev.responsive.kafka.internal.utils.WindowedKey; +import java.time.Duration; import java.util.Collection; import java.util.Map; -import java.util.Optional; import java.util.OptionalLong; import java.util.concurrent.TimeoutException; import java.util.function.Predicate; @@ -81,8 +81,7 @@ public static RemoteWindowOperations create( final ResponsiveWindowParams params, final Map appConfigs, final ResponsiveConfig responsiveConfig, - final Predicate withinRetention, - final Optional> ttlResolver + final Predicate withinRetention ) throws InterruptedException, TimeoutException { final var log = new LogContext( @@ -99,8 +98,8 @@ public static RemoteWindowOperations create( ); final WindowSegmentPartitioner partitioner = new WindowSegmentPartitioner( - params.retentionPeriod(), - StoreUtil.computeSegmentInterval(params.retentionPeriod(), params.numSegments()), + params.retentionPeriodMs(), + StoreUtil.computeSegmentInterval(params.retentionPeriodMs(), params.numSegments()), params.retainDuplicates() ); @@ -124,7 +123,6 @@ public static RemoteWindowOperations create( params, sessionClients, changelog.topic(), - ttlResolver, responsiveMetrics, scopeBuilder ); @@ -240,7 +238,6 @@ private static RemoteWindowTable createRs3( final ResponsiveWindowParams params, final SessionClients sessionClients, final String changelogTopicName, - final Optional> ttlResolver, final ResponsiveMetrics responsiveMetrics, final ResponsiveMetrics.MetricScopeBuilder scopeBuilder ) { @@ -248,9 +245,10 @@ private static RemoteWindowTable createRs3( throw new UnsupportedOperationException("Duplicate retention is not yet supported in RS3"); } + final var defaultTtl = Duration.ofMillis(params.retentionPeriodMs()); return sessionClients.rs3TableFactory().windowTable( params.name().tableName(), - ttlResolver, + defaultTtl, responsiveMetrics, scopeBuilder, () -> numPartitionsForKafkaTopic(sessionClients.admin(), changelogTopicName) diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/stores/ResponsiveWindowStore.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/stores/ResponsiveWindowStore.java index 335ac93d2..0f1c05d86 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/stores/ResponsiveWindowStore.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/stores/ResponsiveWindowStore.java @@ -25,7 +25,6 @@ import dev.responsive.kafka.internal.utils.Iterators; import dev.responsive.kafka.internal.utils.TableName; import java.util.Collection; -import java.util.Optional; import java.util.concurrent.TimeoutException; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.common.utils.Bytes; @@ -69,7 +68,7 @@ public class ResponsiveWindowStore public ResponsiveWindowStore(final ResponsiveWindowParams params) { this.params = params; this.name = params.name(); - this.retentionPeriod = params.retentionPeriod(); + this.retentionPeriod = params.retentionPeriodMs(); this.position = Position.emptyPosition(); this.log = new LogContext( String.format("window-store [%s] ", name.kafkaName()) @@ -103,8 +102,7 @@ public void init(final StateStoreContext storeContext, final StateStore root) { params, appConfigs, config, - window -> window.windowStartMs >= minValidTimestamp(), - Optional.empty() // FIXME: Provide TTLResolver + window -> window.windowStartMs >= minValidTimestamp() ); numBloomFilterWindows = params.retainDuplicates() From e188cd8e366e712352115faedb4e02c46b32c78f Mon Sep 17 00:00:00 2001 From: Jason Gustafson <12502538+hachikuji@users.noreply.github.com> Date: Fri, 25 Apr 2025 09:58:35 -0700 Subject: [PATCH 22/29] Fix failing end to end tests --- .../db/rs3/client/grpc/GrpcRangeKeyCodec.java | 4 +-- .../grpc/GrpcRS3ClientEndToEndTest.java | 28 ++++++++++++++----- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRangeKeyCodec.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRangeKeyCodec.java index ddb4457c3..db2b997d9 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRangeKeyCodec.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRangeKeyCodec.java @@ -89,7 +89,7 @@ class BasicKeyCodec implements GrpcRangeKeyCodec { public KeyValue decodeKeyValue(final Rs3.KeyValue keyValue) { final var kvProto = keyValue.getBasicKv(); final var key = Bytes.wrap(kvProto.getKey().getKey().toByteArray()); - final var value = kvProto.getValue().toByteArray(); + final var value = kvProto.getValue().getValue().toByteArray(); return new KeyValue<>(key, value); } @@ -119,7 +119,7 @@ public Rs3.BasicBound map(final RangeBound.InclusiveBound b) { public Rs3.BasicBound map(final RangeBound.ExclusiveBound b) { final var key = Rs3.BasicKey.newBuilder() .setKey(ByteString.copyFrom(b.key().get())); - return boundProto.setType(Rs3.BoundType.INCLUSIVE) + return boundProto.setType(Rs3.BoundType.EXCLUSIVE) .setKey(key) .build(); } diff --git a/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRS3ClientEndToEndTest.java b/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRS3ClientEndToEndTest.java index 2f1b14293..334943846 100644 --- a/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRS3ClientEndToEndTest.java +++ b/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRS3ClientEndToEndTest.java @@ -414,14 +414,28 @@ public void onNext(final Rs3.WriteWALSegmentRequest req) { current -> Math.max(current, req.getEndOffset()) ); if (req.hasPut()) { - final var kv = req.getPut().getKv().getBasicKv(); - final var keyBytes = Bytes.wrap(kv.getKey().getKey().toByteArray()); - final var valueBytes = Bytes.wrap(kv.getValue().getValue().toByteArray()); - table.put(keyBytes, valueBytes); + final var kv = req.getPut().getKv(); + if (kv.hasBasicKv()) { + final var basicKv = kv.getBasicKv(); + final var keyBytes = Bytes.wrap(basicKv.getKey().getKey().toByteArray()); + final var valueBytes = Bytes.wrap(basicKv.getValue().getValue().toByteArray()); + table.put(keyBytes, valueBytes); + } else if (kv.hasWindowKv()) { + throw new UnsupportedOperationException("Window ops not yet supported"); + } else { + throw new UnsupportedOperationException("Unhandled key-value type in Put"); + } } else if (req.hasDelete()) { - final var key = req.getDelete().getKey().getBasicKey(); - final var keyBytes = Bytes.wrap(key.getKey().toByteArray()); - table.remove(keyBytes); + final var key = req.getDelete().getKey(); + if (key.hasBasicKey()) { + final var basicKey = key.getBasicKey(); + final var keyBytes = Bytes.wrap(basicKey.getKey().toByteArray()); + table.remove(keyBytes); + } else if (key.hasWindowKey()) { + throw new UnsupportedOperationException("Window ops not yet supported"); + } else { + throw new UnsupportedOperationException("Unhandled key-value type in Put"); + } } } From 873b6446f0f1b4c4b5392ec8f655ab7e948d6dda Mon Sep 17 00:00:00 2001 From: Jason Gustafson <12502538+hachikuji@users.noreply.github.com> Date: Fri, 25 Apr 2025 11:43:30 -0700 Subject: [PATCH 23/29] Implement windowed end-to-end test --- .../ResponsiveWindowBytesStoreSupplier.java | 5 +- .../internal/db/rs3/RS3TableFactory.java | 4 +- .../internal/db/rs3/client/RS3ClientUtil.java | 1 - ...va => GrpcRS3ClientBasicEndToEndTest.java} | 238 +++++----------- .../grpc/GrpcRS3ClientWindowEndToEndTest.java | 262 ++++++++++++++++++ .../db/rs3/client/grpc/GrpsRs3TestUtil.java | 24 -- .../rs3/client/grpc/TestGrpcRs3Service.java | 188 +++++++++++++ 7 files changed, 522 insertions(+), 200 deletions(-) rename kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/grpc/{GrpcRS3ClientEndToEndTest.java => GrpcRS3ClientBasicEndToEndTest.java} (53%) create mode 100644 kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRS3ClientWindowEndToEndTest.java create mode 100644 kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/grpc/TestGrpcRs3Service.java diff --git a/kafka-client/src/main/java/dev/responsive/kafka/api/stores/ResponsiveWindowBytesStoreSupplier.java b/kafka-client/src/main/java/dev/responsive/kafka/api/stores/ResponsiveWindowBytesStoreSupplier.java index fdfa924c4..2004a56f9 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/api/stores/ResponsiveWindowBytesStoreSupplier.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/api/stores/ResponsiveWindowBytesStoreSupplier.java @@ -27,7 +27,10 @@ public class ResponsiveWindowBytesStoreSupplier implements WindowBytesStoreSuppl public ResponsiveWindowBytesStoreSupplier(final ResponsiveWindowParams params) { this.params = params; - this.segmentIntervalMs = computeSegmentInterval(params.retentionPeriodMs(), params.numSegments()); + this.segmentIntervalMs = computeSegmentInterval( + params.retentionPeriodMs(), + params.numSegments() + ); } @Override diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3TableFactory.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3TableFactory.java index 15abcf487..9458dd97d 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3TableFactory.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3TableFactory.java @@ -49,7 +49,9 @@ public RemoteKVTable kvTable( final ResponsiveMetrics.MetricScopeBuilder scopeBuilder, final Supplier computeNumKafkaPartitions ) { - final Optional defaultTtl = ttlResolver.isPresent() && ttlResolver.get().defaultTtl().isFinite() + + final Optional defaultTtl = ttlResolver.isPresent() + && ttlResolver.get().defaultTtl().isFinite() ? Optional.of(ttlResolver.get().defaultTtl().duration()) : Optional.empty(); final Optional clockType = ttlResolver.isPresent() diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/RS3ClientUtil.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/RS3ClientUtil.java index 11ecefb3b..f6862439a 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/RS3ClientUtil.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/RS3ClientUtil.java @@ -71,5 +71,4 @@ public LssMetadata fetchLssMetadata(LssId lssId) { return new LssMetadata(lastWrittenOffset, writtenOffsets); } - } diff --git a/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRS3ClientEndToEndTest.java b/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRS3ClientBasicEndToEndTest.java similarity index 53% rename from kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRS3ClientEndToEndTest.java rename to kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRS3ClientBasicEndToEndTest.java index 334943846..15b6dfb5a 100644 --- a/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRS3ClientEndToEndTest.java +++ b/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRS3ClientBasicEndToEndTest.java @@ -6,11 +6,11 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; -import com.google.protobuf.ByteString; import dev.responsive.kafka.internal.db.rs3.client.LssId; import dev.responsive.kafka.internal.db.rs3.client.Put; import dev.responsive.kafka.internal.db.rs3.client.Range; import dev.responsive.kafka.internal.db.rs3.client.RangeBound; +import dev.responsive.kafka.internal.db.rs3.client.WalEntry; import dev.responsive.rs3.RS3Grpc; import dev.responsive.rs3.Rs3; import io.grpc.ClientCall; @@ -20,7 +20,6 @@ import io.grpc.StatusRuntimeException; import io.grpc.inprocess.InProcessChannelBuilder; import io.grpc.inprocess.InProcessServerBuilder; -import io.grpc.stub.StreamObserver; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.Arrays; @@ -29,7 +28,7 @@ import java.util.Optional; import java.util.UUID; import java.util.concurrent.ConcurrentSkipListMap; -import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Stream; import org.apache.kafka.common.utils.Bytes; import org.apache.kafka.common.utils.Time; import org.apache.kafka.streams.state.KeyValueIterator; @@ -38,7 +37,7 @@ import org.junit.jupiter.api.Test; import org.mockito.Mockito; -class GrpcRS3ClientEndToEndTest { +class GrpcRS3ClientBasicEndToEndTest { private static final String SERVER_NAME = "localhost"; private static final long RETRY_TIMEOUT_MS = 10000; @@ -52,9 +51,15 @@ class GrpcRS3ClientEndToEndTest { @BeforeEach public void setUp() throws IOException { + final var service = new TestGrpcRs3Service( + STORE_ID, + LSS_ID, + PSS_ID, + new BasicKeyValueStore() + ); this.server = InProcessServerBuilder .forName(SERVER_NAME) - .addService(new TestRs3Service()) + .addService(service) .build() .start(); this.channel = Mockito.spy(InProcessChannelBuilder @@ -133,7 +138,7 @@ public void shouldScanAllKeyValues() { assertThat(iter.hasNext(), is(false)); } - private void writeWalSegment(long endOffset, List puts) { + private void writeWalSegment(long endOffset, List puts) { final var sendRecv = client.writeWalSegmentAsync( STORE_ID, LSS_ID, @@ -266,195 +271,82 @@ private Put buildPut(String key, String value) { return new Put(keyBytes, valueBytes); } - static class TestRs3Service extends RS3Grpc.RS3ImplBase { - private final AtomicLong offset = new AtomicLong(0); + static class BasicKeyValueStore implements TestGrpcRs3Service.KeyValueStore { private final ConcurrentSkipListMap table = new ConcurrentSkipListMap<>(); @Override - public void getOffsets( - final Rs3.GetOffsetsRequest req, - final StreamObserver responseObserver - ) { - final var storeId = new UUID( - req.getStoreId().getHigh(), - req.getStoreId().getLow() - ); - if (req.getPssId() != PSS_ID - || req.getLssId().getId() != LSS_ID.id() - || !storeId.equals(STORE_ID)) { - responseObserver.onError(new StatusRuntimeException(Status.INVALID_ARGUMENT)); + public void put(final Rs3.KeyValue kvProto) { + if (!kvProto.hasBasicKv()) { + throw new UnsupportedOperationException("Unsupported kv type"); } - - final var currentOffset = offset.get(); - final var result = Rs3.GetOffsetsResult - .newBuilder() - .setFlushedOffset(currentOffset) - .setWrittenOffset(currentOffset) - .build(); - responseObserver.onNext(result); - responseObserver.onCompleted(); + final var kv = kvProto.getBasicKv(); + final var keyBytes = Bytes.wrap(kv.getKey().getKey().toByteArray()); + final var valueBytes = Bytes.wrap(kv.getValue().getValue().toByteArray()); + table.put(keyBytes, valueBytes); } @Override - public void get( - final Rs3.GetRequest req, - final StreamObserver responseObserver - ) { - final var storeId = new UUID( - req.getStoreId().getHigh(), - req.getStoreId().getLow() - ); - if (req.getPssId() != PSS_ID - || req.getLssId().getId() != LSS_ID.id() - || !storeId.equals(STORE_ID)) { - responseObserver.onError(new StatusRuntimeException(Status.INVALID_ARGUMENT)); - return; + public void delete(final Rs3.Key keyProto) { + if (!keyProto.hasBasicKey()) { + throw new UnsupportedOperationException("Unsupported kv type"); } + final var keyBytes = Bytes.wrap(keyProto.getBasicKey().getKey().toByteArray()); + table.remove(keyBytes); + } - if (req.getExpectedWrittenOffset() != GrpcRS3Client.WAL_OFFSET_NONE) { - if (offset.get() < req.getExpectedWrittenOffset()) { - responseObserver.onError(new StatusRuntimeException(Status.INVALID_ARGUMENT)); - return; - } + @Override + public Optional get(final Rs3.Key keyProto) { + if (!keyProto.hasBasicKey()) { + throw new UnsupportedOperationException("Unsupported kv type"); } - - final var key = req.getKey().getBasicKey(); - final var kvBuilder = Rs3.BasicKeyValue.newBuilder().setKey(key); - - final var keyBytes = Bytes.wrap(key.getKey().toByteArray()); + final var keyBytes = Bytes.wrap(keyProto.getBasicKey().getKey().toByteArray()); final var valueBytes = table.get(keyBytes); - if (valueBytes != null) { - final var value = Rs3.BasicValue.newBuilder() - .setValue(ByteString.copyFrom(valueBytes.get())); - kvBuilder.setValue(value); + if (valueBytes == null) { + return Optional.empty(); + } else { + return Optional.of( + Rs3.KeyValue.newBuilder() + .setBasicKv(GrpcRs3Util.basicKeyValueProto(keyBytes.get(), valueBytes.get())) + .build() + ); } - - final var result = Rs3.GetResult - .newBuilder() - .setResult(Rs3.KeyValue.newBuilder().setBasicKv(kvBuilder)) - .build(); - responseObserver.onNext(result); - responseObserver.onCompleted(); } @Override - public void range( - final Rs3.RangeRequest req, - final StreamObserver responseObserver - ) { - final var storeId = new UUID( - req.getStoreId().getHigh(), - req.getStoreId().getLow() - ); - if (req.getPssId() != PSS_ID - || req.getLssId().getId() != LSS_ID.id() - || !storeId.equals(STORE_ID)) { - responseObserver.onError(new StatusRuntimeException(Status.INVALID_ARGUMENT)); - return; + public Stream range(final Rs3.Range rangeProto) { + if (!rangeProto.hasBasicRange()) { + throw new UnsupportedOperationException("Unsupported kv type"); } - if (req.getExpectedWrittenOffset() != GrpcRS3Client.WAL_OFFSET_NONE) { - if (offset.get() < req.getExpectedWrittenOffset()) { - responseObserver.onError(new StatusRuntimeException(Status.INVALID_ARGUMENT)); - return; - } - } - - final var range = GrpsRs3TestUtil.newRangeFromProto(req); - for (final var keyValueEntry : table.entrySet()) { - if (!range.contains(keyValueEntry.getKey())) { - continue; - } - - final var keyValue = GrpcRs3Util.basicKeyValueProto( - keyValueEntry.getKey().get(), - keyValueEntry.getValue().get() - ); - - final var keyValueResult = Rs3.RangeResult.newBuilder() - .setType(Rs3.RangeResult.Type.RESULT) - .setResult(Rs3.KeyValue.newBuilder().setBasicKv(keyValue)) - .build(); - - responseObserver.onNext(keyValueResult); - } + final var basicRange = rangeProto.getBasicRange(); + final var range = new Range<>( + decodeBound(basicRange.getFrom()), + decodeBound(basicRange.getTo()) + ); - final var endOfStream = Rs3.RangeResult.newBuilder() - .setType(Rs3.RangeResult.Type.END_OF_STREAM) - .build(); - responseObserver.onNext(endOfStream); - responseObserver.onCompleted(); + return table.entrySet().stream() + .filter(entry -> range.contains(entry.getKey())) + .map(entry -> Rs3.KeyValue.newBuilder().setBasicKv( + GrpcRs3Util.basicKeyValueProto(entry.getKey().get(), entry.getValue().get()) + ).build()); } - @Override - public StreamObserver writeWALSegmentStream( - final StreamObserver responseObserver - ) { - return new StreamObserver<>() { - @Override - public void onNext(final Rs3.WriteWALSegmentRequest req) { - final var storeId = new UUID( - req.getStoreId().getHigh(), - req.getStoreId().getLow() - ); - if (req.getPssId() != PSS_ID - || req.getLssId().getId() != LSS_ID.id() - || !storeId.equals(STORE_ID)) { - responseObserver.onError(new StatusRuntimeException(Status.INVALID_ARGUMENT)); - } - - if (req.getExpectedWrittenOffset() != GrpcRS3Client.WAL_OFFSET_NONE) { - if (offset.get() < req.getExpectedWrittenOffset()) { - responseObserver.onError(new StatusRuntimeException(Status.INVALID_ARGUMENT)); - return; - } - } - - TestRs3Service.this.offset.getAndUpdate( - current -> Math.max(current, req.getEndOffset()) - ); - if (req.hasPut()) { - final var kv = req.getPut().getKv(); - if (kv.hasBasicKv()) { - final var basicKv = kv.getBasicKv(); - final var keyBytes = Bytes.wrap(basicKv.getKey().getKey().toByteArray()); - final var valueBytes = Bytes.wrap(basicKv.getValue().getValue().toByteArray()); - table.put(keyBytes, valueBytes); - } else if (kv.hasWindowKv()) { - throw new UnsupportedOperationException("Window ops not yet supported"); - } else { - throw new UnsupportedOperationException("Unhandled key-value type in Put"); - } - } else if (req.hasDelete()) { - final var key = req.getDelete().getKey(); - if (key.hasBasicKey()) { - final var basicKey = key.getBasicKey(); - final var keyBytes = Bytes.wrap(basicKey.getKey().toByteArray()); - table.remove(keyBytes); - } else if (key.hasWindowKey()) { - throw new UnsupportedOperationException("Window ops not yet supported"); - } else { - throw new UnsupportedOperationException("Unhandled key-value type in Put"); - } - } - } - - @Override - public void onError(final Throwable throwable) { - responseObserver.onError(throwable); - } - - @Override - public void onCompleted() { - final var result = Rs3.WriteWALSegmentResult - .newBuilder() - .setFlushedOffset(offset.get()) - .build(); - responseObserver.onNext(result); - responseObserver.onCompleted(); + private RangeBound decodeBound(Rs3.BasicBound bound) { + if (bound.getType() == Rs3.BoundType.UNBOUNDED) { + return RangeBound.unbounded(); + } else { + final var key = Bytes.wrap(bound.getKey().getKey().toByteArray()); + if (bound.getType() == Rs3.BoundType.INCLUSIVE) { + return RangeBound.inclusive(key); + } else if (bound.getType() == Rs3.BoundType.EXCLUSIVE) { + return RangeBound.exclusive(key); + } else { + throw new UnsupportedOperationException("Unsupported bound type"); } - }; + } } } + + } \ No newline at end of file diff --git a/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRS3ClientWindowEndToEndTest.java b/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRS3ClientWindowEndToEndTest.java new file mode 100644 index 000000000..f0c17b0a5 --- /dev/null +++ b/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRS3ClientWindowEndToEndTest.java @@ -0,0 +1,262 @@ +package dev.responsive.kafka.internal.db.rs3.client.grpc; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; + +import dev.responsive.kafka.internal.db.rs3.client.LssId; +import dev.responsive.kafka.internal.db.rs3.client.Range; +import dev.responsive.kafka.internal.db.rs3.client.RangeBound; +import dev.responsive.kafka.internal.db.rs3.client.WalEntry; +import dev.responsive.kafka.internal.db.rs3.client.WindowedDelete; +import dev.responsive.kafka.internal.db.rs3.client.WindowedPut; +import dev.responsive.kafka.internal.utils.WindowedKey; +import dev.responsive.rs3.Rs3; +import io.grpc.ManagedChannel; +import io.grpc.Server; +import io.grpc.inprocess.InProcessChannelBuilder; +import io.grpc.inprocess.InProcessServerBuilder; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.ConcurrentSkipListMap; +import java.util.stream.Stream; +import org.apache.kafka.common.utils.Bytes; +import org.apache.kafka.common.utils.Time; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +public class GrpcRS3ClientWindowEndToEndTest { + private static final String SERVER_NAME = "localhost"; + private static final long RETRY_TIMEOUT_MS = 10000; + private static final UUID STORE_ID = UUID.randomUUID(); + private static final int PSS_ID = 0; + private static final LssId LSS_ID = new LssId(PSS_ID); + + private Server server; + private ManagedChannel channel; + private GrpcRS3Client client; + + @BeforeEach + public void setUp() throws IOException { + final var service = new TestGrpcRs3Service( + STORE_ID, + LSS_ID, + PSS_ID, + new WindowKeyValueStore() + ); + this.server = InProcessServerBuilder + .forName(SERVER_NAME) + .addService(service) + .build() + .start(); + this.channel = Mockito.spy(InProcessChannelBuilder + .forName(SERVER_NAME) + .directExecutor() + .build()); + this.client = new GrpcRS3Client( + new PssStubsProvider(this.channel), + Time.SYSTEM, + RETRY_TIMEOUT_MS + ); + } + + @AfterEach + public void tearDown() { + this.channel.shutdownNow(); + this.server.shutdownNow(); + this.client.close(); + } + + @Test + public void shouldPutAndGet() { + final var key = new WindowedKey( + "foo".getBytes(StandardCharsets.UTF_8), + 100L + ); + final var value = "bar".getBytes(StandardCharsets.UTF_8); + writeWalSegment( + 10L, + Collections.singletonList(windowedPut(key, value)) + ); + + final var getResult = client.windowedGet( + STORE_ID, + LSS_ID, + PSS_ID, + Optional.of(5L), + key + ); + assertThat(getResult.isPresent(), is(true)); + final var resultValue = getResult.get(); + assertThat(resultValue, equalTo(value)); + } + + @Test + public void shouldDelete() { + final var key = new WindowedKey( + "foo".getBytes(StandardCharsets.UTF_8), + 100L + ); + final var value = "bar".getBytes(StandardCharsets.UTF_8); + writeWalSegment( + 10L, + Collections.singletonList(windowedPut(key, value)) + ); + + assertThat(client.windowedGet( + STORE_ID, + LSS_ID, + PSS_ID, + Optional.of(5L), + key + ).isPresent(), is(true)); + + writeWalSegment( + 10L, + Collections.singletonList(windowedDelete(key)) + ); + + assertThat(client.windowedGet( + STORE_ID, + LSS_ID, + PSS_ID, + Optional.of(5L), + key + ).isPresent(), is(false)); + } + + private static WindowedPut windowedPut( + WindowedKey key, + byte[] value + ) { + return new WindowedPut( + key.key.get(), + value, + 0L, + key.windowStartMs + ); + } + + private static WindowedDelete windowedDelete( + WindowedKey key + ) { + return new WindowedDelete( + key.key.get(), + key.windowStartMs + ); + } + + private void writeWalSegment(long endOffset, List entries) { + final var sendRecv = client.writeWalSegmentAsync( + STORE_ID, + LSS_ID, + PSS_ID, + Optional.empty(), + endOffset + ); + + entries.forEach(entry -> sendRecv.sender().sendNext(entry)); + sendRecv.sender().finish(); + + final var flushedOffset = sendRecv + .completion() + .toCompletableFuture() + .join(); + assertThat(flushedOffset, is(Optional.of(endOffset))); + } + + static class WindowKeyValueStore implements TestGrpcRs3Service.KeyValueStore { + private final ConcurrentSkipListMap table = new ConcurrentSkipListMap<>(); + + @Override + public void put(final Rs3.KeyValue kvProto) { + if (!kvProto.hasWindowKv()) { + throw new UnsupportedOperationException("Unsupported kv type"); + } + final var windowKv = kvProto.getWindowKv(); + final var windowKey = new WindowedKey( + Bytes.wrap(windowKv.getKey().getKey().toByteArray()), + windowKv.getKey().getWindowTimestamp() + ); + final var valueBytes = Bytes.wrap(windowKv.getValue().getValue().toByteArray()); + table.put(windowKey, valueBytes); + } + + @Override + public void delete(final Rs3.Key keyProto) { + if (!keyProto.hasWindowKey()) { + throw new UnsupportedOperationException("Unsupported kv type"); + } + final var windowKey = new WindowedKey( + Bytes.wrap(keyProto.getWindowKey().getKey().toByteArray()), + keyProto.getWindowKey().getWindowTimestamp() + ); + table.remove(windowKey); + } + + @Override + public Optional get(final Rs3.Key keyProto) { + if (!keyProto.hasWindowKey()) { + throw new UnsupportedOperationException("Unsupported kv type"); + } + final var windowKey = new WindowedKey( + Bytes.wrap(keyProto.getWindowKey().getKey().toByteArray()), + keyProto.getWindowKey().getWindowTimestamp() + ); + + final var valueBytes = table.get(windowKey); + if (valueBytes == null) { + return Optional.empty(); + } else { + return Optional.of( + Rs3.KeyValue.newBuilder() + .setWindowKv(GrpcRs3Util.windowKeyValueProto(windowKey, valueBytes.get())) + .build() + ); + } + } + + @Override + public Stream range(final Rs3.Range rangeProto) { + if (!rangeProto.hasWindowRange()) { + throw new UnsupportedOperationException("Unsupported kv type"); + } + + final var windowRange = rangeProto.getWindowRange(); + final var range = new Range<>( + decodeBound(windowRange.getFrom()), + decodeBound(windowRange.getTo()) + ); + + return table.entrySet().stream() + .filter(entry -> range.contains(entry.getKey())) + .map(entry -> Rs3.KeyValue.newBuilder().setWindowKv( + GrpcRs3Util.windowKeyValueProto(entry.getKey(), entry.getValue().get()) + ).build()); + } + + private RangeBound decodeBound(Rs3.WindowBound bound) { + if (bound.getType() == Rs3.BoundType.UNBOUNDED) { + return RangeBound.unbounded(); + } else { + final var windowKey = new WindowedKey( + Bytes.wrap(bound.getKey().getKey().toByteArray()), + bound.getKey().getWindowTimestamp() + ); + if (bound.getType() == Rs3.BoundType.INCLUSIVE) { + return RangeBound.inclusive(windowKey); + } else if (bound.getType() == Rs3.BoundType.EXCLUSIVE) { + return RangeBound.exclusive(windowKey); + } else { + throw new UnsupportedOperationException("Unsupported bound type"); + } + } + } + } +} diff --git a/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpsRs3TestUtil.java b/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpsRs3TestUtil.java index 7abe46c9c..c2794a2d5 100644 --- a/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpsRs3TestUtil.java +++ b/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpsRs3TestUtil.java @@ -12,11 +12,8 @@ package dev.responsive.kafka.internal.db.rs3.client.grpc; -import dev.responsive.kafka.internal.db.rs3.client.Range; -import dev.responsive.kafka.internal.db.rs3.client.RangeBound; import dev.responsive.rs3.Rs3; import java.nio.charset.StandardCharsets; -import org.apache.kafka.common.utils.Bytes; public class GrpsRs3TestUtil { @@ -31,31 +28,10 @@ public static Rs3.RangeResult newKeyValueResult(String key) { .build(); } - public static Rs3.RangeResult newEndOfStreamResult() { return Rs3.RangeResult.newBuilder() .setType(Rs3.RangeResult.Type.END_OF_STREAM) .build(); } - public static Range newRangeFromProto(Rs3.RangeRequest req) { - final var range = req.getRange().getBasicRange(); - final var startBound = newRangeBoundFromProto(range.getFrom()); - final var endBound = newRangeBoundFromProto(range.getTo()); - return new Range<>(startBound, endBound); - } - - private static RangeBound newRangeBoundFromProto(Rs3.BasicBound bound) { - switch (bound.getType()) { - case EXCLUSIVE: - return RangeBound.exclusive(Bytes.wrap(bound.getKey().getKey().toByteArray())); - case INCLUSIVE: - return RangeBound.inclusive(Bytes.wrap(bound.getKey().getKey().toByteArray())); - case UNBOUNDED: - return RangeBound.unbounded(); - default: - throw new IllegalArgumentException(String.format("Unknown range type %s", bound.getType())); - } - } - } diff --git a/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/grpc/TestGrpcRs3Service.java b/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/grpc/TestGrpcRs3Service.java new file mode 100644 index 000000000..a8446a6b1 --- /dev/null +++ b/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/grpc/TestGrpcRs3Service.java @@ -0,0 +1,188 @@ +package dev.responsive.kafka.internal.db.rs3.client.grpc; + +import dev.responsive.kafka.internal.db.rs3.client.LssId; +import dev.responsive.rs3.RS3Grpc; +import dev.responsive.rs3.Rs3; +import io.grpc.Status; +import io.grpc.StatusRuntimeException; +import io.grpc.stub.StreamObserver; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Stream; + +class TestGrpcRs3Service extends RS3Grpc.RS3ImplBase { + + interface KeyValueStore { + void put(Rs3.KeyValue kv); + + void delete(Rs3.Key key); + + Optional get(Rs3.Key key); + + Stream range(Rs3.Range range); + } + + private final UUID storeId; + private final LssId lssId; + private final int pssId; + + private final AtomicLong offset = new AtomicLong(0); + private final KeyValueStore store; + + public TestGrpcRs3Service( + final UUID storeId, + final LssId lssId, + final int pssId, + final KeyValueStore store + ) { + this.storeId = storeId; + this.lssId = lssId; + this.pssId = pssId; + this.store = store; + } + + @Override + public void getOffsets( + final Rs3.GetOffsetsRequest req, + final StreamObserver responseObserver + ) { + final var storeId = new UUID( + req.getStoreId().getHigh(), + req.getStoreId().getLow() + ); + if (req.getPssId() != pssId + || req.getLssId().getId() != lssId.id() + || !storeId.equals(this.storeId)) { + responseObserver.onError(new StatusRuntimeException(Status.INVALID_ARGUMENT)); + } + + final var currentOffset = offset.get(); + final var result = Rs3.GetOffsetsResult + .newBuilder() + .setFlushedOffset(currentOffset) + .setWrittenOffset(currentOffset) + .build(); + responseObserver.onNext(result); + responseObserver.onCompleted(); + } + + @Override + public void get( + final Rs3.GetRequest req, + final StreamObserver responseObserver + ) { + final var storeId = new UUID( + req.getStoreId().getHigh(), + req.getStoreId().getLow() + ); + if (req.getPssId() != pssId + || req.getLssId().getId() != lssId.id() + || !storeId.equals(this.storeId)) { + responseObserver.onError(new StatusRuntimeException(Status.INVALID_ARGUMENT)); + return; + } + + if (req.getExpectedWrittenOffset() != GrpcRS3Client.WAL_OFFSET_NONE) { + if (offset.get() < req.getExpectedWrittenOffset()) { + responseObserver.onError(new StatusRuntimeException(Status.INVALID_ARGUMENT)); + return; + } + } + + final var resultBldr = Rs3.GetResult.newBuilder(); + final var kv = store.get(req.getKey()); + kv.ifPresent(resultBldr::setResult); + responseObserver.onNext(resultBldr.build()); + responseObserver.onCompleted(); + } + + @Override + public void range( + final Rs3.RangeRequest req, + final StreamObserver responseObserver + ) { + final var storeId = new UUID( + req.getStoreId().getHigh(), + req.getStoreId().getLow() + ); + if (req.getPssId() != pssId + || req.getLssId().getId() != lssId.id() + || !storeId.equals(this.storeId)) { + responseObserver.onError(new StatusRuntimeException(Status.INVALID_ARGUMENT)); + return; + } + + if (req.getExpectedWrittenOffset() != GrpcRS3Client.WAL_OFFSET_NONE) { + if (offset.get() < req.getExpectedWrittenOffset()) { + responseObserver.onError(new StatusRuntimeException(Status.INVALID_ARGUMENT)); + return; + } + } + + store.range(req.getRange()).forEach(kv -> { + final var keyValueResult = Rs3.RangeResult.newBuilder() + .setType(Rs3.RangeResult.Type.RESULT) + .setResult(kv) + .build(); + responseObserver.onNext(keyValueResult); + }); + + final var endOfStream = Rs3.RangeResult.newBuilder() + .setType(Rs3.RangeResult.Type.END_OF_STREAM) + .build(); + responseObserver.onNext(endOfStream); + responseObserver.onCompleted(); + } + + @Override + public StreamObserver writeWALSegmentStream( + final StreamObserver responseObserver + ) { + return new StreamObserver<>() { + @Override + public void onNext(final Rs3.WriteWALSegmentRequest req) { + final var storeId = new UUID( + req.getStoreId().getHigh(), + req.getStoreId().getLow() + ); + if (req.getPssId() != pssId + || req.getLssId().getId() != lssId.id() + || !storeId.equals(TestGrpcRs3Service.this.storeId)) { + responseObserver.onError(new StatusRuntimeException(Status.INVALID_ARGUMENT)); + } + + if (req.getExpectedWrittenOffset() != GrpcRS3Client.WAL_OFFSET_NONE) { + if (offset.get() < req.getExpectedWrittenOffset()) { + responseObserver.onError(new StatusRuntimeException(Status.INVALID_ARGUMENT)); + return; + } + } + + TestGrpcRs3Service.this.offset.getAndUpdate( + current -> Math.max(current, req.getEndOffset()) + ); + if (req.hasPut()) { + store.put(req.getPut().getKv()); + } else if (req.hasDelete()) { + store.delete(req.getDelete().getKey()); + } + } + + @Override + public void onError(final Throwable throwable) { + responseObserver.onError(throwable); + } + + @Override + public void onCompleted() { + final var result = Rs3.WriteWALSegmentResult + .newBuilder() + .setFlushedOffset(offset.get()) + .build(); + responseObserver.onNext(result); + responseObserver.onCompleted(); + } + }; + } +} From e2ae90c6224f1bdffe087425546eaabf8e458c85 Mon Sep 17 00:00:00 2001 From: Jason Gustafson <12502538+hachikuji@users.noreply.github.com> Date: Fri, 25 Apr 2025 11:55:15 -0700 Subject: [PATCH 24/29] fix merge breakage --- .../db/rs3/client/grpc/GrpcRS3Client.java | 2 +- .../db/rs3/client/grpc/TestGrpcRs3Service.java | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRS3Client.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRS3Client.java index ce0e81649..161cf6080 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRS3Client.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRS3Client.java @@ -13,9 +13,9 @@ package dev.responsive.kafka.internal.db.rs3.client.grpc; import static dev.responsive.kafka.internal.db.rs3.client.grpc.GrpcRs3Util.basicKeyProto; -import static dev.responsive.kafka.internal.db.rs3.client.grpc.GrpcRs3Util.windowKeyProto; import static dev.responsive.kafka.internal.db.rs3.client.grpc.GrpcRs3Util.walOffsetFromProto; import static dev.responsive.kafka.internal.db.rs3.client.grpc.GrpcRs3Util.walOffsetProto; +import static dev.responsive.kafka.internal.db.rs3.client.grpc.GrpcRs3Util.windowKeyProto; import static dev.responsive.kafka.internal.utils.Utils.lssIdProto; import static dev.responsive.kafka.internal.utils.Utils.uuidFromProto; import static dev.responsive.kafka.internal.utils.Utils.uuidToProto; diff --git a/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/grpc/TestGrpcRs3Service.java b/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/grpc/TestGrpcRs3Service.java index a8446a6b1..5404eee46 100644 --- a/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/grpc/TestGrpcRs3Service.java +++ b/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/grpc/TestGrpcRs3Service.java @@ -60,8 +60,8 @@ public void getOffsets( final var currentOffset = offset.get(); final var result = Rs3.GetOffsetsResult .newBuilder() - .setFlushedOffset(currentOffset) - .setWrittenOffset(currentOffset) + .setFlushedOffset(GrpcRs3Util.walOffsetProto(currentOffset)) + .setWrittenOffset(GrpcRs3Util.walOffsetProto(currentOffset)) .build(); responseObserver.onNext(result); responseObserver.onCompleted(); @@ -83,8 +83,8 @@ public void get( return; } - if (req.getExpectedWrittenOffset() != GrpcRS3Client.WAL_OFFSET_NONE) { - if (offset.get() < req.getExpectedWrittenOffset()) { + if (req.getExpectedWrittenOffset().getIsWritten()) { + if (offset.get() < req.getExpectedWrittenOffset().getOffset()) { responseObserver.onError(new StatusRuntimeException(Status.INVALID_ARGUMENT)); return; } @@ -113,8 +113,8 @@ public void range( return; } - if (req.getExpectedWrittenOffset() != GrpcRS3Client.WAL_OFFSET_NONE) { - if (offset.get() < req.getExpectedWrittenOffset()) { + if (req.getExpectedWrittenOffset().getIsWritten()) { + if (offset.get() < req.getExpectedWrittenOffset().getOffset()) { responseObserver.onError(new StatusRuntimeException(Status.INVALID_ARGUMENT)); return; } @@ -152,8 +152,8 @@ public void onNext(final Rs3.WriteWALSegmentRequest req) { responseObserver.onError(new StatusRuntimeException(Status.INVALID_ARGUMENT)); } - if (req.getExpectedWrittenOffset() != GrpcRS3Client.WAL_OFFSET_NONE) { - if (offset.get() < req.getExpectedWrittenOffset()) { + if (req.getExpectedWrittenOffset().getIsWritten()) { + if (offset.get() < req.getExpectedWrittenOffset().getOffset()) { responseObserver.onError(new StatusRuntimeException(Status.INVALID_ARGUMENT)); return; } @@ -178,7 +178,7 @@ public void onError(final Throwable throwable) { public void onCompleted() { final var result = Rs3.WriteWALSegmentResult .newBuilder() - .setFlushedOffset(offset.get()) + .setFlushedOffset(GrpcRs3Util.walOffsetProto(offset.get())) .build(); responseObserver.onNext(result); responseObserver.onCompleted(); From 9a8045ef34f5adb0768aa01429597c397299f689 Mon Sep 17 00:00:00 2001 From: Jason Gustafson <12502538+hachikuji@users.noreply.github.com> Date: Fri, 25 Apr 2025 13:35:20 -0700 Subject: [PATCH 25/29] Fix remaining broken tests --- .../kafka/internal/db/rs3/RS3KVTable.java | 2 +- .../internal/db/rs3/RS3TableFactory.java | 4 +-- .../kafka/internal/db/rs3/RS3WindowTable.java | 4 +++ .../internal/db/rs3/client/RangeBound.java | 5 ++- .../rs3/client/grpc/GrpcKeyValueIterator.java | 11 +++--- .../internal/db/rs3/RS3TableFactoryTest.java | 36 +++++++++++++++++-- .../db/rs3/client/grpc/GrpcRS3ClientTest.java | 2 ++ 7 files changed, 53 insertions(+), 11 deletions(-) diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3KVTable.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3KVTable.java index f6e6c7e62..b71635a7f 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3KVTable.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3KVTable.java @@ -155,7 +155,7 @@ public String name() { return name; } - public UUID storedId() { + public UUID storeId() { return storeId; } diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3TableFactory.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3TableFactory.java index 9458dd97d..50d2dc440 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3TableFactory.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3TableFactory.java @@ -62,7 +62,7 @@ public RemoteKVTable kvTable( final UUID storeId = createdStores.computeIfAbsent(storeName, n -> createStore( storeName, - CreateStoreTypes.StoreType.WINDOW, + CreateStoreTypes.StoreType.BASIC, clockType, defaultTtl, computeNumKafkaPartitions.get(), @@ -90,7 +90,7 @@ public RemoteWindowTable windowTable( final var rs3Client = connector.connect(); final UUID storeId = createdStores.computeIfAbsent(storeName, n -> createStore( storeName, - CreateStoreTypes.StoreType.BASIC, + CreateStoreTypes.StoreType.WINDOW, Optional.of(ClockType.STREAM_TIME), Optional.of(defaultTtl), computeNumKafkaPartitions.get(), diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowTable.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowTable.java index 8120fad90..6bfc822d4 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowTable.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowTable.java @@ -85,6 +85,10 @@ public RS3WindowTable( this.pssPartitioner = Objects.requireNonNull(pssPartitioner); } + public UUID storeId() { + return storeId; + } + @Override public WindowFlushManager init(final int kafkaPartition) { if (flushManager != null) { diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/RangeBound.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/RangeBound.java index ae81e71fb..8299c7f89 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/RangeBound.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/RangeBound.java @@ -18,8 +18,9 @@ public interface RangeBound { T map(Mapper mapper); + @SuppressWarnings("unchecked") static Unbounded unbounded() { - return new Unbounded<>(); + return (Unbounded) Unbounded.INSTANCE; } static InclusiveBound inclusive(K key) { @@ -99,6 +100,8 @@ public int hashCode() { } class Unbounded implements RangeBound { + private static Unbounded INSTANCE = new Unbounded<>(); + private Unbounded() {} @Override diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcKeyValueIterator.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcKeyValueIterator.java index d7de8c706..81999f417 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcKeyValueIterator.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcKeyValueIterator.java @@ -38,7 +38,8 @@ public class GrpcKeyValueIterator> implements KeyValueIt private final GrpcRangeRequestProxy requestProxy; private final GrpcRangeKeyCodec keyCodec; private final GrpcMessageQueue queue; - private Range range; + private final RangeBound end; + private RangeBound start; private RangeResultObserver resultObserver; public GrpcKeyValueIterator( @@ -49,7 +50,8 @@ public GrpcKeyValueIterator( this.requestProxy = requestProxy; this.keyCodec = keyCodec; this.queue = new GrpcMessageQueue<>(); - this.range = range; + this.start = range.start(); + this.end = range.end(); sendRangeRequest(); } @@ -78,6 +80,7 @@ static GrpcKeyValueIterator windowed( private void sendRangeRequest() { // Note that backoff on retry is handled internally by the request proxy resultObserver = new RangeResultObserver(); + final var range = new Range<>(start, end); requestProxy.send(range, resultObserver); } @@ -92,9 +95,7 @@ public KeyValue next() { if (nextKeyValue.isPresent()) { queue.poll(); final var keyValue = nextKeyValue.get(); - final RangeBound newStartRange = RangeBound.exclusive(keyValue.key); - final RangeBound newEndRange = this.range.end(); - this.range = new Range<>(newStartRange, newEndRange); + this.start = RangeBound.exclusive(keyValue.key); return keyValue; } else { throw new NoSuchElementException(); diff --git a/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/RS3TableFactoryTest.java b/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/RS3TableFactoryTest.java index a5617d564..404c1c9f1 100644 --- a/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/RS3TableFactoryTest.java +++ b/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/RS3TableFactoryTest.java @@ -28,6 +28,7 @@ import dev.responsive.kafka.internal.db.rs3.client.grpc.GrpcRS3Client.Connector; import dev.responsive.kafka.internal.metrics.ClientVersionMetadata; import dev.responsive.kafka.internal.metrics.ResponsiveMetrics; +import java.time.Duration; import java.util.List; import java.util.Map; import java.util.Optional; @@ -69,7 +70,7 @@ public void setup() { } @Test - public void testTableMapping() { + public void testBasicTableMapping() { final UUID storeId = new UUID(100, 200); final String tableName = "test-table"; final int partitions = 5; @@ -86,7 +87,7 @@ public void testTableMapping() { () -> partitions ); assertEquals(tableName, rs3Table.name()); - assertEquals(storeId, rs3Table.storedId()); + assertEquals(storeId, rs3Table.storeId()); final var expectedOptions = new CreateStoreOptions( partitions, @@ -98,6 +99,37 @@ public void testTableMapping() { verify(client).createStore(tableName, expectedOptions); } + @Test + public void testWindowTableMapping() { + final UUID storeId = new UUID(100, 200); + final String tableName = "test-table"; + final int partitions = 5; + + when(client.createStore(anyString(), any(CreateStoreOptions.class))) + .thenReturn(new CreateStoreResult(storeId, List.of(1, 2, 3, 4, 5))); + + final var factory = newTestFactory(); + final var defaultTtl = Duration.ofMinutes(10); + final RS3WindowTable rs3Table = (RS3WindowTable) factory.windowTable( + tableName, + defaultTtl, + metrics, + scopeBuilder, + () -> partitions + ); + assertEquals(tableName, rs3Table.name()); + assertEquals(storeId, rs3Table.storeId()); + + final var expectedOptions = new CreateStoreOptions( + partitions, + CreateStoreTypes.StoreType.WINDOW, + Optional.of(CreateStoreTypes.ClockType.STREAM_TIME), + Optional.of(defaultTtl.toMillis()), + Optional.empty() + ); + verify(client).createStore(tableName, expectedOptions); + } + private RS3TableFactory newTestFactory() { final var connector = mock(Connector.class); lenient().when(connector.connect()).thenReturn(client); diff --git a/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRS3ClientTest.java b/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRS3ClientTest.java index 62463522c..97356d232 100644 --- a/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRS3ClientTest.java +++ b/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRS3ClientTest.java @@ -824,6 +824,7 @@ public void shouldWindowedGet() { verify(stub).get(Rs3.GetRequest.newBuilder() .setLssId(lssIdProto(LSS_ID)) .setPssId(PSS_ID) + .setExpectedWrittenOffset(GrpcRs3Util.UNWRITTEN_WAL_OFFSET) .setStoreId(uuidToProto(STORE_ID)) .setKey(Rs3.Key.newBuilder().setWindowKey(keyProto)) .build() @@ -864,6 +865,7 @@ public void shouldRetryWindowedGet() { .get(Rs3.GetRequest.newBuilder() .setLssId(lssIdProto(LSS_ID)) .setPssId(PSS_ID) + .setExpectedWrittenOffset(GrpcRs3Util.UNWRITTEN_WAL_OFFSET) .setStoreId(uuidToProto(STORE_ID)) .setKey(Rs3.Key.newBuilder().setWindowKey(keyProto)) .build() From 68e9fa8a4029c1966b6f79ac6181b44e1c08a51e Mon Sep 17 00:00:00 2001 From: Jason Gustafson <12502538+hachikuji@users.noreply.github.com> Date: Fri, 25 Apr 2025 14:59:40 -0700 Subject: [PATCH 26/29] Fix window value iteration with test case --- .../db/rs3/client/grpc/GrpcRangeKeyCodec.java | 2 +- .../grpc/GrpcRS3ClientWindowEndToEndTest.java | 74 +++++++++++++++---- 2 files changed, 61 insertions(+), 15 deletions(-) diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRangeKeyCodec.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRangeKeyCodec.java index db2b997d9..4d47ae0a3 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRangeKeyCodec.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRangeKeyCodec.java @@ -38,7 +38,7 @@ public KeyValue decodeKeyValue(final Rs3.KeyValue keyValue) Bytes.wrap(keyProto.getKey().toByteArray()), keyProto.getWindowTimestamp() ); - final var value = kvProto.getValue().toByteArray(); + final var value = kvProto.getValue().getValue().toByteArray(); return new KeyValue<>(key, value); } diff --git a/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRS3ClientWindowEndToEndTest.java b/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRS3ClientWindowEndToEndTest.java index f0c17b0a5..e264da41c 100644 --- a/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRS3ClientWindowEndToEndTest.java +++ b/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/client/grpc/GrpcRS3ClientWindowEndToEndTest.java @@ -18,6 +18,7 @@ import io.grpc.inprocess.InProcessServerBuilder; import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Optional; @@ -26,6 +27,7 @@ import java.util.stream.Stream; import org.apache.kafka.common.utils.Bytes; import org.apache.kafka.common.utils.Time; +import org.apache.kafka.streams.state.KeyValueIterator; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -75,11 +77,8 @@ public void tearDown() { @Test public void shouldPutAndGet() { - final var key = new WindowedKey( - "foo".getBytes(StandardCharsets.UTF_8), - 100L - ); - final var value = "bar".getBytes(StandardCharsets.UTF_8); + final var key = windowedKey("foo", 100L); + final var value = "bar"; writeWalSegment( 10L, Collections.singletonList(windowedPut(key, value)) @@ -94,19 +93,16 @@ public void shouldPutAndGet() { ); assertThat(getResult.isPresent(), is(true)); final var resultValue = getResult.get(); - assertThat(resultValue, equalTo(value)); + assertThat(new String(resultValue, StandardCharsets.UTF_8), equalTo(value)); } + @Test public void shouldDelete() { - final var key = new WindowedKey( - "foo".getBytes(StandardCharsets.UTF_8), - 100L - ); - final var value = "bar".getBytes(StandardCharsets.UTF_8); + final var key = windowedKey("foo", 100L); writeWalSegment( 10L, - Collections.singletonList(windowedPut(key, value)) + Collections.singletonList(windowedPut(key, "bar")) ); assertThat(client.windowedGet( @@ -131,13 +127,63 @@ public void shouldDelete() { ).isPresent(), is(false)); } + @Test + public void shouldScanValuesInTimeWindowRange() { + writeWalSegment( + 10L, + Arrays.asList( + windowedPut(windowedKey("a", 100L), "1"), + windowedPut(windowedKey("b", 100L), "2"), + windowedPut(windowedKey("a", 200L), "3"), + windowedPut(windowedKey("b", 150L), "4"), + windowedPut(windowedKey("c", 200L), "5"), + windowedPut(windowedKey("a", 300L), "6") + ) + ); + + final var range = new Range<>( + RangeBound.inclusive(windowedKey("a", 100L)), + RangeBound.exclusive(windowedKey("a", 300L)) + ); + + try (final var iter = client.windowedRange( + STORE_ID, + LSS_ID, + PSS_ID, + Optional.empty(), + range + )) { + assertNext(iter, windowedKey("a", 100L), "1"); + assertNext(iter, windowedKey("a", 200L), "3"); + assertThat(iter.hasNext(), is(false)); + } + } + + private void assertNext( + KeyValueIterator iter, + WindowedKey key, + String value + ) { + assertThat(iter.hasNext(), is(true)); + final var keyValue = iter.next(); + assertThat(keyValue.key, is(key)); + assertThat(Bytes.wrap(keyValue.value), is(Bytes.wrap(value.getBytes(StandardCharsets.UTF_8)))); + } + + private static WindowedKey windowedKey(String key, long windowTimestamp) { + return new WindowedKey( + key.getBytes(StandardCharsets.UTF_8), + windowTimestamp + ); + } + private static WindowedPut windowedPut( WindowedKey key, - byte[] value + String value ) { return new WindowedPut( key.key.get(), - value, + value.getBytes(StandardCharsets.UTF_8), 0L, key.windowStartMs ); From b93a5399d8b4edf378cea0f7235ac5c91b45fb27 Mon Sep 17 00:00:00 2001 From: Jason Gustafson <12502538+hachikuji@users.noreply.github.com> Date: Fri, 25 Apr 2025 16:31:33 -0700 Subject: [PATCH 27/29] Get rid of TODO --- .../responsive/kafka/internal/db/rs3/RS3WindowFlushManager.java | 1 - 1 file changed, 1 deletion(-) diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowFlushManager.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowFlushManager.java index e66391e76..14d1e2b95 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowFlushManager.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3WindowFlushManager.java @@ -67,7 +67,6 @@ public TablePartitioner partitioner() { return new PssTablePartitioner<>(pssPartitioner) { @Override public byte[] serialize(final WindowedKey key) { - // TODO: Get rid of this return key.key.get(); } }; From 4c5ee031e1d17437d18a2187cef783c4ca4c9cbb Mon Sep 17 00:00:00 2001 From: Jason Gustafson <12502538+hachikuji@users.noreply.github.com> Date: Fri, 25 Apr 2025 16:33:52 -0700 Subject: [PATCH 28/29] Don't commit jwik database --- kafka-client/.jqwik-database | Bin 4 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 kafka-client/.jqwik-database diff --git a/kafka-client/.jqwik-database b/kafka-client/.jqwik-database deleted file mode 100644 index 711006c3d3b5c6d50049e3f48311f3dbe372803d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4 LcmZ4UmVp%j1%Lsc From db8fe920af63206062bc08467d73b3b1fa71cdda Mon Sep 17 00:00:00 2001 From: Jason Gustafson <12502538+hachikuji@users.noreply.github.com> Date: Fri, 25 Apr 2025 18:28:43 -0700 Subject: [PATCH 29/29] Use wall clock time for now --- kafka-client/.jqwik-database | Bin 0 -> 4 bytes .../kafka/internal/db/rs3/RS3TableFactory.java | 2 +- .../kafka/internal/db/rs3/RS3TableFactoryTest.java | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 kafka-client/.jqwik-database diff --git a/kafka-client/.jqwik-database b/kafka-client/.jqwik-database new file mode 100644 index 0000000000000000000000000000000000000000..711006c3d3b5c6d50049e3f48311f3dbe372803d GIT binary patch literal 4 LcmZ4UmVp%j1%Lsc literal 0 HcmV?d00001 diff --git a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3TableFactory.java b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3TableFactory.java index 50d2dc440..6bc592782 100644 --- a/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3TableFactory.java +++ b/kafka-client/src/main/java/dev/responsive/kafka/internal/db/rs3/RS3TableFactory.java @@ -91,7 +91,7 @@ public RemoteWindowTable windowTable( final UUID storeId = createdStores.computeIfAbsent(storeName, n -> createStore( storeName, CreateStoreTypes.StoreType.WINDOW, - Optional.of(ClockType.STREAM_TIME), + Optional.of(ClockType.WALL_CLOCK), Optional.of(defaultTtl), computeNumKafkaPartitions.get(), rs3Client diff --git a/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/RS3TableFactoryTest.java b/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/RS3TableFactoryTest.java index 404c1c9f1..12e6caf63 100644 --- a/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/RS3TableFactoryTest.java +++ b/kafka-client/src/test/java/dev/responsive/kafka/internal/db/rs3/RS3TableFactoryTest.java @@ -123,7 +123,7 @@ public void testWindowTableMapping() { final var expectedOptions = new CreateStoreOptions( partitions, CreateStoreTypes.StoreType.WINDOW, - Optional.of(CreateStoreTypes.ClockType.STREAM_TIME), + Optional.of(CreateStoreTypes.ClockType.WALL_CLOCK), Optional.of(defaultTtl.toMillis()), Optional.empty() );