From 0ec127b9149848589308000be1e372159549bc66 Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Mon, 17 Feb 2025 11:02:10 -0800 Subject: [PATCH 1/9] save --- .../query/FDBStreamAggregationTest.java | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBStreamAggregationTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBStreamAggregationTest.java index 85a1a16e46..ca19bead14 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBStreamAggregationTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBStreamAggregationTest.java @@ -22,6 +22,8 @@ import com.apple.foundationdb.record.EvaluationContext; import com.apple.foundationdb.record.ExecuteProperties; +import com.apple.foundationdb.record.RecordCursor; +import com.apple.foundationdb.record.RecordCursorResult; import com.apple.foundationdb.record.RecordMetaData; import com.apple.foundationdb.record.TestRecords1Proto; import com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext; @@ -54,6 +56,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.LinkedList; import java.util.List; import java.util.function.BiConsumer; import java.util.function.Function; @@ -280,6 +283,47 @@ void aggregateNoRecordsNoGroupNoAggregate(final boolean useNestedResult) { } } + @ParameterizedTest(name = "[{displayName}-{index}] {0}") + @BooleanSource + void test(final boolean useNestedResult) { + try (final var context = openContext()) { + openSimpleRecordStore(context, NO_HOOK); + + final var plan = + new AggregationPlanBuilder(recordStore.getRecordMetaData(), "MySimpleRecord") + .withAggregateValue("num_value_2", value -> new NumericAggregationValue.Sum(NumericAggregationValue.PhysicalOperator.SUM_I, value)) + .withGroupCriterion("num_value_3_indexed") + .withGroupCriterion("str_value_indexed") + .build(useNestedResult); + + final var result = executePlanWithRecordScanLimit(plan, 2); + assertResults(useNestedResult ? this::assertResultNested : this::assertResultFlattened, result, resultOf(0, "0", 1), resultOf(1, "0", 2), resultOf(1, "1", 3), resultOf(2, "1", 9)); + } + } + + private List executePlanWithRecordScanLimit(final RecordQueryPlan plan, final int recordScanLimit) { + byte[] continuation = null; + List queryResults = new LinkedList<>(); + while (true) { + RecordCursor currentCursor = executePlan(plan, 0, recordScanLimit, continuation); + RecordCursorResult currentCursorResult; + while (true) { + currentCursorResult = currentCursor.getNext(); + continuation = currentCursorResult.getContinuation().toBytes(); + if (!currentCursorResult.hasNext()) { + break; + } + queryResults.add(currentCursorResult.get()); + System.out.println("current result:" + currentCursorResult.get().getMessage()); + } + System.out.println("getNoNextReson:" + currentCursorResult.getNoNextReason()); + if (currentCursorResult.getNoNextReason() == RecordCursor.NoNextReason.SOURCE_EXHAUSTED) { + break; + } + } + return queryResults; + } + private void populateDB(final int numRecords) throws Exception { try (FDBRecordContext context = openContext()) { openSimpleRecordStore(context); @@ -307,6 +351,17 @@ private List executePlan(final RecordQueryPlan plan) { } } + @Nonnull + private List executePlan(final RecordQueryPlan plan) { + final var types = plan.getDynamicTypes(); + final var typeRepository = TypeRepository.newBuilder().addAllTypes(types).build(); + try { + return plan.executePlan(recordStore, EvaluationContext.forTypeRepository(typeRepository), null, ExecuteProperties.SERIAL_EXECUTE).asList().get(); + } catch (final Throwable t) { + throw Assertions.fail(t); + } + } + private void assertResults(@Nonnull final BiConsumer> checkConsumer, @Nonnull final List actual, @Nonnull final List... expected) { Assertions.assertEquals(expected.length, actual.size()); for (var i = 0 ; i < actual.size() ; i++) { From 1ae9d4d34ccaa9e3fa026d70ce99515367c753ab Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Mon, 17 Feb 2025 19:44:57 -0800 Subject: [PATCH 2/9] scan limit not working? --- .../cursors/aggregate/AggregateCursor.java | 10 ++- .../query/FDBStreamAggregationTest.java | 72 +++++++++++++------ 2 files changed, 59 insertions(+), 23 deletions(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/cursors/aggregate/AggregateCursor.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/cursors/aggregate/AggregateCursor.java index 64c9fc0617..ea5e61f6be 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/cursors/aggregate/AggregateCursor.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/cursors/aggregate/AggregateCursor.java @@ -24,6 +24,7 @@ import com.apple.foundationdb.async.AsyncUtil; import com.apple.foundationdb.record.RecordCursor; import com.apple.foundationdb.record.RecordCursorContinuation; +import com.apple.foundationdb.record.RecordCursorEndContinuation; import com.apple.foundationdb.record.RecordCursorResult; import com.apple.foundationdb.record.RecordCursorStartContinuation; import com.apple.foundationdb.record.RecordCursorVisitor; @@ -71,7 +72,9 @@ public AggregateCursor(@Nonnull RecordCursor inner, public CompletableFuture> onNext() { if (previousResult != null && !previousResult.hasNext()) { // we are done - return CompletableFuture.completedFuture(RecordCursorResult.exhausted()); + return CompletableFuture.completedFuture(RecordCursorResult.withoutNextValue(previousResult.getContinuation(), + previousResult.getNoNextReason())); + // return CompletableFuture.completedFuture(RecordCursorResult.exhausted()); } return AsyncUtil.whileTrue(() -> inner.onNext().thenApply(innerResult -> { @@ -84,7 +87,9 @@ public CompletableFuture> onNext() { } else { final QueryResult queryResult = Objects.requireNonNull(innerResult.get()); boolean groupBreak = streamGrouping.apply(queryResult); - previousValidResult = innerResult; + if (!groupBreak) { + previousValidResult = innerResult; + } return (!groupBreak); } }), getExecutor()).thenApply(vignore -> { @@ -98,6 +103,7 @@ public CompletableFuture> onNext() { } // Use the last valid result for the continuation as we need non-terminal one here. RecordCursorContinuation continuation = Verify.verifyNotNull(previousValidResult).getContinuation(); + previousValidResult = previousResult; return RecordCursorResult.withNextValue(QueryResult.ofComputed(streamGrouping.getCompletedGroupResult()), continuation); }); } diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBStreamAggregationTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBStreamAggregationTest.java index ca19bead14..d3700d09d4 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBStreamAggregationTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBStreamAggregationTest.java @@ -22,9 +22,11 @@ import com.apple.foundationdb.record.EvaluationContext; import com.apple.foundationdb.record.ExecuteProperties; +import com.apple.foundationdb.record.ExecuteState; import com.apple.foundationdb.record.RecordCursor; import com.apple.foundationdb.record.RecordCursorResult; import com.apple.foundationdb.record.RecordMetaData; +import com.apple.foundationdb.record.RecordScanLimiterFactory; import com.apple.foundationdb.record.TestRecords1Proto; import com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext; import com.apple.foundationdb.record.query.plan.ScanComparisons; @@ -292,36 +294,56 @@ void test(final boolean useNestedResult) { final var plan = new AggregationPlanBuilder(recordStore.getRecordMetaData(), "MySimpleRecord") .withAggregateValue("num_value_2", value -> new NumericAggregationValue.Sum(NumericAggregationValue.PhysicalOperator.SUM_I, value)) - .withGroupCriterion("num_value_3_indexed") .withGroupCriterion("str_value_indexed") .build(useNestedResult); - final var result = executePlanWithRecordScanLimit(plan, 2); - assertResults(useNestedResult ? this::assertResultNested : this::assertResultFlattened, result, resultOf(0, "0", 1), resultOf(1, "0", 2), resultOf(1, "1", 3), resultOf(2, "1", 9)); + /* + num_value_2 str_value_indexed + 0 "0" + 1 "0" + 2 "0" + */ + /* + RecordCursorResult result1 = executePlanWithRecordScanLimit(plan, 1, null, useNestedResult, + List.of(resultOf("0", 0))); + RecordCursorResult result2 = executePlanWithRecordScanLimit(plan, 2, null, useNestedResult, + List.of(resultOf("0", 1))); + // only scanned 2 rows? + RecordCursorResult result3 = executePlanWithRecordScanLimit(plan, 3, null, useNestedResult, + List.of(resultOf("0", 1))); + + */ + // only scanned 3 rows? + RecordCursorResult result4 = executePlanWithRecordScanLimit(plan, 4, null, useNestedResult, + List.of(resultOf("0", 3))); + /* + RecordCursorResult result5 = executePlanWithRecordScanLimit(plan, 5, null, useNestedResult, + List.of(resultOf("0", 3), resultOf("1", 3))); + RecordCursorResult result6 = executePlanWithRecordScanLimit(plan, 6, null, useNestedResult, + List.of(resultOf("0", 3), resultOf("1", 7))); + RecordCursorResult result7 = executePlanWithRecordScanLimit(plan, 7, null, useNestedResult, + List.of(resultOf("0", 3), resultOf("1", 12))); + + */ } } - private List executePlanWithRecordScanLimit(final RecordQueryPlan plan, final int recordScanLimit) { - byte[] continuation = null; - List queryResults = new LinkedList<>(); + private RecordCursorResult executePlanWithRecordScanLimit(final RecordQueryPlan plan, final int recordScanLimit, byte[] continuation, final boolean useNestedResult, List expectedResult) { + RecordCursorResult currentCursorResult; + List currentQueryResults = new LinkedList<>(); while (true) { RecordCursor currentCursor = executePlan(plan, 0, recordScanLimit, continuation); - RecordCursorResult currentCursorResult; - while (true) { - currentCursorResult = currentCursor.getNext(); - continuation = currentCursorResult.getContinuation().toBytes(); - if (!currentCursorResult.hasNext()) { - break; - } - queryResults.add(currentCursorResult.get()); - System.out.println("current result:" + currentCursorResult.get().getMessage()); - } - System.out.println("getNoNextReson:" + currentCursorResult.getNoNextReason()); - if (currentCursorResult.getNoNextReason() == RecordCursor.NoNextReason.SOURCE_EXHAUSTED) { + currentCursorResult = currentCursor.getNext(); + continuation = currentCursorResult.getContinuation().toBytes(); + if (!currentCursorResult.hasNext()) { break; } + currentQueryResults.add(currentCursorResult.get()); + System.out.println("current result:" + currentCursorResult.get().getMessage()); } - return queryResults; + assertResults(useNestedResult ? this::assertResultNested : this::assertResultFlattened, currentQueryResults, expectedResult.toArray(new List[0])); + System.out.println("getNoNextReson:" + currentCursorResult.getNoNextReason()); + return currentCursorResult; } private void populateDB(final int numRecords) throws Exception { @@ -341,11 +363,19 @@ private void populateDB(final int numRecords) throws Exception { } @Nonnull - private List executePlan(final RecordQueryPlan plan) { + private RecordCursor executePlan(final RecordQueryPlan plan, final int rowLimit, final int recordScanLimit, final byte[] continuation) { final var types = plan.getDynamicTypes(); final var typeRepository = TypeRepository.newBuilder().addAllTypes(types).build(); + ExecuteState executeState; + if (recordScanLimit > 0) { + executeState = new ExecuteState(RecordScanLimiterFactory.enforce(recordScanLimit), null); + } else { + executeState = ExecuteState.NO_LIMITS; + } + ExecuteProperties executeProperties = ExecuteProperties.SERIAL_EXECUTE; + executeProperties = executeProperties.setReturnedRowLimit(rowLimit).setState(executeState); try { - return plan.executePlan(recordStore, EvaluationContext.forTypeRepository(typeRepository), null, ExecuteProperties.SERIAL_EXECUTE).asList().get(); + return plan.executePlan(recordStore, EvaluationContext.forTypeRepository(typeRepository), continuation, executeProperties); } catch (final Throwable t) { throw Assertions.fail(t); } From 3a153b08acd5ed1b73cc9eb2571461e5872caab4 Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Tue, 18 Feb 2025 16:33:09 -0800 Subject: [PATCH 3/9] test work --- .../cursors/aggregate/AggregateCursor.java | 50 ++++-- .../RecordQueryStreamingAggregationPlan.java | 2 +- .../query/FDBStreamAggregationTest.java | 149 +++++++++++++----- 3 files changed, 145 insertions(+), 56 deletions(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/cursors/aggregate/AggregateCursor.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/cursors/aggregate/AggregateCursor.java index ea5e61f6be..eca4e00b21 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/cursors/aggregate/AggregateCursor.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/cursors/aggregate/AggregateCursor.java @@ -22,12 +22,14 @@ import com.apple.foundationdb.annotation.API; import com.apple.foundationdb.async.AsyncUtil; +import com.apple.foundationdb.record.ByteArrayContinuation; import com.apple.foundationdb.record.RecordCursor; import com.apple.foundationdb.record.RecordCursorContinuation; import com.apple.foundationdb.record.RecordCursorEndContinuation; import com.apple.foundationdb.record.RecordCursorResult; import com.apple.foundationdb.record.RecordCursorStartContinuation; import com.apple.foundationdb.record.RecordCursorVisitor; +import com.apple.foundationdb.record.provider.foundationdb.KeyValueCursorBase; import com.apple.foundationdb.record.query.plan.plans.QueryResult; import com.google.common.base.Verify; import com.google.protobuf.Message; @@ -58,13 +60,19 @@ public class AggregateCursor implements RecordCursor previousValidResult; + // last row in last group, is null if the current group is the first group + @Nullable + private RecordCursorResult lastInLastGroup; + byte[] continuation; public AggregateCursor(@Nonnull RecordCursor inner, @Nonnull final StreamGrouping streamGrouping, - boolean isCreateDefaultOnEmpty) { + boolean isCreateDefaultOnEmpty, + byte[] continuation) { this.inner = inner; this.streamGrouping = streamGrouping; this.isCreateDefaultOnEmpty = isCreateDefaultOnEmpty; + this.continuation = continuation; } @Nonnull @@ -74,37 +82,55 @@ public CompletableFuture> onNext() { // we are done return CompletableFuture.completedFuture(RecordCursorResult.withoutNextValue(previousResult.getContinuation(), previousResult.getNoNextReason())); - // return CompletableFuture.completedFuture(RecordCursorResult.exhausted()); } return AsyncUtil.whileTrue(() -> inner.onNext().thenApply(innerResult -> { previousResult = innerResult; if (!innerResult.hasNext()) { if (!isNoRecords() || (isCreateDefaultOnEmpty && streamGrouping.isResultOnEmpty())) { + // the method streamGrouping.finalizeGroup() computes previousCompleteResult and resets the accumulator streamGrouping.finalizeGroup(); } return false; } else { final QueryResult queryResult = Objects.requireNonNull(innerResult.get()); boolean groupBreak = streamGrouping.apply(queryResult); - if (!groupBreak) { + if (groupBreak) { + lastInLastGroup = previousValidResult; + } else { previousValidResult = innerResult; } return (!groupBreak); } }), getExecutor()).thenApply(vignore -> { - if (isNoRecords()) { - // Edge case where there are no records at all - if (isCreateDefaultOnEmpty && streamGrouping.isResultOnEmpty()) { - return RecordCursorResult.withNextValue(QueryResult.ofComputed(streamGrouping.getCompletedGroupResult()), RecordCursorStartContinuation.START); + // either innerResult.hasNext() = false; or groupBreak = true + if (Verify.verifyNotNull(previousResult).hasNext()) { + // in this case groupBreak = true, return aggregated result and continuation + RecordCursorContinuation continuation = Verify.verifyNotNull(previousValidResult).getContinuation(); + previousValidResult = previousResult; + return RecordCursorResult.withNextValue(QueryResult.ofComputed(streamGrouping.getCompletedGroupResult()), continuation); + } else { + if (Verify.verifyNotNull(previousResult).getNoNextReason() == NoNextReason.SOURCE_EXHAUSTED) { + if (previousValidResult == null) { + return RecordCursorResult.exhausted(); + } else { + RecordCursorContinuation continuation =previousValidResult.getContinuation(); + previousValidResult = previousResult; + return RecordCursorResult.withNextValue(QueryResult.ofComputed(streamGrouping.getCompletedGroupResult()), continuation); + } } else { - return RecordCursorResult.exhausted(); + RecordCursorContinuation currentContinuation; + // in the current scan, if current group is the first group, set the continuation to the start of the current scan + // otherwise set the continuation to the last row in the last group + if (lastInLastGroup == null) { + currentContinuation = continuation == null ? RecordCursorStartContinuation.START : ByteArrayContinuation.fromNullable(continuation); + } else { + currentContinuation = lastInLastGroup.getContinuation(); + } + previousValidResult = lastInLastGroup; + return RecordCursorResult.withoutNextValue(currentContinuation, Verify.verifyNotNull(previousResult).getNoNextReason()); } } - // Use the last valid result for the continuation as we need non-terminal one here. - RecordCursorContinuation continuation = Verify.verifyNotNull(previousValidResult).getContinuation(); - previousValidResult = previousResult; - return RecordCursorResult.withNextValue(QueryResult.ofComputed(streamGrouping.getCompletedGroupResult()), continuation); }); } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryStreamingAggregationPlan.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryStreamingAggregationPlan.java index 4342b4b65d..323be325ef 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryStreamingAggregationPlan.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryStreamingAggregationPlan.java @@ -148,7 +148,7 @@ public RecordCursor executePlan(@Nonnull FDBRec (FDBRecordStoreBase)store, context, inner.getAlias()); - return new AggregateCursor<>(innerCursor, streamGrouping, isCreateDefaultOnEmpty) + return new AggregateCursor<>(innerCursor, streamGrouping, isCreateDefaultOnEmpty, continuation) .skipThenLimit(executeProperties.getSkip(), executeProperties.getReturnedRowLimit()); } diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBStreamAggregationTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBStreamAggregationTest.java index d3700d09d4..8254431489 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBStreamAggregationTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBStreamAggregationTest.java @@ -20,11 +20,15 @@ package com.apple.foundationdb.record.provider.foundationdb.query; +import com.apple.foundationdb.record.ByteScanLimiterFactory; import com.apple.foundationdb.record.EvaluationContext; import com.apple.foundationdb.record.ExecuteProperties; import com.apple.foundationdb.record.ExecuteState; import com.apple.foundationdb.record.RecordCursor; +import com.apple.foundationdb.record.RecordCursorContinuation; +import com.apple.foundationdb.record.RecordCursorEndContinuation; import com.apple.foundationdb.record.RecordCursorResult; +import com.apple.foundationdb.record.RecordCursorStartContinuation; import com.apple.foundationdb.record.RecordMetaData; import com.apple.foundationdb.record.RecordScanLimiterFactory; import com.apple.foundationdb.record.TestRecords1Proto; @@ -52,9 +56,13 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import javax.annotation.Nonnull; +import javax.annotation.Nullable; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -62,6 +70,7 @@ import java.util.List; import java.util.function.BiConsumer; import java.util.function.Function; +import java.util.stream.Stream; /** * Tests related to planning and executing queries with string collation. @@ -218,6 +227,26 @@ void aggregateThreeGroupByTwo(final boolean useNestedResult) { } } + @ParameterizedTest(name = "[{displayName}-{index}] {0}") + @MethodSource("provideArguments") + void aggregateOneGroupByThree(final boolean useNestedResult, final int rowLimit) { + // each group only has one row + try (final var context = openContext()) { + openSimpleRecordStore(context, NO_HOOK); + + final var plan = + new AggregationPlanBuilder(recordStore.getRecordMetaData(), "MySimpleRecord") + .withAggregateValue("num_value_2", value -> new NumericAggregationValue.Sum(NumericAggregationValue.PhysicalOperator.SUM_I, value)) + .withGroupCriterion("num_value_3_indexed") + .withGroupCriterion("str_value_indexed") + .withGroupCriterion("num_value_unique") + .build(useNestedResult); + + final var result = executePlanWithRowLimit(plan, rowLimit); + assertResults(useNestedResult ? this::assertResultNested : this::assertResultFlattened, result, resultOf(0, "0", 0, 0), resultOf(0, "0", 1, 1), resultOf(1, "0", 2, 2), resultOf(1, "1", 3, 3), resultOf(2, "1", 4, 4), resultOf(2, "1", 5, 5)); + } + } + @ParameterizedTest(name = "[{displayName}-{index}] {0}") @BooleanSource void aggregateNoRecords(final boolean useNestedResult) { @@ -285,9 +314,8 @@ void aggregateNoRecordsNoGroupNoAggregate(final boolean useNestedResult) { } } - @ParameterizedTest(name = "[{displayName}-{index}] {0}") - @BooleanSource - void test(final boolean useNestedResult) { + @Test + void aggregateHitScanLimitReached() { try (final var context = openContext()) { openSimpleRecordStore(context, NO_HOOK); @@ -295,55 +323,55 @@ void test(final boolean useNestedResult) { new AggregationPlanBuilder(recordStore.getRecordMetaData(), "MySimpleRecord") .withAggregateValue("num_value_2", value -> new NumericAggregationValue.Sum(NumericAggregationValue.PhysicalOperator.SUM_I, value)) .withGroupCriterion("str_value_indexed") - .build(useNestedResult); - - /* - num_value_2 str_value_indexed - 0 "0" - 1 "0" - 2 "0" - */ - /* - RecordCursorResult result1 = executePlanWithRecordScanLimit(plan, 1, null, useNestedResult, - List.of(resultOf("0", 0))); - RecordCursorResult result2 = executePlanWithRecordScanLimit(plan, 2, null, useNestedResult, - List.of(resultOf("0", 1))); - // only scanned 2 rows? - RecordCursorResult result3 = executePlanWithRecordScanLimit(plan, 3, null, useNestedResult, - List.of(resultOf("0", 1))); - - */ - // only scanned 3 rows? - RecordCursorResult result4 = executePlanWithRecordScanLimit(plan, 4, null, useNestedResult, - List.of(resultOf("0", 3))); - /* - RecordCursorResult result5 = executePlanWithRecordScanLimit(plan, 5, null, useNestedResult, - List.of(resultOf("0", 3), resultOf("1", 3))); - RecordCursorResult result6 = executePlanWithRecordScanLimit(plan, 6, null, useNestedResult, - List.of(resultOf("0", 3), resultOf("1", 7))); - RecordCursorResult result7 = executePlanWithRecordScanLimit(plan, 7, null, useNestedResult, - List.of(resultOf("0", 3), resultOf("1", 12))); - - */ + .build(false); + + // In the testing data, there are 2 groups, each group has 3 rows. + // recordScanLimit = 5: scans 3 rows, and the 4th scan hits SCAN_LIMIT_REACHED + // although the first group contains exactly 3 rows, we don't know we've finished the first group before we get to the 4th row, so nothing is returned, continuation is back to START + RecordCursorContinuation continuation1 = executePlanWithRecordScanLimit(plan, 5, null, null); + Assertions.assertEquals(RecordCursorStartContinuation.START, continuation1); + // recordScanLimit = 6: scans 4 rows, and the 5th scan hits SCAN_LIMIT_REACHED, we know that we've finished the 1st group, aggregated result is returned + RecordCursorContinuation continuation2 = executePlanWithRecordScanLimit(plan, 6, continuation1.toBytes(), resultOf("0", 3)); + // continue with recordScanLimit = 5, scans 3 rows and hits SCAN_LIMIT_REACHED + // again, we don't know that we've finished the 2nd group, nothing is returned, continuation is back to where the scan starts + RecordCursorContinuation continuation3 = executePlanWithRecordScanLimit(plan, 5, continuation2.toBytes(), null); + Assertions.assertTrue(Arrays.equals(continuation2.toBytes(), continuation3.toBytes())); + // finish the 2nd group, aggregated result is returned, exhausted the source + RecordCursorContinuation continuation4 = executePlanWithRecordScanLimit(plan, 6, continuation3.toBytes(), resultOf("1", 12)); + Assertions.assertEquals(RecordCursorEndContinuation.END, continuation4); } } - private RecordCursorResult executePlanWithRecordScanLimit(final RecordQueryPlan plan, final int recordScanLimit, byte[] continuation, final boolean useNestedResult, List expectedResult) { + private RecordCursorContinuation executePlanWithRecordScanLimit(final RecordQueryPlan plan, final int recordScanLimit, byte[] continuation, @Nullable List expectedResult) { + List queryResults = new LinkedList<>(); + RecordCursor currentCursor = executePlan(plan, 0, recordScanLimit, continuation); RecordCursorResult currentCursorResult; - List currentQueryResults = new LinkedList<>(); + RecordCursorContinuation cursorContinuation; while (true) { - RecordCursor currentCursor = executePlan(plan, 0, recordScanLimit, continuation); currentCursorResult = currentCursor.getNext(); - continuation = currentCursorResult.getContinuation().toBytes(); + cursorContinuation = currentCursorResult.getContinuation(); if (!currentCursorResult.hasNext()) { break; } - currentQueryResults.add(currentCursorResult.get()); - System.out.println("current result:" + currentCursorResult.get().getMessage()); + queryResults.add(currentCursorResult.get()); } - assertResults(useNestedResult ? this::assertResultNested : this::assertResultFlattened, currentQueryResults, expectedResult.toArray(new List[0])); - System.out.println("getNoNextReson:" + currentCursorResult.getNoNextReason()); - return currentCursorResult; + if (expectedResult == null) { + Assertions.assertTrue(queryResults.isEmpty()); + } else { + assertResults(this::assertResultFlattened, queryResults, expectedResult); + } + return cursorContinuation; + } + + private static Stream provideArguments() { + // (boolean, rowLimit) + // setting rowLimit = 0 is equivalent to no limit + List arguments = new LinkedList<>(); + for (int i = 0; i <= 4; i++) { + arguments.add(Arguments.of(false, i)); + arguments.add(Arguments.of(true, i)); + } + return arguments.stream(); } private void populateDB(final int numRecords) throws Exception { @@ -356,6 +384,7 @@ private void populateDB(final int numRecords) throws Exception { recBuilder.setNumValue2(i); recBuilder.setNumValue3Indexed(i / 2); // some field that changes every 2nd record recBuilder.setStrValueIndexed(Integer.toString(i / 3)); // some field that changes every 3rd record + recBuilder.setNumValueUnique(i); recordStore.saveRecord(recBuilder.build()); } commit(context); @@ -368,7 +397,7 @@ private RecordCursor executePlan(final RecordQueryPlan plan, final final var typeRepository = TypeRepository.newBuilder().addAllTypes(types).build(); ExecuteState executeState; if (recordScanLimit > 0) { - executeState = new ExecuteState(RecordScanLimiterFactory.enforce(recordScanLimit), null); + executeState = new ExecuteState(RecordScanLimiterFactory.enforce(recordScanLimit), ByteScanLimiterFactory.tracking()); } else { executeState = ExecuteState.NO_LIMITS; } @@ -392,6 +421,40 @@ private List executePlan(final RecordQueryPlan plan) { } } + @Nonnull + private RecordCursor executePlan(final RecordQueryPlan plan, final int rowLimit, final byte[] continuation) { + final var types = plan.getDynamicTypes(); + final var typeRepository = TypeRepository.newBuilder().addAllTypes(types).build(); + ExecuteProperties executeProperties = ExecuteProperties.SERIAL_EXECUTE; + executeProperties = executeProperties.setReturnedRowLimit(rowLimit); + try { + return plan.executePlan(recordStore, EvaluationContext.forTypeRepository(typeRepository), continuation, executeProperties); + } catch (final Throwable t) { + throw Assertions.fail(t); + } + } + + private List executePlanWithRowLimit(final RecordQueryPlan plan, final int rowLimit) { + byte[] continuation = null; + List queryResults = new LinkedList<>(); + while (true) { + RecordCursor currentCursor = executePlan(plan, rowLimit, continuation); + RecordCursorResult currentCursorResult; + while (true) { + currentCursorResult = currentCursor.getNext(); + continuation = currentCursorResult.getContinuation().toBytes(); + if (!currentCursorResult.hasNext()) { + break; + } + queryResults.add(currentCursorResult.get()); + } + if (currentCursorResult.getNoNextReason() == RecordCursor.NoNextReason.SOURCE_EXHAUSTED) { + break; + } + } + return queryResults; + } + private void assertResults(@Nonnull final BiConsumer> checkConsumer, @Nonnull final List actual, @Nonnull final List... expected) { Assertions.assertEquals(expected.length, actual.size()); for (var i = 0 ; i < actual.size() ; i++) { From 347e151a3b3e14db8a3f0641fc8c4aa17afb6b28 Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Wed, 19 Feb 2025 08:19:16 -0800 Subject: [PATCH 4/9] pmd --- .../foundationdb/record/cursors/aggregate/AggregateCursor.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/cursors/aggregate/AggregateCursor.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/cursors/aggregate/AggregateCursor.java index eca4e00b21..bfc3265cbf 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/cursors/aggregate/AggregateCursor.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/cursors/aggregate/AggregateCursor.java @@ -25,7 +25,6 @@ import com.apple.foundationdb.record.ByteArrayContinuation; import com.apple.foundationdb.record.RecordCursor; import com.apple.foundationdb.record.RecordCursorContinuation; -import com.apple.foundationdb.record.RecordCursorEndContinuation; import com.apple.foundationdb.record.RecordCursorResult; import com.apple.foundationdb.record.RecordCursorStartContinuation; import com.apple.foundationdb.record.RecordCursorVisitor; @@ -114,7 +113,7 @@ public CompletableFuture> onNext() { if (previousValidResult == null) { return RecordCursorResult.exhausted(); } else { - RecordCursorContinuation continuation =previousValidResult.getContinuation(); + RecordCursorContinuation continuation = previousValidResult.getContinuation(); previousValidResult = previousResult; return RecordCursorResult.withNextValue(QueryResult.ofComputed(streamGrouping.getCompletedGroupResult()), continuation); } From 31da8933f6c8e490b5dccc31a8f67f42fc53b6f0 Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Wed, 19 Feb 2025 13:40:19 -0800 Subject: [PATCH 5/9] release notes --- docs/ReleaseNotes.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/ReleaseNotes.md b/docs/ReleaseNotes.md index cd49e73f04..a5f026caaf 100644 --- a/docs/ReleaseNotes.md +++ b/docs/ReleaseNotes.md @@ -23,6 +23,7 @@ Users performing online updates are encouraged to update from [4.0.559.4](#40559 * **Bug fix** Fix 2 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN) * **Bug fix** Fix 3 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN) * **Bug fix** Fix continuation bug in AggregateCursor when a group is finished [Issue #3172](https://github.com/FoundationDB/fdb-record-layer/issues/3172) +* **Bug fix** Fix incorrect result in AggregateCursor when a scan is stopped in the middle of a group [Issue #3180](https://github.com/FoundationDB/fdb-record-layer/issues/3180) * **Performance** Improvement 1 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN) * **Performance** Improvement 2 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN) * **Performance** Improvement 3 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN) From 58a9aeb1e0cdfa448aa74ad5e9d20f3beae62799 Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Wed, 19 Feb 2025 14:28:32 -0800 Subject: [PATCH 6/9] check style --- .../provider/foundationdb/query/FDBStreamAggregationTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBStreamAggregationTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBStreamAggregationTest.java index 139b8181ff..98fda6dba1 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBStreamAggregationTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBStreamAggregationTest.java @@ -430,7 +430,7 @@ private List executePlanWithRowLimit(final RecordQueryPlan plan, fi return queryResults; } - private void assertResults(@Nonnull final BiConsumer> checkConsumer, @Nonnull final List actual, @Nonnull final List...expected){ + private void assertResults(@Nonnull final BiConsumer> checkConsumer, @Nonnull final List actual, @Nonnull final List... expected) { Assertions.assertEquals(expected.length, actual.size()); for (var i = 0; i < actual.size(); i++) { checkConsumer.accept(actual.get(i), expected[i]); @@ -484,7 +484,7 @@ private void assertResultNested(final QueryResult actual, final List expected } } - private List resultOf(final Object...objects) { + private List resultOf(final Object... objects) { return Arrays.asList(objects); } From 14c8cfac1483d4b20806092a6cf40d9f76c90b07 Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Tue, 25 Feb 2025 17:19:10 -0800 Subject: [PATCH 7/9] revert releasenote change --- docs/sphinx/source/ReleaseNotes.md | 48 +++++++++---------- .../cursors/aggregate/AggregateCursor.java | 3 +- 2 files changed, 26 insertions(+), 25 deletions(-) diff --git a/docs/sphinx/source/ReleaseNotes.md b/docs/sphinx/source/ReleaseNotes.md index a2ffccbe56..8351045a9a 100644 --- a/docs/sphinx/source/ReleaseNotes.md +++ b/docs/sphinx/source/ReleaseNotes.md @@ -20,7 +20,7 @@ Users performing online updates are encouraged to update from [4.0.559.4](#40559

New Features

* FRL respects PLAN_CACHE_*_MAX_ENTRIES - [PR #3156](https://github.com/FoundationDB/fdb-record-layer/pull/3156) -

Bug Fixes

+

Bug Fixes

* Use `Locale.ROOT` in `String.format`, fixing messages if JVM has non-US default locale - [PR #3117](https://github.com/FoundationDB/fdb-record-layer/pull/3117) @@ -62,10 +62,10 @@ Mixed mode testing run against the following previous versions: * Support Lucene index scrubbing of missing entries - [PR #3009](https://github.com/FoundationDB/fdb-record-layer/pull/3009) * Add enum column support to relational server - [PR #3074](https://github.com/FoundationDB/fdb-record-layer/pull/3074) * Add validation to Insert statement parsing to match the column names with supplied values - [PR #3070](https://github.com/FoundationDB/fdb-record-layer/pull/3070) -

Bug Fixes

+

Bug Fixes

* Plan Hash Instability in recursive union query - [PR #3142](https://github.com/FoundationDB/fdb-record-layer/pull/3142) -

Dependency Updates

+

Dependency Updates

* Update grpc-commonProtos, assertj, and junit dependencies - [PR #3115](https://github.com/FoundationDB/fdb-record-layer/pull/3115) * Bump spotbugs gradle plugin from 4.6.1 to 6.1.3 - [PR #3104](https://github.com/FoundationDB/fdb-record-layer/pull/3104) @@ -107,13 +107,13 @@ Mixed mode testing run against the following previous versions:

New Features

* Plan generator now uses Cascades's `FunctionCatalog` - [PR #3061](https://github.com/FoundationDB/fdb-record-layer/pull/3061) -

Bug Fixes

+

Bug Fixes

* Exclude transitive maven information in shaded jar - [PR #3129](https://github.com/FoundationDB/fdb-record-layer/pull/3129) * Fix the publication of `fdb-record-layer-core-shaded` jars so that only the correct artifacts get uploaded - [PR #3125](https://github.com/FoundationDB/fdb-record-layer/pull/3125) * Set skip and limit for the recursive union cursor correctly - [PR #3111](https://github.com/FoundationDB/fdb-record-layer/pull/3111) * Fix infinite continuations when NULL_ON_EMPTY is in the plan - [PR #3092](https://github.com/FoundationDB/fdb-record-layer/pull/3092) -

Dependency Updates

+

Dependency Updates

* Upgrade the versions gradle plugin from 0.38.0 to 0.52.0 - [PR #3106](https://github.com/FoundationDB/fdb-record-layer/pull/3106) * Upgrade de.undercouch.download gradle plugin from 4.1.1 to 5.6.0 - [PR #3088](https://github.com/FoundationDB/fdb-record-layer/pull/3088) @@ -160,7 +160,7 @@ Our API stability annotations have been updated to reflect greater API instabili

Breaking Changes

* Only allow continuation to be returned after a result set has been exhausted - [PR #3038](https://github.com/FoundationDB/fdb-record-layer/pull/3038) -

Dependency Updates

+

Dependency Updates

* Upgrade git-version plugin to the latest version: 3.1.0 - [PR #3089](https://github.com/FoundationDB/fdb-record-layer/pull/3089) * Upgrade jmh plugin to latest version: 0.7.2 - [PR #3090](https://github.com/FoundationDB/fdb-record-layer/pull/3090) @@ -219,7 +219,7 @@ Our API stability annotations have been updated to reflect greater API instabili

New Features

* Support enums in relational operations - [PR #3012](https://github.com/FoundationDB/fdb-record-layer/pull/3012) -

Bug Fixes

+

Bug Fixes

* Break out a helper class from GenerateVisitor annotation processor - [PR #3060](https://github.com/FoundationDB/fdb-record-layer/pull/3060) @@ -287,7 +287,7 @@ Our API stability annotations have been updated to reflect greater API instabili * Add support for continuations to the relational server - [PR #3030](https://github.com/FoundationDB/fdb-record-layer/pull/3030) * Introduce SQL support to Recursive CTEs - [PR #3035](https://github.com/FoundationDB/fdb-record-layer/pull/3035) * Align aggregate query behaviour on empty tables more towards SQL standard - [PR #3029](https://github.com/FoundationDB/fdb-record-layer/pull/3029) -

Dependency Updates

+

Dependency Updates

* Bump black from 22.10.0 to 24.3.0 in /scripts - [PR #3024](https://github.com/FoundationDB/fdb-record-layer/pull/3024) @@ -382,7 +382,7 @@ Our API stability annotations have been updated to reflect greater API instabili ### Breaking Changes -The Apache Commons library has been removed as a dependency. There were a few locations where the `Pair` class from that library was exposed via the API. This has necessitated making API incompatible changes. These have mostly been replaced by classes defined in the repository, or with other JDK classes. +The Apache Commons library has been removed as a dependency. There were a few locations where the `Pair` class from that library was exposed via the API. This has necessitated making API incompatible changes. These have mostly been replaced by classes defined in the repository, or with other JDK classes. ### 3.5.556.0 @@ -405,10 +405,10 @@ Starting with version [3.4.455.0](#344550), the semantics of `UnnestedRecordType ### 3.4.554.0 * `DefaultTextTokenizer` (and `TextTokenizer`s that use it) now uses `Locale.ROOT` for tokenizing instead of the default - locale. This could have an impact if it's being used in an environment with a default locale that is not compatible with - `Locale.ROOT` for this purpose. For example if the default locale is `th`, Thai text may be tokenized differently with - this version. This also means that if you have TEXT indexes using one of these tokenizers, they will need to be rebuilt - with the new code. +locale. This could have an impact if it's being used in an environment with a default locale that is not compatible with +`Locale.ROOT` for this purpose. For example if the default locale is `th`, Thai text may be tokenized differently with +this version. This also means that if you have TEXT indexes using one of these tokenizers, they will need to be rebuilt +with the new code. * **Bug fix** DefaultTextTokenizer behaved differently depending on system locale [(Issue #2966)](https://github.com/FoundationDB/fdb-record-layer/issues/2966) @@ -503,7 +503,7 @@ Starting with version [3.4.455.0](#344550), the semantics of `UnnestedRecordType * **Bug fix** Expose `Tuple`-based `MIN_EVER` and `MAX_EVER` indexes to Cascades [(Issue #2874)](https://github.com/FoundationDB/fdb-record-layer/issues/2874) * **Bug fix** Log Repartitioned records after writing them [(Issue #2867)](https://github.com/FoundationDB/fdb-record-layer/issues/2867) * **Bug fix** Lucene merges: false no merges found [(Issue #2864)](https://github.com/FoundationDB/fdb-record-layer/issues/2864) -* **Feature** Deprecate special IndexingByRecords functions [(Issue #2259)](https://github.com/FoundationDB/fdb-record-layer/issues/2259) +* **Feature** Deprecate special IndexingByRecords functions [(Issue #2259)](https://github.com/FoundationDB/fdb-record-layer/issues/2259) ### 3.4.531.0 @@ -1455,12 +1455,12 @@ The Guava dependency version has been updated to 31.1. Projects may need to chec ### Features This version of the Record Layer allows the FDB API version to be configured through the `FDBDatabaseFactory`. This means that while this version allows the client to be configured to use 7.1 features, it also supports connecting to 6.3 FDB clusters if the API version is set appropriately. Note that setting the API version does restrict the set of potential FDB server versions that can be connected to, so this configuration change should only be made if the FDB server has already been updated. - -New index state "READABLE_UNIQUE_PENDING" - the proper way to roll this feature out is: + +New index state "READABLE_UNIQUE_PENDING" - the proper way to roll this feature out is: 1. The adopter should upgrade to the new Record Layer version and deploy the version everywhere. 2. The format version should be set READABLE_UNIQUE_PENDING_FORMAT_VERSION. 3. Only after all the possible clients are upgraded to support the new state, the adopter may set the allowPendingState on the indexing policy of new index builds. - An index may be in this new state if it is fully built, the unique flag is set, and duplications were found during online indexing. From the code point of view, it is defined as scannable but not readable. +An index may be in this new state if it is fully built, the unique flag is set, and duplications were found during online indexing. From the code point of view, it is defined as scannable but not readable. ### Breaking Changes @@ -1697,7 +1697,7 @@ This version of the Record Layer changes the Java source and target compatibilit ### 3.1.238.0 * **Feature** Support custom additional synonyms. This introduces a new SynonymMapRegistry. - New synonym maps should implement `SynonymMapConfig`. See example `EnglishSynonymMap`. +New synonym maps should implement `SynonymMapConfig`. See example `EnglishSynonymMap`. ### 3.1.237.0 @@ -1719,10 +1719,10 @@ This version of the Record Layer changes the Java source and target compatibilit * **Feature** Expose IndexQueryabilityFilter for Aggregate planning [(Issue #1520)](https://github.com/FoundationDB/fdb-record-layer/issues/1520) * **Breaking change** As part of [(Issue #1520)](https://github.com/FoundationDB/fdb-record-layer/issues/1520) implementers - of `FDBRecordStoreBase` need to implement a new overload of `getSnapshotRecordCountForRecordType` and `evaluateAggregateFunction` - that takes an `IndexQueryabilityFilter`. In addition some methods on `IndexFunctionHelper` and `ComposedBitmapIndexAggregate` - now take an `IndexQueryabilityFilter`; to preserve backwards compatibility, if all indexes are valid, - `IndexQueryabilityFilter.TRUE` can be used. +of `FDBRecordStoreBase` need to implement a new overload of `getSnapshotRecordCountForRecordType` and `evaluateAggregateFunction` +that takes an `IndexQueryabilityFilter`. In addition some methods on `IndexFunctionHelper` and `ComposedBitmapIndexAggregate` +now take an `IndexQueryabilityFilter`; to preserve backwards compatibility, if all indexes are valid, +`IndexQueryabilityFilter.TRUE` can be used. ### 3.1.231.0 @@ -1813,7 +1813,7 @@ Another, smaller change that has been made is that by default, new indexes added ### 3.0.210.0 * **Bug fix** relax conditions enforced in ImplementInUnionRule [(Issue #1369)](https://github.com/FoundationDB/fdb-record-layer/issues/1369) - *OnlineIndexScrubberTest.testScrubberLimits - reduce number of records [(Issue #1363)](https://github.com/FoundationDB/fdb-record-layer/issues/1363) +*OnlineIndexScrubberTest.testScrubberLimits - reduce number of records [(Issue #1363)](https://github.com/FoundationDB/fdb-record-layer/issues/1363) * OnlineIndexScrubber: clear ranges when exhausted [(Issue #1367)](https://github.com/FoundationDB/fdb-record-layer/issues/1367) * **Feature** Prototype non-index sorting [(Issue #1161)](https://github.com/FoundationDB/fdb-record-layer/issues/1161) @@ -2195,7 +2195,7 @@ This version of the Record Layer requires a FoundationDB server version of at le Constructors of the `RecordQueryUnionPlan` and `RecordQueryIntersectionPlan` have been marked as deprecated in favor of static initializers. This will allow for more flexibility as work on the new planner develops. -### Newly Deprecated +### Newly Deprecated The non-static `RecordCursor::flatMapPipelined()` method has been deprecated because it is easy to mis-use (by mistaken analogy to the `mapPipelined()` method) and cannot be used with continuations. See [Issue #665](https://github.com/FoundationDB/fdb-record-layer/issues/665) for further explanation. diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/cursors/aggregate/AggregateCursor.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/cursors/aggregate/AggregateCursor.java index 23a1ca9c3d..eef330b8bd 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/cursors/aggregate/AggregateCursor.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/cursors/aggregate/AggregateCursor.java @@ -61,12 +61,13 @@ public class AggregateCursor implements RecordCursor lastInLastGroup; + @Nullable byte[] continuation; public AggregateCursor(@Nonnull RecordCursor inner, @Nonnull final StreamGrouping streamGrouping, boolean isCreateDefaultOnEmpty, - byte[] continuation) { + @Nullable byte[] continuation) { this.inner = inner; this.streamGrouping = streamGrouping; this.isCreateDefaultOnEmpty = isCreateDefaultOnEmpty; From 41afb52f8ae126a7a34c5e46981af2a85e1200a2 Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Tue, 25 Feb 2025 17:21:49 -0800 Subject: [PATCH 8/9] more --- docs/sphinx/source/ReleaseNotes.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/sphinx/source/ReleaseNotes.md b/docs/sphinx/source/ReleaseNotes.md index 8351045a9a..d591a9cbf2 100644 --- a/docs/sphinx/source/ReleaseNotes.md +++ b/docs/sphinx/source/ReleaseNotes.md @@ -1460,7 +1460,7 @@ New index state "READABLE_UNIQUE_PENDING" - the proper way to roll this feature 1. The adopter should upgrade to the new Record Layer version and deploy the version everywhere. 2. The format version should be set READABLE_UNIQUE_PENDING_FORMAT_VERSION. 3. Only after all the possible clients are upgraded to support the new state, the adopter may set the allowPendingState on the indexing policy of new index builds. -An index may be in this new state if it is fully built, the unique flag is set, and duplications were found during online indexing. From the code point of view, it is defined as scannable but not readable. +An index may be in this new state if it is fully built, the unique flag is set, and duplications were found during online indexing. From the code point of view, it is defined as scannable but not readable. ### Breaking Changes @@ -2956,4 +2956,4 @@ The capability and reliability of text queries on more sophisticated indexes has ### 2.1.10.0 -* **Feature** A new record type key expression allows for structuring data in a record store more akin to how tables are stored in a traditional relational database [(Issue #27)](https://github.com/FoundationDB/fdb-record-layer/issues/27) \ No newline at end of file +* **Feature** A new record type key expression allows for structuring data in a record store more akin to how tables are stored in a traditional relational database [(Issue #27)](https://github.com/FoundationDB/fdb-record-layer/issues/27) \ No newline at end of file From a6ddf598cc5ceb58e8a2a448e4dfb97eac610c52 Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Tue, 25 Feb 2025 17:23:53 -0800 Subject: [PATCH 9/9] new line --- docs/sphinx/source/ReleaseNotes.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/sphinx/source/ReleaseNotes.md b/docs/sphinx/source/ReleaseNotes.md index d591a9cbf2..1beceefab0 100644 --- a/docs/sphinx/source/ReleaseNotes.md +++ b/docs/sphinx/source/ReleaseNotes.md @@ -1459,7 +1459,7 @@ This version of the Record Layer allows the FDB API version to be configured thr New index state "READABLE_UNIQUE_PENDING" - the proper way to roll this feature out is: 1. The adopter should upgrade to the new Record Layer version and deploy the version everywhere. 2. The format version should be set READABLE_UNIQUE_PENDING_FORMAT_VERSION. -3. Only after all the possible clients are upgraded to support the new state, the adopter may set the allowPendingState on the indexing policy of new index builds. +3. Only after all the possible clients are upgraded to support the new state, the adopter may set the allowPendingState on the indexing policy of new index builds. An index may be in this new state if it is fully built, the unique flag is set, and duplications were found during online indexing. From the code point of view, it is defined as scannable but not readable. @@ -2956,4 +2956,4 @@ The capability and reliability of text queries on more sophisticated indexes has ### 2.1.10.0 -* **Feature** A new record type key expression allows for structuring data in a record store more akin to how tables are stored in a traditional relational database [(Issue #27)](https://github.com/FoundationDB/fdb-record-layer/issues/27) \ No newline at end of file +* **Feature** A new record type key expression allows for structuring data in a record store more akin to how tables are stored in a traditional relational database [(Issue #27)](https://github.com/FoundationDB/fdb-record-layer/issues/27)