From b7cdd8019b45dee13ea4b1c771a2b9925d331d61 Mon Sep 17 00:00:00 2001 From: Josef Ezra Date: Wed, 18 Dec 2024 11:09:36 -0500 Subject: [PATCH 01/20] Resolve #3008: Support Lucene index scrubbing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To validate Lucene index validity, support "Report Only" scrubbing for: Dangling Lucene index entries: Iterate "all entries" (similar toLuceneScanAllEntriesTest), validate that all pointers lead to existing records. Missing Lucene index entries: iterate all records, validate that their primary keys are represented in the “primary key to Lucene segment” map, and that the Lucene segment exists --- docs/ReleaseNotes.md | 4 +- .../record/lucene/LuceneIndexMaintainer.java | 14 ++ .../LuceneIndexScrubbingToolsDangling.java | 144 ++++++++++++++++++ .../LuceneIndexScrubbingToolsMissing.java | 120 +++++++++++++++ 4 files changed, 280 insertions(+), 2 deletions(-) create mode 100644 fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneIndexScrubbingToolsDangling.java create mode 100644 fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneIndexScrubbingToolsMissing.java diff --git a/docs/ReleaseNotes.md b/docs/ReleaseNotes.md index d6cc6c4622..322bebceba 100644 --- a/docs/ReleaseNotes.md +++ b/docs/ReleaseNotes.md @@ -33,7 +33,7 @@ Users performing online updates are encouraged to update from [4.0.559.4](#40559 * **Feature** Add enum column support to relational server [(Issue #3073)](https://github.com/FoundationDB/fdb-record-layer/issues/3073) * **Feature** Feature 3 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN) * **Feature** Feature 4 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN) -* **Feature** Feature 5 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN) +* **Feature** Support Lucene index scrubbing [(Issue #3008)](https://github.com/FoundationDB/fdb-record-layer/issues/3008) * **Breaking change** Change 1 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN) * **Breaking change** Change 2 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN) * **Breaking change** Change 3 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN) @@ -58,7 +58,7 @@ Several "FDB relational" sub-projects have been added which present a new relati ### Breaking Changes -Our API stability annotations have been updated to reflect greater API instability. We have degraded existing `STABLE` and `MAINTAINED` APIs to `UNSTABLE`, and the `MAINTAINED` classification has been removed from the project. The new relational sub-projects' APIs are all `EXPERIMENTAL` (or `INTERNAL`). These APIs are expected to evolve in the future as more functionality is moved to the relational APIs. The API annotation class was moved to its own module, to avoid having the new sub-projects depend on the FDB java bindings. +The Apache Commons library has been removed as a dependency. There were a few locations where the `Pair` class from that library was exposed via the API. This has necessitated making API incompatible changes. These have mostly been replaced by classes defined in the repository, or with other JDK classes. ### 4.0.575.0 diff --git a/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneIndexMaintainer.java b/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneIndexMaintainer.java index 7fb25ac366..a22d2be0cf 100644 --- a/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneIndexMaintainer.java +++ b/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneIndexMaintainer.java @@ -58,6 +58,7 @@ import com.apple.foundationdb.record.provider.foundationdb.IndexOperation; import com.apple.foundationdb.record.provider.foundationdb.IndexOperationResult; import com.apple.foundationdb.record.provider.foundationdb.IndexScanBounds; +import com.apple.foundationdb.record.provider.foundationdb.IndexScrubbingTools; import com.apple.foundationdb.record.provider.foundationdb.indexes.InvalidIndexEntry; import com.apple.foundationdb.record.provider.foundationdb.indexes.StandardIndexMaintainer; import com.apple.foundationdb.record.query.QueryToKeyMatcher; @@ -750,4 +751,17 @@ private void logSerializationError(String format, Object ... arguments) { } } } + + @Nullable + @Override + public IndexScrubbingTools getIndexScrubbingTools(final IndexScrubbingTools.ScrubbingType type) { + switch (type) { + case MISSING: + return new LuceneIndexScrubbingToolsMissing(partitioner, state); + case DANGLING: + return new LuceneIndexScrubbingToolsDangling(partitioner, state); + default: + return null; + } + } } diff --git a/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneIndexScrubbingToolsDangling.java b/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneIndexScrubbingToolsDangling.java new file mode 100644 index 0000000000..48f710f32f --- /dev/null +++ b/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneIndexScrubbingToolsDangling.java @@ -0,0 +1,144 @@ +/* + * LuceneIndexScrubbingToolsDangling.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2024 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.lucene; + +import com.apple.foundationdb.record.EvaluationContext; +import com.apple.foundationdb.record.ExecuteProperties; +import com.apple.foundationdb.record.IndexEntry; +import com.apple.foundationdb.record.IsolationLevel; +import com.apple.foundationdb.record.RecordCursor; +import com.apple.foundationdb.record.RecordCursorResult; +import com.apple.foundationdb.record.ScanProperties; +import com.apple.foundationdb.record.TupleRange; +import com.apple.foundationdb.record.cursors.AutoContinuingCursor; +import com.apple.foundationdb.record.logging.KeyValueLogMessage; +import com.apple.foundationdb.record.logging.LogMessageKeys; +import com.apple.foundationdb.record.metadata.Index; +import com.apple.foundationdb.record.metadata.RecordType; +import com.apple.foundationdb.record.provider.foundationdb.FDBDatabaseRunner; +import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStore; +import com.apple.foundationdb.record.provider.foundationdb.FDBStoreTimer; +import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerState; +import com.apple.foundationdb.record.provider.foundationdb.IndexOrphanBehavior; +import com.apple.foundationdb.record.provider.foundationdb.IndexScrubbingTools; +import com.apple.foundationdb.record.query.plan.ScanComparisons; +import com.apple.foundationdb.tuple.Tuple; + +import javax.annotation.Nonnull; +import java.util.Collection; +import java.util.concurrent.CompletableFuture; + +/** + * Index Scrubbing Toolbox for a Lucene index maintainer. Scrub dangling value index entries - i.e. index entries + * pointing to non-existing record(s) + */ +public class LuceneIndexScrubbingToolsDangling implements IndexScrubbingTools { + Index index = null; + boolean isSynthetic = false; + @Nonnull + private final LucenePartitioner partitioner; // non-mutable + @Nonnull + private final IndexMaintainerState state; + + public LuceneIndexScrubbingToolsDangling(@Nonnull final LucenePartitioner partitioner, @Nonnull final IndexMaintainerState state) { + this.partitioner = partitioner; + this.state = state; + } + + @Override + public void presetCommonParams(final Index index, final boolean allowRepair, final boolean isSynthetic, final Collection types) { + this.index = index; + this.isSynthetic = isSynthetic; + } + + @Override + public RecordCursor getCursor(final TupleRange range, final FDBRecordStore store, final int limit) { + // TODO: Range tuple should begin with a null or [groupingKey, timestamp] + FDBDatabaseRunner runner = state.context.newRunner(); + final ScanProperties scanProperties = new ScanProperties(ExecuteProperties.newBuilder() + .setIsolationLevel(IsolationLevel.SERIALIZABLE) + .setReturnedRowLimit(limit) + .build()); + // TODO: start from continuation + LuceneQueryClause search = LuceneQuerySearchClause.MATCH_ALL_DOCS_QUERY; + LuceneScanParameters scan = new LuceneScanQueryParameters( + ScanComparisons.EMPTY, + search, + null, null, null, + null); + // See paritiiionManager + return new AutoContinuingCursor<>( + runner, + (context, continuation) -> { + LuceneScanBounds scanBounds = scan.bind(store, index, EvaluationContext.EMPTY); + return store.scanIndex(index, scanBounds, continuation, scanProperties); + }); + } + + @Override + public Tuple getKeyFromCursorResult(final RecordCursorResult result) { + final IndexEntry indexEntry = result.get(); + return indexEntry == null ? null : indexEntry.getKey(); + // Todo: return tuple that contains groupId, timestmap + + } + + @Override + public CompletableFuture handleOneItem(final FDBRecordStore store, final RecordCursorResult result) { + if (index == null) { + throw new IllegalStateException("presetParams was not called appropriately for this scrubbing tool"); + } + + final IndexEntry indexEntry = result.get(); + if (indexEntry == null) { + return CompletableFuture.completedFuture(null); + } + + if (isSynthetic) { + return store.loadSyntheticRecord(indexEntry.getPrimaryKey()).thenApply(syntheticRecord -> { + if (syntheticRecord.getConstituents().isEmpty()) { + // None of the constituents of this synthetic type are present, so it must be dangling + return scrubDanglingEntry(indexEntry); + } + return null; + }); + } else { + return store.loadIndexEntryRecord(indexEntry, IndexOrphanBehavior.RETURN).thenApply(indexedRecord -> { + if (!indexedRecord.hasStoredRecord()) { + // Here: Oh, No! this index is dangling! + return scrubDanglingEntry(indexEntry); + } + return null; + }); + } + } + + private Issue scrubDanglingEntry(@Nonnull IndexEntry indexEntry) { + // Here: the index entry is dangling. Fix it (if allowed) and report the issue. + final Tuple valueKey = indexEntry.getKey(); + return new Issue( + KeyValueLogMessage.build("Scrubber: dangling index entry", + LogMessageKeys.KEY, valueKey, + LogMessageKeys.PRIMARY_KEY, indexEntry.getPrimaryKey()), + FDBStoreTimer.Counts.INDEX_SCRUBBER_DANGLING_ENTRIES, + null); + } +} diff --git a/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneIndexScrubbingToolsMissing.java b/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneIndexScrubbingToolsMissing.java new file mode 100644 index 0000000000..8e1996411d --- /dev/null +++ b/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneIndexScrubbingToolsMissing.java @@ -0,0 +1,120 @@ +/* + * LuceneIndexScrubbingToolsMissing.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2024 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.lucene; + +import com.apple.foundationdb.record.ExecuteProperties; +import com.apple.foundationdb.record.IsolationLevel; +import com.apple.foundationdb.record.RecordCursor; +import com.apple.foundationdb.record.RecordCursorResult; +import com.apple.foundationdb.record.ScanProperties; +import com.apple.foundationdb.record.TupleRange; +import com.apple.foundationdb.record.logging.KeyValueLogMessage; +import com.apple.foundationdb.record.logging.LogMessageKeys; +import com.apple.foundationdb.record.metadata.Index; +import com.apple.foundationdb.record.metadata.RecordType; +import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStore; +import com.apple.foundationdb.record.provider.foundationdb.FDBStoreTimer; +import com.apple.foundationdb.record.provider.foundationdb.FDBStoredRecord; +import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerState; +import com.apple.foundationdb.record.provider.foundationdb.IndexScrubbingTools; +import com.apple.foundationdb.tuple.Tuple; +import com.google.protobuf.Message; + +import javax.annotation.Nonnull; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.CompletableFuture; + +public class LuceneIndexScrubbingToolsMissing implements IndexScrubbingTools> { + private Collection recordTypes = null; + private Index index; + private boolean isSynthetic; + + @Nonnull + private final LucenePartitioner partitioner; // non-mutable + @Nonnull + private final IndexMaintainerState state; + + public LuceneIndexScrubbingToolsMissing(@Nonnull final LucenePartitioner partitioner, @Nonnull final IndexMaintainerState state) { + this.partitioner = partitioner; + this.state = state; + } + + + @Override + public void presetCommonParams(Index index, boolean allowRepair, boolean isSynthetic, Collection types) { + this.recordTypes = types; + this.index = index; + this.isSynthetic = isSynthetic; + } + + @Override + public RecordCursor> getCursor(final TupleRange range, final FDBRecordStore store, final int limit) { + final IsolationLevel isolationLevel = IsolationLevel.SNAPSHOT; + final ExecuteProperties.Builder executeProperties = ExecuteProperties.newBuilder() + .setIsolationLevel(isolationLevel) + .setReturnedRowLimit(limit); + + final ScanProperties scanProperties = new ScanProperties(executeProperties.build(), false); + return store.scanRecords(range, null, scanProperties); + } + + @Override + public Tuple getKeyFromCursorResult(final RecordCursorResult> result) { + final FDBStoredRecord storedRecord = result.get(); + return storedRecord == null ? null : storedRecord.getPrimaryKey(); + } + + @Override + public CompletableFuture handleOneItem(final FDBRecordStore store, final RecordCursorResult> result) { + if (recordTypes == null || index == null) { + throw new IllegalStateException("presetParams was not called appropriately for this scrubbing tool"); + } + + final FDBStoredRecord rec = result.get(); + if (rec == null || !recordTypes.contains(rec.getRecordType())) { + return CompletableFuture.completedFuture(null); + } + + return getMissingIndexKeys(store, rec) + .thenApply(missingIndexesKeys -> { + if (missingIndexesKeys.isEmpty()) { + return null; + } + // Here: Oh, No! the index is missing!! + // (Maybe) report an error and (maybe) return this record to be index + return new Issue( + KeyValueLogMessage.build("Scrubber: missing index entry", + LogMessageKeys.KEY, rec.getPrimaryKey().toString(), + LogMessageKeys.INDEX_KEY, missingIndexesKeys.toString()), + FDBStoreTimer.Counts.INDEX_SCRUBBER_MISSING_ENTRIES, + null); + }); + } + + private CompletableFuture> getMissingIndexKeys(FDBRecordStore store, FDBStoredRecord rec) { + // follow the logic of LuceneIndexMaintainer::tryDelete + + + + return CompletableFuture.completedFuture(null); + } +} From 76963d0124947e626d24b25091ad0df414f6c55e Mon Sep 17 00:00:00 2001 From: Josef Ezra Date: Mon, 23 Dec 2024 17:55:48 -0500 Subject: [PATCH 02/20] scrub missing --- .../record/logging/LogMessageKeys.java | 1 + .../record/lucene/LuceneIndexMaintainer.java | 10 +- .../LuceneIndexScrubbingToolsDangling.java | 5 +- .../LuceneIndexScrubbingToolsMissing.java | 111 +++++++++++++++--- 4 files changed, 105 insertions(+), 22 deletions(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/logging/LogMessageKeys.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/logging/LogMessageKeys.java index 04ae9ae951..67a4c2cf35 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/logging/LogMessageKeys.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/logging/LogMessageKeys.java @@ -126,6 +126,7 @@ public enum LogMessageKeys { PRIMARY_INDEX, VALUE_KEY, PRIMARY_KEY, + GROUPING_KEY, VALUE, INDEX_OPERATION("operation"), INITIAL_PREFIX, diff --git a/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneIndexMaintainer.java b/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneIndexMaintainer.java index a22d2be0cf..9c18792ce8 100644 --- a/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneIndexMaintainer.java +++ b/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneIndexMaintainer.java @@ -112,6 +112,7 @@ public class LuceneIndexMaintainer extends StandardIndexMaintainer { private static final Logger LOG = LoggerFactory.getLogger(LuceneIndexMaintainer.class); + @Nonnull private final FDBDirectoryManager directoryManager; private final LuceneAnalyzerCombinationProvider indexAnalyzerSelector; private final LuceneAnalyzerCombinationProvider autoCompleteAnalyzerSelector; @@ -757,9 +758,14 @@ private void logSerializationError(String format, Object ... arguments) { public IndexScrubbingTools getIndexScrubbingTools(final IndexScrubbingTools.ScrubbingType type) { switch (type) { case MISSING: - return new LuceneIndexScrubbingToolsMissing(partitioner, state); + final Map options = state.index.getOptions(); + if (Boolean.parseBoolean(options.get(LuceneIndexOptions.PRIMARY_KEY_SEGMENT_INDEX_ENABLED)) || + Boolean.parseBoolean(options.get(LuceneIndexOptions.PRIMARY_KEY_SEGMENT_INDEX_V2_ENABLED))) { + return new LuceneIndexScrubbingToolsMissing(partitioner, directoryManager); + } + return null; case DANGLING: - return new LuceneIndexScrubbingToolsDangling(partitioner, state); + return new LuceneIndexScrubbingToolsDangling(state); default: return null; } diff --git a/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneIndexScrubbingToolsDangling.java b/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneIndexScrubbingToolsDangling.java index 48f710f32f..ae9f986a1e 100644 --- a/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneIndexScrubbingToolsDangling.java +++ b/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneIndexScrubbingToolsDangling.java @@ -54,12 +54,9 @@ public class LuceneIndexScrubbingToolsDangling implements IndexScrubbingTools> { private Collection recordTypes = null; private Index index; - private boolean isSynthetic; @Nonnull - private final LucenePartitioner partitioner; // non-mutable + private final LucenePartitioner partitioner; @Nonnull - private final IndexMaintainerState state; + private final FDBDirectoryManager directoryManager; - public LuceneIndexScrubbingToolsMissing(@Nonnull final LucenePartitioner partitioner, @Nonnull final IndexMaintainerState state) { + public LuceneIndexScrubbingToolsMissing(@Nonnull LucenePartitioner partitioner, @Nonnull FDBDirectoryManager directoryManager) { this.partitioner = partitioner; - this.state = state; + this.directoryManager = directoryManager; } @@ -63,7 +77,6 @@ public LuceneIndexScrubbingToolsMissing(@Nonnull final LucenePartitioner partiti public void presetCommonParams(Index index, boolean allowRepair, boolean isSynthetic, Collection types) { this.recordTypes = types; this.index = index; - this.isSynthetic = isSynthetic; } @Override @@ -83,6 +96,15 @@ public Tuple getKeyFromCursorResult(final RecordCursorResult handleOneItem(final FDBRecordStore store, final RecordCursorResult> result) { if (recordTypes == null || index == null) { @@ -94,27 +116,84 @@ public CompletableFuture handleOneItem(final FDBRecordStore store, final return CompletableFuture.completedFuture(null); } - return getMissingIndexKeys(store, rec) + return detectMissingIndexKeys(rec) .thenApply(missingIndexesKeys -> { - if (missingIndexesKeys.isEmpty()) { + if (missingIndexesKeys == null) { return null; } // Here: Oh, No! the index is missing!! - // (Maybe) report an error and (maybe) return this record to be index + // (Maybe) report an error return new Issue( KeyValueLogMessage.build("Scrubber: missing index entry", - LogMessageKeys.KEY, rec.getPrimaryKey().toString(), - LogMessageKeys.INDEX_KEY, missingIndexesKeys.toString()), + LogMessageKeys.KEY, rec.getPrimaryKey(), + LogMessageKeys.GROUPING_KEY, missingIndexesKeys.getValue(), + LogMessageKeys.REASON, missingIndexesKeys.getKey()), FDBStoreTimer.Counts.INDEX_SCRUBBER_MISSING_ENTRIES, null); }); } - private CompletableFuture> getMissingIndexKeys(FDBRecordStore store, FDBStoredRecord rec) { - // follow the logic of LuceneIndexMaintainer::tryDelete - + public CompletableFuture> detectMissingIndexKeys(FDBStoredRecord rec) { + // return the first missing (if any). + final KeyExpression root = index.getRootExpression(); + final Map> recordFields = LuceneDocumentFromRecord.getRecordFields(root, rec); + if (recordFields.isEmpty()) { + // Could recordFields be an empty map? + return CompletableFuture.completedFuture(Pair.of(MissingIndexReason.EMPTY_RECORDS_FIELDS, null)); + } + if (recordFields.size() == 1) { + // A single grouping key + return checkMissingIndexKey(rec, recordFields.keySet().stream().findFirst().get()); + } + + // Here: more than one grouping key + final Map keys = Collections.synchronizedMap(new HashMap<>()); + return AsyncUtil.whenAll( recordFields.keySet().stream().map(groupingKey -> + checkMissingIndexKey(rec, groupingKey) + .thenApply(missing -> keys.put(missing.getValue(), missing.getKey())) + ).collect(Collectors.toList())) + .thenApply(ignore -> { + final Optional> first = keys.entrySet().stream().findFirst(); + return first.map(tupleStringEntry -> Pair.of(tupleStringEntry.getValue(), tupleStringEntry.getKey())).orElse(null); + }); + } + + private CompletableFuture> checkMissingIndexKey(FDBStoredRecord rec, Tuple groupingKey) { + if (!partitioner.isPartitioningEnabled()) { + return CompletableFuture.completedFuture( + isMissingIndexKey(rec, null, groupingKey) ? + Pair.of(MissingIndexReason.NOT_IN_PK_SEGMENT_INDEX, null) : + null); + } + return partitioner.tryGetPartitionInfo(rec, groupingKey).thenApply(partitionInfo -> { + if (partitionInfo == null) { + return Pair.of(MissingIndexReason.NOT_IN_PARTITION, groupingKey); + } + if (isMissingIndexKey(rec, partitionInfo.getId(), groupingKey)) { + return Pair.of(MissingIndexReason.NOT_IN_PK_SEGMENT_INDEX, groupingKey); + } + return null; + }); + } - return CompletableFuture.completedFuture(null); + private boolean isMissingIndexKey(FDBStoredRecord rec, Integer partitionId, Tuple groupingKey) { + @Nullable final LucenePrimaryKeySegmentIndex segmentIndex = directoryManager.getDirectory(groupingKey, partitionId).getPrimaryKeySegmentIndex(); + if (segmentIndex == null) { + // Here: iternal error, getIndexScrubbingTools should have indicated that scrub missing is not supported. + throw new IllegalStateException("This scrubber should not have been used"); + } + try (DirectoryReader directoryReader = directoryManager.getDirectoryReader(groupingKey, partitionId)) { + final LucenePrimaryKeySegmentIndex.DocumentIndexEntry documentIndexEntry = segmentIndex.findDocument(directoryReader, rec.getPrimaryKey()); + if (documentIndexEntry == null) { + // Here: the document had not been found in the PK segment index + return true; + } + } catch (IOException ex) { + // Here: probably an fdb exception. Unwrap and rethrow. + throw LuceneExceptions.toRecordCoreException("Error while finding document", ex); + } + return false; } + } From 0927a48035094dce57f374f0987216a4bddac228 Mon Sep 17 00:00:00 2001 From: Josef Ezra Date: Fri, 3 Jan 2025 17:02:52 -0500 Subject: [PATCH 03/20] Add negative test (scrub index with missings entries) --- .../provider/foundationdb/IndexScrubbing.java | 1 - .../record/lucene/LuceneIndexMaintainer.java | 2 +- .../LuceneIndexScrubbingToolsMissing.java | 16 ++- .../PrimaryKeyAndStoredFieldsWriter.java | 2 +- .../lucene/LuceneIndexScrubbingTest.java | 130 ++++++++++++++++++ .../directory/InjectedFailureRepository.java | 5 + .../lucene/directory/MockedFDBDirectory.java | 4 + 7 files changed, 155 insertions(+), 5 deletions(-) create mode 100644 fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneIndexScrubbingTest.java diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexScrubbing.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexScrubbing.java index e183c5fb5c..f95106536e 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexScrubbing.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexScrubbing.java @@ -142,7 +142,6 @@ private CompletableFuture indexScrubRangeOnly(@Nonnull FDBRecordStore s throw new UnsupportedOperationException("This index does not support scrubbing type " + scrubbingType); } - return indexScrubRangeOnly(store, recordsScanned, index, tools, maintainer.isIdempotent()); } diff --git a/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneIndexMaintainer.java b/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneIndexMaintainer.java index 9c18792ce8..d3ccb90359 100644 --- a/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneIndexMaintainer.java +++ b/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneIndexMaintainer.java @@ -761,7 +761,7 @@ public IndexScrubbingTools getIndexScrubbingTools(final IndexScrubbingTools.S final Map options = state.index.getOptions(); if (Boolean.parseBoolean(options.get(LuceneIndexOptions.PRIMARY_KEY_SEGMENT_INDEX_ENABLED)) || Boolean.parseBoolean(options.get(LuceneIndexOptions.PRIMARY_KEY_SEGMENT_INDEX_V2_ENABLED))) { - return new LuceneIndexScrubbingToolsMissing(partitioner, directoryManager); + return new LuceneIndexScrubbingToolsMissing(partitioner, directoryManager, indexAnalyzerSelector); } return null; case DANGLING: diff --git a/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneIndexScrubbingToolsMissing.java b/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneIndexScrubbingToolsMissing.java index 2c7910ff61..6e432ac7a4 100644 --- a/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneIndexScrubbingToolsMissing.java +++ b/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneIndexScrubbingToolsMissing.java @@ -66,10 +66,14 @@ public class LuceneIndexScrubbingToolsMissing implements IndexScrubbingTools rec, Integer partitio // Here: iternal error, getIndexScrubbingTools should have indicated that scrub missing is not supported. throw new IllegalStateException("This scrubber should not have been used"); } - try (DirectoryReader directoryReader = directoryManager.getDirectoryReader(groupingKey, partitionId)) { + + try { + // TODO: this is called to initilize the writer, else we get an exception at getDirectoryReader. Should it really be done for a RO operation? + directoryManager.getIndexWriter(groupingKey, partitionId, indexAnalyzerSelector.provideIndexAnalyzer("")); + } catch (IOException e) { + throw LuceneExceptions.toRecordCoreException("failed getIndexWriter", e); + } + try { + DirectoryReader directoryReader = directoryManager.getDirectoryReader(groupingKey, partitionId); final LucenePrimaryKeySegmentIndex.DocumentIndexEntry documentIndexEntry = segmentIndex.findDocument(directoryReader, rec.getPrimaryKey()); if (documentIndexEntry == null) { // Here: the document had not been found in the PK segment index diff --git a/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/codec/PrimaryKeyAndStoredFieldsWriter.java b/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/codec/PrimaryKeyAndStoredFieldsWriter.java index 81c2b5683f..dadcb0d286 100644 --- a/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/codec/PrimaryKeyAndStoredFieldsWriter.java +++ b/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/codec/PrimaryKeyAndStoredFieldsWriter.java @@ -56,7 +56,7 @@ public void finishDocument() throws IOException { public void writeField(FieldInfo info, IndexableField field) throws IOException { super.writeField(info, field); try { - if (LuceneIndexMaintainer.PRIMARY_KEY_FIELD_NAME.equals(info.name)) { + if (LuceneIndexMaintainer.PRIMARY_KEY_FIELD_NAME.equals(info.name) && lucenePrimaryKeySegmentIndex != null) { final byte[] primaryKey = field.binaryValue().bytes; lucenePrimaryKeySegmentIndex.addOrDeletePrimaryKeyEntry(primaryKey, segmentId, documentId, true, info.name); // TODO we store this twice, but we'll probably want to optimize and only store this once diff --git a/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneIndexScrubbingTest.java b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneIndexScrubbingTest.java new file mode 100644 index 0000000000..fb9a7fca57 --- /dev/null +++ b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneIndexScrubbingTest.java @@ -0,0 +1,130 @@ +/* + * LuceneIndexScrubbingTest.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2024 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.lucene; + +import com.apple.foundationdb.record.lucene.directory.InjectedFailureRepository; +import com.apple.foundationdb.record.lucene.directory.MockedLuceneIndexMaintainerFactory; +import com.apple.foundationdb.record.lucene.directory.TestingIndexMaintainerRegistry; +import com.apple.foundationdb.record.metadata.Index; +import com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext; +import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStore; +import com.apple.foundationdb.record.provider.foundationdb.OnlineIndexScrubber; +import com.apple.foundationdb.record.query.plan.QueryPlanner; +import com.apple.foundationdb.record.util.pair.Pair; +import org.junit.jupiter.api.Test; + +import java.util.concurrent.TimeoutException; + +import static com.apple.foundationdb.record.lucene.LuceneIndexTestUtils.SIMPLE_TEXT_SUFFIXES_WITH_PRIMARY_KEY_SEGMENT_INDEX; +import static com.apple.foundationdb.record.lucene.LuceneIndexTestUtils.createComplexDocument; +import static com.apple.foundationdb.record.lucene.LuceneIndexTestUtils.createSimpleDocument; +import static com.apple.foundationdb.record.lucene.directory.InjectedFailureRepository.Methods.LUCENE_GET_PRIMARY_KEY_SEGMENT_INDEX_FORCE_NULL; +import static com.apple.foundationdb.record.provider.foundationdb.indexes.TextIndexTestUtils.SIMPLE_DOC; +import static org.junit.jupiter.api.Assertions.assertEquals; + +class LuceneIndexScrubbingTest extends FDBLuceneTestBase { + + private void rebuildIndexMetaData(final FDBRecordContext context, final String document, final Index index) { + Pair pair = LuceneIndexTestUtils.rebuildIndexMetaData(context, path, document, index, isUseCascadesPlanner()); + this.recordStore = pair.getLeft(); + this.planner = pair.getRight(); + } + + @Test + void luceneIndexScrubMissingSimpleNoIssues() { + // Scrub a valid index, expect zero issues + Index index = SIMPLE_TEXT_SUFFIXES_WITH_PRIMARY_KEY_SEGMENT_INDEX; + try (final FDBRecordContext context = openContext()) { + // Write some records + rebuildIndexMetaData(context, SIMPLE_DOC, index); + recordStore.saveRecord(createSimpleDocument(1623L, ENGINEER_JOKE, 2)); + recordStore.saveRecord(createSimpleDocument(1547L, WAYLON, 1)); + recordStore.saveRecord(createSimpleDocument(2222L, WAYLON + " who?", 1)); + context.commit(); + } + try (final FDBRecordContext context = openContext()) { + // Overwrite + add records + rebuildIndexMetaData(context, SIMPLE_DOC, index); + recordStore.saveRecord(createSimpleDocument(1623L, ENGINEER_JOKE, 2)); + recordStore.saveRecord(createSimpleDocument(7771547L, WAYLON, 1)); + recordStore.saveRecord(createSimpleDocument(7772222L, WAYLON + " who?", 1)); + context.commit(); + } + try (final FDBRecordContext context = openContext()) { + // Scrub issues, assert none + rebuildIndexMetaData(context, SIMPLE_DOC, index); + try (OnlineIndexScrubber indexScrubber = OnlineIndexScrubber.newBuilder() + .setRecordStore(recordStore) + .setIndex(index) + .build()) { + final long missingEntriesCount = indexScrubber.scrubMissingIndexEntries(); + assertEquals(0, missingEntriesCount); + } + } + } + + @Test + void luceneIndexScrubMissingSimple() { + // Scrub an index with missing entries + Index index = SIMPLE_TEXT_SUFFIXES_WITH_PRIMARY_KEY_SEGMENT_INDEX; + + long startTime = System.currentTimeMillis(); + try (final FDBRecordContext context = openContext()) { + // Write some records + rebuildIndexMetaData(context, SIMPLE_DOC, index); + recordStore.saveRecord(createComplexDocument(1623L, WAYLON, 1, startTime)); + recordStore.saveRecord(createComplexDocument(1547L, WAYLON, 1, startTime + 1000)); + recordStore.saveRecord(createComplexDocument(2222L, WAYLON + " who?", 1, startTime + 2000)); + recordStore.saveRecord(createComplexDocument(899L, ENGINEER_JOKE, 1, startTime + 3000)); + context.commit(); + } + + final InjectedFailureRepository injectedFailures = new InjectedFailureRepository(); + final TestingIndexMaintainerRegistry registry = new TestingIndexMaintainerRegistry(); + registry.overrideFactory(new MockedLuceneIndexMaintainerFactory(injectedFailures)); + + try (final FDBRecordContext context = openContext()) { + // Overwrite + add records with an injected key segment index failure + Pair pair = LuceneIndexTestUtils.rebuildIndexMetaData(context, path, SIMPLE_DOC, index, isUseCascadesPlanner(), registry); + this.recordStore = pair.getLeft(); + this.planner = pair.getRight(); + injectedFailures.addFailure(LUCENE_GET_PRIMARY_KEY_SEGMENT_INDEX_FORCE_NULL, + new LuceneConcurrency.AsyncToSyncTimeoutException("", new TimeoutException("Blah")), 3); + + recordStore.saveRecord(createSimpleDocument(1623L, ENGINEER_JOKE, 2)); + recordStore.saveRecord(createSimpleDocument(7771547L, WAYLON, 1)); + recordStore.saveRecord(createSimpleDocument(7772222L, WAYLON + " who?", 1)); + context.commit(); + } + + try (final FDBRecordContext context = openContext()) { + // Scrub issues, assert the number of issues found + rebuildIndexMetaData(context, SIMPLE_DOC, index); + try (OnlineIndexScrubber indexScrubber = OnlineIndexScrubber.newBuilder() + .setRecordStore(recordStore) + .setIndex(index) + .build()) { + final long missingEntriesCount = indexScrubber.scrubMissingIndexEntries(); + assertEquals(3, missingEntriesCount); + } + } + } +} diff --git a/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/directory/InjectedFailureRepository.java b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/directory/InjectedFailureRepository.java index 93639e3b85..29a40e8094 100644 --- a/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/directory/InjectedFailureRepository.java +++ b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/directory/InjectedFailureRepository.java @@ -44,6 +44,7 @@ public enum Methods { LUCENE_GET_FILE_REFERENCE_CACHE_ASYNC, LUCENE_DELETE_FILE_INTERNAL, LUCENE_GET_PRIMARY_KEY_SEGMENT_INDEX, + LUCENE_GET_PRIMARY_KEY_SEGMENT_INDEX_FORCE_NULL, LUCENE_GET_ALL_FIELDS_INFO_STREAM } @@ -65,6 +66,10 @@ public void clear() { invocationCounts.clear(); } + public boolean hasFailure(@Nonnull Methods method) { + return failureDescriptions.get(method) != null; + } + public void checkFailureForIoException(@Nonnull final Methods method) throws IOException { try { checkFailure(method); diff --git a/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/directory/MockedFDBDirectory.java b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/directory/MockedFDBDirectory.java index 54ac75cc0c..dcf87d6832 100644 --- a/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/directory/MockedFDBDirectory.java +++ b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/directory/MockedFDBDirectory.java @@ -40,6 +40,7 @@ import static com.apple.foundationdb.record.lucene.directory.InjectedFailureRepository.Methods.LUCENE_GET_FILE_REFERENCE_CACHE_ASYNC; import static com.apple.foundationdb.record.lucene.directory.InjectedFailureRepository.Methods.LUCENE_GET_INCREMENT; import static com.apple.foundationdb.record.lucene.directory.InjectedFailureRepository.Methods.LUCENE_GET_PRIMARY_KEY_SEGMENT_INDEX; +import static com.apple.foundationdb.record.lucene.directory.InjectedFailureRepository.Methods.LUCENE_GET_PRIMARY_KEY_SEGMENT_INDEX_FORCE_NULL; import static com.apple.foundationdb.record.lucene.directory.InjectedFailureRepository.Methods.LUCENE_LIST_ALL; import static com.apple.foundationdb.record.lucene.directory.InjectedFailureRepository.Methods.LUCENE_READ_BLOCK; @@ -112,6 +113,9 @@ protected boolean deleteFileInternal(@Nonnull final Map Date: Sun, 5 Jan 2025 12:24:47 -0500 Subject: [PATCH 04/20] elaborate shouldFailWithoutException --- .../directory/InjectedFailureRepository.java | 16 ++++++++++++---- .../lucene/directory/MockedFDBDirectory.java | 2 +- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/directory/InjectedFailureRepository.java b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/directory/InjectedFailureRepository.java index 29a40e8094..4e7f40a281 100644 --- a/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/directory/InjectedFailureRepository.java +++ b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/directory/InjectedFailureRepository.java @@ -66,10 +66,6 @@ public void clear() { invocationCounts.clear(); } - public boolean hasFailure(@Nonnull Methods method) { - return failureDescriptions.get(method) != null; - } - public void checkFailureForIoException(@Nonnull final Methods method) throws IOException { try { checkFailure(method); @@ -102,6 +98,18 @@ private void checkFailure(@Nonnull final Methods method) throws Exception { } } + public boolean shouldFailWithoutException(@Nonnull Methods method) { + // Return true "count" times, then return false + // (Note that "checkFailure" will succeed "count" times, the repeatedly throw an exception) + FailureDescription failureDescription = failureDescriptions.get(method); + if (failureDescription != null) { + AtomicLong count = invocationCounts.computeIfAbsent(method, m -> new AtomicLong(0)); + long invocations = count.incrementAndGet(); + return invocations < failureDescription.getCount(); + } + return false; + } + /** * A Failure description is the definition of the failure to be injected. */ diff --git a/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/directory/MockedFDBDirectory.java b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/directory/MockedFDBDirectory.java index dcf87d6832..ded80da220 100644 --- a/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/directory/MockedFDBDirectory.java +++ b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/directory/MockedFDBDirectory.java @@ -113,7 +113,7 @@ protected boolean deleteFileInternal(@Nonnull final Map Date: Sun, 5 Jan 2025 13:10:04 -0500 Subject: [PATCH 05/20] Suppress Warnings "PMD.CloseResource" --- .../record/lucene/LuceneIndexScrubbingToolsMissing.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneIndexScrubbingToolsMissing.java b/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneIndexScrubbingToolsMissing.java index 6e432ac7a4..d351564b99 100644 --- a/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneIndexScrubbingToolsMissing.java +++ b/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneIndexScrubbingToolsMissing.java @@ -180,7 +180,7 @@ private CompletableFuture> checkMissingIndexKey( }); } - + @SuppressWarnings("PMD.CloseResource") private boolean isMissingIndexKey(FDBStoredRecord rec, Integer partitionId, Tuple groupingKey) { @Nullable final LucenePrimaryKeySegmentIndex segmentIndex = directoryManager.getDirectory(groupingKey, partitionId).getPrimaryKeySegmentIndex(); if (segmentIndex == null) { From d2ea59ecba8a79e7b2bcdba51d49f226e7cf3097 Mon Sep 17 00:00:00 2001 From: Josef Ezra Date: Sun, 5 Jan 2025 14:15:23 -0500 Subject: [PATCH 06/20] Reword comment --- .../foundationdb/record/lucene/LuceneIndexScrubbingTest.java | 2 +- .../record/lucene/directory/InjectedFailureRepository.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneIndexScrubbingTest.java b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneIndexScrubbingTest.java index fb9a7fca57..c443b3eed9 100644 --- a/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneIndexScrubbingTest.java +++ b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneIndexScrubbingTest.java @@ -107,7 +107,7 @@ void luceneIndexScrubMissingSimple() { this.recordStore = pair.getLeft(); this.planner = pair.getRight(); injectedFailures.addFailure(LUCENE_GET_PRIMARY_KEY_SEGMENT_INDEX_FORCE_NULL, - new LuceneConcurrency.AsyncToSyncTimeoutException("", new TimeoutException("Blah")), 3); + new LuceneConcurrency.AsyncToSyncTimeoutException("", new TimeoutException("Dummy")), 3); recordStore.saveRecord(createSimpleDocument(1623L, ENGINEER_JOKE, 2)); recordStore.saveRecord(createSimpleDocument(7771547L, WAYLON, 1)); diff --git a/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/directory/InjectedFailureRepository.java b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/directory/InjectedFailureRepository.java index 4e7f40a281..fe421349fb 100644 --- a/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/directory/InjectedFailureRepository.java +++ b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/directory/InjectedFailureRepository.java @@ -100,7 +100,7 @@ private void checkFailure(@Nonnull final Methods method) throws Exception { public boolean shouldFailWithoutException(@Nonnull Methods method) { // Return true "count" times, then return false - // (Note that "checkFailure" will succeed "count" times, the repeatedly throw an exception) + // (Note that it's a reverse logic of "checkFailure" - which will succeed "count" times, then repeatedly throw an exception) FailureDescription failureDescription = failureDescriptions.get(method); if (failureDescription != null) { AtomicLong count = invocationCounts.computeIfAbsent(method, m -> new AtomicLong(0)); From 1f53825c4a97f25544400fccde4857df9d914193 Mon Sep 17 00:00:00 2001 From: Josef Ezra Date: Mon, 13 Jan 2025 12:15:55 -0500 Subject: [PATCH 07/20] Convert pseodo exception to a flag --- .../lucene/LuceneIndexScrubbingTest.java | 8 ++--- .../directory/InjectedFailureRepository.java | 31 +++++++++++-------- .../lucene/directory/MockedFDBDirectory.java | 4 +-- 3 files changed, 22 insertions(+), 21 deletions(-) diff --git a/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneIndexScrubbingTest.java b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneIndexScrubbingTest.java index c443b3eed9..c14e299a38 100644 --- a/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneIndexScrubbingTest.java +++ b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneIndexScrubbingTest.java @@ -31,12 +31,10 @@ import com.apple.foundationdb.record.util.pair.Pair; import org.junit.jupiter.api.Test; -import java.util.concurrent.TimeoutException; - import static com.apple.foundationdb.record.lucene.LuceneIndexTestUtils.SIMPLE_TEXT_SUFFIXES_WITH_PRIMARY_KEY_SEGMENT_INDEX; import static com.apple.foundationdb.record.lucene.LuceneIndexTestUtils.createComplexDocument; import static com.apple.foundationdb.record.lucene.LuceneIndexTestUtils.createSimpleDocument; -import static com.apple.foundationdb.record.lucene.directory.InjectedFailureRepository.Methods.LUCENE_GET_PRIMARY_KEY_SEGMENT_INDEX_FORCE_NULL; +import static com.apple.foundationdb.record.lucene.directory.InjectedFailureRepository.Flags.LUCENE_GET_PRIMARY_KEY_SEGMENT_INDEX_FORCE_NULL; import static com.apple.foundationdb.record.provider.foundationdb.indexes.TextIndexTestUtils.SIMPLE_DOC; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -106,9 +104,7 @@ void luceneIndexScrubMissingSimple() { Pair pair = LuceneIndexTestUtils.rebuildIndexMetaData(context, path, SIMPLE_DOC, index, isUseCascadesPlanner(), registry); this.recordStore = pair.getLeft(); this.planner = pair.getRight(); - injectedFailures.addFailure(LUCENE_GET_PRIMARY_KEY_SEGMENT_INDEX_FORCE_NULL, - new LuceneConcurrency.AsyncToSyncTimeoutException("", new TimeoutException("Dummy")), 3); - + injectedFailures.setFlag(LUCENE_GET_PRIMARY_KEY_SEGMENT_INDEX_FORCE_NULL); recordStore.saveRecord(createSimpleDocument(1623L, ENGINEER_JOKE, 2)); recordStore.saveRecord(createSimpleDocument(7771547L, WAYLON, 1)); recordStore.saveRecord(createSimpleDocument(7772222L, WAYLON + " who?", 1)); diff --git a/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/directory/InjectedFailureRepository.java b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/directory/InjectedFailureRepository.java index fe421349fb..962e60dee2 100644 --- a/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/directory/InjectedFailureRepository.java +++ b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/directory/InjectedFailureRepository.java @@ -25,6 +25,7 @@ import javax.annotation.Nonnull; import java.io.IOException; import java.util.EnumMap; +import java.util.Optional; import java.util.concurrent.atomic.AtomicLong; /** @@ -44,13 +45,17 @@ public enum Methods { LUCENE_GET_FILE_REFERENCE_CACHE_ASYNC, LUCENE_DELETE_FILE_INTERNAL, LUCENE_GET_PRIMARY_KEY_SEGMENT_INDEX, - LUCENE_GET_PRIMARY_KEY_SEGMENT_INDEX_FORCE_NULL, LUCENE_GET_ALL_FIELDS_INFO_STREAM } + public enum Flags { + LUCENE_GET_PRIMARY_KEY_SEGMENT_INDEX_FORCE_NULL, + } + // The injected failure state private EnumMap failureDescriptions = new EnumMap<>(Methods.class); private EnumMap invocationCounts = new EnumMap<>(Methods.class); + private EnumMap flagsMap = new EnumMap<>(Flags.class); public void addFailure(@Nonnull Methods method, @Nonnull Exception exception, long count) { failureDescriptions.put(method, new FailureDescription(method, exception, count)); @@ -66,6 +71,18 @@ public void clear() { invocationCounts.clear(); } + public void setFlag(@Nonnull Flags flag) { + setFlag(flag, true); + } + + public void setFlag(@Nonnull Flags flag, Boolean value) { + flagsMap.put(flag, value); + } + + public boolean hasFlag(@Nonnull Flags flag) { + return Optional.ofNullable(flagsMap.get(flag)).orElse(false); + } + public void checkFailureForIoException(@Nonnull final Methods method) throws IOException { try { checkFailure(method); @@ -98,18 +115,6 @@ private void checkFailure(@Nonnull final Methods method) throws Exception { } } - public boolean shouldFailWithoutException(@Nonnull Methods method) { - // Return true "count" times, then return false - // (Note that it's a reverse logic of "checkFailure" - which will succeed "count" times, then repeatedly throw an exception) - FailureDescription failureDescription = failureDescriptions.get(method); - if (failureDescription != null) { - AtomicLong count = invocationCounts.computeIfAbsent(method, m -> new AtomicLong(0)); - long invocations = count.incrementAndGet(); - return invocations < failureDescription.getCount(); - } - return false; - } - /** * A Failure description is the definition of the failure to be injected. */ diff --git a/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/directory/MockedFDBDirectory.java b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/directory/MockedFDBDirectory.java index ded80da220..0181be867a 100644 --- a/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/directory/MockedFDBDirectory.java +++ b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/directory/MockedFDBDirectory.java @@ -40,7 +40,7 @@ import static com.apple.foundationdb.record.lucene.directory.InjectedFailureRepository.Methods.LUCENE_GET_FILE_REFERENCE_CACHE_ASYNC; import static com.apple.foundationdb.record.lucene.directory.InjectedFailureRepository.Methods.LUCENE_GET_INCREMENT; import static com.apple.foundationdb.record.lucene.directory.InjectedFailureRepository.Methods.LUCENE_GET_PRIMARY_KEY_SEGMENT_INDEX; -import static com.apple.foundationdb.record.lucene.directory.InjectedFailureRepository.Methods.LUCENE_GET_PRIMARY_KEY_SEGMENT_INDEX_FORCE_NULL; +import static com.apple.foundationdb.record.lucene.directory.InjectedFailureRepository.Flags.LUCENE_GET_PRIMARY_KEY_SEGMENT_INDEX_FORCE_NULL; import static com.apple.foundationdb.record.lucene.directory.InjectedFailureRepository.Methods.LUCENE_LIST_ALL; import static com.apple.foundationdb.record.lucene.directory.InjectedFailureRepository.Methods.LUCENE_READ_BLOCK; @@ -113,7 +113,7 @@ protected boolean deleteFileInternal(@Nonnull final Map Date: Mon, 13 Jan 2025 15:58:39 -0500 Subject: [PATCH 08/20] Update releases notes (after rebase..) --- docs/ReleaseNotes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ReleaseNotes.md b/docs/ReleaseNotes.md index 322bebceba..623d1b1c85 100644 --- a/docs/ReleaseNotes.md +++ b/docs/ReleaseNotes.md @@ -15,7 +15,7 @@ Builds and releases have been moved to a new CI system. This includes the resump Users performing online updates are encouraged to update from [4.0.559.4](#405594). The continuations of some queries have changed in ways that may break if continued on other 4.0 builds. See: [Issue #3093](https://github.com/FoundationDB/fdb-record-layer/issues/3093), [PR #3092](https://github.com/FoundationDB/fdb-record-layer/pull/3092) fixing the issue, and [PR #3108](https://github.com/FoundationDB/fdb-record-layer/issues/3108) preparing 4.0.559.4 to accept newer continuations. -