diff --git a/driver-kotlin-coroutine/src/integration/kotlin/com/mongodb/kotlin/client/coroutine/UnifiedCrudTest.kt b/driver-kotlin-coroutine/src/integration/kotlin/com/mongodb/kotlin/client/coroutine/UnifiedCrudTest.kt index 036ec5afcc4..5091058573e 100644 --- a/driver-kotlin-coroutine/src/integration/kotlin/com/mongodb/kotlin/client/coroutine/UnifiedCrudTest.kt +++ b/driver-kotlin-coroutine/src/integration/kotlin/com/mongodb/kotlin/client/coroutine/UnifiedCrudTest.kt @@ -24,7 +24,7 @@ internal class UnifiedCrudTest() : UnifiedTest() { @JvmStatic @Throws(URISyntaxException::class, IOException::class) fun data(): Collection? { - return getTestData("unified-test-format/crud") + return getTestData("unified-test-format/crud", true) } } } diff --git a/driver-kotlin-sync/src/integration/kotlin/com/mongodb/kotlin/client/UnifiedCrudTest.kt b/driver-kotlin-sync/src/integration/kotlin/com/mongodb/kotlin/client/UnifiedCrudTest.kt index eb06f5c1875..f030cb54645 100644 --- a/driver-kotlin-sync/src/integration/kotlin/com/mongodb/kotlin/client/UnifiedCrudTest.kt +++ b/driver-kotlin-sync/src/integration/kotlin/com/mongodb/kotlin/client/UnifiedCrudTest.kt @@ -24,7 +24,7 @@ internal class UnifiedCrudTest() : UnifiedTest() { @JvmStatic @Throws(URISyntaxException::class, IOException::class) fun data(): Collection? { - return getTestData("unified-test-format/crud") + return getTestData("unified-test-format/crud", false) } } } diff --git a/driver-reactive-streams/src/test/functional/com/mongodb/reactivestreams/client/unified/ClientSideOperationTimeoutTest.java b/driver-reactive-streams/src/test/functional/com/mongodb/reactivestreams/client/unified/ClientSideOperationTimeoutTest.java index 168ff4b8f81..a1063f05362 100644 --- a/driver-reactive-streams/src/test/functional/com/mongodb/reactivestreams/client/unified/ClientSideOperationTimeoutTest.java +++ b/driver-reactive-streams/src/test/functional/com/mongodb/reactivestreams/client/unified/ClientSideOperationTimeoutTest.java @@ -99,18 +99,25 @@ The Reactive Streams specification prevents us from allowing a subsequent next c @MethodSource("data") @Override public void shouldPassAllOutcomes( + final String testName, @Nullable final String fileDescription, @Nullable final String testDescription, @Nullable final String directoryName, + final int attemptNumber, + final int totalAttempts, final String schemaVersion, @Nullable final BsonArray runOnRequirements, final BsonArray entitiesArray, final BsonArray initialData, final BsonDocument definition) { try { - super.shouldPassAllOutcomes(fileDescription, + super.shouldPassAllOutcomes( + testName, + fileDescription, testDescription, directoryName, + attemptNumber, + totalAttempts, schemaVersion, runOnRequirements, entitiesArray, diff --git a/driver-reactive-streams/src/test/functional/com/mongodb/reactivestreams/client/unified/UnifiedReactiveStreamsTest.java b/driver-reactive-streams/src/test/functional/com/mongodb/reactivestreams/client/unified/UnifiedReactiveStreamsTest.java index 62c1315e240..28c8a27f8fa 100644 --- a/driver-reactive-streams/src/test/functional/com/mongodb/reactivestreams/client/unified/UnifiedReactiveStreamsTest.java +++ b/driver-reactive-streams/src/test/functional/com/mongodb/reactivestreams/client/unified/UnifiedReactiveStreamsTest.java @@ -24,6 +24,7 @@ import com.mongodb.client.unified.UnifiedTest; import com.mongodb.client.unified.UnifiedTestModifications; import com.mongodb.client.vault.ClientEncryption; +import com.mongodb.lang.NonNull; import com.mongodb.reactivestreams.client.MongoClients; import com.mongodb.reactivestreams.client.gridfs.GridFSBuckets; import com.mongodb.reactivestreams.client.internal.vault.ClientEncryptionImpl; @@ -31,6 +32,11 @@ import com.mongodb.reactivestreams.client.syncadapter.SyncGridFSBucket; import com.mongodb.reactivestreams.client.syncadapter.SyncMongoClient; import com.mongodb.reactivestreams.client.syncadapter.SyncMongoDatabase; +import org.junit.jupiter.params.provider.Arguments; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.util.Collection; import static com.mongodb.client.unified.UnifiedTestModifications.Modifier; import static com.mongodb.client.unified.UnifiedTestModifications.TestDef; @@ -94,4 +100,9 @@ protected void postCleanUp(final TestDef testDef) { disableSleep(); } } + + @NonNull + protected static Collection getTestData(final String directory) throws URISyntaxException, IOException { + return getTestData(directory, true); + } } diff --git a/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedSyncTest.java b/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedSyncTest.java index 37db7cfe907..afcc8e4f1a3 100644 --- a/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedSyncTest.java +++ b/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedSyncTest.java @@ -25,6 +25,12 @@ import com.mongodb.client.gridfs.GridFSBuckets; import com.mongodb.client.internal.ClientEncryptionImpl; import com.mongodb.client.vault.ClientEncryption; +import com.mongodb.lang.NonNull; +import org.junit.jupiter.params.provider.Arguments; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.util.Collection; public abstract class UnifiedSyncTest extends UnifiedTest { protected UnifiedSyncTest() { @@ -44,4 +50,9 @@ protected GridFSBucket createGridFSBucket(final MongoDatabase database) { protected ClientEncryption createClientEncryption(final MongoClient keyVaultClient, final ClientEncryptionSettings clientEncryptionSettings) { return new ClientEncryptionImpl(keyVaultClient, clientEncryptionSettings); } + + @NonNull + protected static Collection getTestData(final String directory) throws URISyntaxException, IOException { + return getTestData(directory, false); + } } diff --git a/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTest.java b/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTest.java index 7ee16484df1..d9bffb29678 100644 --- a/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTest.java +++ b/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTest.java @@ -62,9 +62,11 @@ import java.io.File; import java.io.IOException; import java.net.URISyntaxException; +import java.text.MessageFormat; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.concurrent.ExecutionException; @@ -81,6 +83,8 @@ import static com.mongodb.client.test.CollectionHelper.getCurrentClusterTime; import static com.mongodb.client.test.CollectionHelper.killAllSessions; import static com.mongodb.client.unified.RunOnRequirementsMatcher.runOnRequirementsMet; +import static com.mongodb.client.unified.UnifiedTestModifications.Modifier; +import static com.mongodb.client.unified.UnifiedTestModifications.applyCustomizations; import static com.mongodb.client.unified.UnifiedTestModifications.testDef; import static java.util.Collections.singletonList; import static java.util.stream.Collectors.toList; @@ -91,6 +95,7 @@ import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; +import static org.junit.jupiter.api.Assumptions.abort; import static org.junit.jupiter.api.Assumptions.assumeFalse; import static org.junit.jupiter.api.Assumptions.assumeTrue; import static util.JsonPoweredTestHelper.getTestDocument; @@ -101,6 +106,9 @@ public abstract class UnifiedTest { private static final Set PRESTART_POOL_ASYNC_WORK_MANAGER_FILE_DESCRIPTIONS = Collections.singleton( "wait queue timeout errors include details about checked out connections"); + public static final int ATTEMPTS = 3; + private static Set ignoreRemaining = new HashSet<>(); + @Nullable private String fileDescription; private String schemaVersion; @@ -156,32 +164,47 @@ public Entities getEntities() { } @NonNull - protected static Collection getTestData(final String directory) throws URISyntaxException, IOException { + protected static Collection getTestData(final String directory, final boolean isReactive) + throws URISyntaxException, IOException { List data = new ArrayList<>(); for (File file : getTestFiles("/" + directory + "/")) { BsonDocument fileDocument = getTestDocument(file); - for (BsonValue cur : fileDocument.getArray("tests")) { - data.add(UnifiedTest.createTestData(directory, fileDocument, cur.asDocument())); + + final BsonDocument testDocument = cur.asDocument(); + String testDescription = testDocument.getString("description").getValue(); + String fileDescription = fileDocument.getString("description").getValue(); + TestDef testDef = testDef(directory, fileDescription, testDescription, isReactive); + applyCustomizations(testDef); + + boolean forceFlaky = testDef.wasAssignedModifier(Modifier.FORCE_FLAKY); + boolean retry = forceFlaky || testDef.wasAssignedModifier(Modifier.RETRY); + + int attempts = retry ? ATTEMPTS : 1; + if (forceFlaky) { + attempts = 10; + } + + for (int attempt = 1; attempt <= attempts; attempt++) { + String testName = MessageFormat.format("{0}: {1}", fileDescription, testDescription); + data.add(Arguments.of( + testName, + fileDescription, + testDescription, + directory, + attempt, + attempts * (forceFlaky ? -1 : 1), + fileDocument.getString("schemaVersion").getValue(), + fileDocument.getArray("runOnRequirements", null), + fileDocument.getArray("createEntities", new BsonArray()), + fileDocument.getArray("initialData", new BsonArray()), + testDocument)); + } } } return data; } - @NonNull - private static Arguments createTestData( - final String directory, final BsonDocument fileDocument, final BsonDocument testDocument) { - return Arguments.of( - fileDocument.getString("description").getValue(), - testDocument.getString("description").getValue(), - directory, - fileDocument.getString("schemaVersion").getValue(), - fileDocument.getArray("runOnRequirements", null), - fileDocument.getArray("createEntities", new BsonArray()), - fileDocument.getArray("initialData", new BsonArray()), - testDocument); - } - protected BsonDocument getDefinition() { return definition; } @@ -194,9 +217,12 @@ protected BsonDocument getDefinition() { @BeforeEach public void setUp( + final String testName, @Nullable final String fileDescription, @Nullable final String testDescription, @Nullable final String directoryName, + final int attemptNumber, + final int totalAttempts, final String schemaVersion, @Nullable final BsonArray runOnRequirements, final BsonArray entitiesArray, @@ -218,9 +244,9 @@ public void setUp( ignoreExtraEvents = false; if (directoryName != null && fileDescription != null && testDescription != null) { testDef = testDef(directoryName, fileDescription, testDescription, isReactive()); - UnifiedTestModifications.doSkips(testDef); + applyCustomizations(testDef); - boolean skip = testDef.wasAssignedModifier(UnifiedTestModifications.Modifier.SKIP); + boolean skip = testDef.wasAssignedModifier(Modifier.SKIP); assumeFalse(skip, "Skipping test"); } skips(fileDescription, testDescription); @@ -295,8 +321,9 @@ protected void postCleanUp(final TestDef testDef) { } /** - * This method is called once per {@link #setUp(String, String, String, String, org.bson.BsonArray, org.bson.BsonArray, org.bson.BsonArray, org.bson.BsonDocument)}, - * unless {@link #setUp(String, String, String, String, org.bson.BsonArray, org.bson.BsonArray, org.bson.BsonArray, org.bson.BsonDocument)} fails unexpectedly. + * This method is called once per + * {@link #setUp(String, String, String, String, int, int, String, org.bson.BsonArray, org.bson.BsonArray, org.bson.BsonArray, org.bson.BsonDocument)}, unless + * {@link #setUp(String, String, String, String, int, int, String, org.bson.BsonArray, org.bson.BsonArray, org.bson.BsonArray, org.bson.BsonDocument)} fails unexpectedly. */ protected void skips(final String fileDescription, final String testDescription) { } @@ -305,40 +332,66 @@ protected boolean isReactive() { return false; } - @ParameterizedTest(name = "{0}: {1}") + @ParameterizedTest(name = "{0}") @MethodSource("data") public void shouldPassAllOutcomes( + final String testName, @Nullable final String fileDescription, @Nullable final String testDescription, @Nullable final String directoryName, + final int attemptNumber, + final int totalAttempts, final String schemaVersion, @Nullable final BsonArray runOnRequirements, final BsonArray entitiesArray, final BsonArray initialData, final BsonDocument definition) { - BsonArray operations = definition.getArray("operations"); - for (int i = 0; i < operations.size(); i++) { - BsonValue cur = operations.get(i); - assertOperation(rootContext, cur.asDocument(), i); + boolean forceFlaky = totalAttempts < 0; + if (!forceFlaky) { + boolean ignoreThisTest = ignoreRemaining.contains(testName); + assumeFalse(ignoreThisTest, "Skipping a retryable test that already succeeded"); + ignoreRemaining.add(testName); } + try { + BsonArray operations = definition.getArray("operations"); + for (int i = 0; i < operations.size(); i++) { + BsonValue cur = operations.get(i); + assertOperation(rootContext, cur.asDocument(), i); + } - if (definition.containsKey("outcome")) { - assertOutcome(rootContext); - } + if (definition.containsKey("outcome")) { + assertOutcome(rootContext); + } - if (definition.containsKey("expectEvents")) { - compareEvents(rootContext, definition); - } + if (definition.containsKey("expectEvents")) { + compareEvents(rootContext, definition); + } - if (definition.containsKey("expectLogMessages")) { - ArrayList tweaks = new ArrayList<>(singletonList( - // `LogMessage.Entry.Name.OPERATION` is not supported, therefore we skip matching its value - LogMatcher.Tweak.skip(LogMessage.Entry.Name.OPERATION))); - if (getMongoClientSettings().getClusterSettings() - .getHosts().stream().anyMatch(serverAddress -> serverAddress instanceof UnixServerAddress)) { - tweaks.add(LogMatcher.Tweak.skip(LogMessage.Entry.Name.SERVER_PORT)); + if (definition.containsKey("expectLogMessages")) { + ArrayList tweaks = new ArrayList<>(singletonList( + // `LogMessage.Entry.Name.OPERATION` is not supported, therefore we skip matching its value + LogMatcher.Tweak.skip(LogMessage.Entry.Name.OPERATION))); + if (getMongoClientSettings().getClusterSettings() + .getHosts().stream().anyMatch(serverAddress -> serverAddress instanceof UnixServerAddress)) { + tweaks.add(LogMatcher.Tweak.skip(LogMessage.Entry.Name.SERVER_PORT)); + } + compareLogMessages(rootContext, definition, tweaks); + } + } catch (Throwable e) { + if (forceFlaky) { + throw e; + } + if (!testDef.matchesThrowable(e)) { + // if the throwable is not matched, test definitions were not intended to apply; rethrow it + throw e; } - compareLogMessages(rootContext, definition, tweaks); + boolean isLastAttempt = attemptNumber == Math.abs(totalAttempts); + if (isLastAttempt) { + throw e; + } + + ignoreRemaining.remove(testName); + abort("Ignoring failure and retrying attempt " + attemptNumber); } } diff --git a/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTestFailureValidator.java b/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTestFailureValidator.java index 88458f8af8e..0472ef8e6ce 100644 --- a/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTestFailureValidator.java +++ b/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTestFailureValidator.java @@ -36,9 +36,12 @@ final class UnifiedTestFailureValidator extends UnifiedSyncTest { @Override @BeforeEach public void setUp( + final String testName, @Nullable final String fileDescription, @Nullable final String testDescription, final String directoryName, + final int attemptNumber, + final int totalAttempts, final String schemaVersion, @Nullable final BsonArray runOnRequirements, final BsonArray entitiesArray, @@ -46,9 +49,12 @@ public void setUp( final BsonDocument definition) { try { super.setUp( + testName, fileDescription, testDescription, directoryName, + attemptNumber, + totalAttempts, schemaVersion, runOnRequirements, entitiesArray, @@ -63,9 +69,12 @@ public void setUp( @ParameterizedTest @MethodSource("data") public void shouldPassAllOutcomes( + final String testName, @Nullable final String fileDescription, @Nullable final String testDescription, @Nullable final String directoryName, + final int attemptNumber, + final int totalAttempts, final String schemaVersion, @Nullable final BsonArray runOnRequirements, final BsonArray entitiesArray, @@ -74,9 +83,12 @@ public void shouldPassAllOutcomes( if (exception == null) { try { super.shouldPassAllOutcomes( + testName, fileDescription, testDescription, directoryName, + attemptNumber, + totalAttempts, schemaVersion, runOnRequirements, entitiesArray, diff --git a/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTestModifications.java b/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTestModifications.java index 5184fd699be..e48c96972ef 100644 --- a/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTestModifications.java +++ b/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTestModifications.java @@ -17,10 +17,12 @@ package com.mongodb.client.unified; import com.mongodb.assertions.Assertions; +import org.opentest4j.AssertionFailedError; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.function.Function; import java.util.function.Supplier; import static com.mongodb.ClusterFixture.isDataLakeTest; @@ -30,13 +32,27 @@ import static com.mongodb.ClusterFixture.serverVersionLessThan; import static com.mongodb.assertions.Assertions.assertNotNull; import static com.mongodb.client.unified.UnifiedTestModifications.Modifier.IGNORE_EXTRA_EVENTS; +import static com.mongodb.client.unified.UnifiedTestModifications.Modifier.RETRY; import static com.mongodb.client.unified.UnifiedTestModifications.Modifier.SKIP; import static com.mongodb.client.unified.UnifiedTestModifications.Modifier.SLEEP_AFTER_CURSOR_CLOSE; import static com.mongodb.client.unified.UnifiedTestModifications.Modifier.SLEEP_AFTER_CURSOR_OPEN; import static com.mongodb.client.unified.UnifiedTestModifications.Modifier.WAIT_FOR_BATCH_CURSOR_CREATION; public final class UnifiedTestModifications { - public static void doSkips(final TestDef def) { + public static void applyCustomizations(final TestDef def) { + + // TODO reasons for retry + // Exception in encryption library: ChangeCipherSpec message sequence violation + def.retry("TODO reason") + .whenFailureContains("ChangeCipherSpec message sequence violation") + .test("client-side-encryption", "namedKMS-createDataKey", "create datakey with named KMIP KMS provider"); + + def.retry("TODO reason") + .whenFailureContains("Number of checked out connections must match expected") + .test("load-balancers", "cursors are correctly pinned to connections for load-balanced clusters", "pinned connections are returned after a network error during a killCursors request"); + + def.retry("TODO reason") + .test("client-side-encryption", "namedKMS-rewrapManyDataKey", "rewrap to kmip:name1"); // atlas-data-lake @@ -259,6 +275,7 @@ public static final class TestDef { private final boolean reactive; private final List modifiers = new ArrayList<>(); + private Function matchesThrowable; private TestDef(final String dir, final String file, final String test, final boolean reactive) { this.dir = assertNotNull(dir); @@ -322,6 +339,14 @@ public TestApplicator skipUnknownReason(final String reason) { return new TestApplicator(this, reason, SKIP); } + + /** + * The test will be retried, for the reason provided + */ + public TestApplicator retry(final String reason) { + return new TestApplicator(this, reason, RETRY); + } + public TestApplicator modify(final Modifier... modifiers) { return new TestApplicator(this, null, modifiers); } @@ -333,6 +358,13 @@ public boolean isReactive() { public boolean wasAssignedModifier(final Modifier modifier) { return this.modifiers.contains(modifier); } + + public boolean matchesThrowable(final Throwable e) { + if (matchesThrowable != null) { + return matchesThrowable.apply(e); + } + return false; + } } /** @@ -340,17 +372,19 @@ public boolean wasAssignedModifier(final Modifier modifier) { */ public static final class TestApplicator { private final TestDef testDef; - private final List modifiersToApply; private Supplier precondition; private boolean matchWasPerformed = false; + private final List modifiersToApply; + private Function matchesThrowable; + private TestApplicator( final TestDef testDef, final String reason, final Modifier... modifiersToApply) { this.testDef = testDef; this.modifiersToApply = Arrays.asList(modifiersToApply); - if (this.modifiersToApply.contains(SKIP)) { + if (this.modifiersToApply.contains(SKIP) || this.modifiersToApply.contains(RETRY)) { assertNotNull(reason); } } @@ -362,6 +396,7 @@ private TestApplicator onMatch(final boolean match) { } if (match) { this.testDef.modifiers.addAll(this.modifiersToApply); + this.testDef.matchesThrowable = this.matchesThrowable; } return this; } @@ -453,6 +488,24 @@ public TestApplicator when(final Supplier precondition) { this.precondition = precondition; return this; } + + /** + * The modification, if it is a RETRY, will only be applied when the + * failure message contains the provided message fragment. If an + * {@code AssertionFailedError} occurs, and has a cause, the cause's + * message will be checked. Otherwise, the throwable will be checked. + */ + public TestApplicator whenFailureContains(final String messageFragment) { + this.matchesThrowable = (final Throwable e) -> { + // inspect the cause for failed assertions with a cause + if (e instanceof AssertionFailedError && e.getCause() != null) { + return e.getCause().getMessage().contains(messageFragment); + } else { + return e.getMessage().contains(messageFragment); + } + }; + return this; + } } public enum Modifier { @@ -478,5 +531,17 @@ public enum Modifier { * Skip the test. */ SKIP, + /** + * Ignore results and retry the test on failure. Will not repeat the + * test if the test succeeds. Multiple copies of the test are used to + * facilitate retries. + */ + RETRY, + /** + * The test will be retried multiple times, without the results being + * ignored. This is a helper that can be used, in patches, to check + * if certain tests are (still) flaky. + */ + FORCE_FLAKY, } } diff --git a/driver-workload-executor/src/main/com/mongodb/workload/WorkloadExecutor.java b/driver-workload-executor/src/main/com/mongodb/workload/WorkloadExecutor.java index 0e995cb34fd..7aba736aebc 100644 --- a/driver-workload-executor/src/main/com/mongodb/workload/WorkloadExecutor.java +++ b/driver-workload-executor/src/main/com/mongodb/workload/WorkloadExecutor.java @@ -98,18 +98,24 @@ protected boolean terminateLoop() { BsonArray createEntities = fileDocument.getArray("createEntities", new BsonArray()); BsonArray initialData = fileDocument.getArray("initialData", new BsonArray()); unifiedTest.setUp( + "", null, null, null, + 1, + 1, schemaVersion, runOnRequirements, createEntities, initialData, testDocument); unifiedTest.shouldPassAllOutcomes( + "", null, null, null, + 1, + 1, schemaVersion, runOnRequirements, createEntities,