From 7fc1507cde670f65f6084ae27f609831651dacf5 Mon Sep 17 00:00:00 2001 From: Maxim Katcharov Date: Wed, 20 Nov 2024 13:48:30 -0700 Subject: [PATCH 01/10] Retry flaky unified tests JAVA-5393 --- .../client/coroutine/UnifiedCrudTest.kt | 2 +- .../mongodb/kotlin/client/UnifiedCrudTest.kt | 2 +- .../ClientSideOperationTimeoutTest.java | 9 +- .../unified/UnifiedReactiveStreamsTest.java | 11 ++ .../client/unified/UnifiedSyncTest.java | 11 ++ .../mongodb/client/unified/UnifiedTest.java | 124 ++++++++++++------ .../unified/UnifiedTestFailureValidator.java | 12 ++ .../unified/UnifiedTestModifications.java | 17 +++ .../mongodb/workload/WorkloadExecutor.java | 6 + 9 files changed, 153 insertions(+), 41 deletions(-) diff --git a/driver-kotlin-coroutine/src/integration/kotlin/com/mongodb/kotlin/client/coroutine/UnifiedCrudTest.kt b/driver-kotlin-coroutine/src/integration/kotlin/com/mongodb/kotlin/client/coroutine/UnifiedCrudTest.kt index 036ec5afcc4..5091058573e 100644 --- a/driver-kotlin-coroutine/src/integration/kotlin/com/mongodb/kotlin/client/coroutine/UnifiedCrudTest.kt +++ b/driver-kotlin-coroutine/src/integration/kotlin/com/mongodb/kotlin/client/coroutine/UnifiedCrudTest.kt @@ -24,7 +24,7 @@ internal class UnifiedCrudTest() : UnifiedTest() { @JvmStatic @Throws(URISyntaxException::class, IOException::class) fun data(): Collection? { - return getTestData("unified-test-format/crud") + return getTestData("unified-test-format/crud", true) } } } diff --git a/driver-kotlin-sync/src/integration/kotlin/com/mongodb/kotlin/client/UnifiedCrudTest.kt b/driver-kotlin-sync/src/integration/kotlin/com/mongodb/kotlin/client/UnifiedCrudTest.kt index eb06f5c1875..f030cb54645 100644 --- a/driver-kotlin-sync/src/integration/kotlin/com/mongodb/kotlin/client/UnifiedCrudTest.kt +++ b/driver-kotlin-sync/src/integration/kotlin/com/mongodb/kotlin/client/UnifiedCrudTest.kt @@ -24,7 +24,7 @@ internal class UnifiedCrudTest() : UnifiedTest() { @JvmStatic @Throws(URISyntaxException::class, IOException::class) fun data(): Collection? { - return getTestData("unified-test-format/crud") + return getTestData("unified-test-format/crud", false) } } } diff --git a/driver-reactive-streams/src/test/functional/com/mongodb/reactivestreams/client/unified/ClientSideOperationTimeoutTest.java b/driver-reactive-streams/src/test/functional/com/mongodb/reactivestreams/client/unified/ClientSideOperationTimeoutTest.java index 168ff4b8f81..a1063f05362 100644 --- a/driver-reactive-streams/src/test/functional/com/mongodb/reactivestreams/client/unified/ClientSideOperationTimeoutTest.java +++ b/driver-reactive-streams/src/test/functional/com/mongodb/reactivestreams/client/unified/ClientSideOperationTimeoutTest.java @@ -99,18 +99,25 @@ The Reactive Streams specification prevents us from allowing a subsequent next c @MethodSource("data") @Override public void shouldPassAllOutcomes( + final String testName, @Nullable final String fileDescription, @Nullable final String testDescription, @Nullable final String directoryName, + final int attemptNumber, + final int totalAttempts, final String schemaVersion, @Nullable final BsonArray runOnRequirements, final BsonArray entitiesArray, final BsonArray initialData, final BsonDocument definition) { try { - super.shouldPassAllOutcomes(fileDescription, + super.shouldPassAllOutcomes( + testName, + fileDescription, testDescription, directoryName, + attemptNumber, + totalAttempts, schemaVersion, runOnRequirements, entitiesArray, diff --git a/driver-reactive-streams/src/test/functional/com/mongodb/reactivestreams/client/unified/UnifiedReactiveStreamsTest.java b/driver-reactive-streams/src/test/functional/com/mongodb/reactivestreams/client/unified/UnifiedReactiveStreamsTest.java index 62c1315e240..28c8a27f8fa 100644 --- a/driver-reactive-streams/src/test/functional/com/mongodb/reactivestreams/client/unified/UnifiedReactiveStreamsTest.java +++ b/driver-reactive-streams/src/test/functional/com/mongodb/reactivestreams/client/unified/UnifiedReactiveStreamsTest.java @@ -24,6 +24,7 @@ import com.mongodb.client.unified.UnifiedTest; import com.mongodb.client.unified.UnifiedTestModifications; import com.mongodb.client.vault.ClientEncryption; +import com.mongodb.lang.NonNull; import com.mongodb.reactivestreams.client.MongoClients; import com.mongodb.reactivestreams.client.gridfs.GridFSBuckets; import com.mongodb.reactivestreams.client.internal.vault.ClientEncryptionImpl; @@ -31,6 +32,11 @@ import com.mongodb.reactivestreams.client.syncadapter.SyncGridFSBucket; import com.mongodb.reactivestreams.client.syncadapter.SyncMongoClient; import com.mongodb.reactivestreams.client.syncadapter.SyncMongoDatabase; +import org.junit.jupiter.params.provider.Arguments; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.util.Collection; import static com.mongodb.client.unified.UnifiedTestModifications.Modifier; import static com.mongodb.client.unified.UnifiedTestModifications.TestDef; @@ -94,4 +100,9 @@ protected void postCleanUp(final TestDef testDef) { disableSleep(); } } + + @NonNull + protected static Collection getTestData(final String directory) throws URISyntaxException, IOException { + return getTestData(directory, true); + } } diff --git a/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedSyncTest.java b/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedSyncTest.java index 37db7cfe907..afcc8e4f1a3 100644 --- a/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedSyncTest.java +++ b/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedSyncTest.java @@ -25,6 +25,12 @@ import com.mongodb.client.gridfs.GridFSBuckets; import com.mongodb.client.internal.ClientEncryptionImpl; import com.mongodb.client.vault.ClientEncryption; +import com.mongodb.lang.NonNull; +import org.junit.jupiter.params.provider.Arguments; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.util.Collection; public abstract class UnifiedSyncTest extends UnifiedTest { protected UnifiedSyncTest() { @@ -44,4 +50,9 @@ protected GridFSBucket createGridFSBucket(final MongoDatabase database) { protected ClientEncryption createClientEncryption(final MongoClient keyVaultClient, final ClientEncryptionSettings clientEncryptionSettings) { return new ClientEncryptionImpl(keyVaultClient, clientEncryptionSettings); } + + @NonNull + protected static Collection getTestData(final String directory) throws URISyntaxException, IOException { + return getTestData(directory, false); + } } diff --git a/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTest.java b/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTest.java index 7ee16484df1..194cbd73a29 100644 --- a/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTest.java +++ b/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTest.java @@ -57,14 +57,17 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; +import org.opentest4j.AssertionFailedError; import org.opentest4j.TestAbortedException; import java.io.File; import java.io.IOException; import java.net.URISyntaxException; +import java.text.MessageFormat; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.concurrent.ExecutionException; @@ -81,6 +84,7 @@ import static com.mongodb.client.test.CollectionHelper.getCurrentClusterTime; import static com.mongodb.client.test.CollectionHelper.killAllSessions; import static com.mongodb.client.unified.RunOnRequirementsMatcher.runOnRequirementsMet; +import static com.mongodb.client.unified.UnifiedTestModifications.doSkips; import static com.mongodb.client.unified.UnifiedTestModifications.testDef; import static java.util.Collections.singletonList; import static java.util.stream.Collectors.toList; @@ -101,6 +105,9 @@ public abstract class UnifiedTest { private static final Set PRESTART_POOL_ASYNC_WORK_MANAGER_FILE_DESCRIPTIONS = Collections.singleton( "wait queue timeout errors include details about checked out connections"); + public static final int ATTEMPTS = 3; + private static Set completed = new HashSet<>(); + @Nullable private String fileDescription; private String schemaVersion; @@ -156,32 +163,51 @@ public Entities getEntities() { } @NonNull - protected static Collection getTestData(final String directory) throws URISyntaxException, IOException { + protected static Collection getTestData(final String directory, final boolean isReactive) + throws URISyntaxException, IOException { List data = new ArrayList<>(); for (File file : getTestFiles("/" + directory + "/")) { BsonDocument fileDocument = getTestDocument(file); - for (BsonValue cur : fileDocument.getArray("tests")) { - data.add(UnifiedTest.createTestData(directory, fileDocument, cur.asDocument())); + + final BsonDocument testDocument = cur.asDocument(); + String testDescription = testDocument.getString("description").getValue(); + String fileDescription = fileDocument.getString("description").getValue(); + TestDef testDef = testDef(directory, fileDescription, testDescription, isReactive); + doSkips(testDef); + + boolean forceFlaky = testDef.wasAssignedModifier(UnifiedTestModifications.Modifier.FORCE_FLAKY); + boolean retry = forceFlaky || testDef.wasAssignedModifier(UnifiedTestModifications.Modifier.RETRY); + + int attempts = retry ? ATTEMPTS : 1; + if (forceFlaky) { + attempts = 10; + } + + for (int attempt = 1; attempt <= attempts; attempt++) { + String testName = !retry + ? MessageFormat.format("{0}: {1}", fileDescription, testDescription) + : MessageFormat.format( + "{0}: {1} ({2} of {3})", + fileDescription, testDescription, attempt, attempts); + data.add(Arguments.of( + testName, + fileDescription, + testDescription, + directory, + attempt, + attempts * (forceFlaky ? -1 : 1), + fileDocument.getString("schemaVersion").getValue(), + fileDocument.getArray("runOnRequirements", null), + fileDocument.getArray("createEntities", new BsonArray()), + fileDocument.getArray("initialData", new BsonArray()), + testDocument)); + } } } return data; } - @NonNull - private static Arguments createTestData( - final String directory, final BsonDocument fileDocument, final BsonDocument testDocument) { - return Arguments.of( - fileDocument.getString("description").getValue(), - testDocument.getString("description").getValue(), - directory, - fileDocument.getString("schemaVersion").getValue(), - fileDocument.getArray("runOnRequirements", null), - fileDocument.getArray("createEntities", new BsonArray()), - fileDocument.getArray("initialData", new BsonArray()), - testDocument); - } - protected BsonDocument getDefinition() { return definition; } @@ -194,9 +220,12 @@ protected BsonDocument getDefinition() { @BeforeEach public void setUp( + final String testName, @Nullable final String fileDescription, @Nullable final String testDescription, @Nullable final String directoryName, + final int attemptNumber, + final int totalAttempts, final String schemaVersion, @Nullable final BsonArray runOnRequirements, final BsonArray entitiesArray, @@ -295,8 +324,9 @@ protected void postCleanUp(final TestDef testDef) { } /** - * This method is called once per {@link #setUp(String, String, String, String, org.bson.BsonArray, org.bson.BsonArray, org.bson.BsonArray, org.bson.BsonDocument)}, - * unless {@link #setUp(String, String, String, String, org.bson.BsonArray, org.bson.BsonArray, org.bson.BsonArray, org.bson.BsonDocument)} fails unexpectedly. + * This method is called once per + * {@link #setUp(String, String, String, String, int, int, String, org.bson.BsonArray, org.bson.BsonArray, org.bson.BsonArray, org.bson.BsonDocument)}, unless + * {@link #setUp(String, String, String, String, int, int, String, org.bson.BsonArray, org.bson.BsonArray, org.bson.BsonArray, org.bson.BsonDocument)} fails unexpectedly. */ protected void skips(final String fileDescription, final String testDescription) { } @@ -305,40 +335,58 @@ protected boolean isReactive() { return false; } - @ParameterizedTest(name = "{0}: {1}") + @ParameterizedTest(name = "{0}") @MethodSource("data") public void shouldPassAllOutcomes( + final String testName, @Nullable final String fileDescription, @Nullable final String testDescription, @Nullable final String directoryName, + final int attemptNumber, + final int totalAttempts, final String schemaVersion, @Nullable final BsonArray runOnRequirements, final BsonArray entitiesArray, final BsonArray initialData, final BsonDocument definition) { - BsonArray operations = definition.getArray("operations"); - for (int i = 0; i < operations.size(); i++) { - BsonValue cur = operations.get(i); - assertOperation(rootContext, cur.asDocument(), i); + boolean forceFlaky = totalAttempts < 0; + if (!forceFlaky) { + assumeFalse(completed.contains(testName), "Skipping retryable test that succeeded"); + completed.add(testName); } + try { + BsonArray operations = definition.getArray("operations"); + for (int i = 0; i < operations.size(); i++) { + BsonValue cur = operations.get(i); + assertOperation(rootContext, cur.asDocument(), i); + } - if (definition.containsKey("outcome")) { - assertOutcome(rootContext); - } + if (definition.containsKey("outcome")) { + assertOutcome(rootContext); + } - if (definition.containsKey("expectEvents")) { - compareEvents(rootContext, definition); - } + if (definition.containsKey("expectEvents")) { + compareEvents(rootContext, definition); + } - if (definition.containsKey("expectLogMessages")) { - ArrayList tweaks = new ArrayList<>(singletonList( - // `LogMessage.Entry.Name.OPERATION` is not supported, therefore we skip matching its value - LogMatcher.Tweak.skip(LogMessage.Entry.Name.OPERATION))); - if (getMongoClientSettings().getClusterSettings() - .getHosts().stream().anyMatch(serverAddress -> serverAddress instanceof UnixServerAddress)) { - tweaks.add(LogMatcher.Tweak.skip(LogMessage.Entry.Name.SERVER_PORT)); + if (definition.containsKey("expectLogMessages")) { + ArrayList tweaks = new ArrayList<>(singletonList( + // `LogMessage.Entry.Name.OPERATION` is not supported, therefore we skip matching its value + LogMatcher.Tweak.skip(LogMessage.Entry.Name.OPERATION))); + if (getMongoClientSettings().getClusterSettings() + .getHosts().stream().anyMatch(serverAddress -> serverAddress instanceof UnixServerAddress)) { + tweaks.add(LogMatcher.Tweak.skip(LogMessage.Entry.Name.SERVER_PORT)); + } + compareLogMessages(rootContext, definition, tweaks); + } + } catch (AssertionFailedError e) { + completed.remove(testName); + boolean lastAttempt = attemptNumber == Math.abs(totalAttempts); + if (forceFlaky || lastAttempt) { + throw e; + } else { + assumeFalse(completed.contains(testName), "Ignoring failure and retrying attempt " + attemptNumber); } - compareLogMessages(rootContext, definition, tweaks); } } diff --git a/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTestFailureValidator.java b/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTestFailureValidator.java index 88458f8af8e..0472ef8e6ce 100644 --- a/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTestFailureValidator.java +++ b/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTestFailureValidator.java @@ -36,9 +36,12 @@ final class UnifiedTestFailureValidator extends UnifiedSyncTest { @Override @BeforeEach public void setUp( + final String testName, @Nullable final String fileDescription, @Nullable final String testDescription, final String directoryName, + final int attemptNumber, + final int totalAttempts, final String schemaVersion, @Nullable final BsonArray runOnRequirements, final BsonArray entitiesArray, @@ -46,9 +49,12 @@ public void setUp( final BsonDocument definition) { try { super.setUp( + testName, fileDescription, testDescription, directoryName, + attemptNumber, + totalAttempts, schemaVersion, runOnRequirements, entitiesArray, @@ -63,9 +69,12 @@ public void setUp( @ParameterizedTest @MethodSource("data") public void shouldPassAllOutcomes( + final String testName, @Nullable final String fileDescription, @Nullable final String testDescription, @Nullable final String directoryName, + final int attemptNumber, + final int totalAttempts, final String schemaVersion, @Nullable final BsonArray runOnRequirements, final BsonArray entitiesArray, @@ -74,9 +83,12 @@ public void shouldPassAllOutcomes( if (exception == null) { try { super.shouldPassAllOutcomes( + testName, fileDescription, testDescription, directoryName, + attemptNumber, + totalAttempts, schemaVersion, runOnRequirements, entitiesArray, diff --git a/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTestModifications.java b/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTestModifications.java index 5184fd699be..f1bf20dd788 100644 --- a/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTestModifications.java +++ b/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTestModifications.java @@ -28,6 +28,7 @@ import static com.mongodb.ClusterFixture.isServerlessTest; import static com.mongodb.ClusterFixture.isSharded; import static com.mongodb.ClusterFixture.serverVersionLessThan; +import static com.mongodb.client.unified.UnifiedTestModifications.Modifier.FORCE_FLAKY; import static com.mongodb.assertions.Assertions.assertNotNull; import static com.mongodb.client.unified.UnifiedTestModifications.Modifier.IGNORE_EXTRA_EVENTS; import static com.mongodb.client.unified.UnifiedTestModifications.Modifier.SKIP; @@ -38,6 +39,14 @@ public final class UnifiedTestModifications { public static void doSkips(final TestDef def) { + // TODO reasons for retry + def.modify(FORCE_FLAKY) + // Exception in encryption library: ChangeCipherSpec message sequence violation + .test("client-side-encryption", "namedKMS-createDataKey", "create datakey with named KMIP KMS provider") + // Number of checked out connections must match expected + .test("load-balancers", "cursors are correctly pinned to connections for load-balanced clusters", "pinned connections are returned after a network error during a killCursors request") + .test("client-side-encryption", "namedKMS-rewrapManyDataKey", "rewrap to kmip:name1"); + // atlas-data-lake def.skipAccordingToSpec("Data lake tests should only run on data lake") @@ -478,5 +487,13 @@ public enum Modifier { * Skip the test. */ SKIP, + /** + * Retry the test on failure. + */ + RETRY, + /** + * Retry the test multiple times, without ignoring failures. + */ + FORCE_FLAKY, } } diff --git a/driver-workload-executor/src/main/com/mongodb/workload/WorkloadExecutor.java b/driver-workload-executor/src/main/com/mongodb/workload/WorkloadExecutor.java index 0e995cb34fd..7aba736aebc 100644 --- a/driver-workload-executor/src/main/com/mongodb/workload/WorkloadExecutor.java +++ b/driver-workload-executor/src/main/com/mongodb/workload/WorkloadExecutor.java @@ -98,18 +98,24 @@ protected boolean terminateLoop() { BsonArray createEntities = fileDocument.getArray("createEntities", new BsonArray()); BsonArray initialData = fileDocument.getArray("initialData", new BsonArray()); unifiedTest.setUp( + "", null, null, null, + 1, + 1, schemaVersion, runOnRequirements, createEntities, initialData, testDocument); unifiedTest.shouldPassAllOutcomes( + "", null, null, null, + 1, + 1, schemaVersion, runOnRequirements, createEntities, From 96e9bd970dee906986e62ec591fbc80118d5715f Mon Sep 17 00:00:00 2001 From: Maxim Katcharov Date: Mon, 2 Dec 2024 15:24:11 -0700 Subject: [PATCH 02/10] Remove parenthetical number from test name --- .../functional/com/mongodb/client/unified/UnifiedTest.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTest.java b/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTest.java index 194cbd73a29..e4a27a366ed 100644 --- a/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTest.java +++ b/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTest.java @@ -185,11 +185,7 @@ protected static Collection getTestData(final String directory, final } for (int attempt = 1; attempt <= attempts; attempt++) { - String testName = !retry - ? MessageFormat.format("{0}: {1}", fileDescription, testDescription) - : MessageFormat.format( - "{0}: {1} ({2} of {3})", - fileDescription, testDescription, attempt, attempts); + String testName = MessageFormat.format("{0}: {1}", fileDescription, testDescription); data.add(Arguments.of( testName, fileDescription, From bb294cc4ff85d41206c5235f74342d943603d4a3 Mon Sep 17 00:00:00 2001 From: Maxim Katcharov Date: Wed, 4 Dec 2024 17:01:51 -0700 Subject: [PATCH 03/10] Add whenExceptionContains, fixes --- .../mongodb/client/unified/UnifiedTest.java | 19 ++++--- .../unified/UnifiedTestModifications.java | 51 +++++++++++++++---- 2 files changed, 55 insertions(+), 15 deletions(-) diff --git a/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTest.java b/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTest.java index e4a27a366ed..5592fe64a56 100644 --- a/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTest.java +++ b/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTest.java @@ -84,7 +84,8 @@ import static com.mongodb.client.test.CollectionHelper.getCurrentClusterTime; import static com.mongodb.client.test.CollectionHelper.killAllSessions; import static com.mongodb.client.unified.RunOnRequirementsMatcher.runOnRequirementsMet; -import static com.mongodb.client.unified.UnifiedTestModifications.doSkips; +import static com.mongodb.client.unified.UnifiedTestModifications.Modifier; +import static com.mongodb.client.unified.UnifiedTestModifications.applyCustomizations; import static com.mongodb.client.unified.UnifiedTestModifications.testDef; import static java.util.Collections.singletonList; import static java.util.stream.Collectors.toList; @@ -174,10 +175,10 @@ protected static Collection getTestData(final String directory, final String testDescription = testDocument.getString("description").getValue(); String fileDescription = fileDocument.getString("description").getValue(); TestDef testDef = testDef(directory, fileDescription, testDescription, isReactive); - doSkips(testDef); + applyCustomizations(testDef); - boolean forceFlaky = testDef.wasAssignedModifier(UnifiedTestModifications.Modifier.FORCE_FLAKY); - boolean retry = forceFlaky || testDef.wasAssignedModifier(UnifiedTestModifications.Modifier.RETRY); + boolean forceFlaky = testDef.wasAssignedModifier(Modifier.FORCE_FLAKY); + boolean retry = forceFlaky || testDef.wasAssignedModifier(Modifier.RETRY); int attempts = retry ? ATTEMPTS : 1; if (forceFlaky) { @@ -243,9 +244,9 @@ public void setUp( ignoreExtraEvents = false; if (directoryName != null && fileDescription != null && testDescription != null) { testDef = testDef(directoryName, fileDescription, testDescription, isReactive()); - UnifiedTestModifications.doSkips(testDef); + applyCustomizations(testDef); - boolean skip = testDef.wasAssignedModifier(UnifiedTestModifications.Modifier.SKIP); + boolean skip = testDef.wasAssignedModifier(Modifier.SKIP); assumeFalse(skip, "Skipping test"); } skips(fileDescription, testDescription); @@ -376,6 +377,12 @@ public void shouldPassAllOutcomes( compareLogMessages(rootContext, definition, tweaks); } } catch (AssertionFailedError e) { + assertTrue(testDef.wasAssignedModifier(Modifier.RETRY)); + if (!testDef.matchesError(e)) { + // if the error is not matched, test definitions were not intended to apply; throw it + throw e; + } + completed.remove(testName); boolean lastAttempt = attemptNumber == Math.abs(totalAttempts); if (forceFlaky || lastAttempt) { diff --git a/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTestModifications.java b/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTestModifications.java index f1bf20dd788..e33d7064b23 100644 --- a/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTestModifications.java +++ b/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTestModifications.java @@ -17,10 +17,12 @@ package com.mongodb.client.unified; import com.mongodb.assertions.Assertions; +import org.opentest4j.AssertionFailedError; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.function.Function; import java.util.function.Supplier; import static com.mongodb.ClusterFixture.isDataLakeTest; @@ -28,23 +30,28 @@ import static com.mongodb.ClusterFixture.isServerlessTest; import static com.mongodb.ClusterFixture.isSharded; import static com.mongodb.ClusterFixture.serverVersionLessThan; -import static com.mongodb.client.unified.UnifiedTestModifications.Modifier.FORCE_FLAKY; import static com.mongodb.assertions.Assertions.assertNotNull; import static com.mongodb.client.unified.UnifiedTestModifications.Modifier.IGNORE_EXTRA_EVENTS; +import static com.mongodb.client.unified.UnifiedTestModifications.Modifier.RETRY; import static com.mongodb.client.unified.UnifiedTestModifications.Modifier.SKIP; import static com.mongodb.client.unified.UnifiedTestModifications.Modifier.SLEEP_AFTER_CURSOR_CLOSE; import static com.mongodb.client.unified.UnifiedTestModifications.Modifier.SLEEP_AFTER_CURSOR_OPEN; import static com.mongodb.client.unified.UnifiedTestModifications.Modifier.WAIT_FOR_BATCH_CURSOR_CREATION; public final class UnifiedTestModifications { - public static void doSkips(final TestDef def) { + public static void applyCustomizations(final TestDef def) { // TODO reasons for retry - def.modify(FORCE_FLAKY) - // Exception in encryption library: ChangeCipherSpec message sequence violation - .test("client-side-encryption", "namedKMS-createDataKey", "create datakey with named KMIP KMS provider") - // Number of checked out connections must match expected - .test("load-balancers", "cursors are correctly pinned to connections for load-balanced clusters", "pinned connections are returned after a network error during a killCursors request") + // Exception in encryption library: ChangeCipherSpec message sequence violation + def.retry("TODO reason") + .whenExceptionContains("ChangeCipherSpec message sequence violation") + .test("client-side-encryption", "namedKMS-createDataKey", "create datakey with named KMIP KMS provider"); + + def.retry("TODO reason") + .whenExceptionContains("Number of checked out connections must match expected") + .test("load-balancers", "cursors are correctly pinned to connections for load-balanced clusters", "pinned connections are returned after a network error during a killCursors request"); + + def.retry("TODO reason") .test("client-side-encryption", "namedKMS-rewrapManyDataKey", "rewrap to kmip:name1"); // atlas-data-lake @@ -268,6 +275,7 @@ public static final class TestDef { private final boolean reactive; private final List modifiers = new ArrayList<>(); + private Function matchesError; private TestDef(final String dir, final String file, final String test, final boolean reactive) { this.dir = assertNotNull(dir); @@ -331,6 +339,14 @@ public TestApplicator skipUnknownReason(final String reason) { return new TestApplicator(this, reason, SKIP); } + + /** + * The test will be retried, for the reason provided + */ + public TestApplicator retry(final String reason) { + return new TestApplicator(this, reason, RETRY); + } + public TestApplicator modify(final Modifier... modifiers) { return new TestApplicator(this, null, modifiers); } @@ -342,6 +358,13 @@ public boolean isReactive() { public boolean wasAssignedModifier(final Modifier modifier) { return this.modifiers.contains(modifier); } + + public boolean matchesError(final AssertionFailedError e) { + if (matchesError != null) { + return matchesError.apply(e); + } + return false; + } } /** @@ -349,17 +372,19 @@ public boolean wasAssignedModifier(final Modifier modifier) { */ public static final class TestApplicator { private final TestDef testDef; - private final List modifiersToApply; private Supplier precondition; private boolean matchWasPerformed = false; + private final List modifiersToApply; + private Function matchesError; + private TestApplicator( final TestDef testDef, final String reason, final Modifier... modifiersToApply) { this.testDef = testDef; this.modifiersToApply = Arrays.asList(modifiersToApply); - if (this.modifiersToApply.contains(SKIP)) { + if (this.modifiersToApply.contains(SKIP) || this.modifiersToApply.contains(RETRY)) { assertNotNull(reason); } } @@ -371,6 +396,7 @@ private TestApplicator onMatch(final boolean match) { } if (match) { this.testDef.modifiers.addAll(this.modifiersToApply); + this.testDef.matchesError = this.matchesError; } return this; } @@ -462,6 +488,13 @@ public TestApplicator when(final Supplier precondition) { this.precondition = precondition; return this; } + + public TestApplicator whenExceptionContains(final String fragment) { + this.matchesError = (final AssertionFailedError e) -> { + return e.getCause().getMessage().contains(fragment); + }; + return this; + } } public enum Modifier { From 6402460fdad798570d573a954b691da52223aaf3 Mon Sep 17 00:00:00 2001 From: Maxim Katcharov Date: Thu, 5 Dec 2024 16:15:31 -0700 Subject: [PATCH 04/10] Assertion fixes, clarity --- .../mongodb/client/unified/UnifiedTest.java | 30 ++++++++++--------- .../unified/UnifiedTestModifications.java | 27 ++++++++++------- 2 files changed, 32 insertions(+), 25 deletions(-) diff --git a/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTest.java b/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTest.java index 5592fe64a56..7541d5d2895 100644 --- a/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTest.java +++ b/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTest.java @@ -57,7 +57,6 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; -import org.opentest4j.AssertionFailedError; import org.opentest4j.TestAbortedException; import java.io.File; @@ -96,6 +95,7 @@ import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; +import static org.junit.jupiter.api.Assumptions.abort; import static org.junit.jupiter.api.Assumptions.assumeFalse; import static org.junit.jupiter.api.Assumptions.assumeTrue; import static util.JsonPoweredTestHelper.getTestDocument; @@ -107,7 +107,7 @@ public abstract class UnifiedTest { "wait queue timeout errors include details about checked out connections"); public static final int ATTEMPTS = 3; - private static Set completed = new HashSet<>(); + private static Set ignoreRemaining = new HashSet<>(); @Nullable private String fileDescription; @@ -348,8 +348,9 @@ public void shouldPassAllOutcomes( final BsonDocument definition) { boolean forceFlaky = totalAttempts < 0; if (!forceFlaky) { - assumeFalse(completed.contains(testName), "Skipping retryable test that succeeded"); - completed.add(testName); + boolean ignoreThisTest = ignoreRemaining.contains(testName); + assumeFalse(ignoreThisTest, "Skipping a retryable test that already succeeded"); + ignoreRemaining.add(testName); } try { BsonArray operations = definition.getArray("operations"); @@ -376,20 +377,21 @@ public void shouldPassAllOutcomes( } compareLogMessages(rootContext, definition, tweaks); } - } catch (AssertionFailedError e) { - assertTrue(testDef.wasAssignedModifier(Modifier.RETRY)); - if (!testDef.matchesError(e)) { - // if the error is not matched, test definitions were not intended to apply; throw it + } catch (Throwable e) { + if (forceFlaky) { throw e; } - - completed.remove(testName); - boolean lastAttempt = attemptNumber == Math.abs(totalAttempts); - if (forceFlaky || lastAttempt) { + if (!testDef.matchesThrowable(e)) { + // if the throwable is not matched, test definitions were not intended to apply; rethrow it + throw e; + } + boolean isLastAttempt = attemptNumber == Math.abs(totalAttempts); + if (isLastAttempt) { throw e; - } else { - assumeFalse(completed.contains(testName), "Ignoring failure and retrying attempt " + attemptNumber); } + + ignoreRemaining.remove(testName); + abort("Ignoring failure and retrying attempt " + attemptNumber); } } diff --git a/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTestModifications.java b/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTestModifications.java index e33d7064b23..0e895f4c807 100644 --- a/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTestModifications.java +++ b/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTestModifications.java @@ -44,11 +44,11 @@ public static void applyCustomizations(final TestDef def) { // TODO reasons for retry // Exception in encryption library: ChangeCipherSpec message sequence violation def.retry("TODO reason") - .whenExceptionContains("ChangeCipherSpec message sequence violation") + .whenFailureContains("ChangeCipherSpec message sequence violation") .test("client-side-encryption", "namedKMS-createDataKey", "create datakey with named KMIP KMS provider"); def.retry("TODO reason") - .whenExceptionContains("Number of checked out connections must match expected") + .whenFailureContains("Number of checked out connections must match expected") .test("load-balancers", "cursors are correctly pinned to connections for load-balanced clusters", "pinned connections are returned after a network error during a killCursors request"); def.retry("TODO reason") @@ -275,7 +275,7 @@ public static final class TestDef { private final boolean reactive; private final List modifiers = new ArrayList<>(); - private Function matchesError; + private Function matchesThrowable; private TestDef(final String dir, final String file, final String test, final boolean reactive) { this.dir = assertNotNull(dir); @@ -359,9 +359,9 @@ public boolean wasAssignedModifier(final Modifier modifier) { return this.modifiers.contains(modifier); } - public boolean matchesError(final AssertionFailedError e) { - if (matchesError != null) { - return matchesError.apply(e); + public boolean matchesThrowable(final Throwable e) { + if (matchesThrowable != null) { + return matchesThrowable.apply(e); } return false; } @@ -376,7 +376,7 @@ public static final class TestApplicator { private boolean matchWasPerformed = false; private final List modifiersToApply; - private Function matchesError; + private Function matchesThrowable; private TestApplicator( final TestDef testDef, @@ -396,7 +396,7 @@ private TestApplicator onMatch(final boolean match) { } if (match) { this.testDef.modifiers.addAll(this.modifiersToApply); - this.testDef.matchesError = this.matchesError; + this.testDef.matchesThrowable = this.matchesThrowable; } return this; } @@ -489,9 +489,14 @@ public TestApplicator when(final Supplier precondition) { return this; } - public TestApplicator whenExceptionContains(final String fragment) { - this.matchesError = (final AssertionFailedError e) -> { - return e.getCause().getMessage().contains(fragment); + public TestApplicator whenFailureContains(final String messageFragment) { + this.matchesThrowable = (final Throwable e) -> { + // inspect the cause for failed assertions with a cause + if (e instanceof AssertionFailedError && e.getCause() != null) { + return e.getCause().getMessage().contains(messageFragment); + } else { + return e.getMessage().contains(messageFragment); + } }; return this; } From f62539ed21aabcef2494e51831cb2b64a91933d5 Mon Sep 17 00:00:00 2001 From: Maxim Katcharov Date: Thu, 5 Dec 2024 16:57:16 -0700 Subject: [PATCH 05/10] Add docs --- .../com/mongodb/client/unified/UnifiedTest.java | 2 +- .../client/unified/UnifiedTestModifications.java | 14 ++++++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTest.java b/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTest.java index 7541d5d2895..d9bffb29678 100644 --- a/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTest.java +++ b/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTest.java @@ -389,7 +389,7 @@ public void shouldPassAllOutcomes( if (isLastAttempt) { throw e; } - + ignoreRemaining.remove(testName); abort("Ignoring failure and retrying attempt " + attemptNumber); } diff --git a/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTestModifications.java b/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTestModifications.java index 0e895f4c807..e48c96972ef 100644 --- a/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTestModifications.java +++ b/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTestModifications.java @@ -489,6 +489,12 @@ public TestApplicator when(final Supplier precondition) { return this; } + /** + * The modification, if it is a RETRY, will only be applied when the + * failure message contains the provided message fragment. If an + * {@code AssertionFailedError} occurs, and has a cause, the cause's + * message will be checked. Otherwise, the throwable will be checked. + */ public TestApplicator whenFailureContains(final String messageFragment) { this.matchesThrowable = (final Throwable e) -> { // inspect the cause for failed assertions with a cause @@ -526,11 +532,15 @@ public enum Modifier { */ SKIP, /** - * Retry the test on failure. + * Ignore results and retry the test on failure. Will not repeat the + * test if the test succeeds. Multiple copies of the test are used to + * facilitate retries. */ RETRY, /** - * Retry the test multiple times, without ignoring failures. + * The test will be retried multiple times, without the results being + * ignored. This is a helper that can be used, in patches, to check + * if certain tests are (still) flaky. */ FORCE_FLAKY, } From 2d4f4124d1464ba7a081881b6ca091c5efd9831c Mon Sep 17 00:00:00 2001 From: Maxim Katcharov Date: Wed, 26 Feb 2025 09:15:37 -0700 Subject: [PATCH 06/10] PR fixes --- .../main/com/mongodb/assertions/Assertions.java | 13 +++++++++++++ .../com/mongodb/client/unified/UnifiedTest.java | 16 +++++++++++----- .../client/unified/UnifiedTestModifications.java | 7 +++++-- 3 files changed, 29 insertions(+), 7 deletions(-) diff --git a/driver-core/src/main/com/mongodb/assertions/Assertions.java b/driver-core/src/main/com/mongodb/assertions/Assertions.java index a40b4e4b7b6..bf38638dc6d 100644 --- a/driver-core/src/main/com/mongodb/assertions/Assertions.java +++ b/driver-core/src/main/com/mongodb/assertions/Assertions.java @@ -179,6 +179,19 @@ public static boolean assertTrue(final boolean value) throws AssertionError { return true; } + /** + * @param value A value to check. + * @param message The message. + * @return {@code true}. + * @throws AssertionError If {@code value} is {@code false}. + */ + public static boolean assertTrue(final boolean value, final String message) throws AssertionError { + if (!value) { + throw new AssertionError(message); + } + return true; + } + /** * @param value A value to check. * @return {@code false}. diff --git a/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTest.java b/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTest.java index d9bffb29678..b4b00934a62 100644 --- a/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTest.java +++ b/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTest.java @@ -107,7 +107,7 @@ public abstract class UnifiedTest { "wait queue timeout errors include details about checked out connections"); public static final int ATTEMPTS = 3; - private static Set ignoreRemaining = new HashSet<>(); + private static final Set ATTEMPTED_TESTS_TO_HENCEFORTH_IGNORE = new HashSet<>(); @Nullable private String fileDescription; @@ -348,9 +348,12 @@ public void shouldPassAllOutcomes( final BsonDocument definition) { boolean forceFlaky = totalAttempts < 0; if (!forceFlaky) { - boolean ignoreThisTest = ignoreRemaining.contains(testName); + boolean ignoreThisTest = ATTEMPTED_TESTS_TO_HENCEFORTH_IGNORE.contains(testName); assumeFalse(ignoreThisTest, "Skipping a retryable test that already succeeded"); - ignoreRemaining.add(testName); + // The attempt is what counts, since a test may fail with + // something like "ignored", and would not be retried. + // Only failures should trigger another attempt. + ATTEMPTED_TESTS_TO_HENCEFORTH_IGNORE.add(testName); } try { BsonArray operations = definition.getArray("operations"); @@ -377,11 +380,14 @@ public void shouldPassAllOutcomes( } compareLogMessages(rootContext, definition, tweaks); } + } catch (TestAbortedException e) { + // if a test is ignored, we do not retry + throw e; } catch (Throwable e) { if (forceFlaky) { throw e; } - if (!testDef.matchesThrowable(e)) { + if (testDef!= null && !testDef.matchesThrowable(e)) { // if the throwable is not matched, test definitions were not intended to apply; rethrow it throw e; } @@ -390,7 +396,7 @@ public void shouldPassAllOutcomes( throw e; } - ignoreRemaining.remove(testName); + ATTEMPTED_TESTS_TO_HENCEFORTH_IGNORE.remove(testName); abort("Ignoring failure and retrying attempt " + attemptNumber); } } diff --git a/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTestModifications.java b/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTestModifications.java index e48c96972ef..2ce2daa4b66 100644 --- a/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTestModifications.java +++ b/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTestModifications.java @@ -16,7 +16,6 @@ package com.mongodb.client.unified; -import com.mongodb.assertions.Assertions; import org.opentest4j.AssertionFailedError; import java.util.ArrayList; @@ -31,12 +30,14 @@ import static com.mongodb.ClusterFixture.isSharded; import static com.mongodb.ClusterFixture.serverVersionLessThan; import static com.mongodb.assertions.Assertions.assertNotNull; +import static com.mongodb.assertions.Assertions.assertTrue; import static com.mongodb.client.unified.UnifiedTestModifications.Modifier.IGNORE_EXTRA_EVENTS; import static com.mongodb.client.unified.UnifiedTestModifications.Modifier.RETRY; import static com.mongodb.client.unified.UnifiedTestModifications.Modifier.SKIP; import static com.mongodb.client.unified.UnifiedTestModifications.Modifier.SLEEP_AFTER_CURSOR_CLOSE; import static com.mongodb.client.unified.UnifiedTestModifications.Modifier.SLEEP_AFTER_CURSOR_OPEN; import static com.mongodb.client.unified.UnifiedTestModifications.Modifier.WAIT_FOR_BATCH_CURSOR_CREATION; +import static java.lang.String.format; public final class UnifiedTestModifications { public static void applyCustomizations(final TestDef def) { @@ -291,7 +292,7 @@ private TestDef(final String dir, final String file, final String test, final bo * @param ticket reason for skipping the test; must start with a Jira URL */ public TestApplicator skipJira(final String ticket) { - Assertions.assertTrue(ticket.startsWith("https://jira.mongodb.org/browse/JAVA-")); + assertTrue(ticket.startsWith("https://jira.mongodb.org/browse/JAVA-")); return new TestApplicator(this, ticket, SKIP); } @@ -496,6 +497,8 @@ public TestApplicator when(final Supplier precondition) { * message will be checked. Otherwise, the throwable will be checked. */ public TestApplicator whenFailureContains(final String messageFragment) { + assertTrue(this.modifiersToApply.contains(RETRY), + format("Modifier %s was not specified before calling whenFailureContains", RETRY)); this.matchesThrowable = (final Throwable e) -> { // inspect the cause for failed assertions with a cause if (e instanceof AssertionFailedError && e.getCause() != null) { From 919845afb66be1f8488146f5c8a767ee02508491 Mon Sep 17 00:00:00 2001 From: Maxim Katcharov Date: Thu, 27 Feb 2025 14:45:06 -0700 Subject: [PATCH 07/10] checkstyle fix --- .../test/functional/com/mongodb/client/unified/UnifiedTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTest.java b/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTest.java index b4b00934a62..879fcd512bc 100644 --- a/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTest.java +++ b/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTest.java @@ -387,7 +387,7 @@ public void shouldPassAllOutcomes( if (forceFlaky) { throw e; } - if (testDef!= null && !testDef.matchesThrowable(e)) { + if (testDef != null && !testDef.matchesThrowable(e)) { // if the throwable is not matched, test definitions were not intended to apply; rethrow it throw e; } From 85d662ec0ad02468a5fcaad6e6844b4f79168980 Mon Sep 17 00:00:00 2001 From: Maxim Katcharov Date: Fri, 7 Mar 2025 11:32:42 -0700 Subject: [PATCH 08/10] PR fixes --- ...tClientSideOperationsTimeoutProseTest.java | 12 ++++++------ .../mongodb/client/unified/UnifiedTest.java | 19 +++++++++++-------- .../unified/UnifiedTestModifications.java | 19 +++++++++---------- 3 files changed, 26 insertions(+), 24 deletions(-) diff --git a/driver-sync/src/test/functional/com/mongodb/client/AbstractClientSideOperationsTimeoutProseTest.java b/driver-sync/src/test/functional/com/mongodb/client/AbstractClientSideOperationsTimeoutProseTest.java index 8eb47aa0a6c..2bb1b2d9c68 100644 --- a/driver-sync/src/test/functional/com/mongodb/client/AbstractClientSideOperationsTimeoutProseTest.java +++ b/driver-sync/src/test/functional/com/mongodb/client/AbstractClientSideOperationsTimeoutProseTest.java @@ -134,7 +134,7 @@ protected int postSessionCloseSleep() { } @SuppressWarnings("try") - @FlakyTest(maxAttempts = 3) + @FlakyTest(maxAttempts = 3) // TODO-JAVA-5809 @DisplayName("4. Background Connection Pooling - timeoutMS used for handshake commands") public void testBackgroundConnectionPoolingTimeoutMSUsedForHandshakeCommands() { assumeTrue(serverVersionAtLeast(4, 4)); @@ -171,7 +171,7 @@ public void testBackgroundConnectionPoolingTimeoutMSUsedForHandshakeCommands() { } @SuppressWarnings("try") - @FlakyTest(maxAttempts = 3) + @FlakyTest(maxAttempts = 3) // TODO-JAVA-5809 @DisplayName("4. Background Connection Pooling - timeoutMS is refreshed for each handshake command") public void testBackgroundConnectionPoolingTimeoutMSIsRefreshedForEachHandshakeCommand() { assumeTrue(serverVersionAtLeast(4, 4)); @@ -205,7 +205,7 @@ public void testBackgroundConnectionPoolingTimeoutMSIsRefreshedForEachHandshakeC } } - @FlakyTest(maxAttempts = 3) + @FlakyTest(maxAttempts = 3) // TODO-JAVA-5809 @DisplayName("5. Blocking Iteration Methods - Tailable cursors") public void testBlockingIterationMethodsTailableCursor() { assumeTrue(serverVersionAtLeast(4, 4)); @@ -242,7 +242,7 @@ public void testBlockingIterationMethodsTailableCursor() { } } - @FlakyTest(maxAttempts = 3) + @FlakyTest(maxAttempts = 3) // TODO-JAVA-5809 @DisplayName("5. Blocking Iteration Methods - Change Streams") public void testBlockingIterationMethodsChangeStream() { assumeTrue(serverVersionAtLeast(4, 4)); @@ -290,7 +290,7 @@ public void testBlockingIterationMethodsChangeStream() { } @DisplayName("6. GridFS Upload - uploads via openUploadStream can be timed out") - @FlakyTest(maxAttempts = 3) + @FlakyTest(maxAttempts = 3) // TODO-JAVA-5809 public void testGridFSUploadViaOpenUploadStreamTimeout() { assumeTrue(serverVersionAtLeast(4, 4)); long rtt = ClusterFixture.getPrimaryRTT(); @@ -469,7 +469,7 @@ public void test8ServerSelectionHandshake(final String ignoredTestName, final in @SuppressWarnings("try") @DisplayName("9. End Session. The timeout specified via the MongoClient timeoutMS option") - @FlakyTest(maxAttempts = 3) + @FlakyTest(maxAttempts = 3) // TODO-JAVA-5809 public void test9EndSessionClientTimeout() { assumeTrue(serverVersionAtLeast(4, 4)); assumeFalse(isStandalone()); diff --git a/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTest.java b/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTest.java index 879fcd512bc..a437084ac1d 100644 --- a/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTest.java +++ b/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTest.java @@ -106,7 +106,8 @@ public abstract class UnifiedTest { private static final Set PRESTART_POOL_ASYNC_WORK_MANAGER_FILE_DESCRIPTIONS = Collections.singleton( "wait queue timeout errors include details about checked out connections"); - public static final int ATTEMPTS = 3; + public static final int RETRY_ATTEMPTS = 3; + public static final int FORCE_FLAKY_ATTEMPTS = 10; private static final Set ATTEMPTED_TESTS_TO_HENCEFORTH_IGNORE = new HashSet<>(); @Nullable @@ -180,9 +181,11 @@ protected static Collection getTestData(final String directory, final boolean forceFlaky = testDef.wasAssignedModifier(Modifier.FORCE_FLAKY); boolean retry = forceFlaky || testDef.wasAssignedModifier(Modifier.RETRY); - int attempts = retry ? ATTEMPTS : 1; - if (forceFlaky) { - attempts = 10; + int attempts; + if (retry) { + attempts = forceFlaky ? FORCE_FLAKY_ATTEMPTS : RETRY_ATTEMPTS; + } else { + attempts = 1; } for (int attempt = 1; attempt <= attempts; attempt++) { @@ -193,12 +196,12 @@ protected static Collection getTestData(final String directory, final testDescription, directory, attempt, - attempts * (forceFlaky ? -1 : 1), + attempts, fileDocument.getString("schemaVersion").getValue(), fileDocument.getArray("runOnRequirements", null), fileDocument.getArray("createEntities", new BsonArray()), fileDocument.getArray("initialData", new BsonArray()), - testDocument)); + testDocument.clone())); } } } @@ -346,7 +349,7 @@ public void shouldPassAllOutcomes( final BsonArray entitiesArray, final BsonArray initialData, final BsonDocument definition) { - boolean forceFlaky = totalAttempts < 0; + boolean forceFlaky = testDef.wasAssignedModifier(Modifier.FORCE_FLAKY); if (!forceFlaky) { boolean ignoreThisTest = ATTEMPTED_TESTS_TO_HENCEFORTH_IGNORE.contains(testName); assumeFalse(ignoreThisTest, "Skipping a retryable test that already succeeded"); @@ -391,7 +394,7 @@ public void shouldPassAllOutcomes( // if the throwable is not matched, test definitions were not intended to apply; rethrow it throw e; } - boolean isLastAttempt = attemptNumber == Math.abs(totalAttempts); + boolean isLastAttempt = attemptNumber == totalAttempts; if (isLastAttempt) { throw e; } diff --git a/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTestModifications.java b/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTestModifications.java index 2ce2daa4b66..613d84b9209 100644 --- a/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTestModifications.java +++ b/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTestModifications.java @@ -42,17 +42,16 @@ public final class UnifiedTestModifications { public static void applyCustomizations(final TestDef def) { - // TODO reasons for retry // Exception in encryption library: ChangeCipherSpec message sequence violation - def.retry("TODO reason") + def.retry("") // TODO-JAVA-5809 .whenFailureContains("ChangeCipherSpec message sequence violation") .test("client-side-encryption", "namedKMS-createDataKey", "create datakey with named KMIP KMS provider"); - def.retry("TODO reason") + def.retry("") // TODO-JAVA-5809 .whenFailureContains("Number of checked out connections must match expected") .test("load-balancers", "cursors are correctly pinned to connections for load-balanced clusters", "pinned connections are returned after a network error during a killCursors request"); - def.retry("TODO reason") + def.retry("") // TODO-JAVA-5809 .test("client-side-encryption", "namedKMS-rewrapManyDataKey", "rewrap to kmip:name1"); // atlas-data-lake @@ -62,7 +61,7 @@ public static void applyCustomizations(final TestDef def) { .directory("atlas-data-lake-testing"); // change-streams - def.skipNoncompliantReactive("error required from change stream initialization") // TODO reason? + def.skipNoncompliantReactive("error required from change stream initialization") // TODO-JAVA-5711 reason? .test("change-streams", "change-streams", "Test with document comment - pre 4.4"); def.skipNoncompliantReactive("event sensitive tests. We can't guarantee the amount of GetMore commands sent in the reactive driver") .test("change-streams", "change-streams", "Test that comment is set on getMore") @@ -78,17 +77,17 @@ public static void applyCustomizations(final TestDef def) { // client-side-operation-timeout (CSOT) - // TODO + // TODO-JAVA-5712 // collection-management - def.skipNoncompliant("") // TODO reason? + def.skipNoncompliant("") // TODO-JAVA-5711 reason? .test("collection-management", "modifyCollection-pre_and_post_images", "modifyCollection to changeStreamPreAndPostImages enabled"); // command-logging-and-monitoring - def.skipNoncompliant("TODO") - .when(() -> !def.isReactive() && isServerlessTest()) // TODO why reactive check? + def.skipNoncompliant("") // TODO-JAVA-5711 + .when(() -> !def.isReactive() && isServerlessTest()) // TODO-JAVA-5711 why reactive check? .directory("command-logging") .directory("command-monitoring"); @@ -101,7 +100,7 @@ public static void applyCustomizations(final TestDef def) { // connection-monitoring-and-pooling - // TODO reason, jira + // TODO-JAVA-5711 reason, jira // added as part of https://jira.mongodb.org/browse/JAVA-4976 , but unknown Jira to complete // The implementation of the functionality related to clearing the connection pool before closing the connection // will be carried out once the specification is finalized and ready. From db1e58721e772c9814e74513e96c75696d6fbc08 Mon Sep 17 00:00:00 2001 From: Maxim Katcharov Date: Mon, 10 Mar 2025 08:01:46 -0600 Subject: [PATCH 09/10] Update unified test retries --- .../unified/UnifiedTestModifications.java | 26 ++++++++++++++++--- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTestModifications.java b/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTestModifications.java index 613d84b9209..39e63a39660 100644 --- a/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTestModifications.java +++ b/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTestModifications.java @@ -43,17 +43,28 @@ public final class UnifiedTestModifications { public static void applyCustomizations(final TestDef def) { // Exception in encryption library: ChangeCipherSpec message sequence violation - def.retry("") // TODO-JAVA-5809 + def.retryReactive("") // TODO-JAVA-5809 .whenFailureContains("ChangeCipherSpec message sequence violation") .test("client-side-encryption", "namedKMS-createDataKey", "create datakey with named KMIP KMS provider"); - def.retry("") // TODO-JAVA-5809 + def.retryReactive("") // TODO-JAVA-5809 .whenFailureContains("Number of checked out connections must match expected") .test("load-balancers", "cursors are correctly pinned to connections for load-balanced clusters", "pinned connections are returned after a network error during a killCursors request"); - def.retry("") // TODO-JAVA-5809 + def.retryReactive("") // TODO-JAVA-5809 .test("client-side-encryption", "namedKMS-rewrapManyDataKey", "rewrap to kmip:name1"); + def.retry("") // TODO-JAVA-5809 + .whenFailureContains("Read timed out") + .test("client-side-encryption", "rewrapManyDataKey", "rewrap with new Azure KMS provider") + .test("client-side-encryption", "rewrapManyDataKey", "rewrap with new KMIP KMS provider"); + + def.retryReactive("") // TODO-JAVA-5809 + .test("load-balancers", "cursors are correctly pinned to connections for load-balanced clusters", "pinned connections are returned after a network error during a killCursors request"); + + def.retryReactive("") // TODO-JAVA-5809 + .test("server-selection.logging", "operation-id", "Failed bulkWrite operation: log messages have operationIds"); + // atlas-data-lake def.skipAccordingToSpec("Data lake tests should only run on data lake") @@ -339,7 +350,6 @@ public TestApplicator skipUnknownReason(final String reason) { return new TestApplicator(this, reason, SKIP); } - /** * The test will be retried, for the reason provided */ @@ -347,6 +357,14 @@ public TestApplicator retry(final String reason) { return new TestApplicator(this, reason, RETRY); } + /** + * The reactive test will be retried, for the reason provided + */ + public TestApplicator retryReactive(final String reason) { + return new TestApplicator(this, reason, RETRY) + .when(this::isReactive); + } + public TestApplicator modify(final Modifier... modifiers) { return new TestApplicator(this, null, modifiers); } From 920de497877477deb55de2835eaec09d9196ea43 Mon Sep 17 00:00:00 2001 From: Maxim Katcharov Date: Mon, 10 Mar 2025 15:06:47 -0600 Subject: [PATCH 10/10] Remove skips and TODO labels --- ...tClientSideOperationsTimeoutProseTest.java | 12 +++++----- .../unified/UnifiedTestModifications.java | 23 ------------------- 2 files changed, 6 insertions(+), 29 deletions(-) diff --git a/driver-sync/src/test/functional/com/mongodb/client/AbstractClientSideOperationsTimeoutProseTest.java b/driver-sync/src/test/functional/com/mongodb/client/AbstractClientSideOperationsTimeoutProseTest.java index 2bb1b2d9c68..8eb47aa0a6c 100644 --- a/driver-sync/src/test/functional/com/mongodb/client/AbstractClientSideOperationsTimeoutProseTest.java +++ b/driver-sync/src/test/functional/com/mongodb/client/AbstractClientSideOperationsTimeoutProseTest.java @@ -134,7 +134,7 @@ protected int postSessionCloseSleep() { } @SuppressWarnings("try") - @FlakyTest(maxAttempts = 3) // TODO-JAVA-5809 + @FlakyTest(maxAttempts = 3) @DisplayName("4. Background Connection Pooling - timeoutMS used for handshake commands") public void testBackgroundConnectionPoolingTimeoutMSUsedForHandshakeCommands() { assumeTrue(serverVersionAtLeast(4, 4)); @@ -171,7 +171,7 @@ public void testBackgroundConnectionPoolingTimeoutMSUsedForHandshakeCommands() { } @SuppressWarnings("try") - @FlakyTest(maxAttempts = 3) // TODO-JAVA-5809 + @FlakyTest(maxAttempts = 3) @DisplayName("4. Background Connection Pooling - timeoutMS is refreshed for each handshake command") public void testBackgroundConnectionPoolingTimeoutMSIsRefreshedForEachHandshakeCommand() { assumeTrue(serverVersionAtLeast(4, 4)); @@ -205,7 +205,7 @@ public void testBackgroundConnectionPoolingTimeoutMSIsRefreshedForEachHandshakeC } } - @FlakyTest(maxAttempts = 3) // TODO-JAVA-5809 + @FlakyTest(maxAttempts = 3) @DisplayName("5. Blocking Iteration Methods - Tailable cursors") public void testBlockingIterationMethodsTailableCursor() { assumeTrue(serverVersionAtLeast(4, 4)); @@ -242,7 +242,7 @@ public void testBlockingIterationMethodsTailableCursor() { } } - @FlakyTest(maxAttempts = 3) // TODO-JAVA-5809 + @FlakyTest(maxAttempts = 3) @DisplayName("5. Blocking Iteration Methods - Change Streams") public void testBlockingIterationMethodsChangeStream() { assumeTrue(serverVersionAtLeast(4, 4)); @@ -290,7 +290,7 @@ public void testBlockingIterationMethodsChangeStream() { } @DisplayName("6. GridFS Upload - uploads via openUploadStream can be timed out") - @FlakyTest(maxAttempts = 3) // TODO-JAVA-5809 + @FlakyTest(maxAttempts = 3) public void testGridFSUploadViaOpenUploadStreamTimeout() { assumeTrue(serverVersionAtLeast(4, 4)); long rtt = ClusterFixture.getPrimaryRTT(); @@ -469,7 +469,7 @@ public void test8ServerSelectionHandshake(final String ignoredTestName, final in @SuppressWarnings("try") @DisplayName("9. End Session. The timeout specified via the MongoClient timeoutMS option") - @FlakyTest(maxAttempts = 3) // TODO-JAVA-5809 + @FlakyTest(maxAttempts = 3) public void test9EndSessionClientTimeout() { assumeTrue(serverVersionAtLeast(4, 4)); assumeFalse(isStandalone()); diff --git a/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTestModifications.java b/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTestModifications.java index 39e63a39660..df9706b8dd7 100644 --- a/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTestModifications.java +++ b/driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTestModifications.java @@ -42,29 +42,6 @@ public final class UnifiedTestModifications { public static void applyCustomizations(final TestDef def) { - // Exception in encryption library: ChangeCipherSpec message sequence violation - def.retryReactive("") // TODO-JAVA-5809 - .whenFailureContains("ChangeCipherSpec message sequence violation") - .test("client-side-encryption", "namedKMS-createDataKey", "create datakey with named KMIP KMS provider"); - - def.retryReactive("") // TODO-JAVA-5809 - .whenFailureContains("Number of checked out connections must match expected") - .test("load-balancers", "cursors are correctly pinned to connections for load-balanced clusters", "pinned connections are returned after a network error during a killCursors request"); - - def.retryReactive("") // TODO-JAVA-5809 - .test("client-side-encryption", "namedKMS-rewrapManyDataKey", "rewrap to kmip:name1"); - - def.retry("") // TODO-JAVA-5809 - .whenFailureContains("Read timed out") - .test("client-side-encryption", "rewrapManyDataKey", "rewrap with new Azure KMS provider") - .test("client-side-encryption", "rewrapManyDataKey", "rewrap with new KMIP KMS provider"); - - def.retryReactive("") // TODO-JAVA-5809 - .test("load-balancers", "cursors are correctly pinned to connections for load-balanced clusters", "pinned connections are returned after a network error during a killCursors request"); - - def.retryReactive("") // TODO-JAVA-5809 - .test("server-selection.logging", "operation-id", "Failed bulkWrite operation: log messages have operationIds"); - // atlas-data-lake def.skipAccordingToSpec("Data lake tests should only run on data lake")