Skip to content

Commit 8e8d7a3

Browse files
katcharovkatcharov
committed
Retry flaky unified tests
JAVA-5393
1 parent 90976e4 commit 8e8d7a3

File tree

9 files changed

+156
-66
lines changed

9 files changed

+156
-66
lines changed

‎driver-kotlin-coroutine/src/integration/kotlin/com/mongodb/kotlin/client/coroutine/UnifiedCrudTest.kt

+1-1
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ internal class UnifiedCrudTest() : UnifiedTest() {
2424
@JvmStatic
2525
@Throws(URISyntaxException::class, IOException::class)
2626
fun data(): Collection<Arguments>? {
27-
return getTestData("unified-test-format/crud")
27+
return getTestData("unified-test-format/crud", true)
2828
}
2929
}
3030
}

‎driver-kotlin-sync/src/integration/kotlin/com/mongodb/kotlin/client/UnifiedCrudTest.kt

+1-1
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ internal class UnifiedCrudTest() : UnifiedTest() {
2424
@JvmStatic
2525
@Throws(URISyntaxException::class, IOException::class)
2626
fun data(): Collection<Arguments>? {
27-
return getTestData("unified-test-format/crud")
27+
return getTestData("unified-test-format/crud", false)
2828
}
2929
}
3030
}

‎driver-reactive-streams/src/test/functional/com/mongodb/reactivestreams/client/unified/ClientSideOperationTimeoutTest.java

+8-1
Original file line numberDiff line numberDiff line change
@@ -99,18 +99,25 @@ The Reactive Streams specification prevents us from allowing a subsequent next c
9999
@MethodSource("data")
100100
@Override
101101
public void shouldPassAllOutcomes(
102+
final String testName,
102103
@Nullable final String fileDescription,
103104
@Nullable final String testDescription,
104105
@Nullable final String directoryName,
106+
final int attemptNumber,
107+
final int totalAttempts,
105108
final String schemaVersion,
106109
@Nullable final BsonArray runOnRequirements,
107110
final BsonArray entitiesArray,
108111
final BsonArray initialData,
109112
final BsonDocument definition) {
110113
try {
111-
super.shouldPassAllOutcomes(fileDescription,
114+
super.shouldPassAllOutcomes(
115+
testName,
116+
fileDescription,
112117
testDescription,
113118
directoryName,
119+
attemptNumber,
120+
totalAttempts,
114121
schemaVersion,
115122
runOnRequirements,
116123
entitiesArray,

‎driver-reactive-streams/src/test/functional/com/mongodb/reactivestreams/client/unified/UnifiedReactiveStreamsTest.java

+11
Original file line numberDiff line numberDiff line change
@@ -24,13 +24,19 @@
2424
import com.mongodb.client.unified.UnifiedTest;
2525
import com.mongodb.client.unified.UnifiedTestModifications;
2626
import com.mongodb.client.vault.ClientEncryption;
27+
import com.mongodb.lang.NonNull;
2728
import com.mongodb.reactivestreams.client.MongoClients;
2829
import com.mongodb.reactivestreams.client.gridfs.GridFSBuckets;
2930
import com.mongodb.reactivestreams.client.internal.vault.ClientEncryptionImpl;
3031
import com.mongodb.reactivestreams.client.syncadapter.SyncClientEncryption;
3132
import com.mongodb.reactivestreams.client.syncadapter.SyncGridFSBucket;
3233
import com.mongodb.reactivestreams.client.syncadapter.SyncMongoClient;
3334
import com.mongodb.reactivestreams.client.syncadapter.SyncMongoDatabase;
35+
import org.junit.jupiter.params.provider.Arguments;
36+
37+
import java.io.IOException;
38+
import java.net.URISyntaxException;
39+
import java.util.Collection;
3440

3541
import static com.mongodb.client.unified.UnifiedTestModifications.Modifier;
3642
import static com.mongodb.client.unified.UnifiedTestModifications.TestDef;
@@ -94,4 +100,9 @@ protected void postCleanUp(final TestDef testDef) {
94100
disableSleep();
95101
}
96102
}
103+
104+
@NonNull
105+
protected static Collection<Arguments> getTestData(final String directory) throws URISyntaxException, IOException {
106+
return getTestData(directory, true);
107+
}
97108
}

‎driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedSyncTest.java

+11
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,12 @@
2525
import com.mongodb.client.gridfs.GridFSBuckets;
2626
import com.mongodb.client.internal.ClientEncryptionImpl;
2727
import com.mongodb.client.vault.ClientEncryption;
28+
import com.mongodb.lang.NonNull;
29+
import org.junit.jupiter.params.provider.Arguments;
30+
31+
import java.io.IOException;
32+
import java.net.URISyntaxException;
33+
import java.util.Collection;
2834

2935
public abstract class UnifiedSyncTest extends UnifiedTest {
3036
protected UnifiedSyncTest() {
@@ -44,4 +50,9 @@ protected GridFSBucket createGridFSBucket(final MongoDatabase database) {
4450
protected ClientEncryption createClientEncryption(final MongoClient keyVaultClient, final ClientEncryptionSettings clientEncryptionSettings) {
4551
return new ClientEncryptionImpl(keyVaultClient, clientEncryptionSettings);
4652
}
53+
54+
@NonNull
55+
protected static Collection<Arguments> getTestData(final String directory) throws URISyntaxException, IOException {
56+
return getTestData(directory, false);
57+
}
4758
}

‎driver-sync/src/test/functional/com/mongodb/client/unified/UnifiedTest.java

+88-43
Original file line numberDiff line numberDiff line change
@@ -21,27 +21,27 @@
2121
import com.mongodb.MongoNamespace;
2222
import com.mongodb.ReadPreference;
2323
import com.mongodb.UnixServerAddress;
24-
import com.mongodb.client.unified.UnifiedTestModifications.TestDef;
25-
import com.mongodb.event.TestServerMonitorListener;
26-
import com.mongodb.internal.logging.LogMessage;
27-
import com.mongodb.logging.TestLoggingInterceptor;
2824
import com.mongodb.WriteConcern;
2925
import com.mongodb.client.ClientSession;
3026
import com.mongodb.client.MongoClient;
3127
import com.mongodb.client.MongoDatabase;
3228
import com.mongodb.client.gridfs.GridFSBucket;
3329
import com.mongodb.client.model.Filters;
3430
import com.mongodb.client.test.CollectionHelper;
31+
import com.mongodb.client.unified.UnifiedTestModifications.TestDef;
3532
import com.mongodb.client.vault.ClientEncryption;
3633
import com.mongodb.connection.ClusterDescription;
3734
import com.mongodb.connection.ClusterType;
3835
import com.mongodb.connection.ServerDescription;
3936
import com.mongodb.event.CommandEvent;
4037
import com.mongodb.event.CommandStartedEvent;
38+
import com.mongodb.event.TestServerMonitorListener;
4139
import com.mongodb.internal.connection.TestCommandListener;
4240
import com.mongodb.internal.connection.TestConnectionPoolListener;
41+
import com.mongodb.internal.logging.LogMessage;
4342
import com.mongodb.lang.NonNull;
4443
import com.mongodb.lang.Nullable;
44+
import com.mongodb.logging.TestLoggingInterceptor;
4545
import com.mongodb.test.AfterBeforeParameterResolver;
4646
import org.bson.BsonArray;
4747
import org.bson.BsonBoolean;
@@ -57,14 +57,17 @@
5757
import org.junit.jupiter.params.ParameterizedTest;
5858
import org.junit.jupiter.params.provider.Arguments;
5959
import org.junit.jupiter.params.provider.MethodSource;
60+
import org.opentest4j.AssertionFailedError;
6061
import org.opentest4j.TestAbortedException;
6162

6263
import java.io.File;
6364
import java.io.IOException;
6465
import java.net.URISyntaxException;
66+
import java.text.MessageFormat;
6567
import java.util.ArrayList;
6668
import java.util.Collection;
6769
import java.util.Collections;
70+
import java.util.HashSet;
6871
import java.util.List;
6972
import java.util.Set;
7073
import java.util.concurrent.ExecutionException;
@@ -81,6 +84,7 @@
8184
import static com.mongodb.client.test.CollectionHelper.getCurrentClusterTime;
8285
import static com.mongodb.client.test.CollectionHelper.killAllSessions;
8386
import static com.mongodb.client.unified.RunOnRequirementsMatcher.runOnRequirementsMet;
87+
import static com.mongodb.client.unified.UnifiedTestModifications.doSkips;
8488
import static com.mongodb.client.unified.UnifiedTestModifications.testDef;
8589
import static java.util.Collections.singletonList;
8690
import static java.util.stream.Collectors.toList;
@@ -91,6 +95,7 @@
9195
import static org.junit.jupiter.api.Assertions.assertNull;
9296
import static org.junit.jupiter.api.Assertions.assertTrue;
9397
import static org.junit.jupiter.api.Assertions.fail;
98+
import static org.junit.jupiter.api.Assumptions.assumeFalse;
9499
import static org.junit.jupiter.api.Assumptions.assumeTrue;
95100
import static util.JsonPoweredTestHelper.getTestDocument;
96101
import static util.JsonPoweredTestHelper.getTestFiles;
@@ -100,6 +105,9 @@ public abstract class UnifiedTest {
100105
private static final Set<String> PRESTART_POOL_ASYNC_WORK_MANAGER_FILE_DESCRIPTIONS = Collections.singleton(
101106
"wait queue timeout errors include details about checked out connections");
102107

108+
public static final int ATTEMPTS = 3;
109+
private static Set<String> completed = new HashSet<>();
110+
103111
@Nullable
104112
private String fileDescription;
105113
private String schemaVersion;
@@ -154,32 +162,47 @@ public Entities getEntities() {
154162
}
155163

156164
@NonNull
157-
protected static Collection<Arguments> getTestData(final String directory) throws URISyntaxException, IOException {
165+
protected static Collection<Arguments> getTestData(final String directory, final boolean isReactive)
166+
throws URISyntaxException, IOException {
158167
List<Arguments> data = new ArrayList<>();
159168
for (File file : getTestFiles("/" + directory + "/")) {
160169
BsonDocument fileDocument = getTestDocument(file);
161-
162170
for (BsonValue cur : fileDocument.getArray("tests")) {
163-
data.add(UnifiedTest.createTestData(directory, fileDocument, cur.asDocument()));
171+
172+
final BsonDocument testDocument = cur.asDocument();
173+
String testDescription = testDocument.getString("description").getValue();
174+
String fileDescription = fileDocument.getString("description").getValue();
175+
TestDef testDef = testDef(directory, fileDescription, testDescription, isReactive);
176+
doSkips(testDef);
177+
178+
boolean retry = testDef.wasAssignedModifier(UnifiedTestModifications.Modifier.RETRY);
179+
int attempts = retry ? ATTEMPTS : 1;
180+
181+
182+
for (int attempt = 1; attempt <= attempts; attempt++) {
183+
String testName = !retry
184+
? MessageFormat.format("{0}: {1}", fileDescription, testDescription)
185+
: MessageFormat.format(
186+
"{0}: {1} ({2} of {3})",
187+
fileDescription, testDescription, attempt, attempts);
188+
data.add(Arguments.of(
189+
testName,
190+
fileDescription,
191+
testDescription,
192+
directory,
193+
attempt,
194+
attempts,
195+
fileDocument.getString("schemaVersion").getValue(),
196+
fileDocument.getArray("runOnRequirements", null),
197+
fileDocument.getArray("createEntities", new BsonArray()),
198+
fileDocument.getArray("initialData", new BsonArray()),
199+
testDocument));
200+
}
164201
}
165202
}
166203
return data;
167204
}
168205

169-
@NonNull
170-
private static Arguments createTestData(
171-
final String directory, final BsonDocument fileDocument, final BsonDocument testDocument) {
172-
return Arguments.of(
173-
fileDocument.getString("description").getValue(),
174-
testDocument.getString("description").getValue(),
175-
directory,
176-
fileDocument.getString("schemaVersion").getValue(),
177-
fileDocument.getArray("runOnRequirements", null),
178-
fileDocument.getArray("createEntities", new BsonArray()),
179-
fileDocument.getArray("initialData", new BsonArray()),
180-
testDocument);
181-
}
182-
183206
protected BsonDocument getDefinition() {
184207
return definition;
185208
}
@@ -192,9 +215,12 @@ protected BsonDocument getDefinition() {
192215

193216
@BeforeEach
194217
public void setUp(
218+
final String testName,
195219
@Nullable final String fileDescription,
196220
@Nullable final String testDescription,
197221
@Nullable final String directoryName,
222+
final int attemptNumber,
223+
final int totalAttempts,
198224
final String schemaVersion,
199225
@Nullable final BsonArray runOnRequirements,
200226
final BsonArray entitiesArray,
@@ -216,7 +242,11 @@ public void setUp(
216242
ignoreExtraEvents = false;
217243
testDef = testDef(directoryName, fileDescription, testDescription, isReactive());
218244
UnifiedTestModifications.doSkips(testDef);
245+
246+
boolean skip = testDef.wasAssignedModifier(UnifiedTestModifications.Modifier.SKIP);
247+
assumeFalse(skip, "Skipping test");
219248
skips(fileDescription, testDescription);
249+
220250
assertTrue(
221251
schemaVersion.equals("1.0")
222252
|| schemaVersion.equals("1.1")
@@ -283,8 +313,9 @@ protected void postCleanUp(final TestDef testDef) {
283313
}
284314

285315
/**
286-
* This method is called once per {@link #setUp(String, String, String, String, org.bson.BsonArray, org.bson.BsonArray, org.bson.BsonArray, org.bson.BsonDocument)},
287-
* unless {@link #setUp(String, String, String, String, org.bson.BsonArray, org.bson.BsonArray, org.bson.BsonArray, org.bson.BsonDocument)} fails unexpectedly.
316+
* This method is called once per
317+
* {@link #setUp(String, String, String, String, int, int, String, org.bson.BsonArray, org.bson.BsonArray, org.bson.BsonArray, org.bson.BsonDocument)}, unless
318+
* {@link #setUp(String, String, String, String, int, int, String, org.bson.BsonArray, org.bson.BsonArray, org.bson.BsonArray, org.bson.BsonDocument)} fails unexpectedly.
288319
*/
289320
protected void skips(final String fileDescription, final String testDescription) {
290321
}
@@ -293,40 +324,54 @@ protected boolean isReactive() {
293324
return false;
294325
}
295326

296-
@ParameterizedTest(name = "{0}: {1}")
327+
@ParameterizedTest(name = "{0}")
297328
@MethodSource("data")
298329
public void shouldPassAllOutcomes(
330+
final String testName,
299331
@Nullable final String fileDescription,
300332
@Nullable final String testDescription,
301333
@Nullable final String directoryName,
334+
final int attemptNumber,
335+
final int totalAttempts,
302336
final String schemaVersion,
303337
@Nullable final BsonArray runOnRequirements,
304338
final BsonArray entitiesArray,
305339
final BsonArray initialData,
306340
final BsonDocument definition) {
307-
BsonArray operations = definition.getArray("operations");
308-
for (int i = 0; i < operations.size(); i++) {
309-
BsonValue cur = operations.get(i);
310-
assertOperation(rootContext, cur.asDocument(), i);
311-
}
341+
assumeFalse(completed.contains(testName), "Skipping test already performed");
342+
completed.add(testName);
343+
try {
344+
BsonArray operations = definition.getArray("operations");
345+
for (int i = 0; i < operations.size(); i++) {
346+
BsonValue cur = operations.get(i);
347+
assertOperation(rootContext, cur.asDocument(), i);
348+
}
312349

313-
if (definition.containsKey("outcome")) {
314-
assertOutcome(rootContext);
315-
}
350+
if (definition.containsKey("outcome")) {
351+
assertOutcome(rootContext);
352+
}
316353

317-
if (definition.containsKey("expectEvents")) {
318-
compareEvents(rootContext, definition);
319-
}
354+
if (definition.containsKey("expectEvents")) {
355+
compareEvents(rootContext, definition);
356+
}
320357

321-
if (definition.containsKey("expectLogMessages")) {
322-
ArrayList<LogMatcher.Tweak> tweaks = new ArrayList<>(singletonList(
323-
// `LogMessage.Entry.Name.OPERATION` is not supported, therefore we skip matching its value
324-
LogMatcher.Tweak.skip(LogMessage.Entry.Name.OPERATION)));
325-
if (getMongoClientSettings().getClusterSettings()
326-
.getHosts().stream().anyMatch(serverAddress -> serverAddress instanceof UnixServerAddress)) {
327-
tweaks.add(LogMatcher.Tweak.skip(LogMessage.Entry.Name.SERVER_PORT));
358+
if (definition.containsKey("expectLogMessages")) {
359+
ArrayList<LogMatcher.Tweak> tweaks = new ArrayList<>(singletonList(
360+
// `LogMessage.Entry.Name.OPERATION` is not supported, therefore we skip matching its value
361+
LogMatcher.Tweak.skip(LogMessage.Entry.Name.OPERATION)));
362+
if (getMongoClientSettings().getClusterSettings()
363+
.getHosts().stream().anyMatch(serverAddress -> serverAddress instanceof UnixServerAddress)) {
364+
tweaks.add(LogMatcher.Tweak.skip(LogMessage.Entry.Name.SERVER_PORT));
365+
}
366+
compareLogMessages(rootContext, definition, tweaks);
367+
}
368+
} catch (AssertionFailedError e) {
369+
completed.remove(testName);
370+
if (attemptNumber == totalAttempts) { // last attempt
371+
throw e;
372+
} else {
373+
assumeFalse(completed.contains(testName), "Ignoring failure and retrying attempt " + attemptNumber);
328374
}
329-
compareLogMessages(rootContext, definition, tweaks);
330375
}
331376
}
332377

0 commit comments

Comments
 (0)