Fix CI failures by adding sleep backoff (#870)

* Standardize suite setup

* Add sleepWithBackoff

* Fix deleted token

* Tweak

* Kick CI

* Add warning to current sleep

* Improve update fixtures sleep

* Typo

* Docs

* More docs

Author: pokey

src/test/suite/backwardCompatibility.test.ts

@@ -1,16 +1,11 @@
 import * as assert from "assert";
 import * as vscode from "vscode";
-import * as sinon from "sinon";
 import { getCursorlessApi } from "../../util/getExtensionApi";
 import { openNewEditor } from "../openNewEditor";
+import { standardSuiteSetup } from "./standardSuiteSetup";
 
 suite("Backward compatibility", async function () {
-  this.timeout("100s");
-  this.retries(3);
-
-  teardown(() => {
-    sinon.restore();
-  });
+  standardSuiteSetup(this);
 
   test("Backward compatibility", runTest);
 });

src/test/suite/breakpoints.test.ts

@@ -1,21 +1,16 @@
 import * as assert from "assert";
 import * as vscode from "vscode";
-import * as sinon from "sinon";
 import { getCursorlessApi } from "../../util/getExtensionApi";
 import { openNewEditor } from "../openNewEditor";
+import { standardSuiteSetup } from "./standardSuiteSetup";
 
 suite("breakpoints", async function () {
-  this.timeout("100s");
-  this.retries(3);
+  standardSuiteSetup(this);
 
   setup(() => {
     removeBreakpoints();
   });
 
-  teardown(() => {
-    sinon.restore();
-  });
-
   suiteTeardown(() => {
     removeBreakpoints();
   });

src/test/suite/crossCellsSetSelection.test.ts

@@ -1,18 +1,12 @@
 import * as assert from "assert";
 import * as vscode from "vscode";
-import * as sinon from "sinon";
 import { getCursorlessApi } from "../../util/getExtensionApi";
 import { openNewNotebookEditor } from "../openNewEditor";
-import sleep from "../../util/sleep";
+import { sleepWithBackoff, standardSuiteSetup } from "./standardSuiteSetup";
 
 // Check that setSelection is able to focus the correct cell
 suite("Cross-cell set selection", async function () {
-  this.timeout("100s");
-  this.retries(3);
-
-  teardown(() => {
-    sinon.restore();
-  });
+  standardSuiteSetup(this);
 
   test("Cross-cell set selection", runTest);
 });
@@ -24,7 +18,7 @@ async function runTest() {
 
   // FIXME: There seems to be some timing issue when you create a notebook
   // editor
-  await sleep(1000);
+  await sleepWithBackoff(1000);
 
   await graph.hatTokenMap.addDecorations();
 

src/test/suite/editNewCell.test.ts

@@ -1,20 +1,14 @@
 import * as assert from "assert";
 import * as vscode from "vscode";
-import * as sinon from "sinon";
 import { getCursorlessApi } from "../../util/getExtensionApi";
-import { openNewNotebookEditor } from "../openNewEditor";
-import sleep from "../../util/sleep";
 import { getCellIndex } from "../../util/notebook";
+import { openNewNotebookEditor } from "../openNewEditor";
 import { getPlainNotebookContents } from "../util/notebook";
+import { sleepWithBackoff, standardSuiteSetup } from "./standardSuiteSetup";
 
 // Check that setSelection is able to focus the correct cell
 suite("Edit new cell", async function () {
-  this.timeout("100s");
-  this.retries(3);
-
-  teardown(() => {
-    sinon.restore();
-  });
+  standardSuiteSetup(this);
 
   test("drink cell", () =>
     runTest("drink cell", "editNewLineBefore", 0, ["", "hello"]));
@@ -33,7 +27,7 @@ async function runTest(
 
   // FIXME: There seems to be some timing issue when you create a notebook
   // editor
-  await sleep(1000);
+  await sleepWithBackoff(1000);
 
   await graph.hatTokenMap.addDecorations();
 

src/test/suite/fold.test.ts

@@ -1,15 +1,10 @@
 import * as assert from "assert";
 import * as vscode from "vscode";
-import * as sinon from "sinon";
 import { openNewEditor } from "../openNewEditor";
+import { standardSuiteSetup } from "./standardSuiteSetup";
 
 suite("fold", async function () {
-  this.timeout("100s");
-  this.retries(3);
-
-  teardown(() => {
-    sinon.restore();
-  });
+  standardSuiteSetup(this);
 
   test("fold made", foldMade);
   test("unfold made", unfoldMade);

src/test/suite/followLink.test.ts

@@ -1,17 +1,12 @@
 import * as assert from "assert";
-import * as vscode from "vscode";
-import * as sinon from "sinon";
 import * as os from "os";
+import * as vscode from "vscode";
 import { openNewEditor } from "../openNewEditor";
 import { getFixturePath } from "../util/getFixturePaths";
+import { standardSuiteSetup } from "./standardSuiteSetup";
 
 suite("followLink", async function () {
-  this.timeout("100s");
-  this.retries(3);
-
-  teardown(() => {
-    sinon.restore();
-  });
+  standardSuiteSetup(this);
 
   test("follow definition", followDefinition);
   test("follow link", followLink);

src/test/suite/groupByDocument.test.ts

@@ -1,16 +1,11 @@
 import * as assert from "assert";
 import * as vscode from "vscode";
-import * as sinon from "sinon";
-import { getCursorlessApi } from "../../util/getExtensionApi";
 import HatTokenMap from "../../core/HatTokenMap";
+import { getCursorlessApi } from "../../util/getExtensionApi";
+import { standardSuiteSetup } from "./standardSuiteSetup";
 
 suite("Group by document", async function () {
-  this.timeout("100s");
-  this.retries(3);
-
-  teardown(() => {
-    sinon.restore();
-  });
+  standardSuiteSetup(this);
 
   test("Group by document", runTest);
 });

src/test/suite/intraCellSetSelection.test.ts

@@ -1,18 +1,12 @@
 import * as assert from "assert";
 import * as vscode from "vscode";
-import * as sinon from "sinon";
 import { getCursorlessApi } from "../../util/getExtensionApi";
 import { openNewNotebookEditor } from "../openNewEditor";
-import sleep from "../../util/sleep";
+import { sleepWithBackoff, standardSuiteSetup } from "./standardSuiteSetup";
 
 // Check that setSelection is able to focus the correct cell
 suite("Within cell set selection", async function () {
-  this.timeout("100s");
-  this.retries(3);
-
-  teardown(() => {
-    sinon.restore();
-  });
+  standardSuiteSetup(this);
 
   test("Within cell set selection", runTest);
 });
@@ -24,7 +18,7 @@ async function runTest() {
 
   // FIXME: There seems to be some timing issue when you create a notebook
   // editor
-  await sleep(1000);
+  await sleepWithBackoff(1000);
 
   await graph.hatTokenMap.addDecorations();
 

src/test/suite/prePhraseSnapshot.test.ts

@@ -1,10 +1,10 @@
 import * as assert from "assert";
 import * as vscode from "vscode";
-import * as sinon from "sinon";
-import { getCursorlessApi } from "../../util/getExtensionApi";
 import { selectionToPlainObject } from "../../testUtil/toPlainObject";
+import { getCursorlessApi } from "../../util/getExtensionApi";
 import { mockPrePhraseGetVersion } from "../mockPrePhraseGetVersion";
 import { openNewEditor } from "../openNewEditor";
+import { standardSuiteSetup } from "./standardSuiteSetup";
 
 /**
  * The selections we expect when the pre-phrase snapshot is used
@@ -17,12 +17,7 @@ const snapshotExpectedSelections = [new vscode.Selection(0, 6, 0, 11)];
 const noSnapshotExpectedSelections = [new vscode.Selection(1, 6, 1, 11)];
 
 suite("Pre-phrase snapshots", async function () {
-  this.timeout("100s");
-  this.retries(3);
-
-  teardown(() => {
-    sinon.restore();
-  });
+  standardSuiteSetup(this);
 
   test("Pre-phrase snapshot; single phrase", () =>
     runTest(true, false, snapshotExpectedSelections));

src/test/suite/recorded.test.ts

@@ -1,7 +1,6 @@
 import * as assert from "assert";
 import { promises as fsp } from "fs";
 import * as yaml from "js-yaml";
-import * as sinon from "sinon";
 import * as vscode from "vscode";
 import HatTokenMap from "../../core/HatTokenMap";
 import { ReadOnlyHatMap } from "../../core/IndividualHatMap";
@@ -22,10 +21,11 @@ import {
 } from "../../testUtil/toPlainObject";
 import { Clipboard } from "../../util/Clipboard";
 import { getCursorlessApi } from "../../util/getExtensionApi";
-import sleep from "../../util/sleep";
 import { openNewEditor } from "../openNewEditor";
 import asyncSafety from "../util/asyncSafety";
 import { getRecordedTestPaths } from "../util/getFixturePaths";
+import shouldUpdateFixtures from "./shouldUpdateFixtures";
+import { sleepWithBackoff, standardSuiteSetup } from "./standardSuiteSetup";
 
 function createPosition(position: PositionPlainObject) {
   return new vscode.Position(position.line, position.character);
@@ -38,12 +38,7 @@ function createSelection(selection: SelectionPlainObject): vscode.Selection {
 }
 
 suite("recorded test cases", async function () {
-  this.timeout("100s");
-  this.retries(5);
-
-  teardown(() => {
-    sinon.restore();
-  });
+  standardSuiteSetup(this);
 
   suiteSetup(async () => {
     // Necessary because opening a notebook opens the panel for some reason
@@ -77,7 +72,7 @@ async function runTest(file: string) {
   );
 
   if (fixture.postEditorOpenSleepTimeMs != null) {
-    await sleep(fixture.postEditorOpenSleepTimeMs);
+    await sleepWithBackoff(fixture.postEditorOpenSleepTimeMs);
   }
 
   editor.selections = fixture.initialState.selections.map(createSelection);
@@ -138,7 +133,7 @@ async function runTest(file: string) {
   );
 
   if (fixture.postCommandSleepTimeMs != null) {
-    await sleep(fixture.postCommandSleepTimeMs);
+    await sleepWithBackoff(fixture.postCommandSleepTimeMs);
   }
 
   const marks =
@@ -170,7 +165,7 @@ async function runTest(file: string) {
       ? undefined
       : testDecorationsToPlainObject(graph.editStyles.testDecorations);
 
-  if (process.env.CURSORLESS_TEST_UPDATE_FIXTURES === "true") {
+  if (shouldUpdateFixtures()) {
     const outputFixture = {
       ...fixture,
       finalState: resultState,

src/test/suite/scroll.test.ts

@@ -1,15 +1,10 @@
 import * as assert from "assert";
 import * as vscode from "vscode";
-import * as sinon from "sinon";
 import { openNewEditor } from "../openNewEditor";
+import { standardSuiteSetup } from "./standardSuiteSetup";
 
 suite("scroll", async function () {
-  this.timeout("100s");
-  this.retries(3);
-
-  teardown(() => {
-    sinon.restore();
-  });
+  standardSuiteSetup(this);
 
   test("top whale", topWhale);
   test("bottom whale", bottomWhale);

src/test/suite/shouldUpdateFixtures.ts

@@ -0,0 +1,7 @@
+/**
+ * Used to check weather the update fixtures launch config was used.
+ * @returns `true` if developer used to the update fixtures launch config
+ */
+export default function shouldUpdateFixtures() {
+  return process.env.CURSORLESS_TEST_UPDATE_FIXTURES === "true";
+}

src/test/suite/standardSuiteSetup.ts

@@ -0,0 +1,50 @@
+import { Context } from "mocha";
+import * as sinon from "sinon";
+import sleep from "../../util/sleep";
+import shouldUpdateFixtures from "./shouldUpdateFixtures";
+
+/**
+ * The number of times the current test has been retried. Will be 0 the first
+ * time the test runs and increase by 1 each time the test fails and needs to be
+ * rerun.
+ */
+let retryCount = -1;
+
+/**
+ * The title of the previously run test. Used to keep track of
+ * {@link retryCount}.
+ */
+let previousTestTitle = "";
+
+export function standardSuiteSetup(suite: Mocha.Suite) {
+  suite.timeout("100s");
+  suite.retries(5);
+
+  teardown(() => {
+    sinon.restore();
+  });
+
+  setup(async function (this: Context) {
+    const title = this.test!.fullTitle();
+    retryCount = title === previousTestTitle ? retryCount + 1 : 0;
+    previousTestTitle = title;
+  });
+}
+
+/**
+ * Sleep function for use in tests that will be retried. Doubles the amount of
+ * time it sleeps each time a test is run, starting from {@link ms} / 4.
+ *
+ * If the developer used the update fixtures launch config, we sleep for {@link ms} *
+ * 2 every time so that they don't get spurious updates to fixtures due to not
+ * sleeping enough.
+ * @param ms The baseline number of milliseconds to sleep.
+ * @returns A promise that will resolve when the sleep is over
+ */
+export function sleepWithBackoff(ms: number) {
+  const timeToSleep = shouldUpdateFixtures()
+    ? ms * 2
+    : ms * Math.pow(2, retryCount - 2);
+
+  return sleep(timeToSleep);
+}

src/test/suite/toggleDecorations.test.ts

@@ -1,16 +1,11 @@
 import * as assert from "assert";
 import * as vscode from "vscode";
-import * as sinon from "sinon";
 import { getCursorlessApi } from "../../util/getExtensionApi";
 import { openNewEditor } from "../openNewEditor";
+import { standardSuiteSetup } from "./standardSuiteSetup";
 
 suite("toggle decorations", async function () {
-  this.timeout("100s");
-  this.retries(3);
-
-  teardown(() => {
-    sinon.restore();
-  });
+  standardSuiteSetup(this);
 
   test("toggle decorations", () => runTest());
 });

src/util/sleep.ts

@@ -1,3 +1,13 @@
 import { promisify } from "util";
 
-export default promisify(setTimeout);
+/**
+ * Sleep function that returns a promise that resolves when the sleep is
+ * complete.
+ *
+ * WARNING: Please do not use this function in tests, because we retry our tests
+ * on failure, and this function will sleep the same amount of time every time
+ * the test is retried.  Prefer {@link sleepWithBackoff} instead.
+ */
+const sleep = promisify(setTimeout);
+
+export default sleep;