Skip to content

Remove docker and user tests, related infra #53118

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Mar 7, 2023
Merged

Remove docker and user tests, related infra #53118

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
572eab4
Remove docker and user tests, infra
jakebailey Mar 6, 2023
f5bc862
Drop out of dotfiles
jakebailey Mar 6, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/accept-baselines-fix-lints.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ jobs:
git config user.email "[email protected]"
git config user.name "TypeScript Bot"
npm install
git rm -r --quiet tests/baselines/reference :^tests/baselines/reference/docker :^tests/baselines/reference/user
git rm -r --quiet tests/baselines/reference
npx hereby runtests-parallel --ci --fix || true
npx hereby baseline-accept
git add ./src
Expand Down
23 changes: 1 addition & 22 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,6 @@ tests/webTestServer.js.map
tests/webhost/*.d.ts
tests/webhost/webtsc.js
tests/cases/**/*.js
!tests/cases/docker/*.js/
tests/cases/**/*.js.map
*.config
scripts/eslint/built/
Expand All @@ -59,29 +58,9 @@ internal/
yarn.lock
yarn-error.log
.parallelperf.*
tests/cases/user/*/package-lock.json
tests/cases/user/*/node_modules/
tests/cases/user/*/**/*.js
tests/cases/user/*/**/*.js.map
tests/cases/user/*/**/*.d.ts
!tests/cases/user/zone.js/
!tests/cases/user/bignumber.js/
!tests/cases/user/discord.js/
tests/baselines/reference/dt
.failed-tests
TEST-results.xml
package-lock.json
tests/cases/user/npm/npm
tests/cases/user/TypeScript-React-Starter/TypeScript-React-Starter
tests/cases/user/TypeScript-Node-Starter/TypeScript-Node-Starter
tests/cases/user/TypeScript-React-Native-Starter/TypeScript-React-Native-Starter
tests/cases/user/TypeScript-Vue-Starter/TypeScript-Vue-Starter
tests/cases/user/TypeScript-WeChat-Starter/TypeScript-WeChat-Starter
tests/cases/user/create-react-app/create-react-app
tests/cases/user/fp-ts/fp-ts
tests/cases/user/webpack/webpack
tests/cases/user/puppeteer/puppeteer
tests/cases/user/axios-src/axios-src
tests/cases/user/prettier/prettier
.eslintcache
*v8.log
*v8.log
5 changes: 1 addition & 4 deletions CONTRIBUTING.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,10 +81,7 @@ hereby tests # Build the test infrastructure using the built compile
hereby runtests # Run tests using the built compiler and test infrastructure.
# You can override the specific suite runner used or specify a test for this command.
# Use --tests=<testPath> for a specific test and/or --runner=<runnerName> for a specific suite.
# Valid runners include conformance, compiler, fourslash, project, user, and docker
# The user and docker runners are extended test suite runners - the user runner
# works on disk in the tests/cases/user directory, while the docker runner works in containers.
# You'll need to have the docker executable in your system path for the docker runner to work.
# Valid runners include conformance, compiler, fourslash, and project
hereby runtests-parallel # Like runtests, but split across multiple threads. Uses a number of threads equal to the system
# core count by default. Use --workers=<number> to adjust this.
hereby baseline-accept # This replaces the baseline test results with the results obtained from hereby runtests.
Expand Down
2 changes: 1 addition & 1 deletion src/harness/runnerbase.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import {
} from "./_namespaces/Harness";
import * as ts from "./_namespaces/ts";

export type TestRunnerKind = CompilerTestKind | FourslashTestKind | "project" | "rwc" | "test262" | "user" | "dt" | "docker";
export type TestRunnerKind = CompilerTestKind | FourslashTestKind | "project" | "rwc" | "test262" | "dt";
export type CompilerTestKind = "conformance" | "compiler";
export type FourslashTestKind = "fourslash" | "fourslash-shims" | "fourslash-shims-pp" | "fourslash-server";

Expand Down
202 changes: 0 additions & 202 deletions src/testRunner/externalCompileRunner.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,180 +113,6 @@ abstract class ExternalCompileRunnerBase extends RunnerBase {
}
}

export class UserCodeRunner extends ExternalCompileRunnerBase {
readonly testDir = "tests/cases/user/";
kind(): TestRunnerKind {
return "user";
}
report(result: ExecResult) {
// eslint-disable-next-line no-null/no-null
return result.status === 0 && !result.stdout.length && !result.stderr.length ? null : `Exit Code: ${result.status}
Standard output:
${sortErrors(stripAbsoluteImportPaths(result.stdout.toString().replace(/\r\n/g, "\n")))}


Standard error:
${stripAbsoluteImportPaths(result.stderr.toString().replace(/\r\n/g, "\n"))}`;
}
}

export class DockerfileRunner extends ExternalCompileRunnerBase {
readonly testDir = "tests/cases/docker/";
kind(): TestRunnerKind {
return "docker";
}
override initializeTests(): void {
// Read in and evaluate the test list
const testList = this.tests && this.tests.length ? this.tests : this.getTestFiles();

// eslint-disable-next-line @typescript-eslint/no-this-alias
const cls = this;
describe(`${this.kind()} code samples`, function (this: Mocha.Suite) {
this.timeout(cls.timeout); // 20 minutes
before(() => {
// cached because workspace is hashed to determine cacheability
cls.exec("docker", ["build", ".", "-t", "typescript/typescript", "-f", cls.testDir + "Dockerfile"], {
cwd: IO.getWorkspaceRoot(),
env: { ...process.env, DOCKER_BUILDKIT: "1" }, // We need buildkit to allow Dockerfile.dockerignore to work.
});
});
for (const test of testList) {
const directory = typeof test === "string" ? test : test.file;
const cwd = path.join(IO.getWorkspaceRoot(), cls.testDir, directory);
it(`should build ${directory} successfully`, () => {
const imageName = `tstest/${directory}`;
cls.exec("docker", ["build", "--no-cache", ".", "-t", imageName], { cwd }); // --no-cache so the latest version of the repos referenced is always fetched
const cp: typeof import("child_process") = require("child_process");
Baseline.runBaseline(`${cls.kind()}/${directory}.log`, cls.report(cp.spawnSync(`docker`, ["run", imageName], { cwd, timeout: cls.timeout, shell: true })));
});
}
});
}

private timeout = 1_200_000; // 20 minutes;
private exec(command: string, args: string[], options: { cwd: string; env?: NodeJS.ProcessEnv }): void {
const cp: typeof import("child_process") = require("child_process");
const stdio = isWorker ? "pipe" : "inherit";
const res = cp.spawnSync(isWorker ? `${command} 2>&1` : command, args, { timeout: this.timeout, shell: true, stdio, ...options });
if (res.status !== 0) {
throw new Error(`${command} ${args.join(" ")} for ${options.cwd} failed: ${res.stdout && res.stdout.toString()}`);
}
}
report(result: ExecResult) {
// eslint-disable-next-line no-null/no-null
return result.status === 0 && !result.stdout.length && !result.stderr.length ? null : `Exit Code: ${result.status}
Standard output:
${sanitizeDockerfileOutput(result.stdout.toString())}


Standard error:
${sanitizeDockerfileOutput(result.stderr.toString())}`;
}
}

function sanitizeDockerfileOutput(result: string): string {
return [
normalizeNewlines,
stripANSIEscapes,
stripRushStageNumbers,
stripWebpackHash,
sanitizeVersionSpecifiers,
sanitizeTimestamps,
sanitizeSizes,
sanitizeUnimportantGulpOutput,
stripAbsoluteImportPaths,
].reduce((result, f) => f(result), result);
}

function normalizeNewlines(result: string): string {
return result.replace(/\r\n/g, "\n");
}

function stripANSIEscapes(result: string): string {
return result.replace(/\x1b\[[0-9;]*[a-zA-Z]/g, "");
}

function stripRushStageNumbers(result: string): string {
return result.replace(/\d+ of \d+:/g, "XX of XX:");
}

function stripWebpackHash(result: string): string {
return result.replace(/Hash: \w+/g, "Hash: [redacted]");
}

function sanitizeSizes(result: string): string {
return result.replace(/\d+(\.\d+)? ((Ki|M)B|bytes)/g, "X KiB");
}

/**
* Gulp's output order within a `parallel` block is nondeterministic (and there's no way to configure it to execute in series),
* so we purge as much of the gulp output as we can
*/
function sanitizeUnimportantGulpOutput(result: string): string {
return result.replace(/^.*(\] (Starting)|(Finished)).*$/gm, "") // "gulp" task start/end messages (nondeterministic order)
.replace(/^.*(\] . (finished)|(started)).*$/gm, "") // "just" task start/end messages (nondeterministic order)
.replace(/^.*\] Respawned to PID: \d+.*$/gm, "") // PID of child is OS and system-load dependent (likely stableish in a container but still dangerous)
.replace(/\n+/g, "\n")
.replace(/\/tmp\/yarn--.*?\/node/g, "");
}

function sanitizeTimestamps(result: string): string {
return result.replace(/\[\d?\d:\d\d:\d\d (A|P)M\]/g, "[XX:XX:XX XM]")
.replace(/\[\d?\d:\d\d:\d\d\]/g, "[XX:XX:XX]")
.replace(/\/\d+-\d+-[\d_TZ]+-debug.log/g, "\/XXXX-XX-XXXXXXXXX-debug.log")
.replace(/\d+\/\d+\/\d+ \d+:\d+:\d+ (AM|PM)/g, "XX/XX/XX XX:XX:XX XM")
.replace(/\d+(\.\d+)? sec(onds?)?/g, "? seconds")
.replace(/\d+(\.\d+)? min(utes?)?/g, "")
.replace(/\d+(\.\d+)? ?m?s/g, "?s")
.replace(/ \(\?s\)/g, "");
}

function sanitizeVersionSpecifiers(result: string): string {
return result
.replace(/\d+.\d+.\d+-insiders.\d\d\d\d\d\d\d\d/g, "X.X.X-insiders.xxxxxxxx")
.replace(/Rush Multi-Project Build Tool (\d+)\.\d+\.\d+/g, "Rush Multi-Project Build Tool $1.X.X")
.replace(/([@v\()])\d+\.\d+\.\d+/g, "$1X.X.X")
.replace(/webpack (\d+)\.\d+\.\d+/g, "webpack $1.X.X")
.replace(/Webpack version: (\d+)\.\d+\.\d+/g, "Webpack version: $1.X.X");
}

/**
* Import types and some other error messages use absolute paths in errors as they have no context to be written relative to;
* This is problematic for error baselines, so we grep for them and strip them out.
*/
function stripAbsoluteImportPaths(result: string) {
const workspaceRegexp = new RegExp(IO.getWorkspaceRoot().replace(/\\/g, "\\\\"), "g");
return result
.replace(/import\(".*?\/tests\/cases\/user\//g, `import("/`)
.replace(/Module '".*?\/tests\/cases\/user\//g, `Module '"/`)
.replace(workspaceRegexp, "../../..");
}

function sortErrors(result: string) {
return ts.flatten(splitBy(result.split("\n"), s => /^\S+/.test(s)).sort(compareErrorStrings)).join("\n");
}

const errorRegexp = /^(.+\.[tj]sx?)\((\d+),(\d+)\)(: error TS.*)/;
function compareErrorStrings(a: string[], b: string[]) {
ts.Debug.assertGreaterThanOrEqual(a.length, 1);
ts.Debug.assertGreaterThanOrEqual(b.length, 1);
const matchA = a[0].match(errorRegexp);
if (!matchA) {
return -1;
}
const matchB = b[0].match(errorRegexp);
if (!matchB) {
return 1;
}
const [, errorFileA, lineNumberStringA, columnNumberStringA, remainderA] = matchA;
const [, errorFileB, lineNumberStringB, columnNumberStringB, remainderB] = matchB;
return ts.comparePathsCaseSensitive(errorFileA, errorFileB) ||
ts.compareValues(parseInt(lineNumberStringA), parseInt(lineNumberStringB)) ||
ts.compareValues(parseInt(columnNumberStringA), parseInt(columnNumberStringB)) ||
ts.compareStringsCaseSensitive(remainderA, remainderB) ||
ts.compareStringsCaseSensitive(a.slice(1).join("\n"), b.slice(1).join("\n"));
}

export class DefinitelyTypedRunner extends ExternalCompileRunnerBase {
readonly testDir = "../DefinitelyTyped/types/";
override workingDirectory = this.testDir;
Expand All @@ -304,31 +130,3 @@ Standard error:
${result.stderr.toString().replace(/\r\n/g, "\n")}`;
}
}

/**
* Split an array into multiple arrays whenever `isStart` returns true.
* @example
* splitBy([1,2,3,4,5,6], isOdd)
* ==> [[1, 2], [3, 4], [5, 6]]
* where
* const isOdd = n => !!(n % 2)
*/
function splitBy<T>(xs: T[], isStart: (x: T) => boolean): T[][] {
const result = [];
let group: T[] = [];
for (const x of xs) {
if (isStart(x)) {
if (group.length) {
result.push(group);
}
group = [x];
}
else {
group.push(x);
}
}
if (group.length) {
result.push(group);
}
return result;
}
18 changes: 0 additions & 18 deletions src/testRunner/runner.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ import {
CompilerBaselineRunner,
CompilerTestType,
DefinitelyTypedRunner,
DockerfileRunner,
FourSlashRunner,
GeneratedFourslashRunner,
IO,
Expand All @@ -14,7 +13,6 @@ import {
setShards,
Test262BaselineRunner,
TestRunnerKind,
UserCodeRunner,
} from "./_namespaces/Harness";
import * as project from "./_namespaces/project";
import * as RWC from "./_namespaces/RWC";
Expand Down Expand Up @@ -80,12 +78,8 @@ export function createRunner(kind: TestRunnerKind): RunnerBase {
return new RWC.RWCRunner();
case "test262":
return new Test262BaselineRunner();
case "user":
return new UserCodeRunner();
case "dt":
return new DefinitelyTypedRunner();
case "docker":
return new DockerfileRunner();
}
return ts.Debug.fail(`Unknown runner kind ${kind}`);
}
Expand Down Expand Up @@ -223,15 +217,9 @@ function handleTestConfig() {
case "test262":
runners.push(new Test262BaselineRunner());
break;
case "user":
runners.push(new UserCodeRunner());
break;
case "dt":
runners.push(new DefinitelyTypedRunner());
break;
case "docker":
runners.push(new DockerfileRunner());
break;
}
}
}
Expand All @@ -250,12 +238,6 @@ function handleTestConfig() {
runners.push(new FourSlashRunner(FourSlash.FourSlashTestType.ShimsWithPreprocess));
runners.push(new FourSlashRunner(FourSlash.FourSlashTestType.Server));
// runners.push(new GeneratedFourslashRunner());

// CRON-only tests
if (process.env.TRAVIS_EVENT_TYPE === "cron") {
runners.push(new UserCodeRunner());
runners.push(new DockerfileRunner());
}
}
if (runUnitTests === undefined) {
runUnitTests = runners.length !== 1; // Don't run unit tests when running only one runner if unit tests were not explicitly asked for
Expand Down
Loading