test_runner: add initial CLI runner

This commit introduces an initial version of a CLI-based
test runner.

Author: cjihrig

doc/api/cli.md

@@ -1052,6 +1052,15 @@ minimum allocation from the secure heap. The minimum value is `2`.
 The maximum value is the lesser of `--secure-heap` or `2147483647`.
 The value given must be a power of two.
 
+### `--test`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+Starts the Node.js command line test runner. See the documentation on
+[running tests from the command line][] for more details.
+
 ### `--test-only`
 
 <!-- YAML
@@ -2033,6 +2042,7 @@ $ node --max-old-space-size=1536 index.js
 [jitless]: https://v8.dev/blog/jitless
 [libuv threadpool documentation]: https://docs.libuv.org/en/latest/threadpool.html
 [remote code execution]: https://www.owasp.org/index.php/Code_Injection
+[running tests from the command line]: test.md#running-tests-from-the-command-line
 [security warning]: #warning-binding-inspector-to-a-public-ipport-combination-is-insecure
 [timezone IDs]: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones
 [ways that `TZ` is handled in other environments]: https://www.gnu.org/software/libc/manual/html_node/TZ-Variable.html

doc/api/test.md

@@ -219,6 +219,49 @@ test('a test that creates asynchronous activity', (t) => {
 });
 ```
 
+## Running tests from the command line
+
+The Node.js test runner can be invoked from the command line by passing the
+[`--test`][] flag. By default, Node.js will recursively search the current
+directory for test files:
+
+```bash
+node --test
+```
+
+Alternatively, one or more paths can be provided as the final argument(s) to
+the Node.js command, as shown below.
+
+```bash
+node --test test1.js test2.mjs test/
+```
+
+In this example, the test runner will execute the files `test1.js` and
+`test2.mjs`. The test runner will also recursively search the `test/` directory
+for test files to execute.
+
+When searching for test files to execute, the test runner behaves as follows:
+
+* Any files explicitly provided by the user are executed.
+* `node_modules` directories are skipped unless explicitly provided by the
+  user.
+* `.js`, `.cjs`, and `.mjs` filenames matching the following patterns are
+  treated as test files:
+  * `test` - Files whose basename is the string `'test'`. Examples: `test.js`,
+    `test.cjs`, `test.mjs`.
+  * `test-.+` - Files whose basename starts with the string `'test-'` followed
+    by one or more characters. Examples: `test-example.js`,
+    `test-another-example.mjs`.
+  * `.+[\.\-\_]test` - Files whose basename ends with one or more characters
+    followed by `.test`, `-test`, or `_test`. Examples: `example.test.js`,
+    `example-test.cjs`, `example_test.mjs`.
+
+Each matching test file is executed in a separate child process. If the child
+process finishes with an exit code of 0, the test is considered passing.
+Otherwise, the test is considered to be a failure. Test files must be
+executable by Node.js, but are not required to use the `node:test` module
+internally.
+
 ## `test([name][, options][, fn])`
 
 <!-- YAML
@@ -368,5 +411,6 @@ behaves in the same fashion as the top level [`test()`][] function.
 
 [TAP]: https://testanything.org/
 [`--test-only`]: cli.md#--test-only
+[`--test`]: cli.md#--test
 [`TestContext`]: #class-testcontext
 [`test()`]: #testname-options-fn

doc/node.1

@@ -381,6 +381,9 @@ the secure heap. The default is 0. The value must be a power of two.
 .It Fl -secure-heap-min Ns = Ns Ar n
 Specify the minimum allocation from the OpenSSL secure heap. The default is 2. The value must be a power of two.
 .
+.It Fl -test
+Starts the Node.js command line test runner.
+.
 .It Fl -test-only
 Configures the test runner to only execute top level tests that have the `only`
 option set.

lib/internal/errors.js

@@ -207,6 +207,16 @@ function isErrorStackTraceLimitWritable() {
     desc.set !== undefined;
 }
 
+function inspectWithNoCustomRetry(obj, options) {
+  const utilInspect = lazyInternalUtilInspect();
+
+  try {
+    return utilInspect.inspect(obj, options);
+  } catch {
+    return utilInspect.inspect(obj, { ...options, customInspect: false });
+  }
+}
+
 // A specialized Error that includes an additional info property with
 // additional information about the error condition.
 // It has the properties present in a UVException but with a custom error
@@ -862,6 +872,7 @@ module.exports = {
   getMessage,
   hideInternalStackFrames,
   hideStackFrames,
+  inspectWithNoCustomRetry,
   isErrorStackTraceLimitWritable,
   isStackOverflowError,
   kEnhanceStackBeforeInspector,
@@ -1549,11 +1560,14 @@ E('ERR_TEST_FAILURE', function(error, failureType) {
   assert(typeof failureType === 'string',
          "The 'failureType' argument must be of type string.");
 
-  const msg = error?.message ?? lazyInternalUtilInspect().inspect(error);
+  let msg = error?.message ?? error;
 
-  this.failureType = error?.failureType ?? failureType;
-  this.cause = error;
+  if (typeof msg !== 'string') {
+    msg = inspectWithNoCustomRetry(msg);
+  }
 
+  this.failureType = failureType;
+  this.cause = error;
   return msg;
 }, Error);
 E('ERR_TLS_CERT_ALTNAME_FORMAT', 'Invalid subject alternative name string',

lib/internal/main/test_runner.js

@@ -0,0 +1,135 @@
+'use strict';
+const {
+  ArrayFrom,
+  ArrayPrototypeFilter,
+  ArrayPrototypeIncludes,
+  ArrayPrototypePush,
+  ArrayPrototypeSlice,
+  ArrayPrototypeSort,
+  Promise,
+  SafeSet,
+  StringPrototypeEndsWith,
+} = primordials;
+const {
+  prepareMainThreadExecution,
+} = require('internal/bootstrap/pre_execution');
+const { spawn } = require('child_process');
+const { readdirSync, statSync } = require('fs');
+const console = require('internal/console/global');
+const {
+  codes: {
+    ERR_TEST_FAILURE,
+  },
+} = require('internal/errors');
+const test = require('internal/test_runner/harness');
+const { kSubtestsFailed } = require('internal/test_runner/test');
+const { doesPathMatchFilter } = require('internal/test_runner/utils');
+const { join, resolve } = require('path');
+const kFilterArgs = ['--test'];
+
+prepareMainThreadExecution(false);
+markBootstrapComplete();
+
+// TODO(cjihrig): Replace this with recursive readdir once it lands.
+function processPath(path, testFiles, userSupplied = false) {
+  const stats = statSync(path);
+
+  if (stats.isFile()) {
+    if (userSupplied || doesPathMatchFilter(path)) {
+      testFiles.add(path);
+    }
+  } else if (stats.isDirectory()) {
+    if (!userSupplied && StringPrototypeEndsWith(path, 'node_modules')) {
+      return;
+    }
+
+    const entries = readdirSync(path);
+
+    for (let i = 0; i < entries.length; i++) {
+      processPath(join(path, entries[i]), testFiles);
+    }
+  }
+}
+
+function createTestFileList() {
+  const testPaths = process.argv.length > 1 ?
+    ArrayPrototypeSlice(process.argv, 1) : ['.'];
+  const testFiles = new SafeSet();
+
+  try {
+    for (let i = 0; i < testPaths.length; i++) {
+      const absolutePath = resolve(testPaths[i]);
+
+      processPath(absolutePath, testFiles, true);
+    }
+  } catch (err) {
+    if (err?.code === 'ENOENT') {
+      console.error(`Could not find '${err.path}'`);
+      process.exit(1);
+    }
+
+    throw err;
+  }
+
+  return ArrayPrototypeSort(ArrayFrom(testFiles));
+}
+
+function runTestFile(path) {
+  return test(path, () => {
+    return new Promise((resolve, reject) => {
+      const args = ArrayPrototypeFilter(process.execArgv, (arg) => {
+        return !ArrayPrototypeIncludes(kFilterArgs, arg);
+      });
+      ArrayPrototypePush(args, path);
+
+      const child = spawn(process.execPath, args);
+      // TODO(cjihrig): Implement a TAP parser to read the child's stdout
+      // instead of just displaying it all if the child fails.
+      let stdout = '';
+      let stderr = '';
+      let err;
+
+      child.on('error', (error) => {
+        err = error;
+      });
+
+      child.stdout.setEncoding('utf8');
+      child.stderr.setEncoding('utf8');
+
+      child.stdout.on('data', (chunk) => {
+        stdout += chunk;
+      });
+
+      child.stderr.on('data', (chunk) => {
+        stderr += chunk;
+      });
+
+      child.once('exit', (code, signal) => {
+        if (code !== 0 || signal !== null) {
+          if (!err) {
+            err = new ERR_TEST_FAILURE('test failed', kSubtestsFailed);
+            err.exitCode = code;
+            err.signal = signal;
+            err.stdout = stdout;
+            err.stderr = stderr;
+            // The stack will not be useful since the failures came from tests
+            // in a child process.
+            err.stack = undefined;
+          }
+
+          return reject(err);
+        }
+
+        resolve();
+      });
+    });
+  });
+}
+
+(async function main() {
+  const testFiles = createTestFileList();
+
+  for (let i = 0; i < testFiles.length; i++) {
+    runTestFile(testFiles[i]);
+  }
+})();