test_runner: add support for coverage thresholds

Co-Authored-By: Marco Ippolito <[email protected]>
PR-URL: #54429
Reviewed-By: Matteo Collina <[email protected]>
Reviewed-By: Paolo Insogna <[email protected]>
Reviewed-By: Moshe Atlow <[email protected]>
Reviewed-By: Colin Ihrig <[email protected]>
Reviewed-By: Marco Ippolito <[email protected]>

Author: 2 people

doc/api/cli.md

@@ -2218,6 +2218,17 @@ concurrently. If `--experimental-test-isolation` is set to `'none'`, this flag
 is ignored and concurrency is one. Otherwise, concurrency defaults to
 `os.availableParallelism() - 1`.
 
+### `--test-coverage-branches=threshold`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+> Stability: 1 - Experimental
+
+Require a minimum percent of covered branches. If code coverage does not reach
+the threshold specified, the process will exit with code `1`.
+
 ### `--test-coverage-exclude`
 
 <!-- YAML
@@ -2235,6 +2246,17 @@ This option may be specified multiple times to exclude multiple glob patterns.
 If both `--test-coverage-exclude` and `--test-coverage-include` are provided,
 files must meet **both** criteria to be included in the coverage report.
 
+### `--test-coverage-functions=threshold`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+> Stability: 1 - Experimental
+
+Require a minimum percent of covered functions. If code coverage does not reach
+the threshold specified, the process will exit with code `1`.
+
 ### `--test-coverage-include`
 
 <!-- YAML
@@ -2252,6 +2274,17 @@ This option may be specified multiple times to include multiple glob patterns.
 If both `--test-coverage-exclude` and `--test-coverage-include` are provided,
 files must meet **both** criteria to be included in the coverage report.
 
+### `--test-coverage-lines=threshold`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+> Stability: 1 - Experimental
+
+Require a minimum percent of covered lines. If code coverage does not reach
+the threshold specified, the process will exit with code `1`.
+
 ### `--test-force-exit`
 
 <!-- YAML
@@ -3047,8 +3080,11 @@ one is included in the list below.
 * `--secure-heap-min`
 * `--secure-heap`
 * `--snapshot-blob`
+* `--test-coverage-branches`
 * `--test-coverage-exclude`
+* `--test-coverage-functions`
 * `--test-coverage-include`
+* `--test-coverage-lines`
 * `--test-name-pattern`
 * `--test-only`
 * `--test-reporter-destination`

doc/node.1

@@ -450,12 +450,21 @@ Starts the Node.js command line test runner.
 The maximum number of test files that the test runner CLI will execute
 concurrently.
 .
+.It Fl -test-coverage-branches Ns = Ns Ar threshold
+Require a minimum threshold for branch coverage (0 - 100).
+.
 .It Fl -test-coverage-exclude
 A glob pattern that excludes matching files from the coverage report
 .
+.It Fl -test-coverage-functions Ns = Ns Ar threshold
+Require a minimum threshold for function coverage (0 - 100).
+.
 .It Fl -test-coverage-include
 A glob pattern that only includes matching files in the coverage report
 .
+.It Fl -test-coverage-lines Ns = Ns Ar threshold
+Require a minimum threshold for line coverage (0 - 100).
+.
 .It Fl -test-force-exit
 Configures the test runner to exit the process once all known tests have
 finished executing even if the event loop would otherwise remain active.

lib/internal/test_runner/test.js

@@ -11,6 +11,7 @@ const {
   FunctionPrototype,
   MathMax,
   Number,
+  NumberPrototypeToFixed,
   ObjectDefineProperty,
   ObjectSeal,
   PromisePrototypeThen,
@@ -28,6 +29,7 @@ const {
   SymbolDispose,
 } = primordials;
 const { getCallerLocation } = internalBinding('util');
+const { exitCodes: { kGenericUserError } } = internalBinding('errors');
 const { addAbortListener } = require('internal/events/abort_listener');
 const { queueMicrotask } = require('internal/process/task_queues');
 const { AsyncResource } = require('async_hooks');
@@ -1009,6 +1011,25 @@ class Test extends AsyncResource {
 
       if (coverage) {
         reporter.coverage(nesting, loc, coverage);
+
+        const coverages = [
+          { __proto__: null, actual: coverage.totals.coveredLinePercent,
+            threshold: this.config.lineCoverage, name: 'line' },
+
+          { __proto__: null, actual: coverage.totals.coveredBranchPercent,
+            threshold: this.config.branchCoverage, name: 'branch' },
+
+          { __proto__: null, actual: coverage.totals.coveredFunctionPercent,
+            threshold: this.config.functionCoverage, name: 'function' },
+        ];
+
+        for (let i = 0; i < coverages.length; i++) {
+          const { threshold, actual, name } = coverages[i];
+          if (actual < threshold) {
+            process.exitCode = kGenericUserError;
+            reporter.diagnostic(nesting, loc, `Error: ${NumberPrototypeToFixed(actual, 2)}% ${name} coverage does not meet threshold of ${threshold}%.`);
+          }
+        }
       }
 
       if (harness.watching) {

lib/internal/test_runner/utils.js

@@ -39,6 +39,7 @@ const {
   kIsNodeError,
 } = require('internal/errors');
 const { compose } = require('stream');
+const { validateInteger } = require('internal/validators');
 
 const coverageColors = {
   __proto__: null,
@@ -194,6 +195,9 @@ function parseCommandLine() {
   let concurrency;
   let coverageExcludeGlobs;
   let coverageIncludeGlobs;
+  let lineCoverage;
+  let branchCoverage;
+  let functionCoverage;
   let destinations;
   let isolation;
   let only = getOptionValue('--test-only');
@@ -278,6 +282,14 @@ function parseCommandLine() {
   if (coverage) {
     coverageExcludeGlobs = getOptionValue('--test-coverage-exclude');
     coverageIncludeGlobs = getOptionValue('--test-coverage-include');
+
+    branchCoverage = getOptionValue('--test-coverage-branches');
+    lineCoverage = getOptionValue('--test-coverage-lines');
+    functionCoverage = getOptionValue('--test-coverage-functions');
+
+    validateInteger(branchCoverage, '--test-coverage-branches', 0, 100);
+    validateInteger(lineCoverage, '--test-coverage-lines', 0, 100);
+    validateInteger(functionCoverage, '--test-coverage-functions', 0, 100);
   }
 
   const setup = reporterScope.bind(async (rootReporter) => {
@@ -299,6 +311,9 @@ function parseCommandLine() {
     destinations,
     forceExit,
     isolation,
+    branchCoverage,
+    functionCoverage,
+    lineCoverage,
     only,
     reporters,
     setup,

src/node_options.cc

@@ -670,6 +670,19 @@ EnvironmentOptionsParser::EnvironmentOptionsParser() {
   AddOption("--experimental-test-coverage",
             "enable code coverage in the test runner",
             &EnvironmentOptions::test_runner_coverage);
+  AddOption("--test-coverage-branches",
+            "the branch coverage minimum threshold",
+            &EnvironmentOptions::test_coverage_branches,
+            kAllowedInEnvvar);
+  AddOption("--test-coverage-functions",
+            "the function coverage minimum threshold",
+            &EnvironmentOptions::test_coverage_functions,
+            kAllowedInEnvvar);
+  AddOption("--test-coverage-lines",
+            "the line coverage minimum threshold",
+            &EnvironmentOptions::test_coverage_lines,
+            kAllowedInEnvvar);
+
   AddOption("--experimental-test-isolation",
             "configures the type of test isolation used in the test runner",
             &EnvironmentOptions::test_isolation);

src/node_options.h

@@ -184,6 +184,9 @@ class EnvironmentOptions : public Options {
   uint64_t test_runner_timeout = 0;
   bool test_runner_coverage = false;
   bool test_runner_force_exit = false;
+  uint64_t test_coverage_branches = 0;
+  uint64_t test_coverage_functions = 0;
+  uint64_t test_coverage_lines = 0;
   bool test_runner_module_mocks = false;
   bool test_runner_snapshots = false;
   bool test_runner_update_snapshots = false;

test/parallel/test-runner-coverage-thresholds.js

@@ -0,0 +1,110 @@
+'use strict';
+const common = require('../common');
+const assert = require('node:assert');
+const { spawnSync } = require('node:child_process');
+const { readdirSync } = require('node:fs');
+const { test } = require('node:test');
+const fixtures = require('../common/fixtures');
+const tmpdir = require('../common/tmpdir');
+
+common.skipIfInspectorDisabled();
+tmpdir.refresh();
+
+function findCoverageFileForPid(pid) {
+  const pattern = `^coverage\\-${pid}\\-(\\d{13})\\-(\\d+)\\.json$`;
+  const regex = new RegExp(pattern);
+
+  return readdirSync(tmpdir.path).find((file) => {
+    return regex.test(file);
+  });
+}
+
+function getTapCoverageFixtureReport() {
+  /* eslint-disable @stylistic/js/max-len */
+  const report = [
+    '# start of coverage report',
+    '# -------------------------------------------------------------------------------------------------------------------',
+    '# file                                     | line % | branch % | funcs % | uncovered lines',
+    '# -------------------------------------------------------------------------------------------------------------------',
+    '# test/fixtures/test-runner/coverage.js    |  78.65 |    38.46 |   60.00 | 12-13 16-22 27 39 43-44 61-62 66-67 71-72',
+    '# test/fixtures/test-runner/invalid-tap.js | 100.00 |   100.00 |  100.00 | ',
+    '# test/fixtures/v8-coverage/throw.js       |  71.43 |    50.00 |  100.00 | 5-6',
+    '# -------------------------------------------------------------------------------------------------------------------',
+    '# all files                                |  78.35 |    43.75 |   60.00 |',
+    '# -------------------------------------------------------------------------------------------------------------------',
+    '# end of coverage report',
+  ].join('\n');
+  /* eslint-enable @stylistic/js/max-len */
+
+  if (common.isWindows) {
+    return report.replaceAll('/', '\\');
+  }
+
+  return report;
+}
+
+const fixture = fixtures.path('test-runner', 'coverage.js');
+const neededArguments = [
+  '--experimental-test-coverage',
+  '--test-reporter', 'tap',
+];
+
+const coverages = [
+  { flag: '--test-coverage-lines', name: 'line', actual: 78.35 },
+  { flag: '--test-coverage-functions', name: 'function', actual: 60.00 },
+  { flag: '--test-coverage-branches', name: 'branch', actual: 43.75 },
+];
+
+for (const coverage of coverages) {
+  test(`test passing ${coverage.flag}`, async (t) => {
+    const result = spawnSync(process.execPath, [
+      ...neededArguments,
+      `${coverage.flag}=25`,
+      fixture,
+    ]);
+
+    const stdout = result.stdout.toString();
+    assert(stdout.includes(getTapCoverageFixtureReport()));
+    assert.doesNotMatch(stdout, RegExp(`Error: [\\d\\.]+% ${coverage.name} coverage`));
+    assert.strictEqual(result.status, 0);
+    assert(!findCoverageFileForPid(result.pid));
+  });
+
+  test(`test failing ${coverage.flag}`, async (t) => {
+    const result = spawnSync(process.execPath, [
+      ...neededArguments,
+      `${coverage.flag}=99`,
+      fixture,
+    ]);
+
+    const stdout = result.stdout.toString();
+    assert(stdout.includes(getTapCoverageFixtureReport()));
+    assert.match(stdout, RegExp(`Error: ${coverage.actual.toFixed(2)}% ${coverage.name} coverage does not meet threshold of 99%`));
+    assert.strictEqual(result.status, 1);
+    assert(!findCoverageFileForPid(result.pid));
+  });
+
+  test(`test out-of-range ${coverage.flag} (too high)`, async (t) => {
+    const result = spawnSync(process.execPath, [
+      ...neededArguments,
+      `${coverage.flag}=101`,
+      fixture,
+    ]);
+
+    assert.match(result.stderr.toString(), RegExp(`The value of "${coverage.flag}`));
+    assert.strictEqual(result.status, 1);
+    assert(!findCoverageFileForPid(result.pid));
+  });
+
+  test(`test out-of-range ${coverage.flag} (too low)`, async (t) => {
+    const result = spawnSync(process.execPath, [
+      ...neededArguments,
+      `${coverage.flag}=-1`,
+      fixture,
+    ]);
+
+    assert.match(result.stderr.toString(), RegExp(`The value of "${coverage.flag}`));
+    assert.strictEqual(result.status, 1);
+    assert(!findCoverageFileForPid(result.pid));
+  });
+}