test_runner: add support for coverage via run()

PR-URL: https://github.com/nodejs/node/pull/53937
Fixes: https://github.com/nodejs/node/issues/53867
Refs: https://github.com/nodejs/node/issues/53924
Reviewed-By: Colin Ihrig <cjihrig@gmail.com>
Reviewed-By: Moshe Atlow <moshe@atlow.co.il>
Reviewed-By: Matteo Collina <matteo.collina@gmail.com>

doc/api/test.md

@@ -1246,6 +1246,9 @@ added:
   - v18.9.0
   - v16.19.0
 changes:
+  - version: REPLACEME
+    pr-url: https://github.com/nodejs/node/pull/53937
+    description: Added coverage options.
   - version: v22.8.0
     pr-url: https://github.com/nodejs/node/pull/53927
     description: Added the `isolation` option.
@@ -1319,6 +1322,29 @@ changes:
       that specifies the index of the shard to run. This option is _required_.
     * `total` {number} is a positive integer that specifies the total number
       of shards to split the test files to. This option is _required_.
+  * `coverage` {boolean} enable [code coverage][] collection.
+    **Default:** `false`.
+  * `coverageExcludeGlobs` {string|Array} Excludes specific files from code coverage
+    using a glob pattern, which can match both absolute and relative file paths.
+    This property is only applicable when `coverage` was set to `true`.
+    If both `coverageExcludeGlobs` and `coverageIncludeGlobs` are provided,
+    files must meet **both** criteria to be included in the coverage report.
+    **Default:** `undefined`.
+  * `coverageIncludeGlobs` {string|Array} Includes specific files in code coverage
+    using a glob pattern, which can match both absolute and relative file paths.
+    This property is only applicable when `coverage` was set to `true`.
+    If both `coverageExcludeGlobs` and `coverageIncludeGlobs` are provided,
+    files must meet **both** criteria to be included in the coverage report.
+    **Default:** `undefined`.
+  * `lineCoverage` {number} Require a minimum percent of covered lines. If code
+    coverage does not reach the threshold specified, the process will exit with code `1`.
+    **Default:** `0`.
+  * `branchCoverage` {number} Require a minimum percent of covered branches. If code
+    coverage does not reach the threshold specified, the process will exit with code `1`.
+    **Default:** `0`.
+  * `functionCoverage` {number} Require a minimum percent of covered functions. If code
+    coverage does not reach the threshold specified, the process will exit with code `1`.
+    **Default:** `0`.
 * Returns: {TestsStream}
 
 **Note:** `shard` is used to horizontally parallelize test running across
@@ -3537,6 +3563,7 @@ Can be used to abort test subtasks when the test has been aborted.
 [`run()`]: #runoptions
 [`suite()`]: #suitename-options-fn
 [`test()`]: #testname-options-fn
+[code coverage]: #collecting-code-coverage
 [describe options]: #describename-options-fn
 [it options]: #testname-options-fn
 [stream.compose]: stream.md#streamcomposestreams

lib/internal/test_runner/runner.js

@@ -55,6 +55,7 @@ const {
   validateObject,
   validateOneOf,
   validateInteger,
+  validateStringArray,
 } = require('internal/validators');
 const { getInspectPort, isUsingInspector, isInspectorMessage } = require('internal/util/inspector');
 const { isRegExp } = require('internal/util/types');
@@ -524,7 +525,13 @@ function watchFiles(testFiles, opts) {
 function run(options = kEmptyObject) {
   validateObject(options, 'options');
 
-  let { testNamePatterns, testSkipPatterns, shard } = options;
+  let {
+    testNamePatterns,
+    testSkipPatterns,
+    shard,
+    coverageExcludeGlobs,
+    coverageIncludeGlobs,
+  } = options;
   const {
     concurrency,
     timeout,
@@ -537,6 +544,10 @@ function run(options = kEmptyObject) {
     setup,
     only,
     globPatterns,
+    coverage = false,
+    lineCoverage = 0,
+    branchCoverage = 0,
+    functionCoverage = 0,
   } = options;
 
   if (files != null) {
@@ -615,6 +626,22 @@ function run(options = kEmptyObject) {
     });
   }
   validateOneOf(isolation, 'options.isolation', ['process', 'none']);
+  validateBoolean(coverage, 'options.coverage');
+  if (coverageExcludeGlobs != null) {
+    if (!ArrayIsArray(coverageExcludeGlobs)) {
+      coverageExcludeGlobs = [coverageExcludeGlobs];
+    }
+    validateStringArray(coverageExcludeGlobs, 'options.coverageExcludeGlobs');
+  }
+  if (coverageIncludeGlobs != null) {
+    if (!ArrayIsArray(coverageIncludeGlobs)) {
+      coverageIncludeGlobs = [coverageIncludeGlobs];
+    }
+    validateStringArray(coverageIncludeGlobs, 'options.coverageIncludeGlobs');
+  }
+  validateInteger(lineCoverage, 'options.lineCoverage', 0, 100);
+  validateInteger(branchCoverage, 'options.branchCoverage', 0, 100);
+  validateInteger(functionCoverage, 'options.functionCoverage', 0, 100);
 
   const rootTestOptions = { __proto__: null, concurrency, timeout, signal };
   const globalOptions = {
@@ -623,6 +650,12 @@ function run(options = kEmptyObject) {
     // behavior has relied on it, so removing it must be done in a semver major.
     ...parseCommandLine(),
     setup,  // This line can be removed when parseCommandLine() is removed here.
+    coverage,
+    coverageExcludeGlobs,
+    coverageIncludeGlobs,
+    lineCoverage: lineCoverage,
+    branchCoverage: branchCoverage,
+    functionCoverage: functionCoverage,
   };
   const root = createTestTree(rootTestOptions, globalOptions);
 

test/parallel/test-runner-run-coverage.mjs

@@ -0,0 +1,186 @@
+import * as common from '../common/index.mjs';
+import * as fixtures from '../common/fixtures.mjs';
+import { describe, it, run } from 'node:test';
+import assert from 'node:assert';
+import { sep } from 'node:path';
+
+const files = [fixtures.path('test-runner', 'coverage.js')];
+const abortedSignal = AbortSignal.abort();
+
+describe('require(\'node:test\').run coverage settings', { concurrency: true }, async () => {
+  await describe('validation', async () => {
+    await it('should only allow boolean in options.coverage', async () => {
+      [Symbol(), {}, () => {}, 0, 1, 0n, 1n, '', '1', Promise.resolve(true), []]
+        .forEach((coverage) => assert.throws(() => run({ coverage }), {
+          code: 'ERR_INVALID_ARG_TYPE'
+        }));
+    });
+
+    await it('should only allow string|string[] in options.coverageExcludeGlobs', async () => {
+      [Symbol(), {}, () => {}, 0, 1, 0n, 1n, Promise.resolve([]), true, false]
+        .forEach((coverageExcludeGlobs) => {
+          assert.throws(() => run({ coverage: true, coverageExcludeGlobs }), {
+            code: 'ERR_INVALID_ARG_TYPE'
+          });
+          assert.throws(() => run({ coverage: true, coverageExcludeGlobs: [coverageExcludeGlobs] }), {
+            code: 'ERR_INVALID_ARG_TYPE'
+          });
+        });
+      run({ files: [], signal: abortedSignal, coverage: true, coverageExcludeGlobs: [''] });
+      run({ files: [], signal: abortedSignal, coverage: true, coverageExcludeGlobs: '' });
+    });
+
+    await it('should only allow string|string[] in options.coverageIncludeGlobs', async () => {
+      [Symbol(), {}, () => {}, 0, 1, 0n, 1n, Promise.resolve([]), true, false]
+        .forEach((coverageIncludeGlobs) => {
+          assert.throws(() => run({ coverage: true, coverageIncludeGlobs }), {
+            code: 'ERR_INVALID_ARG_TYPE'
+          });
+          assert.throws(() => run({ coverage: true, coverageIncludeGlobs: [coverageIncludeGlobs] }), {
+            code: 'ERR_INVALID_ARG_TYPE'
+          });
+        });
+
+      run({ files: [], signal: abortedSignal, coverage: true, coverageIncludeGlobs: [''] });
+      run({ files: [], signal: abortedSignal, coverage: true, coverageIncludeGlobs: '' });
+    });
+
+    await it('should only allow an int within range in options.lineCoverage', async () => {
+      [Symbol(), {}, () => {}, [], 0n, 1n, Promise.resolve([]), true, false]
+        .forEach((lineCoverage) => {
+          assert.throws(() => run({ coverage: true, lineCoverage }), {
+            code: 'ERR_INVALID_ARG_TYPE'
+          });
+          assert.throws(() => run({ coverage: true, lineCoverage: [lineCoverage] }), {
+            code: 'ERR_INVALID_ARG_TYPE'
+          });
+        });
+      assert.throws(() => run({ coverage: true, lineCoverage: -1 }), { code: 'ERR_OUT_OF_RANGE' });
+      assert.throws(() => run({ coverage: true, lineCoverage: 101 }), { code: 'ERR_OUT_OF_RANGE' });
+
+      run({ files: [], signal: abortedSignal, coverage: true, lineCoverage: 0 });
+    });
+
+    await it('should only allow an int within range in options.branchCoverage', async () => {
+      [Symbol(), {}, () => {}, [], 0n, 1n, Promise.resolve([]), true, false]
+        .forEach((branchCoverage) => {
+          assert.throws(() => run({ coverage: true, branchCoverage }), {
+            code: 'ERR_INVALID_ARG_TYPE'
+          });
+          assert.throws(() => run({ coverage: true, branchCoverage: [branchCoverage] }), {
+            code: 'ERR_INVALID_ARG_TYPE'
+          });
+        });
+
+      assert.throws(() => run({ coverage: true, branchCoverage: -1 }), { code: 'ERR_OUT_OF_RANGE' });
+      assert.throws(() => run({ coverage: true, branchCoverage: 101 }), { code: 'ERR_OUT_OF_RANGE' });
+
+      run({ files: [], signal: abortedSignal, coverage: true, branchCoverage: 0 });
+    });
+
+    await it('should only allow an int within range in options.functionCoverage', async () => {
+      [Symbol(), {}, () => {}, [], 0n, 1n, Promise.resolve([]), true, false]
+        .forEach((functionCoverage) => {
+          assert.throws(() => run({ coverage: true, functionCoverage }), {
+            code: 'ERR_INVALID_ARG_TYPE'
+          });
+          assert.throws(() => run({ coverage: true, functionCoverage: [functionCoverage] }), {
+            code: 'ERR_INVALID_ARG_TYPE'
+          });
+        });
+
+      assert.throws(() => run({ coverage: true, functionCoverage: -1 }), { code: 'ERR_OUT_OF_RANGE' });
+      assert.throws(() => run({ coverage: true, functionCoverage: 101 }), { code: 'ERR_OUT_OF_RANGE' });
+
+      run({ files: [], signal: abortedSignal, coverage: true, functionCoverage: 0 });
+    });
+  });
+
+  const options = { concurrency: false, skip: !process.features.inspector ? 'inspector disabled' : false };
+  await describe('run with coverage', options, async () => {
+    await it('should run with coverage', async () => {
+      const stream = run({ files, coverage: true });
+      stream.on('test:fail', common.mustNotCall());
+      stream.on('test:pass', common.mustCall());
+      stream.on('test:coverage', common.mustCall());
+      // eslint-disable-next-line no-unused-vars
+      for await (const _ of stream);
+    });
+
+    await it('should run with coverage and exclude by glob', async () => {
+      const stream = run({ files, coverage: true, coverageExcludeGlobs: ['test/*/test-runner/invalid-tap.js'] });
+      stream.on('test:fail', common.mustNotCall());
+      stream.on('test:pass', common.mustCall(1));
+      stream.on('test:coverage', common.mustCall(({ summary: { files } }) => {
+        const filesPaths = files.map(({ path }) => path);
+        assert.strictEqual(filesPaths.some((path) => path.includes(`test-runner${sep}invalid-tap.js`)), false);
+      }));
+      // eslint-disable-next-line no-unused-vars
+      for await (const _ of stream);
+    });
+
+    await it('should run with coverage and include by glob', async () => {
+      const stream = run({
+        files,
+        coverage: true,
+        coverageIncludeGlobs: ['test/fixtures/test-runner/coverage.js', 'test/*/v8-coverage/throw.js'],
+      });
+      stream.on('test:fail', common.mustNotCall());
+      stream.on('test:pass', common.mustCall(1));
+      stream.on('test:coverage', common.mustCall(({ summary: { files } }) => {
+        const filesPaths = files.map(({ path }) => path);
+        assert.strictEqual(filesPaths.some((path) => path.includes(`v8-coverage${sep}throw.js`)), true);
+      }));
+      // eslint-disable-next-line no-unused-vars
+      for await (const _ of stream);
+    });
+
+    await it('should run while including and excluding globs', async () => {
+      const stream = run({
+        files: [...files, fixtures.path('test-runner/invalid-tap.js')],
+        coverage: true,
+        coverageIncludeGlobs: ['test/fixtures/test-runner/*.js'],
+        coverageExcludeGlobs: ['test/fixtures/test-runner/*-tap.js']
+      });
+      stream.on('test:fail', common.mustNotCall());
+      stream.on('test:pass', common.mustCall(2));
+      stream.on('test:coverage', common.mustCall(({ summary: { files } }) => {
+        const filesPaths = files.map(({ path }) => path);
+        assert.strictEqual(filesPaths.every((path) => !path.includes(`test-runner${sep}invalid-tap.js`)), true);
+        assert.strictEqual(filesPaths.some((path) => path.includes(`test-runner${sep}coverage.js`)), true);
+      }));
+      // eslint-disable-next-line no-unused-vars
+      for await (const _ of stream);
+    });
+
+    await it('should run with coverage and fail when below line threshold', async () => {
+      const thresholdErrors = [];
+      const originalExitCode = process.exitCode;
+      assert.notStrictEqual(originalExitCode, 1);
+      const stream = run({ files, coverage: true, lineCoverage: 99, branchCoverage: 99, functionCoverage: 99 });
+      stream.on('test:fail', common.mustNotCall());
+      stream.on('test:pass', common.mustCall(1));
+      stream.on('test:diagnostic', ({ message }) => {
+        const match = message.match(/Error: \d{2}\.\d{2}% (line|branch|function) coverage does not meet threshold of 99%/);
+        if (match) {
+          thresholdErrors.push(match[1]);
+        }
+      });
+      // eslint-disable-next-line no-unused-vars
+      for await (const _ of stream);
+      assert.deepStrictEqual(thresholdErrors.sort(), ['branch', 'function', 'line']);
+      assert.strictEqual(process.exitCode, 1);
+      process.exitCode = originalExitCode;
+    });
+  });
+});
+
+
+// exitHandler doesn't run until after the tests / after hooks finish.
+process.on('exit', () => {
+  assert.strictEqual(process.listeners('uncaughtException').length, 0);
+  assert.strictEqual(process.listeners('unhandledRejection').length, 0);
+  assert.strictEqual(process.listeners('beforeExit').length, 0);
+  assert.strictEqual(process.listeners('SIGINT').length, 0);
+  assert.strictEqual(process.listeners('SIGTERM').length, 0);
+});