From 8950229fd9103806e96e96397e30eead1a0c0087 Mon Sep 17 00:00:00 2001
From: Moshe Atlow <moshe@atlow.co.il>
Date: Mon, 3 Oct 2022 12:56:25 +0300
Subject: [PATCH 1/6] test_runner: support programmatically running `--test`

PR-URL: https://github.com/nodejs/node/pull/44241
Backport-PR-URL: https://github.com/nodejs/node/pull/44873
Fixes: https://github.com/nodejs/node/issues/44023
Fixes: https://github.com/nodejs/node/issues/43675
Reviewed-By: Benjamin Gruenbaum <benjamingr@gmail.com>
Reviewed-By: Antoine du Hamel <duhamelantoine1995@gmail.com>
---
 doc/api/test.md                        |  74 +++++++++++
 lib/internal/main/test_runner.js       | 144 +---------------------
 lib/internal/test_runner/harness.js    |  89 +++++---------
 lib/internal/test_runner/runner.js     | 163 +++++++++++++++++++++++++
 lib/internal/test_runner/tap_stream.js |  34 +++---
 lib/internal/test_runner/test.js       |  58 +++++++--
 lib/test.js                            |  19 +--
 test/message/test_runner_no_tests.out  |   2 +-
 test/parallel/test-runner-run.mjs      |  69 +++++++++++
 tools/doc/type-parser.mjs              |   2 +
 10 files changed, 424 insertions(+), 230 deletions(-)
 create mode 100644 lib/internal/test_runner/runner.js
 create mode 100644 test/parallel/test-runner-run.mjs

diff --git a/doc/api/test.md b/doc/api/test.md
index 196e854eb9c0a9..fcffd932df53c1 100644
--- a/doc/api/test.md
+++ b/doc/api/test.md
@@ -316,6 +316,35 @@ Otherwise, the test is considered to be a failure. Test files must be
 executable by Node.js, but are not required to use the `node:test` module
 internally.
 
+## `run([options])`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+* `options` {Object} Configuration options for running tests. The following
+  properties are supported:
+  * `concurrency` {number|boolean} If a number is provided,
+    then that many files would run in parallel.
+    If truthy, it would run (number of cpu cores - 1)
+    files in parallel.
+    If falsy, it would only run one file at a time.
+    If unspecified, subtests inherit this value from their parent.
+    **Default:** `true`.
+  * `files`: {Array} An array containing the list of files to run.
+    **Default** matching files from [test runner execution model][].
+  * `signal` {AbortSignal} Allows aborting an in-progress test execution.
+  * `timeout` {number} A number of milliseconds the test execution will
+    fail after.
+    If unspecified, subtests inherit this value from their parent.
+    **Default:** `Infinity`.
+* Returns: {TapStream}
+
+```js
+run({ files: [path.resolve('./tests/test.js')] })
+  .pipe(process.stdout);
+```
+
 ## `test([name][, options][, fn])`
 
 <!-- YAML
@@ -560,6 +589,47 @@ describe('tests', async () => {
 });
 ```
 
+## Class: `TapStream`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+* Extends {ReadableStream}
+
+A successful call to [`run()`][] method will return a new {TapStream}
+object, streaming a [TAP][] output
+`TapStream` will emit events, in the order of the tests definition
+
+### Event: `'test:diagnostic'`
+
+* `message` {string} The diagnostic message.
+
+Emitted when [`context.diagnostic`][] is called.
+
+### Event: `'test:fail'`
+
+* `data` {Object}
+  * `duration` {number} The test duration.
+  * `error` {Error} The failure casing test to fail.
+  * `name` {string} The test name.
+  * `testNumber` {number} The ordinal number of the test.
+  * `todo` {string|undefined} Present if [`context.todo`][] is called
+  * `skip` {string|undefined} Present if [`context.skip`][] is called
+
+Emitted when a test fails.
+
+### Event: `'test:pass'`
+
+* `data` {Object}
+  * `duration` {number} The test duration.
+  * `name` {string} The test name.
+  * `testNumber` {number} The ordinal number of the test.
+  * `todo` {string|undefined} Present if [`context.todo`][] is called
+  * `skip` {string|undefined} Present if [`context.skip`][] is called
+
+Emitted when a test passes.
+
 ## Class: `TestContext`
 
 <!-- YAML
@@ -825,6 +895,10 @@ added: v16.17.0
 [`--test`]: cli.md#--test
 [`SuiteContext`]: #class-suitecontext
 [`TestContext`]: #class-testcontext
+[`context.diagnostic`]: #contextdiagnosticmessage
+[`context.skip`]: #contextskipmessage
+[`context.todo`]: #contexttodomessage
+[`run()`]: #runoptions
 [`test()`]: #testname-options-fn
 [describe options]: #describename-options-fn
 [it options]: #testname-options-fn
diff --git a/lib/internal/main/test_runner.js b/lib/internal/main/test_runner.js
index c1066cfca35323..c965a6a95fbab5 100644
--- a/lib/internal/main/test_runner.js
+++ b/lib/internal/main/test_runner.js
@@ -1,146 +1,14 @@
 'use strict';
-const {
-  ArrayFrom,
-  ArrayPrototypeFilter,
-  ArrayPrototypeIncludes,
-  ArrayPrototypeJoin,
-  ArrayPrototypePush,
-  ArrayPrototypeSlice,
-  ArrayPrototypeSort,
-  SafePromiseAll,
-  SafeSet,
-} = primordials;
 const {
   prepareMainThreadExecution,
 } = require('internal/bootstrap/pre_execution');
-const { spawn } = require('child_process');
-const { readdirSync, statSync } = require('fs');
-const console = require('internal/console/global');
-const {
-  codes: {
-    ERR_TEST_FAILURE,
-  },
-} = require('internal/errors');
-const { test } = require('internal/test_runner/harness');
-const { kSubtestsFailed } = require('internal/test_runner/test');
-const {
-  isSupportedFileType,
-  doesPathMatchFilter,
-} = require('internal/test_runner/utils');
-const { basename, join, resolve } = require('path');
-const { once } = require('events');
-const kFilterArgs = ['--test'];
+const { run } = require('internal/test_runner/runner');
 
 prepareMainThreadExecution(false);
 markBootstrapComplete();
 
-// TODO(cjihrig): Replace this with recursive readdir once it lands.
-function processPath(path, testFiles, options) {
-  const stats = statSync(path);
-
-  if (stats.isFile()) {
-    if (options.userSupplied ||
-        (options.underTestDir && isSupportedFileType(path)) ||
-        doesPathMatchFilter(path)) {
-      testFiles.add(path);
-    }
-  } else if (stats.isDirectory()) {
-    const name = basename(path);
-
-    if (!options.userSupplied && name === 'node_modules') {
-      return;
-    }
-
-    // 'test' directories get special treatment. Recursively add all .js,
-    // .cjs, and .mjs files in the 'test' directory.
-    const isTestDir = name === 'test';
-    const { underTestDir } = options;
-    const entries = readdirSync(path);
-
-    if (isTestDir) {
-      options.underTestDir = true;
-    }
-
-    options.userSupplied = false;
-
-    for (let i = 0; i < entries.length; i++) {
-      processPath(join(path, entries[i]), testFiles, options);
-    }
-
-    options.underTestDir = underTestDir;
-  }
-}
-
-function createTestFileList() {
-  const cwd = process.cwd();
-  const hasUserSuppliedPaths = process.argv.length > 1;
-  const testPaths = hasUserSuppliedPaths ?
-    ArrayPrototypeSlice(process.argv, 1) : [cwd];
-  const testFiles = new SafeSet();
-
-  try {
-    for (let i = 0; i < testPaths.length; i++) {
-      const absolutePath = resolve(testPaths[i]);
-
-      processPath(absolutePath, testFiles, { userSupplied: true });
-    }
-  } catch (err) {
-    if (err?.code === 'ENOENT') {
-      console.error(`Could not find '${err.path}'`);
-      process.exit(1);
-    }
-
-    throw err;
-  }
-
-  return ArrayPrototypeSort(ArrayFrom(testFiles));
-}
-
-function filterExecArgv(arg) {
-  return !ArrayPrototypeIncludes(kFilterArgs, arg);
-}
-
-function runTestFile(path) {
-  return test(path, async (t) => {
-    const args = ArrayPrototypeFilter(process.execArgv, filterExecArgv);
-    ArrayPrototypePush(args, path);
-
-    const child = spawn(process.execPath, args, { signal: t.signal, encoding: 'utf8' });
-    // TODO(cjihrig): Implement a TAP parser to read the child's stdout
-    // instead of just displaying it all if the child fails.
-    let err;
-
-    child.on('error', (error) => {
-      err = error;
-    });
-
-    const { 0: { 0: code, 1: signal }, 1: stdout, 2: stderr } = await SafePromiseAll([
-      once(child, 'exit', { signal: t.signal }),
-      child.stdout.toArray({ signal: t.signal }),
-      child.stderr.toArray({ signal: t.signal }),
-    ]);
-
-    if (code !== 0 || signal !== null) {
-      if (!err) {
-        err = new ERR_TEST_FAILURE('test failed', kSubtestsFailed);
-        err.exitCode = code;
-        err.signal = signal;
-        err.stdout = ArrayPrototypeJoin(stdout, '');
-        err.stderr = ArrayPrototypeJoin(stderr, '');
-        // The stack will not be useful since the failures came from tests
-        // in a child process.
-        err.stack = undefined;
-      }
-
-      throw err;
-    }
-  });
-}
-
-(async function main() {
-  const testFiles = createTestFileList();
-
-  for (let i = 0; i < testFiles.length; i++) {
-    runTestFile(testFiles[i]);
-  }
-})();
+const tapStream = run();
+tapStream.pipe(process.stdout);
+tapStream.once('test:fail', () => {
+  process.exitCode = 1;
+});
diff --git a/lib/internal/test_runner/harness.js b/lib/internal/test_runner/harness.js
index 360fe141fe70a1..62d26a1a1bd8bf 100644
--- a/lib/internal/test_runner/harness.js
+++ b/lib/internal/test_runner/harness.js
@@ -1,8 +1,8 @@
 'use strict';
 const {
   ArrayPrototypeForEach,
-  FunctionPrototypeBind,
   SafeMap,
+  SafeWeakSet,
 } = primordials;
 const {
   createHook,
@@ -13,13 +13,18 @@ const {
     ERR_TEST_FAILURE,
   },
 } = require('internal/errors');
+const { kEmptyObject } = require('internal/util');
 const { getOptionValue } = require('internal/options');
 const { kCancelledByParent, Test, ItTest, Suite } = require('internal/test_runner/test');
+const { bigint: hrtime } = process.hrtime;
 
-const isTestRunner = getOptionValue('--test');
+const isTestRunnerCli = getOptionValue('--test');
 const testResources = new SafeMap();
-const root = new Test({ __proto__: null, name: '<root>' });
-let wasRootSetup = false;
+const wasRootSetup = new SafeWeakSet();
+
+function createTestTree(options = kEmptyObject) {
+  return setup(new Test({ __proto__: null, ...options, name: '<root>' }));
+}
 
 function createProcessEventHandler(eventName, rootTest) {
   return (err) => {
@@ -48,7 +53,7 @@ function createProcessEventHandler(eventName, rootTest) {
 }
 
 function setup(root) {
-  if (wasRootSetup) {
+  if (wasRootSetup.has(root)) {
     return root;
   }
   const hook = createHook({
@@ -81,52 +86,9 @@ function setup(root) {
       'Promise resolution is still pending but the event loop has already resolved',
       kCancelledByParent));
 
-    let passCount = 0;
-    let failCount = 0;
-    let skipCount = 0;
-    let todoCount = 0;
-    let cancelledCount = 0;
-
-    for (let i = 0; i < root.subtests.length; i++) {
-      const test = root.subtests[i];
-
-      // Check SKIP and TODO tests first, as those should not be counted as
-      // failures.
-      if (test.skipped) {
-        skipCount++;
-      } else if (test.isTodo) {
-        todoCount++;
-      } else if (test.cancelled) {
-        cancelledCount++;
-      } else if (!test.passed) {
-        failCount++;
-      } else {
-        passCount++;
-      }
-    }
-
-    root.reporter.plan(root.indent, root.subtests.length);
-
-    for (let i = 0; i < root.diagnostics.length; i++) {
-      root.reporter.diagnostic(root.indent, root.diagnostics[i]);
-    }
-
-    root.reporter.diagnostic(root.indent, `tests ${root.subtests.length}`);
-    root.reporter.diagnostic(root.indent, `pass ${passCount}`);
-    root.reporter.diagnostic(root.indent, `fail ${failCount}`);
-    root.reporter.diagnostic(root.indent, `cancelled ${cancelledCount}`);
-    root.reporter.diagnostic(root.indent, `skipped ${skipCount}`);
-    root.reporter.diagnostic(root.indent, `todo ${todoCount}`);
-    root.reporter.diagnostic(root.indent, `duration_ms ${process.uptime()}`);
-
-    root.reporter.push(null);
     hook.disable();
     process.removeListener('unhandledRejection', rejectionHandler);
     process.removeListener('uncaughtException', exceptionHandler);
-
-    if (failCount > 0 || cancelledCount > 0) {
-      process.exitCode = 1;
-    }
   };
 
   const terminationHandler = () => {
@@ -137,29 +99,41 @@ function setup(root) {
   process.on('uncaughtException', exceptionHandler);
   process.on('unhandledRejection', rejectionHandler);
   process.on('beforeExit', exitHandler);
-  // TODO(MoLow): Make it configurable to hook when isTestRunner === false.
-  if (isTestRunner) {
+  // TODO(MoLow): Make it configurable to hook when isTestRunnerCli === false.
+  if (isTestRunnerCli) {
     process.on('SIGINT', terminationHandler);
     process.on('SIGTERM', terminationHandler);
   }
 
-  root.reporter.pipe(process.stdout);
+  root.startTime = hrtime();
   root.reporter.version();
 
-  wasRootSetup = true;
+  wasRootSetup.add(root);
   return root;
 }
 
+let globalRoot;
+function getGlobalRoot() {
+  if (!globalRoot) {
+    globalRoot = createTestTree();
+    globalRoot.reporter.pipe(process.stdout);
+    globalRoot.reporter.once('test:fail', () => {
+      process.exitCode = 1;
+    });
+  }
+  return globalRoot;
+}
+
 function test(name, options, fn) {
-  const subtest = setup(root).createSubtest(Test, name, options, fn);
+  const subtest = getGlobalRoot().createSubtest(Test, name, options, fn);
   return subtest.start();
 }
 
 function runInParentContext(Factory) {
   function run(name, options, fn, overrides) {
-    const parent = testResources.get(executionAsyncId()) || setup(root);
+    const parent = testResources.get(executionAsyncId()) || getGlobalRoot();
     const subtest = parent.createSubtest(Factory, name, options, fn, overrides);
-    if (parent === root) {
+    if (parent === getGlobalRoot()) {
       subtest.start();
     }
   }
@@ -178,13 +152,14 @@ function runInParentContext(Factory) {
 
 function hook(hook) {
   return (fn, options) => {
-    const parent = testResources.get(executionAsyncId()) || setup(root);
+    const parent = testResources.get(executionAsyncId()) || getGlobalRoot();
     parent.createHook(hook, fn, options);
   };
 }
 
 module.exports = {
-  test: FunctionPrototypeBind(test, root),
+  createTestTree,
+  test,
   describe: runInParentContext(Suite),
   it: runInParentContext(ItTest),
   before: hook('before'),
diff --git a/lib/internal/test_runner/runner.js b/lib/internal/test_runner/runner.js
new file mode 100644
index 00000000000000..a4afbcb0a7710f
--- /dev/null
+++ b/lib/internal/test_runner/runner.js
@@ -0,0 +1,163 @@
+'use strict';
+const {
+  ArrayFrom,
+  ArrayPrototypeConcat,
+  ArrayPrototypeFilter,
+  ArrayPrototypeIncludes,
+  ArrayPrototypeJoin,
+  ArrayPrototypeSlice,
+  ArrayPrototypeSort,
+  ObjectAssign,
+  PromisePrototypeThen,
+  SafePromiseAll,
+  SafeSet,
+} = primordials;
+
+const { spawn } = require('child_process');
+const { readdirSync, statSync } = require('fs');
+const console = require('internal/console/global');
+const {
+  codes: {
+    ERR_TEST_FAILURE,
+  },
+} = require('internal/errors');
+const { validateArray } = require('internal/validators');
+const { kEmptyObject } = require('internal/util');
+const { createTestTree } = require('internal/test_runner/harness');
+const { kSubtestsFailed, Test } = require('internal/test_runner/test');
+const {
+  isSupportedFileType,
+  doesPathMatchFilter,
+} = require('internal/test_runner/utils');
+const { basename, join, resolve } = require('path');
+const { once } = require('events');
+
+const kFilterArgs = ['--test'];
+
+// TODO(cjihrig): Replace this with recursive readdir once it lands.
+function processPath(path, testFiles, options) {
+  const stats = statSync(path);
+
+  if (stats.isFile()) {
+    if (options.userSupplied ||
+        (options.underTestDir && isSupportedFileType(path)) ||
+        doesPathMatchFilter(path)) {
+      testFiles.add(path);
+    }
+  } else if (stats.isDirectory()) {
+    const name = basename(path);
+
+    if (!options.userSupplied && name === 'node_modules') {
+      return;
+    }
+
+    // 'test' directories get special treatment. Recursively add all .js,
+    // .cjs, and .mjs files in the 'test' directory.
+    const isTestDir = name === 'test';
+    const { underTestDir } = options;
+    const entries = readdirSync(path);
+
+    if (isTestDir) {
+      options.underTestDir = true;
+    }
+
+    options.userSupplied = false;
+
+    for (let i = 0; i < entries.length; i++) {
+      processPath(join(path, entries[i]), testFiles, options);
+    }
+
+    options.underTestDir = underTestDir;
+  }
+}
+
+function createTestFileList() {
+  const cwd = process.cwd();
+  const hasUserSuppliedPaths = process.argv.length > 1;
+  const testPaths = hasUserSuppliedPaths ?
+    ArrayPrototypeSlice(process.argv, 1) : [cwd];
+  const testFiles = new SafeSet();
+
+  try {
+    for (let i = 0; i < testPaths.length; i++) {
+      const absolutePath = resolve(testPaths[i]);
+
+      processPath(absolutePath, testFiles, { userSupplied: true });
+    }
+  } catch (err) {
+    if (err?.code === 'ENOENT') {
+      console.error(`Could not find '${err.path}'`);
+      process.exit(1);
+    }
+
+    throw err;
+  }
+
+  return ArrayPrototypeSort(ArrayFrom(testFiles));
+}
+
+function filterExecArgv(arg) {
+  return !ArrayPrototypeIncludes(kFilterArgs, arg);
+}
+
+function runTestFile(path, root) {
+  const subtest = root.createSubtest(Test, path, async (t) => {
+    const args = ArrayPrototypeConcat(
+      ArrayPrototypeFilter(process.execArgv, filterExecArgv),
+      path);
+
+    const child = spawn(process.execPath, args, { signal: t.signal, encoding: 'utf8' });
+    // TODO(cjihrig): Implement a TAP parser to read the child's stdout
+    // instead of just displaying it all if the child fails.
+    let err;
+
+    child.on('error', (error) => {
+      err = error;
+    });
+
+    const { 0: { 0: code, 1: signal }, 1: stdout, 2: stderr } = await SafePromiseAll([
+      once(child, 'exit', { signal: t.signal }),
+      child.stdout.toArray({ signal: t.signal }),
+      child.stderr.toArray({ signal: t.signal }),
+    ]);
+
+    if (code !== 0 || signal !== null) {
+      if (!err) {
+        err = ObjectAssign(new ERR_TEST_FAILURE('test failed', kSubtestsFailed), {
+          __proto__: null,
+          exitCode: code,
+          signal: signal,
+          stdout: ArrayPrototypeJoin(stdout, ''),
+          stderr: ArrayPrototypeJoin(stderr, ''),
+          // The stack will not be useful since the failures came from tests
+          // in a child process.
+          stack: undefined,
+        });
+      }
+
+      throw err;
+    }
+  });
+  return subtest.start();
+}
+
+function run(options) {
+  if (options === null || typeof options !== 'object') {
+    options = kEmptyObject;
+  }
+  const { concurrency, timeout, signal, files } = options;
+
+  if (files != null) {
+    validateArray(files, 'options.files');
+  }
+
+  const root = createTestTree({ concurrency, timeout, signal });
+  const testFiles = files ?? createTestFileList();
+
+  PromisePrototypeThen(SafePromiseAll(testFiles, (path) => runTestFile(path, root)),
+                       () => root.postRun());
+
+  return root.reporter;
+}
+
+module.exports = { run };
diff --git a/lib/internal/test_runner/tap_stream.js b/lib/internal/test_runner/tap_stream.js
index 1aba42d31a4819..ebd825a3dc9236 100644
--- a/lib/internal/test_runner/tap_stream.js
+++ b/lib/internal/test_runner/tap_stream.js
@@ -2,6 +2,7 @@
 const {
   ArrayPrototypeForEach,
   ArrayPrototypeJoin,
+  ArrayPrototypeMap,
   ArrayPrototypePush,
   ArrayPrototypeShift,
   ObjectEntries,
@@ -11,7 +12,7 @@ const {
 } = primordials;
 const { inspectWithNoCustomRetry } = require('internal/errors');
 const Readable = require('internal/streams/readable');
-const { isError } = require('internal/util');
+const { isError, kEmptyObject } = require('internal/util');
 const kFrameStartRegExp = /^ {4}at /;
 const kLineBreakRegExp = /\n|\r\n/;
 const inspectOptions = { colors: false, breakLength: Infinity };
@@ -49,12 +50,16 @@ class TapStream extends Readable {
     this.#tryPush(`Bail out!${message ? ` ${tapEscape(message)}` : ''}\n`);
   }
 
-  fail(indent, testNumber, description, directive) {
-    this.#test(indent, testNumber, 'not ok', description, directive);
+  fail(indent, testNumber, name, duration, error, directive) {
+    this.emit('test:fail', { __proto__: null, name, testNumber, duration, ...directive, error });
+    this.#test(indent, testNumber, 'not ok', name, directive);
+    this.#details(indent, duration, error);
   }
 
-  ok(indent, testNumber, description, directive) {
-    this.#test(indent, testNumber, 'ok', description, directive);
+  ok(indent, testNumber, name, duration, directive) {
+    this.emit('test:pass', { __proto__: null, name, testNumber, duration, ...directive });
+    this.#test(indent, testNumber, 'ok', name, directive);
+    this.#details(indent, duration, null);
   }
 
   plan(indent, count, explanation) {
@@ -64,18 +69,18 @@ class TapStream extends Readable {
   }
 
   getSkip(reason) {
-    return `SKIP${reason ? ` ${tapEscape(reason)}` : ''}`;
+    return { __proto__: null, skip: reason };
   }
 
   getTodo(reason) {
-    return `TODO${reason ? ` ${tapEscape(reason)}` : ''}`;
+    return { __proto__: null, todo: reason };
   }
 
   subtest(indent, name) {
     this.#tryPush(`${indent}# Subtest: ${tapEscape(name)}\n`);
   }
 
-  details(indent, duration, error) {
+  #details(indent, duration, error) {
     let details = `${indent}  ---\n`;
 
     details += jsToYaml(indent, 'duration_ms', duration);
@@ -85,6 +90,7 @@ class TapStream extends Readable {
   }
 
   diagnostic(indent, message) {
+    this.emit('test:diagnostic', message);
     this.#tryPush(`${indent}# ${tapEscape(message)}\n`);
   }
 
@@ -92,16 +98,16 @@ class TapStream extends Readable {
     this.#tryPush('TAP version 13\n');
   }
 
-  #test(indent, testNumber, status, description, directive) {
+  #test(indent, testNumber, status, name, directive = kEmptyObject) {
     let line = `${indent}${status} ${testNumber}`;
 
-    if (description) {
-      line += ` ${tapEscape(description)}`;
+    if (name) {
+      line += ` ${tapEscape(`- ${name}`)}`;
     }
 
-    if (directive) {
-      line += ` # ${directive}`;
-    }
+    line += ArrayPrototypeJoin(ArrayPrototypeMap(ObjectEntries(directive), ({ 0: key, 1: value }) => (
+      ` # ${key.toUpperCase()}${value ? ` ${tapEscape(value)}` : ''}`
+    )), '');
 
     line += '\n';
     this.#tryPush(line);
diff --git a/lib/internal/test_runner/test.js b/lib/internal/test_runner/test.js
index 83b353d5808366..27ce298d456def 100644
--- a/lib/internal/test_runner/test.js
+++ b/lib/internal/test_runner/test.js
@@ -180,7 +180,7 @@ class Test extends AsyncResource {
 
       case 'boolean':
         if (concurrency) {
-          this.concurrency = isTestRunner ? MathMax(cpus().length - 1, 1) : Infinity;
+          this.concurrency = parent === null ? MathMax(cpus().length - 1, 1) : Infinity;
         } else {
           this.concurrency = 1;
         }
@@ -516,7 +516,7 @@ class Test extends AsyncResource {
   }
 
   postRun(pendingSubtestsError) {
-    let failedSubtests = 0;
+    const counters = { __proto__: null, failed: 0, passed: 0, cancelled: 0, skipped: 0, todo: 0, totalFailed: 0 };
 
     // If the test was failed before it even started, then the end time will
     // be earlier than the start time. Correct that here.
@@ -536,14 +536,28 @@ class Test extends AsyncResource {
         subtest.postRun(pendingSubtestsError);
       }
 
+      // Check SKIP and TODO tests first, as those should not be counted as
+      // failures.
+      if (subtest.skipped) {
+        counters.skipped++;
+      } else if (subtest.isTodo) {
+        counters.todo++;
+      } else if (subtest.cancelled) {
+        counters.cancelled++;
+      } else if (!subtest.passed) {
+        counters.failed++;
+      } else {
+        counters.passed++;
+      }
+
       if (!subtest.passed) {
-        failedSubtests++;
+        counters.totalFailed++;
       }
     }
 
-    if (this.passed && failedSubtests > 0) {
-      const subtestString = `subtest${failedSubtests > 1 ? 's' : ''}`;
-      const msg = `${failedSubtests} ${subtestString} failed`;
+    if ((this.passed || this.parent === null) && counters.totalFailed > 0) {
+      const subtestString = `subtest${counters.totalFailed > 1 ? 's' : ''}`;
+      const msg = `${counters.totalFailed} ${subtestString} failed`;
 
       this.fail(new ERR_TEST_FAILURE(msg, kSubtestsFailed));
     }
@@ -555,6 +569,22 @@ class Test extends AsyncResource {
       this.parent.addReadySubtest(this);
       this.parent.processReadySubtestRange(false);
       this.parent.processPendingSubtests();
+    } else if (!this.reported) {
+      this.reported = true;
+      this.reporter.plan(this.indent, this.subtests.length);
+
+      for (let i = 0; i < this.diagnostics.length; i++) {
+        this.reporter.diagnostic(this.indent, this.diagnostics[i]);
+      }
+
+      this.reporter.diagnostic(this.indent, `tests ${this.subtests.length}`);
+      this.reporter.diagnostic(this.indent, `pass ${counters.passed}`);
+      this.reporter.diagnostic(this.indent, `fail ${counters.failed}`);
+      this.reporter.diagnostic(this.indent, `cancelled ${counters.cancelled}`);
+      this.reporter.diagnostic(this.indent, `skipped ${counters.skipped}`);
+      this.reporter.diagnostic(this.indent, `todo ${counters.todo}`);
+      this.reporter.diagnostic(this.indent, `duration_ms ${this.#duration()}`);
+      this.reporter.push(null);
     }
   }
 
@@ -588,10 +618,12 @@ class Test extends AsyncResource {
     this.finished = true;
   }
 
-  report() {
+  #duration() {
     // Duration is recorded in BigInt nanoseconds. Convert to seconds.
-    const duration = Number(this.endTime - this.startTime) / 1_000_000_000;
-    const message = `- ${this.name}`;
+    return Number(this.endTime - this.startTime) / 1_000_000_000;
+  }
+
+  report() {
     let directive;
 
     if (this.skipped) {
@@ -601,13 +633,11 @@ class Test extends AsyncResource {
     }
 
     if (this.passed) {
-      this.reporter.ok(this.indent, this.testNumber, message, directive);
+      this.reporter.ok(this.indent, this.testNumber, this.name, this.#duration(), directive);
     } else {
-      this.reporter.fail(this.indent, this.testNumber, message, directive);
+      this.reporter.fail(this.indent, this.testNumber, this.name, this.#duration(), this.error, directive);
     }
 
-    this.reporter.details(this.indent, duration, this.error);
-
     for (let i = 0; i < this.diagnostics.length; i++) {
       this.reporter.diagnostic(this.indent, this.diagnostics[i]);
     }
@@ -630,6 +660,8 @@ class TestHook extends Test {
   getRunArgs() {
     return this.#args;
   }
+  postRun() {
+  }
 }
 
 class ItTest extends Test {
diff --git a/lib/test.js b/lib/test.js
index a365eef6d45f91..c8a4bae9148ab4 100644
--- a/lib/test.js
+++ b/lib/test.js
@@ -1,14 +1,19 @@
 'use strict';
+const { ObjectAssign } = primordials;
 const { test, describe, it, before, after, beforeEach, afterEach } = require('internal/test_runner/harness');
+const { run } = require('internal/test_runner/runner');
 const { emitExperimentalWarning } = require('internal/util');
 
 emitExperimentalWarning('The test runner');
 
 module.exports = test;
-module.exports.test = test;
-module.exports.describe = describe;
-module.exports.it = it;
-module.exports.before = before;
-module.exports.after = after;
-module.exports.beforeEach = beforeEach;
-module.exports.afterEach = afterEach;
+ObjectAssign(module.exports, {
+  after,
+  afterEach,
+  before,
+  beforeEach,
+  describe,
+  it,
+  run,
+  test,
+});
diff --git a/test/message/test_runner_no_tests.out b/test/message/test_runner_no_tests.out
index 9f84e58dc125f8..9daeafb9864cf4 100644
--- a/test/message/test_runner_no_tests.out
+++ b/test/message/test_runner_no_tests.out
@@ -1 +1 @@
-bound test
+test
diff --git a/test/parallel/test-runner-run.mjs b/test/parallel/test-runner-run.mjs
new file mode 100644
index 00000000000000..4a0d4f025bf7bf
--- /dev/null
+++ b/test/parallel/test-runner-run.mjs
@@ -0,0 +1,69 @@
+import * as common from '../common/index.mjs';
+import * as fixtures from '../common/fixtures.mjs';
+import { join } from 'node:path';
+import { describe, it, run } from 'node:test';
+import assert from 'node:assert';
+
+const testFixtures = fixtures.path('test-runner');
+
+describe('require(\'node:test\').run', { concurrency: true }, () => {
+
+  it('should run with no tests', async () => {
+    const stream = run({ files: [] });
+    stream.setEncoding('utf8');
+    stream.on('test:fail', common.mustNotCall());
+    stream.on('test:pass', common.mustNotCall());
+    // eslint-disable-next-line no-unused-vars
+    for await (const _ of stream); // TODO(MoLow): assert.snapshot
+  });
+
+  it('should fail with non existing file', async () => {
+    const stream = run({ files: ['a-random-file-that-does-not-exist.js'] });
+    stream.on('test:fail', common.mustCall(1));
+    stream.on('test:pass', common.mustNotCall());
+    // eslint-disable-next-line no-unused-vars
+    for await (const _ of stream); // TODO(MoLow): assert.snapshot
+  });
+
+  it('should succeed with a file', async () => {
+    const stream = run({ files: [join(testFixtures, 'test/random.cjs')] });
+    stream.on('test:fail', common.mustNotCall());
+    stream.on('test:pass', common.mustCall(1));
+    // eslint-disable-next-line no-unused-vars
+    for await (const _ of stream); // TODO(MoLow): assert.snapshot
+  });
+
+  it('should run same file twice', async () => {
+    const stream = run({ files: [join(testFixtures, 'test/random.cjs'), join(testFixtures, 'test/random.cjs')] });
+    stream.on('test:fail', common.mustNotCall());
+    stream.on('test:pass', common.mustCall(2));
+    // eslint-disable-next-line no-unused-vars
+    for await (const _ of stream); // TODO(MoLow): assert.snapshot
+  });
+
+  it('should run a failed test', async () => {
+    const stream = run({ files: [testFixtures] });
+    stream.on('test:fail', common.mustCall(1));
+    stream.on('test:pass', common.mustNotCall());
+    // eslint-disable-next-line no-unused-vars
+    for await (const _ of stream); // TODO(MoLow): assert.snapshot
+  });
+
+  it('should support timeout', async () => {
+    const stream = run({ timeout: 50, files: [
+      fixtures.path('test-runner', 'never_ending_sync.js'),
+      fixtures.path('test-runner', 'never_ending_async.js'),
+    ] });
+    stream.on('test:fail', common.mustCall(2));
+    stream.on('test:pass', common.mustNotCall());
+    // eslint-disable-next-line no-unused-vars
+    for await (const _ of stream); // TODO(MoLow): assert.snapshot
+  });
+
+  it('should validate files', async () => {
+    [Symbol(), {}, () => {}, 0, 1, 0n, 1n, '', '1', Promise.resolve([]), true, false]
+      .forEach((files) => assert.throws(() => run({ files }), {
+        code: 'ERR_INVALID_ARG_TYPE'
+      }));
+  });
+});
diff --git a/tools/doc/type-parser.mjs b/tools/doc/type-parser.mjs
index c77cba7de19fc0..da24512b52d137 100644
--- a/tools/doc/type-parser.mjs
+++ b/tools/doc/type-parser.mjs
@@ -201,6 +201,8 @@ const customTypesMap = {
   'Timeout': 'timers.html#class-timeout',
   'Timer': 'timers.html#timers',
 
+  'TapStream': 'test.html#class-tapstream',
+
   'tls.SecureContext': 'tls.html#tlscreatesecurecontextoptions',
   'tls.Server': 'tls.html#class-tlsserver',
   'tls.TLSSocket': 'tls.html#class-tlstlssocket',

From 9037120c473909f162acfd1812df6c029736912d Mon Sep 17 00:00:00 2001
From: Moshe Atlow <moshe@atlow.co.il>
Date: Mon, 3 Oct 2022 13:08:56 +0300
Subject: [PATCH 2/6] test_runner: fix `duration_ms` to be milliseconds

PR-URL: https://github.com/nodejs/node/pull/44450
Backport-PR-URL: https://github.com/nodejs/node/pull/44873
Reviewed-By: Benjamin Gruenbaum <benjamingr@gmail.com>
Reviewed-By: Antoine du Hamel <duhamelantoine1995@gmail.com>
Reviewed-By: Franziska Hinkelmann <franziska.hinkelmann@gmail.com>
Reviewed-By: Rich Trott <rtrott@gmail.com>
Reviewed-By: Minwoo Jung <nodecorelab@gmail.com>
---
 lib/internal/test_runner/test.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/internal/test_runner/test.js b/lib/internal/test_runner/test.js
index 27ce298d456def..5c33d760dd0b6e 100644
--- a/lib/internal/test_runner/test.js
+++ b/lib/internal/test_runner/test.js
@@ -619,8 +619,8 @@ class Test extends AsyncResource {
   }
 
   #duration() {
-    // Duration is recorded in BigInt nanoseconds. Convert to seconds.
-    return Number(this.endTime - this.startTime) / 1_000_000_000;
+    // Duration is recorded in BigInt nanoseconds. Convert to milliseconds.
+    return Number(this.endTime - this.startTime) / 1_000_000;
   }
 
   report() {

From 6e980bfb295f40c63e607cda183ebd87868c416d Mon Sep 17 00:00:00 2001
From: Moshe Atlow <moshe@atlow.co.il>
Date: Mon, 3 Oct 2022 13:11:58 +0300
Subject: [PATCH 3/6] test_runner: support using `--inspect` with `--test`

PR-URL: https://github.com/nodejs/node/pull/44520
Backport-PR-URL: https://github.com/nodejs/node/pull/44873
Reviewed-By: Benjamin Gruenbaum <benjamingr@gmail.com>
Reviewed-By: Antoine du Hamel <duhamelantoine1995@gmail.com>
---
 doc/api/test.md                               |   5 +
 lib/internal/cluster/primary.js               |   1 +
 lib/internal/main/test_runner.js              |  13 +-
 lib/internal/test_runner/runner.js            |  59 +++++--
 lib/internal/test_runner/test.js              |   5 +-
 lib/internal/util/inspector.js                |  42 +++++
 src/node_options.cc                           |   3 -
 test/common/index.mjs                         |   3 +
 test/fixtures/test-runner/run_inspect.js      |  40 +++++
 .../test-runner/run_inspect_assert.js         |  19 ++
 test/parallel/test-runner-cli.js              |   6 -
 test/parallel/test-runner-inspect.mjs         |  54 ++++++
 test/sequential/test-runner-run-inspect.mjs   | 164 ++++++++++++++++++
 13 files changed, 391 insertions(+), 23 deletions(-)
 create mode 100644 test/fixtures/test-runner/run_inspect.js
 create mode 100644 test/fixtures/test-runner/run_inspect_assert.js
 create mode 100644 test/parallel/test-runner-inspect.mjs
 create mode 100644 test/sequential/test-runner-run-inspect.mjs

diff --git a/doc/api/test.md b/doc/api/test.md
index fcffd932df53c1..7da51afff4b7f9 100644
--- a/doc/api/test.md
+++ b/doc/api/test.md
@@ -338,6 +338,11 @@ added: REPLACEME
     fail after.
     If unspecified, subtests inherit this value from their parent.
     **Default:** `Infinity`.
+  * `inspectPort` {number|Function} Sets inspector port of test child process.
+    This can be a number, or a function that takes no arguments and returns a
+    number. If a nullish value is provided, each process gets its own port,
+    incremented from the primary's `process.debugPort`.
+    **Default:** `undefined`.
 * Returns: {TapStream}
 
 ```js
diff --git a/lib/internal/cluster/primary.js b/lib/internal/cluster/primary.js
index 3940d094e7b76d..749f537734947d 100644
--- a/lib/internal/cluster/primary.js
+++ b/lib/internal/cluster/primary.js
@@ -120,6 +120,7 @@ function createWorkerProcess(id, env) {
   const debugArgRegex = /--inspect(?:-brk|-port)?|--debug-port/;
   const nodeOptions = process.env.NODE_OPTIONS || '';
 
+  // TODO(MoLow): Use getInspectPort from internal/util/inspector
   if (ArrayPrototypeSome(execArgv,
                          (arg) => RegExpPrototypeExec(debugArgRegex, arg) !== null) ||
       RegExpPrototypeExec(debugArgRegex, nodeOptions) !== null) {
diff --git a/lib/internal/main/test_runner.js b/lib/internal/main/test_runner.js
index c965a6a95fbab5..bf68b6933bed76 100644
--- a/lib/internal/main/test_runner.js
+++ b/lib/internal/main/test_runner.js
@@ -2,12 +2,23 @@
 const {
   prepareMainThreadExecution,
 } = require('internal/bootstrap/pre_execution');
+const { isUsingInspector } = require('internal/util/inspector');
 const { run } = require('internal/test_runner/runner');
 
 prepareMainThreadExecution(false);
 markBootstrapComplete();
 
-const tapStream = run();
+let concurrency = true;
+let inspectPort;
+
+if (isUsingInspector()) {
+  process.emitWarning('Using the inspector with --test forces running at a concurrency of 1. ' +
+  'Use the inspectPort option to run with concurrency');
+  concurrency = 1;
+  inspectPort = process.debugPort;
+}
+
+const tapStream = run({ concurrency, inspectPort });
 tapStream.pipe(process.stdout);
 tapStream.once('test:fail', () => {
   process.exitCode = 1;
diff --git a/lib/internal/test_runner/runner.js b/lib/internal/test_runner/runner.js
index a4afbcb0a7710f..911a700d68d58d 100644
--- a/lib/internal/test_runner/runner.js
+++ b/lib/internal/test_runner/runner.js
@@ -1,18 +1,22 @@
 'use strict';
 const {
   ArrayFrom,
-  ArrayPrototypeConcat,
   ArrayPrototypeFilter,
   ArrayPrototypeIncludes,
   ArrayPrototypeJoin,
+  ArrayPrototypePop,
+  ArrayPrototypePush,
   ArrayPrototypeSlice,
   ArrayPrototypeSort,
   ObjectAssign,
   PromisePrototypeThen,
+  RegExpPrototypeSymbolSplit,
   SafePromiseAll,
   SafeSet,
+  StringPrototypeEndsWith,
 } = primordials;
 
+const { Buffer } = require('buffer');
 const { spawn } = require('child_process');
 const { readdirSync, statSync } = require('fs');
 const console = require('internal/console/global');
@@ -22,6 +26,7 @@ const {
   },
 } = require('internal/errors');
 const { validateArray } = require('internal/validators');
+const { getInspectPort, isUsingInspector, isInspectorMessage } = require('internal/util/inspector');
 const { kEmptyObject } = require('internal/util');
 const { createTestTree } = require('internal/test_runner/harness');
 const { kSubtestsFailed, Test } = require('internal/test_runner/test');
@@ -100,25 +105,59 @@ function filterExecArgv(arg) {
   return !ArrayPrototypeIncludes(kFilterArgs, arg);
 }
 
-function runTestFile(path, root) {
+function getRunArgs({ path, inspectPort }) {
+  const argv = ArrayPrototypeFilter(process.execArgv, filterExecArgv);
+  if (isUsingInspector()) {
+    ArrayPrototypePush(argv, `--inspect-port=${getInspectPort(inspectPort)}`);
+  }
+  ArrayPrototypePush(argv, path);
+  return argv;
+}
+
+function makeStderrCallback(callback) {
+  if (!isUsingInspector()) {
+    return callback;
+  }
+  let buffer = Buffer.alloc(0);
+  return (data) => {
+    callback(data);
+    const newData = Buffer.concat([buffer, data]);
+    const str = newData.toString('utf8');
+    let lines = str;
+    if (StringPrototypeEndsWith(lines, '\n')) {
+      buffer = Buffer.alloc(0);
+    } else {
+      lines = RegExpPrototypeSymbolSplit(/\r?\n/, str);
+      buffer = Buffer.from(ArrayPrototypePop(lines), 'utf8');
+      lines = ArrayPrototypeJoin(lines, '\n');
+    }
+    if (isInspectorMessage(lines)) {
+      process.stderr.write(lines);
+    }
+  };
+}
+
+function runTestFile(path, root, inspectPort) {
   const subtest = root.createSubtest(Test, path, async (t) => {
-    const args = ArrayPrototypeConcat(
-      ArrayPrototypeFilter(process.execArgv, filterExecArgv),
-      path);
+    const args = getRunArgs({ path, inspectPort });
 
     const child = spawn(process.execPath, args, { signal: t.signal, encoding: 'utf8' });
     // TODO(cjihrig): Implement a TAP parser to read the child's stdout
     // instead of just displaying it all if the child fails.
     let err;
+    let stderr = '';
 
     child.on('error', (error) => {
       err = error;
     });
 
-    const { 0: { 0: code, 1: signal }, 1: stdout, 2: stderr } = await SafePromiseAll([
+    child.stderr.on('data', makeStderrCallback((data) => {
+      stderr += data;
+    }));
+
+    const { 0: { 0: code, 1: signal }, 1: stdout } = await SafePromiseAll([
       once(child, 'exit', { signal: t.signal }),
       child.stdout.toArray({ signal: t.signal }),
-      child.stderr.toArray({ signal: t.signal }),
     ]);
 
     if (code !== 0 || signal !== null) {
@@ -128,7 +167,7 @@ function runTestFile(path, root) {
           exitCode: code,
           signal: signal,
           stdout: ArrayPrototypeJoin(stdout, ''),
-          stderr: ArrayPrototypeJoin(stderr, ''),
+          stderr,
           // The stack will not be useful since the failures came from tests
           // in a child process.
           stack: undefined,
@@ -145,7 +184,7 @@ function run(options) {
   if (options === null || typeof options !== 'object') {
     options = kEmptyObject;
   }
-  const { concurrency, timeout, signal, files } = options;
+  const { concurrency, timeout, signal, files, inspectPort } = options;
 
   if (files != null) {
     validateArray(files, 'options.files');
@@ -154,7 +193,7 @@ function run(options) {
   const root = createTestTree({ concurrency, timeout, signal });
   const testFiles = files ?? createTestFileList();
 
-  PromisePrototypeThen(SafePromiseAll(testFiles, (path) => runTestFile(path, root)),
+  PromisePrototypeThen(SafePromiseAll(testFiles, (path) => runTestFile(path, root, inspectPort)),
                        () => root.postRun());
 
   return root.reporter;
diff --git a/lib/internal/test_runner/test.js b/lib/internal/test_runner/test.js
index 5c33d760dd0b6e..f02c173bff35eb 100644
--- a/lib/internal/test_runner/test.js
+++ b/lib/internal/test_runner/test.js
@@ -58,8 +58,6 @@ const kDefaultTimeout = null;
 const noop = FunctionPrototype;
 const isTestRunner = getOptionValue('--test');
 const testOnlyFlag = !isTestRunner && getOptionValue('--test-only');
-// TODO(cjihrig): Use uv_available_parallelism() once it lands.
-const rootConcurrency = isTestRunner ? MathMax(cpus().length - 1, 1) : 1;
 const kShouldAbort = Symbol('kShouldAbort');
 const kRunHook = Symbol('kRunHook');
 const kHookNames = ObjectSeal(['before', 'after', 'beforeEach', 'afterEach']);
@@ -150,7 +148,7 @@ class Test extends AsyncResource {
     }
 
     if (parent === null) {
-      this.concurrency = rootConcurrency;
+      this.concurrency = 1;
       this.indent = '';
       this.indentString = kDefaultIndent;
       this.only = testOnlyFlag;
@@ -180,6 +178,7 @@ class Test extends AsyncResource {
 
       case 'boolean':
         if (concurrency) {
+          // TODO(cjihrig): Use uv_available_parallelism() once it lands.
           this.concurrency = parent === null ? MathMax(cpus().length - 1, 1) : Infinity;
         } else {
           this.concurrency = 1;
diff --git a/lib/internal/util/inspector.js b/lib/internal/util/inspector.js
index 95ef370c3c495f..f1bb21d859d437 100644
--- a/lib/internal/util/inspector.js
+++ b/lib/internal/util/inspector.js
@@ -2,12 +2,47 @@
 
 const {
   ArrayPrototypeConcat,
+  ArrayPrototypeSome,
   FunctionPrototypeBind,
   ObjectDefineProperty,
   ObjectKeys,
   ObjectPrototypeHasOwnProperty,
+  RegExpPrototypeExec,
 } = primordials;
 
+const { validatePort } = require('internal/validators');
+
+const kMinPort = 1024;
+const kMaxPort = 65535;
+const kInspectArgRegex = /--inspect(?:-brk|-port)?|--debug-port/;
+const kInspectMsgRegex = /Debugger listening on ws:\/\/\[?(.+?)\]?:(\d+)\/|Debugger attached|Waiting for the debugger to disconnect\.\.\./;
+
+let _isUsingInspector;
+function isUsingInspector() {
+  _isUsingInspector ??=
+    ArrayPrototypeSome(process.execArgv, (arg) => RegExpPrototypeExec(kInspectArgRegex, arg) !== null) ||
+    RegExpPrototypeExec(kInspectArgRegex, process.env.NODE_OPTIONS) !== null;
+  return _isUsingInspector;
+}
+
+let debugPortOffset = 1;
+function getInspectPort(inspectPort) {
+  if (!isUsingInspector()) {
+    return null;
+  }
+  if (typeof inspectPort === 'function') {
+    inspectPort = inspectPort();
+  } else if (inspectPort == null) {
+    inspectPort = process.debugPort + debugPortOffset;
+    if (inspectPort > kMaxPort)
+      inspectPort = inspectPort - kMaxPort + kMinPort - 1;
+    debugPortOffset++;
+  }
+  validatePort(inspectPort);
+
+  return inspectPort;
+}
+
 let session;
 function sendInspectorCommand(cb, onError) {
   const { hasInspector } = internalBinding('config');
@@ -22,6 +57,10 @@ function sendInspectorCommand(cb, onError) {
   }
 }
 
+function isInspectorMessage(string) {
+  return isUsingInspector() && RegExpPrototypeExec(kInspectMsgRegex, string) !== null;
+}
+
 // Create a special require function for the inspector command line API
 function installConsoleExtensions(commandLineApi) {
   if (commandLineApi.require) { return; }
@@ -65,7 +104,10 @@ function wrapConsole(consoleFromNode, consoleFromVM) {
 // Stores the console from VM, should be set during bootstrap.
 let consoleFromVM;
 module.exports = {
+  getInspectPort,
   installConsoleExtensions,
+  isInspectorMessage,
+  isUsingInspector,
   sendInspectorCommand,
   wrapConsole,
   get consoleFromVM() {
diff --git a/src/node_options.cc b/src/node_options.cc
index ba6a666e88a082..913c0a84bdc7a5 100644
--- a/src/node_options.cc
+++ b/src/node_options.cc
@@ -161,9 +161,6 @@ void EnvironmentOptions::CheckOptions(std::vector<std::string>* errors) {
       errors->push_back("either --test or --watch can be used, not both");
     }
 
-    if (debug_options_.inspector_enabled) {
-      errors->push_back("the inspector cannot be used with --test");
-    }
 #ifndef ALLOW_ATTACHING_DEBUGGER_IN_TEST_RUNNER
     debug_options_.allow_attaching_debugger = false;
 #endif
diff --git a/test/common/index.mjs b/test/common/index.mjs
index 2b30f499343cc4..e77b1b298cbc30 100644
--- a/test/common/index.mjs
+++ b/test/common/index.mjs
@@ -52,6 +52,8 @@ const {
   spawnPromisified,
 } = common;
 
+const getPort = () => common.PORT;
+
 export {
   isMainThread,
   isWindows,
@@ -100,4 +102,5 @@ export {
   runWithInvalidFD,
   createRequire,
   spawnPromisified,
+  getPort,
 };
diff --git a/test/fixtures/test-runner/run_inspect.js b/test/fixtures/test-runner/run_inspect.js
new file mode 100644
index 00000000000000..1586b6aaccf082
--- /dev/null
+++ b/test/fixtures/test-runner/run_inspect.js
@@ -0,0 +1,40 @@
+'use strict';
+
+const common = require('../../common');
+const fixtures = require('../../common/fixtures');
+const { run } = require('node:test');
+const assert = require('node:assert');
+
+const badPortError = { name: 'RangeError', code: 'ERR_SOCKET_BAD_PORT' };
+let inspectPort = 'inspectPort' in process.env ? Number(process.env.inspectPort) : undefined;
+let expectedError;
+
+if (process.env.inspectPort === 'addTwo') {
+  inspectPort = common.mustCall(() => { return process.debugPort += 2; });
+} else if (process.env.inspectPort === 'string') {
+  inspectPort = 'string';
+  expectedError = badPortError;
+} else if (process.env.inspectPort === 'null') {
+  inspectPort = null;
+} else if (process.env.inspectPort === 'bignumber') {
+  inspectPort = 1293812;
+  expectedError = badPortError;
+} else if (process.env.inspectPort === 'negativenumber') {
+  inspectPort = -9776;
+  expectedError = badPortError;
+} else if (process.env.inspectPort === 'bignumberfunc') {
+  inspectPort = common.mustCall(() => 123121);
+  expectedError = badPortError;
+} else if (process.env.inspectPort === 'strfunc') {
+  inspectPort = common.mustCall(() => 'invalidPort');
+  expectedError = badPortError;
+}
+
+const stream = run({ files: [fixtures.path('test-runner/run_inspect_assert.js')], inspectPort });
+if (expectedError) {
+  stream.on('test:fail', common.mustCall(({ error }) => {
+    assert.deepStrictEqual({ name: error.cause.name, code: error.cause.code }, expectedError);
+  }));
+} else {
+  stream.on('test:fail', common.mustNotCall());
+}
diff --git a/test/fixtures/test-runner/run_inspect_assert.js b/test/fixtures/test-runner/run_inspect_assert.js
new file mode 100644
index 00000000000000..daea9b3b6656c1
--- /dev/null
+++ b/test/fixtures/test-runner/run_inspect_assert.js
@@ -0,0 +1,19 @@
+'use strict';
+
+const assert = require('node:assert');
+
+const { expectedPort, expectedInitialPort, expectedHost } = process.env;
+const debugOptions =
+  require('internal/options').getOptionValue('--inspect-port');
+
+if ('expectedPort' in process.env) {
+  assert.strictEqual(process.debugPort, +expectedPort);
+}
+
+if ('expectedInitialPort' in process.env) {
+  assert.strictEqual(debugOptions.port, +expectedInitialPort);
+}
+
+if ('expectedHost' in process.env) {
+  assert.strictEqual(debugOptions.host, expectedHost);
+}
diff --git a/test/parallel/test-runner-cli.js b/test/parallel/test-runner-cli.js
index 552d64d7c40ba2..8c1f6b3b0bee69 100644
--- a/test/parallel/test-runner-cli.js
+++ b/test/parallel/test-runner-cli.js
@@ -104,12 +104,6 @@ const testFixtures = fixtures.path('test-runner');
     ['--print', 'console.log("should not print")', '--test'],
   ];
 
-  if (process.features.inspector) {
-    flags.push(
-      ['--inspect', '--test'],
-      ['--inspect-brk', '--test'],
-    );
-  }
 
   flags.forEach((args) => {
     const child = spawnSync(process.execPath, args);
diff --git a/test/parallel/test-runner-inspect.mjs b/test/parallel/test-runner-inspect.mjs
new file mode 100644
index 00000000000000..67095291e2acd3
--- /dev/null
+++ b/test/parallel/test-runner-inspect.mjs
@@ -0,0 +1,54 @@
+import * as common from '../common/index.mjs';
+import * as tmpdir from '../common/tmpdir.js';
+import * as fixtures from '../common/fixtures.mjs';
+import assert from 'node:assert';
+import { NodeInstance } from '../common/inspector-helper.js';
+
+
+common.skipIfInspectorDisabled();
+tmpdir.refresh();
+
+{
+  const child = new NodeInstance(['--test', '--inspect-brk=0'], undefined, fixtures.path('test-runner/index.test.js'));
+
+  let stdout = '';
+  let stderr = '';
+  child.on('stdout', (line) => stdout += line);
+  child.on('stderr', (line) => stderr += line);
+
+  const session = await child.connectInspectorSession();
+
+  await session.send([
+    { method: 'Runtime.enable' },
+    { method: 'Runtime.runIfWaitingForDebugger' }]);
+
+  session.disconnect();
+  assert.match(stderr,
+               /Warning: Using the inspector with --test forces running at a concurrency of 1\. Use the inspectPort option to run with concurrency/);
+}
+
+
+{
+  const args = ['--test', '--inspect=0', fixtures.path('test-runner/index.js')];
+  const { stderr, stdout, code, signal } = await common.spawnPromisified(process.execPath, args);
+
+  assert.match(stderr,
+               /Warning: Using the inspector with --test forces running at a concurrency of 1\. Use the inspectPort option to run with concurrency/);
+  assert.match(stdout, /not ok 1 - .+index\.js/);
+  assert.match(stdout, /stderr: \|-\r?\n\s+Debugger listening on/);
+  assert.strictEqual(code, 1);
+  assert.strictEqual(signal, null);
+}
+
+
+{
+  // File not found.
+  const args = ['--test', '--inspect=0', 'a-random-file-that-does-not-exist.js'];
+  const { stderr, stdout, code, signal } = await common.spawnPromisified(process.execPath, args);
+
+  assert.strictEqual(stdout, '');
+  assert.match(stderr, /^Could not find/);
+  assert.doesNotMatch(stderr, /Warning: Using the inspector with --test forces running at a concurrency of 1\. Use the inspectPort option to run with concurrency/);
+  assert.strictEqual(code, 1);
+  assert.strictEqual(signal, null);
+}
diff --git a/test/sequential/test-runner-run-inspect.mjs b/test/sequential/test-runner-run-inspect.mjs
new file mode 100644
index 00000000000000..35c380c8d4e33e
--- /dev/null
+++ b/test/sequential/test-runner-run-inspect.mjs
@@ -0,0 +1,164 @@
+import * as common from '../common/index.mjs';
+import * as fixtures from '../common/fixtures.mjs';
+import assert from 'node:assert';
+
+common.skipIfInspectorDisabled();
+
+
+const debuggerPort = common.getPort();
+
+async function spawnRunner({ execArgv, expectedPort, expectedHost, expectedInitialPort, inspectPort }) {
+  const { code, signal } = await common.spawnPromisified(
+    process.execPath,
+    ['--expose-internals', '--no-warnings', ...execArgv, fixtures.path('test-runner/run_inspect.js')], {
+      env: { ...process.env,
+             expectedPort,
+             inspectPort,
+             expectedHost,
+             expectedInitialPort }
+    });
+  assert.strictEqual(code, 0);
+  assert.strictEqual(signal, null);
+}
+
+let offset = 0;
+
+const defaultPortCase = spawnRunner({
+  execArgv: ['--inspect'],
+  expectedPort: 9230,
+});
+
+await spawnRunner({
+  execArgv: ['--inspect=65535'],
+  expectedPort: 1024,
+});
+
+let port = debuggerPort + offset++ * 5;
+
+await spawnRunner({
+  execArgv: [`--inspect=${port}`],
+  expectedPort: port + 1,
+});
+
+port = debuggerPort + offset++ * 5;
+
+await spawnRunner({
+  execArgv: ['--inspect', `--inspect-port=${port}`],
+  expectedPort: port + 1,
+});
+
+port = debuggerPort + offset++ * 5;
+
+await spawnRunner({
+  execArgv: ['--inspect', `--debug-port=${port}`],
+  expectedPort: port + 1,
+});
+
+port = debuggerPort + offset++ * 5;
+
+await spawnRunner({
+  execArgv: [`--inspect=0.0.0.0:${port}`],
+  expectedPort: port + 1, expectedHost: '0.0.0.0',
+});
+
+port = debuggerPort + offset++ * 5;
+
+await spawnRunner({
+  execArgv: [`--inspect=127.0.0.1:${port}`],
+  expectedPort: port + 1, expectedHost: '127.0.0.1'
+});
+
+if (common.hasIPv6) {
+  port = debuggerPort + offset++ * 5;
+
+  await spawnRunner({
+    execArgv: [`--inspect=[::]:${port}`],
+    expectedPort: port + 1, expectedHost: '::'
+  });
+
+  port = debuggerPort + offset++ * 5;
+
+  await spawnRunner({
+    execArgv: [`--inspect=[::1]:${port}`],
+    expectedPort: port + 1, expectedHost: '::1'
+  });
+}
+
+// These tests check that setting inspectPort in run
+// would take effect and override port incrementing behavior
+
+port = debuggerPort + offset++ * 5;
+
+await spawnRunner({
+  execArgv: [`--inspect=${port}`],
+  inspectPort: port + 2,
+  expectedPort: port + 2,
+});
+
+port = debuggerPort + offset++ * 5;
+
+await spawnRunner({
+  execArgv: [`--inspect=${port}`],
+  inspectPort: 'addTwo',
+  expectedPort: port + 2,
+});
+
+port = debuggerPort + offset++ * 5;
+
+await spawnRunner({
+  execArgv: [`--inspect=${port}`],
+  inspectPort: 'string',
+});
+
+port = debuggerPort + offset++ * 5;
+
+await spawnRunner({
+  execArgv: [`--inspect=${port}`],
+  inspectPort: 'null',
+  expectedPort: port + 1,
+});
+
+port = debuggerPort + offset++ * 5;
+
+await spawnRunner({
+  execArgv: [`--inspect=${port}`],
+  inspectPort: 'bignumber',
+});
+
+port = debuggerPort + offset++ * 5;
+
+await spawnRunner({
+  execArgv: [`--inspect=${port}`],
+  inspectPort: 'negativenumber',
+});
+
+port = debuggerPort + offset++ * 5;
+
+await spawnRunner({
+  execArgv: [`--inspect=${port}`],
+  inspectPort: 'bignumberfunc'
+});
+
+port = debuggerPort + offset++ * 5;
+
+await spawnRunner({
+  execArgv: [`--inspect=${port}`],
+  inspectPort: 'strfunc',
+});
+
+port = debuggerPort + offset++ * 5;
+
+await spawnRunner({
+  execArgv: [`--inspect=${port}`],
+  inspectPort: 0,
+  expectedInitialPort: 0,
+});
+
+await defaultPortCase;
+
+port = debuggerPort + offset++ * 5;
+await spawnRunner({
+  execArgv: ['--inspect'],
+  inspectPort: port + 2,
+  expectedInitialPort: port + 2,
+});

From 3c4c0cda4078b4706fb18a6255254eea814f67c6 Mon Sep 17 00:00:00 2001
From: Moshe Atlow <moshe@atlow.co.il>
Date: Mon, 3 Oct 2022 13:12:53 +0300
Subject: [PATCH 4/6] test: fix test-runner-inspect

PR-URL: https://github.com/nodejs/node/pull/44620
Backport-PR-URL: https://github.com/nodejs/node/pull/44873
Fixes: https://github.com/nodejs/node/issues/44600
Reviewed-By: Benjamin Gruenbaum <benjamingr@gmail.com>
Reviewed-By: Antoine du Hamel <duhamelantoine1995@gmail.com>
---
 lib/internal/test_runner/runner.js | 41 ++++++++++--------------------
 1 file changed, 13 insertions(+), 28 deletions(-)

diff --git a/lib/internal/test_runner/runner.js b/lib/internal/test_runner/runner.js
index 911a700d68d58d..9994ac12ecf3a4 100644
--- a/lib/internal/test_runner/runner.js
+++ b/lib/internal/test_runner/runner.js
@@ -4,21 +4,19 @@ const {
   ArrayPrototypeFilter,
   ArrayPrototypeIncludes,
   ArrayPrototypeJoin,
-  ArrayPrototypePop,
   ArrayPrototypePush,
   ArrayPrototypeSlice,
   ArrayPrototypeSort,
   ObjectAssign,
   PromisePrototypeThen,
-  RegExpPrototypeSymbolSplit,
   SafePromiseAll,
   SafeSet,
-  StringPrototypeEndsWith,
 } = primordials;
 
-const { Buffer } = require('buffer');
 const { spawn } = require('child_process');
 const { readdirSync, statSync } = require('fs');
+// TODO(aduh95): switch to internal/readline/interface when backporting to Node.js 16.x is no longer a concern.
+const { createInterface } = require('readline');
 const console = require('internal/console/global');
 const {
   codes: {
@@ -114,28 +112,6 @@ function getRunArgs({ path, inspectPort }) {
   return argv;
 }
 
-function makeStderrCallback(callback) {
-  if (!isUsingInspector()) {
-    return callback;
-  }
-  let buffer = Buffer.alloc(0);
-  return (data) => {
-    callback(data);
-    const newData = Buffer.concat([buffer, data]);
-    const str = newData.toString('utf8');
-    let lines = str;
-    if (StringPrototypeEndsWith(lines, '\n')) {
-      buffer = Buffer.alloc(0);
-    } else {
-      lines = RegExpPrototypeSymbolSplit(/\r?\n/, str);
-      buffer = Buffer.from(ArrayPrototypePop(lines), 'utf8');
-      lines = ArrayPrototypeJoin(lines, '\n');
-    }
-    if (isInspectorMessage(lines)) {
-      process.stderr.write(lines);
-    }
-  };
-}
 
 function runTestFile(path, root, inspectPort) {
   const subtest = root.createSubtest(Test, path, async (t) => {
@@ -151,9 +127,18 @@ function runTestFile(path, root, inspectPort) {
       err = error;
     });
 
-    child.stderr.on('data', makeStderrCallback((data) => {
+    child.stderr.on('data', (data) => {
       stderr += data;
-    }));
+    });
+
+    if (isUsingInspector()) {
+      const rl = createInterface({ input: child.stderr });
+      rl.on('line', (line) => {
+        if (isInspectorMessage(line)) {
+          process.stderr.write(line + '\n');
+        }
+      });
+    }
 
     const { 0: { 0: code, 1: signal }, 1: stdout } = await SafePromiseAll([
       once(child, 'exit', { signal: t.signal }),

From 36be71e177efde67baba9d740dc9e175b8daf283 Mon Sep 17 00:00:00 2001
From: Moshe Atlow <moshe@atlow.co.il>
Date: Tue, 13 Sep 2022 09:41:19 +0300
Subject: [PATCH 5/6] cluster: use inspector utils

PR-URL: https://github.com/nodejs/node/pull/44592
Backport-PR-URL: https://github.com/nodejs/node/pull/44873
Reviewed-By: Matteo Collina <matteo.collina@gmail.com>
Reviewed-By: Benjamin Gruenbaum <benjamingr@gmail.com>
---
 lib/internal/cluster/primary.js | 38 ++++++++++-----------------------
 lib/internal/util/inspector.js  | 18 ++++++++--------
 2 files changed, 20 insertions(+), 36 deletions(-)

diff --git a/lib/internal/cluster/primary.js b/lib/internal/cluster/primary.js
index 749f537734947d..fc366680fb7bf7 100644
--- a/lib/internal/cluster/primary.js
+++ b/lib/internal/cluster/primary.js
@@ -6,10 +6,14 @@ const {
   ArrayPrototypeSome,
   ObjectKeys,
   ObjectValues,
-  RegExpPrototypeExec,
   SafeMap,
   StringPrototypeStartsWith,
 } = primordials;
+const {
+  codes: {
+    ERR_SOCKET_BAD_PORT,
+  }
+} = require('internal/errors');
 
 const assert = require('internal/assert');
 const { fork } = require('child_process');
@@ -18,14 +22,12 @@ const EventEmitter = require('events');
 const RoundRobinHandle = require('internal/cluster/round_robin_handle');
 const SharedHandle = require('internal/cluster/shared_handle');
 const Worker = require('internal/cluster/worker');
+const { getInspectPort, isUsingInspector } = require('internal/util/inspector');
 const { internal, sendHelper } = require('internal/cluster/utils');
 const cluster = new EventEmitter();
 const intercom = new EventEmitter();
 const SCHED_NONE = 1;
 const SCHED_RR = 2;
-const minPort = 1024;
-const maxPort = 65535;
-const { validatePort } = require('internal/validators');
 
 module.exports = cluster;
 
@@ -40,7 +42,6 @@ cluster.SCHED_NONE = SCHED_NONE;  // Leave it to the operating system.
 cluster.SCHED_RR = SCHED_RR;      // Primary distributes connections.
 
 let ids = 0;
-let debugPortOffset = 1;
 let initialized = false;
 
 // XXX(bnoordhuis) Fold cluster.schedulingPolicy into cluster.settings?
@@ -117,29 +118,12 @@ function setupSettingsNT(settings) {
 function createWorkerProcess(id, env) {
   const workerEnv = { ...process.env, ...env, NODE_UNIQUE_ID: `${id}` };
   const execArgv = [...cluster.settings.execArgv];
-  const debugArgRegex = /--inspect(?:-brk|-port)?|--debug-port/;
-  const nodeOptions = process.env.NODE_OPTIONS || '';
-
-  // TODO(MoLow): Use getInspectPort from internal/util/inspector
-  if (ArrayPrototypeSome(execArgv,
-                         (arg) => RegExpPrototypeExec(debugArgRegex, arg) !== null) ||
-      RegExpPrototypeExec(debugArgRegex, nodeOptions) !== null) {
-    let inspectPort;
-    if ('inspectPort' in cluster.settings) {
-      if (typeof cluster.settings.inspectPort === 'function')
-        inspectPort = cluster.settings.inspectPort();
-      else
-        inspectPort = cluster.settings.inspectPort;
-
-      validatePort(inspectPort);
-    } else {
-      inspectPort = process.debugPort + debugPortOffset;
-      if (inspectPort > maxPort)
-        inspectPort = inspectPort - maxPort + minPort - 1;
-      debugPortOffset++;
-    }
 
-    ArrayPrototypePush(execArgv, `--inspect-port=${inspectPort}`);
+  if (cluster.settings.inspectPort === null) {
+    throw new ERR_SOCKET_BAD_PORT('Port', null, true);
+  }
+  if (isUsingInspector(cluster.settings.execArgv)) {
+    ArrayPrototypePush(execArgv, `--inspect-port=${getInspectPort(cluster.settings.inspectPort)}`);
   }
 
   return fork(cluster.settings.exec, cluster.settings.args, {
diff --git a/lib/internal/util/inspector.js b/lib/internal/util/inspector.js
index f1bb21d859d437..0514aece96d95e 100644
--- a/lib/internal/util/inspector.js
+++ b/lib/internal/util/inspector.js
@@ -8,6 +8,7 @@ const {
   ObjectKeys,
   ObjectPrototypeHasOwnProperty,
   RegExpPrototypeExec,
+  SafeWeakMap,
 } = primordials;
 
 const { validatePort } = require('internal/validators');
@@ -17,19 +18,18 @@ const kMaxPort = 65535;
 const kInspectArgRegex = /--inspect(?:-brk|-port)?|--debug-port/;
 const kInspectMsgRegex = /Debugger listening on ws:\/\/\[?(.+?)\]?:(\d+)\/|Debugger attached|Waiting for the debugger to disconnect\.\.\./;
 
-let _isUsingInspector;
-function isUsingInspector() {
-  _isUsingInspector ??=
-    ArrayPrototypeSome(process.execArgv, (arg) => RegExpPrototypeExec(kInspectArgRegex, arg) !== null) ||
-    RegExpPrototypeExec(kInspectArgRegex, process.env.NODE_OPTIONS) !== null;
-  return _isUsingInspector;
+const _isUsingInspector = new SafeWeakMap();
+function isUsingInspector(execArgv = process.execArgv) {
+  if (!_isUsingInspector.has(execArgv)) {
+    _isUsingInspector.set(execArgv,
+                          ArrayPrototypeSome(execArgv, (arg) => RegExpPrototypeExec(kInspectArgRegex, arg) !== null) ||
+      RegExpPrototypeExec(kInspectArgRegex, process.env.NODE_OPTIONS) !== null);
+  }
+  return _isUsingInspector.get(execArgv);
 }
 
 let debugPortOffset = 1;
 function getInspectPort(inspectPort) {
-  if (!isUsingInspector()) {
-    return null;
-  }
   if (typeof inspectPort === 'function') {
     inspectPort = inspectPort();
   } else if (inspectPort == null) {

From 880b27366a90ce00b9dd4ed31af65afd0700b616 Mon Sep 17 00:00:00 2001
From: Pierrick Bouvier <101587250+pbo-linaro@users.noreply.github.com>
Date: Thu, 20 Oct 2022 11:06:09 +0200
Subject: [PATCH 6/6] test: mark test-watch-mode* as flaky on all platforms
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This was confirmed flaky on those platforms:
- Windows x64
- Windows on Arm
- Linux
- Freebsd

Tests randomly fail because of bad order in messages expected, which
seems related to threads scheduling at execution.

PR-URL: https://github.com/nodejs/node/pull/45049
Reviewed-By: Tobias Nießen <tniessen@tnie.de>
Reviewed-By: Moshe Atlow <moshe@atlow.co.il>
Reviewed-By: Richard Lau <rlau@redhat.com>
Reviewed-By: Darshan Sen <raisinten@gmail.com>
Reviewed-By: James M Snell <jasnell@gmail.com>
---
 test/sequential/sequential.status | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/test/sequential/sequential.status b/test/sequential/sequential.status
index 56f9667d7a3058..5d88747b7066ce 100644
--- a/test/sequential/sequential.status
+++ b/test/sequential/sequential.status
@@ -7,6 +7,9 @@ prefix sequential
 [true] # This section applies to all platforms
 # https://github.com/nodejs/node/issues/27611#issuecomment-613100468
 test-cpu-prof-dir-worker: PASS, FLAKY
+# https://github.com/nodejs/node/issues/44898
+test-watch-mode: PASS, FLAKY
+test-watch-mode-inspect: PASS, FLAKY
 
 [$system==win32]