test_runner: add test:complete event to reflect execution order

PR-URL: #51909 Fixes: #51907 Reviewed-By: Chemi Atlow <[email protected]> Reviewed-By: Colin Ihrig <[email protected]> Reviewed-By: Benjamin Gruenbaum <[email protected]>
nodejs · Mar 1, 2024 · aac5ad9 · aac5ad9
1 parent 30c9181
commit aac5ad9
Show file tree

Hide file tree

Showing 4 changed files with 84 additions and 13 deletions.
diff --git a/doc/api/test.md b/doc/api/test.md
@@ -2438,6 +2438,9 @@ A successful call to [`run()`][] method will return a new {TestsStream}
 object, streaming a series of events representing the execution of the tests.
 `TestsStream` will emit events, in the order of the tests definition
 
+Some of the events are guaranteed to be emitted in the same order as the tests
+are defined, while others are emitted in the order that the tests execute.
+
 ### Event: `'test:coverage'`
 
 * `data` {Object}
@@ -2484,6 +2487,34 @@ object, streaming a series of events representing the execution of the tests.
 
 Emitted when code coverage is enabled and all tests have completed.
 
+### Event: `'test:complete'`
+
+* `data` {Object}
+  * `column` {number|undefined} The column number where the test is defined, or
+    `undefined` if the test was run through the REPL.
+  * `details` {Object} Additional execution metadata.
+    * `passed` {boolean} Whether the test passed or not.
+    * `duration_ms` {number} The duration of the test in milliseconds.
+    * `error` {Error|undefined} An error wrapping the error thrown by the test
+      if it did not pass.
+      * `cause` {Error} The actual error thrown by the test.
+    * `type` {string|undefined} The type of the test, used to denote whether
+      this is a suite.
+  * `file` {string|undefined} The path of the test file,
+    `undefined` if test was run through the REPL.
+  * `line` {number|undefined} The line number where the test is defined, or
+    `undefined` if the test was run through the REPL.
+  * `name` {string} The test name.
+  * `nesting` {number} The nesting level of the test.
+  * `testNumber` {number} The ordinal number of the test.
+  * `todo` {string|boolean|undefined} Present if [`context.todo`][] is called
+  * `skip` {string|boolean|undefined} Present if [`context.skip`][] is called
+
+Emitted when a test completes its execution.
+This event is not emitted in the same order as the tests are
+defined.
+The corresponding declaration ordered events are `'test:pass'` and `'test:fail'`.
+
 ### Event: `'test:dequeue'`
 
 * `data` {Object}
@@ -2497,6 +2528,8 @@ Emitted when code coverage is enabled and all tests have completed.
   * `nesting` {number} The nesting level of the test.
 
 Emitted when a test is dequeued, right before it is executed.
+This event is not guaranteed to be emitted in the same order as the tests are
+defined. The corresponding declaration ordered event is `'test:start'`.
 
 ### Event: `'test:diagnostic'`
 
@@ -2511,6 +2544,8 @@ Emitted when a test is dequeued, right before it is executed.
   * `nesting` {number} The nesting level of the test.
 
 Emitted when [`context.diagnostic`][] is called.
+This event is guaranteed to be emitted in the same order as the tests are
+defined.
 
 ### Event: `'test:enqueue'`
 
@@ -2548,6 +2583,9 @@ Emitted when a test is enqueued for execution.
   * `skip` {string|boolean|undefined} Present if [`context.skip`][] is called
 
 Emitted when a test fails.
+This event is guaranteed to be emitted in the same order as the tests are
+defined.
+The corresponding execution ordered event is `'test:complete'`.
 
 ### Event: `'test:pass'`
 
@@ -2569,6 +2607,9 @@ Emitted when a test fails.
   * `skip` {string|boolean|undefined} Present if [`context.skip`][] is called
 
 Emitted when a test passes.
+This event is guaranteed to be emitted in the same order as the tests are
+defined.
+The corresponding execution ordered event is `'test:complete'`.
 
 ### Event: `'test:plan'`
 
@@ -2583,6 +2624,8 @@ Emitted when a test passes.
   * `count` {number} The number of subtests that have ran.
 
 Emitted when all subtests have completed for a given test.
+This event is guaranteed to be emitted in the same order as the tests are
+defined.
 
 ### Event: `'test:start'`
 
@@ -2599,6 +2642,7 @@ Emitted when all subtests have completed for a given test.
 Emitted when a test starts reporting its own and its subtests status.
 This event is guaranteed to be emitted in the same order as the tests are
 defined.
+The corresponding execution ordered event is `'test:dequeue'`.
 
 ### Event: `'test:stderr'`
 
@@ -2612,6 +2656,8 @@ defined.
 
 Emitted when a running test writes to `stderr`.
 This event is only emitted if `--test` flag is passed.
+This event is not guaranteed to be emitted in the same order as the tests are
+defined.
 
 ### Event: `'test:stdout'`
 
@@ -2625,6 +2671,8 @@ This event is only emitted if `--test` flag is passed.
 
 Emitted when a running test writes to `stdout`.
 This event is only emitted if `--test` flag is passed.
+This event is not guaranteed to be emitted in the same order as the tests are
+defined.
 
 ### Event: `'test:watch:drained'`
 

diff --git a/lib/internal/test_runner/test.js b/lib/internal/test_runner/test.js
@@ -726,6 +726,10 @@ class Test extends AsyncResource {
     this.mock?.reset();
 
     if (this.parent !== null) {
+      const report = this.getReportDetails();
+      report.details.passed = this.passed;
+      this.reporter.complete(this.nesting, this.loc, this.testNumber, this.name, report.details, report.directive);
+
       this.parent.activeSubtests--;
       this.parent.addReadySubtest(this);
       this.parent.processReadySubtestRange(false);
@@ -806,13 +810,7 @@ class Test extends AsyncResource {
     return Number(this.endTime - this.startTime) / 1_000_000;
   }
 
-  report() {
-    countCompletedTest(this);
-    if (this.subtests.length > 0) {
-      this.reporter.plan(this.subtests[0].nesting, this.loc, this.subtests.length);
-    } else {
-      this.reportStarted();
-    }
+  getReportDetails() {
     let directive;
     const details = { __proto__: null, duration_ms: this.duration() };
 
@@ -825,12 +823,25 @@ class Test extends AsyncResource {
     if (this.reportedType) {
       details.type = this.reportedType;
     }
+    if (!this.passed) {
+      details.error = this.error;
+    }
+    return { __proto__: null, details, directive };
+  }
+
+  report() {
+    countCompletedTest(this);
+    if (this.subtests.length > 0) {
+      this.reporter.plan(this.subtests[0].nesting, this.loc, this.subtests.length);
+    } else {
+      this.reportStarted();
+    }
+    const report = this.getReportDetails();
 
     if (this.passed) {
-      this.reporter.ok(this.nesting, this.loc, this.testNumber, this.name, details, directive);
+      this.reporter.ok(this.nesting, this.loc, this.testNumber, this.name, report.details, report.directive);
     } else {
-      details.error = this.error;
-      this.reporter.fail(this.nesting, this.loc, this.testNumber, this.name, details, directive);
+      this.reporter.fail(this.nesting, this.loc, this.testNumber, this.name, report.details, report.directive);
     }
 
     for (let i = 0; i < this.diagnostics.length; i++) {

diff --git a/lib/internal/test_runner/tests_stream.js b/lib/internal/test_runner/tests_stream.js
@@ -53,6 +53,18 @@ class TestsStream extends Readable {
     });
   }
 
+  complete(nesting, loc, testNumber, name, details, directive) {
+    this[kEmitMessage]('test:complete', {
+      __proto__: null,
+      name,
+      nesting,
+      testNumber,
+      details,
+      ...loc,
+      ...directive,
+    });
+  }
+
   plan(nesting, loc, count) {
     this[kEmitMessage]('test:plan', {
       __proto__: null,

diff --git a/test/parallel/test-runner-reporters.js b/test/parallel/test-runner-reporters.js
@@ -96,7 +96,7 @@ describe('node:test reporters', { concurrency: true }, () => {
                                testFile]);
       assert.strictEqual(child.stderr.toString(), '');
       const stdout = child.stdout.toString();
-      assert.match(stdout, /{"test:enqueue":5,"test:dequeue":5,"test:start":4,"test:pass":2,"test:fail":2,"test:plan":2,"test:diagnostic":\d+}$/);
+      assert.match(stdout, /{"test:enqueue":5,"test:dequeue":5,"test:complete":5,"test:start":4,"test:pass":2,"test:fail":2,"test:plan":2,"test:diagnostic":\d+}$/);
       assert.strictEqual(stdout.slice(0, filename.length + 2), `${filename} {`);
     });
   });
@@ -108,7 +108,7 @@ describe('node:test reporters', { concurrency: true }, () => {
     assert.strictEqual(child.stderr.toString(), '');
     assert.match(
       child.stdout.toString(),
-      /^package: reporter-cjs{"test:enqueue":5,"test:dequeue":5,"test:start":4,"test:pass":2,"test:fail":2,"test:plan":2,"test:diagnostic":\d+}$/,
+      /^package: reporter-cjs{"test:enqueue":5,"test:dequeue":5,"test:complete":5,"test:start":4,"test:pass":2,"test:fail":2,"test:plan":2,"test:diagnostic":\d+}$/,
     );
   });
 
@@ -119,7 +119,7 @@ describe('node:test reporters', { concurrency: true }, () => {
     assert.strictEqual(child.stderr.toString(), '');
     assert.match(
       child.stdout.toString(),
-      /^package: reporter-esm{"test:enqueue":5,"test:dequeue":5,"test:start":4,"test:pass":2,"test:fail":2,"test:plan":2,"test:diagnostic":\d+}$/,
+      /^package: reporter-esm{"test:enqueue":5,"test:dequeue":5,"test:complete":5,"test:start":4,"test:pass":2,"test:fail":2,"test:plan":2,"test:diagnostic":\d+}$/,
     );
   });