test_runner: test runner bail

Author: marco-ippolito

doc/api/cli.md

@@ -1441,6 +1441,16 @@ Starts the Node.js command line test runner. This flag cannot be combined with
 See the documentation on [running tests from the command line][]
 for more details.
 
+### `--test-bail`
+
+<!-- YAML
+added:
+  - REPLACEME
+-->
+
+Specifies the bailout behavior of the test runner when running tests.
+See the documentation on [test bailout][] for more details.
+
 ### `--test-name-pattern`
 
 <!-- YAML
@@ -2643,6 +2653,7 @@ done
 [security warning]: #warning-binding-inspector-to-a-public-ipport-combination-is-insecure
 [semi-space]: https://www.memorymanagement.org/glossary/s.html#semi.space
 [single executable application]: single-executable-applications.md
+[test bailout]: test.md#test-bail
 [test reporters]: test.md#test-reporters
 [timezone IDs]: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones
 [tracking issue for user-land snapshots]: https://github.com/nodejs/node/issues/44014

doc/api/errors.md

@@ -2738,6 +2738,13 @@ the token causing the error is available via the `cause` property.
 
 This error represents a failed TAP validation.
 
+<a id="ERR_TEST_BAILOUT"></a>
+
+### `ERR_TEST_BAILOUT`
+
+This error represents a test that has bailed out after failure.
+This error occurs only when the flag `--test-bail` is passed.
+
 <a id="ERR_TEST_FAILURE"></a>
 
 ### `ERR_TEST_FAILURE`

doc/api/test.md

@@ -602,6 +602,24 @@ test('mocks setTimeout to be executed synchronously without having to actually w
 });
 ```
 
+## Test bail
+
+<!-- YAML
+added:
+  - REPLACEME
+-->
+
+```bash
+node --test-bail
+```
+
+The `--test-bail` flag provides a way to stop the test execution
+as soon as a test fails.
+By enabling this flag, the test runner will exit the test suite early
+when it encounters the first failing test, preventing
+the execution of subsequent tests.
+**Default:** `false`.
+
 ## Test reporters
 
 <!-- YAML

lib/internal/errors.js

@@ -1613,6 +1613,10 @@ E('ERR_TAP_VALIDATION_ERROR', function(errorMsg) {
   hideInternalStackFrames(this);
   return errorMsg;
 }, Error);
+E('ERR_TEST_BAILOUT', function(errorMsg) {
+  hideInternalStackFrames(this);
+  return errorMsg;
+}, Error);
 E('ERR_TEST_FAILURE', function(error, failureType) {
   hideInternalStackFrames(this);
   assert(typeof failureType === 'string' || typeof failureType === 'symbol',

lib/internal/main/test_runner.js

@@ -57,7 +57,13 @@ if (shardOption) {
   };
 }
 
-run({ concurrency, inspectPort, watch: getOptionValue('--watch'), setup: setupTestReporters, shard })
-.once('test:fail', () => {
+run({
+  concurrency,
+  inspectPort,
+  watch: getOptionValue('--watch'),
+  setup: setupTestReporters,
+  shard,
+  bail: getOptionValue('--test-bail'),
+}).once('test:fail', () => {
   process.exitCode = kGenericUserError;
 });

lib/internal/test_runner/test.js

@@ -28,6 +28,7 @@ const {
   codes: {
     ERR_INVALID_ARG_TYPE,
     ERR_TEST_FAILURE,
+    ERR_TEST_BAILOUT,
   },
   AbortError,
 } = require('internal/errors');
@@ -71,8 +72,9 @@ const kHookNames = ObjectSeal(['before', 'after', 'beforeEach', 'afterEach']);
 const kUnwrapErrors = new SafeSet()
   .add(kTestCodeFailure).add(kHookFailure)
   .add('uncaughtException').add('unhandledRejection');
-const { testNamePatterns, testOnlyFlag } = parseCommandLine();
+const { testNamePatterns, testOnlyFlag, bail } = parseCommandLine();
 let kResistStopPropagation;
+let bailedOut = false;
 
 function stopTest(timeout, signal) {
   if (timeout === kDefaultTimeout) {
@@ -421,11 +423,13 @@ class Test extends AsyncResource {
       return;
     }
 
+    const unknownError = bailedOut ? new ERR_TEST_BAILOUT('test bailed out') : new ERR_TEST_FAILURE(
+      'test did not finish before its parent and was cancelled',
+      kCancelledByParent,
+    );
+
     this.fail(error ||
-      new ERR_TEST_FAILURE(
-        'test did not finish before its parent and was cancelled',
-        kCancelledByParent,
-      ),
+      unknownError,
     );
     this.startTime = this.startTime || this.endTime; // If a test was canceled before it was started, e.g inside a hook
     this.cancelled = true;
@@ -444,6 +448,7 @@ class Test extends AsyncResource {
   }
 
   fail(err) {
+    bailedOut = bail;
     if (this.error !== null) {
       return;
     }
@@ -526,6 +531,10 @@ class Test extends AsyncResource {
   }
 
   async run(pendingSubtestsError) {
+    if (bailedOut) {
+      return;
+    }
+
     if (this.parent !== null) {
       this.parent.activeSubtests++;
     }

lib/internal/test_runner/utils.js

@@ -179,6 +179,7 @@ function parseCommandLine() {
   }
 
   const isTestRunner = getOptionValue('--test');
+  const bail = getOptionValue('--test-bail');
   const coverage = getOptionValue('--experimental-test-coverage');
   const isChildProcess = process.env.NODE_TEST_CONTEXT === 'child';
   const isChildProcessV8 = process.env.NODE_TEST_CONTEXT === 'child-v8';
@@ -230,6 +231,7 @@ function parseCommandLine() {
   globalTestOptions = {
     __proto__: null,
     isTestRunner,
+    bail,
     coverage,
     testOnlyFlag,
     testNamePatterns,

src/node_options.cc

@@ -567,6 +567,9 @@ EnvironmentOptionsParser::EnvironmentOptionsParser() {
             "profile generated with --heap-prof. (default: 512 * 1024)",
             &EnvironmentOptions::heap_prof_interval);
 #endif  // HAVE_INSPECTOR
+  AddOption("--test-bail",
+            "stop test execution when given number of tests have failed",
+            &EnvironmentOptions::test_bail);
   AddOption("--max-http-header-size",
             "set the maximum size of HTTP headers (default: 16384 (16KB))",
             &EnvironmentOptions::max_http_header_size,

src/node_options.h

@@ -159,6 +159,7 @@ class EnvironmentOptions : public Options {
   std::string redirect_warnings;
   std::string diagnostic_dir;
   bool test_runner = false;
+  bool test_bail = false;
   bool test_runner_coverage = false;
   std::vector<std::string> test_name_pattern;
   std::vector<std::string> test_reporter;

test/fixtures/test-runner/bail/bail.js

@@ -0,0 +1,18 @@
+const assert = require('assert');
+const test = require('node:test');
+
+test('nested', (t) => {
+  t.test('first', () => {});
+  t.test('second', () => {
+    throw new Error();
+  });
+  t.test('third', () => {});
+});
+
+test('top level', (t) => {
+  t.test('forth', () => {});
+  t.test('fifth', () => {
+    throw new Error();
+  });
+});
+