test_runner: support 'only' tests

This commit introduces a CLI flag and test runner functionality
to support running a subset of tests that are indicated by an
'only' option passed to the test.

PR-URL: nodejs/node#42514
Backport-PR-URL: nodejs/node#43904
Reviewed-By: James M Snell <[email protected]>
Reviewed-By: Antoine du Hamel <[email protected]>

Author: cjihrig

doc/api/cli.md

@@ -1064,6 +1064,15 @@ minimum allocation from the secure heap. The minimum value is `2`.
 The maximum value is the lesser of `--secure-heap` or `2147483647`.
 The value given must be a power of two.
 
+### `--test-only`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+Configures the test runner to only execute top level tests that have the `only`
+option set.
+
 ### `--throw-deprecation`
 
 <!-- YAML
@@ -1654,6 +1663,7 @@ Node.js options that are allowed are:
 * `--require`, `-r`
 * `--secure-heap-min`
 * `--secure-heap`
+* `--test-only`
 * `--throw-deprecation`
 * `--title`
 * `--tls-cipher-list`

doc/api/test.md

@@ -148,6 +148,42 @@ test('skip() method with message', (t) => {
 });
 ```
 
+### `only` tests
+
+If Node.js is started with the [`--test-only`][] command-line option, it is
+possible to skip all top level tests except for a selected subset by passing
+the `only` option to the tests that should be run. When a test with the `only`
+option set is run, all subtests are also run. The test context's `runOnly()`
+method can be used to implement the same behavior at the subtest level.
+
+```js
+// Assume Node.js is run with the --test-only command-line option.
+// The 'only' option is set, so this test is run.
+test('this test is run', { only: true }, async (t) => {
+  // Within this test, all subtests are run by default.
+  await t.test('running subtest');
+
+  // The test context can be updated to run subtests with the 'only' option.
+  t.runOnly(true);
+  await t.test('this subtest is now skipped');
+  await t.test('this subtest is run', { only: true });
+
+  // Switch the context back to execute all tests.
+  t.runOnly(false);
+  await t.test('this subtest is now run');
+
+  // Explicitly do not run these tests.
+  await t.test('skipped subtest 3', { only: false });
+  await t.test('skipped subtest 4', { skip: true });
+});
+
+// The 'only' option is not set, so this test is skipped.
+test('this test is not run', () => {
+  // This code is not run.
+  throw new Error('fail');
+});
+```
+
 ## Extraneous asynchronous activity
 
 Once a test function finishes executing, the TAP results are output as quickly
@@ -197,6 +233,9 @@ added: REPLACEME
   * `concurrency` {number} The number of tests that can be run at the same time.
     If unspecified, subtests inherit this value from their parent.
     **Default:** `1`.
+  * `only` {boolean} If truthy, and the test context is configured to run
+    `only` tests, then this test will be run. Otherwise, the test is skipped.
+    **Default:** `false`.
   * `skip` {boolean|string} If truthy, the test is skipped. If a string is
     provided, that string is displayed in the test results as the reason for
     skipping the test. **Default:** `false`.
@@ -257,6 +296,19 @@ This function is used to write TAP diagnostics to the output. Any diagnostic
 information is included at the end of the test's results. This function does
 not return a value.
 
+### `context.runOnly(shouldRunOnlyTests)`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+* `shouldRunOnlyTests` {boolean} Whether or not to run `only` tests.
+
+If `shouldRunOnlyTests` is truthy, the test context will only run tests that
+have the `only` option set. Otherwise, all tests are run. If Node.js was not
+started with the [`--test-only`][] command-line option, this function is a
+no-op.
+
 ### `context.skip([message])`
 
 <!-- YAML
@@ -296,6 +348,9 @@ added: REPLACEME
   * `concurrency` {number} The number of tests that can be run at the same time.
     If unspecified, subtests inherit this value from their parent.
     **Default:** `1`.
+  * `only` {boolean} If truthy, and the test context is configured to run
+    `only` tests, then this test will be run. Otherwise, the test is skipped.
+    **Default:** `false`.
   * `skip` {boolean|string} If truthy, the test is skipped. If a string is
     provided, that string is displayed in the test results as the reason for
     skipping the test. **Default:** `false`.
@@ -312,5 +367,6 @@ This function is used to create subtests under the current test. This function
 behaves in the same fashion as the top level [`test()`][] function.
 
 [TAP]: https://testanything.org/
+[`--test-only`]: cli.md#--test-only
 [`TestContext`]: #class-testcontext
 [`test()`]: #testname-options-fn

doc/node.1

@@ -378,6 +378,10 @@ the secure heap. The default is 0. The value must be a power of two.
 .It Fl -secure-heap-min Ns = Ns Ar n
 Specify the minimum allocation from the OpenSSL secure heap. The default is 2. The value must be a power of two.
 .
+.It Fl -test-only
+Configures the test runner to only execute top level tests that have the `only`
+option set.
+.
 .It Fl -throw-deprecation
 Throw errors for deprecations.
 .

lib/internal/test_runner/test.js

@@ -13,6 +13,7 @@ const {
     ERR_TEST_FAILURE,
   },
 } = require('internal/errors');
+const { getOptionValue } = require('internal/options');
 const { TapStream } = require('internal/test_runner/tap_stream');
 const { createDeferredPromise } = require('internal/util');
 const { isPromise } = require('internal/util/types');
@@ -26,6 +27,7 @@ const kSubtestsFailed = 'subtestsFailed';
 const kTestCodeFailure = 'testCodeFailure';
 const kDefaultIndent = '    ';
 const noop = FunctionPrototype;
+const testOnlyFlag = getOptionValue('--test-only');
 
 class TestContext {
   #test;
@@ -38,6 +40,10 @@ class TestContext {
     this.#test.diagnostic(message);
   }
 
+  runOnly(value) {
+    this.#test.runOnlySubtests = !!value;
+  }
+
   skip(message) {
     this.#test.skip(message);
   }
@@ -57,8 +63,8 @@ class Test extends AsyncResource {
   constructor(options) {
     super('Test');
 
-    let { fn, name, parent } = options;
-    const { concurrency, skip, todo } = options;
+    let { fn, name, parent, skip } = options;
+    const { concurrency, only, todo } = options;
 
     if (typeof fn !== 'function') {
       fn = noop;
@@ -72,19 +78,13 @@ class Test extends AsyncResource {
       parent = null;
     }
 
-    if (skip) {
-      fn = noop;
-    }
-
-    this.fn = fn;
-    this.name = name;
-    this.parent = parent;
-
     if (parent === null) {
       this.concurrency = 1;
       this.indent = '';
       this.indentString = kDefaultIndent;
+      this.only = testOnlyFlag;
       this.reporter = new TapStream();
+      this.runOnlySubtests = this.only;
       this.testNumber = 0;
     } else {
       const indent = parent.parent === null ? parent.indent :
@@ -93,14 +93,27 @@ class Test extends AsyncResource {
       this.concurrency = parent.concurrency;
       this.indent = indent;
       this.indentString = parent.indentString;
+      this.only = only ?? !parent.runOnlySubtests;
       this.reporter = parent.reporter;
+      this.runOnlySubtests = !this.only;
       this.testNumber = parent.subtests.length + 1;
     }
 
     if (isUint32(concurrency) && concurrency !== 0) {
       this.concurrency = concurrency;
     }
 
+    if (testOnlyFlag && !this.only) {
+      skip = '\'only\' option not set';
+    }
+
+    if (skip) {
+      fn = noop;
+    }
+
+    this.fn = fn;
+    this.name = name;
+    this.parent = parent;
     this.cancelled = false;
     this.skipped = !!skip;
     this.isTodo = !!todo;

src/node_options.cc

@@ -503,6 +503,10 @@ EnvironmentOptionsParser::EnvironmentOptionsParser() {
             "write warnings to file instead of stderr",
             &EnvironmentOptions::redirect_warnings,
             kAllowedInEnvironment);
+  AddOption("--test-only",
+            "run tests with 'only' option set",
+            &EnvironmentOptions::test_only,
+            kAllowedInEnvironment);
   AddOption("--test-udp-no-try-send", "",  // For testing only.
             &EnvironmentOptions::test_udp_no_try_send);
   AddOption("--throw-deprecation",

src/node_options.h

@@ -149,6 +149,7 @@ class EnvironmentOptions : public Options {
 #endif  // HAVE_INSPECTOR
   std::string redirect_warnings;
   std::string diagnostic_dir;
+  bool test_only = false;
   bool test_udp_no_try_send = false;
   bool throw_deprecation = false;
   bool trace_atomics_wait = false;

test/message/test_runner_only_tests.js

@@ -0,0 +1,48 @@
+// Flags: --no-warnings --test-only
+'use strict';
+require('../common');
+const test = require('node:test');
+
+// These tests should be skipped based on the 'only' option.
+test('only = undefined');
+test('only = undefined, skip = string', { skip: 'skip message' });
+test('only = undefined, skip = true', { skip: true });
+test('only = undefined, skip = false', { skip: false });
+test('only = false', { only: false });
+test('only = false, skip = string', { only: false, skip: 'skip message' });
+test('only = false, skip = true', { only: false, skip: true });
+test('only = false, skip = false', { only: false, skip: false });
+
+// These tests should be skipped based on the 'skip' option.
+test('only = true, skip = string', { only: true, skip: 'skip message' });
+test('only = true, skip = true', { only: true, skip: true });
+
+// An 'only' test with subtests.
+test('only = true, with subtests', { only: true }, async (t) => {
+  // These subtests should run.
+  await t.test('running subtest 1');
+  await t.test('running subtest 2');
+
+  // Switch the context to only execute 'only' tests.
+  t.runOnly(true);
+  await t.test('skipped subtest 1');
+  await t.test('skipped subtest 2');
+  await t.test('running subtest 3', { only: true });
+
+  // Switch the context back to execute all tests.
+  t.runOnly(false);
+  await t.test('running subtest 4', async (t) => {
+    // These subtests should run.
+    await t.test('running sub-subtest 1');
+    await t.test('running sub-subtest 2');
+
+    // Switch the context to only execute 'only' tests.
+    t.runOnly(true);
+    await t.test('skipped sub-subtest 1');
+    await t.test('skipped sub-subtest 2');
+  });
+
+  // Explicitly do not run these tests.
+  await t.test('skipped subtest 3', { only: false });
+  await t.test('skipped subtest 4', { skip: true });
+});

test/message/test_runner_only_tests.out

@@ -0,0 +1,102 @@
+TAP version 13
+ok 1 - only = undefined # SKIP 'only' option not set
+  ---
+  duration_ms: *
+  ...
+ok 2 - only = undefined, skip = string # SKIP 'only' option not set
+  ---
+  duration_ms: *
+  ...
+ok 3 - only = undefined, skip = true # SKIP 'only' option not set
+  ---
+  duration_ms: *
+  ...
+ok 4 - only = undefined, skip = false # SKIP 'only' option not set
+  ---
+  duration_ms: *
+  ...
+ok 5 - only = false # SKIP 'only' option not set
+  ---
+  duration_ms: *
+  ...
+ok 6 - only = false, skip = string # SKIP 'only' option not set
+  ---
+  duration_ms: *
+  ...
+ok 7 - only = false, skip = true # SKIP 'only' option not set
+  ---
+  duration_ms: *
+  ...
+ok 8 - only = false, skip = false # SKIP 'only' option not set
+  ---
+  duration_ms: *
+  ...
+ok 9 - only = true, skip = string # SKIP skip message
+  ---
+  duration_ms: *
+  ...
+ok 10 - only = true, skip = true # SKIP
+  ---
+  duration_ms: *
+  ...
+    ok 1 - running subtest 1
+      ---
+      duration_ms: *
+      ...
+    ok 2 - running subtest 2
+      ---
+      duration_ms: *
+      ...
+    ok 3 - skipped subtest 1 # SKIP 'only' option not set
+      ---
+      duration_ms: *
+      ...
+    ok 4 - skipped subtest 2 # SKIP 'only' option not set
+      ---
+      duration_ms: *
+      ...
+    ok 5 - running subtest 3
+      ---
+      duration_ms: *
+      ...
+        ok 1 - running sub-subtest 1
+          ---
+          duration_ms: *
+          ...
+        ok 2 - running sub-subtest 2
+          ---
+          duration_ms: *
+          ...
+        ok 3 - skipped sub-subtest 1 # SKIP 'only' option not set
+          ---
+          duration_ms: *
+          ...
+        ok 4 - skipped sub-subtest 2 # SKIP 'only' option not set
+          ---
+          duration_ms: *
+          ...
+        1..4
+    ok 6 - running subtest 4
+      ---
+      duration_ms: *
+      ...
+    ok 7 - skipped subtest 3 # SKIP 'only' option not set
+      ---
+      duration_ms: *
+      ...
+    ok 8 - skipped subtest 4 # SKIP
+      ---
+      duration_ms: *
+      ...
+    1..8
+ok 11 - only = true, with subtests
+  ---
+  duration_ms: *
+  ...
+1..11
+# tests 11
+# pass 1
+# fail 0
+# skipped 10
+# todo 0
+# duration_ms *

test/message/test_runner_output.js

@@ -286,3 +286,13 @@ test('callback async throw after done', (t, done) => {
 
   done();
 });
+
+test('only is set but not in only mode', { only: true }, async (t) => {
+  // All of these subtests should run.
+  await t.test('running subtest 1');
+  t.runOnly(true);
+  await t.test('running subtest 2');
+  await t.test('running subtest 3', { only: true });
+  t.runOnly(false);
+  await t.test('running subtest 4');
+});