test_runner: add initial TAP parser

Work in progress

PR-URL: #43525
Refs: #43344
Reviewed-By: Franziska Hinkelmann <[email protected]>
Reviewed-By: Colin Ihrig <[email protected]>
Reviewed-By: Moshe Atlow <[email protected]>

Author: manekinekko

doc/api/errors.md

@@ -2682,6 +2682,25 @@ An unspecified or non-specific system error has occurred within the Node.js
 process. The error object will have an `err.info` object property with
 additional details.
 
+<a id="ERR_TAP_LEXER_ERROR"></a>
+
+### `ERR_TAP_LEXER_ERROR`
+
+An error representing a failing lexer state.
+
+<a id="ERR_TAP_PARSER_ERROR"></a>
+
+### `ERR_TAP_PARSER_ERROR`
+
+An error representing a failing parser state. Additional information about
+the token causing the error is available via the `cause` property.
+
+<a id="ERR_TAP_VALIDATION_ERROR"></a>
+
+### `ERR_TAP_VALIDATION_ERROR`
+
+This error represents a failed TAP validation.
+
 <a id="ERR_TEST_FAILURE"></a>
 
 ### `ERR_TEST_FAILURE`

doc/api/test.md

@@ -1028,8 +1028,7 @@ Emitted when [`context.diagnostic`][] is called.
 ### Event: `'test:fail'`
 
 * `data` {Object}
-  * `duration` {number} The test duration.
-  * `error` {Error} The failure casing test to fail.
+  * `details` {Object} Additional execution metadata.
   * `name` {string} The test name.
   * `testNumber` {number} The ordinal number of the test.
   * `todo` {string|undefined} Present if [`context.todo`][] is called
@@ -1040,7 +1039,7 @@ Emitted when a test fails.
 ### Event: `'test:pass'`
 
 * `data` {Object}
-  * `duration` {number} The test duration.
+  * `details` {Object} Additional execution metadata.
   * `name` {string} The test name.
   * `testNumber` {number} The ordinal number of the test.
   * `todo` {string|undefined} Present if [`context.todo`][] is called

lib/internal/errors.js

@@ -1597,6 +1597,21 @@ E('ERR_STREAM_WRAP', 'Stream has StringDecoder set or is in objectMode', Error);
 E('ERR_STREAM_WRITE_AFTER_END', 'write after end', Error);
 E('ERR_SYNTHETIC', 'JavaScript Callstack', Error);
 E('ERR_SYSTEM_ERROR', 'A system error occurred', SystemError);
+E('ERR_TAP_LEXER_ERROR', function(errorMsg) {
+  hideInternalStackFrames(this);
+  return errorMsg;
+}, Error);
+E('ERR_TAP_PARSER_ERROR', function(errorMsg, details, tokenCausedError, source) {
+  hideInternalStackFrames(this);
+  this.cause = tokenCausedError;
+  const { column, line, start, end } = tokenCausedError.location;
+  const errorDetails = `${details} at line ${line}, column ${column} (start ${start}, end ${end})`;
+  return errorMsg + errorDetails;
+}, SyntaxError);
+E('ERR_TAP_VALIDATION_ERROR', function(errorMsg) {
+  hideInternalStackFrames(this);
+  return errorMsg;
+}, Error);
 E('ERR_TEST_FAILURE', function(error, failureType) {
   hideInternalStackFrames(this);
   assert(typeof failureType === 'string',

lib/internal/test_runner/runner.js

@@ -2,6 +2,7 @@
 const {
   ArrayFrom,
   ArrayPrototypeFilter,
+  ArrayPrototypeForEach,
   ArrayPrototypeIncludes,
   ArrayPrototypeJoin,
   ArrayPrototypePush,
@@ -14,6 +15,7 @@ const {
   SafePromiseAllSettledReturnVoid,
   SafeMap,
   SafeSet,
+  StringPrototypeRepeat,
 } = primordials;
 
 const { spawn } = require('child_process');
@@ -31,7 +33,10 @@ const { validateArray, validateBoolean } = require('internal/validators');
 const { getInspectPort, isUsingInspector, isInspectorMessage } = require('internal/util/inspector');
 const { kEmptyObject } = require('internal/util');
 const { createTestTree } = require('internal/test_runner/harness');
-const { kSubtestsFailed, Test } = require('internal/test_runner/test');
+const { kDefaultIndent, kSubtestsFailed, Test } = require('internal/test_runner/test');
+const { TapParser } = require('internal/test_runner/tap_parser');
+const { TokenKind } = require('internal/test_runner/tap_lexer');
+
 const {
   isSupportedFileType,
   doesPathMatchFilter,
@@ -119,11 +124,103 @@ function getRunArgs({ path, inspectPort }) {
   return argv;
 }
 
+class FileTest extends Test {
+  #buffer = [];
+  #handleReportItem({ kind, node, nesting = 0 }) {
+    const indent = StringPrototypeRepeat(kDefaultIndent, nesting + 1);
+
+    const details = (diagnostic) => {
+      return (
+        diagnostic && {
+          __proto__: null,
+          yaml:
+            `${indent}  ` +
+            ArrayPrototypeJoin(diagnostic, `\n${indent}  `) +
+            '\n',
+        }
+      );
+    };
+
+    switch (kind) {
+      case TokenKind.TAP_VERSION:
+        // TODO(manekinekko): handle TAP version coming from the parser.
+        // this.reporter.version(node.version);
+        break;
+
+      case TokenKind.TAP_PLAN:
+        this.reporter.plan(indent, node.end - node.start + 1);
+        break;
+
+      case TokenKind.TAP_SUBTEST_POINT:
+        this.reporter.subtest(indent, node.name);
+        break;
+
+      case TokenKind.TAP_TEST_POINT:
+        // eslint-disable-next-line no-case-declarations
+        const { todo, skip, pass } = node.status;
+        // eslint-disable-next-line no-case-declarations
+        let directive;
+
+        if (skip) {
+          directive = this.reporter.getSkip(node.reason);
+        } else if (todo) {
+          directive = this.reporter.getTodo(node.reason);
+        } else {
+          directive = kEmptyObject;
+        }
+
+        if (pass) {
+          this.reporter.ok(
+            indent,
+            node.id,
+            node.description,
+            details(node.diagnostics),
+            directive
+          );
+        } else {
+          this.reporter.fail(
+            indent,
+            node.id,
+            node.description,
+            details(node.diagnostics),
+            directive
+          );
+        }
+        break;
+
+      case TokenKind.COMMENT:
+        if (indent === kDefaultIndent) {
+          // Ignore file top level diagnostics
+          break;
+        }
+        this.reporter.diagnostic(indent, node.comment);
+        break;
+
+      case TokenKind.UNKNOWN:
+        this.reporter.diagnostic(indent, node.value);
+        break;
+    }
+  }
+  addToReport(ast) {
+    if (!this.isClearToSend()) {
+      ArrayPrototypePush(this.#buffer, ast);
+      return;
+    }
+    this.reportSubtest();
+    this.#handleReportItem(ast);
+  }
+  report() {
+    this.reportSubtest();
+    ArrayPrototypeForEach(this.#buffer, (ast) => this.#handleReportItem(ast));
+    super.report();
+  }
+}
+
 const runningProcesses = new SafeMap();
 const runningSubtests = new SafeMap();
 
 function runTestFile(path, root, inspectPort, filesWatcher) {
-  const subtest = root.createSubtest(Test, path, async (t) => {
+  const subtest = root.createSubtest(FileTest, path, async (t) => {
     const args = getRunArgs({ path, inspectPort });
     const stdio = ['pipe', 'pipe', 'pipe'];
     const env = { ...process.env };
@@ -134,8 +231,7 @@ function runTestFile(path, root, inspectPort, filesWatcher) {
 
     const child = spawn(process.execPath, args, { signal: t.signal, encoding: 'utf8', env, stdio });
     runningProcesses.set(path, child);
-    // TODO(cjihrig): Implement a TAP parser to read the child's stdout
-    // instead of just displaying it all if the child fails.
+
     let err;
     let stderr = '';
 
@@ -158,6 +254,17 @@ function runTestFile(path, root, inspectPort, filesWatcher) {
       });
     }
 
+    const parser = new TapParser();
+    child.stderr.pipe(parser).on('data', (ast) => {
+      if (ast.lexeme && isInspectorMessage(ast.lexeme)) {
+        process.stderr.write(ast.lexeme + '\n');
+      }
+    });
+
+    child.stdout.pipe(parser).on('data', (ast) => {
+      subtest.addToReport(ast);
+    });
+
     const { 0: { 0: code, 1: signal }, 1: stdout } = await SafePromiseAll([
       once(child, 'exit', { signal: t.signal }),
       child.stdout.toArray({ signal: t.signal }),

lib/internal/test_runner/tap_checker.js

@@ -0,0 +1,155 @@
+'use strict';
+
+const {
+  ArrayPrototypeFilter,
+  ArrayPrototypeFind,
+  NumberParseInt,
+} = primordials;
+const {
+  codes: { ERR_TAP_VALIDATION_ERROR },
+} = require('internal/errors');
+const { TokenKind } = require('internal/test_runner/tap_lexer');
+
+// TODO(@manekinekko): add more validation rules based on the TAP14 spec.
+// See https://testanything.org/tap-version-14-specification.html
+class TAPValidationStrategy {
+  validate(ast) {
+    this.#validateVersion(ast);
+    this.#validatePlan(ast);
+    this.#validateTestPoints(ast);
+
+    return true;
+  }
+
+  #validateVersion(ast) {
+    const entry = ArrayPrototypeFind(
+      ast,
+      (node) => node.kind === TokenKind.TAP_VERSION
+    );
+
+    if (!entry) {
+      throw new ERR_TAP_VALIDATION_ERROR('missing TAP version');
+    }
+
+    const { version } = entry.node;
+
+    // TAP14 specification is compatible with observed behavior of existing TAP13 consumers and producers
+    if (version !== '14' && version !== '13') {
+      throw new ERR_TAP_VALIDATION_ERROR('TAP version should be 13 or 14');
+    }
+  }
+
+  #validatePlan(ast) {
+    const entry = ArrayPrototypeFind(
+      ast,
+      (node) => node.kind === TokenKind.TAP_PLAN
+    );
+
+    if (!entry) {
+      throw new ERR_TAP_VALIDATION_ERROR('missing TAP plan');
+    }
+
+    const plan = entry.node;
+
+    if (!plan.start) {
+      throw new ERR_TAP_VALIDATION_ERROR('missing plan start');
+    }
+
+    if (!plan.end) {
+      throw new ERR_TAP_VALIDATION_ERROR('missing plan end');
+    }
+
+    const planStart = NumberParseInt(plan.start, 10);
+    const planEnd = NumberParseInt(plan.end, 10);
+
+    if (planEnd !== 0 && planStart > planEnd) {
+      throw new ERR_TAP_VALIDATION_ERROR(
+        `plan start ${planStart} is greater than plan end ${planEnd}`
+      );
+    }
+  }
+
+  // TODO(@manekinekko): since we are dealing with a flat AST, we need to
+  // validate test points grouped by their "nesting" level. This is because a set of
+  // Test points belongs to a TAP document. Each new subtest block creates a new TAP document.
+  // https://testanything.org/tap-version-14-specification.html#subtests
+  #validateTestPoints(ast) {
+    const bailoutEntry = ArrayPrototypeFind(
+      ast,
+      (node) => node.kind === TokenKind.TAP_BAIL_OUT
+    );
+    const planEntry = ArrayPrototypeFind(
+      ast,
+      (node) => node.kind === TokenKind.TAP_PLAN
+    );
+    const testPointEntries = ArrayPrototypeFilter(
+      ast,
+      (node) => node.kind === TokenKind.TAP_TEST_POINT
+    );
+
+    const plan = planEntry.node;
+
+    const planStart = NumberParseInt(plan.start, 10);
+    const planEnd = NumberParseInt(plan.end, 10);
+
+    if (planEnd === 0 && testPointEntries.length > 0) {
+      throw new ERR_TAP_VALIDATION_ERROR(
+        `found ${testPointEntries.length} Test Point${
+          testPointEntries.length > 1 ? 's' : ''
+        } but plan is ${planStart}..0`
+      );
+    }
+
+    if (planEnd > 0) {
+      if (testPointEntries.length === 0) {
+        throw new ERR_TAP_VALIDATION_ERROR('missing Test Points');
+      }
+
+      if (!bailoutEntry && testPointEntries.length !== planEnd) {
+        throw new ERR_TAP_VALIDATION_ERROR(
+          `test Points count ${testPointEntries.length} does not match plan count ${planEnd}`
+        );
+      }
+
+      for (let i = 0; i < testPointEntries.length; i++) {
+        const test = testPointEntries[i].node;
+        const testId = NumberParseInt(test.id, 10);
+
+        if (testId < planStart || testId > planEnd) {
+          throw new ERR_TAP_VALIDATION_ERROR(
+            `test ${testId} is out of plan range ${planStart}..${planEnd}`
+          );
+        }
+      }
+    }
+  }
+}
+
+// TAP14 and TAP13 are compatible with each other
+class TAP13ValidationStrategy extends TAPValidationStrategy {}
+class TAP14ValidationStrategy extends TAPValidationStrategy {}
+
+class TapChecker {
+  static TAP13 = '13';
+  static TAP14 = '14';
+
+  constructor({ specs }) {
+    switch (specs) {
+      case TapChecker.TAP13:
+        this.strategy = new TAP13ValidationStrategy();
+        break;
+      default:
+        this.strategy = new TAP14ValidationStrategy();
+    }
+  }
+
+  check(ast) {
+    return this.strategy.validate(ast);
+  }
+}
+
+module.exports = {
+  TapChecker,
+  TAP14ValidationStrategy,
+  TAP13ValidationStrategy,
+};