Skip to content

Commit

Permalink
test_runner: add initial TAP parser
Browse files Browse the repository at this point in the history
Work in progress

PR-URL: #43525
Refs: #43344
Reviewed-By: Franziska Hinkelmann <franziska.hinkelmann@gmail.com>
Reviewed-By: Colin Ihrig <cjihrig@gmail.com>
Reviewed-By: Moshe Atlow <moshe@atlow.co.il>
  • Loading branch information
manekinekko authored and danielleadams committed Jan 5, 2023
1 parent 1264986 commit 87c902f
Show file tree
Hide file tree
Showing 19 changed files with 4,418 additions and 31 deletions.
19 changes: 19 additions & 0 deletions doc/api/errors.md
Original file line number Diff line number Diff line change
Expand Up @@ -2682,6 +2682,25 @@ An unspecified or non-specific system error has occurred within the Node.js
process. The error object will have an `err.info` object property with
additional details.

<a id="ERR_TAP_LEXER_ERROR"></a>

### `ERR_TAP_LEXER_ERROR`

An error representing a failing lexer state.

<a id="ERR_TAP_PARSER_ERROR"></a>

### `ERR_TAP_PARSER_ERROR`

An error representing a failing parser state. Additional information about
the token causing the error is available via the `cause` property.

<a id="ERR_TAP_VALIDATION_ERROR"></a>

### `ERR_TAP_VALIDATION_ERROR`

This error represents a failed TAP validation.

<a id="ERR_TEST_FAILURE"></a>

### `ERR_TEST_FAILURE`
Expand Down
5 changes: 2 additions & 3 deletions doc/api/test.md
Original file line number Diff line number Diff line change
Expand Up @@ -1028,8 +1028,7 @@ Emitted when [`context.diagnostic`][] is called.
### Event: `'test:fail'`

* `data` {Object}
* `duration` {number} The test duration.
* `error` {Error} The failure casing test to fail.
* `details` {Object} Additional execution metadata.
* `name` {string} The test name.
* `testNumber` {number} The ordinal number of the test.
* `todo` {string|undefined} Present if [`context.todo`][] is called
Expand All @@ -1040,7 +1039,7 @@ Emitted when a test fails.
### Event: `'test:pass'`

* `data` {Object}
* `duration` {number} The test duration.
* `details` {Object} Additional execution metadata.
* `name` {string} The test name.
* `testNumber` {number} The ordinal number of the test.
* `todo` {string|undefined} Present if [`context.todo`][] is called
Expand Down
15 changes: 15 additions & 0 deletions lib/internal/errors.js
Original file line number Diff line number Diff line change
Expand Up @@ -1597,6 +1597,21 @@ E('ERR_STREAM_WRAP', 'Stream has StringDecoder set or is in objectMode', Error);
E('ERR_STREAM_WRITE_AFTER_END', 'write after end', Error);
E('ERR_SYNTHETIC', 'JavaScript Callstack', Error);
E('ERR_SYSTEM_ERROR', 'A system error occurred', SystemError);
E('ERR_TAP_LEXER_ERROR', function(errorMsg) {
hideInternalStackFrames(this);
return errorMsg;
}, Error);
E('ERR_TAP_PARSER_ERROR', function(errorMsg, details, tokenCausedError, source) {
hideInternalStackFrames(this);
this.cause = tokenCausedError;
const { column, line, start, end } = tokenCausedError.location;
const errorDetails = `${details} at line ${line}, column ${column} (start ${start}, end ${end})`;
return errorMsg + errorDetails;
}, SyntaxError);
E('ERR_TAP_VALIDATION_ERROR', function(errorMsg) {
hideInternalStackFrames(this);
return errorMsg;
}, Error);
E('ERR_TEST_FAILURE', function(error, failureType) {
hideInternalStackFrames(this);
assert(typeof failureType === 'string',
Expand Down
115 changes: 111 additions & 4 deletions lib/internal/test_runner/runner.js
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
const {
ArrayFrom,
ArrayPrototypeFilter,
ArrayPrototypeForEach,
ArrayPrototypeIncludes,
ArrayPrototypeJoin,
ArrayPrototypePush,
Expand All @@ -14,6 +15,7 @@ const {
SafePromiseAllSettledReturnVoid,
SafeMap,
SafeSet,
StringPrototypeRepeat,
} = primordials;

const { spawn } = require('child_process');
Expand All @@ -31,7 +33,10 @@ const { validateArray, validateBoolean } = require('internal/validators');
const { getInspectPort, isUsingInspector, isInspectorMessage } = require('internal/util/inspector');
const { kEmptyObject } = require('internal/util');
const { createTestTree } = require('internal/test_runner/harness');
const { kSubtestsFailed, Test } = require('internal/test_runner/test');
const { kDefaultIndent, kSubtestsFailed, Test } = require('internal/test_runner/test');
const { TapParser } = require('internal/test_runner/tap_parser');
const { TokenKind } = require('internal/test_runner/tap_lexer');

const {
isSupportedFileType,
doesPathMatchFilter,
Expand Down Expand Up @@ -119,11 +124,103 @@ function getRunArgs({ path, inspectPort }) {
return argv;
}

class FileTest extends Test {
#buffer = [];
#handleReportItem({ kind, node, nesting = 0 }) {
const indent = StringPrototypeRepeat(kDefaultIndent, nesting + 1);

const details = (diagnostic) => {
return (
diagnostic && {
__proto__: null,
yaml:
`${indent} ` +
ArrayPrototypeJoin(diagnostic, `\n${indent} `) +
'\n',
}
);
};

switch (kind) {
case TokenKind.TAP_VERSION:
// TODO(manekinekko): handle TAP version coming from the parser.
// this.reporter.version(node.version);
break;

case TokenKind.TAP_PLAN:
this.reporter.plan(indent, node.end - node.start + 1);
break;

case TokenKind.TAP_SUBTEST_POINT:
this.reporter.subtest(indent, node.name);
break;

case TokenKind.TAP_TEST_POINT:
// eslint-disable-next-line no-case-declarations
const { todo, skip, pass } = node.status;
// eslint-disable-next-line no-case-declarations
let directive;

if (skip) {
directive = this.reporter.getSkip(node.reason);
} else if (todo) {
directive = this.reporter.getTodo(node.reason);
} else {
directive = kEmptyObject;
}

if (pass) {
this.reporter.ok(
indent,
node.id,
node.description,
details(node.diagnostics),
directive
);
} else {
this.reporter.fail(
indent,
node.id,
node.description,
details(node.diagnostics),
directive
);
}
break;

case TokenKind.COMMENT:
if (indent === kDefaultIndent) {
// Ignore file top level diagnostics
break;
}
this.reporter.diagnostic(indent, node.comment);
break;

case TokenKind.UNKNOWN:
this.reporter.diagnostic(indent, node.value);
break;
}
}
addToReport(ast) {
if (!this.isClearToSend()) {
ArrayPrototypePush(this.#buffer, ast);
return;
}
this.reportSubtest();
this.#handleReportItem(ast);
}
report() {
this.reportSubtest();
ArrayPrototypeForEach(this.#buffer, (ast) => this.#handleReportItem(ast));
super.report();
}
}

const runningProcesses = new SafeMap();
const runningSubtests = new SafeMap();

function runTestFile(path, root, inspectPort, filesWatcher) {
const subtest = root.createSubtest(Test, path, async (t) => {
const subtest = root.createSubtest(FileTest, path, async (t) => {
const args = getRunArgs({ path, inspectPort });
const stdio = ['pipe', 'pipe', 'pipe'];
const env = { ...process.env };
Expand All @@ -134,8 +231,7 @@ function runTestFile(path, root, inspectPort, filesWatcher) {

const child = spawn(process.execPath, args, { signal: t.signal, encoding: 'utf8', env, stdio });
runningProcesses.set(path, child);
// TODO(cjihrig): Implement a TAP parser to read the child's stdout
// instead of just displaying it all if the child fails.

let err;
let stderr = '';

Expand All @@ -158,6 +254,17 @@ function runTestFile(path, root, inspectPort, filesWatcher) {
});
}

const parser = new TapParser();
child.stderr.pipe(parser).on('data', (ast) => {
if (ast.lexeme && isInspectorMessage(ast.lexeme)) {
process.stderr.write(ast.lexeme + '\n');
}
});

child.stdout.pipe(parser).on('data', (ast) => {
subtest.addToReport(ast);
});

const { 0: { 0: code, 1: signal }, 1: stdout } = await SafePromiseAll([
once(child, 'exit', { signal: t.signal }),
child.stdout.toArray({ signal: t.signal }),
Expand Down
155 changes: 155 additions & 0 deletions lib/internal/test_runner/tap_checker.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,155 @@
'use strict';

const {
ArrayPrototypeFilter,
ArrayPrototypeFind,
NumberParseInt,
} = primordials;
const {
codes: { ERR_TAP_VALIDATION_ERROR },
} = require('internal/errors');
const { TokenKind } = require('internal/test_runner/tap_lexer');

// TODO(@manekinekko): add more validation rules based on the TAP14 spec.
// See https://testanything.org/tap-version-14-specification.html
class TAPValidationStrategy {
validate(ast) {
this.#validateVersion(ast);
this.#validatePlan(ast);
this.#validateTestPoints(ast);

return true;
}

#validateVersion(ast) {
const entry = ArrayPrototypeFind(
ast,
(node) => node.kind === TokenKind.TAP_VERSION
);

if (!entry) {
throw new ERR_TAP_VALIDATION_ERROR('missing TAP version');
}

const { version } = entry.node;

// TAP14 specification is compatible with observed behavior of existing TAP13 consumers and producers
if (version !== '14' && version !== '13') {
throw new ERR_TAP_VALIDATION_ERROR('TAP version should be 13 or 14');
}
}

#validatePlan(ast) {
const entry = ArrayPrototypeFind(
ast,
(node) => node.kind === TokenKind.TAP_PLAN
);

if (!entry) {
throw new ERR_TAP_VALIDATION_ERROR('missing TAP plan');
}

const plan = entry.node;

if (!plan.start) {
throw new ERR_TAP_VALIDATION_ERROR('missing plan start');
}

if (!plan.end) {
throw new ERR_TAP_VALIDATION_ERROR('missing plan end');
}

const planStart = NumberParseInt(plan.start, 10);
const planEnd = NumberParseInt(plan.end, 10);

if (planEnd !== 0 && planStart > planEnd) {
throw new ERR_TAP_VALIDATION_ERROR(
`plan start ${planStart} is greater than plan end ${planEnd}`
);
}
}

// TODO(@manekinekko): since we are dealing with a flat AST, we need to
// validate test points grouped by their "nesting" level. This is because a set of
// Test points belongs to a TAP document. Each new subtest block creates a new TAP document.
// https://testanything.org/tap-version-14-specification.html#subtests
#validateTestPoints(ast) {
const bailoutEntry = ArrayPrototypeFind(
ast,
(node) => node.kind === TokenKind.TAP_BAIL_OUT
);
const planEntry = ArrayPrototypeFind(
ast,
(node) => node.kind === TokenKind.TAP_PLAN
);
const testPointEntries = ArrayPrototypeFilter(
ast,
(node) => node.kind === TokenKind.TAP_TEST_POINT
);

const plan = planEntry.node;

const planStart = NumberParseInt(plan.start, 10);
const planEnd = NumberParseInt(plan.end, 10);

if (planEnd === 0 && testPointEntries.length > 0) {
throw new ERR_TAP_VALIDATION_ERROR(
`found ${testPointEntries.length} Test Point${
testPointEntries.length > 1 ? 's' : ''
} but plan is ${planStart}..0`
);
}

if (planEnd > 0) {
if (testPointEntries.length === 0) {
throw new ERR_TAP_VALIDATION_ERROR('missing Test Points');
}

if (!bailoutEntry && testPointEntries.length !== planEnd) {
throw new ERR_TAP_VALIDATION_ERROR(
`test Points count ${testPointEntries.length} does not match plan count ${planEnd}`
);
}

for (let i = 0; i < testPointEntries.length; i++) {
const test = testPointEntries[i].node;
const testId = NumberParseInt(test.id, 10);

if (testId < planStart || testId > planEnd) {
throw new ERR_TAP_VALIDATION_ERROR(
`test ${testId} is out of plan range ${planStart}..${planEnd}`
);
}
}
}
}
}

// TAP14 and TAP13 are compatible with each other
class TAP13ValidationStrategy extends TAPValidationStrategy {}
class TAP14ValidationStrategy extends TAPValidationStrategy {}

class TapChecker {
static TAP13 = '13';
static TAP14 = '14';

constructor({ specs }) {
switch (specs) {
case TapChecker.TAP13:
this.strategy = new TAP13ValidationStrategy();
break;
default:
this.strategy = new TAP14ValidationStrategy();
}
}

check(ast) {
return this.strategy.validate(ast);
}
}

module.exports = {
TapChecker,
TAP14ValidationStrategy,
TAP13ValidationStrategy,
};
Loading

0 comments on commit 87c902f

Please sign in to comment.