From 93faa202368c2d6a02955f6469c3a9c3c149e8ab Mon Sep 17 00:00:00 2001 From: Moshe Atlow Date: Mon, 11 Sep 2023 21:26:48 +0300 Subject: [PATCH 1/2] test_runner: add junit reporter --- lib/internal/test_runner/reporter/junit.js | 158 ++++++ lib/internal/test_runner/utils.js | 1 + lib/test/reporters.js | 10 + .../test-runner/output/junit_reporter.js | 7 + .../output/junit_reporter.snapshot | 488 ++++++++++++++++++ test/parallel/test-runner-output.mjs | 16 + 6 files changed, 680 insertions(+) create mode 100644 lib/internal/test_runner/reporter/junit.js create mode 100644 test/fixtures/test-runner/output/junit_reporter.js create mode 100644 test/fixtures/test-runner/output/junit_reporter.snapshot diff --git a/lib/internal/test_runner/reporter/junit.js b/lib/internal/test_runner/reporter/junit.js new file mode 100644 index 00000000000000..b45c233861c000 --- /dev/null +++ b/lib/internal/test_runner/reporter/junit.js @@ -0,0 +1,158 @@ +'use strict'; +const { + ArrayPrototypeFilter, + ArrayPrototypeMap, + ArrayPrototypeJoin, + ArrayPrototypePush, + ArrayPrototypeSome, + NumberPrototypeToFixed, + ObjectEntries, + RegExpPrototypeSymbolReplace, + String, + StringPrototypeRepeat, +} = primordials; + +const { inspectWithNoCustomRetry } = require('internal/errors'); +const { hostname } = require('os'); + +const inspectOptions = { __proto__: null, colors: false, breakLength: Infinity }; +const HOSTNAME = hostname(); + +function escapeAttribute(s = '') { + return escapeContent(RegExpPrototypeSymbolReplace(/"/g, RegExpPrototypeSymbolReplace(/\n/g, s, ''), '"')); +} + +function escapeContent(s = '') { + return RegExpPrototypeSymbolReplace(/\n`; + } + const attrsString = ArrayPrototypeJoin(ArrayPrototypeMap(ObjectEntries(attrs) + , ({ 0: key, 1: value }) => `${key}="${escapeAttribute(String(value))}"`) + , ' '); + if (!children?.length) { + return `${indent}<${tag} ${attrsString}/>\n`; + } + const childrenString = ArrayPrototypeJoin(ArrayPrototypeMap(children ?? [], treeToXML), ''); + return `${indent}<${tag} ${attrsString}>\n${childrenString}${indent}\n`; +} + +function isFailure(node) { + return (node?.children && ArrayPrototypeSome(node.children, (c) => c.tag === 'failure')) || node?.attrs?.failures; +} + +function isSkipped(node) { + return (node?.children && ArrayPrototypeSome(node.children, (c) => c.tag === 'skipped')) || node?.attrs?.failures; +} + +module.exports = async function* junitReporter(source) { + yield '\n'; + yield '\n'; + let currentSuite = null; + const roots = []; + + function startTest(event) { + const originalSuite = currentSuite; + currentSuite = { + __proto__: null, + attrs: { __proto__: null, name: event.data.name }, + nesting: event.data.nesting, + parent: currentSuite, + children: [], + }; + if (originalSuite?.children) { + ArrayPrototypePush(originalSuite.children, currentSuite); + } + if (!currentSuite.parent) { + ArrayPrototypePush(roots, currentSuite); + } + } + + for await (const event of source) { + switch (event.type) { + case 'test:start': { + startTest(event); + break; + } + case 'test:pass': + case 'test:fail': { + if (!currentSuite) { + startTest({ __proto__: null, data: { __proto__: null, name: 'root', nesting: 0 } }); + } + if (currentSuite.attrs.name !== event.data.name || + currentSuite.nesting !== event.data.nesting) { + startTest(event); + } + const currentTest = currentSuite; + if (currentSuite?.nesting === event.data.nesting) { + currentSuite = currentSuite.parent; + } + currentTest.attrs.time = NumberPrototypeToFixed(event.data.details.duration_ms / 1000, 6); + const nonCommentChildren = ArrayPrototypeFilter(currentTest.children, (c) => c.comment == null); + if (nonCommentChildren.length > 0) { + currentTest.tag = 'testsuite'; + currentTest.attrs.disabled = 0; + currentTest.attrs.errors = 0; + currentTest.attrs.tests = nonCommentChildren.length; + currentTest.attrs.failures = ArrayPrototypeFilter(currentTest.children, isFailure).length; + currentTest.attrs.skipped = ArrayPrototypeFilter(currentTest.children, isSkipped).length; + currentTest.attrs.hostname = HOSTNAME; + } else { + currentTest.tag = 'testcase'; + currentTest.attrs.classname = event.data.classname ?? 'test'; + if (event.data.skip) { + ArrayPrototypePush(currentTest.children, { + __proto__: null, nesting: event.data.nesting + 1, tag: 'skipped', + attrs: { __proto__: null, type: 'skipped', message: event.data.skip }, + }); + } + if (event.data.todo) { + ArrayPrototypePush(currentTest.children, { + __proto__: null, nesting: event.data.nesting + 1, tag: 'skipped', + attrs: { __proto__: null, type: 'todo', message: event.data.todo }, + }); + } + if (event.type === 'test:fail') { + const error = event.data.details?.error; + currentTest.children.push({ + __proto__: null, + nesting: event.data.nesting + 1, + tag: 'failure', + attrs: { __proto__: null, type: error?.failureType || error?.code, message: error?.message ?? '' }, + children: [inspectWithNoCustomRetry(error, inspectOptions)], + }); + currentTest.failures = 1; + currentTest.attrs.failure = error?.message ?? ''; + } + } + break; + } + case 'test:diagnostic': { + const parent = currentSuite?.children ?? roots; + ArrayPrototypePush(parent, { + __proto__: null, nesting: event.data.nesting, comment: event.data.message, + }); + break; + } default: + break; + } + } + for (const suite of roots) { + yield treeToXML(suite); + } + yield '\n'; +}; diff --git a/lib/internal/test_runner/utils.js b/lib/internal/test_runner/utils.js index 7923732f04dfcb..971534af3e8e8b 100644 --- a/lib/internal/test_runner/utils.js +++ b/lib/internal/test_runner/utils.js @@ -110,6 +110,7 @@ const kBuiltinReporters = new SafeMap([ ['spec', 'internal/test_runner/reporter/spec'], ['dot', 'internal/test_runner/reporter/dot'], ['tap', 'internal/test_runner/reporter/tap'], + ['junit', 'internal/test_runner/reporter/junit'], ]); const kDefaultReporter = process.stdout.isTTY ? 'spec' : 'tap'; diff --git a/lib/test/reporters.js b/lib/test/reporters.js index 86aea679b52a7a..06a0b27ee58275 100644 --- a/lib/test/reporters.js +++ b/lib/test/reporters.js @@ -3,6 +3,7 @@ const { ObjectDefineProperties, ReflectConstruct } = primordials; let dot; +let junit; let spec; let tap; @@ -17,6 +18,15 @@ ObjectDefineProperties(module.exports, { return dot; }, }, + junit: { + __proto__: null, + configurable: true, + enumerable: true, + get() { + junit ??= require('internal/test_runner/reporter/junit'); + return junit; + }, + }, spec: { __proto__: null, configurable: true, diff --git a/test/fixtures/test-runner/output/junit_reporter.js b/test/fixtures/test-runner/output/junit_reporter.js new file mode 100644 index 00000000000000..1f49b3f6042d97 --- /dev/null +++ b/test/fixtures/test-runner/output/junit_reporter.js @@ -0,0 +1,7 @@ +'use strict'; +require('../../../common'); +const fixtures = require('../../../common/fixtures'); +const spawn = require('node:child_process').spawn; + +spawn(process.execPath, + ['--no-warnings', '--test-reporter', 'junit', fixtures.path('test-runner/output/output.js')], { stdio: 'inherit' }); diff --git a/test/fixtures/test-runner/output/junit_reporter.snapshot b/test/fixtures/test-runner/output/junit_reporter.snapshot new file mode 100644 index 00000000000000..6516387e7ed582 --- /dev/null +++ b/test/fixtures/test-runner/output/junit_reporter.snapshot @@ -0,0 +1,488 @@ + + + + + + + + + + + +[Error [ERR_TEST_FAILURE]: thrown from sync fail todo] { + failureType: 'testCodeFailure', + cause: Error: thrown from sync fail todo + * + * + * + * + * + * + at async Test.processPendingSubtests (node:internal/test_runner/test:374:7), + code: 'ERR_TEST_FAILURE' +} + + + + + +[Error [ERR_TEST_FAILURE]: thrown from sync fail todo with message] { + failureType: 'testCodeFailure', + cause: Error: thrown from sync fail todo with message + * + * + * + * + * + * + at async Test.processPendingSubtests (node:internal/test_runner/test:374:7), + code: 'ERR_TEST_FAILURE' +} + + + + + + + + + + + + +[Error [ERR_TEST_FAILURE]: thrown from sync throw fail] { + failureType: 'testCodeFailure', + cause: Error: thrown from sync throw fail + * + * + * + * + * + * + at async Test.processPendingSubtests (node:internal/test_runner/test:374:7), + code: 'ERR_TEST_FAILURE' +} + + + + + + + + +[Error [ERR_TEST_FAILURE]: thrown from async throw fail] { + failureType: 'testCodeFailure', + cause: Error: thrown from async throw fail + * + * + * + * + * + * + at async Test.processPendingSubtests (node:internal/test_runner/test:374:7), + code: 'ERR_TEST_FAILURE' +} + + + + + +[Error [ERR_TEST_FAILURE]: thrown from async throw fail] { + failureType: 'testCodeFailure', + cause: Error: thrown from async throw fail + * + * + * + * + * + * + at async Test.processPendingSubtests (node:internal/test_runner/test:374:7), + code: 'ERR_TEST_FAILURE' +} + + + + +[Error [ERR_TEST_FAILURE]: Expected values to be strictly equal: + +true !== false +] { + failureType: 'testCodeFailure', + cause: AssertionError [ERR_ASSERTION]: Expected values to be strictly equal: + + true !== false + + * + * + * + * + * + * + * { + generatedMessage: true, + code: 'ERR_ASSERTION', + actual: true, + expected: false, + operator: 'strictEqual' + }, + code: 'ERR_TEST_FAILURE' +} + + + + + +[Error [ERR_TEST_FAILURE]: rejected from reject fail] { + failureType: 'testCodeFailure', + cause: Error: rejected from reject fail + * + * + * + * + * + * + at async Test.processPendingSubtests (node:internal/test_runner/test:374:7), + code: 'ERR_TEST_FAILURE' +} + + + + + + + + + + +Error [ERR_TEST_FAILURE]: thrown from subtest sync throw fail + * { + failureType: 'testCodeFailure', + cause: Error: thrown from subtest sync throw fail + * + * + * + * + * + * + * + * + * + at Test.postRun (node:internal/test_runner/test:715:19), + code: 'ERR_TEST_FAILURE' +} + + + + + + +[Error [ERR_TEST_FAILURE]: Symbol(thrown symbol from sync throw non-error fail)] { failureType: 'testCodeFailure', cause: Symbol(thrown symbol from sync throw non-error fail), code: 'ERR_TEST_FAILURE' } + + + + + + + + + + + +[Error [ERR_TEST_FAILURE]: test did not finish before its parent and was cancelled] { failureType: 'cancelledByParent', cause: 'test did not finish before its parent and was cancelled', code: 'ERR_TEST_FAILURE' } + + + + + + + + + + + + + + + +[Error [ERR_TEST_FAILURE]: this should be executed] { + failureType: 'testCodeFailure', + cause: Error: this should be executed + * + * + * + * + * + * + at async Test.processPendingSubtests (node:internal/test_runner/test:374:7), + code: 'ERR_TEST_FAILURE' +} + + + + + + + + + + + + + + + + + + + +[Error [ERR_TEST_FAILURE]: callback failure] { + failureType: 'testCodeFailure', + cause: Error: callback failure + * + at process.processImmediate (node:internal/timers:478:21), + code: 'ERR_TEST_FAILURE' +} + + + + + + + +[Error [ERR_TEST_FAILURE]: passed a callback but also returned a Promise] { failureType: 'callbackAndPromisePresent', cause: 'passed a callback but also returned a Promise', code: 'ERR_TEST_FAILURE' } + + + + +[Error [ERR_TEST_FAILURE]: thrown from callback throw] { + failureType: 'testCodeFailure', + cause: Error: thrown from callback throw + * + * + * + * + * + * + at async Test.processPendingSubtests (node:internal/test_runner/test:374:7), + code: 'ERR_TEST_FAILURE' +} + + + + +Error [ERR_TEST_FAILURE]: callback invoked multiple times + * + * { + failureType: 'multipleCallbackInvocations', + cause: 'callback invoked multiple times', + code: 'ERR_TEST_FAILURE' +} + + + + + +Error [ERR_TEST_FAILURE]: callback invoked multiple times + * { + failureType: 'uncaughtException', + cause: Error [ERR_TEST_FAILURE]: callback invoked multiple times + * { + failureType: 'multipleCallbackInvocations', + cause: 'callback invoked multiple times', + code: 'ERR_TEST_FAILURE' + }, + code: 'ERR_TEST_FAILURE' +} + + + + +Error [ERR_TEST_FAILURE]: thrown from callback async throw + * { + failureType: 'uncaughtException', + cause: Error: thrown from callback async throw + * + at process.processImmediate (node:internal/timers:478:21), + code: 'ERR_TEST_FAILURE' +} + + + + + + + + + + + + + + +[Error [ERR_TEST_FAILURE]: customized] { failureType: 'testCodeFailure', cause: customized, code: 'ERR_TEST_FAILURE' } + + + + +[Error [ERR_TEST_FAILURE]: { + foo: 1, + [Symbol(nodejs.util.inspect.custom)]: [Function: [nodejs.util.inspect.custom]] +}] { + failureType: 'testCodeFailure', + cause: { foo: 1, [Symbol(nodejs.util.inspect.custom)]: [Function: [nodejs.util.inspect.custom]] }, + code: 'ERR_TEST_FAILURE' +} + + + + + +Error [ERR_TEST_FAILURE]: thrown from subtest sync throw fails at first + * { + failureType: 'testCodeFailure', + cause: Error: thrown from subtest sync throw fails at first + * + * + * + * + * + * + * + * + * + at Test.postRun (node:internal/test_runner/test:715:19), + code: 'ERR_TEST_FAILURE' +} + + + + +Error [ERR_TEST_FAILURE]: thrown from subtest sync throw fails at second + * { + failureType: 'testCodeFailure', + cause: Error: thrown from subtest sync throw fails at second + * + * + * + * + * + * + * + * + * + at async Test.run (node:internal/test_runner/test:632:9), + code: 'ERR_TEST_FAILURE' +} + + + + + +[Error [ERR_TEST_FAILURE]: test timed out after 5ms] { failureType: 'testTimeoutFailure', cause: 'test timed out after 5ms', code: 'ERR_TEST_FAILURE' } + + + + +[Error [ERR_TEST_FAILURE]: test timed out after 5ms] { failureType: 'testTimeoutFailure', cause: 'test timed out after 5ms', code: 'ERR_TEST_FAILURE' } + + + + + + + +[Error [ERR_TEST_FAILURE]: custom error] { failureType: 'testCodeFailure', cause: 'custom error', code: 'ERR_TEST_FAILURE' } + + + + +Error [ERR_TEST_FAILURE]: foo + * { + failureType: 'uncaughtException', + cause: Error: foo + * + * + at process.processTimers (node:internal/timers:514:7), + code: 'ERR_TEST_FAILURE' +} + + + + +Error [ERR_TEST_FAILURE]: bar + * { + failureType: 'unhandledRejection', + cause: Error: bar + * + * + at process.processTimers (node:internal/timers:514:7), + code: 'ERR_TEST_FAILURE' +} + + + + +[Error [ERR_TEST_FAILURE]: Expected values to be loosely deep-equal: + +{ + bar: 1, + foo: 1 +} + +should loosely deep-equal + +<ref *1> { + bar: 2, + c: [Circular *1] +}] { + failureType: 'testCodeFailure', + cause: AssertionError [ERR_ASSERTION]: Expected values to be loosely deep-equal: + + { + bar: 1, + foo: 1 + } + + should loosely deep-equal + + <ref *1> { + bar: 2, + c: [Circular *1] + } + * { + generatedMessage: true, + code: 'ERR_ASSERTION', + actual: [Object], + expected: [Object], + operator: 'deepEqual' + }, + code: 'ERR_TEST_FAILURE' +} + + + + +Error [ERR_TEST_FAILURE]: test could not be started because its parent finished + * { + failureType: 'parentAlreadyFinished', + cause: 'test could not be started because its parent finished', + code: 'ERR_TEST_FAILURE' +} + + + + + + + + + + + + + + + + + + diff --git a/test/parallel/test-runner-output.mjs b/test/parallel/test-runner-output.mjs index a45ac62d5f0eb7..fff6fed92655e9 100644 --- a/test/parallel/test-runner-output.mjs +++ b/test/parallel/test-runner-output.mjs @@ -2,6 +2,7 @@ import * as common from '../common/index.mjs'; import * as fixtures from '../common/fixtures.mjs'; import * as snapshot from '../common/assertSnapshot.js'; import { describe, it } from 'node:test'; +import { hostname } from 'node:os'; const skipForceColors = process.config.variables.icu_gyp_path !== 'tools/icu/icu-generic.gyp' || @@ -25,6 +26,15 @@ function replaceSpecDuration(str) { .replace(stackTraceBasePath, '$3'); } +function replaceJunitDuration(str) { + return str + .replaceAll(/time="0"/g, 'time="ZERO"') + .replaceAll(/time="[0-9.]+"/g, 'time="*"') + .replaceAll(/duration_ms [0-9.]+/g, 'duration_ms *') + .replaceAll(hostname(), 'HOSTNAME') + .replace(stackTraceBasePath, '$3'); +} + function removeWindowsPathEscaping(str) { return common.isWindows ? str.replaceAll(/\\\\/g, '\\') : str; } @@ -47,6 +57,11 @@ const specTransform = snapshot.transform( snapshot.replaceWindowsLineEndings, snapshot.replaceStackTrace, ); +const junitTransform = snapshot.transform( + replaceJunitDuration, + snapshot.replaceWindowsLineEndings, + snapshot.replaceStackTrace, +); const tests = [ { name: 'test-runner/output/abort.js' }, @@ -64,6 +79,7 @@ const tests = [ { name: 'test-runner/output/no_tests.js' }, { name: 'test-runner/output/only_tests.js' }, { name: 'test-runner/output/dot_reporter.js' }, + { name: 'test-runner/output/junit_reporter.js', transform: junitTransform }, { name: 'test-runner/output/spec_reporter_successful.js', transform: specTransform }, { name: 'test-runner/output/spec_reporter.js', transform: specTransform }, { name: 'test-runner/output/spec_reporter_cli.js', transform: specTransform }, From e89bc87edec334c1705d57a479acd8efbb54e1f6 Mon Sep 17 00:00:00 2001 From: Moshe Atlow Date: Thu, 14 Sep 2023 08:42:07 +0300 Subject: [PATCH 2/2] add docs --- doc/api/test.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/doc/api/test.md b/doc/api/test.md index cbc7083bbbd0ec..d84d4c9f81a501 100644 --- a/doc/api/test.md +++ b/doc/api/test.md @@ -632,6 +632,9 @@ The following built-reporters are supported: where each passing test is represented by a `.`, and each failing test is represented by a `X`. +* `junit` + The junit reporter outputs test results in a jUnit XML format + When `stdout` is a [TTY][], the `spec` reporter is used by default. Otherwise, the `tap` reporter is used by default. @@ -643,11 +646,11 @@ to the test runner's output is required, use the events emitted by the The reporters are available via the `node:test/reporters` module: ```mjs -import { tap, spec, dot } from 'node:test/reporters'; +import { tap, spec, dot, junit } from 'node:test/reporters'; ``` ```cjs -const { tap, spec, dot } = require('node:test/reporters'); +const { tap, spec, dot, junit } = require('node:test/reporters'); ``` ### Custom reporters