From be7774120f1c1dc978f679e64697ce8bb805f83e Mon Sep 17 00:00:00 2001 From: RedYetiDev <38299977+RedYetiDev@users.noreply.github.com> Date: Sun, 8 Sep 2024 16:54:01 -0400 Subject: [PATCH 1/3] test_runner: report coverage thresholds in `test:coverage` --- doc/api/test.md | 5 ++ lib/internal/test_runner/test.js | 6 +- .../test-runner-coverage-thresholds.js | 57 +++++++++++++++---- 3 files changed, 54 insertions(+), 14 deletions(-) diff --git a/doc/api/test.md b/doc/api/test.md index 19819202d1d2e1..4a126787241cfd 100644 --- a/doc/api/test.md +++ b/doc/api/test.md @@ -2814,6 +2814,11 @@ are defined, while others are emitted in the order that the tests execute. numbers and the number of times they were covered. * `line` {number} The line number. * `count` {number} The number of times the line was covered. + * `thresholds` {Object} An object containing whether or not the coverage for + each coverage type. + * `function` {number} The user-specified `function` threshold value. + * `branch` {number} The user-specified `branch` threshold value. + * `line` {number} The user-specified `line` threshold value. * `totals` {Object} An object containing a summary of coverage for all files. * `totalLineCount` {number} The total number of lines. diff --git a/lib/internal/test_runner/test.js b/lib/internal/test_runner/test.js index 09387f89c36c34..49064b9b9a314f 100644 --- a/lib/internal/test_runner/test.js +++ b/lib/internal/test_runner/test.js @@ -1010,8 +1010,6 @@ class Test extends AsyncResource { reporter.diagnostic(nesting, loc, `duration_ms ${this.duration()}`); if (coverage) { - reporter.coverage(nesting, loc, coverage); - const coverages = [ { __proto__: null, actual: coverage.totals.coveredLinePercent, threshold: this.config.lineCoverage, name: 'line' }, @@ -1023,13 +1021,17 @@ class Test extends AsyncResource { threshold: this.config.functionCoverage, name: 'function' }, ]; + coverage.thresholds = { __proto__: null }; for (let i = 0; i < coverages.length; i++) { const { threshold, actual, name } = coverages[i]; + coverage.thresholds[name] = threshold; if (actual < threshold) { process.exitCode = kGenericUserError; reporter.diagnostic(nesting, loc, `Error: ${NumberPrototypeToFixed(actual, 2)}% ${name} coverage does not meet threshold of ${threshold}%.`); } } + + reporter.coverage(nesting, loc, coverage); } if (harness.watching) { diff --git a/test/parallel/test-runner-coverage-thresholds.js b/test/parallel/test-runner-coverage-thresholds.js index 8ee15f4c65b7fe..a57231ea2e661a 100644 --- a/test/parallel/test-runner-coverage-thresholds.js +++ b/test/parallel/test-runner-coverage-thresholds.js @@ -44,10 +44,7 @@ function getTapCoverageFixtureReport() { } const fixture = fixtures.path('test-runner', 'coverage.js'); -const neededArguments = [ - '--experimental-test-coverage', - '--test-reporter', 'tap', -]; +const reporter = fixtures.fileURL('test-runner/custom_reporters/coverage.mjs'); const coverages = [ { flag: '--test-coverage-lines', name: 'line', actual: 78.35 }, @@ -56,10 +53,12 @@ const coverages = [ ]; for (const coverage of coverages) { - test(`test passing ${coverage.flag}`, async (t) => { + test(`test passing ${coverage.flag}`, () => { const result = spawnSync(process.execPath, [ - ...neededArguments, + '--test', + '--experimental-test-coverage', `${coverage.flag}=25`, + '--test-reporter', 'tap', fixture, ]); @@ -70,10 +69,27 @@ for (const coverage of coverages) { assert(!findCoverageFileForPid(result.pid)); }); - test(`test failing ${coverage.flag}`, async (t) => { + test(`test passing ${coverage.flag} with custom reporter`, () => { + const result = spawnSync(process.execPath, [ + '--test', + '--experimental-test-coverage', + `${coverage.flag}=25`, + '--test-reporter', reporter, + fixture, + ]); + + const stdout = JSON.parse(result.stdout.toString()); + assert.strictEqual(stdout.summary.thresholds[coverage.name], 25); + assert.strictEqual(result.status, 0); + assert(!findCoverageFileForPid(result.pid)); + }); + + test(`test failing ${coverage.flag}`, () => { const result = spawnSync(process.execPath, [ - ...neededArguments, + '--test', + '--experimental-test-coverage', `${coverage.flag}=99`, + '--test-reporter', 'tap', fixture, ]); @@ -84,9 +100,25 @@ for (const coverage of coverages) { assert(!findCoverageFileForPid(result.pid)); }); - test(`test out-of-range ${coverage.flag} (too high)`, async (t) => { + test(`test failing ${coverage.flag} with custom reporter`, () => { + const result = spawnSync(process.execPath, [ + '--test', + '--experimental-test-coverage', + `${coverage.flag}=99`, + '--test-reporter', reporter, + fixture, + ]); + + const stdout = JSON.parse(result.stdout.toString()); + assert.strictEqual(stdout.summary.thresholds[coverage.name], 99); + assert.strictEqual(result.status, 1); + assert(!findCoverageFileForPid(result.pid)); + }); + + test(`test out-of-range ${coverage.flag} (too high)`, () => { const result = spawnSync(process.execPath, [ - ...neededArguments, + '--test', + '--experimental-test-coverage', `${coverage.flag}=101`, fixture, ]); @@ -96,9 +128,10 @@ for (const coverage of coverages) { assert(!findCoverageFileForPid(result.pid)); }); - test(`test out-of-range ${coverage.flag} (too low)`, async (t) => { + test(`test out-of-range ${coverage.flag} (too low)`, () => { const result = spawnSync(process.execPath, [ - ...neededArguments, + '--test', + '--experimental-test-coverage', `${coverage.flag}=-1`, fixture, ]); From f86abda0375094c117ba31f104bb4ad7b4b2f3d7 Mon Sep 17 00:00:00 2001 From: Aviv Keller Date: Sun, 8 Sep 2024 17:07:42 -0400 Subject: [PATCH 2/3] Update doc/api/test.md Co-authored-by: Colin Ihrig --- doc/api/test.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/api/test.md b/doc/api/test.md index 4a126787241cfd..3bd790a32d5e38 100644 --- a/doc/api/test.md +++ b/doc/api/test.md @@ -2816,9 +2816,9 @@ are defined, while others are emitted in the order that the tests execute. * `count` {number} The number of times the line was covered. * `thresholds` {Object} An object containing whether or not the coverage for each coverage type. - * `function` {number} The user-specified `function` threshold value. - * `branch` {number} The user-specified `branch` threshold value. - * `line` {number} The user-specified `line` threshold value. + * `function` {number} The function coverage threshold. + * `branch` {number} The branch coverage threshold. + * `line` {number} The line coverage threshold. * `totals` {Object} An object containing a summary of coverage for all files. * `totalLineCount` {number} The total number of lines. From 45db682c4100d1ea03065941115ce80d0d1da3a4 Mon Sep 17 00:00:00 2001 From: RedYetiDev <38299977+RedYetiDev@users.noreply.github.com> Date: Sun, 8 Sep 2024 17:13:55 -0400 Subject: [PATCH 3/3] move to `coverage.js` --- lib/internal/test_runner/coverage.js | 10 +++++++++- lib/internal/test_runner/test.js | 2 -- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/lib/internal/test_runner/coverage.js b/lib/internal/test_runner/coverage.js index 7d78bd69886566..e4f0f79ebf3e24 100644 --- a/lib/internal/test_runner/coverage.js +++ b/lib/internal/test_runner/coverage.js @@ -50,12 +50,13 @@ class CoverageLine { } class TestCoverage { - constructor(coverageDirectory, originalCoverageDirectory, workingDirectory, excludeGlobs, includeGlobs) { + constructor(coverageDirectory, originalCoverageDirectory, workingDirectory, excludeGlobs, includeGlobs, thresholds) { this.coverageDirectory = coverageDirectory; this.originalCoverageDirectory = originalCoverageDirectory; this.workingDirectory = workingDirectory; this.excludeGlobs = excludeGlobs; this.includeGlobs = includeGlobs; + this.thresholds = thresholds; } #sourceLines = new SafeMap(); @@ -142,6 +143,7 @@ class TestCoverage { coveredBranchPercent: 0, coveredFunctionPercent: 0, }, + thresholds: this.thresholds, }; if (!coverage) { @@ -497,6 +499,12 @@ function setupCoverage(options) { cwd, options.coverageExcludeGlobs, options.coverageIncludeGlobs, + { + __proto__: null, + line: options.lineCoverage, + branch: options.branchCoverage, + function: options.functionCoverage, + }, ); } diff --git a/lib/internal/test_runner/test.js b/lib/internal/test_runner/test.js index 49064b9b9a314f..0d909c938e8e7b 100644 --- a/lib/internal/test_runner/test.js +++ b/lib/internal/test_runner/test.js @@ -1021,10 +1021,8 @@ class Test extends AsyncResource { threshold: this.config.functionCoverage, name: 'function' }, ]; - coverage.thresholds = { __proto__: null }; for (let i = 0; i < coverages.length; i++) { const { threshold, actual, name } = coverages[i]; - coverage.thresholds[name] = threshold; if (actual < threshold) { process.exitCode = kGenericUserError; reporter.diagnostic(nesting, loc, `Error: ${NumberPrototypeToFixed(actual, 2)}% ${name} coverage does not meet threshold of ${threshold}%.`);