test_runner: report coverage thresholds in test:coverage

nodejs · Sep 8, 2024 · c411a61 · c411a61
1 parent b77476d
commit c411a61
Show file tree

Hide file tree

Showing 4 changed files with 55 additions and 15 deletions.
diff --git a/doc/api/test.md b/doc/api/test.md
@@ -2814,6 +2814,11 @@ are defined, while others are emitted in the order that the tests execute.
         numbers and the number of times they were covered.
         * `line` {number} The line number.
         * `count` {number} The number of times the line was covered.
+    * `thresholds` {Object} An object containing whether or not the coverage for
+      each coverage type.
+      * `function` {number} The user-specified `function` threshold value.
+      * `branch` {number} The user-specified `branch` threshold value.
+      * `line` {number} The user-specified `line` threshold value.
     * `totals` {Object} An object containing a summary of coverage for all
       files.
       * `totalLineCount` {number} The total number of lines.

diff --git a/lib/internal/test_runner/test.js b/lib/internal/test_runner/test.js
@@ -1010,8 +1010,6 @@ class Test extends AsyncResource {
       reporter.diagnostic(nesting, loc, `duration_ms ${this.duration()}`);
 
       if (coverage) {
-        reporter.coverage(nesting, loc, coverage);
-
         const coverages = [
           { __proto__: null, actual: coverage.totals.coveredLinePercent,
             threshold: this.config.lineCoverage, name: 'line' },
@@ -1023,13 +1021,17 @@ class Test extends AsyncResource {
             threshold: this.config.functionCoverage, name: 'function' },
         ];
 
+        coverage.thresholds = { __proto__: null };
         for (let i = 0; i < coverages.length; i++) {
           const { threshold, actual, name } = coverages[i];
+          coverage.thresholds[name] = threshold;
           if (actual < threshold) {
             process.exitCode = kGenericUserError;
             reporter.diagnostic(nesting, loc, `Error: ${NumberPrototypeToFixed(actual, 2)}% ${name} coverage does not meet threshold of ${threshold}%.`);
           }
         }
+
+        reporter.coverage(nesting, loc, coverage);
       }
 
       if (harness.watching) {

diff --git a/test/parallel/test-runner-coverage-thresholds.js b/test/parallel/test-runner-coverage-thresholds.js
@@ -44,10 +44,7 @@ function getTapCoverageFixtureReport() {
 }
 
 const fixture = fixtures.path('test-runner', 'coverage.js');
-const neededArguments = [
-  '--experimental-test-coverage',
-  '--test-reporter', 'tap',
-];
+const reporter = fixtures.fileURL('test-runner/custom_reporters/coverage.mjs');
 
 const coverages = [
   { flag: '--test-coverage-lines', name: 'line', actual: 78.35 },
@@ -56,10 +53,12 @@ const coverages = [
 ];
 
 for (const coverage of coverages) {
-  test(`test passing ${coverage.flag}`, async (t) => {
+  test(`test passing ${coverage.flag}`, () => {
     const result = spawnSync(process.execPath, [
-      ...neededArguments,
+      '--test',
+      '--experimental-test-coverage',
       `${coverage.flag}=25`,
+      '--test-reporter', 'tap',
       fixture,
     ]);
 
@@ -70,10 +69,27 @@ for (const coverage of coverages) {
     assert(!findCoverageFileForPid(result.pid));
   });
 
-  test(`test failing ${coverage.flag}`, async (t) => {
+  test(`test passing ${coverage.flag} with custom reporter`, () => {
+    const result = spawnSync(process.execPath, [
+      '--test',
+      '--experimental-test-coverage',
+      `${coverage.flag}=25`,
+      '--test-reporter', reporter,
+      fixture,
+    ]);
+
+    const stdout = JSON.parse(result.stdout.toString());
+    assert.strictEqual(stdout.summary.thresholds[coverage.name], 25);
+    assert.strictEqual(result.status, 0);
+    assert(!findCoverageFileForPid(result.pid));
+  });
+
+  test(`test failing ${coverage.flag}`, () => {
     const result = spawnSync(process.execPath, [
-      ...neededArguments,
+      '--test',
+      '--experimental-test-coverage',
       `${coverage.flag}=99`,
+      '--test-reporter', 'tap',
       fixture,
     ]);
 
@@ -84,9 +100,25 @@ for (const coverage of coverages) {
     assert(!findCoverageFileForPid(result.pid));
   });
 
-  test(`test out-of-range ${coverage.flag} (too high)`, async (t) => {
+  test(`test failing ${coverage.flag} with custom reporter`, () => {
+    const result = spawnSync(process.execPath, [
+      '--test',
+      '--experimental-test-coverage',
+      `${coverage.flag}=99`,
+      '--test-reporter', reporter,
+      fixture,
+    ]);
+
+    const stdout = JSON.parse(result.stdout.toString());
+    assert.strictEqual(stdout.summary.thresholds[coverage.name], 99);
+    assert.strictEqual(result.status, 1);
+    assert(!findCoverageFileForPid(result.pid));
+  });
+
+  test(`test out-of-range ${coverage.flag} (too high)`, () => {
     const result = spawnSync(process.execPath, [
-      ...neededArguments,
+      '--test',
+      '--experimental-test-coverage',
       `${coverage.flag}=101`,
       fixture,
     ]);
@@ -96,9 +128,10 @@ for (const coverage of coverages) {
     assert(!findCoverageFileForPid(result.pid));
   });
 
-  test(`test out-of-range ${coverage.flag} (too low)`, async (t) => {
+  test(`test out-of-range ${coverage.flag} (too low)`, () => {
     const result = spawnSync(process.execPath, [
-      ...neededArguments,
+      '--test',
+      '--experimental-test-coverage',
       `${coverage.flag}=-1`,
       fixture,
     ]);

diff --git a/test/parallel/test-runner-coverage.js b/test/parallel/test-runner-coverage.js
@@ -467,7 +467,7 @@ test('coverage with included and excluded files', skipIfNoInspector, () => {
   assert(!findCoverageFileForPid(result.pid));
 });
 
-test('properly accounts for line endings in source maps', skipIfNoInspector, () => {
+test.only('properly accounts for line endings in source maps', skipIfNoInspector, () => {
   const fixture = fixtures.path('test-runner', 'source-map-line-lengths', 'index.js');
   const args = [
     '--test', '--experimental-test-coverage', '--test-reporter', 'tap',