Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

test_runner: support glob matching coverage files #53553

Merged
merged 1 commit into from
Jul 14, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 36 additions & 0 deletions doc/api/cli.md
Original file line number Diff line number Diff line change
Expand Up @@ -490,6 +490,40 @@ For example, to run a module with "development" resolutions:
node -C development app.js
RedYetiDev marked this conversation as resolved.
Show resolved Hide resolved
```

### `--test-coverage-exclude`

<!-- YAML
added:
- REPLACEME
-->

> Stability: 1 - Experimental

Excludes specific files from code coverage using a glob pattern, which can match
both absolute and relative file paths.

This option may be specified multiple times to exclude multiple glob patterns.

If both `--test-coverage-exclude` and `--test-coverage-include` are provided,
files must meet **both** criteria to be included in the coverage report.

### `--test-coverage-include`

<!-- YAML
added:
- REPLACEME
-->

> Stability: 1 - Experimental

Includes specific files in code coverage using a glob pattern, which can match
both absolute and relative file paths.

This option may be specified multiple times to include multiple glob patterns.

If both `--test-coverage-exclude` and `--test-coverage-include` are provided,
files must meet **both** criteria to be included in the coverage report.

### `--cpu-prof`

<!-- YAML
Expand Down Expand Up @@ -2908,6 +2942,8 @@ one is included in the list below.
* `--secure-heap-min`
* `--secure-heap`
* `--snapshot-blob`
* `--test-coverage-exclude`
* `--test-coverage-include`
* `--test-only`
* `--test-reporter-destination`
* `--test-reporter`
Expand Down
5 changes: 0 additions & 5 deletions doc/api/test.md
Original file line number Diff line number Diff line change
Expand Up @@ -511,11 +511,6 @@ node --test --experimental-test-coverage --test-reporter=lcov --test-reporter-de
* No test results are reported by this reporter.
* This reporter should ideally be used alongside another reporter.

### Limitations

The test runner's code coverage functionality does not support excluding
specific files or directories from the coverage report.

## Mocking

The `node:test` module supports mocking during testing via a top-level `mock`
Expand Down
6 changes: 6 additions & 0 deletions doc/node.1
Original file line number Diff line number Diff line change
Expand Up @@ -435,6 +435,12 @@ Starts the Node.js command line test runner.
The maximum number of test files that the test runner CLI will execute
concurrently.
.
.It Fl -test-coverage-exclude
A glob pattern that excludes matching files from the coverage report
.
.It Fl -test-coverage-include
A glob pattern that only includes matching files in the coverage report
.
.It Fl -test-force-exit
Configures the test runner to exit the process once all known tests have
finished executing even if the event loop would otherwise remain active.
Expand Down
45 changes: 34 additions & 11 deletions lib/internal/test_runner/coverage.js
Original file line number Diff line number Diff line change
Expand Up @@ -25,15 +25,18 @@ const {
readFileSync,
} = require('fs');
const { setupCoverageHooks } = require('internal/util');
const { getOptionValue } = require('internal/options');
const { tmpdir } = require('os');
const { join, resolve } = require('path');
const { join, resolve, relative, matchesGlob } = require('path');
const { fileURLToPath } = require('internal/url');
const { kMappings, SourceMap } = require('internal/source_map/source_map');
const kCoverageFileRegex = /^coverage-(\d+)-(\d{13})-(\d+)\.json$/;
const kIgnoreRegex = /\/\* node:coverage ignore next (?<count>\d+ )?\*\//;
const kLineEndingRegex = /\r?\n$/u;
const kLineSplitRegex = /(?<=\r?\n)/u;
const kStatusRegex = /\/\* node:coverage (?<status>enable|disable) \*\//;
const excludeFileGlobs = getOptionValue('--test-coverage-exclude');
const includeFileGlobs = getOptionValue('--test-coverage-include');

class CoverageLine {
constructor(line, startOffset, src, length = src?.length) {
Expand Down Expand Up @@ -308,7 +311,7 @@ class TestCoverage {

const coverageFile = join(this.coverageDirectory, entry.name);
const coverage = JSONParse(readFileSync(coverageFile, 'utf8'));
mergeCoverage(result, this.mapCoverageWithSourceMap(coverage));
mergeCoverage(result, this.mapCoverageWithSourceMap(coverage), this.workingDirectory);
}

return ArrayFrom(result.values());
Expand All @@ -331,7 +334,7 @@ class TestCoverage {
const script = result[i];
const { url, functions } = script;

if (shouldSkipFileCoverage(url) || sourceMapCache[url] == null) {
if (shouldSkipFileCoverage(url, this.workingDirectory) || sourceMapCache[url] == null) {
newResult.set(url, script);
continue;
}
Expand Down Expand Up @@ -485,22 +488,42 @@ function mapRangeToLines(range, lines) {
return { __proto__: null, lines: mappedLines, ignoredLines };
}

function shouldSkipFileCoverage(url) {
// The first part of this check filters out the node_modules/ directory
// from the results. This filter is applied first because most real world
// applications will be dominated by third party dependencies. The second
// part of the check filters out core modules, which start with 'node:' in
function shouldSkipFileCoverage(url, workingDirectory) {
// This check filters out core modules, which start with 'node:' in
// coverage reports, as well as any invalid coverages which have been
// observed on Windows.
return StringPrototypeIncludes(url, '/node_modules/') || !StringPrototypeStartsWith(url, 'file:');
if (!StringPrototypeStartsWith(url, 'file:')) return true;

const absolutePath = fileURLToPath(url);
const relativePath = relative(workingDirectory, absolutePath);

// This check filters out files that match the exclude globs.
if (excludeFileGlobs?.length > 0) {
for (let i = 0; i < excludeFileGlobs.length; ++i) {
if (matchesGlob(relativePath, excludeFileGlobs[i]) ||
matchesGlob(absolutePath, excludeFileGlobs[i])) return true;
}
}

// This check filters out files that do not match the include globs.
if (includeFileGlobs?.length > 0) {
for (let i = 0; i < includeFileGlobs.length; ++i) {
if (matchesGlob(relativePath, includeFileGlobs[i]) ||
matchesGlob(absolutePath, includeFileGlobs[i])) return false;
}
return true;
}

// This check filters out the node_modules/ directory, unless it is explicitly included.
return StringPrototypeIncludes(url, '/node_modules/');
}

function mergeCoverage(merged, coverage) {
function mergeCoverage(merged, coverage, workingDirectory) {
for (let i = 0; i < coverage.length; ++i) {
const newScript = coverage[i];
const { url } = newScript;

if (shouldSkipFileCoverage(url)) {
if (shouldSkipFileCoverage(url, workingDirectory)) {
continue;
}

Expand Down
8 changes: 8 additions & 0 deletions src/node_options.cc
Original file line number Diff line number Diff line change
Expand Up @@ -670,6 +670,14 @@ EnvironmentOptionsParser::EnvironmentOptionsParser() {
AddOption("--test-skip-pattern",
"run tests whose name do not match this regular expression",
&EnvironmentOptions::test_skip_pattern);
AddOption("--test-coverage-include",
"include files in coverage report that match this glob pattern",
&EnvironmentOptions::coverage_include_pattern,
kAllowedInEnvvar);
AddOption("--test-coverage-exclude",
"exclude files from coverage report that match this glob pattern",
&EnvironmentOptions::coverage_exclude_pattern,
kAllowedInEnvvar);
AddOption("--test-udp-no-try-send", "", // For testing only.
&EnvironmentOptions::test_udp_no_try_send);
AddOption("--throw-deprecation",
Expand Down
2 changes: 2 additions & 0 deletions src/node_options.h
Original file line number Diff line number Diff line change
Expand Up @@ -186,6 +186,8 @@ class EnvironmentOptions : public Options {
bool test_udp_no_try_send = false;
std::string test_shard;
std::vector<std::string> test_skip_pattern;
std::vector<std::string> coverage_include_pattern;
std::vector<std::string> coverage_exclude_pattern;
bool throw_deprecation = false;
bool trace_deprecation = false;
bool trace_exit = false;
Expand Down
93 changes: 93 additions & 0 deletions test/parallel/test-runner-coverage.js
Original file line number Diff line number Diff line change
Expand Up @@ -335,3 +335,96 @@ test('coverage with ESM hook - source transpiled', skipIfNoInspector, () => {
assert(result.stdout.toString().includes(report));
assert.strictEqual(result.status, 0);
});

test('coverage with excluded files', skipIfNoInspector, () => {
const fixture = fixtures.path('test-runner', 'coverage.js');
const args = [
'--experimental-test-coverage', '--test-reporter', 'tap',
'--test-coverage-exclude=test/*/test-runner/invalid-tap.js',
fixture];
const result = spawnSync(process.execPath, args);
const report = [
'# start of coverage report',
'# ' + '-'.repeat(112),
'# file | line % | branch % | funcs % | uncovered lines',
'# ' + '-'.repeat(112),
'# test/fixtures/test-runner/coverage.js | 78.65 | 38.46 | 60.00 | 12-13 16-22 27 39 43-44 61-62 66-67 71-72',
'# test/fixtures/v8-coverage/throw.js | 71.43 | 50.00 | 100.00 | 5-6',
'# ' + '-'.repeat(112),
'# all files | 78.13 | 40.00 | 60.00 |',
'# ' + '-'.repeat(112),
'# end of coverage report',
].join('\n');


if (common.isWindows) {
return report.replaceAll('/', '\\');
}

assert(result.stdout.toString().includes(report));
assert.strictEqual(result.status, 0);
assert(!findCoverageFileForPid(result.pid));
});

test('coverage with included files', skipIfNoInspector, () => {
const fixture = fixtures.path('test-runner', 'coverage.js');
const args = [
'--experimental-test-coverage', '--test-reporter', 'tap',
'--test-coverage-include=test/fixtures/test-runner/coverage.js',
'--test-coverage-include=test/fixtures/v8-coverage/throw.js',
fixture,
];
const result = spawnSync(process.execPath, args);
const report = [
'# start of coverage report',
'# ' + '-'.repeat(112),
'# file | line % | branch % | funcs % | uncovered lines',
'# ' + '-'.repeat(112),
'# test/fixtures/test-runner/coverage.js | 78.65 | 38.46 | 60.00 | 12-13 16-22 27 39 43-44 61-62 66-67 71-72',
'# test/fixtures/v8-coverage/throw.js | 71.43 | 50.00 | 100.00 | 5-6',
'# ' + '-'.repeat(112),
'# all files | 78.13 | 40.00 | 60.00 |',
'# ' + '-'.repeat(112),
'# end of coverage report',
].join('\n');


if (common.isWindows) {
return report.replaceAll('/', '\\');
}

assert(result.stdout.toString().includes(report));
assert.strictEqual(result.status, 0);
assert(!findCoverageFileForPid(result.pid));
});

test('coverage with included and excluded files', skipIfNoInspector, () => {
const fixture = fixtures.path('test-runner', 'coverage.js');
const args = [
'--experimental-test-coverage', '--test-reporter', 'tap',
'--test-coverage-include=test/fixtures/test-runner/*.js',
'--test-coverage-exclude=test/fixtures/test-runner/*-tap.js',
fixture,
];
const result = spawnSync(process.execPath, args);
const report = [
'# start of coverage report',
'# ' + '-'.repeat(112),
'# file | line % | branch % | funcs % | uncovered lines',
'# ' + '-'.repeat(112),
'# test/fixtures/test-runner/coverage.js | 78.65 | 38.46 | 60.00 | 12-13 16-22 27 39 43-44 61-62 66-67 71-72',
'# ' + '-'.repeat(112),
'# all files | 78.65 | 38.46 | 60.00 |',
'# ' + '-'.repeat(112),
'# end of coverage report',
].join('\n');


if (common.isWindows) {
return report.replaceAll('/', '\\');
}

assert(result.stdout.toString().includes(report));
assert.strictEqual(result.status, 0);
assert(!findCoverageFileForPid(result.pid));
});