From a0a677f75018ddcbdda99c84e3b0d5a7adeecc48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ari=20Perkki=C3=B6?= Date: Tue, 8 Oct 2024 19:09:02 +0300 Subject: [PATCH] refactor(coverage): move re-usable parts to base provider --- packages/coverage-istanbul/package.json | 1 + packages/coverage-istanbul/src/provider.ts | 354 ++---------- packages/coverage-v8/package.json | 1 + packages/coverage-v8/src/provider.ts | 330 ++---------- packages/vitest/src/utils/coverage.ts | 503 +++++++++++++----- pnpm-lock.yaml | 11 + .../test/threshold-auto-update.unit.test.ts | 20 +- 7 files changed, 460 insertions(+), 760 deletions(-) diff --git a/packages/coverage-istanbul/package.json b/packages/coverage-istanbul/package.json index fee3def67ed6..68d4d682898e 100644 --- a/packages/coverage-istanbul/package.json +++ b/packages/coverage-istanbul/package.json @@ -62,6 +62,7 @@ "@types/istanbul-lib-report": "^3.0.3", "@types/istanbul-lib-source-maps": "^4.0.4", "@types/istanbul-reports": "^3.0.4", + "@types/test-exclude": "^6.0.2", "pathe": "^1.1.2", "vitest": "workspace:*" } diff --git a/packages/coverage-istanbul/src/provider.ts b/packages/coverage-istanbul/src/provider.ts index bde4ddd84850..5db3408e03c4 100644 --- a/packages/coverage-istanbul/src/provider.ts +++ b/packages/coverage-istanbul/src/provider.ts @@ -1,133 +1,41 @@ -import { - existsSync, - promises as fs, - readdirSync, - writeFileSync, -} from 'node:fs' +import { promises as fs } from 'node:fs' import { resolve } from 'pathe' -import type { - AfterSuiteRunMeta, - CoverageIstanbulOptions, - CoverageProvider, - ReportContext, - ResolvedCoverageOptions, - Vitest, -} from 'vitest' -import { - coverageConfigDefaults, -} from 'vitest/config' +import type { CoverageProvider, ReportContext, ResolvedCoverageOptions, Vitest } from 'vitest/node' import { BaseCoverageProvider } from 'vitest/coverage' import c from 'tinyrainbow' import { parseModule } from 'magicast' import createDebug from 'debug' +import TestExclude from 'test-exclude' import libReport from 'istanbul-lib-report' import reports from 'istanbul-reports' -import type { CoverageMap } from 'istanbul-lib-coverage' -import libCoverage from 'istanbul-lib-coverage' +import libCoverage, { type CoverageMap } from 'istanbul-lib-coverage' import libSourceMaps from 'istanbul-lib-source-maps' import { type Instrumenter, createInstrumenter } from 'istanbul-lib-instrument' // @ts-expect-error missing types import { defaults as istanbulDefaults } from '@istanbuljs/schema' -// @ts-expect-error missing types -import _TestExclude from 'test-exclude' import { version } from '../package.json' with { type: 'json' } import { COVERAGE_STORE_KEY } from './constants' -type Options = ResolvedCoverageOptions<'istanbul'> - -/** - * Holds info about raw coverage results that are stored on file system: - * - * ```json - * "project-a": { - * "web": { - * "tests/math.test.ts": "coverage-1.json", - * "tests/utils.test.ts": "coverage-2.json", - * // ^^^^^^^^^^^^^^^ Raw coverage on file system - * }, - * "ssr": { ... }, - * "browser": { ... }, - * }, - * "project-b": ... - * ``` - */ -type CoverageFiles = Map< - NonNullable | typeof DEFAULT_PROJECT, - Record< - AfterSuiteRunMeta['transformMode'], - { [TestFilenames: string]: string } - > -> - -interface TestExclude { - new (opts: { - cwd?: string | string[] - include?: string | string[] - exclude?: string | string[] - extension?: string | string[] - excludeNodeModules?: boolean - relativePath?: boolean - }): { - shouldInstrument: (filePath: string) => boolean - glob: (cwd: string) => Promise - } -} - -const DEFAULT_PROJECT: unique symbol = Symbol.for('default-project') const debug = createDebug('vitest:coverage') -let uniqueId = 0 - -export class IstanbulCoverageProvider extends BaseCoverageProvider implements CoverageProvider { - name = 'istanbul' - ctx!: Vitest - options!: Options +export class IstanbulCoverageProvider extends BaseCoverageProvider> implements CoverageProvider { + name = 'istanbul' as const + version = version instrumenter!: Instrumenter - testExclude!: InstanceType - - coverageFiles: CoverageFiles = new Map() - coverageFilesDirectory!: string - pendingPromises: Promise[] = [] + testExclude!: InstanceType initialize(ctx: Vitest): void { - const config: CoverageIstanbulOptions = ctx.config.coverage - - if (ctx.version !== version) { - ctx.logger.warn( - c.yellow( - `Loaded ${c.inverse(c.yellow(` vitest@${ctx.version} `))} and ${c.inverse(c.yellow(` @vitest/coverage-istanbul@${version} `))}.` - + '\nRunning mixed versions is not supported and may lead into bugs' - + '\nUpdate your dependencies and make sure the versions match.', - ), - ) - } + this._initialize(ctx) - this.ctx = ctx - this.options = { - ...coverageConfigDefaults, - - // User's options - ...config, - - // Resolved fields - provider: 'istanbul', - reportsDirectory: resolve( - ctx.config.root, - config.reportsDirectory || coverageConfigDefaults.reportsDirectory, - ), - reporter: this.resolveReporters( - config.reporter || coverageConfigDefaults.reporter, - ), - - thresholds: config.thresholds && { - ...config.thresholds, - lines: config.thresholds['100'] ? 100 : config.thresholds.lines, - branches: config.thresholds['100'] ? 100 : config.thresholds.branches, - functions: config.thresholds['100'] ? 100 : config.thresholds.functions, - statements: config.thresholds['100'] ? 100 : config.thresholds.statements, - }, - } + this.testExclude = new TestExclude({ + cwd: ctx.config.root, + include: this.options.include, + exclude: this.options.exclude, + excludeNodeModules: true, + extension: this.options.extension, + relativePath: !this.options.allowExternal, + }) this.instrumenter = createInstrumenter({ produceSourceMap: true, @@ -147,35 +55,9 @@ export class IstanbulCoverageProvider extends BaseCoverageProvider implements Co importAttributesKeyword: 'with', }, }) - - this.testExclude = new _TestExclude({ - cwd: ctx.config.root, - include: this.options.include, - exclude: this.options.exclude, - excludeNodeModules: true, - extension: this.options.extension, - relativePath: !this.options.allowExternal, - }) - - const shard = this.ctx.config.shard - const tempDirectory = `.tmp${ - shard ? `-${shard.index}-${shard.count}` : '' - }` - - this.coverageFilesDirectory = resolve( - this.options.reportsDirectory, - tempDirectory, - ) } - resolveOptions(): Options { - return this.options - } - - onFileTransform(sourceCode: string, id: string, pluginCtx: any): { - code: string - map: any - } | undefined { + onFileTransform(sourceCode: string, id: string, pluginCtx: any): { code: string; map: any } | undefined { if (!this.testExclude.shouldInstrument(id)) { return } @@ -199,115 +81,34 @@ export class IstanbulCoverageProvider extends BaseCoverageProvider implements Co return { code, map } } - /* - * Coverage and meta information passed from Vitest runners. - * Note that adding new entries here and requiring on those without - * backwards compatibility is a breaking change. - */ - onAfterSuiteRun({ coverage, transformMode, projectName, testFiles }: AfterSuiteRunMeta): void { - if (!coverage) { - return - } - - if (transformMode !== 'web' && transformMode !== 'ssr' && transformMode !== 'browser') { - throw new Error(`Invalid transform mode: ${transformMode}`) - } - - let entry = this.coverageFiles.get(projectName || DEFAULT_PROJECT) - - if (!entry) { - entry = { web: {}, ssr: {}, browser: {} } - this.coverageFiles.set(projectName || DEFAULT_PROJECT, entry) - } - - const testFilenames = testFiles.join() - const filename = resolve( - this.coverageFilesDirectory, - `coverage-${uniqueId++}.json`, - ) - - // If there's a result from previous run, overwrite it - entry[transformMode][testFilenames] = filename - - const promise = fs.writeFile(filename, JSON.stringify(coverage), 'utf-8') - this.pendingPromises.push(promise) - } - - async clean(clean = true): Promise { - if (clean && existsSync(this.options.reportsDirectory)) { - await fs.rm(this.options.reportsDirectory, { - recursive: true, - force: true, - maxRetries: 10, - }) - } - - if (existsSync(this.coverageFilesDirectory)) { - await fs.rm(this.coverageFilesDirectory, { - recursive: true, - force: true, - maxRetries: 10, - }) - } - - await fs.mkdir(this.coverageFilesDirectory, { recursive: true }) - - this.coverageFiles = new Map() - this.pendingPromises = [] + createCoverageMap() { + return libCoverage.createCoverageMap({}) } async generateCoverage({ allTestsRun }: ReportContext): Promise { - const coverageMap = libCoverage.createCoverageMap({}) - let index = 0 - const total = this.pendingPromises.length - - await Promise.all(this.pendingPromises) - this.pendingPromises = [] - - for (const coveragePerProject of this.coverageFiles.values()) { - for (const coverageByTestfiles of [ - coveragePerProject.ssr, - coveragePerProject.web, - coveragePerProject.browser, - ]) { - const coverageMapByTransformMode = libCoverage.createCoverageMap({}) - const filenames = Object.values(coverageByTestfiles) - - for (const chunk of this.toSlices( - filenames, - this.options.processingConcurrency, - )) { - if (debug.enabled) { - index += chunk.length - debug('Covered files %d/%d', index, total) - } - - await Promise.all( - chunk.map(async (filename) => { - const contents = await fs.readFile(filename, 'utf-8') - const coverage = JSON.parse(contents) as CoverageMap - - coverageMapByTransformMode.merge(coverage) - }), - ) - } + const coverageMap = this.createCoverageMap() + let coverageMapByTransformMode = this.createCoverageMap() + await this.readCoverageFiles({ + onFileRead(coverage) { + coverageMapByTransformMode.merge(coverage) + }, + onFinished: async () => { // Source maps can change based on projectName and transform mode. // Coverage transform re-uses source maps so we need to separate transforms from each other. - const transformedCoverage = await transformCoverage( - coverageMapByTransformMode, - ) + const transformedCoverage = await transformCoverage(coverageMapByTransformMode) coverageMap.merge(transformedCoverage) - } - } + + coverageMapByTransformMode = this.createCoverageMap() + }, + onDebug: debug, + }) // Include untested files when all tests were run (not a single file re-run) // or if previous results are preserved by "cleanOnRerun: false" if (this.options.all && (allTestsRun || !this.options.cleanOnRerun)) { const coveredFiles = coverageMap.files() - const uncoveredCoverage = await this.getCoverageMapForUncoveredFiles( - coveredFiles, - ) + const uncoveredCoverage = await this.getCoverageMapForUncoveredFiles(coveredFiles) coverageMap.merge(await transformCoverage(uncoveredCoverage)) } @@ -319,40 +120,7 @@ export class IstanbulCoverageProvider extends BaseCoverageProvider implements Co return coverageMap } - async reportCoverage(coverageMap: unknown, { allTestsRun }: ReportContext): Promise { - await this.generateReports( - (coverageMap as CoverageMap) || libCoverage.createCoverageMap({}), - allTestsRun, - ) - - // In watch mode we need to preserve the previous results if cleanOnRerun is disabled - const keepResults = !this.options.cleanOnRerun && this.ctx.config.watch - - if (!keepResults) { - await this.cleanAfterRun() - } - } - - private async cleanAfterRun() { - this.coverageFiles = new Map() - await fs.rm(this.coverageFilesDirectory, { recursive: true }) - - // Remove empty reports directory, e.g. when only text-reporter is used - if (readdirSync(this.options.reportsDirectory).length === 0) { - await fs.rm(this.options.reportsDirectory, { recursive: true }) - } - } - - async onTestFailure() { - if (!this.options.reportOnFailure) { - await this.cleanAfterRun() - } - } - - async generateReports( - coverageMap: CoverageMap, - allTestsRun: boolean | undefined, - ): Promise { + async generateReports(coverageMap: CoverageMap, allTestsRun: boolean | undefined): Promise { const context = libReport.createContext({ dir: this.options.reportsDirectory, coverageMap, @@ -377,54 +145,14 @@ export class IstanbulCoverageProvider extends BaseCoverageProvider implements Co } if (this.options.thresholds) { - const resolvedThresholds = this.resolveThresholds({ - coverageMap, - thresholds: this.options.thresholds, - createCoverageMap: () => libCoverage.createCoverageMap({}), - root: this.ctx.config.root, - }) - - this.checkThresholds({ - thresholds: resolvedThresholds, - perFile: this.options.thresholds.perFile, - onError: error => this.ctx.logger.error(error), - }) - - if (this.options.thresholds.autoUpdate && allTestsRun) { - if (!this.ctx.server.config.configFile) { - throw new Error( - 'Missing configurationFile. The "coverage.thresholds.autoUpdate" can only be enabled when configuration file is used.', - ) - } - - const configFilePath = this.ctx.server.config.configFile - const configModule = parseModule( - await fs.readFile(configFilePath, 'utf8'), - ) - - this.updateThresholds({ - thresholds: resolvedThresholds, - perFile: this.options.thresholds.perFile, - configurationFile: configModule, - onUpdate: () => - writeFileSync( - configFilePath, - configModule.generate().code, - 'utf-8', - ), - }) - } + await this.reportThresholds(coverageMap, allTestsRun) } } - async mergeReports(coverageMaps: unknown[]): Promise { - const coverageMap = libCoverage.createCoverageMap({}) - - for (const coverage of coverageMaps) { - coverageMap.merge(coverage as CoverageMap) - } - - await this.generateReports(coverageMap, true) + async parseConfigModule(configFilePath: string) { + return parseModule( + await fs.readFile(configFilePath, 'utf8'), + ) } private async getCoverageMapForUncoveredFiles(coveredFiles: string[]) { @@ -444,7 +172,7 @@ export class IstanbulCoverageProvider extends BaseCoverageProvider implements Co .sort() const cacheKey = new Date().getTime() - const coverageMap = libCoverage.createCoverageMap({}) + const coverageMap = this.createCoverageMap() const transform = this.createUncoveredFileTransformer(this.ctx) diff --git a/packages/coverage-v8/package.json b/packages/coverage-v8/package.json index 3e9eabcb361f..61e47212b3f1 100644 --- a/packages/coverage-v8/package.json +++ b/packages/coverage-v8/package.json @@ -73,6 +73,7 @@ "@types/istanbul-lib-report": "^3.0.3", "@types/istanbul-lib-source-maps": "^4.0.4", "@types/istanbul-reports": "^3.0.4", + "@types/test-exclude": "^6.0.2", "@vitest/browser": "workspace:*", "pathe": "^1.1.2", "v8-to-istanbul": "^9.3.0", diff --git a/packages/coverage-v8/src/provider.ts b/packages/coverage-v8/src/provider.ts index bb84e712f6b8..b6ef76806e2f 100644 --- a/packages/coverage-v8/src/provider.ts +++ b/packages/coverage-v8/src/provider.ts @@ -1,86 +1,32 @@ -import { - existsSync, - promises as fs, - readdirSync, - writeFileSync, -} from 'node:fs' import type { Profiler } from 'node:inspector' import { fileURLToPath, pathToFileURL } from 'node:url' +import { promises as fs } from 'node:fs' import v8ToIstanbul from 'v8-to-istanbul' import { mergeProcessCovs } from '@bcoe/v8-coverage' +import libCoverage from 'istanbul-lib-coverage' import libReport from 'istanbul-lib-report' +import libSourceMaps from 'istanbul-lib-source-maps' import reports from 'istanbul-reports' import type { CoverageMap } from 'istanbul-lib-coverage' -import libCoverage from 'istanbul-lib-coverage' -import libSourceMaps from 'istanbul-lib-source-maps' -import MagicString from 'magic-string' -import { parseModule } from 'magicast' -import remapping from '@ampproject/remapping' import { normalize, resolve } from 'pathe' -import c from 'tinyrainbow' import { provider } from 'std-env' +import c from 'tinyrainbow' import createDebug from 'debug' +import MagicString from 'magic-string' +import TestExclude from 'test-exclude' +import remapping from '@ampproject/remapping' +import { BaseCoverageProvider } from 'vitest/coverage' import { cleanUrl } from 'vite-node/utils' +import type { AfterSuiteRunMeta } from 'vitest' +import type { CoverageProvider, ReportContext, ResolvedCoverageOptions, Vitest, WorkspaceProject } from 'vitest/node' import type { EncodedSourceMap, FetchResult } from 'vite-node' -import { coverageConfigDefaults } from 'vitest/config' -import { BaseCoverageProvider } from 'vitest/coverage' -import type { Vitest, WorkspaceProject } from 'vitest/node' -import type { - AfterSuiteRunMeta, - CoverageProvider, - CoverageV8Options, - ReportContext, - ResolvedCoverageOptions, -} from 'vitest' -// @ts-expect-error missing types -import _TestExclude from 'test-exclude' +import { parseModule } from 'magicast' import { version } from '../package.json' with { type: 'json' } -interface TestExclude { - new (opts: { - cwd?: string | string[] - include?: string | string[] - exclude?: string | string[] - extension?: string | string[] - excludeNodeModules?: boolean - relativePath?: boolean - }): { - shouldInstrument: (filePath: string) => boolean - glob: (cwd: string) => Promise - } -} - -type Options = ResolvedCoverageOptions<'v8'> type TransformResults = Map type RawCoverage = Profiler.TakePreciseCoverageReturnType -/** - * Holds info about raw coverage results that are stored on file system: - * - * ```json - * "project-a": { - * "web": { - * "tests/math.test.ts": "coverage-1.json", - * "tests/utils.test.ts": "coverage-2.json", - * // ^^^^^^^^^^^^^^^ Raw coverage on file system - * }, - * "ssr": { ... }, - * "browser": { ... }, - * }, - * "project-b": ... - * ``` - */ -type CoverageFiles = Map< - NonNullable | typeof DEFAULT_PROJECT, - Record< - AfterSuiteRunMeta['transformMode'], - { [TestFilenames: string]: string } - > -> - -type Entries = [keyof T, T[keyof T]][] - // TODO: vite-node should export this const WRAPPER_LENGTH = 185 @@ -89,63 +35,19 @@ const VITE_EXPORTS_LINE_PATTERN = /Object\.defineProperty\(__vite_ssr_exports__.*\n/g const DECORATOR_METADATA_PATTERN = /_ts_metadata\("design:paramtypes", \[[^\]]*\]\),*/g -const DEFAULT_PROJECT: unique symbol = Symbol.for('default-project') const FILE_PROTOCOL = 'file://' const debug = createDebug('vitest:coverage') -let uniqueId = 0 - -export class V8CoverageProvider extends BaseCoverageProvider implements CoverageProvider { - name = 'v8' - ctx!: Vitest - options!: Options - testExclude!: InstanceType - - coverageFiles: CoverageFiles = new Map() - coverageFilesDirectory!: string - pendingPromises: Promise[] = [] +export class V8CoverageProvider extends BaseCoverageProvider> implements CoverageProvider { + name = 'v8' as const + version = version + testExclude!: InstanceType initialize(ctx: Vitest): void { - const config: CoverageV8Options = ctx.config.coverage - - if (ctx.version !== version) { - ctx.logger.warn( - c.yellow( - `Loaded ${c.inverse(c.yellow(` vitest@${ctx.version} `))} and ${c.inverse(c.yellow(` @vitest/coverage-v8@${version} `))}.` - + '\nRunning mixed versions is not supported and may lead into bugs' - + '\nUpdate your dependencies and make sure the versions match.', - ), - ) - } - - this.ctx = ctx - this.options = { - ...coverageConfigDefaults, - - // User's options - ...config, - - // Resolved fields - provider: 'v8', - reporter: this.resolveReporters( - config.reporter || coverageConfigDefaults.reporter, - ), - reportsDirectory: resolve( - ctx.config.root, - config.reportsDirectory || coverageConfigDefaults.reportsDirectory, - ), - - thresholds: config.thresholds && { - ...config.thresholds, - lines: config.thresholds['100'] ? 100 : config.thresholds.lines, - branches: config.thresholds['100'] ? 100 : config.thresholds.branches, - functions: config.thresholds['100'] ? 100 : config.thresholds.functions, - statements: config.thresholds['100'] ? 100 : config.thresholds.statements, - }, - } + this._initialize(ctx) - this.testExclude = new _TestExclude({ + this.testExclude = new TestExclude({ cwd: ctx.config.root, include: this.options.include, exclude: this.options.exclude, @@ -153,105 +55,21 @@ export class V8CoverageProvider extends BaseCoverageProvider implements Coverage extension: this.options.extension, relativePath: !this.options.allowExternal, }) - - const shard = this.ctx.config.shard - const tempDirectory = `.tmp${ - shard ? `-${shard.index}-${shard.count}` : '' - }` - - this.coverageFilesDirectory = resolve( - this.options.reportsDirectory, - tempDirectory, - ) - } - - resolveOptions(): Options { - return this.options - } - - async clean(clean = true): Promise { - if (clean && existsSync(this.options.reportsDirectory)) { - await fs.rm(this.options.reportsDirectory, { - recursive: true, - force: true, - maxRetries: 10, - }) - } - - if (existsSync(this.coverageFilesDirectory)) { - await fs.rm(this.coverageFilesDirectory, { - recursive: true, - force: true, - maxRetries: 10, - }) - } - - await fs.mkdir(this.coverageFilesDirectory, { recursive: true }) - - this.coverageFiles = new Map() - this.pendingPromises = [] } - /* - * Coverage and meta information passed from Vitest runners. - * Note that adding new entries here and requiring on those without - * backwards compatibility is a breaking change. - */ - onAfterSuiteRun({ coverage, transformMode, projectName, testFiles }: AfterSuiteRunMeta): void { - if (transformMode !== 'web' && transformMode !== 'ssr' && transformMode !== 'browser') { - throw new Error(`Invalid transform mode: ${transformMode}`) - } - - let entry = this.coverageFiles.get(projectName || DEFAULT_PROJECT) - - if (!entry) { - entry = { web: { }, ssr: { }, browser: { } } - this.coverageFiles.set(projectName || DEFAULT_PROJECT, entry) - } - - const testFilenames = testFiles.join() - const filename = resolve( - this.coverageFilesDirectory, - `coverage-${uniqueId++}.json`, - ) - - // If there's a result from previous run, overwrite it - entry[transformMode][testFilenames] = filename - - const promise = fs.writeFile(filename, JSON.stringify(coverage), 'utf-8') - this.pendingPromises.push(promise) + createCoverageMap() { + return libCoverage.createCoverageMap({}) } async generateCoverage({ allTestsRun }: ReportContext): Promise { - const coverageMap = libCoverage.createCoverageMap({}) - let index = 0 - const total = this.pendingPromises.length - - await Promise.all(this.pendingPromises) - this.pendingPromises = [] - - for (const [projectName, coveragePerProject] of this.coverageFiles.entries()) { - for (const [transformMode, coverageByTestfiles] of Object.entries(coveragePerProject) as Entries) { - let merged: RawCoverage = { result: [] } - - const filenames = Object.values(coverageByTestfiles) - const project = this.ctx.projects.find(p => p.getName() === projectName) || this.ctx.getCoreWorkspaceProject() - - for (const chunk of this.toSlices(filenames, this.options.processingConcurrency)) { - if (debug.enabled) { - index += chunk.length - debug('Covered files %d/%d', index, total) - } - - await Promise.all(chunk.map(async (filename) => { - const contents = await fs.readFile(filename, 'utf-8') - const coverage = JSON.parse(contents) as RawCoverage - - merged = mergeProcessCovs([merged, coverage]) - }), - ) - } + const coverageMap = this.createCoverageMap() + let merged: RawCoverage = { result: [] } + await this.readCoverageFiles({ + onFileRead(coverage) { + merged = mergeProcessCovs([merged, coverage]) + }, + onFinished: async (project, transformMode) => { const converted = await this.convertCoverage( merged, project, @@ -262,8 +80,11 @@ export class V8CoverageProvider extends BaseCoverageProvider implements Coverage // Coverage transform re-uses source maps so we need to separate transforms from each other. const transformedCoverage = await transformCoverage(converted) coverageMap.merge(transformedCoverage) - } - } + + merged = { result: [] } + }, + onDebug: debug, + }) // Include untested files when all tests were run (not a single file re-run) // or if previous results are preserved by "cleanOnRerun: false" @@ -282,7 +103,7 @@ export class V8CoverageProvider extends BaseCoverageProvider implements Coverage return coverageMap } - async reportCoverage(coverageMap: unknown, { allTestsRun }: ReportContext): Promise { + async generateReports(coverageMap: CoverageMap, allTestsRun?: boolean): Promise { if (provider === 'stackblitz') { this.ctx.logger.log( c.blue(' % ') @@ -292,36 +113,6 @@ export class V8CoverageProvider extends BaseCoverageProvider implements Coverage ) } - await this.generateReports( - (coverageMap as CoverageMap) || libCoverage.createCoverageMap({}), - allTestsRun, - ) - - // In watch mode we need to preserve the previous results if cleanOnRerun is disabled - const keepResults = !this.options.cleanOnRerun && this.ctx.config.watch - - if (!keepResults) { - await this.cleanAfterRun() - } - } - - private async cleanAfterRun() { - this.coverageFiles = new Map() - await fs.rm(this.coverageFilesDirectory, { recursive: true }) - - // Remove empty reports directory, e.g. when only text-reporter is used - if (readdirSync(this.options.reportsDirectory).length === 0) { - await fs.rm(this.options.reportsDirectory, { recursive: true }) - } - } - - async onTestFailure() { - if (!this.options.reportOnFailure) { - await this.cleanAfterRun() - } - } - - async generateReports(coverageMap: CoverageMap, allTestsRun?: boolean): Promise { const context = libReport.createContext({ dir: this.options.reportsDirectory, coverageMap, @@ -346,54 +137,14 @@ export class V8CoverageProvider extends BaseCoverageProvider implements Coverage } if (this.options.thresholds) { - const resolvedThresholds = this.resolveThresholds({ - coverageMap, - thresholds: this.options.thresholds, - createCoverageMap: () => libCoverage.createCoverageMap({}), - root: this.ctx.config.root, - }) - - this.checkThresholds({ - thresholds: resolvedThresholds, - perFile: this.options.thresholds.perFile, - onError: error => this.ctx.logger.error(error), - }) - - if (this.options.thresholds.autoUpdate && allTestsRun) { - if (!this.ctx.server.config.configFile) { - throw new Error( - 'Missing configurationFile. The "coverage.thresholds.autoUpdate" can only be enabled when configuration file is used.', - ) - } - - const configFilePath = this.ctx.server.config.configFile - const configModule = parseModule( - await fs.readFile(configFilePath, 'utf8'), - ) - - this.updateThresholds({ - thresholds: resolvedThresholds, - perFile: this.options.thresholds.perFile, - configurationFile: configModule, - onUpdate: () => - writeFileSync( - configFilePath, - configModule.generate().code, - 'utf-8', - ), - }) - } + await this.reportThresholds(coverageMap, allTestsRun) } } - async mergeReports(coverageMaps: unknown[]): Promise { - const coverageMap = libCoverage.createCoverageMap({}) - - for (const coverage of coverageMaps) { - coverageMap.merge(coverage as CoverageMap) - } - - await this.generateReports(coverageMap, true) + async parseConfigModule(configFilePath: string) { + return parseModule( + await fs.readFile(configFilePath, 'utf8'), + ) } private async getUntestedFiles(testedFiles: string[]): Promise { @@ -420,10 +171,7 @@ export class V8CoverageProvider extends BaseCoverageProvider implements Coverage let merged: RawCoverage = { result: [] } let index = 0 - for (const chunk of this.toSlices( - uncoveredFiles, - this.options.processingConcurrency, - )) { + for (const chunk of this.toSlices(uncoveredFiles, this.options.processingConcurrency)) { if (debug.enabled) { index += chunk.length debug('Uncovered files %d/%d', index, uncoveredFiles.length) @@ -579,7 +327,7 @@ export class V8CoverageProvider extends BaseCoverageProvider implements Coverage } } - const coverageMap = libCoverage.createCoverageMap({}) + const coverageMap = this.createCoverageMap() let index = 0 for (const chunk of this.toSlices(scriptCoverages, this.options.processingConcurrency)) { diff --git a/packages/vitest/src/utils/coverage.ts b/packages/vitest/src/utils/coverage.ts index 285a2ba59299..cbbb1904b2ef 100644 --- a/packages/vitest/src/utils/coverage.ts +++ b/packages/vitest/src/utils/coverage.ts @@ -1,8 +1,12 @@ -import { relative } from 'pathe' +import { existsSync, promises as fs, readdirSync, writeFileSync } from 'node:fs' +import { relative, resolve } from 'pathe' import mm from 'micromatch' +import c from 'tinyrainbow' import type { CoverageMap } from 'istanbul-lib-coverage' +import { coverageConfigDefaults } from '../defaults' +import type { AfterSuiteRunMeta } from '../types/general' import type { Vitest } from '../node/core' -import type { BaseCoverageOptions, ResolvedCoverageOptions } from '../node/types/coverage' +import type { BaseCoverageOptions, ReportContext, ResolvedCoverageOptions } from '../node/types/coverage' import { resolveCoverageReporters } from '../node/config/resolveConfig' type Threshold = 'lines' | 'functions' | 'statements' | 'branches' @@ -13,6 +17,31 @@ interface ResolvedThreshold { thresholds: Partial> } +/** + * Holds info about raw coverage results that are stored on file system: + * + * ```json + * "project-a": { + * "web": { + * "tests/math.test.ts": "coverage-1.json", + * "tests/utils.test.ts": "coverage-2.json", + * // ^^^^^^^^^^^^^^^ Raw coverage on file system + * }, + * "ssr": { ... }, + * "browser": { ... }, + * }, + * "project-b": ... + * ``` + */ +type CoverageFiles = Map< + NonNullable | symbol, + Record< + AfterSuiteRunMeta['transformMode'], + { [TestFilenames: string]: string } + > +> +type Entries = [keyof T, T[keyof T]][] + const THRESHOLD_KEYS: Readonly = [ 'lines', 'functions', @@ -20,89 +49,297 @@ const THRESHOLD_KEYS: Readonly = [ 'branches', ] const GLOBAL_THRESHOLDS_KEY = 'global' +const DEFAULT_PROJECT: unique symbol = Symbol.for('default-project') +let uniqueId = 0 + +export class BaseCoverageProvider> { + ctx!: Vitest + readonly name!: 'v8' | 'istanbul' + version!: string + options!: Options + + coverageFiles: CoverageFiles = new Map() + pendingPromises: Promise[] = [] + coverageFilesDirectory!: string + + _initialize(ctx: Vitest) { + this.ctx = ctx + + if (ctx.version !== this.version) { + ctx.logger.warn( + c.yellow( + `Loaded ${c.inverse(c.yellow(` vitest@${ctx.version} `))} and ${c.inverse(c.yellow(` @vitest/coverage-${this.name}@${this.version} `))}.` + + '\nRunning mixed versions is not supported and may lead into bugs' + + '\nUpdate your dependencies and make sure the versions match.', + ), + ) + } -export class BaseCoverageProvider { - /** - * Check if current coverage is above configured thresholds and bump the thresholds if needed - */ - updateThresholds({ - thresholds: allThresholds, - perFile, - configurationFile, - onUpdate, - }: { - thresholds: ResolvedThreshold[] - perFile?: boolean - configurationFile: unknown // ProxifiedModule from magicast - onUpdate: () => void + const config = ctx.config.coverage as Options + + this.options = { + ...coverageConfigDefaults, + + // User's options + ...config, + + // Resolved fields + provider: this.name, + reportsDirectory: resolve( + ctx.config.root, + config.reportsDirectory || coverageConfigDefaults.reportsDirectory, + ), + reporter: resolveCoverageReporters( + config.reporter || coverageConfigDefaults.reporter, + ), + thresholds: config.thresholds && { + ...config.thresholds, + lines: config.thresholds['100'] ? 100 : config.thresholds.lines, + branches: config.thresholds['100'] ? 100 : config.thresholds.branches, + functions: config.thresholds['100'] ? 100 : config.thresholds.functions, + statements: config.thresholds['100'] ? 100 : config.thresholds.statements, + }, + } + + const shard = this.ctx.config.shard + const tempDirectory = `.tmp${ + shard ? `-${shard.index}-${shard.count}` : '' + }` + + this.coverageFilesDirectory = resolve( + this.options.reportsDirectory, + tempDirectory, + ) + } + + createCoverageMap(): CoverageMap { + throw new Error('BaseReporter\'s createCoverageMap was not overwritten') + } + + async generateReports(_: CoverageMap, __: boolean | undefined) { + throw new Error('BaseReporter\'s generateReports was not overwritten') + } + + async parseConfigModule(_: string): Promise<{ generate: () => { code: string } }> { + throw new Error('BaseReporter\'s parseConfigModule was not overwritten') + } + + resolveOptions() { + return this.options + } + + async clean(clean = true): Promise { + if (clean && existsSync(this.options.reportsDirectory)) { + await fs.rm(this.options.reportsDirectory, { + recursive: true, + force: true, + maxRetries: 10, + }) + } + + if (existsSync(this.coverageFilesDirectory)) { + await fs.rm(this.coverageFilesDirectory, { + recursive: true, + force: true, + maxRetries: 10, + }) + } + + await fs.mkdir(this.coverageFilesDirectory, { recursive: true }) + + this.coverageFiles = new Map() + this.pendingPromises = [] + } + + onAfterSuiteRun({ coverage, transformMode, projectName, testFiles }: AfterSuiteRunMeta): void { + if (!coverage) { + return + } + + if (transformMode !== 'web' && transformMode !== 'ssr' && transformMode !== 'browser') { + throw new Error(`Invalid transform mode: ${transformMode}`) + } + + let entry = this.coverageFiles.get(projectName || DEFAULT_PROJECT) + + if (!entry) { + entry = { web: {}, ssr: {}, browser: {} } + this.coverageFiles.set(projectName || DEFAULT_PROJECT, entry) + } + + const testFilenames = testFiles.join() + const filename = resolve( + this.coverageFilesDirectory, + `coverage-${uniqueId++}.json`, + ) + + // If there's a result from previous run, overwrite it + entry[transformMode][testFilenames] = filename + + const promise = fs.writeFile(filename, JSON.stringify(coverage), 'utf-8') + this.pendingPromises.push(promise) + } + + async readCoverageFiles({ onFileRead, onFinished, onDebug }: { + /** Callback invoked with a single coverage result */ + onFileRead: (data: CoverageType) => void + /** Callback invoked once all results of a project for specific transform mode are read */ + onFinished: (project: Vitest['projects'][number], transformMode: AfterSuiteRunMeta['transformMode']) => Promise + onDebug: ((...logs: any[]) => void) & { enabled: boolean } }) { - let updatedThresholds = false + let index = 0 + const total = this.pendingPromises.length - const config = resolveConfig(configurationFile) - assertConfigurationModule(config) + await Promise.all(this.pendingPromises) + this.pendingPromises = [] - for (const { coverageMap, thresholds, name } of allThresholds) { - const summaries = perFile - ? coverageMap - .files() - .map((file: string) => - coverageMap.fileCoverageFor(file).toSummary(), - ) - : [coverageMap.getCoverageSummary()] + for (const [projectName, coveragePerProject] of this.coverageFiles.entries()) { + for (const [transformMode, coverageByTestfiles] of Object.entries(coveragePerProject) as Entries) { + const filenames = Object.values(coverageByTestfiles) + const project = this.ctx.getProjectByName(projectName as string) - const thresholdsToUpdate: [Threshold, number][] = [] + for (const chunk of this.toSlices(filenames, this.options.processingConcurrency)) { + if (onDebug.enabled) { + index += chunk.length + onDebug('Covered files %d/%d', index, total) + } - for (const key of THRESHOLD_KEYS) { - const threshold = thresholds[key] ?? 100 - const actual = Math.min( - ...summaries.map(summary => summary[key].pct), - ) + await Promise.all(chunk.map(async (filename) => { + const contents = await fs.readFile(filename, 'utf-8') + const coverage = JSON.parse(contents) - if (actual > threshold) { - thresholdsToUpdate.push([key, actual]) + onFileRead(coverage) + }), + ) } + + await onFinished(project, transformMode) } + } + } - if (thresholdsToUpdate.length === 0) { + async cleanAfterRun() { + this.coverageFiles = new Map() + await fs.rm(this.coverageFilesDirectory, { recursive: true }) + + // Remove empty reports directory, e.g. when only text-reporter is used + if (readdirSync(this.options.reportsDirectory).length === 0) { + await fs.rm(this.options.reportsDirectory, { recursive: true }) + } + } + + async onTestFailure() { + if (!this.options.reportOnFailure) { + await this.cleanAfterRun() + } + } + + async reportCoverage(coverageMap: unknown, { allTestsRun }: ReportContext): Promise { + await this.generateReports( + (coverageMap as CoverageMap) || this.createCoverageMap(), + allTestsRun, + ) + + // In watch mode we need to preserve the previous results if cleanOnRerun is disabled + const keepResults = !this.options.cleanOnRerun && this.ctx.config.watch + + if (!keepResults) { + await this.cleanAfterRun() + } + } + + async reportThresholds(coverageMap: CoverageMap, allTestsRun: boolean | undefined) { + const resolvedThresholds = this.resolveThresholds(coverageMap) + this.checkThresholds(resolvedThresholds) + + if (this.options.thresholds?.autoUpdate && allTestsRun) { + if (!this.ctx.server.config.configFile) { + throw new Error( + 'Missing configurationFile. The "coverage.thresholds.autoUpdate" can only be enabled when configuration file is used.', + ) + } + + const configFilePath = this.ctx.server.config.configFile + const configModule = await this.parseConfigModule(configFilePath) + + await this.updateThresholds({ + thresholds: resolvedThresholds, + configurationFile: configModule, + onUpdate: () => + writeFileSync( + configFilePath, + configModule.generate().code, + 'utf-8', + ), + + }) + } + } + + /** + * Constructs collected coverage and users' threshold options into separate sets + * where each threshold set holds their own coverage maps. Threshold set is either + * for specific files defined by glob pattern or global for all other files. + */ + private resolveThresholds(coverageMap: CoverageMap): ResolvedThreshold[] { + const resolvedThresholds: ResolvedThreshold[] = [] + const files = coverageMap.files() + const globalCoverageMap = this.createCoverageMap() + + for (const key of Object.keys(this.options.thresholds!) as `${keyof NonNullable}`[]) { + if ( + key === 'perFile' + || key === 'autoUpdate' + || key === '100' + || THRESHOLD_KEYS.includes(key) + ) { continue } - updatedThresholds = true + const glob = key + const globThresholds = resolveGlobThresholds(this.options.thresholds![glob]) + const globCoverageMap = this.createCoverageMap() - for (const [threshold, newValue] of thresholdsToUpdate) { - if (name === GLOBAL_THRESHOLDS_KEY) { - config.test.coverage.thresholds[threshold] = newValue - } - else { - const glob = config.test.coverage.thresholds[ - name as Threshold - ] as ResolvedThreshold['thresholds'] - glob[threshold] = newValue - } + const matchingFiles = files.filter(file => + mm.isMatch(relative(this.ctx.config.root, file), glob), + ) + + for (const file of matchingFiles) { + const fileCoverage = coverageMap.fileCoverageFor(file) + globCoverageMap.addFileCoverage(fileCoverage) } + + resolvedThresholds.push({ + name: glob, + coverageMap: globCoverageMap, + thresholds: globThresholds, + }) } - if (updatedThresholds) { - // eslint-disable-next-line no-console - console.log( - 'Updating thresholds to configuration file. You may want to push with updated coverage thresholds.', - ) - onUpdate() + // Global threshold is for all files, even if they are included by glob patterns + for (const file of files) { + const fileCoverage = coverageMap.fileCoverageFor(file) + globalCoverageMap.addFileCoverage(fileCoverage) } + + resolvedThresholds.unshift({ + name: GLOBAL_THRESHOLDS_KEY, + coverageMap: globalCoverageMap, + thresholds: { + branches: this.options.thresholds?.branches, + functions: this.options.thresholds?.functions, + lines: this.options.thresholds?.lines, + statements: this.options.thresholds?.statements, + }, + }) + + return resolvedThresholds } /** * Check collected coverage against configured thresholds. Sets exit code to 1 when thresholds not reached. */ - checkThresholds({ - thresholds: allThresholds, - perFile, - onError, - }: { - thresholds: ResolvedThreshold[] - perFile?: boolean - onError: (error: string) => void - }) { + private checkThresholds(allThresholds: ResolvedThreshold[]) { for (const { coverageMap, thresholds, name } of allThresholds) { if ( thresholds.branches === undefined @@ -114,26 +351,16 @@ export class BaseCoverageProvider { } // Construct list of coverage summaries where thresholds are compared against - const summaries = perFile + const summaries = this.options.thresholds?.perFile ? coverageMap.files().map((file: string) => ({ file, summary: coverageMap.fileCoverageFor(file).toSummary(), })) - : [ - { - file: null, - summary: coverageMap.getCoverageSummary(), - }, - ] + : [{ file: null, summary: coverageMap.getCoverageSummary() }] // Check thresholds of each summary for (const { summary, file } of summaries) { - for (const thresholdKey of [ - 'lines', - 'functions', - 'statements', - 'branches', - ] as const) { + for (const thresholdKey of THRESHOLD_KEYS) { const threshold = thresholds[thresholdKey] if (threshold !== undefined) { @@ -151,14 +378,11 @@ export class BaseCoverageProvider { name === GLOBAL_THRESHOLDS_KEY ? name : `"${name}"` } threshold (${threshold}%)` - if (perFile && file) { - errorMessage += ` for ${relative('./', file).replace( - /\\/g, - '/', - )}` + if (this.options.thresholds?.perFile && file) { + errorMessage += ` for ${relative('./', file).replace(/\\/g, '/')}` } - onError(errorMessage) + this.ctx.logger.error(errorMessage) } } } @@ -167,84 +391,71 @@ export class BaseCoverageProvider { } /** - * Constructs collected coverage and users' threshold options into separate sets - * where each threshold set holds their own coverage maps. Threshold set is either - * for specific files defined by glob pattern or global for all other files. + * Check if current coverage is above configured thresholds and bump the thresholds if needed */ - resolveThresholds({ - coverageMap, - thresholds, - createCoverageMap, - root, - }: { - coverageMap: CoverageMap - thresholds: NonNullable - createCoverageMap: () => CoverageMap - root: string - }): ResolvedThreshold[] { - const resolvedThresholds: ResolvedThreshold[] = [] - const files = coverageMap.files() - const globalCoverageMap = createCoverageMap() + async updateThresholds({ thresholds: allThresholds, onUpdate, configurationFile }: { + thresholds: ResolvedThreshold[] + configurationFile: unknown // ProxifiedModule from magicast + onUpdate: () => void + }) { + let updatedThresholds = false - for (const key of Object.keys( - thresholds, - ) as `${keyof typeof thresholds}`[]) { - if ( - key === 'perFile' - || key === 'autoUpdate' - || key === '100' - || THRESHOLD_KEYS.includes(key) - ) { - continue - } + const config = resolveConfig(configurationFile) + assertConfigurationModule(config) - const glob = key - const globThresholds = resolveGlobThresholds(thresholds[glob]) - const globCoverageMap = createCoverageMap() + for (const { coverageMap, thresholds, name } of allThresholds) { + const summaries = this.options.thresholds?.perFile + ? coverageMap + .files() + .map((file: string) => + coverageMap.fileCoverageFor(file).toSummary(), + ) + : [coverageMap.getCoverageSummary()] - const matchingFiles = files.filter(file => - mm.isMatch(relative(root, file), glob), - ) + const thresholdsToUpdate: [Threshold, number][] = [] - for (const file of matchingFiles) { - const fileCoverage = coverageMap.fileCoverageFor(file) - globCoverageMap.addFileCoverage(fileCoverage) + for (const key of THRESHOLD_KEYS) { + const threshold = thresholds[key] ?? 100 + const actual = Math.min( + ...summaries.map(summary => summary[key].pct), + ) + + if (actual > threshold) { + thresholdsToUpdate.push([key, actual]) + } } - resolvedThresholds.push({ - name: glob, - coverageMap: globCoverageMap, - thresholds: globThresholds, - }) + if (thresholdsToUpdate.length === 0) { + continue + } + + updatedThresholds = true + + for (const [threshold, newValue] of thresholdsToUpdate) { + if (name === GLOBAL_THRESHOLDS_KEY) { + config.test.coverage.thresholds[threshold] = newValue + } + else { + const glob = config.test.coverage.thresholds[name as Threshold] as ResolvedThreshold['thresholds'] + glob[threshold] = newValue + } + } } - // Global threshold is for all files, even if they are included by glob patterns - for (const file of files) { - const fileCoverage = coverageMap.fileCoverageFor(file) - globalCoverageMap.addFileCoverage(fileCoverage) + if (updatedThresholds) { + this.ctx.logger.log('Updating thresholds to configuration file. You may want to push with updated coverage thresholds.') + onUpdate() } + } - resolvedThresholds.unshift({ - name: GLOBAL_THRESHOLDS_KEY, - coverageMap: globalCoverageMap, - thresholds: { - branches: thresholds.branches, - functions: thresholds.functions, - lines: thresholds.lines, - statements: thresholds.statements, - }, - }) + async mergeReports(coverageMaps: unknown[]): Promise { + const coverageMap = this.createCoverageMap() - return resolvedThresholds - } + for (const coverage of coverageMaps) { + coverageMap.merge(coverage as CoverageMap) + } - /** - * Resolve reporters from various configuration options - */ - resolveReporters( - configReporters: NonNullable, - ): ResolvedCoverageOptions['reporter'] { - return resolveCoverageReporters(configReporters) as any + await this.generateReports(coverageMap, true) } hasTerminalReporter(reporters: ResolvedCoverageOptions['reporter']) { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c9c977ee37bf..ea42f499ec2a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -476,6 +476,9 @@ importers: '@types/istanbul-reports': specifier: ^3.0.4 version: 3.0.4 + '@types/test-exclude': + specifier: ^6.0.2 + version: 6.0.2 pathe: specifier: ^1.1.2 version: 1.1.2 @@ -537,6 +540,9 @@ importers: '@types/istanbul-reports': specifier: ^3.0.4 version: 3.0.4 + '@types/test-exclude': + specifier: ^6.0.2 + version: 6.0.2 '@vitest/browser': specifier: workspace:* version: link:../browser @@ -3863,6 +3869,9 @@ packages: '@types/tern@0.23.4': resolution: {integrity: sha512-JAUw1iXGO1qaWwEOzxTKJZ/5JxVeON9kvGZ/osgZaJImBnyjyn0cjovPsf6FNLmyGY8Vw9DoXZCMlfMkMwHRWg==} + '@types/test-exclude@6.0.2': + resolution: {integrity: sha512-s/ec1VRsab6A/N4Zdn3MkBLKDaIjnXMHJOg2MfbfC6Vp4Z+UDV/slrA35JQJIjuQ4wFirRcqdX/c+rm8eqhh0w==} + '@types/through@0.0.30': resolution: {integrity: sha512-FvnCJljyxhPM3gkRgWmxmDZyAQSiBQQWLI0A0VFL0K7W1oRUrPJSqNO0NvTnLkBcotdlp3lKvaT0JrnyRDkzOg==} @@ -12443,6 +12452,8 @@ snapshots: dependencies: '@types/estree': 1.0.5 + '@types/test-exclude@6.0.2': {} + '@types/through@0.0.30': dependencies: '@types/node': 20.14.15 diff --git a/test/coverage-test/test/threshold-auto-update.unit.test.ts b/test/coverage-test/test/threshold-auto-update.unit.test.ts index 12e5620a6602..3a7558c869be 100644 --- a/test/coverage-test/test/threshold-auto-update.unit.test.ts +++ b/test/coverage-test/test/threshold-auto-update.unit.test.ts @@ -148,15 +148,15 @@ async function updateThresholds(configurationFile: ReturnType { const provider = new BaseCoverageProvider() - try { - provider.updateThresholds({ - thresholds, - configurationFile, - onUpdate: () => resolve(configurationFile.generate().code), - }) - } - catch (error) { - reject(error) - } + provider._initialize({ + config: { coverage: { } }, + logger: { log: () => {} }, + } as any) + + provider.updateThresholds({ + thresholds, + configurationFile, + onUpdate: () => resolve(configurationFile.generate().code), + }).catch(error => reject(error)) }) }