From 13d85bd19ab4b394ee01ebe03130dcf06568f0a1 Mon Sep 17 00:00:00 2001 From: Vladimir Date: Tue, 30 Jul 2024 13:21:40 +0200 Subject: [PATCH] feat: introduce experimental reported tasks (#6149) --- packages/browser/src/node/rpc.ts | 2 +- packages/runner/src/collect.ts | 14 +- packages/runner/src/setup.ts | 3 +- .../ui/client/composables/client/static.ts | 3 +- packages/utils/src/diff/index.ts | 10 +- packages/utils/src/error.ts | 27 +- packages/utils/src/index.ts | 2 + packages/utils/src/source-map.ts | 2 +- packages/utils/src/types.ts | 25 +- packages/vitest/rollup.config.js | 2 +- packages/vitest/src/api/setup.ts | 4 - packages/vitest/src/api/types.ts | 1 - packages/vitest/src/node/core.ts | 2 +- packages/vitest/src/node/error.ts | 2 +- packages/vitest/src/node/pools/forks.ts | 2 +- packages/vitest/src/node/pools/rpc.ts | 2 +- packages/vitest/src/node/pools/threads.ts | 2 +- packages/vitest/src/node/pools/typecheck.ts | 8 +- packages/vitest/src/node/pools/vmForks.ts | 2 +- packages/vitest/src/node/pools/vmThreads.ts | 2 +- .../src/node/reported-workspace-project.ts | 77 +++ packages/vitest/src/node/reporters/index.ts | 14 + .../src/node/reporters/reported-tasks.ts | 507 ++++++++++++++++++ packages/vitest/src/node/state.ts | 40 +- packages/vitest/src/node/types/config.ts | 22 +- packages/vitest/src/node/workspace.ts | 44 +- packages/vitest/src/public/index.ts | 30 +- packages/vitest/src/public/node.ts | 15 + packages/ws-client/src/index.ts | 2 +- packages/ws-client/src/state.ts | 134 +++++ .../fixtures/reported-tasks/1_first.test.ts | 84 +++ test/cli/test/reported-tasks.test.ts | 246 +++++++++ .../__snapshots__/jest-expect.test.ts.snap | 12 +- test/core/test/serialize.test.ts | 26 +- 34 files changed, 1258 insertions(+), 112 deletions(-) create mode 100644 packages/vitest/src/node/reported-workspace-project.ts create mode 100644 packages/vitest/src/node/reporters/reported-tasks.ts create mode 100644 packages/ws-client/src/state.ts create mode 100644 test/cli/fixtures/reported-tasks/1_first.test.ts create mode 100644 test/cli/test/reported-tasks.test.ts diff --git a/packages/browser/src/node/rpc.ts b/packages/browser/src/node/rpc.ts index 9db8a86490cc..17740981733d 100644 --- a/packages/browser/src/node/rpc.ts +++ b/packages/browser/src/node/rpc.ts @@ -70,7 +70,7 @@ export function setupBrowserRpc( ctx.state.catchError(error, type) }, async onCollected(files) { - ctx.state.collectFiles(files) + ctx.state.collectFiles(project, files) await ctx.report('onCollected', files) }, async onTaskUpdate(packs) { diff --git a/packages/runner/src/collect.ts b/packages/runner/src/collect.ts index c30a86220c75..55ac27428546 100644 --- a/packages/runner/src/collect.ts +++ b/packages/runner/src/collect.ts @@ -1,4 +1,5 @@ import { processError } from '@vitest/utils/error' +import { toArray } from '@vitest/utils' import type { File, SuiteHooks } from './types/tasks' import type { VitestRunner } from './types/runner' import { @@ -34,11 +35,18 @@ export async function collectTests( clearCollectorContext(filepath, runner) try { - const setupStart = now() - await runSetupFiles(config, runner) + const setupFiles = toArray(config.setupFiles) + if (setupFiles.length) { + const setupStart = now() + await runSetupFiles(config, setupFiles, runner) + const setupEnd = now() + file.setupDuration = setupEnd - setupStart + } + else { + file.setupDuration = 0 + } const collectStart = now() - file.setupDuration = collectStart - setupStart await runner.importFile(filepath, 'collect') diff --git a/packages/runner/src/setup.ts b/packages/runner/src/setup.ts index ea3a129263e2..f6ea1d722f77 100644 --- a/packages/runner/src/setup.ts +++ b/packages/runner/src/setup.ts @@ -1,11 +1,10 @@ -import { toArray } from '@vitest/utils' import type { VitestRunner, VitestRunnerConfig } from './types/runner' export async function runSetupFiles( config: VitestRunnerConfig, + files: string[], runner: VitestRunner, ): Promise { - const files = toArray(config.setupFiles) if (config.sequence.setupFiles === 'parallel') { await Promise.all( files.map(async (fsPath) => { diff --git a/packages/ui/client/composables/client/static.ts b/packages/ui/client/composables/client/static.ts index cc32711984fa..21d8020d0441 100644 --- a/packages/ui/client/composables/client/static.ts +++ b/packages/ui/client/composables/client/static.ts @@ -9,7 +9,7 @@ import type { } from 'vitest' import { parse } from 'flatted' import { decompressSync, strFromU8 } from 'fflate' -import { StateManager } from '../../../../vitest/src/node/state' +import { StateManager } from '../../../../ws-client/src/state' interface HTMLReportMetadata { paths: string[] @@ -55,7 +55,6 @@ export function createStaticClient(): VitestClient { }, getTransformResult: asyncNoop, onDone: noop, - onCollected: asyncNoop, onTaskUpdate: noop, writeFile: asyncNoop, rerun: asyncNoop, diff --git a/packages/utils/src/diff/index.ts b/packages/utils/src/diff/index.ts index 82777b6c4de0..d6dc8a084a83 100644 --- a/packages/utils/src/diff/index.ts +++ b/packages/utils/src/diff/index.ts @@ -69,7 +69,7 @@ const FALLBACK_FORMAT_OPTIONS = { * @param options Diff options * @returns {string | null} a string diff */ -export function diff(a: any, b: any, options?: DiffOptions): string | null { +export function diff(a: any, b: any, options?: DiffOptions): string | undefined { if (Object.is(a, b)) { return '' } @@ -80,11 +80,11 @@ export function diff(a: any, b: any, options?: DiffOptions): string | null { if (aType === 'object' && typeof a.asymmetricMatch === 'function') { if (a.$$typeof !== Symbol.for('jest.asymmetricMatcher')) { // Do not know expected type of user-defined asymmetric matcher. - return null + return undefined } if (typeof a.getExpectedType !== 'function') { // For example, expect.anything() matches either null or undefined - return null + return undefined } expectedType = a.getExpectedType() // Primitive types boolean and number omit difference below. @@ -104,7 +104,7 @@ export function diff(a: any, b: any, options?: DiffOptions): string | null { } if (omitDifference) { - return null + return undefined } switch (aType) { @@ -234,7 +234,7 @@ export function printDiffOrStringify( expected: unknown, received: unknown, options?: DiffOptions, -): string | null { +): string | undefined { const { aAnnotation, bAnnotation } = normalizeDiffOptions(options) if ( diff --git a/packages/utils/src/error.ts b/packages/utils/src/error.ts index 2e0cf0c57152..f675d949137c 100644 --- a/packages/utils/src/error.ts +++ b/packages/utils/src/error.ts @@ -1,5 +1,6 @@ import { type DiffOptions, printDiffOrStringify } from './diff' import { format, stringify } from './display' +import type { TestError } from './types' // utils is bundled for any environment and might not support `Element` declare class Element { @@ -26,7 +27,7 @@ function getUnserializableMessage(err: unknown) { } // https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API/Structured_clone_algorithm -export function serializeError(val: any, seen: WeakMap = new WeakMap()): any { +export function serializeValue(val: any, seen: WeakMap = new WeakMap()): any { if (!val || typeof val === 'string') { return val } @@ -41,7 +42,7 @@ export function serializeError(val: any, seen: WeakMap = new WeakM } // cannot serialize immutables as immutables if (isImmutable(val)) { - return serializeError(val.toJSON(), seen) + return serializeValue(val.toJSON(), seen) } if ( val instanceof Promise @@ -56,7 +57,7 @@ export function serializeError(val: any, seen: WeakMap = new WeakM return `${val.toString()} ${format(val.sample)}` } if (typeof val.toJSON === 'function') { - return serializeError(val.toJSON(), seen) + return serializeValue(val.toJSON(), seen) } if (seen.has(val)) { @@ -69,7 +70,7 @@ export function serializeError(val: any, seen: WeakMap = new WeakM seen.set(val, clone) val.forEach((e, i) => { try { - clone[i] = serializeError(e, seen) + clone[i] = serializeValue(e, seen) } catch (err) { clone[i] = getUnserializableMessage(err) @@ -90,7 +91,7 @@ export function serializeError(val: any, seen: WeakMap = new WeakM return } try { - clone[key] = serializeError(val[key], seen) + clone[key] = serializeValue(val[key], seen) } catch (err) { // delete in case it has a setter from prototype that might throw @@ -104,18 +105,22 @@ export function serializeError(val: any, seen: WeakMap = new WeakM } } +export { serializeValue as serializeError } + function normalizeErrorMessage(message: string) { return message.replace(/__(vite_ssr_import|vi_import)_\d+__\./g, '') } export function processError( - err: any, + _err: any, diffOptions?: DiffOptions, seen: WeakSet = new WeakSet(), ): any { - if (!err || typeof err !== 'object') { - return { message: err } + if (!_err || typeof _err !== 'object') { + return { message: String(_err) } } + const err = _err as TestError + // stack is not serialized in worker communication // we stringify it first if (err.stack) { @@ -133,7 +138,7 @@ export function processError( ) { err.diff = printDiffOrStringify(err.actual, err.expected, { ...diffOptions, - ...err.diffOptions, + ...err.diffOptions as DiffOptions, }) } @@ -163,10 +168,10 @@ export function processError( catch {} try { - return serializeError(err) + return serializeValue(err) } catch (e: any) { - return serializeError( + return serializeValue( new Error( `Failed to fully serialize error: ${e?.message}\nInner error message: ${err?.message}`, ), diff --git a/packages/utils/src/index.ts b/packages/utils/src/index.ts index dce716edf4b4..6b87a5c4fcb2 100644 --- a/packages/utils/src/index.ts +++ b/packages/utils/src/index.ts @@ -47,4 +47,6 @@ export type { Constructable, ParsedStack, ErrorWithDiff, + SerializedError, + TestError, } from './types' diff --git a/packages/utils/src/source-map.ts b/packages/utils/src/source-map.ts index c549c290772e..f1cd145ef419 100644 --- a/packages/utils/src/source-map.ts +++ b/packages/utils/src/source-map.ts @@ -15,7 +15,7 @@ export interface StackTraceParserOptions { ignoreStackEntries?: (RegExp | string)[] getSourceMap?: (file: string) => unknown getFileName?: (id: string) => string - frameFilter?: (error: Error, frame: ParsedStack) => boolean | void + frameFilter?: (error: ErrorWithDiff, frame: ParsedStack) => boolean | void } const CHROME_IE_STACK_REGEXP = /^\s*at .*(?:\S:\d+|\(native\))/m diff --git a/packages/utils/src/types.ts b/packages/utils/src/types.ts index e5c249bceaa4..8e3c3c229de8 100644 --- a/packages/utils/src/types.ts +++ b/packages/utils/src/types.ts @@ -32,8 +32,29 @@ export interface ParsedStack { column: number } -export interface ErrorWithDiff extends Error { - name: string +export interface SerializedError { + message: string + stack?: string + name?: string + stacks?: ParsedStack[] + cause?: SerializedError + [key: string]: unknown +} + +export interface TestError extends SerializedError { + cause?: TestError + diff?: string + actual?: string + expected?: string +} + +/** + * @deprecated Use `TestError` instead + */ +export interface ErrorWithDiff { + message: string + name?: string + cause?: unknown nameStr?: string stack?: string stackStr?: string diff --git a/packages/vitest/rollup.config.js b/packages/vitest/rollup.config.js index 46e6417ae7a0..4652deaec581 100644 --- a/packages/vitest/rollup.config.js +++ b/packages/vitest/rollup.config.js @@ -95,7 +95,7 @@ const plugins = [ json(), commonjs(), esbuild({ - target: 'node14', + target: 'node18', }), ] diff --git a/packages/vitest/src/api/setup.ts b/packages/vitest/src/api/setup.ts index c837d44aba53..2f380a56d00a 100644 --- a/packages/vitest/src/api/setup.ts +++ b/packages/vitest/src/api/setup.ts @@ -46,10 +46,6 @@ export function setup(ctx: Vitest, _server?: ViteDevServer) { function setupClient(ws: WebSocket) { const rpc = createBirpc( { - async onCollected(files) { - ctx.state.collectFiles(files) - await ctx.report('onCollected', files) - }, async onTaskUpdate(packs) { ctx.state.updateTasks(packs) await ctx.report('onTaskUpdate', packs) diff --git a/packages/vitest/src/api/types.ts b/packages/vitest/src/api/types.ts index 14102be34761..60a3a48ef522 100644 --- a/packages/vitest/src/api/types.ts +++ b/packages/vitest/src/api/types.ts @@ -27,7 +27,6 @@ export interface TransformResultWithSource { } export interface WebSocketHandlers { - onCollected: (files?: File[]) => Promise onTaskUpdate: (packs: TaskResultPack[]) => void getFiles: () => File[] getTestFiles: () => Promise diff --git a/packages/vitest/src/node/core.ts b/packages/vitest/src/node/core.ts index e1967d78daa0..ed5a813edcf2 100644 --- a/packages/vitest/src/node/core.ts +++ b/packages/vitest/src/node/core.ts @@ -440,7 +440,7 @@ export class Vitest { files.forEach((file) => { file.logs?.forEach(log => this.state.updateUserLog(log)) }) - this.state.collectFiles(files) + this.state.collectFiles(project, files) } await this.report('onCollected', files).catch(noop) diff --git a/packages/vitest/src/node/error.ts b/packages/vitest/src/node/error.ts index d0f07767823d..6597dc902c22 100644 --- a/packages/vitest/src/node/error.ts +++ b/packages/vitest/src/node/error.ts @@ -306,7 +306,7 @@ function printModuleWarningForSourceCode(logger: Logger, path: string) { ) } -export function displayDiff(diff: string | null, console: Console) { +export function displayDiff(diff: string | undefined, console: Console) { if (diff) { console.error(`\n${diff}\n`) } diff --git a/packages/vitest/src/node/pools/forks.ts b/packages/vitest/src/node/pools/forks.ts index 69a8de2c706b..b7185520b886 100644 --- a/packages/vitest/src/node/pools/forks.ts +++ b/packages/vitest/src/node/pools/forks.ts @@ -137,7 +137,7 @@ export function createForksPool( && error instanceof Error && /The task has been cancelled/.test(error.message) ) { - ctx.state.cancelFiles(files, ctx.config.root, project.config.name) + ctx.state.cancelFiles(files, project) } else { throw error diff --git a/packages/vitest/src/node/pools/rpc.ts b/packages/vitest/src/node/pools/rpc.ts index 32196af49e72..36a2b369921d 100644 --- a/packages/vitest/src/node/pools/rpc.ts +++ b/packages/vitest/src/node/pools/rpc.ts @@ -76,7 +76,7 @@ export function createMethodsRPC(project: WorkspaceProject, options: MethodsOpti return ctx.report('onPathsCollected', paths) }, onCollected(files) { - ctx.state.collectFiles(files) + ctx.state.collectFiles(project, files) return ctx.report('onCollected', files) }, onAfterSuiteRun(meta) { diff --git a/packages/vitest/src/node/pools/threads.ts b/packages/vitest/src/node/pools/threads.ts index c76109baf492..9613d47d198f 100644 --- a/packages/vitest/src/node/pools/threads.ts +++ b/packages/vitest/src/node/pools/threads.ts @@ -135,7 +135,7 @@ export function createThreadsPool( && error instanceof Error && /The task has been cancelled/.test(error.message) ) { - ctx.state.cancelFiles(files, ctx.config.root, project.config.name) + ctx.state.cancelFiles(files, project) } else { throw error diff --git a/packages/vitest/src/node/pools/typecheck.ts b/packages/vitest/src/node/pools/typecheck.ts index a88c29ae5912..b0ce984e0f49 100644 --- a/packages/vitest/src/node/pools/typecheck.ts +++ b/packages/vitest/src/node/pools/typecheck.ts @@ -61,7 +61,7 @@ export function createTypecheckPool(ctx: Vitest): ProcessPool { checker.setFiles(files) checker.onParseStart(async () => { - ctx.state.collectFiles(checker.getTestFiles()) + ctx.state.collectFiles(project, checker.getTestFiles()) await ctx.report('onCollected') }) @@ -80,7 +80,7 @@ export function createTypecheckPool(ctx: Vitest): ProcessPool { } await checker.collectTests() - ctx.state.collectFiles(checker.getTestFiles()) + ctx.state.collectFiles(project, checker.getTestFiles()) await ctx.report('onTaskUpdate', checker.getTestPacks()) await ctx.report('onCollected') @@ -107,7 +107,7 @@ export function createTypecheckPool(ctx: Vitest): ProcessPool { const checker = await createWorkspaceTypechecker(project, files) checker.setFiles(files) await checker.collectTests() - ctx.state.collectFiles(checker.getTestFiles()) + ctx.state.collectFiles(project, checker.getTestFiles()) await ctx.report('onCollected') } } @@ -135,7 +135,7 @@ export function createTypecheckPool(ctx: Vitest): ProcessPool { }) const triggered = await _p if (project.typechecker && !triggered) { - ctx.state.collectFiles(project.typechecker.getTestFiles()) + ctx.state.collectFiles(project, project.typechecker.getTestFiles()) await ctx.report('onCollected') await onParseEnd(project, project.typechecker.getResult()) continue diff --git a/packages/vitest/src/node/pools/vmForks.ts b/packages/vitest/src/node/pools/vmForks.ts index 46ea4c3c96b6..249da53e503c 100644 --- a/packages/vitest/src/node/pools/vmForks.ts +++ b/packages/vitest/src/node/pools/vmForks.ts @@ -147,7 +147,7 @@ export function createVmForksPool( && error instanceof Error && /The task has been cancelled/.test(error.message) ) { - ctx.state.cancelFiles(files, ctx.config.root, project.config.name) + ctx.state.cancelFiles(files, project) } else { throw error diff --git a/packages/vitest/src/node/pools/vmThreads.ts b/packages/vitest/src/node/pools/vmThreads.ts index ea245696db89..700eab6c0a60 100644 --- a/packages/vitest/src/node/pools/vmThreads.ts +++ b/packages/vitest/src/node/pools/vmThreads.ts @@ -141,7 +141,7 @@ export function createVmThreadsPool( && error instanceof Error && /The task has been cancelled/.test(error.message) ) { - ctx.state.cancelFiles(files, ctx.config.root, project.config.name) + ctx.state.cancelFiles(files, project) } else { throw error diff --git a/packages/vitest/src/node/reported-workspace-project.ts b/packages/vitest/src/node/reported-workspace-project.ts new file mode 100644 index 000000000000..0d22aa809d94 --- /dev/null +++ b/packages/vitest/src/node/reported-workspace-project.ts @@ -0,0 +1,77 @@ +import type { ProvidedContext } from '../types/general' +import type { ResolvedConfig, ResolvedProjectConfig, SerializedConfig } from './types/config' +import type { WorkspaceProject } from './workspace' +import type { Vitest } from './core' + +export class TestProject { + /** + * The global vitest instance. + * @experimental The public Vitest API is experimental and does not follow semver. + */ + public readonly vitest: Vitest + /** + * The workspace project this test project is associated with. + * @experimental The public Vitest API is experimental and does not follow semver. + */ + public readonly workspaceProject: WorkspaceProject + + /** + * Resolved project configuration. + */ + public readonly config: ResolvedProjectConfig + /** + * Resolved global configuration. If there are no workspace projects, this will be the same as `config`. + */ + public readonly globalConfig: ResolvedConfig + + /** + * The name of the project or an empty string if not set. + */ + public readonly name: string + + constructor(workspaceProject: WorkspaceProject) { + this.workspaceProject = workspaceProject + this.vitest = workspaceProject.ctx + this.globalConfig = workspaceProject.ctx.config + this.config = workspaceProject.config + this.name = workspaceProject.getName() + } + + /** + * Serialized project configuration. This is the config that tests receive. + */ + public get serializedConfig() { + return this.workspaceProject.getSerializableConfig() + } + + /** + * Custom context provided to the project. + */ + public context(): ProvidedContext { + return this.workspaceProject.getProvidedContext() + } + + /** + * Provide a custom context to the project. This context will be available for tests once they run. + */ + public provide( + key: T, + value: ProvidedContext[T], + ): void { + this.workspaceProject.provide(key, value) + } + + public toJSON(): SerializedTestProject { + return { + name: this.name, + serializedConfig: this.serializedConfig, + context: this.context(), + } + } +} + +interface SerializedTestProject { + name: string + serializedConfig: SerializedConfig + context: ProvidedContext +} diff --git a/packages/vitest/src/node/reporters/index.ts b/packages/vitest/src/node/reporters/index.ts index 37000f174bbb..c066a3126c76 100644 --- a/packages/vitest/src/node/reporters/index.ts +++ b/packages/vitest/src/node/reporters/index.ts @@ -28,6 +28,20 @@ export { } export type { BaseReporter, Reporter } +export { TestCase, TestFile, TestSuite } from './reported-tasks' +export type { + TestCollection, + + TaskOptions, + TestDiagnostic, + FileDiagnostic, + + TestResult, + TestResultFailed, + TestResultPassed, + TestResultSkipped, +} from './reported-tasks' + export type { JsonAssertionResult, JsonTestResult, diff --git a/packages/vitest/src/node/reporters/reported-tasks.ts b/packages/vitest/src/node/reporters/reported-tasks.ts new file mode 100644 index 000000000000..7e6a317e0270 --- /dev/null +++ b/packages/vitest/src/node/reporters/reported-tasks.ts @@ -0,0 +1,507 @@ +import type { + Custom as RunnerCustomCase, + Task as RunnerTask, + Test as RunnerTestCase, + File as RunnerTestFile, + Suite as RunnerTestSuite, + TaskMeta, +} from '@vitest/runner' +import type { TestError } from '@vitest/utils' +import { getTestName } from '../../utils/tasks' +import type { WorkspaceProject } from '../workspace' +import { TestProject } from '../reported-workspace-project' + +class ReportedTaskImplementation { + /** + * Task instance. + * @experimental Public runner task API is experimental and does not follow semver. + */ + public readonly task: RunnerTask + + /** + * The project assosiacted with the test or suite. + */ + public readonly project: TestProject + + /** + * Unique identifier. + * This ID is deterministic and will be the same for the same test across multiple runs. + * The ID is based on the file path and test position. + */ + public readonly id: string + + /** + * Location in the file where the test or suite is defined. + */ + public readonly location: { line: number; column: number } | undefined + + protected constructor( + task: RunnerTask, + project: WorkspaceProject, + ) { + this.task = task + this.project = project.testProject || (project.testProject = new TestProject(project)) + this.id = task.id + this.location = task.location + } + + /** + * Creates a new reported task instance and stores it in the project's state for future use. + */ + static register(task: RunnerTask, project: WorkspaceProject) { + const state = new this(task, project) as TestCase | TestSuite | TestFile + storeTask(project, task, state) + return state + } +} + +export class TestCase extends ReportedTaskImplementation { + #fullName: string | undefined + + declare public readonly task: RunnerTestCase | RunnerCustomCase + public readonly type: 'test' | 'custom' = 'test' + + /** + * Direct reference to the test file where the test or suite is defined. + */ + public readonly file: TestFile + + /** + * Name of the test. + */ + public readonly name: string + + /** + * Options that the test was initiated with. + */ + public readonly options: TaskOptions + + /** + * Parent suite. If suite was called directly inside the file, the parent will be the file. + */ + public readonly parent: TestSuite | TestFile + + protected constructor(task: RunnerTestSuite | RunnerTestFile, project: WorkspaceProject) { + super(task, project) + + this.name = task.name + this.file = getReportedTask(project, task.file) as TestFile + const suite = this.task.suite + if (suite) { + this.parent = getReportedTask(project, suite) as TestSuite + } + else { + this.parent = this.file + } + this.options = buildOptions(task) + } + + /** + * Full name of the test including all parent suites separated with `>`. + */ + public get fullName(): string { + if (this.#fullName === undefined) { + this.#fullName = getTestName(this.task, ' > ') + } + return this.#fullName + } + + /** + * Result of the test. Will be `undefined` if test is not finished yet or was just collected. + */ + public result(): TestResult | undefined { + const result = this.task.result + if (!result || result.state === 'run') { + return undefined + } + const state = result.state === 'fail' + ? 'failed' + : result.state === 'pass' + ? 'passed' + : 'skipped' + return { + state, + errors: result.errors as TestError[] | undefined, + } as TestResult + } + + /** + * Checks if the test passed successfully. + * If the test is not finished yet or was skipped, it will return `true`. + */ + public ok(): boolean { + const result = this.result() + return !result || result.state !== 'failed' + } + + /** + * Custom metadata that was attached to the test during its execution. + */ + public meta(): TaskMeta { + return this.task.meta + } + + /** + * Useful information about the test like duration, memory usage, etc. + * Diagnostic is only available after the test has finished. + */ + public diagnostic(): TestDiagnostic | undefined { + const result = this.task.result + // startTime should always be available if the test has properly finished + if (!result || result.state === 'run' || !result.startTime) { + return undefined + } + return { + heap: result.heap, + duration: result.duration!, + startTime: result.startTime, + retryCount: result.retryCount ?? 0, + repeatCount: result.repeatCount ?? 0, + flaky: !!result.retryCount && result.state === 'pass' && result.retryCount > 0, + } + } +} + +class TestCollection { + #task: RunnerTestSuite | RunnerTestFile + #project: WorkspaceProject + + constructor(task: RunnerTestSuite | RunnerTestFile, project: WorkspaceProject) { + this.#task = task + this.#project = project + } + + /** + * Test or a suite at a specific index in the array. + */ + at(index: number): TestCase | TestSuite | undefined { + if (index < 0) { + index = this.size + index + } + return getReportedTask(this.#project, this.#task.tasks[index]) as TestCase | TestSuite | undefined + } + + /** + * The number of tests and suites in the collection. + */ + get size(): number { + return this.#task.tasks.length + } + + /** + * The same collection, but in an array form for easier manipulation. + */ + array(): (TestCase | TestSuite)[] { + return Array.from(this) + } + + /** + * Iterates over all tests and suites in the collection. + */ + *values(): IterableIterator { + return this[Symbol.iterator]() + } + + /** + * Filters all tests that are part of this collection's suite and its children. + */ + *allTests(state?: TestResult['state'] | 'running'): IterableIterator { + for (const child of this) { + if (child.type === 'suite') { + yield * child.children.allTests(state) + } + else if (state) { + const testState = getTestState(child) + if (state === testState) { + yield child + } + } + else { + yield child + } + } + } + + /** + * Filters only tests that are part of this collection. + */ + *tests(state?: TestResult['state'] | 'running'): IterableIterator { + for (const child of this) { + if (child.type !== 'test') { + continue + } + + if (state) { + const testState = getTestState(child) + if (state === testState) { + yield child + } + } + else { + yield child + } + } + } + + /** + * Filters only suites that are part of this collection. + */ + *suites(): IterableIterator { + for (const child of this) { + if (child.type === 'suite') { + yield child + } + } + } + + /** + * Filters all suites that are part of this collection's suite and its children. + */ + *allSuites(): IterableIterator { + for (const child of this) { + if (child.type === 'suite') { + yield child + yield * child.children.allSuites() + } + } + } + + *[Symbol.iterator](): IterableIterator { + for (const task of this.#task.tasks) { + yield getReportedTask(this.#project, task) as TestSuite | TestCase + } + } +} + +export type { TestCollection } + +abstract class SuiteImplementation extends ReportedTaskImplementation { + declare public readonly task: RunnerTestSuite | RunnerTestFile + + /** + * Collection of suites and tests that are part of this suite. + */ + public readonly children: TestCollection + + protected constructor(task: RunnerTestSuite | RunnerTestFile, project: WorkspaceProject) { + super(task, project) + this.children = new TestCollection(task, project) + } +} + +export class TestSuite extends SuiteImplementation { + #fullName: string | undefined + + declare public readonly task: RunnerTestSuite + public readonly type = 'suite' + + /** + * Name of the test or the suite. + */ + public readonly name: string + + /** + * Direct reference to the test file where the test or suite is defined. + */ + public readonly file: TestFile + + /** + * Parent suite. If suite was called directly inside the file, the parent will be the file. + */ + public readonly parent: TestSuite | TestFile + + /** + * Options that suite was initiated with. + */ + public readonly options: TaskOptions + + protected constructor(task: RunnerTestSuite, project: WorkspaceProject) { + super(task, project) + + this.name = task.name + this.file = getReportedTask(project, task.file) as TestFile + const suite = this.task.suite + if (suite) { + this.parent = getReportedTask(project, suite) as TestSuite + } + else { + this.parent = this.file + } + this.options = buildOptions(task) + } + + /** + * Full name of the suite including all parent suites separated with `>`. + */ + public get fullName(): string { + if (this.#fullName === undefined) { + this.#fullName = getTestName(this.task, ' > ') + } + return this.#fullName + } +} + +export class TestFile extends SuiteImplementation { + declare public readonly task: RunnerTestFile + declare public readonly location: undefined + public readonly type = 'file' + + /** + * This is usually an absolute UNIX file path. + * It can be a virtual id if the file is not on the disk. + * This value corresponds to Vite's `ModuleGraph` id. + */ + public readonly moduleId: string + + protected constructor(task: RunnerTestFile, project: WorkspaceProject) { + super(task, project) + this.moduleId = task.filepath + } + + /** + * Useful information about the file like duration, memory usage, etc. + * If the file was not executed yet, all diagnostic values will return `0`. + */ + public diagnostic(): FileDiagnostic { + const setupDuration = this.task.setupDuration || 0 + const collectDuration = this.task.collectDuration || 0 + const prepareDuration = this.task.prepareDuration || 0 + const environmentSetupDuration = this.task.environmentLoad || 0 + const duration = this.task.result?.duration || 0 + return { + environmentSetupDuration, + prepareDuration, + collectDuration, + setupDuration, + duration, + } + } +} + +export interface TaskOptions { + each: boolean | undefined + concurrent: boolean | undefined + shuffle: boolean | undefined + retry: number | undefined + repeats: number | undefined + mode: 'run' | 'only' | 'skip' | 'todo' +} + +function buildOptions(task: RunnerTestCase | RunnerCustomCase | RunnerTestFile | RunnerTestSuite): TaskOptions { + return { + each: task.each, + concurrent: task.concurrent, + shuffle: task.shuffle, + retry: task.retry, + repeats: task.repeats, + mode: task.mode, + } +} + +export type TestResult = TestResultPassed | TestResultFailed | TestResultSkipped + +export interface TestResultPassed { + /** + * The test passed successfully. + */ + state: 'passed' + /** + * Errors that were thrown during the test execution. + * + * **Note**: If test was retried successfully, errors will still be reported. + */ + errors: TestError[] | undefined +} + +export interface TestResultFailed { + /** + * The test failed to execute. + */ + state: 'failed' + /** + * Errors that were thrown during the test execution. + */ + errors: TestError[] +} + +export interface TestResultSkipped { + /** + * The test was skipped with `only`, `skip` or `todo` flag. + * You can see which one was used in the `mode` option. + */ + state: 'skipped' + /** + * Skipped tests have no errors. + */ + errors: undefined +} + +export interface TestDiagnostic { + /** + * The amount of memory used by the test in bytes. + * This value is only available if the test was executed with `logHeapUsage` flag. + */ + heap: number | undefined + /** + * The time it takes to execute the test in ms. + */ + duration: number + /** + * The time in ms when the test started. + */ + startTime: number + /** + * The amount of times the test was retried. + */ + retryCount: number + /** + * The amount of times the test was repeated as configured by `repeats` option. + * This value can be lower if the test failed during the repeat and no `retry` is configured. + */ + repeatCount: number + /** + * If test passed on a second retry. + */ + flaky: boolean +} + +export interface FileDiagnostic { + /** + * The time it takes to import and initiate an environment. + */ + environmentSetupDuration: number + /** + * The time it takes Vitest to setup test harness (runner, mocks, etc.). + */ + prepareDuration: number + /** + * The time it takes to import the test file. + * This includes importing everything in the file and executing suite callbacks. + */ + collectDuration: number + /** + * The time it takes to import the setup file. + */ + setupDuration: number + /** + * Accumulated duration of all tests and hooks in the file. + */ + duration: number +} + +function getTestState(test: TestCase): TestResult['state'] | 'running' { + const result = test.result() + return result ? result.state : 'running' +} + +function storeTask(project: WorkspaceProject, runnerTask: RunnerTask, reportedTask: TestCase | TestSuite | TestFile): void { + project.ctx.state.reportedTasksMap.set(runnerTask, reportedTask) +} + +function getReportedTask(project: WorkspaceProject, runnerTask: RunnerTask): TestCase | TestSuite | TestFile { + const reportedTask = project.ctx.state.getReportedEntity(runnerTask) + if (!reportedTask) { + throw new Error(`Task instance was not found for ${runnerTask.type} "${runnerTask.name}"`) + } + return reportedTask +} diff --git a/packages/vitest/src/node/state.ts b/packages/vitest/src/node/state.ts index 919af2bc6f0e..dc7b79fafe91 100644 --- a/packages/vitest/src/node/state.ts +++ b/packages/vitest/src/node/state.ts @@ -1,10 +1,9 @@ import type { File, Task, TaskResultPack } from '@vitest/runner' - -// can't import actual functions from utils, because it's incompatible with @vitest/browsers import { createFileTask } from '@vitest/runner/utils' import type { AggregateError as AggregateErrorPonyfill } from '../utils/base' import type { UserConsoleLog } from '../types/general' import type { WorkspaceProject } from './workspace' +import { TestCase, TestFile, TestSuite } from './reporters/reported-tasks' export function isAggregateError(err: unknown): err is AggregateErrorPonyfill { if (typeof AggregateError !== 'undefined' && err instanceof AggregateError) { @@ -14,7 +13,6 @@ export function isAggregateError(err: unknown): err is AggregateErrorPonyfill { return err instanceof Error && 'errors' in err } -// Note this file is shared for both node and browser, be aware to avoid node specific logic export class StateManager { filesMap = new Map() pathsSet: Set = new Set() @@ -22,6 +20,7 @@ export class StateManager { taskFileMap = new WeakMap() errorsSet = new Set() processTimeoutCauses = new Set() + reportedTasksMap = new WeakMap() catchError(err: unknown, type: string): void { if (isAggregateError(err)) { @@ -98,7 +97,7 @@ export class StateManager { }) } - collectFiles(files: File[] = []) { + collectFiles(project: WorkspaceProject, files: File[] = []) { files.forEach((file) => { const existing = this.filesMap.get(file.filepath) || [] const otherProject = existing.filter( @@ -114,16 +113,14 @@ export class StateManager { } otherProject.push(file) this.filesMap.set(file.filepath, otherProject) - this.updateId(file) + this.updateId(file, project) }) } - // this file is reused by ws-client, and should not rely on heavy dependencies like workspace clearFiles( - _project: { config: { name: string | undefined; root: string } }, + project: WorkspaceProject, paths: string[] = [], ) { - const project = _project as WorkspaceProject paths.forEach((path) => { const files = this.filesMap.get(path) const fileTask = createFileTask( @@ -132,6 +129,7 @@ export class StateManager { project.config.name, ) fileTask.local = true + TestFile.register(fileTask, project) this.idMap.set(fileTask.id, fileTask) if (!files) { this.filesMap.set(path, [fileTask]) @@ -150,18 +148,33 @@ export class StateManager { }) } - updateId(task: Task) { + updateId(task: Task, project: WorkspaceProject) { if (this.idMap.get(task.id) === task) { return } + + if (task.type === 'suite' && 'filepath' in task) { + TestFile.register(task, project) + } + else if (task.type === 'suite') { + TestSuite.register(task, project) + } + else { + TestCase.register(task, project) + } + this.idMap.set(task.id, task) if (task.type === 'suite') { task.tasks.forEach((task) => { - this.updateId(task) + this.updateId(task, project) }) } } + getReportedEntity(task: Task) { + return this.reportedTasksMap.get(task) + } + updateTasks(packs: TaskResultPack[]) { for (const [id, result, meta] of packs) { const task = this.idMap.get(id) @@ -192,9 +205,12 @@ export class StateManager { ).length } - cancelFiles(files: string[], root: string, projectName: string) { + cancelFiles(files: string[], project: WorkspaceProject) { this.collectFiles( - files.map(filepath => createFileTask(filepath, root, projectName)), + project, + files.map(filepath => + createFileTask(filepath, project.config.root, project.config.name), + ), ) } } diff --git a/packages/vitest/src/node/types/config.ts b/packages/vitest/src/node/types/config.ts index 57cf97cce725..bf866ec4fa8a 100644 --- a/packages/vitest/src/node/types/config.ts +++ b/packages/vitest/src/node/types/config.ts @@ -10,7 +10,7 @@ import type { } from '../reporters' import type { TestSequencerConstructor } from '../sequencers/types' import type { ChaiConfig } from '../../integrations/chai/config' -import type { Arrayable, ParsedStack } from '../../types/general' +import type { Arrayable, ErrorWithDiff, ParsedStack } from '../../types/general' import type { JSDOMOptions } from '../../types/jsdom-options' import type { HappyDOMOptions } from '../../types/happy-dom-options' import type { EnvironmentOptions } from '../../types/environment' @@ -620,7 +620,7 @@ export interface InlineConfig { * * Return `false` to omit the frame. */ - onStackTrace?: (error: Error, frame: ParsedStack) => boolean | void + onStackTrace?: (error: ErrorWithDiff, frame: ParsedStack) => boolean | void /** * Indicates if CSS files should be processed. @@ -1019,9 +1019,7 @@ export interface ResolvedConfig minWorkers: number } -export type ProjectConfig = Omit< - UserConfig, - | 'sequencer' +type NonProjectOptions = | 'shard' | 'watch' | 'run' @@ -1029,7 +1027,6 @@ export type ProjectConfig = Omit< | 'update' | 'reporters' | 'outputFile' - | 'poolOptions' | 'teardownTimeout' | 'silent' | 'forceRerunTriggers' @@ -1047,11 +1044,17 @@ export type ProjectConfig = Omit< | 'slowTestThreshold' | 'inspect' | 'inspectBrk' - | 'deps' | 'coverage' | 'maxWorkers' | 'minWorkers' | 'fileParallelism' + +export type ProjectConfig = Omit< + UserConfig, + NonProjectOptions + | 'sequencer' + | 'deps' + | 'poolOptions' > & { sequencer?: Omit deps?: Omit @@ -1065,4 +1068,9 @@ export type ProjectConfig = Omit< } } +export type ResolvedProjectConfig = Omit< + ResolvedConfig, + NonProjectOptions +> + export type { UserWorkspaceConfig } from '../../public/config' diff --git a/packages/vitest/src/node/workspace.ts b/packages/vitest/src/node/workspace.ts index 749f8a8a069f..40a6a21b5ed8 100644 --- a/packages/vitest/src/node/workspace.ts +++ b/packages/vitest/src/node/workspace.ts @@ -1,6 +1,6 @@ import { promises as fs } from 'node:fs' -import { rm } from 'node:fs/promises' import { tmpdir } from 'node:os' +import { rm } from 'node:fs/promises' import fg from 'fast-glob' import mm from 'micromatch' import { @@ -24,6 +24,7 @@ import { setup } from '../api/setup' import type { ProvidedContext } from '../types/general' import type { ResolvedConfig, + SerializedConfig, UserConfig, UserWorkspaceConfig, } from './types/config' @@ -37,6 +38,7 @@ import { MocksPlugins } from './plugins/mocks' import { CoverageTransform } from './plugins/coverageTransform' import { serializeConfig } from './config/serializeConfig' import type { Vitest } from './core' +import { TestProject } from './reported-workspace-project' interface InitializeProjectOptions extends UserWorkspaceConfig { workspaceConfigPath: string @@ -96,6 +98,8 @@ export class WorkspaceProject { testFilesList: string[] | null = null + public testProject!: TestProject + public readonly id = nanoid() public readonly tmpDir = join(tmpdir(), this.id) @@ -206,12 +210,6 @@ export class WorkspaceProject { return mod?.ssrTransformResult?.map || mod?.transformResult?.map } - getBrowserSourceMapModuleById( - id: string, - ): TransformResult['map'] | undefined { - return this.browser?.vite.moduleGraph.getModuleById(id)?.transformResult?.map - } - get reporters() { return this.ctx.reporters } @@ -366,6 +364,7 @@ export class WorkspaceProject { project.server = ctx.server project.runner = ctx.runner project.config = ctx.config + project.testProject = new TestProject(project) return project } @@ -385,6 +384,7 @@ export class WorkspaceProject { server.config, this.ctx.logger, ) + this.testProject = new TestProject(this) this.server = server @@ -404,28 +404,24 @@ export class WorkspaceProject { await this.initBrowserServer(this.server.config.configFile) } - isBrowserEnabled() { + isBrowserEnabled(): boolean { return isBrowserEnabled(this.config) } - getSerializableConfig(method: 'run' | 'collect' = 'run') { - // TODO: call `serializeConfig` only once - const config = deepMerge(serializeConfig( + getSerializableConfig(): SerializedConfig { + // TODO: serialize the config _once_ or when needed + const config = serializeConfig( this.config, this.ctx.config, - this.server?.config, - ), (this.ctx.configOverride || {})) - - // disable heavy features when collecting because they are not needed - if (method === 'collect') { - if (this.config.browser.provider && this.config.browser.provider !== 'preview') { - config.browser.headless = true - } - config.snapshotSerializers = [] - config.diff = undefined + this.server.config, + ) + if (!this.ctx.configOverride) { + return config } - - return config + return deepMerge( + config, + this.ctx.configOverride, + ) } close() { @@ -444,7 +440,7 @@ export class WorkspaceProject { private async clearTmpDir() { try { - await rm(this.tmpDir, { force: true, recursive: true }) + await rm(this.tmpDir, { recursive: true }) } catch {} } diff --git a/packages/vitest/src/public/index.ts b/packages/vitest/src/public/index.ts index 33f17ad0e200..3b0293034179 100644 --- a/packages/vitest/src/public/index.ts +++ b/packages/vitest/src/public/index.ts @@ -2,6 +2,13 @@ import '../node/types/vite' import '../types/global' +import type { + Custom as Custom_, + File as File_, + Suite as Suite_, + Task as Task_, + Test as Test_, +} from '@vitest/runner' import type { CollectLineNumbers as CollectLineNumbers_, CollectLines as CollectLines_, @@ -120,16 +127,28 @@ export type RootAndTarget = RootAndTarget_ /** @deprecated import `TypeCheckContext` from `vitest/node` instead */ export type Context = Context_ +/** @deprecated use `RunnerTestSuite` instead */ +export type Suite = Suite_ +/** @deprecated use `RunnerTestFile` instead */ +export type File = File_ +/** @deprecated use `RunnerTestCase` instead */ +export type Test = Test_ +/** @deprecated use `RunnerCustomCase` instead */ +export type Custom = Custom_ +/** @deprecated use `RunnerTask` instead */ +export type Task = Task_ + export type { RunMode, TaskState, TaskBase, TaskResult, TaskResultPack, - Suite, - File, - Test, - Task, + Suite as RunnerTestSuite, + File as RunnerTestFile, + Test as RunnerTestCase, + Task as RunnerTask, + Custom as RunnerCustomCase, DoneCallback, TestFunction, TestOptions, @@ -144,7 +163,6 @@ export type { TestContext, TaskContext, ExtendedContext, - Custom, TaskCustomOptions, OnTestFailedHandler, TaskMeta, @@ -201,6 +219,8 @@ export type { AfterSuiteRunMeta, } from '../types/general' +export type { TestError, SerializedError } from '@vitest/utils' + /** @deprecated import from `vitest/environments` instead */ export type EnvironmentReturn = EnvironmentReturn_ /** @deprecated import from `vitest/environments` instead */ diff --git a/packages/vitest/src/public/node.ts b/packages/vitest/src/public/node.ts index 7373e0916550..d0d1bcb5d613 100644 --- a/packages/vitest/src/public/node.ts +++ b/packages/vitest/src/public/node.ts @@ -48,6 +48,20 @@ export type { HTMLOptions } from '../node/reporters/html' export { isFileServingAllowed, createServer, parseAst, parseAstAsync } from 'vite' export type * as Vite from 'vite' +export { TestCase, TestFile, TestSuite } from '../node/reporters/reported-tasks' +export { TestProject } from '../node/reported-workspace-project' +export type { + TestCollection, + + TaskOptions, + TestDiagnostic, + FileDiagnostic, + TestResult, + TestResultPassed, + TestResultFailed, + TestResultSkipped, +} from '../node/reporters/reported-tasks' + export type { SequenceHooks, SequenceSetupFiles, @@ -68,6 +82,7 @@ export type { UserConfig, ResolvedConfig, ProjectConfig, + ResolvedProjectConfig, UserWorkspaceConfig, RuntimeConfig, } from '../node/types/config' diff --git a/packages/ws-client/src/index.ts b/packages/ws-client/src/index.ts index 8301c19b719c..ecd65739426e 100644 --- a/packages/ws-client/src/index.ts +++ b/packages/ws-client/src/index.ts @@ -4,7 +4,7 @@ import { parse, stringify } from 'flatted' // eslint-disable-next-line no-restricted-imports import type { WebSocketEvents, WebSocketHandlers } from 'vitest' -import { StateManager } from '../../vitest/src/node/state' +import { StateManager } from './state' export * from '../../vitest/src/utils/tasks' diff --git a/packages/ws-client/src/state.ts b/packages/ws-client/src/state.ts new file mode 100644 index 000000000000..e00b92333d45 --- /dev/null +++ b/packages/ws-client/src/state.ts @@ -0,0 +1,134 @@ +import type { File, Task, TaskResultPack } from '@vitest/runner' +// eslint-disable-next-line no-restricted-imports +import type { UserConsoleLog } from 'vitest' + +// can't import actual functions from utils, because it's incompatible with @vitest/browsers +import { createFileTask } from '@vitest/runner/utils' + +// Note this file is shared for both node and browser, be aware to avoid node specific logic +export class StateManager { + filesMap = new Map() + pathsSet: Set = new Set() + idMap = new Map() + + getPaths() { + return Array.from(this.pathsSet) + } + + /** + * Return files that were running or collected. + */ + getFiles(keys?: string[]): File[] { + if (keys) { + return keys + .map(key => this.filesMap.get(key)!) + .flat() + .filter(file => file && !file.local) + } + return Array.from(this.filesMap.values()).flat().filter(file => !file.local) + } + + getFilepaths(): string[] { + return Array.from(this.filesMap.keys()) + } + + getFailedFilepaths() { + return this.getFiles() + .filter(i => i.result?.state === 'fail') + .map(i => i.filepath) + } + + collectPaths(paths: string[] = []) { + paths.forEach((path) => { + this.pathsSet.add(path) + }) + } + + collectFiles(files: File[] = []) { + files.forEach((file) => { + const existing = this.filesMap.get(file.filepath) || [] + const otherProject = existing.filter( + i => i.projectName !== file.projectName, + ) + const currentFile = existing.find( + i => i.projectName === file.projectName, + ) + // keep logs for the previous file because it should always be initiated before the collections phase + // which means that all logs are collected during the collection and not inside tests + if (currentFile) { + file.logs = currentFile.logs + } + otherProject.push(file) + this.filesMap.set(file.filepath, otherProject) + this.updateId(file) + }) + } + + // this file is reused by ws-client, and should not rely on heavy dependencies like workspace + clearFiles( + _project: { config: { name: string | undefined; root: string } }, + paths: string[] = [], + ) { + const project = _project + paths.forEach((path) => { + const files = this.filesMap.get(path) + const fileTask = createFileTask( + path, + project.config.root, + project.config.name || '', + ) + fileTask.local = true + this.idMap.set(fileTask.id, fileTask) + if (!files) { + this.filesMap.set(path, [fileTask]) + return + } + const filtered = files.filter( + file => file.projectName !== project.config.name, + ) + // always keep a File task, so we can associate logs with it + if (!filtered.length) { + this.filesMap.set(path, [fileTask]) + } + else { + this.filesMap.set(path, [...filtered, fileTask]) + } + }) + } + + updateId(task: Task) { + if (this.idMap.get(task.id) === task) { + return + } + this.idMap.set(task.id, task) + if (task.type === 'suite') { + task.tasks.forEach((task) => { + this.updateId(task) + }) + } + } + + updateTasks(packs: TaskResultPack[]) { + for (const [id, result, meta] of packs) { + const task = this.idMap.get(id) + if (task) { + task.result = result + task.meta = meta + // skipped with new PendingError + if (result?.state === 'skip') { + task.mode = 'skip' + } + } + } + } + + updateUserLog(log: UserConsoleLog) { + const task = log.taskId && this.idMap.get(log.taskId) + if (task) { + if (!task.logs) { + task.logs = [] + } + task.logs.push(log) + } + } +} diff --git a/test/cli/fixtures/reported-tasks/1_first.test.ts b/test/cli/fixtures/reported-tasks/1_first.test.ts new file mode 100644 index 000000000000..1c52b5e3ee50 --- /dev/null +++ b/test/cli/fixtures/reported-tasks/1_first.test.ts @@ -0,0 +1,84 @@ +import { describe, expect, it } from 'vitest' + +it('runs a test', async () => { + await new Promise(r => setTimeout(r, 10)) + expect(1).toBe(1) +}) + +it('fails a test', async () => { + await new Promise(r => setTimeout(r, 10)) + expect(1).toBe(2) +}) + +it('fails multiple times', () => { + expect.soft(1).toBe(2) + expect.soft(3).toBe(3) + expect.soft(2).toBe(3) +}) + +it('skips an option test', { skip: true }) +it.skip('skips a .modifier test') + +it('todos an option test', { todo: true }) +it.todo('todos a .modifier test') + +it('retries a test', { retry: 5 }, () => { + expect(1).toBe(2) +}) + +let counter = 0 +it('retries a test with success', { retry: 5 }, () => { + expect(counter++).toBe(2) +}) + +it('repeats a test', { repeats: 5 }, () => { + expect(1).toBe(2) +}) + +describe('a group', () => { + it('runs a test in a group', () => { + expect(1).toBe(1) + }) + + it('todos an option test in a group', { todo: true }) + + describe('a nested group', () => { + it('runs a test in a nested group', () => { + expect(1).toBe(1) + }) + + it('fails a test in a nested group', () => { + expect(1).toBe(2) + }) + + it.concurrent('runs first concurrent test in a nested group', () => { + expect(1).toBe(1) + }) + + it.concurrent('runs second concurrent test in a nested group', () => { + expect(1).toBe(1) + }) + }) +}) + +describe.shuffle('shuffled group', () => { + it('runs a test in a shuffled group', () => { + expect(1).toBe(1) + }) +}) + +describe.each([1])('each group %s', (groupValue) => { + it.each([2])('each test %s', (itValue) => { + expect(groupValue + itValue).toBe(3) + }) +}) + +it('registers a metadata', (ctx) => { + ctx.task.meta.key = 'value' +}) + +declare module 'vitest' { + interface TaskMeta { + key?: string + } +} diff --git a/test/cli/test/reported-tasks.test.ts b/test/cli/test/reported-tasks.test.ts new file mode 100644 index 000000000000..716d9f96b3c9 --- /dev/null +++ b/test/cli/test/reported-tasks.test.ts @@ -0,0 +1,246 @@ +import { beforeAll, expect, it } from 'vitest' +import { resolve } from 'pathe' +import type { File } from 'vitest' +import type { StateManager } from 'vitest/src/node/state.js' +import type { WorkspaceProject } from 'vitest/node' +import { runVitest } from '../../test-utils' +import type { TestCase, TestCollection, TestFile } from '../../../packages/vitest/src/node/reporters/reported-tasks' + +const now = new Date() +// const finishedFiles: File[] = [] +const collectedFiles: File[] = [] +let state: StateManager +let project: WorkspaceProject +let files: File[] +let testFile: TestFile + +beforeAll(async () => { + const { ctx } = await runVitest({ + root: resolve(__dirname, '..', 'fixtures', 'reported-tasks'), + include: ['**/*.test.ts'], + reporters: [ + 'verbose', + { + // onFinished(files) { + // finishedFiles.push(...files || []) + // }, + onCollected(files) { + collectedFiles.push(...files || []) + }, + }, + ], + includeTaskLocation: true, + logHeapUsage: true, + }) + state = ctx!.state + project = ctx!.getCoreWorkspaceProject() + files = state.getFiles() + expect(files).toHaveLength(1) + testFile = state.getReportedEntity(files[0])! as TestFile + expect(testFile).toBeDefined() +}) + +it('correctly reports a file', () => { + // suite properties not available on file + expect(testFile).not.toHaveProperty('parent') + expect(testFile).not.toHaveProperty('options') + expect(testFile).not.toHaveProperty('file') + expect(testFile).not.toHaveProperty('fullName') + expect(testFile).not.toHaveProperty('name') + + expect(testFile.type).toBe('file') + expect(testFile.task).toBe(files[0]) + expect(testFile.id).toBe(files[0].id) + expect(testFile.location).toBeUndefined() + expect(testFile.moduleId).toBe(resolve('./fixtures/reported-tasks/1_first.test.ts')) + expect(testFile.project.workspaceProject).toBe(project) + expect(testFile.children.size).toBe(14) + + const tests = [...testFile.children.tests()] + expect(tests).toHaveLength(11) + const deepTests = [...testFile.children.allTests()] + expect(deepTests).toHaveLength(19) + + const suites = [...testFile.children.suites()] + expect(suites).toHaveLength(3) + const deepSuites = [...testFile.children.allSuites()] + expect(deepSuites).toHaveLength(4) + + const diagnostic = testFile.diagnostic() + expect(diagnostic).toBeDefined() + expect(diagnostic.environmentSetupDuration).toBeGreaterThan(0) + expect(diagnostic.prepareDuration).toBeGreaterThan(0) + expect(diagnostic.collectDuration).toBeGreaterThan(0) + expect(diagnostic.duration).toBeGreaterThan(0) + // doesn't have a setup file + expect(diagnostic.setupDuration).toBe(0) +}) + +it('correctly reports a passed test', () => { + const passedTest = findTest(testFile.children, 'runs a test') + expect(passedTest.type).toBe('test') + expect(passedTest.task).toBe(files[0].tasks[0]) + expect(passedTest.name).toBe('runs a test') + expect(passedTest.fullName).toBe('runs a test') + expect(passedTest.file).toBe(testFile) + expect(passedTest.parent).toBe(testFile) + expect(passedTest.options).toEqual({ + each: undefined, + concurrent: undefined, + shuffle: undefined, + retry: undefined, + repeats: undefined, + mode: 'run', + }) + expect(passedTest.meta()).toEqual({}) + + const result = passedTest.result()! + expect(result).toBeDefined() + expect(result.state).toBe('passed') + expect(result.errors).toBeUndefined() + + const diagnostic = passedTest.diagnostic()! + expect(diagnostic).toBeDefined() + expect(diagnostic.heap).toBeGreaterThan(0) + expect(diagnostic.duration).toBeGreaterThan(0) + expect(date(new Date(diagnostic.startTime))).toBe(date(now)) + expect(diagnostic.flaky).toBe(false) + expect(diagnostic.repeatCount).toBe(0) + expect(diagnostic.repeatCount).toBe(0) +}) + +it('correctly reports failed test', () => { + const passedTest = findTest(testFile.children, 'fails a test') + expect(passedTest.type).toBe('test') + expect(passedTest.task).toBe(files[0].tasks[1]) + expect(passedTest.name).toBe('fails a test') + expect(passedTest.fullName).toBe('fails a test') + expect(passedTest.file).toBe(testFile) + expect(passedTest.parent).toBe(testFile) + expect(passedTest.options).toEqual({ + each: undefined, + concurrent: undefined, + shuffle: undefined, + retry: undefined, + repeats: undefined, + mode: 'run', + }) + expect(passedTest.meta()).toEqual({}) + + const result = passedTest.result()! + expect(result).toBeDefined() + expect(result.state).toBe('failed') + expect(result.errors).toHaveLength(1) + expect(result.errors![0]).toMatchObject({ + diff: expect.any(String), + message: 'expected 1 to be 2 // Object.is equality', + ok: false, + stack: expect.stringContaining('expected 1 to be 2 // Object.is equality'), + stacks: [ + { + column: 13, + file: resolve('./fixtures/reported-tasks/1_first.test.ts'), + line: 10, + method: '', + }, + ], + }) + + const diagnostic = passedTest.diagnostic()! + expect(diagnostic).toBeDefined() + expect(diagnostic.heap).toBeGreaterThan(0) + expect(diagnostic.duration).toBeGreaterThan(0) + expect(date(new Date(diagnostic.startTime))).toBe(date(now)) + expect(diagnostic.flaky).toBe(false) + expect(diagnostic.repeatCount).toBe(0) + expect(diagnostic.repeatCount).toBe(0) +}) + +it('correctly reports multiple failures', () => { + const testCase = findTest(testFile.children, 'fails multiple times') + const result = testCase.result()! + expect(result).toBeDefined() + expect(result.state).toBe('failed') + expect(result.errors).toHaveLength(2) + expect(result.errors![0]).toMatchObject({ + message: 'expected 1 to be 2 // Object.is equality', + }) + expect(result.errors![1]).toMatchObject({ + message: 'expected 2 to be 3 // Object.is equality', + }) +}) + +it('correctly reports test assigned options', () => { + const testOptionSkip = findTest(testFile.children, 'skips an option test') + expect(testOptionSkip.options.mode).toBe('skip') + const testModifierSkip = findTest(testFile.children, 'skips a .modifier test') + expect(testModifierSkip.options.mode).toBe('skip') + + const testOptionTodo = findTest(testFile.children, 'todos an option test') + expect(testOptionTodo.options.mode).toBe('todo') + const testModifierTodo = findTest(testFile.children, 'todos a .modifier test') + expect(testModifierTodo.options.mode).toBe('todo') +}) + +it('correctly reports retried tests', () => { + const testRetry = findTest(testFile.children, 'retries a test') + expect(testRetry.options.retry).toBe(5) + expect(testRetry.options.repeats).toBeUndefined() + expect(testRetry.result()!.state).toBe('failed') +}) + +it('correctly reports flaky tests', () => { + const testFlaky = findTest(testFile.children, 'retries a test with success') + const diagnostic = testFlaky.diagnostic()! + expect(diagnostic.flaky).toBe(true) + expect(diagnostic.retryCount).toBe(2) + expect(diagnostic.repeatCount).toBe(0) + const result = testFlaky.result()! + expect(result.state).toBe('passed') + expect(result.errors).toHaveLength(2) +}) + +it('correctly reports repeated tests', () => { + const testRepeated = findTest(testFile.children, 'repeats a test') + const diagnostic = testRepeated.diagnostic()! + expect(diagnostic.flaky).toBe(false) + expect(diagnostic.retryCount).toBe(0) + expect(diagnostic.repeatCount).toBe(5) + const result = testRepeated.result()! + expect(result.state).toBe('failed') + expect(result.errors).toHaveLength(6) +}) + +it('correctly passed down metadata', () => { + const testMetadata = findTest(testFile.children, 'registers a metadata') + const meta = testMetadata.meta() + expect(meta).toHaveProperty('key', 'value') +}) + +function date(time: Date) { + return `${time.getDate()}/${time.getMonth() + 1}/${time.getFullYear()}` +} + +function deepFind(children: TestCollection, name: string): TestCase | undefined { + for (const task of children) { + if (task.type === 'test') { + if (task.name === name) { + return task + } + } + if (task.type === 'suite') { + const result = deepFind(task.children, name) + if (result) { + return result + } + } + } +} + +function findTest(children: TestCollection, name: string): TestCase { + const testCase = deepFind(children, name) + if (!testCase) { + throw new Error(`Test "${name}" not found`) + } + return testCase +} diff --git a/test/core/test/__snapshots__/jest-expect.test.ts.snap b/test/core/test/__snapshots__/jest-expect.test.ts.snap index a8aff8bc9627..85b30fbf13c6 100644 --- a/test/core/test/__snapshots__/jest-expect.test.ts.snap +++ b/test/core/test/__snapshots__/jest-expect.test.ts.snap @@ -3,7 +3,7 @@ exports[`asymmetric matcher error 1`] = ` { "actual": "hello", - "diff": null, + "diff": undefined, "expected": "StringContaining "xx"", "message": "expected 'hello' to deeply equal StringContaining "xx"", } @@ -12,7 +12,7 @@ exports[`asymmetric matcher error 1`] = ` exports[`asymmetric matcher error 2`] = ` { "actual": "hello", - "diff": null, + "diff": undefined, "expected": "StringNotContaining "ll"", "message": "expected 'hello' to deeply equal StringNotContaining "ll"", } @@ -200,7 +200,7 @@ exports[`asymmetric matcher error 13`] = ` exports[`asymmetric matcher error 14`] = ` { "actual": "hello", - "diff": null, + "diff": undefined, "expected": "StringMatching /xx/", "message": "expected 'hello' to deeply equal StringMatching /xx/", } @@ -222,7 +222,7 @@ exports[`asymmetric matcher error 15`] = ` exports[`asymmetric matcher error 16`] = ` { "actual": "hello", - "diff": null, + "diff": undefined, "expected": "StringContaining "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"", "message": "expected 'hello' to deeply equal StringContaining{…}", } @@ -231,7 +231,7 @@ exports[`asymmetric matcher error 16`] = ` exports[`asymmetric matcher error 17`] = ` { "actual": "hello", - "diff": null, + "diff": undefined, "expected": "StringContaining "xx"", "message": "expected error to match asymmetric matcher", } @@ -253,7 +253,7 @@ stringContainingCustom exports[`asymmetric matcher error 19`] = ` { "actual": "hello", - "diff": null, + "diff": undefined, "expected": "StringContaining "ll"", "message": "expected error not to match asymmetric matcher", } diff --git a/test/core/test/serialize.test.ts b/test/core/test/serialize.test.ts index af2721376d44..387d8de7f631 100644 --- a/test/core/test/serialize.test.ts +++ b/test/core/test/serialize.test.ts @@ -1,15 +1,15 @@ // @vitest-environment jsdom -import { serializeError } from '@vitest/utils/error' +import { serializeValue } from '@vitest/utils/error' import { describe, expect, it } from 'vitest' describe('error serialize', () => { it('works', () => { - expect(serializeError(undefined)).toEqual(undefined) - expect(serializeError(null)).toEqual(null) - expect(serializeError('hi')).toEqual('hi') + expect(serializeValue(undefined)).toEqual(undefined) + expect(serializeValue(null)).toEqual(null) + expect(serializeValue('hi')).toEqual('hi') - expect(serializeError({ + expect(serializeValue({ foo: 'hi', promise: new Promise(() => {}), fn: () => {}, @@ -35,7 +35,7 @@ describe('error serialize', () => { error.whateverArray = [error, error] error.whateverArrayClone = error.whateverArray - expect(serializeError(error)).toMatchSnapshot() + expect(serializeValue(error)).toMatchSnapshot() }) it('Should handle object with getter/setter correctly', () => { @@ -51,7 +51,7 @@ describe('error serialize', () => { }, } - expect(serializeError(user)).toEqual({ + expect(serializeValue(user)).toEqual({ name: 'John', surname: 'Smith', fullName: 'John Smith', @@ -70,7 +70,7 @@ describe('error serialize', () => { Object.defineProperty(user, 'fullName', { enumerable: false, value: 'John Smith' }) - const serialized = serializeError(user) + const serialized = serializeValue(user) expect(serialized).not.toBe(user) expect(serialized).toEqual({ name: 'John', @@ -86,7 +86,7 @@ describe('error serialize', () => { // `MessagePort`, so the serialized error object should have been recreated as plain object. const error = new Error('test') - const serialized = serializeError(error) + const serialized = serializeValue(error) expect(Object.getPrototypeOf(serialized)).toBe(null) expect(serialized).toEqual({ constructor: 'Function', @@ -114,7 +114,7 @@ describe('error serialize', () => { }, }], }) - expect(serializeError(error)).toEqual({ + expect(serializeValue(error)).toEqual({ array: [ { name: ': name cannot be accessed', @@ -131,7 +131,7 @@ describe('error serialize', () => { it('can serialize DOMException', () => { const err = new DOMException('You failed', 'InvalidStateError') - expect(serializeError(err)).toMatchObject({ + expect(serializeValue(err)).toMatchObject({ NETWORK_ERR: 19, name: 'InvalidStateError', message: 'You failed', @@ -160,7 +160,7 @@ describe('error serialize', () => { immutableRecord, }) - expect(serializeError(error)).toMatchObject({ + expect(serializeValue(error)).toMatchObject({ stack: expect.stringContaining('Error: test'), immutableList: ['foo'], immutableRecord: { foo: 'bar' }, @@ -186,7 +186,7 @@ describe('error serialize', () => { }, } - const serialized = serializeError(error) + const serialized = serializeValue(error) expect(serialized).toEqual({ key: 'value',