From 7ff27600b4662745cdca864287ec91c0a874e8c9 Mon Sep 17 00:00:00 2001 From: Pavel Feldman Date: Fri, 20 Jan 2023 18:24:15 -0800 Subject: [PATCH] chore: experimental oop loader (#20269) --- packages/playwright-test/src/loaderHost.ts | 36 ++++++ packages/playwright-test/src/loaderRunner.ts | 48 ++++++++ packages/playwright-test/src/process.ts | 33 +----- packages/playwright-test/src/processHost.ts | 27 ++--- packages/playwright-test/src/runner.ts | 98 ++++------------ packages/playwright-test/src/suiteUtils.ts | 37 +++++- packages/playwright-test/src/test.ts | 116 +++++++++++++++---- packages/playwright-test/src/testLoader.ts | 22 +++- packages/playwright-test/src/workerHost.ts | 5 +- packages/playwright-test/src/workerRunner.ts | 33 +++++- tests/playwright-test/reporter-html.spec.ts | 15 ++- tests/playwright-test/web-server.spec.ts | 24 ++-- 12 files changed, 321 insertions(+), 173 deletions(-) create mode 100644 packages/playwright-test/src/loaderHost.ts create mode 100644 packages/playwright-test/src/loaderRunner.ts diff --git a/packages/playwright-test/src/loaderHost.ts b/packages/playwright-test/src/loaderHost.ts new file mode 100644 index 0000000000000..d2bdfd3c900c9 --- /dev/null +++ b/packages/playwright-test/src/loaderHost.ts @@ -0,0 +1,36 @@ +/** + * Copyright Microsoft Corporation. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { TestError } from '../reporter'; +import type { SerializedConfig } from './ipc'; +import { ProcessHost } from './processHost'; +import { Suite } from './test'; + +export class LoaderHost extends ProcessHost { + constructor() { + super(require.resolve('./loaderRunner.js'), 'loader'); + } + + async start(config: SerializedConfig) { + await this.startRunner(config, true, {}); + } + + async loadTestFiles(files: string[], loadErrors: TestError[]): Promise { + const result = await this.sendMessage({ method: 'loadTestFiles', params: { files } }) as any; + loadErrors.push(...result.loadErrors); + return Suite._deepParse(result.rootSuite); + } +} diff --git a/packages/playwright-test/src/loaderRunner.ts b/packages/playwright-test/src/loaderRunner.ts new file mode 100644 index 0000000000000..7157945049251 --- /dev/null +++ b/packages/playwright-test/src/loaderRunner.ts @@ -0,0 +1,48 @@ +/** + * Copyright Microsoft Corporation. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { SerializedConfig } from './ipc'; +import type { TestError } from '../reporter'; +import { ConfigLoader } from './configLoader'; +import { ProcessRunner } from './process'; +import { loadTestFilesInProcess } from './testLoader'; +import { setFatalErrorSink } from './globals'; + +export class LoaderRunner extends ProcessRunner { + private _config: SerializedConfig; + private _configLoaderPromise: Promise | undefined; + + constructor(config: SerializedConfig) { + super(); + this._config = config; + } + + private _configLoader(): Promise { + if (!this._configLoaderPromise) + this._configLoaderPromise = ConfigLoader.deserialize(this._config); + return this._configLoaderPromise; + } + + async loadTestFiles(params: { files: string[] }) { + const loadErrors: TestError[] = []; + setFatalErrorSink(loadErrors); + const configLoader = await this._configLoader(); + const rootSuite = await loadTestFilesInProcess(configLoader.fullConfig(), params.files, loadErrors); + return { rootSuite: rootSuite._deepSerialize(), loadErrors }; + } +} + +export const create = (config: SerializedConfig) => new LoaderRunner(config); diff --git a/packages/playwright-test/src/process.ts b/packages/playwright-test/src/process.ts index 43c9710879b5b..aededb85f6e34 100644 --- a/packages/playwright-test/src/process.ts +++ b/packages/playwright-test/src/process.ts @@ -15,8 +15,7 @@ */ import type { WriteStream } from 'tty'; -import * as util from 'util'; -import type { ProcessInitParams, TeardownErrorsPayload, TestOutputPayload, TtyParams } from './ipc'; +import type { ProcessInitParams, TeardownErrorsPayload, TtyParams } from './ipc'; import { startProfiling, stopProfiling } from './profiler'; import type { TestInfoError } from './types'; import { serializeError } from './util'; @@ -29,7 +28,7 @@ export type ProtocolRequest = { export type ProtocolResponse = { id?: number; - error?: string; + error?: TestInfoError; method?: string; params?: any; result?: any; @@ -49,24 +48,6 @@ let closed = false; sendMessageToParent({ method: 'ready' }); -process.stdout.write = (chunk: string | Buffer) => { - const outPayload: TestOutputPayload = { - ...chunkToParams(chunk) - }; - sendMessageToParent({ method: 'stdOut', params: outPayload }); - return true; -}; - -if (!process.env.PW_RUNNER_DEBUG) { - process.stderr.write = (chunk: string | Buffer) => { - const outPayload: TestOutputPayload = { - ...chunkToParams(chunk) - }; - sendMessageToParent({ method: 'stdErr', params: outPayload }); - return true; - }; -} - process.on('disconnect', gracefullyCloseAndExit); process.on('SIGINT', () => {}); process.on('SIGTERM', () => {}); @@ -94,7 +75,7 @@ process.on('message', async message => { const response: ProtocolResponse = { id, result }; sendMessageToParent({ method: '__dispatch__', params: response }); } catch (e) { - const response: ProtocolResponse = { id, error: e.toString() }; + const response: ProtocolResponse = { id, error: serializeError(e) }; sendMessageToParent({ method: '__dispatch__', params: response }); } } @@ -132,14 +113,6 @@ function sendMessageToParent(message: { method: string, params?: any }) { } } -function chunkToParams(chunk: Buffer | string): { text?: string, buffer?: string } { - if (chunk instanceof Buffer) - return { buffer: chunk.toString('base64') }; - if (typeof chunk !== 'string') - return { text: util.inspect(chunk) }; - return { text: chunk }; -} - function setTtyParams(stream: WriteStream, params: TtyParams) { stream.isTTY = true; if (params.rows) diff --git a/packages/playwright-test/src/processHost.ts b/packages/playwright-test/src/processHost.ts index a752dc2d5ad4f..200a98acdc552 100644 --- a/packages/playwright-test/src/processHost.ts +++ b/packages/playwright-test/src/processHost.ts @@ -16,6 +16,7 @@ import child_process from 'child_process'; import { EventEmitter } from 'events'; +import { debug } from 'playwright-core/lib/utilsBundle'; import type { ProcessInitParams } from './ipc'; import type { ProtocolResponse } from './process'; @@ -41,17 +42,11 @@ export class ProcessHost extends EventEmitter { this._processName = processName; } - protected async startRunner(runnerParams: InitParams) { + protected async startRunner(runnerParams: InitParams, inheritStdio: boolean, env: NodeJS.ProcessEnv) { this.process = child_process.fork(require.resolve('./process'), { detached: false, - env: { - FORCE_COLOR: '1', - DEBUG_COLORS: '1', - PW_PROCESS_RUNNER_SCRIPT: this._runnerScript, - ...process.env - }, - // Can't pipe since piping slows down termination for some reason. - stdio: ['ignore', 'ignore', process.env.PW_RUNNER_DEBUG ? 'inherit' : 'ignore', 'ipc'] + env: { ...process.env, ...env }, + stdio: inheritStdio ? ['ignore', 'inherit', 'inherit', 'ipc'] : ['ignore', 'ignore', process.env.PW_RUNNER_DEBUG ? 'inherit' : 'ignore', 'ipc'], }); this.process.on('exit', (code, signal) => { this.didExit = true; @@ -59,15 +54,20 @@ export class ProcessHost extends EventEmitter { }); this.process.on('error', e => {}); // do not yell at a send to dead process. this.process.on('message', (message: any) => { + if (debug.enabled('pw:test:protocol')) + debug('pw:test:protocol')('◀ RECV ' + JSON.stringify(message)); if (message.method === '__dispatch__') { const { id, error, method, params, result } = message.params as ProtocolResponse; if (id && this._callbacks.has(id)) { const { resolve, reject } = this._callbacks.get(id)!; this._callbacks.delete(id); - if (error) - reject(new Error(error)); - else + if (error) { + const errorObject = new Error(error.message); + errorObject.stack = error.stack; + reject(errorObject); + } else { resolve(result); + } } else { this.emit(method!, params); } @@ -140,7 +140,8 @@ export class ProcessHost extends EventEmitter { } private send(message: { method: string, params?: any }) { - // This is a great place for debug logging. + if (debug.enabled('pw:test:protocol')) + debug('pw:test:protocol')('SEND ► ' + JSON.stringify(message)); this.process.send(message); } } diff --git a/packages/playwright-test/src/runner.ts b/packages/playwright-test/src/runner.ts index 21ed8bdbfaba2..80c1306818eee 100644 --- a/packages/playwright-test/src/runner.ts +++ b/packages/playwright-test/src/runner.ts @@ -45,9 +45,9 @@ import type { Config, FullConfigInternal, FullProjectInternal, ReporterInternal import { createFileMatcher, createFileMatcherFromFilters, createTitleMatcher, serializeError } from './util'; import type { Matcher, TestFileFilter } from './util'; import { setFatalErrorSink } from './globals'; -import { TestLoader } from './testLoader'; -import { buildFileSuiteForProject, filterTests } from './suiteUtils'; -import { PoolBuilder } from './poolBuilder'; +import { buildFileSuiteForProject, filterOnly, filterSuite, filterSuiteWithOnlySemantics, filterTestsRemoveEmptySuites } from './suiteUtils'; +import { LoaderHost } from './loaderHost'; +import { loadTestFilesInProcess } from './testLoader'; const removeFolderAsync = promisify(rimraf); const readDirAsync = promisify(fs.readdir); @@ -271,27 +271,23 @@ export class Runner { const config = this._configLoader.fullConfig(); const projects = this._collectProjects(options.projectFilter); const filesByProject = await this._collectFiles(projects, options.testFileFilters); - const result = await this._createFilteredRootSuite(options, filesByProject); - this._fatalErrors.push(...result.fatalErrors); - const { rootSuite } = result; + const rootSuite = await this._createFilteredRootSuite(options, filesByProject); const testGroups = createTestGroups(rootSuite.suites, config.workers); return { rootSuite, testGroups }; } - private async _createFilteredRootSuite(options: RunOptions, filesByProject: Map): Promise<{rootSuite: Suite, fatalErrors: TestError[]}> { + private async _createFilteredRootSuite(options: RunOptions, filesByProject: Map): Promise { const config = this._configLoader.fullConfig(); - const fatalErrors: TestError[] = []; const allTestFiles = new Set(); for (const files of filesByProject.values()) files.forEach(file => allTestFiles.add(file)); // Load all tests. - const { rootSuite: preprocessRoot, loadErrors } = await this._loadTests(allTestFiles); - fatalErrors.push(...loadErrors); + const preprocessRoot = await this._loadTests(allTestFiles); // Complain about duplicate titles. - fatalErrors.push(...createDuplicateTitlesErrors(config, preprocessRoot)); + this._fatalErrors.push(...createDuplicateTitlesErrors(config, preprocessRoot)); // Filter tests to respect line/column filter. filterByFocusedLine(preprocessRoot, options.testFileFilters); @@ -300,7 +296,7 @@ export class Runner { if (config.forbidOnly) { const onlyTestsAndSuites = preprocessRoot._getOnlyItems(); if (onlyTestsAndSuites.length > 0) - fatalErrors.push(...createForbidOnlyErrors(config, onlyTestsAndSuites)); + this._fatalErrors.push(...createForbidOnlyErrors(config, onlyTestsAndSuites)); } // Filter only. @@ -335,30 +331,26 @@ export class Runner { continue; for (let repeatEachIndex = 0; repeatEachIndex < project.repeatEach; repeatEachIndex++) { const builtSuite = buildFileSuiteForProject(project, fileSuite, repeatEachIndex); - if (!filterTests(builtSuite, titleMatcher)) + if (!filterTestsRemoveEmptySuites(builtSuite, titleMatcher)) continue; projectSuite._addSuite(builtSuite); } } } - return { rootSuite, fatalErrors }; + return rootSuite; } - private async _loadTests(testFiles: Set): Promise<{ rootSuite: Suite, loadErrors: TestError[] }> { - const config = this._configLoader.fullConfig(); - const testLoader = new TestLoader(config); - const loadErrors: TestError[] = []; - const rootSuite = new Suite('', 'root'); - for (const file of testFiles) { - const fileSuite = await testLoader.loadTestFile(file, 'loader'); - if (fileSuite._loadError) - loadErrors.push(fileSuite._loadError); - // We have to clone only if there maybe subsequent calls of this method. - rootSuite._addSuite(fileSuite); + private async _loadTests(testFiles: Set): Promise { + if (process.env.PWTEST_OOP_LOADER) { + const loaderHost = new LoaderHost(); + await loaderHost.start(this._configLoader.serializedConfig()); + try { + return await loaderHost.loadTestFiles([...testFiles], this._fatalErrors); + } finally { + await loaderHost.stop(); + } } - // Generate hashes. - PoolBuilder.buildForLoader(rootSuite); - return { rootSuite, loadErrors }; + return loadTestFilesInProcess(this._configLoader.fullConfig(), [...testFiles], this._fatalErrors); } private _filterForCurrentShard(rootSuite: Suite, testGroups: TestGroup[]) { @@ -404,8 +396,6 @@ export class Runner { // Filtering with "only semantics" does not work when we have zero tests - it leaves all the tests. // We need an empty suite in this case. rootSuite._entries = []; - rootSuite.suites = []; - rootSuite.tests = []; } else { filterSuiteWithOnlySemantics(rootSuite, () => false, test => shardTests.has(test)); } @@ -493,23 +483,6 @@ export class Runner { return 'success'; } - private _skipTestsFromMatchingGroups(testGroups: TestGroup[], groupFilter: (g: TestGroup) => boolean): TestGroup[] { - const result = []; - for (const group of testGroups) { - if (groupFilter(group)) { - for (const test of group.tests) { - const result = test._appendTestResult(); - this._reporter.onTestBegin?.(test, result); - result.status = 'skipped'; - this._reporter.onTestEnd?.(test, result); - } - } else { - result.push(group); - } - } - return result; - } - private async _removeOutputDirs(options: RunOptions): Promise { const config = this._configLoader.fullConfig(); const outputDirs = new Set(); @@ -616,14 +589,6 @@ export class Runner { } } -function filterOnly(suite: Suite) { - if (!suite._getOnlyItems().length) - return; - const suiteFilter = (suite: Suite) => suite._only; - const testFilter = (test: TestCase) => test._only; - return filterSuiteWithOnlySemantics(suite, suiteFilter, testFilter); -} - function createFileMatcherFromFilter(filter: TestFileFilter) { const fileMatcher = createFileMatcher(filter.re || filter.exact || ''); return (testFileName: string, testLine: number, testColumn: number) => @@ -640,29 +605,6 @@ function filterByFocusedLine(suite: Suite, focusedTestFileLines: TestFileFilter[ return filterSuite(suite, suiteFilter, testFilter); } -function filterSuiteWithOnlySemantics(suite: Suite, suiteFilter: (suites: Suite) => boolean, testFilter: (test: TestCase) => boolean) { - const onlySuites = suite.suites.filter(child => filterSuiteWithOnlySemantics(child, suiteFilter, testFilter) || suiteFilter(child)); - const onlyTests = suite.tests.filter(testFilter); - const onlyEntries = new Set([...onlySuites, ...onlyTests]); - if (onlyEntries.size) { - suite.suites = onlySuites; - suite.tests = onlyTests; - suite._entries = suite._entries.filter(e => onlyEntries.has(e)); // Preserve the order. - return true; - } - return false; -} - -function filterSuite(suite: Suite, suiteFilter: (suites: Suite) => boolean, testFilter: (test: TestCase) => boolean) { - for (const child of suite.suites) { - if (!suiteFilter(child)) - filterSuite(child, suiteFilter, testFilter); - } - suite.tests = suite.tests.filter(testFilter); - const entries = new Set([...suite.suites, ...suite.tests]); - suite._entries = suite._entries.filter(e => entries.has(e)); // Preserve the order. -} - async function collectFiles(testDir: string, respectGitIgnore: boolean): Promise { if (!fs.existsSync(testDir)) return []; diff --git a/packages/playwright-test/src/suiteUtils.ts b/packages/playwright-test/src/suiteUtils.ts index 502d649496139..de1bbcb22e540 100644 --- a/packages/playwright-test/src/suiteUtils.ts +++ b/packages/playwright-test/src/suiteUtils.ts @@ -19,10 +19,20 @@ import { calculateSha1 } from 'playwright-core/lib/utils'; import type { Suite, TestCase } from './test'; import type { FullProjectInternal } from './types'; -export function filterTests(suite: Suite, filter: (test: TestCase) => boolean): boolean { - suite.suites = suite.suites.filter(child => filterTests(child, filter)); - suite.tests = suite.tests.filter(filter); - const entries = new Set([...suite.suites, ...suite.tests]); +export function filterSuite(suite: Suite, suiteFilter: (suites: Suite) => boolean, testFilter: (test: TestCase) => boolean) { + for (const child of suite.suites) { + if (!suiteFilter(child)) + filterSuite(child, suiteFilter, testFilter); + } + const filteredTests = suite.tests.filter(testFilter); + const entries = new Set([...suite.suites, ...filteredTests]); + suite._entries = suite._entries.filter(e => entries.has(e)); // Preserve the order. +} + +export function filterTestsRemoveEmptySuites(suite: Suite, filter: (test: TestCase) => boolean): boolean { + const filteredSuites = suite.suites.filter(child => filterTestsRemoveEmptySuites(child, filter)); + const filteredTests = suite.tests.filter(filter); + const entries = new Set([...filteredSuites, ...filteredTests]); suite._entries = suite._entries.filter(e => entries.has(e)); // Preserve the order. return !!suite._entries.length; } @@ -59,3 +69,22 @@ export function buildFileSuiteForProject(project: FullProjectInternal, suite: Su return result; } + +export function filterOnly(suite: Suite) { + if (!suite._getOnlyItems().length) + return; + const suiteFilter = (suite: Suite) => suite._only; + const testFilter = (test: TestCase) => test._only; + return filterSuiteWithOnlySemantics(suite, suiteFilter, testFilter); +} + +export function filterSuiteWithOnlySemantics(suite: Suite, suiteFilter: (suites: Suite) => boolean, testFilter: (test: TestCase) => boolean) { + const onlySuites = suite.suites.filter(child => filterSuiteWithOnlySemantics(child, suiteFilter, testFilter) || suiteFilter(child)); + const onlyTests = suite.tests.filter(testFilter); + const onlyEntries = new Set([...onlySuites, ...onlyTests]); + if (onlyEntries.size) { + suite._entries = suite._entries.filter(e => onlyEntries.has(e)); // Preserve the order. + return true; + } + return false; +} diff --git a/packages/playwright-test/src/test.ts b/packages/playwright-test/src/test.ts index 33e2fb8f0ae55..1a5648d20f7c0 100644 --- a/packages/playwright-test/src/test.ts +++ b/packages/playwright-test/src/test.ts @@ -17,6 +17,7 @@ import type { FixturePool } from './fixtures'; import type * as reporterTypes from '../types/testReporter'; import type { TestTypeImpl } from './testType'; +import { rootTestType } from './testType'; import type { Annotation, FixturesWithLocation, FullProject, FullProjectInternal, Location } from './types'; class Base { @@ -37,8 +38,6 @@ export type Modifier = { }; export class Suite extends Base implements reporterTypes.Suite { - suites: Suite[] = []; - tests: TestCase[] = []; location?: Location; parent?: Suite; _use: FixturesWithLocation[] = []; @@ -51,7 +50,6 @@ export class Suite extends Base implements reporterTypes.Suite { _modifiers: Modifier[] = []; _parallelMode: 'default' | 'serial' | 'parallel' = 'default'; _projectConfig: FullProjectInternal | undefined; - _loadError?: reporterTypes.TestError; _fileId: string | undefined; readonly _type: 'root' | 'project' | 'file' | 'describe'; @@ -60,15 +58,21 @@ export class Suite extends Base implements reporterTypes.Suite { this._type = type; } + get suites(): Suite[] { + return this._entries.filter(entry => entry instanceof Suite) as Suite[]; + } + + get tests(): TestCase[] { + return this._entries.filter(entry => entry instanceof TestCase) as TestCase[]; + } + _addTest(test: TestCase) { test.parent = this; - this.tests.push(test); this._entries.push(test); } _addSuite(suite: Suite) { suite.parent = this; - this.suites.push(suite); this._entries.push(suite); } @@ -115,6 +119,29 @@ export class Suite extends Base implements reporterTypes.Suite { return suite; } + _deepSerialize(): any { + const suite = this._serialize(); + suite.entries = []; + for (const entry of this._entries) { + if (entry instanceof Suite) + suite.entries.push(entry._deepSerialize()); + else + suite.entries.push(entry._serialize()); + } + return suite; + } + + static _deepParse(data: any): Suite { + const suite = Suite._parse(data); + for (const entry of data.entries) { + if (entry.kind === 'suite') + suite._addSuite(Suite._deepParse(entry)); + else + suite._addTest(TestCase._parse(entry)); + } + return suite; + } + forEachTest(visitor: (test: TestCase, suite: Suite) => void) { for (const entry of this._entries) { if (entry instanceof Suite) @@ -124,20 +151,45 @@ export class Suite extends Base implements reporterTypes.Suite { } } + _serialize(): any { + return { + kind: 'suite', + title: this.title, + type: this._type, + location: this.location, + only: this._only, + requireFile: this._requireFile, + timeout: this._timeout, + retries: this._retries, + annotations: this._annotations.slice(), + modifiers: this._modifiers.slice(), + parallelMode: this._parallelMode, + skipped: this._skipped, + hooks: this._hooks.map(h => ({ type: h.type, location: h.location })), + }; + } + + static _parse(data: any): Suite { + const suite = new Suite(data.title, data.type); + suite.location = data.location; + suite._only = data.only; + suite._requireFile = data.requireFile; + suite._timeout = data.timeout; + suite._retries = data.retries; + suite._annotations = data.annotations; + suite._modifiers = data.modifiers; + suite._parallelMode = data.parallelMode; + suite._skipped = data.skipped; + suite._hooks = data.hooks.map((h: any) => ({ type: h.type, location: h.location, fn: () => { } })); + return suite; + } + _clone(): Suite { - const suite = new Suite(this.title, this._type); - suite._only = this._only; - suite.location = this.location; - suite._requireFile = this._requireFile; + const data = this._serialize(); + const suite = Suite._parse(data); suite._use = this._use.slice(); suite._hooks = this._hooks.slice(); - suite._timeout = this._timeout; - suite._retries = this._retries; - suite._annotations = this._annotations.slice(); - suite._modifiers = this._modifiers.slice(); - suite._parallelMode = this._parallelMode; suite._projectConfig = this._projectConfig; - suite._skipped = this._skipped; return suite; } @@ -197,14 +249,34 @@ export class TestCase extends Base implements reporterTypes.TestCase { return status === 'expected' || status === 'flaky' || status === 'skipped'; } + _serialize(): any { + return { + kind: 'test', + title: this.title, + location: this.location, + only: this._only, + requireFile: this._requireFile, + poolDigest: this._poolDigest, + expectedStatus: this.expectedStatus, + annotations: this.annotations.slice(), + }; + } + + static _parse(data: any): TestCase { + const test = new TestCase(data.title, () => {}, rootTestType, data.location); + test._only = data.only; + test._requireFile = data.requireFile; + test._poolDigest = data.poolDigest; + test.expectedStatus = data.expectedStatus; + test.annotations = data.annotations; + return test; + } + _clone(): TestCase { - const test = new TestCase(this.title, this.fn, this._testType, this.location); - test._only = this._only; - test._requireFile = this._requireFile; - test._poolDigest = this._poolDigest; - test.expectedStatus = this.expectedStatus; - test.annotations = this.annotations.slice(); - test._annotateWithInheritence = this._annotateWithInheritence; + const data = this._serialize(); + const test = TestCase._parse(data); + test._testType = this._testType; + test.fn = this.fn; return test; } diff --git a/packages/playwright-test/src/testLoader.ts b/packages/playwright-test/src/testLoader.ts index 815696420e932..d9d181d800b39 100644 --- a/packages/playwright-test/src/testLoader.ts +++ b/packages/playwright-test/src/testLoader.ts @@ -14,11 +14,13 @@ * limitations under the License. */ -import * as path from 'path'; +import path from 'path'; +import type { TestError } from '../reporter'; +import type { FullConfigInternal } from './types'; import { setCurrentlyLoadingFileSuite } from './globals'; +import { PoolBuilder } from './poolBuilder'; import { Suite } from './test'; import { requireOrImport } from './transform'; -import type { FullConfigInternal } from './types'; import { serializeError } from './util'; export const defaultTimeout = 30000; @@ -34,7 +36,7 @@ export class TestLoader { this._fullConfig = fullConfig; } - async loadTestFile(file: string, environment: 'loader' | 'worker'): Promise { + async loadTestFile(file: string, environment: 'loader' | 'worker', loadErrors: TestError[]): Promise { if (cachedFileSuites.has(file)) return cachedFileSuites.get(file)!; const suite = new Suite(path.relative(this._fullConfig.rootDir, file) || path.basename(file), 'file'); @@ -48,7 +50,7 @@ export class TestLoader { } catch (e) { if (environment === 'worker') throw e; - suite._loadError = serializeError(e); + loadErrors.push(serializeError(e)); } finally { setCurrentlyLoadingFileSuite(undefined); } @@ -76,3 +78,15 @@ export class TestLoader { return suite; } } + +export async function loadTestFilesInProcess(config: FullConfigInternal, testFiles: string[], loadErrors: TestError[]): Promise { + const testLoader = new TestLoader(config); + const rootSuite = new Suite('', 'root'); + for (const file of testFiles) { + const fileSuite = await testLoader.loadTestFile(file, 'loader', loadErrors); + rootSuite._addSuite(fileSuite); + } + // Generate hashes. + PoolBuilder.buildForLoader(rootSuite); + return rootSuite; +} diff --git a/packages/playwright-test/src/workerHost.ts b/packages/playwright-test/src/workerHost.ts index 24ba0cbd022ac..19ddff60ffa80 100644 --- a/packages/playwright-test/src/workerHost.ts +++ b/packages/playwright-test/src/workerHost.ts @@ -44,7 +44,10 @@ export class WorkerHost extends ProcessHost { } async start() { - await this.startRunner(this._params); + await this.startRunner(this._params, false, { + FORCE_COLOR: '1', + DEBUG_COLORS: '1', + }); } runTestGroup(runPayload: RunPayload) { diff --git a/packages/playwright-test/src/workerRunner.ts b/packages/playwright-test/src/workerRunner.ts index 94cf38e670e6d..2329810c2cd16 100644 --- a/packages/playwright-test/src/workerRunner.ts +++ b/packages/playwright-test/src/workerRunner.ts @@ -17,7 +17,7 @@ import { colors, rimraf } from 'playwright-core/lib/utilsBundle'; import util from 'util'; import { debugTest, formatLocation, relativeFilePath, serializeError } from './util'; -import type { TestBeginPayload, TestEndPayload, RunPayload, DonePayload, WorkerInitParams, TeardownErrorsPayload } from './ipc'; +import type { TestBeginPayload, TestEndPayload, RunPayload, DonePayload, WorkerInitParams, TeardownErrorsPayload, TestOutputPayload } from './ipc'; import { setCurrentTestInfo } from './globals'; import { ConfigLoader } from './configLoader'; import type { Suite, TestCase } from './test'; @@ -29,7 +29,7 @@ import type { TimeSlot } from './timeoutManager'; import { TimeoutManager } from './timeoutManager'; import { ProcessRunner } from './process'; import { TestLoader } from './testLoader'; -import { buildFileSuiteForProject, filterTests } from './suiteUtils'; +import { buildFileSuiteForProject, filterTestsRemoveEmptySuites } from './suiteUtils'; import { PoolBuilder } from './poolBuilder'; const removeFolderAsync = util.promisify(rimraf); @@ -76,6 +76,23 @@ export class WorkerRunner extends ProcessRunner { process.on('unhandledRejection', reason => this.unhandledError(reason)); process.on('uncaughtException', error => this.unhandledError(error)); + process.stdout.write = (chunk: string | Buffer) => { + const outPayload: TestOutputPayload = { + ...chunkToParams(chunk) + }; + this.dispatchEvent('stdOut', outPayload); + return true; + }; + + if (!process.env.PW_RUNNER_DEBUG) { + process.stderr.write = (chunk: string | Buffer) => { + const outPayload: TestOutputPayload = { + ...chunkToParams(chunk) + }; + this.dispatchEvent('stdErr', outPayload); + return true; + }; + } } private _stop(): Promise { @@ -184,9 +201,9 @@ export class WorkerRunner extends ProcessRunner { let fatalUnknownTestIds; try { await this._loadIfNeeded(); - const fileSuite = await this._testLoader.loadTestFile(runPayload.file, 'worker'); + const fileSuite = await this._testLoader.loadTestFile(runPayload.file, 'worker', []); const suite = buildFileSuiteForProject(this._project, fileSuite, this._params.repeatEachIndex); - const hasEntries = filterTests(suite, test => entries.has(test.id)); + const hasEntries = filterTestsRemoveEmptySuites(suite, test => entries.has(test.id)); if (hasEntries) { this._poolBuilder.buildPools(suite); this._extraSuiteAnnotations = new Map(); @@ -618,4 +635,12 @@ function formatTestTitle(test: TestCase, projectName: string) { return `${projectTitle}${location} › ${titles.join(' › ')}`; } +function chunkToParams(chunk: Buffer | string): { text?: string, buffer?: string } { + if (chunk instanceof Buffer) + return { buffer: chunk.toString('base64') }; + if (typeof chunk !== 'string') + return { text: util.inspect(chunk) }; + return { text: chunk }; +} + export const create = (params: WorkerInitParams) => new WorkerRunner(params); diff --git a/tests/playwright-test/reporter-html.spec.ts b/tests/playwright-test/reporter-html.spec.ts index 921cb86452623..89d2e30009f6c 100644 --- a/tests/playwright-test/reporter-html.spec.ts +++ b/tests/playwright-test/reporter-html.spec.ts @@ -775,11 +775,14 @@ test.describe('gitCommitInfo plugin', () => { const result = await runInlineTest({ 'uncommitted.txt': `uncommitted file`, - 'playwright.config.ts': `export default {};`, - 'example.spec.ts': ` + 'playwright.config.ts': ` import { gitCommitInfo } from '@playwright/test/lib/plugins'; const { test, _addRunnerPlugin } = pwt; _addRunnerPlugin(gitCommitInfo()); + export default {}; + `, + 'example.spec.ts': ` + const { test } = pwt; test('sample', async ({}) => { expect(2).toBe(2); }); `, }, { reporter: 'dot,html' }, { PW_TEST_HTML_REPORT_OPEN: 'never', GITHUB_REPOSITORY: 'microsoft/playwright-example-for-test', GITHUB_RUN_ID: 'example-run-id', GITHUB_SERVER_URL: 'https://playwright.dev', GITHUB_SHA: 'example-sha' }, undefined, beforeRunPlaywrightTest); @@ -805,9 +808,6 @@ test.describe('gitCommitInfo plugin', () => { const result = await runInlineTest({ 'uncommitted.txt': `uncommitted file`, 'playwright.config.ts': ` - export default {}; - `, - 'example.spec.ts': ` import { gitCommitInfo } from '@playwright/test/lib/plugins'; const { test, _addRunnerPlugin } = pwt; _addRunnerPlugin(gitCommitInfo({ @@ -819,6 +819,11 @@ test.describe('gitCommitInfo plugin', () => { 'revision.email': 'shakespeare@example.local', }, })); + export default {}; + `, + 'example.spec.ts': ` + import { gitCommitInfo } from '@playwright/test/lib/plugins'; + const { test } = pwt; test('sample', async ({}) => { expect(2).toBe(2); }); `, }, { reporter: 'dot,html' }, { PW_TEST_HTML_REPORT_OPEN: 'never', GITHUB_REPOSITORY: 'microsoft/playwright-example-for-test', GITHUB_RUN_ID: 'example-run-id', GITHUB_SERVER_URL: 'https://playwright.dev', GITHUB_SHA: 'example-sha' }, undefined); diff --git a/tests/playwright-test/web-server.spec.ts b/tests/playwright-test/web-server.spec.ts index 10c4c5ccf9058..f0b5ef9087b02 100644 --- a/tests/playwright-test/web-server.spec.ts +++ b/tests/playwright-test/web-server.spec.ts @@ -21,7 +21,7 @@ import { test, expect } from './playwright-test-fixtures'; const SIMPLE_SERVER_PATH = path.join(__dirname, 'assets', 'simple-server.js'); test('should create a server', async ({ runInlineTest }, { workerIndex }) => { - const port = workerIndex + 10500; + const port = workerIndex * 2 + 10500; const result = await runInlineTest({ 'test.spec.ts': ` const { test } = pwt; @@ -87,7 +87,7 @@ test('should create a server', async ({ runInlineTest }, { workerIndex }) => { }); test('should create a server with environment variables', async ({ runInlineTest }, { workerIndex }) => { - const port = workerIndex + 10500; + const port = workerIndex * 2 + 10500; const result = await runInlineTest({ 'test.spec.ts': ` const { test } = pwt; @@ -117,7 +117,7 @@ test('should create a server with environment variables', async ({ runInlineTest }); test('should default cwd to config directory', async ({ runInlineTest }, testInfo) => { - const port = testInfo.workerIndex + 10500; + const port = testInfo.workerIndex * 2 + 10500; const configDir = testInfo.outputPath('foo'); const relativeSimpleServerPath = path.relative(configDir, SIMPLE_SERVER_PATH); const result = await runInlineTest({ @@ -145,7 +145,7 @@ test('should default cwd to config directory', async ({ runInlineTest }, testInf }); test('should resolve cwd wrt config directory', async ({ runInlineTest }, testInfo) => { - const port = testInfo.workerIndex + 10500; + const port = testInfo.workerIndex * 2 + 10500; const testdir = testInfo.outputPath(); const relativeSimpleServerPath = path.relative(testdir, SIMPLE_SERVER_PATH); const result = await runInlineTest({ @@ -175,7 +175,7 @@ test('should resolve cwd wrt config directory', async ({ runInlineTest }, testIn test('should create a server with url', async ({ runInlineTest }, { workerIndex }) => { - const port = workerIndex + 10500; + const port = workerIndex * 2 + 10500; const result = await runInlineTest({ 'test.spec.ts': ` const { test } = pwt; @@ -200,7 +200,7 @@ test('should create a server with url', async ({ runInlineTest }, { workerIndex }); test('should time out waiting for a server', async ({ runInlineTest }, { workerIndex }) => { - const port = workerIndex + 10500; + const port = workerIndex * 2 + 10500; const result = await runInlineTest({ 'test.spec.ts': ` const { test } = pwt; @@ -225,7 +225,7 @@ test('should time out waiting for a server', async ({ runInlineTest }, { workerI }); test('should time out waiting for a server with url', async ({ runInlineTest }, { workerIndex }) => { - const port = workerIndex + 10500; + const port = workerIndex * 2 + 10500; const result = await runInlineTest({ 'test.spec.ts': ` const { test } = pwt; @@ -250,7 +250,7 @@ test('should time out waiting for a server with url', async ({ runInlineTest }, }); test('should be able to specify the baseURL without the server', async ({ runInlineTest }, { workerIndex }) => { - const port = workerIndex + 10500; + const port = workerIndex * 2 + 10500; const server = http.createServer((req: http.IncomingMessage, res: http.ServerResponse) => { res.end('hello'); }); @@ -313,7 +313,7 @@ test('should be able to specify a custom baseURL with the server', async ({ runI }); test('should be able to use an existing server when reuseExistingServer:true', async ({ runInlineTest }, { workerIndex }) => { - const port = workerIndex + 10500; + const port = workerIndex * 2 + 10500; const server = http.createServer((req: http.IncomingMessage, res: http.ServerResponse) => { res.end('hello'); }); @@ -346,7 +346,7 @@ test('should be able to use an existing server when reuseExistingServer:true', a }); test('should throw when a server is already running on the given port and strict is true', async ({ runInlineTest }, { workerIndex }) => { - const port = workerIndex + 10500; + const port = workerIndex * 2 + 10500; const server = http.createServer((req: http.IncomingMessage, res: http.ServerResponse) => { res.end('hello'); }); @@ -378,7 +378,7 @@ test('should throw when a server is already running on the given port and strict for (const host of ['localhost', '127.0.0.1', '0.0.0.0']) { test(`should detect the server if a web-server is already running on ${host}`, async ({ runInlineTest }, { workerIndex }) => { - const port = workerIndex + 10500; + const port = workerIndex * 2 + 10500; const server = http.createServer((req: http.IncomingMessage, res: http.ServerResponse) => { res.end('hello'); }); @@ -581,7 +581,7 @@ test.describe('baseURL with plugins', () => { }); test('should treat 3XX as available server', async ({ runInlineTest }, { workerIndex }) => { - const port = workerIndex + 10500; + const port = workerIndex * 2 + 10500; const result = await runInlineTest({ 'test.spec.ts': ` const { test } = pwt;