Skip to content

Commit

Permalink
feat: Exit with 1 if a test failed (#58)
Browse files Browse the repository at this point in the history
  • Loading branch information
Jason3S authored Jul 13, 2024
1 parent 706d3b2 commit b7d8519
Show file tree
Hide file tree
Showing 10 changed files with 364 additions and 264 deletions.
1 change: 0 additions & 1 deletion examples/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
"description": "Example perf tests.",
"type": "module",
"private": true,
"packageManager": "pnpm@8.15.7",
"engines": {
"node": ">=18"
},
Expand Down
19 changes: 19 additions & 0 deletions examples/src/fail.perf.mts
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
import { suite } from 'perf-insight';

// Use 2 seconds as the default timeout for tests in the suite.
// The `--timeout` option can override this value.
const defaultTimeout = 100;

// ts-check
suite('fail', 'Example with tests that fail or throw exceptions.', async (test) => {
test('ok', () => {
let a = '';
for (let i = 0; i < 1000; ++i) {
a = a + 'a';
}
});

test('fail', () => {
throw new Error('This test failed.');
});
}).setTimeout(defaultTimeout); // set the default timeout for this suite.
4 changes: 2 additions & 2 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@
"@eslint/js": "^9.7.0",
"@tsconfig/node20": "^20.1.4",
"@types/node": "^20.14.10",
"@vitest/coverage-v8": "^1.6.0",
"@vitest/coverage-v8": "^2.0.2",
"cspell": "^8.10.4",
"cspell-trie-lib": "^8.10.4",
"eslint": "^9.7.0",
Expand All @@ -68,6 +68,6 @@
"typescript": "^5.5.3",
"typescript-eslint": "^7.16.0",
"vite": "^5.3.3",
"vitest": "^1.6.0"
"vitest": "^2.0.2"
}
}
3 changes: 1 addition & 2 deletions packages/perf-insight/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,8 @@
"version": "1.1.1",
"description": "Performance benchmarking tool for NodeJS.",
"type": "module",
"packageManager": "pnpm@8.15.7",
"engines": {
"node": ">=18"
"node": ">=18.18"
},
"bin": {
"insight": "./bin.mjs",
Expand Down
13 changes: 11 additions & 2 deletions packages/perf-insight/src/app.mts
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ interface AppOptions {
suite?: string[];
test?: string[];
register?: string[];
failFast?: boolean;
}

const urlRunnerCli = new URL('./runBenchmarkCli.mjs', import.meta.url).toString();
Expand All @@ -34,6 +35,7 @@ export async function app(program = defaultCommand): Promise<Command> {
.option('-t, --timeout <timeout>', 'Override the timeout for each test suite.', (v) => Number(v))
.option('-s, --suite <suite...>', 'Run only matching suites.', appendValue)
.option('-T, --test <test...>', 'Run only matching test found in suites', appendValue)
.option('--fail-fast', 'Stop on first failure.', false)
.option('--repeat <count>', 'Repeat the tests.', (v) => Number(v), 1)
.option('--register <loader>', 'Register a module loader. (e.g. ts-node/esm)', appendValue)
.action(async (suiteNamesToRun: string[], options: AppOptions, command: Command) => {
Expand Down Expand Up @@ -62,7 +64,7 @@ export async function app(program = defaultCommand): Promise<Command> {

await spawnRunners(files, options);

console.log(chalk.green('done.'));
process.exitCode ? console.log(chalk.red('failed.')) : console.log(chalk.green('done.'));
});

program.showHelpAfterError();
Expand Down Expand Up @@ -97,9 +99,16 @@ async function spawnRunners(files: string[], options: AppOptions): Promise<void>
for (const file of files) {
try {
const code = await spawnRunner([file, ...cliOptions]);
code && console.error('Runner failed with "%s" code: %d', file, code);
if (code) {
// console.error('Runner failed with "%s" code: %d', file, code);
process.exitCode ??= code;
if (options.failFast) {
break;
}
}
} catch (e) {
console.error('Failed to spawn runner.', e);
process.exitCode ??= 1;
}
}
}
Expand Down
2 changes: 2 additions & 0 deletions packages/perf-insight/src/perfSuite.mts
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,7 @@ export interface RunnerResult {
name: string;
description: string | undefined;
results: TestResult[];
hadFailures: boolean;
}

type TestMethod = () => void | Promise<void> | unknown | Promise<unknown>;
Expand Down Expand Up @@ -473,6 +474,7 @@ async function runTests(
name,
description,
results,
hadFailures: results.some((r) => !!r.error),
};
}

Expand Down
53 changes: 41 additions & 12 deletions packages/perf-insight/src/run.mts
Original file line number Diff line number Diff line change
Expand Up @@ -11,22 +11,33 @@ export interface RunOptions {
tests?: string[] | undefined;
}

export interface RunBenchmarkSuitesResult {
hadFailures: boolean;
numSuitesRun: number;
}

/**
*
* @param suiteNames
* @param options
*/
export async function runBenchmarkSuites(suiteToRun?: (string | PerfSuite)[], options?: RunOptions) {
export async function runBenchmarkSuites(
suiteToRun?: (string | PerfSuite)[],
options?: RunOptions,
): Promise<RunBenchmarkSuitesResult> {
const suites = getActiveSuites();

let numSuitesRun = 0;
let showRepeatMsg = false;
let hadErrors = false;

for (let repeat = options?.repeat || 1; repeat > 0; repeat--) {
if (showRepeatMsg) {
console.log(chalk.yellow(`Repeating tests: ${repeat} more time${repeat > 1 ? 's' : ''}.`));
}
numSuitesRun = await runTestSuites(suites, suiteToRun || suites, options || {});
const r = await runTestSuites(suites, suiteToRun || suites, options || {});
numSuitesRun = r.numSuitesRun;
hadErrors ||= r.hadFailures;
if (!numSuitesRun) break;
showRepeatMsg = true;
}
Expand All @@ -44,18 +55,32 @@ export async function runBenchmarkSuites(suiteToRun?: (string | PerfSuite)[], op
.map((line) => ` ${line}`)
.join('\n'),
);

hadErrors = true;
}

return { hadFailures: hadErrors, numSuitesRun };
}

interface Result {
hadFailures: boolean;
}

interface RunTestSuitesResults extends Result {
numSuitesRun: number;
}

async function runTestSuites(
suites: PerfSuite[],
suitesToRun: (string | PerfSuite)[],
options: RunOptions,
): Promise<number> {
): Promise<RunTestSuitesResults> {
const timeout = options.timeout || undefined;
const suitesRun = new Set<PerfSuite>();
let hadFailures = false;

async function _runSuite(suites: PerfSuite[]) {
async function _runSuite(suites: PerfSuite[]): Promise<Result> {
let hadFailures = false;
for (const suite of suites) {
if (suitesRun.has(suite)) continue;
if (!filterSuite(suite)) {
Expand All @@ -64,32 +89,36 @@ async function runTestSuites(
}
suitesRun.add(suite);
console.log(chalk.green(`Running Perf Suite: ${suite.name}`));
await suite.setTimeout(timeout).runTests({ tests: options.tests });
const result = await suite.setTimeout(timeout).runTests({ tests: options.tests });
if (result.hadFailures) {
hadFailures = true;
}
}

return { hadFailures: hadFailures };
}

async function runSuite(name: string | PerfSuite) {
async function runSuite(name: string | PerfSuite): Promise<Result> {
if (typeof name !== 'string') {
return await _runSuite([name]);
}

if (name === 'all') {
await _runSuite(suites);
return;
return await _runSuite(suites);
}
const matching = suites.filter((suite) => suite.name.toLowerCase().startsWith(name.toLowerCase()));
if (!matching.length) {
console.log(chalk.red(`Unknown test method: ${name}`));
return;
return { hadFailures: true };
}
await _runSuite(matching);
return await _runSuite(matching);
}

for (const name of suitesToRun) {
await runSuite(name);
hadFailures ||= (await runSuite(name)).hadFailures;
}

return suitesRun.size;
return { hadFailures, numSuitesRun: suitesRun.size };

function filterSuite(suite: PerfSuite): boolean {
const { suites } = options;
Expand Down
37 changes: 30 additions & 7 deletions packages/perf-insight/src/runBenchmarkCli.mts
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,16 @@ import { parseArgs } from 'node:util';

import { runBenchmarkSuites } from './run.mjs';

const cwdUrl = pathToFileURL(process.cwd() + '/');
interface RunResult {
error?: Error;
/**
* Indicates if there were any failures in the benchmark suites.
* This is not an error, since nothing went wrong in the insight code.
*/
hadFailures: boolean;
}

async function run(args: string[]) {
export async function run(args: string[]): Promise<RunResult> {
const parseConfig = {
args,
strict: true,
Expand All @@ -22,16 +29,18 @@ async function run(args: string[]) {
test: { type: 'string', short: 'T', multiple: true },
suite: { type: 'string', short: 'S', multiple: true },
register: { type: 'string', multiple: true },
root: { type: 'string' },
},
} as const satisfies ParseArgsConfig;

const parsed = parseArgs(parseConfig);
const cwdUrl = parsed.values.root ? new URL(parsed.values.root) : pathToFileURL(process.cwd() + '/');

const repeat = Number(parsed.values['repeat'] || '0') || undefined;
const timeout = Number(parsed.values['timeout'] || '0') || undefined;
const tests = parsed.values['test'];
const suites = parsed.values['suite'];
await registerLoaders(parsed.values['register']);
await registerLoaders(parsed.values['register'], cwdUrl);

const errors: Error[] = [];

Expand All @@ -54,13 +63,21 @@ async function run(args: string[]) {
console.error('Errors:');
errors.forEach((err) => console.error('- %s\n%o', err.message, err.cause));
process.exitCode = 1;
return;
return { error: errors[0], hadFailures: true };
}

await runBenchmarkSuites(undefined, { repeat, timeout, tests, suites });
try {
const r = await runBenchmarkSuites(undefined, { repeat, timeout, tests, suites });
process.exitCode = process.exitCode || (r.hadFailures ? 1 : 0);
return { error: errors[0], hadFailures: r.hadFailures };
} catch (e) {
// console.error('Failed to run benchmark suites.', e);
process.exitCode = 1;
return { error: e as Error, hadFailures: true };
}
}

async function registerLoaders(loaders: string[] | undefined) {
async function registerLoaders(loaders: string[] | undefined, cwdUrl: URL) {
if (!loaders?.length) return;

const module = await import('module');
Expand All @@ -79,4 +96,10 @@ async function registerLoaders(loaders: string[] | undefined) {
loaders.forEach(registerLoader);
}

run(process.argv.slice(2));
// console.error('args: %o', process.argv);

if ((process.argv[1] ?? '').endsWith('runBenchmarkCli.mjs')) {
run(process.argv.slice(2)).then((result) => {
if (result.error) process.exitCode = 1;
});
}
21 changes: 21 additions & 0 deletions packages/perf-insight/src/runBenchmarkCli.test.mts
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
import { describe, expect, test } from 'vitest';

import { run } from './runBenchmarkCli.mjs';

describe('runBenchmarkCli', () => {
test.each`
file | args | root
${'src/exampleMap.perf.mts'} | ${'--suite map -t 500'} | ${'../../../examples/'}
`('runBenchmarkCli $file $args', async ({ file, args, root }) => {
expect(run).toBeTypeOf('function');

args = typeof args === 'string' ? args.split(/\s+/g) : args;
const r = new URL(root, import.meta.url).href;
const fileUrl = new URL(file, r).href;

await expect(run([fileUrl, '--root', r, ...args])).resolves.toEqual({
error: undefined,
hadFailures: true,
});
});
});
Loading

0 comments on commit b7d8519

Please sign in to comment.