From 3e805bc082c8da90c33a4dff9d3e1b0804a4386d Mon Sep 17 00:00:00 2001 From: Bryan Nicholls Date: Thu, 6 Dec 2018 12:48:10 -0500 Subject: [PATCH] allow `bail` setting to control when to bail out of a failing test run (#7335) --- CHANGELOG.md | 1 + TestUtils.js | 2 +- docs/CLI.md | 2 +- docs/Configuration.md | 8 +-- docs/WatchPlugins.md | 3 +- .../__snapshots__/show_config.test.js.snap | 2 +- packages/jest-cli/src/TestScheduler.js | 5 +- .../src/__tests__/TestScheduler.test.js | 64 +++++++++++++++++++ packages/jest-cli/src/cli/args.js | 4 +- .../__tests__/__snapshots__/init.test.js.snap | 4 +- .../jest-cli/src/lib/update_global_config.js | 6 +- packages/jest-config/src/Defaults.js | 2 +- packages/jest-config/src/Descriptions.js | 2 +- packages/jest-config/src/ValidConfig.js | 2 +- packages/jest-config/src/normalize.js | 15 ++++- .../src/__tests__/fixtures/jestConfig.js | 4 +- types/Argv.js | 2 +- types/Config.js | 6 +- 18 files changed, 108 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1953f9889dd0..d77318d81d1e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,7 @@ - `[jest-config]` Add `haste.computeSha1` option to compute the sha-1 of the files in the haste map ([#7345](https://github.com/facebook/jest/pull/7345)) - `[expect]` `expect(Infinity).toBeCloseTo(Infinity)` Treats `Infinity` as equal in toBeCloseTo matcher ([#7405](https://github.com/facebook/jest/pull/7405)) - `[jest-worker]` Add node worker-thread support to jest-worker ([#7408](https://github.com/facebook/jest/pull/7408)) +- `[jest-config]` Allow `bail` setting to be configured with a number allowing tests to abort after `n` of failures ([#7335](https://github.com/facebook/jest/pull/7335)) ### Fixes diff --git a/TestUtils.js b/TestUtils.js index 3734d8ec5a55..9203fad967e4 100644 --- a/TestUtils.js +++ b/TestUtils.js @@ -12,7 +12,7 @@ import type {GlobalConfig, ProjectConfig} from 'types/Config'; const DEFAULT_GLOBAL_CONFIG: GlobalConfig = { - bail: false, + bail: 0, changedFilesWithAncestor: false, changedSince: '', collectCoverage: false, diff --git a/docs/CLI.md b/docs/CLI.md index 95ae33e053d5..f19cf7579a1a 100644 --- a/docs/CLI.md +++ b/docs/CLI.md @@ -97,7 +97,7 @@ When you run `jest` with an argument, that argument is treated as a regular expr ### `--bail` -Alias: `-b`. Exit the test suite immediately upon the first failing test suite. +Alias: `-b`. Exit the test suite immediately upon `n` number of failing test suite. Defaults to `1`. ### `--cache` diff --git a/docs/Configuration.md b/docs/Configuration.md index 8cc5493f7864..0c2a96832dcf 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -29,7 +29,7 @@ When using the `--config` option, the JSON file must not contain a "jest" key: ```json { - "bail": true, + "bail": 1, "verbose": true } ``` @@ -99,11 +99,11 @@ _Note: Core modules, like `fs`, are not mocked by default. They can be mocked ex _Note: Automocking has a performance cost most noticeable in large projects. See [here](troubleshooting.html#tests-are-slow-when-leveraging-automocking) for details and a workaround._ -### `bail` [boolean] +### `bail` [number | boolean] -Default: `false` +Default: `0` -By default, Jest runs all tests and produces all errors into the console upon completion. The bail config option can be used here to have Jest stop running tests after the first failure. +By default, Jest runs all tests and produces all errors into the console upon completion. The bail config option can be used here to have Jest stop running tests after `n` failures. Setting bail to `true` is the same as setting bail to `1`. ### `browser` [boolean] diff --git a/docs/WatchPlugins.md b/docs/WatchPlugins.md index 695b35bcc476..05818e95b555 100644 --- a/docs/WatchPlugins.md +++ b/docs/WatchPlugins.md @@ -1,7 +1,6 @@ --- id: watch-plugins title: Watch Plugins -original_id: watch-plugins --- The Jest watch plugin system provides a way to hook into specific parts of Jest and to define watch mode menu prompts that execute code on key press. Combined, these features allow you to develop interactive experiences custom for your workflow. @@ -155,7 +154,7 @@ class MyWatchPlugin { For stability and safety reasons, only part of the global configuration keys can be updated with `updateConfigAndRun`. The current white list is as follows: -- [`bail`](configuration.html#bail-boolean) +- [`bail`](configuration.html#bail-number-boolean) - [`collectCoverage`](configuration.html#collectcoverage-boolean) - [`collectCoverageFrom`](configuration.html#collectcoveragefrom-array) - [`collectCoverageOnlyFrom`](configuration.html#collectcoverageonlyfrom-array) diff --git a/e2e/__tests__/__snapshots__/show_config.test.js.snap b/e2e/__tests__/__snapshots__/show_config.test.js.snap index 9e3a43419f56..3baf9f6336b1 100644 --- a/e2e/__tests__/__snapshots__/show_config.test.js.snap +++ b/e2e/__tests__/__snapshots__/show_config.test.js.snap @@ -77,7 +77,7 @@ exports[`--showConfig outputs config info and exits 1`] = ` } ], \\"globalConfig\\": { - \\"bail\\": false, + \\"bail\\": 0, \\"changedFilesWithAncestor\\": false, \\"collectCoverage\\": false, \\"collectCoverageFrom\\": null, diff --git a/packages/jest-cli/src/TestScheduler.js b/packages/jest-cli/src/TestScheduler.js index 498f90b98f98..682cde647d1e 100644 --- a/packages/jest-cli/src/TestScheduler.js +++ b/packages/jest-cli/src/TestScheduler.js @@ -351,7 +351,10 @@ export default class TestScheduler { aggregatedResults: AggregatedResult, watcher: TestWatcher, ): Promise { - if (this._globalConfig.bail && aggregatedResults.numFailedTests !== 0) { + if ( + this._globalConfig.bail !== 0 && + aggregatedResults.numFailedTests >= this._globalConfig.bail + ) { if (watcher.isWatchMode()) { watcher.setState({interrupted: true}); } else { diff --git a/packages/jest-cli/src/__tests__/TestScheduler.test.js b/packages/jest-cli/src/__tests__/TestScheduler.test.js index e700d4d17aa4..d73eb9684f2d 100644 --- a/packages/jest-cli/src/__tests__/TestScheduler.test.js +++ b/packages/jest-cli/src/__tests__/TestScheduler.test.js @@ -27,6 +27,10 @@ jest.mock('jest-runner-parallel', () => jest.fn(() => mockParallelRunner), { virtual: true, }); +beforeEach(() => { + mockSerialRunner.runTests.mockClear(); +}); + test('config for reporters supports `default`', () => { const undefinedReportersScheduler = new TestScheduler( { @@ -116,3 +120,63 @@ test('schedule tests run in serial if the runner flags them', async () => { expect(mockSerialRunner.runTests).toHaveBeenCalled(); expect(mockSerialRunner.runTests.mock.calls[0][5].serial).toBeTruthy(); }); + +test('should bail after `n` failures', async () => { + const scheduler = new TestScheduler({bail: 2}, {}); + const test = { + context: { + config: { + rootDir: './', + runner: 'jest-runner-serial', + }, + hasteFS: { + matchFiles: jest.fn(() => []), + }, + }, + path: './test/path.js', + }; + + const tests = [test]; + const setState = jest.fn(); + await scheduler.scheduleTests(tests, { + isInterrupted: jest.fn(), + isWatchMode: () => true, + setState, + }); + await mockSerialRunner.runTests.mock.calls[0][3](test, { + numFailingTests: 2, + snapshot: {}, + testResults: [{}], + }); + expect(setState).toBeCalledWith({interrupted: true}); +}); + +test('should not bail if less than `n` failures', async () => { + const scheduler = new TestScheduler({bail: 2}, {}); + const test = { + context: { + config: { + rootDir: './', + runner: 'jest-runner-serial', + }, + hasteFS: { + matchFiles: jest.fn(() => []), + }, + }, + path: './test/path.js', + }; + + const tests = [test]; + const setState = jest.fn(); + await scheduler.scheduleTests(tests, { + isInterrupted: jest.fn(), + isWatchMode: () => true, + setState, + }); + await mockSerialRunner.runTests.mock.calls[0][3](test, { + numFailingTests: 1, + snapshot: {}, + testResults: [{}], + }); + expect(setState).not.toBeCalled(); +}); diff --git a/packages/jest-cli/src/cli/args.js b/packages/jest-cli/src/cli/args.js index 7782f549e360..713a36f3c604 100644 --- a/packages/jest-cli/src/cli/args.js +++ b/packages/jest-cli/src/cli/args.js @@ -85,8 +85,8 @@ export const options = { bail: { alias: 'b', default: undefined, - description: 'Exit the test suite immediately upon the first failing test.', - type: 'boolean', + description: + 'Exit the test suite immediately after `n` number of failing tests.', }, browser: { default: undefined, diff --git a/packages/jest-cli/src/lib/__tests__/__snapshots__/init.test.js.snap b/packages/jest-cli/src/lib/__tests__/__snapshots__/init.test.js.snap index d0b63fa1c18a..31757651ce28 100644 --- a/packages/jest-cli/src/lib/__tests__/__snapshots__/init.test.js.snap +++ b/packages/jest-cli/src/lib/__tests__/__snapshots__/init.test.js.snap @@ -26,8 +26,8 @@ module.exports = { // All imported modules in your tests should be mocked automatically // automock: false, - // Stop running tests after the first failure - // bail: false, + // Stop running tests after \`n\` failures + // bail: 0, // Respect \\"browser\\" field in package.json when resolving modules // browser: false, diff --git a/packages/jest-cli/src/lib/update_global_config.js b/packages/jest-cli/src/lib/update_global_config.js index 3762eeaf6308..ccad3ed5c180 100644 --- a/packages/jest-cli/src/lib/update_global_config.js +++ b/packages/jest-cli/src/lib/update_global_config.js @@ -66,8 +66,10 @@ export default (globalConfig: GlobalConfig, options: Options): GlobalConfig => { !newConfig.testNamePattern && !newConfig.testPathPattern; - if (options.bail !== undefined) { - newConfig.bail = options.bail || false; + if (typeof options.bail === 'boolean') { + newConfig.bail = options.bail ? 1 : 0; + } else if (options.bail !== undefined) { + newConfig.bail = options.bail; } if (options.changedSince !== undefined) { diff --git a/packages/jest-config/src/Defaults.js b/packages/jest-config/src/Defaults.js index 85b29b28f577..cb82c808267a 100644 --- a/packages/jest-config/src/Defaults.js +++ b/packages/jest-config/src/Defaults.js @@ -17,7 +17,7 @@ const NODE_MODULES_REGEXP = replacePathSepForRegex(NODE_MODULES); export default ({ automock: false, - bail: false, + bail: 0, browser: false, cache: true, cacheDirectory: getCacheDirectory(), diff --git a/packages/jest-config/src/Descriptions.js b/packages/jest-config/src/Descriptions.js index 6ff881d1501d..304e16ff5aff 100644 --- a/packages/jest-config/src/Descriptions.js +++ b/packages/jest-config/src/Descriptions.js @@ -9,7 +9,7 @@ export default ({ automock: 'All imported modules in your tests should be mocked automatically', - bail: 'Stop running tests after the first failure', + bail: 'Stop running tests after `n` failures', browser: 'Respect "browser" field in package.json when resolving modules', cacheDirectory: 'The directory where Jest should store its cached dependency information', diff --git a/packages/jest-config/src/ValidConfig.js b/packages/jest-config/src/ValidConfig.js index c3da446f08a5..8ffb1455055c 100644 --- a/packages/jest-config/src/ValidConfig.js +++ b/packages/jest-config/src/ValidConfig.js @@ -17,7 +17,7 @@ const NODE_MODULES_REGEXP = replacePathSepForRegex(NODE_MODULES); export default ({ automock: false, - bail: false, + bail: (multipleValidOptions(false, 0): any), browser: false, cache: true, cacheDirectory: '/tmp/user/jest', diff --git a/packages/jest-config/src/normalize.js b/packages/jest-config/src/normalize.js index b29e3e690091..7a16917bcb06 100644 --- a/packages/jest-config/src/normalize.js +++ b/packages/jest-config/src/normalize.js @@ -617,8 +617,21 @@ export default function normalize(options: InitialOptions, argv: Argv) { break; } + case 'bail': { + if (typeof options[key] === 'boolean') { + value = options[key] ? 1 : 0; + } else if (typeof options[key] === 'string') { + value = 1; + // If Jest is invoked as `jest --bail someTestPattern` then need to + // move the pattern from the `bail` configuration and into `argv._` + // to be processed as an extra parameter + argv._.push(options[key]); + } else { + value = options[key]; + } + break; + } case 'automock': - case 'bail': case 'browser': case 'cache': case 'changedSince': diff --git a/packages/jest-validate/src/__tests__/fixtures/jestConfig.js b/packages/jest-validate/src/__tests__/fixtures/jestConfig.js index 31a04cedb0d2..fd98e6aa40e5 100644 --- a/packages/jest-validate/src/__tests__/fixtures/jestConfig.js +++ b/packages/jest-validate/src/__tests__/fixtures/jestConfig.js @@ -22,7 +22,7 @@ const NODE_MODULES_REGEXP = replacePathSepForRegex(NODE_MODULES); const defaultConfig = { automock: false, - bail: false, + bail: 0, browser: false, cacheDirectory: path.join(os.tmpdir(), 'jest'), clearMocks: false, @@ -62,7 +62,7 @@ const defaultConfig = { const validConfig = { automock: false, - bail: false, + bail: 0, browser: false, cache: true, cacheDirectory: '/tmp/user/jest', diff --git a/types/Argv.js b/types/Argv.js index 30a67a4fa44f..15492b5a477f 100644 --- a/types/Argv.js +++ b/types/Argv.js @@ -13,7 +13,7 @@ export type Argv = {| $0: string, all: boolean, automock: boolean, - bail: boolean, + bail: boolean | number, browser: boolean, cache: boolean, cacheDirectory: string, diff --git a/types/Config.js b/types/Config.js index 39eac782b764..9eff1a77796e 100644 --- a/types/Config.js +++ b/types/Config.js @@ -24,7 +24,7 @@ export type ConfigGlobals = Object; export type DefaultOptions = {| automock: boolean, - bail: boolean, + bail: number, browser: boolean, cache: boolean, cacheDirectory: Path, @@ -92,7 +92,7 @@ export type DefaultOptions = {| export type InitialOptions = { automock?: boolean, - bail?: boolean, + bail?: boolean | number, browser?: boolean, cache?: boolean, cacheDirectory?: Path, @@ -190,7 +190,7 @@ export type InitialOptions = { export type SnapshotUpdateState = 'all' | 'new' | 'none'; export type GlobalConfig = {| - bail: boolean, + bail: number, changedSince: string, changedFilesWithAncestor: boolean, collectCoverage: boolean,