diff --git a/docs/index.md b/docs/index.md index cc3d9eb2ab..b6614016a3 100644 --- a/docs/index.md +++ b/docs/index.md @@ -915,6 +915,10 @@ To ensure your tests aren't leaving messes around, here are some ideas to get st - Try something like [wtfnode][npm-wtfnode] - Use [`.only`](#exclusive-tests) until you find the test that causes Mocha to hang +### `--pass-on-failing-test-suite` + +If set on true, Mocha returns zero code even if there are failed tests during run. + ### `--fail-zero` > _New in v9.1.0_ Fail test run if no tests are encountered with `exit-code: 1`. diff --git a/lib/cli/run-helpers.js b/lib/cli/run-helpers.js index 4ea1a403c8..e31563afbc 100644 --- a/lib/cli/run-helpers.js +++ b/lib/cli/run-helpers.js @@ -21,62 +21,49 @@ const {UnmatchedFile} = require('./collect-files'); /** * Exits Mocha when tests + code under test has finished execution (default) - * @param {number} failOnFailingTestSuite - Exit code; typically # of failures + * @param {number} code - Exit code; typically # of failures * @ignore * @private */ -const exitMochaLater = failOnFailingTestSuite => { - return code => { - process.on('exit', () => { - if (failOnFailingTestSuite) { - process.exitCode = Math.min(code, 255); - } else { - process.exitCode = 0; - } - }); - }; +const exitMochaLater = code => { + process.on('exit', () => { + process.exitCode = Math.min(code, 255); + }); }; /** * Exits Mocha when Mocha itself has finished execution, regardless of * what the tests or code under test is doing. - * @param {number} failOnFailingTestSuite - Exit code; typically # of failures + * @param {number} code - Exit code; typically # of failures * @ignore * @private */ -const exitMocha = failOnFailingTestSuite => { - return code => { - let clampedCode; - if (failOnFailingTestSuite) { - clampedCode = Math.min(code, 255); - } else { - clampedCode = 0; +const exitMocha = code => { + const clampedCode = Math.min(code, 255); + let draining = 0; + + // Eagerly set the process's exit code in case stream.write doesn't + // execute its callback before the process terminates. + process.exitCode = clampedCode; + + // flush output for Node.js Windows pipe bug + // https://github.com/joyent/node/issues/6247 is just one bug example + // https://github.com/visionmedia/mocha/issues/333 has a good discussion + const done = () => { + if (!draining--) { + process.exit(clampedCode); } - let draining = 0; - - // Eagerly set the process's exit code in case stream.write doesn't - // execute its callback before the process terminates. - process.exitCode = clampedCode; - - // flush output for Node.js Windows pipe bug - // https://github.com/joyent/node/issues/6247 is just one bug example - // https://github.com/visionmedia/mocha/issues/333 has a good discussion - const done = () => { - if (!draining--) { - process.exit(clampedCode); - } - }; + }; - const streams = [process.stdout, process.stderr]; + const streams = [process.stdout, process.stderr]; - streams.forEach(stream => { - // submit empty write request and wait for completion - draining += 1; - stream.write('', done); - }); + streams.forEach(stream => { + // submit empty write request and wait for completion + draining += 1; + stream.write('', done); + }); - done(); - }; + done(); }; /** @@ -152,7 +139,7 @@ const handleUnmatchedFiles = (mocha, unmatchedFiles) => { * @param {Mocha} mocha - Mocha instance * @param {Options} [opts] - Command line options * @param {boolean} [opts.exit] - Whether or not to force-exit after tests are complete - * @param {boolean} [opts.failOnFailingTestSuite] - Whether or not to fail test run if tests were failed + * @param {boolean} [opts.passOnFailingTestSuite] - Whether or not to fail test run if tests were failed * @param {Object} fileCollectParams - Parameters that control test * file collection. See `lib/cli/collect-files.js`. * @returns {Promise} @@ -160,7 +147,7 @@ const handleUnmatchedFiles = (mocha, unmatchedFiles) => { */ const singleRun = async ( mocha, - {exit, failOnFailingTestSuite}, + {exit, passOnFailingTestSuite}, fileCollectParams ) => { const fileCollectionObj = collectFiles(fileCollectParams); @@ -175,9 +162,7 @@ const singleRun = async ( // handles ESM modules await mocha.loadFilesAsync(); return mocha.run( - exit - ? exitMocha(failOnFailingTestSuite) - : exitMochaLater(failOnFailingTestSuite) + createExitHandler({exit, passOnFailingTestSuite}) ); }; @@ -209,9 +194,7 @@ const parallelRun = async (mocha, options, fileCollectParams) => { // note that we DO NOT load any files here; this is handled by the worker return mocha.run( - options.exit - ? exitMocha(options.failOnFailingTestSuite) - : exitMochaLater(options.failOnFailingTestSuite) + createExitHandler(options) ); }; @@ -308,3 +291,15 @@ exports.validateLegacyPlugin = (opts, pluginType, map = {}) => { } } }; + +const createExitHandler = ({ exit, passOnFailingTestSuite }) => { + return code => { + const clampedCode = passOnFailingTestSuite + ? 0 + : Math.min(code, 255); + + return exit + ? exitMocha(clampedCode) + : exitMochaLater(clampedCode); + }; +}; diff --git a/lib/cli/run-option-metadata.js b/lib/cli/run-option-metadata.js index 9084dca581..83aa70dd7a 100644 --- a/lib/cli/run-option-metadata.js +++ b/lib/cli/run-option-metadata.js @@ -35,7 +35,7 @@ const TYPES = (exports.types = { 'diff', 'dry-run', 'exit', - 'fail-on-failing-test-suite', + 'pass-on-failing-test-suite', 'fail-zero', 'forbid-only', 'forbid-pending', diff --git a/lib/cli/run.js b/lib/cli/run.js index fccd89210f..3389e6df6e 100644 --- a/lib/cli/run.js +++ b/lib/cli/run.js @@ -98,9 +98,9 @@ exports.builder = yargs => requiresArg: true, coerce: list }, - 'fail-on-failing-test-suite': { - default: true, - description: 'Fail test run if tests were failed', + 'pass-on-failing-test-suite': { + default: false, + description: 'Not fail test run if tests were failed', group: GROUPS.RULES }, 'fail-zero': { diff --git a/lib/mocha.js b/lib/mocha.js index 6940e22f63..c6ee248561 100644 --- a/lib/mocha.js +++ b/lib/mocha.js @@ -157,7 +157,7 @@ exports.run = function (...args) { * @param {boolean} [options.delay] - Delay root suite execution? * @param {boolean} [options.diff] - Show diff on failure? * @param {boolean} [options.dryRun] - Report tests without running them? - * @param {boolean} [options.failOnFailingTestSuite] - Fail test run if tests were failed? + * @param {boolean} [options.passOnFailingTestSuite] - Fail test run if tests were failed? * @param {boolean} [options.failZero] - Fail test run if zero tests? * @param {string} [options.fgrep] - Test filter given string. * @param {boolean} [options.forbidOnly] - Tests marked `only` fail the suite? @@ -217,7 +217,7 @@ function Mocha(options = {}) { 'delay', 'diff', 'dryRun', - 'failOnFailingTestSuite', + 'passOnFailingTestSuite', 'failZero', 'forbidOnly', 'forbidPending', @@ -876,13 +876,13 @@ Mocha.prototype.failZero = function (failZero) { * Fail test run if tests were failed. * * @public - * @see [CLI option](../#-fail-on-failing-test-suite) - * @param {boolean} [failOnFailingTestSuite=true] - Whether to fail test run. + * @see [CLI option](../#-pass-on-failing-test-suite) + * @param {boolean} [passOnFailingTestSuite=false] - Whether to fail test run. * @return {Mocha} this * @chainable */ -Mocha.prototype.failOnFailingTestSuite = function(failOnFailingTestSuite) { - this.options.failOnFailingTestSuite = failOnFailingTestSuite !== false; +Mocha.prototype.passOnFailingTestSuite = function(passOnFailingTestSuite) { + this.options.passOnFailingTestSuite = passOnFailingTestSuite === true; return this; }; diff --git a/test/integration/fixtures/require-undefined.fixture.js b/test/integration/fixtures/failing-sync.fixture.js similarity index 62% rename from test/integration/fixtures/require-undefined.fixture.js rename to test/integration/fixtures/failing-sync.fixture.js index 3afbeab5ba..f81f9653a7 100644 --- a/test/integration/fixtures/require-undefined.fixture.js +++ b/test/integration/fixtures/failing-sync.fixture.js @@ -1,5 +1,3 @@ -// this generic fixture does nothing special and will be used if no fixture is supplied - 'use strict'; var assert = require('assert'); diff --git a/test/integration/options/failOnFailingTestSuite.spec.js b/test/integration/options/failOnFailingTestSuite.spec.js deleted file mode 100644 index 7834ef693b..0000000000 --- a/test/integration/options/failOnFailingTestSuite.spec.js +++ /dev/null @@ -1,22 +0,0 @@ -'use strict'; - -var helpers = require('../helpers'); -var runMochaJSON = helpers.runMochaJSON; - -describe('--fail-on-failing-test-suite', function() { - var args = ['--fail-on-failing-test-suite=false']; - - it('should success', function(done) { - var fixture = 'require-undefined.fixture.js'; - runMochaJSON(fixture, args, function(err, res) { - if (err) { - return done(err); - } - - expect(res, 'to have passed test count', 0) - .and('to have test count', 1) - .and('to have exit code', 0); - done(); - }); - }); -}); diff --git a/test/integration/options/passOnFailingTestSuite.spec.js b/test/integration/options/passOnFailingTestSuite.spec.js new file mode 100644 index 0000000000..9f902da72f --- /dev/null +++ b/test/integration/options/passOnFailingTestSuite.spec.js @@ -0,0 +1,40 @@ +'use strict'; + +var helpers = require('../helpers'); +var runMochaJSON = helpers.runMochaJSON; + +describe('Enabled --pass-on-failing-test-suite', function() { + var args = ['--pass-on-failing-test-suite=true']; + + it('Test should finish with zero code with disabled option', function(done) { + var fixture = 'failing-sync.fixture.js'; + runMochaJSON(fixture, args, function(err, res) { + if (err) { + return done(err); + } + + expect(res, 'to have passed test count', 0) + .and('to have test count', 1) + .and('to have exit code', 0); + done(); + }); + }); +}); + +describe('Disabled --pass-on-failing-test-suite', function() { + var args = ['--pass-on-failing-test-suite=false']; + + it('Test should return non-zero code with enabled option', function(done) { + var fixture = 'failing-sync.fixture.js'; + runMochaJSON(fixture, args, function(err, res) { + if (err) { + return done(err); + } + + expect(res, 'to have passed test count', 0) + .and('to have test count', 1) + .and('to have exit code', 1); + done(); + }); + }); +}); diff --git a/test/unit/mocha.spec.js b/test/unit/mocha.spec.js index 9a768d62d3..e0f603574e 100644 --- a/test/unit/mocha.spec.js +++ b/test/unit/mocha.spec.js @@ -376,24 +376,24 @@ describe('Mocha', function () { }); }); - describe('failOnFailingTestSuite()', function() { - it('should set the failOnFailingTestSuite option to true', function() { - mocha.failOnFailingTestSuite(); + describe('passOnFailingTestSuite()', function() { + it('should set the passOnFailingTestSuite option to false', function() { + mocha.passOnFailingTestSuite(); expect( mocha.options, 'to have property', - 'failOnFailingTestSuite', - true + 'passOnFailingTestSuite', + false ); }); - it('should set the failOnFailingTestSuite option to false', function() { - mocha.failOnFailingTestSuite(false); + it('should set the passOnFailingTestSuite option to true', function() { + mocha.passOnFailingTestSuite(true); expect( mocha.options, 'to have property', - 'failOnFailingTestSuite', - false + 'passOnFailingTestSuite', + true ); }); });