fix(runner): Make exit code configurable when tests are failing

lib/browser_collection.js

@@ -46,7 +46,24 @@ class BrowserCollection {
     return this.browsers.map((browser) => browser.serialize())
   }
 
-  getResults () {
+  calculateExitCode (results, singleRunBrowserNotCaptured, failOnEmptyTestSuite, failOnFailingTestSuite) {
+    if (results.disconnected || singleRunBrowserNotCaptured) {
+      return 1
+    }
+    if (results.success + results.failed === 0 && !failOnEmptyTestSuite) {
+      return 0
+    }
+    if (results.error) {
+      return 1
+    }
+    if (failOnFailingTestSuite === false) {
+      // Tests executed without infrastructure error, exit with 0 independent of test status.
+      return 0
+    }
+    return results.failed ? 1 : 0
+  }
+
+  getResults (singleRunBrowserNotCaptured, failOnEmptyTestSuite, failOnFailingTestSuite) {
     const results = this.browsers.reduce((previous, current) => {
       previous.success += current.lastResult.success
       previous.failed += current.lastResult.failed
@@ -56,7 +73,7 @@ class BrowserCollection {
     }, {success: 0, failed: 0, error: false, disconnected: false, exitCode: 0})
 
     // compute exit status code
-    results.exitCode = results.failed || results.error || results.disconnected ? 1 : 0
+    results.exitCode = this.calculateExitCode(results, singleRunBrowserNotCaptured, failOnEmptyTestSuite, failOnFailingTestSuite)
 
     return results
   }

lib/cli.js

@@ -46,6 +46,10 @@ function processArgs (argv, options, fs, path) {
     options.failOnEmptyTestSuite = options.failOnEmptyTestSuite === 'true'
   }
 
+  if (helper.isString(options.failOnFailingTestSuite)) {
+    options.failOnFailingTestSuite = options.failOnFailingTestSuite === 'true'
+  }
+
   if (helper.isString(options.formatError)) {
     let required
     try {
@@ -196,6 +200,8 @@ function describeStart () {
     .describe('report-slower-than', '<integer> Report tests that are slower than given time [ms].')
     .describe('fail-on-empty-test-suite', 'Fail on empty test suite.')
     .describe('no-fail-on-empty-test-suite', 'Do not fail on empty test suite.')
+    .describe('fail-on-failing-test-suite', 'Fail on failing test suite.')
+    .describe('no-fail-on-failing-test-suite', 'Do not fail on failing test suite.')
     .describe('help', 'Print usage and options.')
 }
 

lib/server.js

@@ -247,14 +247,7 @@ class Server extends KarmaEventEmitter {
 
     const emitRunCompleteIfAllBrowsersDone = () => {
       if (Object.keys(singleRunDoneBrowsers).every((key) => singleRunDoneBrowsers[key])) {
-        const results = singleRunBrowsers.getResults()
-        if (singleRunBrowserNotCaptured) {
-          results.exitCode = 1
-        } else if (results.success + results.failed === 0 && !config.failOnEmptyTestSuite) {
-          results.exitCode = 0
-          this.log.warn('Test suite was empty.')
-        }
-        this.emit('run_complete', singleRunBrowsers, results)
+        this.emit('run_complete', singleRunBrowsers, singleRunBrowsers.getResults(singleRunBrowserNotCaptured, config.failOnEmptyTestSuite, config.failOnFailingTestSuite))
       }
     }
 

test/unit/browser_collection.spec.js

@@ -258,4 +258,90 @@ describe('BrowserCollection', () => {
       })
     })
   })
+
+  // ============================================================================
+  // server.calculateExitCode
+  // ============================================================================
+  describe('calculateExitCode', () => {
+    const EXIT_CODE_ERROR = 1
+    const EXIT_CODE_SUCCESS = 0
+
+    describe('no tests', () => {
+      const results = {
+        success: 0,
+        failed: 0,
+        error: true
+      }
+      it('shall pass if failOnEmptyTestSuite not is set', () => {
+        const res = collection.calculateExitCode(results)
+        expect(res).to.be.equal(EXIT_CODE_SUCCESS)
+      })
+      it('shall pass if failOnEmptyTestSuite is false', () => {
+        const res = collection.calculateExitCode(results, false, false)
+        expect(res).to.be.equal(EXIT_CODE_SUCCESS)
+      })
+      it('shall fail if failOnEmptyTestSuite is true', () => {
+        const res = collection.calculateExitCode(results, false, true)
+        expect(res).to.be.equal(EXIT_CODE_ERROR)
+      })
+      it('shall fail if failOnFailingTestSuite is set', () => {
+        const res = collection.calculateExitCode(results, false, true, true)
+        expect(res).to.be.equal(EXIT_CODE_ERROR)
+      })
+    })
+    describe('all test passed, no errors', () => {
+      const results = {
+        success: 10,
+        failed: 0,
+        error: false
+      }
+      it('shall fail if singleRunBrowserNotCaptured is true', () => {
+        const res = collection.calculateExitCode(results, true)
+        expect(res).to.be.equal(EXIT_CODE_ERROR)
+      })
+      it('shall pass if failOnEmptyTestSuite not is set', () => {
+        const res = collection.calculateExitCode(results, false)
+        expect(res).to.be.equal(EXIT_CODE_SUCCESS)
+      })
+      it('shall pass if failOnEmptyTestSuite not is set', () => {
+        const res = collection.calculateExitCode(results, false, false)
+        expect(res).to.be.equal(EXIT_CODE_SUCCESS)
+      })
+      it('shall pass if failOnFailingTestSuite is true', () => {
+        const res = collection.calculateExitCode(results, false, true, true)
+        expect(res).to.be.equal(EXIT_CODE_SUCCESS)
+      })
+      it('shall pass if failOnFailingTestSuite is false', () => {
+        const res = collection.calculateExitCode(results, false, true, false)
+        expect(res).to.be.equal(EXIT_CODE_SUCCESS)
+      })
+    })
+    describe('all test passed, with error', () => {
+      const results = {
+        success: 10,
+        failed: 5,
+        error: false
+      }
+      it('shall fail if singleRunBrowserNotCaptured is true', () => {
+        const res = collection.calculateExitCode(results, true)
+        expect(res).to.be.equal(EXIT_CODE_ERROR)
+      })
+      it('shall fail if failOnEmptyTestSuite not is set', () => {
+        const res = collection.calculateExitCode(results, false)
+        expect(res).to.be.equal(EXIT_CODE_ERROR)
+      })
+      it('shall fail if failOnEmptyTestSuite not is set', () => {
+        const res = collection.calculateExitCode(results, false, false)
+        expect(res).to.be.equal(EXIT_CODE_ERROR)
+      })
+      it('shall fail if failOnFailingTestSuite is true', () => {
+        const res = collection.calculateExitCode(results, false, true, true)
+        expect(res).to.be.equal(EXIT_CODE_ERROR)
+      })
+      it('shall pass if failOnFailingTestSuite is false', () => {
+        const res = collection.calculateExitCode(results, false, true, false)
+        expect(res).to.be.equal(EXIT_CODE_SUCCESS)
+      })
+    })
+  })
 })