CI should fail when something went wrong. (#5779)

* Added unit test script wrapper.

* Removed bin-up to make lint-coffee fail.

* Made mocha tests return non-zero value on failure for reporter and runner

* Added empty space to test ci failure.

* remove extra whitespace

* use Promise.delay instead of hand-rolled sleep function

* include root-level tests

* exit with number of failures

package.json

@@ -57,6 +57,7 @@
     "test-s3-api": "node -r ./packages/coffee/register -r ./packages/ts/register scripts/binary/s3-api-demo.ts",
     "test-scripts": "mocha -r packages/coffee/register -r packages/ts/register --reporter spec 'scripts/unit/**/*spec.js'",
     "test-scripts-watch": "npm run test-scripts -- --watch --watch-extensions 'ts,js,coffee'",
+    "test-unit": "node ./scripts/test-unit",
     "watch": "npm run all watch"
   },
   "husky": {

packages/reporter/package.json

@@ -16,7 +16,7 @@
     "lint": "bin-up eslint --fix lib/*.js src/*.js* src/**/*.js*",
     "preclean": "npm run check-deps-pre",
     "pretest": "npm run check-deps-pre",
-    "test": "mocha 'src/**/*.spec.*'",
+    "test": "node ../../scripts/test-unit",
     "test-watch": "npm run test -- --watch",
     "prewatch": "npm run check-deps-pre",
     "watch": "npm run build -- --watch --progress"

packages/runner/package.json

@@ -13,7 +13,7 @@
     "clean-deps": "rm -rf node_modules",
     "postinstall": "echo '@packages/runner needs: npm run build'",
     "pretest": "npm run check-deps-pre",
-    "test": "mocha src/**/*.spec.*",
+    "test": "node ../../scripts/test-unit",
     "test-watch": "npm run test -- --watch",
     "prewatch": "npm run check-deps-pre",
     "watch": "webpack --watch --progress"

packages/server/package.json

@@ -12,7 +12,7 @@
     "codecov": "codecov",
     "predev": "npm run check-deps-pre",
     "dev": "node index.js",
-    "lint-coffee": "bin-up coffeelint test/*.coffee test/unit/*.coffee test/integration/*.coffee",
+    "lint-coffee": "../../node_modules/.bin/coffeelint test/*.coffee test/unit/*.coffee test/integration/*.coffee",
     "prerepl": "npm run check-deps-pre",
     "repl": "node repl.js",
     "prestart": "npm run check-deps-pre",

scripts/test-unit.js

@@ -0,0 +1,32 @@
+// This file exists because mocha doesn't return non-zero value when there is a failed test.
+// When https://github.com/mochajs/mocha/issues/3893 is fixed, it will be removed.
+
+const { spawn } = require('child_process')
+const Promise = require('bluebird')
+
+// Test result on console.
+let log = ''
+const proc = spawn(`node`, ['./node_modules/.bin/mocha', 'src/**/*.spec.*'], {})
+
+proc.stdout.on('data', (data) => {
+  log += data
+})
+
+proc.stdout.on('end', async () => {
+  await Promise.delay(500)
+
+  const result = log.match(/\d+ passing.*\n\s*(\d+) failing/)
+  const numFailing = result && parseInt(result[1], 10)
+
+  if (!isNaN(numFailing)) {
+    process.exit(numFailing)
+  }
+
+  process.exit(0)
+})
+
+// Show result on console.
+spawn(`node`, ['./node_modules/.bin/mocha', '"src/*.spec.*" "src/**/*.spec.*"'], {
+  stdio: 'inherit',
+  shell: true,
+})