From e19bcc038a6da3ed0255284881c43622b7d25c01 Mon Sep 17 00:00:00 2001
From: Kukhyeon Heo <sainthkh@naver.com>
Date: Thu, 28 Nov 2019 02:41:45 +0900
Subject: [PATCH] CI should fail when something went wrong. (#5779)

* Added unit test script wrapper.

* Removed bin-up to make lint-coffee fail.

* Made mocha tests return non-zero value on failure for reporter and runner

* Added empty space to test ci failure.

* remove extra whitespace

* use Promise.delay instead of hand-rolled sleep function

* include root-level tests

* exit with number of failures
---
 package.json                   |  1 +
 packages/reporter/package.json |  2 +-
 packages/runner/package.json   |  2 +-
 packages/server/package.json   |  2 +-
 scripts/test-unit.js           | 32 ++++++++++++++++++++++++++++++++
 5 files changed, 36 insertions(+), 3 deletions(-)
 create mode 100644 scripts/test-unit.js

diff --git a/package.json b/package.json
index e508fe605e37..6ef80d8da6ab 100644
--- a/package.json
+++ b/package.json
@@ -57,6 +57,7 @@
     "test-s3-api": "node -r ./packages/coffee/register -r ./packages/ts/register scripts/binary/s3-api-demo.ts",
     "test-scripts": "mocha -r packages/coffee/register -r packages/ts/register --reporter spec 'scripts/unit/**/*spec.js'",
     "test-scripts-watch": "npm run test-scripts -- --watch --watch-extensions 'ts,js,coffee'",
+    "test-unit": "node ./scripts/test-unit",
     "watch": "npm run all watch"
   },
   "husky": {
diff --git a/packages/reporter/package.json b/packages/reporter/package.json
index 5f5d277df164..03457b01b4bc 100644
--- a/packages/reporter/package.json
+++ b/packages/reporter/package.json
@@ -16,7 +16,7 @@
     "lint": "bin-up eslint --fix lib/*.js src/*.js* src/**/*.js*",
     "preclean": "npm run check-deps-pre",
     "pretest": "npm run check-deps-pre",
-    "test": "mocha 'src/**/*.spec.*'",
+    "test": "node ../../scripts/test-unit",
     "test-watch": "npm run test -- --watch",
     "prewatch": "npm run check-deps-pre",
     "watch": "npm run build -- --watch --progress"
diff --git a/packages/runner/package.json b/packages/runner/package.json
index 54bd993a1662..69468f27cb56 100644
--- a/packages/runner/package.json
+++ b/packages/runner/package.json
@@ -13,7 +13,7 @@
     "clean-deps": "rm -rf node_modules",
     "postinstall": "echo '@packages/runner needs: npm run build'",
     "pretest": "npm run check-deps-pre",
-    "test": "mocha src/**/*.spec.*",
+    "test": "node ../../scripts/test-unit",
     "test-watch": "npm run test -- --watch",
     "prewatch": "npm run check-deps-pre",
     "watch": "webpack --watch --progress"
diff --git a/packages/server/package.json b/packages/server/package.json
index 0eb2cd584be9..e62665383bfb 100644
--- a/packages/server/package.json
+++ b/packages/server/package.json
@@ -12,7 +12,7 @@
     "codecov": "codecov",
     "predev": "npm run check-deps-pre",
     "dev": "node index.js",
-    "lint-coffee": "bin-up coffeelint test/*.coffee test/unit/*.coffee test/integration/*.coffee",
+    "lint-coffee": "../../node_modules/.bin/coffeelint test/*.coffee test/unit/*.coffee test/integration/*.coffee",
     "prerepl": "npm run check-deps-pre",
     "repl": "node repl.js",
     "prestart": "npm run check-deps-pre",
diff --git a/scripts/test-unit.js b/scripts/test-unit.js
new file mode 100644
index 000000000000..0e50d94c2a7d
--- /dev/null
+++ b/scripts/test-unit.js
@@ -0,0 +1,32 @@
+// This file exists because mocha doesn't return non-zero value when there is a failed test.
+// When https://github.com/mochajs/mocha/issues/3893 is fixed, it will be removed.
+
+const { spawn } = require('child_process')
+const Promise = require('bluebird')
+
+// Test result on console.
+let log = ''
+const proc = spawn(`node`, ['./node_modules/.bin/mocha', 'src/**/*.spec.*'], {})
+
+proc.stdout.on('data', (data) => {
+  log += data
+})
+
+proc.stdout.on('end', async () => {
+  await Promise.delay(500)
+
+  const result = log.match(/\d+ passing.*\n\s*(\d+) failing/)
+  const numFailing = result && parseInt(result[1], 10)
+
+  if (!isNaN(numFailing)) {
+    process.exit(numFailing)
+  }
+
+  process.exit(0)
+})
+
+// Show result on console.
+spawn(`node`, ['./node_modules/.bin/mocha', '"src/*.spec.*" "src/**/*.spec.*"'], {
+  stdio: 'inherit',
+  shell: true,
+})