From b0711de092a856ccee6ca97ce376687d6094afc3 Mon Sep 17 00:00:00 2001 From: rafer45 Date: Fri, 7 Aug 2020 17:09:40 -0400 Subject: [PATCH 01/13] Implement PoC for e2e test reporter --- build-system/tasks/mocha-ci-json-reporter.js | 107 +++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 build-system/tasks/mocha-ci-json-reporter.js diff --git a/build-system/tasks/mocha-ci-json-reporter.js b/build-system/tasks/mocha-ci-json-reporter.js new file mode 100644 index 000000000000..ed8bb924b58d --- /dev/null +++ b/build-system/tasks/mocha-ci-json-reporter.js @@ -0,0 +1,107 @@ +/** + * Copyright 2020 The AMP HTML Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS-IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +'use strict'; + +const fs = require('fs').promises; +const { + EVENT_TEST_PASS, + EVENT_TEST_FAIL, + EVENT_RUN_END, + EVENT_TEST_PENDING, + EVENT_SUITE_BEGIN, + EVENT_SUITE_END, +} = require('mocha').Runner; +const {Base} = require('mocha').reporters; +const {inherits} = require('mocha').utils; + +async function writeOutput(output, filename) { + try { + await fs.writeFile(filename, JSON.stringify(output, null, 4)); + } catch (error) { + process.stdout.write( + Base.color('fail', `Could not write to file '${filename}'`) + ); + } +} + +/** + * Custom Mocha reporter for CI builds. + * Mimics the structured karma reporter, but for Karma. + * @param {*} runner + */ +function ciJsonReporter(runner) { + Base.call(this, runner); + const testResults = []; + let suiteList = []; + + runner.on(EVENT_SUITE_BEGIN, function (suite) { + suiteList.push(suite.title); + }); + + runner.on(EVENT_SUITE_END, function () { + // We need a fresh copy every time we make a new suite, + // so we can't use pop here (or the suiteList info of previous + // tests would be destroyed) + suiteList = suiteList.slice(0, -1); + }); + + runner.on(EVENT_TEST_PASS, function (test) { + testResults.push( + formatForApi(test, suiteList, { + success: true, + skipped: false, + }) + ); + }); + + runner.on(EVENT_TEST_FAIL, function (test) { + testResults.push( + formatForApi(test, suiteList, { + success: false, + skipped: false, + }) + ); + }); + + runner.on(EVENT_TEST_PENDING, function (test) { + testResults.push( + formatForApi(test, suiteList, { + success: false, + skipped: true, + }) + ); + }); + + runner.on(EVENT_RUN_END, async function () { + // Apparently we'll need to add a --no-exit flag when calling this + // to allow for the asynchronous reporter. + // See https://github.com/mochajs/mocha/issues/812 + await writeOutput({testResults}, `result-reports/e2e.json`); + }); + + function formatForApi(test, suite, {success, skipped}) { + return { + description: test.fullTitle(), + suite, + success, + skipped, + time: test.duration, // in milliseconds + }; + } +} + +inherits(ciJsonReporter, Base); +module.exports = ciJsonReporter; From 4304f891163efbcd07798ed139b7c0c4391cd1f2 Mon Sep 17 00:00:00 2001 From: rafer45 Date: Mon, 10 Aug 2020 17:53:11 -0400 Subject: [PATCH 02/13] Improve wording in comment & err message --- build-system/tasks/mocha-ci-json-reporter.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build-system/tasks/mocha-ci-json-reporter.js b/build-system/tasks/mocha-ci-json-reporter.js index ed8bb924b58d..43401fd87714 100644 --- a/build-system/tasks/mocha-ci-json-reporter.js +++ b/build-system/tasks/mocha-ci-json-reporter.js @@ -32,14 +32,14 @@ async function writeOutput(output, filename) { await fs.writeFile(filename, JSON.stringify(output, null, 4)); } catch (error) { process.stdout.write( - Base.color('fail', `Could not write to file '${filename}'`) + Base.color('fail', `Could not write test result report to file '${filename}'`) ); } } /** * Custom Mocha reporter for CI builds. - * Mimics the structured karma reporter, but for Karma. + * Mimics the structured karma reporter, but for Mocha. * @param {*} runner */ function ciJsonReporter(runner) { From f32d9a8640ca09eeafdec0350b5da0b1608193e0 Mon Sep 17 00:00:00 2001 From: rafer45 Date: Mon, 10 Aug 2020 17:57:55 -0400 Subject: [PATCH 03/13] Lint --- build-system/tasks/mocha-ci-json-reporter.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/build-system/tasks/mocha-ci-json-reporter.js b/build-system/tasks/mocha-ci-json-reporter.js index 43401fd87714..f3950e0f2905 100644 --- a/build-system/tasks/mocha-ci-json-reporter.js +++ b/build-system/tasks/mocha-ci-json-reporter.js @@ -32,7 +32,10 @@ async function writeOutput(output, filename) { await fs.writeFile(filename, JSON.stringify(output, null, 4)); } catch (error) { process.stdout.write( - Base.color('fail', `Could not write test result report to file '${filename}'`) + Base.color( + 'fail', + `Could not write test result report to file '${filename}'` + ) ); } } From c1bdd8a91c5fa5363e34a7263e66e95e03de0170 Mon Sep 17 00:00:00 2001 From: rafer45 Date: Mon, 10 Aug 2020 17:59:54 -0400 Subject: [PATCH 04/13] Refactor for clarity, fix fullTitle typo --- build-system/tasks/mocha-ci-json-reporter.js | 43 ++++++-------------- 1 file changed, 13 insertions(+), 30 deletions(-) diff --git a/build-system/tasks/mocha-ci-json-reporter.js b/build-system/tasks/mocha-ci-json-reporter.js index f3950e0f2905..91994cec1a80 100644 --- a/build-system/tasks/mocha-ci-json-reporter.js +++ b/build-system/tasks/mocha-ci-json-reporter.js @@ -47,7 +47,7 @@ async function writeOutput(output, filename) { */ function ciJsonReporter(runner) { Base.call(this, runner); - const testResults = []; + const testEvents = []; let suiteList = []; runner.on(EVENT_SUITE_BEGIN, function (suite) { @@ -57,53 +57,36 @@ function ciJsonReporter(runner) { runner.on(EVENT_SUITE_END, function () { // We need a fresh copy every time we make a new suite, // so we can't use pop here (or the suiteList info of previous - // tests would be destroyed) + // tests would be changed) suiteList = suiteList.slice(0, -1); }); runner.on(EVENT_TEST_PASS, function (test) { - testResults.push( - formatForApi(test, suiteList, { - success: true, - skipped: false, - }) - ); + testEvents.push({test, suiteList, EVENT_TEST_PASS}); }); runner.on(EVENT_TEST_FAIL, function (test) { - testResults.push( - formatForApi(test, suiteList, { - success: false, - skipped: false, - }) - ); + testEvents.push({test, suiteList, EVENT_TEST_FAIL}); }); runner.on(EVENT_TEST_PENDING, function (test) { - testResults.push( - formatForApi(test, suiteList, { - success: false, - skipped: true, - }) - ); + testEvents.push({test, suiteList, EVENT_TEST_PENDING}); }); runner.on(EVENT_RUN_END, async function () { + const testResults = testEvents.map(({test, suiteList, event}) => ({ + description: test.title, + suite: suiteList, + success: event === EVENT_TEST_PASS, + skipped: event === EVENT_TEST_PENDING, + time: test.duration, // in milliseconds + })); + // Apparently we'll need to add a --no-exit flag when calling this // to allow for the asynchronous reporter. // See https://github.com/mochajs/mocha/issues/812 await writeOutput({testResults}, `result-reports/e2e.json`); }); - - function formatForApi(test, suite, {success, skipped}) { - return { - description: test.fullTitle(), - suite, - success, - skipped, - time: test.duration, // in milliseconds - }; - } } inherits(ciJsonReporter, Base); From 52b19393b26ac91dc0fb8cc631d82801a8880e47 Mon Sep 17 00:00:00 2001 From: rafer45 Date: Tue, 11 Aug 2020 12:51:27 -0400 Subject: [PATCH 05/13] Move reporters to e2e folder, allowing imports to break for now --- build-system/tasks/{ => e2e}/mocha-ci-json-reporter.js | 0 build-system/tasks/{ => e2e}/mocha-ci-reporter.js | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename build-system/tasks/{ => e2e}/mocha-ci-json-reporter.js (100%) rename build-system/tasks/{ => e2e}/mocha-ci-reporter.js (100%) diff --git a/build-system/tasks/mocha-ci-json-reporter.js b/build-system/tasks/e2e/mocha-ci-json-reporter.js similarity index 100% rename from build-system/tasks/mocha-ci-json-reporter.js rename to build-system/tasks/e2e/mocha-ci-json-reporter.js diff --git a/build-system/tasks/mocha-ci-reporter.js b/build-system/tasks/e2e/mocha-ci-reporter.js similarity index 100% rename from build-system/tasks/mocha-ci-reporter.js rename to build-system/tasks/e2e/mocha-ci-reporter.js From cec228bc0695f0f0c151176443f271a5b9413047 Mon Sep 17 00:00:00 2001 From: rafer45 Date: Tue, 11 Aug 2020 12:52:15 -0400 Subject: [PATCH 06/13] Fix broken imports --- build-system/tasks/e2e/index.js | 2 +- build-system/tasks/e2e/mocha-ci-reporter.js | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/build-system/tasks/e2e/index.js b/build-system/tasks/e2e/index.js index 80dc6430d0f3..5776a0a399cc 100644 --- a/build-system/tasks/e2e/index.js +++ b/build-system/tasks/e2e/index.js @@ -17,7 +17,7 @@ 'use strict'; const argv = require('minimist')(process.argv.slice(2)); -const ciReporter = require('../mocha-ci-reporter'); +const ciReporter = require('./mocha-ci-reporter'); const config = require('../../test-configs/config'); const glob = require('glob'); const log = require('fancy-log'); diff --git a/build-system/tasks/e2e/mocha-ci-reporter.js b/build-system/tasks/e2e/mocha-ci-reporter.js index a67c32d44188..96f2bf999e85 100644 --- a/build-system/tasks/e2e/mocha-ci-reporter.js +++ b/build-system/tasks/e2e/mocha-ci-reporter.js @@ -17,8 +17,8 @@ const {Base} = require('mocha').reporters; const {inherits} = require('mocha').utils; -const {reportTestFinished} = require('./report-test-status'); -const {symbols} = require('./karma.conf').mochaReporter; +const {reportTestFinished} = require('../report-test-status'); +const {symbols} = require('../karma.conf').mochaReporter; /** * Custom Mocha reporter for CI builds. From 739b4d70e5753b759f9932438abc4d00df4f9948 Mon Sep 17 00:00:00 2001 From: rafer45 Date: Tue, 11 Aug 2020 13:02:29 -0400 Subject: [PATCH 07/13] Rename mocha ci reporter, breaking imports --- .../{mocha-ci-json-reporter.js => mocha-custom-json-reporter.js} | 0 .../e2e/{mocha-ci-reporter.js => mocha-karma-mimic-reporter.js} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename build-system/tasks/e2e/{mocha-ci-json-reporter.js => mocha-custom-json-reporter.js} (100%) rename build-system/tasks/e2e/{mocha-ci-reporter.js => mocha-karma-mimic-reporter.js} (100%) diff --git a/build-system/tasks/e2e/mocha-ci-json-reporter.js b/build-system/tasks/e2e/mocha-custom-json-reporter.js similarity index 100% rename from build-system/tasks/e2e/mocha-ci-json-reporter.js rename to build-system/tasks/e2e/mocha-custom-json-reporter.js diff --git a/build-system/tasks/e2e/mocha-ci-reporter.js b/build-system/tasks/e2e/mocha-karma-mimic-reporter.js similarity index 100% rename from build-system/tasks/e2e/mocha-ci-reporter.js rename to build-system/tasks/e2e/mocha-karma-mimic-reporter.js From 1f194825a17b578163437b2e201dcf44a6c34a90 Mon Sep 17 00:00:00 2001 From: rafer45 Date: Tue, 11 Aug 2020 13:17:05 -0400 Subject: [PATCH 08/13] Make ci reporter that joins together the reporters --- build-system/tasks/e2e/mocha-ci-reporter.js | 30 +++++++++++++++++++ .../tasks/e2e/mocha-custom-json-reporter.js | 6 ++-- .../tasks/e2e/mocha-karma-mimic-reporter.js | 6 ++-- 3 files changed, 36 insertions(+), 6 deletions(-) create mode 100644 build-system/tasks/e2e/mocha-ci-reporter.js diff --git a/build-system/tasks/e2e/mocha-ci-reporter.js b/build-system/tasks/e2e/mocha-ci-reporter.js new file mode 100644 index 000000000000..e95d2a073394 --- /dev/null +++ b/build-system/tasks/e2e/mocha-ci-reporter.js @@ -0,0 +1,30 @@ +/** + * Copyright 2020 The AMP HTML Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS-IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const JsonReporter = require('./mocha-custom-json-reporter'); +const KarmaMimicReporter = require('./mocha-karma-mimic-reporter'); +const mocha = require('mocha'); +const {Base} = mocha.reporters; + +function ciReporter(runner, options) { + Base.call(this, runner, options); + this._karmaMimicReporter = new KarmaMimicReporter(runner, options); + this._jsonReporter = new JsonReporter(runner, options); + return this; +} +ciReporter.prototype.__proto__ = Base.prototype; + +module.exports = ciReporter; diff --git a/build-system/tasks/e2e/mocha-custom-json-reporter.js b/build-system/tasks/e2e/mocha-custom-json-reporter.js index 91994cec1a80..ad5f84243213 100644 --- a/build-system/tasks/e2e/mocha-custom-json-reporter.js +++ b/build-system/tasks/e2e/mocha-custom-json-reporter.js @@ -45,7 +45,7 @@ async function writeOutput(output, filename) { * Mimics the structured karma reporter, but for Mocha. * @param {*} runner */ -function ciJsonReporter(runner) { +function JsonReporter(runner) { Base.call(this, runner); const testEvents = []; let suiteList = []; @@ -89,5 +89,5 @@ function ciJsonReporter(runner) { }); } -inherits(ciJsonReporter, Base); -module.exports = ciJsonReporter; +inherits(JsonReporter, Base); +module.exports = JsonReporter; diff --git a/build-system/tasks/e2e/mocha-karma-mimic-reporter.js b/build-system/tasks/e2e/mocha-karma-mimic-reporter.js index 96f2bf999e85..50d001eb4788 100644 --- a/build-system/tasks/e2e/mocha-karma-mimic-reporter.js +++ b/build-system/tasks/e2e/mocha-karma-mimic-reporter.js @@ -25,7 +25,7 @@ const {symbols} = require('../karma.conf').mochaReporter; * Mimics the style of the Karma reporter on Travis. * @param {*} runner */ -function ciReporter(runner) { +function KarmaMimicReporter(runner) { Base.call(this, runner); const self = this; @@ -62,5 +62,5 @@ function ciReporter(runner) { } } -inherits(ciReporter, Base); -module.exports = ciReporter; +inherits(KarmaMimicReporter, Base); +module.exports = KarmaMimicReporter; From 8f84767847fe8408b45311fc3e671b72a4901b31 Mon Sep 17 00:00:00 2001 From: rafer45 Date: Tue, 11 Aug 2020 14:20:35 -0400 Subject: [PATCH 09/13] Rename KarmaMimicReporter in files --- build-system/tasks/e2e/mocha-ci-reporter.js | 4 ++-- build-system/tasks/e2e/mocha-karma-mimic-reporter.js | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/build-system/tasks/e2e/mocha-ci-reporter.js b/build-system/tasks/e2e/mocha-ci-reporter.js index e95d2a073394..292bd8c7eed3 100644 --- a/build-system/tasks/e2e/mocha-ci-reporter.js +++ b/build-system/tasks/e2e/mocha-ci-reporter.js @@ -15,13 +15,13 @@ */ const JsonReporter = require('./mocha-custom-json-reporter'); -const KarmaMimicReporter = require('./mocha-karma-mimic-reporter'); const mocha = require('mocha'); +const MochaDotsReporter = require('./mocha-karma-mimic-reporter'); const {Base} = mocha.reporters; function ciReporter(runner, options) { Base.call(this, runner, options); - this._karmaMimicReporter = new KarmaMimicReporter(runner, options); + this._karmaMimicReporter = new MochaDotsReporter(runner, options); this._jsonReporter = new JsonReporter(runner, options); return this; } diff --git a/build-system/tasks/e2e/mocha-karma-mimic-reporter.js b/build-system/tasks/e2e/mocha-karma-mimic-reporter.js index 50d001eb4788..03a08e286fe2 100644 --- a/build-system/tasks/e2e/mocha-karma-mimic-reporter.js +++ b/build-system/tasks/e2e/mocha-karma-mimic-reporter.js @@ -25,7 +25,7 @@ const {symbols} = require('../karma.conf').mochaReporter; * Mimics the style of the Karma reporter on Travis. * @param {*} runner */ -function KarmaMimicReporter(runner) { +function MochaDotsReporter(runner) { Base.call(this, runner); const self = this; @@ -62,5 +62,5 @@ function KarmaMimicReporter(runner) { } } -inherits(KarmaMimicReporter, Base); -module.exports = KarmaMimicReporter; +inherits(MochaDotsReporter, Base); +module.exports = MochaDotsReporter; From 0e222fa8fab882f2bae3371d9d23487e615d02fd Mon Sep 17 00:00:00 2001 From: rafer45 Date: Tue, 11 Aug 2020 14:22:09 -0400 Subject: [PATCH 10/13] Rename karma-mimic-reporter file --- build-system/tasks/e2e/mocha-ci-reporter.js | 2 +- .../{mocha-karma-mimic-reporter.js => mocha-dots-reporter.js} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename build-system/tasks/e2e/{mocha-karma-mimic-reporter.js => mocha-dots-reporter.js} (100%) diff --git a/build-system/tasks/e2e/mocha-ci-reporter.js b/build-system/tasks/e2e/mocha-ci-reporter.js index 292bd8c7eed3..694307addbbd 100644 --- a/build-system/tasks/e2e/mocha-ci-reporter.js +++ b/build-system/tasks/e2e/mocha-ci-reporter.js @@ -16,7 +16,7 @@ const JsonReporter = require('./mocha-custom-json-reporter'); const mocha = require('mocha'); -const MochaDotsReporter = require('./mocha-karma-mimic-reporter'); +const MochaDotsReporter = require('./mocha-dots-reporter'); const {Base} = mocha.reporters; function ciReporter(runner, options) { diff --git a/build-system/tasks/e2e/mocha-karma-mimic-reporter.js b/build-system/tasks/e2e/mocha-dots-reporter.js similarity index 100% rename from build-system/tasks/e2e/mocha-karma-mimic-reporter.js rename to build-system/tasks/e2e/mocha-dots-reporter.js From 35a1bd5bb53b04a030e02d43d5e6e630542acc9f Mon Sep 17 00:00:00 2001 From: rafer45 Date: Tue, 11 Aug 2020 14:25:39 -0400 Subject: [PATCH 11/13] Rename private _karmaMimicReporter --- build-system/tasks/e2e/mocha-ci-reporter.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-system/tasks/e2e/mocha-ci-reporter.js b/build-system/tasks/e2e/mocha-ci-reporter.js index 694307addbbd..5679abc74d2a 100644 --- a/build-system/tasks/e2e/mocha-ci-reporter.js +++ b/build-system/tasks/e2e/mocha-ci-reporter.js @@ -21,7 +21,7 @@ const {Base} = mocha.reporters; function ciReporter(runner, options) { Base.call(this, runner, options); - this._karmaMimicReporter = new MochaDotsReporter(runner, options); + this._mochaDotsReporter = new MochaDotsReporter(runner, options); this._jsonReporter = new JsonReporter(runner, options); return this; } From 01480c61d22da856de2bba74b5c29521cec3df49 Mon Sep 17 00:00:00 2001 From: rafer45 Date: Wed, 12 Aug 2020 17:21:59 -0400 Subject: [PATCH 12/13] Simplify event insertion --- .../tasks/e2e/mocha-custom-json-reporter.js | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/build-system/tasks/e2e/mocha-custom-json-reporter.js b/build-system/tasks/e2e/mocha-custom-json-reporter.js index ad5f84243213..0f323ad00c83 100644 --- a/build-system/tasks/e2e/mocha-custom-json-reporter.js +++ b/build-system/tasks/e2e/mocha-custom-json-reporter.js @@ -61,16 +61,10 @@ function JsonReporter(runner) { suiteList = suiteList.slice(0, -1); }); - runner.on(EVENT_TEST_PASS, function (test) { - testEvents.push({test, suiteList, EVENT_TEST_PASS}); - }); - - runner.on(EVENT_TEST_FAIL, function (test) { - testEvents.push({test, suiteList, EVENT_TEST_FAIL}); - }); - - runner.on(EVENT_TEST_PENDING, function (test) { - testEvents.push({test, suiteList, EVENT_TEST_PENDING}); + [EVENT_TEST_PASS, EVENT_TEST_FAIL, EVENT_TEST_PENDING].forEach((event) => { + runner.on(event, (test) => { + testEvents.push({test, suiteList, event}); + }); }); runner.on(EVENT_RUN_END, async function () { From ad65609366356ab38295a42bbe0919906b768792 Mon Sep 17 00:00:00 2001 From: rafer45 Date: Thu, 13 Aug 2020 11:46:28 -0400 Subject: [PATCH 13/13] Only write json report if report flag set --- build-system/tasks/e2e/index.js | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/build-system/tasks/e2e/index.js b/build-system/tasks/e2e/index.js index 5776a0a399cc..17351ade54b3 100644 --- a/build-system/tasks/e2e/index.js +++ b/build-system/tasks/e2e/index.js @@ -19,6 +19,7 @@ const argv = require('minimist')(process.argv.slice(2)); const ciReporter = require('./mocha-ci-reporter'); const config = require('../../test-configs/config'); +const dotsReporter = require('./mocha-dots-reporter'); const glob = require('glob'); const log = require('fancy-log'); const Mocha = require('mocha'); @@ -52,11 +53,20 @@ async function cleanUp_() { } function createMocha_() { + let reporter; + if (argv.testnames || argv.watch) { + reporter = ''; + } else if (argv.report) { + reporter = ciReporter; + } else { + reporter = dotsReporter; + } + const mocha = new Mocha({ // e2e tests have a different standard for when a test is too slow, // so we set a non-default threshold. slow: SLOW_TEST_THRESHOLD_MS, - reporter: argv.testnames || argv.watch ? '' : ciReporter, + reporter, retries: TEST_RETRIES, fullStackTrace: true, }); @@ -163,4 +173,5 @@ e2e.flags = { 'Options are `puppeteer` or `selenium`. Default: `selenium`', 'headless': ' Runs the browser in headless mode', 'debug': ' Prints debugging information while running tests', + 'report': ' Write test result report to a local file', };