diff --git a/build-system/tasks/e2e/index.js b/build-system/tasks/e2e/index.js index 80dc6430d0f3..17351ade54b3 100644 --- a/build-system/tasks/e2e/index.js +++ b/build-system/tasks/e2e/index.js @@ -17,8 +17,9 @@ 'use strict'; const argv = require('minimist')(process.argv.slice(2)); -const ciReporter = require('../mocha-ci-reporter'); +const ciReporter = require('./mocha-ci-reporter'); const config = require('../../test-configs/config'); +const dotsReporter = require('./mocha-dots-reporter'); const glob = require('glob'); const log = require('fancy-log'); const Mocha = require('mocha'); @@ -52,11 +53,20 @@ async function cleanUp_() { } function createMocha_() { + let reporter; + if (argv.testnames || argv.watch) { + reporter = ''; + } else if (argv.report) { + reporter = ciReporter; + } else { + reporter = dotsReporter; + } + const mocha = new Mocha({ // e2e tests have a different standard for when a test is too slow, // so we set a non-default threshold. slow: SLOW_TEST_THRESHOLD_MS, - reporter: argv.testnames || argv.watch ? '' : ciReporter, + reporter, retries: TEST_RETRIES, fullStackTrace: true, }); @@ -163,4 +173,5 @@ e2e.flags = { 'Options are `puppeteer` or `selenium`. Default: `selenium`', 'headless': ' Runs the browser in headless mode', 'debug': ' Prints debugging information while running tests', + 'report': ' Write test result report to a local file', }; diff --git a/build-system/tasks/e2e/mocha-ci-reporter.js b/build-system/tasks/e2e/mocha-ci-reporter.js new file mode 100644 index 000000000000..5679abc74d2a --- /dev/null +++ b/build-system/tasks/e2e/mocha-ci-reporter.js @@ -0,0 +1,30 @@ +/** + * Copyright 2020 The AMP HTML Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS-IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const JsonReporter = require('./mocha-custom-json-reporter'); +const mocha = require('mocha'); +const MochaDotsReporter = require('./mocha-dots-reporter'); +const {Base} = mocha.reporters; + +function ciReporter(runner, options) { + Base.call(this, runner, options); + this._mochaDotsReporter = new MochaDotsReporter(runner, options); + this._jsonReporter = new JsonReporter(runner, options); + return this; +} +ciReporter.prototype.__proto__ = Base.prototype; + +module.exports = ciReporter; diff --git a/build-system/tasks/e2e/mocha-custom-json-reporter.js b/build-system/tasks/e2e/mocha-custom-json-reporter.js new file mode 100644 index 000000000000..0f323ad00c83 --- /dev/null +++ b/build-system/tasks/e2e/mocha-custom-json-reporter.js @@ -0,0 +1,87 @@ +/** + * Copyright 2020 The AMP HTML Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS-IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +'use strict'; + +const fs = require('fs').promises; +const { + EVENT_TEST_PASS, + EVENT_TEST_FAIL, + EVENT_RUN_END, + EVENT_TEST_PENDING, + EVENT_SUITE_BEGIN, + EVENT_SUITE_END, +} = require('mocha').Runner; +const {Base} = require('mocha').reporters; +const {inherits} = require('mocha').utils; + +async function writeOutput(output, filename) { + try { + await fs.writeFile(filename, JSON.stringify(output, null, 4)); + } catch (error) { + process.stdout.write( + Base.color( + 'fail', + `Could not write test result report to file '${filename}'` + ) + ); + } +} + +/** + * Custom Mocha reporter for CI builds. + * Mimics the structured karma reporter, but for Mocha. + * @param {*} runner + */ +function JsonReporter(runner) { + Base.call(this, runner); + const testEvents = []; + let suiteList = []; + + runner.on(EVENT_SUITE_BEGIN, function (suite) { + suiteList.push(suite.title); + }); + + runner.on(EVENT_SUITE_END, function () { + // We need a fresh copy every time we make a new suite, + // so we can't use pop here (or the suiteList info of previous + // tests would be changed) + suiteList = suiteList.slice(0, -1); + }); + + [EVENT_TEST_PASS, EVENT_TEST_FAIL, EVENT_TEST_PENDING].forEach((event) => { + runner.on(event, (test) => { + testEvents.push({test, suiteList, event}); + }); + }); + + runner.on(EVENT_RUN_END, async function () { + const testResults = testEvents.map(({test, suiteList, event}) => ({ + description: test.title, + suite: suiteList, + success: event === EVENT_TEST_PASS, + skipped: event === EVENT_TEST_PENDING, + time: test.duration, // in milliseconds + })); + + // Apparently we'll need to add a --no-exit flag when calling this + // to allow for the asynchronous reporter. + // See https://github.com/mochajs/mocha/issues/812 + await writeOutput({testResults}, `result-reports/e2e.json`); + }); +} + +inherits(JsonReporter, Base); +module.exports = JsonReporter; diff --git a/build-system/tasks/mocha-ci-reporter.js b/build-system/tasks/e2e/mocha-dots-reporter.js similarity index 88% rename from build-system/tasks/mocha-ci-reporter.js rename to build-system/tasks/e2e/mocha-dots-reporter.js index a67c32d44188..03a08e286fe2 100644 --- a/build-system/tasks/mocha-ci-reporter.js +++ b/build-system/tasks/e2e/mocha-dots-reporter.js @@ -17,15 +17,15 @@ const {Base} = require('mocha').reporters; const {inherits} = require('mocha').utils; -const {reportTestFinished} = require('./report-test-status'); -const {symbols} = require('./karma.conf').mochaReporter; +const {reportTestFinished} = require('../report-test-status'); +const {symbols} = require('../karma.conf').mochaReporter; /** * Custom Mocha reporter for CI builds. * Mimics the style of the Karma reporter on Travis. * @param {*} runner */ -function ciReporter(runner) { +function MochaDotsReporter(runner) { Base.call(this, runner); const self = this; @@ -62,5 +62,5 @@ function ciReporter(runner) { } } -inherits(ciReporter, Base); -module.exports = ciReporter; +inherits(MochaDotsReporter, Base); +module.exports = MochaDotsReporter;