diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index f237ea300888..87bb4362fee1 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -19,7 +19,7 @@ jobs: runs-on: ubuntu-latest env: # The smokehouse tests run by job `test set 1`. `test set 2` will run the rest. - SMOKE_BATCH_1: a11y oopif pwa pwa2 pwa3 dbw redirects errors offline + SMOKE_BATCH_1: a11y oopif pwa dbw redirects errors offline # Job named e.g. "Chrome stable, test set 1". name: Chrome ${{ matrix.chrome-channel }}, batch ${{ matrix.smoke-test-invert == false && '1' || '2' }} diff --git a/lighthouse-cli/test/smokehouse/frontends/lib.js b/lighthouse-cli/test/smokehouse/frontends/lib.js index 1f77e4ca48cb..70f05fe60486 100644 --- a/lighthouse-cli/test/smokehouse/frontends/lib.js +++ b/lighthouse-cli/test/smokehouse/frontends/lib.js @@ -16,7 +16,6 @@ const cloneDeep = require('lodash.clonedeep'); const smokeTests = require('../test-definitions/core-tests.js'); const {runSmokehouse} = require('../smokehouse.js'); -const {updateTestDefnFormat} = require('./back-compat-util.js'); /** * @param {Smokehouse.SmokehouseLibOptions} options @@ -24,8 +23,7 @@ const {updateTestDefnFormat} = require('./back-compat-util.js'); async function smokehouse(options) { const {urlFilterRegex, skip, modify, ...smokehouseOptions} = options; - const updatedCoreTests = updateTestDefnFormat(smokeTests); - const clonedTests = cloneDeep(updatedCoreTests); + const clonedTests = cloneDeep(smokeTests); const modifiedTests = []; for (const test of clonedTests) { if (urlFilterRegex && !test.expectations.lhr.requestedUrl.match(urlFilterRegex)) { diff --git a/lighthouse-cli/test/smokehouse/smokehouse.js b/lighthouse-cli/test/smokehouse/smokehouse.js index 990c69260892..dec9c91204db 100644 --- a/lighthouse-cli/test/smokehouse/smokehouse.js +++ b/lighthouse-cli/test/smokehouse/smokehouse.js @@ -74,7 +74,7 @@ async function runSmokehouse(smokeTestDefns, smokehouseOptions) { const failingDefns = testResults.filter(result => result.failed); if (failingDefns.length) { const testNames = failingDefns.map(d => d.id).join(', '); - console.error(log.redify(`We have ${failingDefns.length} failing smoketest(s): ${testNames}`)); + console.log(log.redify(`We have ${failingDefns.length} failing smoketest(s): ${testNames}`)); return {success: false, testResults}; } diff --git a/lighthouse-cli/test/smokehouse/test-definitions/a11y/expectations.js b/lighthouse-cli/test/smokehouse/test-definitions/a11y/expectations.js index 0ff8c192f451..160ecec9b54f 100644 --- a/lighthouse-cli/test/smokehouse/test-definitions/a11y/expectations.js +++ b/lighthouse-cli/test/smokehouse/test-definitions/a11y/expectations.js @@ -8,691 +8,689 @@ /* eslint-disable max-len */ /** - * @type {Array} + * @type {Smokehouse.ExpectedRunnerResult} * Expected Lighthouse audit values for byte efficiency tests */ -const expectations = [ - { - lhr: { - requestedUrl: 'http://localhost:10200/a11y/a11y_tester.html', - finalUrl: 'http://localhost:10200/a11y/a11y_tester.html', - audits: { - 'aria-allowed-attr': { - score: 0, - details: { - items: [ - { - node: { - 'type': 'node', - 'selector': 'body > section > div#aria-allowed-attr', - 'snippet': '