diff --git a/.appveyor.yml b/.appveyor.yml index b24f9680246c..23abb62765e4 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -39,7 +39,6 @@ test_script: - yarn unit - yarn type-check - yarn smoke - - yarn smokehouse cache: #- chrome-win32 -> appveyor.yml,package.json diff --git a/.travis.yml b/.travis.yml index 136b8c69f71e..1968e14bf679 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,7 +37,6 @@ script: - yarn type-check - yarn closure - yarn smoke - - yarn smokehouse - yarn test-extension # _JAVA_OPTIONS is breaking parsing of compiler output. See #3338. - unset _JAVA_OPTIONS diff --git a/lighthouse-cli/test/smokehouse/a11y/run-tests.sh b/lighthouse-cli/test/smokehouse/a11y/run-tests.sh deleted file mode 100755 index 604b434437d9..000000000000 --- a/lighthouse-cli/test/smokehouse/a11y/run-tests.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash - -node lighthouse-cli/test/fixtures/static-server.js & - -sleep 0.5s - -config="lighthouse-cli/test/smokehouse/a11y/a11y-config.js" -expectations="lighthouse-cli/test/smokehouse/a11y/expectations.js" - -yarn smokehouse --config-path=$config --expectations-path=$expectations -exit_code=$? - -# kill test servers -kill $(jobs -p) - -exit "$exit_code" diff --git a/lighthouse-cli/test/smokehouse/byte-efficiency/run-tests.sh b/lighthouse-cli/test/smokehouse/byte-efficiency/run-tests.sh deleted file mode 100755 index 960cb6f8c12f..000000000000 --- a/lighthouse-cli/test/smokehouse/byte-efficiency/run-tests.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash - -node lighthouse-cli/test/fixtures/static-server.js & - -sleep 0.5s - -config="lighthouse-cli/test/smokehouse/byte-config.js" -expectations="lighthouse-cli/test/smokehouse/byte-efficiency/expectations.js" - -yarn smokehouse --config-path=$config --expectations-path=$expectations -exit_code=$? - -# kill test servers -kill $(jobs -p) - -exit "$exit_code" diff --git a/lighthouse-cli/test/smokehouse/dobetterweb/dbw-expectations.js b/lighthouse-cli/test/smokehouse/dobetterweb/dbw-expectations.js index 897cdabfe772..91b80ee58756 100644 --- a/lighthouse-cli/test/smokehouse/dobetterweb/dbw-expectations.js +++ b/lighthouse-cli/test/smokehouse/dobetterweb/dbw-expectations.js @@ -186,46 +186,5 @@ module.exports = [ }, }, }, - }, { - initialUrl: 'http://localhost:10200/online-only.html', - url: 'http://localhost:10200/online-only.html', - audits: { - 'is-on-https': { - score: 1, - }, - 'uses-http2': { - score: 0, - }, - 'external-anchors-use-rel-noopener': { - score: 1, - }, - 'appcache-manifest': { - score: 1, - }, - 'geolocation-on-start': { - score: 1, - }, - 'link-blocking-first-paint': { - score: 1, - }, - 'no-document-write': { - score: 1, - }, - 'no-mutation-events': { - score: 1, - }, - 'no-websql': { - score: 1, - }, - 'script-blocking-first-paint': { - score: 1, - }, - 'uses-passive-event-listeners': { - score: 1, - }, - 'password-inputs-can-be-pasted-into': { - score: 1, - }, - }, }, ]; diff --git a/lighthouse-cli/test/smokehouse/dobetterweb/run-tests.sh b/lighthouse-cli/test/smokehouse/dobetterweb/run-tests.sh deleted file mode 100755 index fd7acaa12ac3..000000000000 --- a/lighthouse-cli/test/smokehouse/dobetterweb/run-tests.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash - -node lighthouse-cli/test/fixtures/static-server.js & - -sleep 0.5s - -config="lighthouse-cli/test/smokehouse/dbw-config.js" -expectations="lighthouse-cli/test/smokehouse/dobetterweb/dbw-expectations.js" - -# run smoketest on DBW test page. -yarn smokehouse --config-path=$config --expectations-path=$expectations -exit_code=$? - -# kill test servers -kill $(jobs -p) - -exit "$exit_code" diff --git a/lighthouse-cli/test/smokehouse/offline-config.js b/lighthouse-cli/test/smokehouse/offline-config.js index 0811e21f5941..982ea0973edb 100644 --- a/lighthouse-cli/test/smokehouse/offline-config.js +++ b/lighthouse-cli/test/smokehouse/offline-config.js @@ -11,6 +11,9 @@ module.exports = { extends: 'lighthouse:default', settings: { + onlyCategories: [ + 'best-practices', + ], onlyAudits: [ 'is-on-https', 'redirects-http', @@ -20,6 +23,8 @@ module.exports = { 'without-javascript', 'user-timings', 'critical-request-chains', + 'link-blocking-first-paint', + 'script-blocking-first-paint', 'webapp-install-banner', 'splash-screen', 'themed-omnibox', diff --git a/lighthouse-cli/test/smokehouse/offline-local/offline-expectations.js b/lighthouse-cli/test/smokehouse/offline-local/offline-expectations.js index e5bacc308919..63f7c9a4f049 100644 --- a/lighthouse-cli/test/smokehouse/offline-local/offline-expectations.js +++ b/lighthouse-cli/test/smokehouse/offline-local/offline-expectations.js @@ -18,6 +18,39 @@ module.exports = [ 'is-on-https': { score: 1, }, + 'uses-http2': { + score: 0, + }, + 'external-anchors-use-rel-noopener': { + score: 1, + }, + 'appcache-manifest': { + score: 1, + }, + 'geolocation-on-start': { + score: 1, + }, + 'link-blocking-first-paint': { + score: 1, + }, + 'no-document-write': { + score: 1, + }, + 'no-mutation-events': { + score: 1, + }, + 'no-websql': { + score: 1, + }, + 'script-blocking-first-paint': { + score: 1, + }, + 'uses-passive-event-listeners': { + score: 1, + }, + 'password-inputs-can-be-pasted-into': { + score: 1, + }, 'redirects-http': { score: 0, }, diff --git a/lighthouse-cli/test/smokehouse/offline-local/run-tests.sh b/lighthouse-cli/test/smokehouse/offline-local/run-tests.sh deleted file mode 100755 index b2a89d5c7994..000000000000 --- a/lighthouse-cli/test/smokehouse/offline-local/run-tests.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash - -node lighthouse-cli/test/fixtures/static-server.js & - -sleep 0.5s - -config="lighthouse-cli/test/smokehouse/offline-config.js" -expectations="lighthouse-cli/test/smokehouse/offline-local/offline-expectations.js" - -# run smoketest, expecting results found in offline-expectations -yarn smokehouse --config-path=$config --expectations-path=$expectations -exit_code=$? - -# kill test servers -kill $(jobs -p) - -exit "$exit_code" diff --git a/lighthouse-cli/test/smokehouse/perf/run-tests.sh b/lighthouse-cli/test/smokehouse/perf/run-tests.sh deleted file mode 100644 index be2313de0f56..000000000000 --- a/lighthouse-cli/test/smokehouse/perf/run-tests.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash - -node lighthouse-cli/test/fixtures/static-server.js & - -sleep 0.5s - -config="lighthouse-core/config/perf-config.js" -expectations="lighthouse-cli/test/smokehouse/perf/expectations.js" - -yarn smokehouse --config-path=$config --expectations-path=$expectations -exit_code=$? - -# kill test servers -kill $(jobs -p) - -exit "$exit_code" diff --git a/lighthouse-cli/test/smokehouse/pwa-expectations.js b/lighthouse-cli/test/smokehouse/pwa-expectations.js index 5ee99937744a..2d820f772b12 100644 --- a/lighthouse-cli/test/smokehouse/pwa-expectations.js +++ b/lighthouse-cli/test/smokehouse/pwa-expectations.js @@ -219,323 +219,4 @@ module.exports = [ }, }, }, - - { - initialUrl: 'https://jakearchibald.github.io/svgomg/', - url: 'https://jakearchibald.github.io/svgomg/', - audits: { - 'is-on-https': { - score: 1, - }, - 'redirects-http': { - // Note: relies on JS redirect. - // see https://github.com/GoogleChrome/lighthouse/issues/2383 - score: 0, - }, - 'service-worker': { - score: 1, - }, - 'works-offline': { - score: 1, - }, - 'viewport': { - score: 1, - }, - 'without-javascript': { - score: 1, - }, - 'load-fast-enough-for-pwa': { - // Ignore speed test; just verify that it ran. - }, - 'webapp-install-banner': { - score: 0, - extendedInfo: { - value: { - manifestValues: { - allChecks: [ - {id: 'hasStartUrl', passing: true}, - {id: 'hasIconsAtLeast192px', passing: true}, - {id: 'hasIconsAtLeast512px', passing: true}, - {id: 'hasPWADisplayValue', passing: true}, - {id: 'hasBackgroundColor', passing: true}, - {id: 'hasThemeColor', passing: true}, - {id: 'hasShortName', passing: false}, - {id: 'shortNameLength', passing: false}, - {id: 'hasName', passing: true}, - ], - }, - }, - }, - }, - 'splash-screen': { - score: 1, - extendedInfo: { - value: { - manifestValues: { - allChecks: [ - {id: 'hasStartUrl', passing: true}, - {id: 'hasIconsAtLeast192px', passing: true}, - {id: 'hasIconsAtLeast512px', passing: true}, - {id: 'hasPWADisplayValue', passing: true}, - {id: 'hasBackgroundColor', passing: true}, - {id: 'hasThemeColor', passing: true}, - {id: 'hasShortName', passing: false}, - {id: 'shortNameLength', passing: false}, - {id: 'hasName', passing: true}, - ], - }, - }, - }, - }, - 'themed-omnibox': { - score: 1, - extendedInfo: { - value: { - manifestValues: { - allChecks: [ - {id: 'hasStartUrl', passing: true}, - {id: 'hasIconsAtLeast192px', passing: true}, - {id: 'hasIconsAtLeast512px', passing: true}, - {id: 'hasPWADisplayValue', passing: true}, - {id: 'hasBackgroundColor', passing: true}, - {id: 'hasThemeColor', passing: true}, - {id: 'hasShortName', passing: false}, - {id: 'shortNameLength', passing: false}, - {id: 'hasName', passing: true}, - ], - }, - }, - }, - }, - 'content-width': { - score: 1, - }, - - // "manual" audits. Just verify in the results. - 'pwa-cross-browser': { - score: 0, - manual: true, - }, - 'pwa-page-transitions': { - score: 0, - manual: true, - }, - 'pwa-each-page-has-url': { - score: 0, - manual: true, - }, - }, - }, - - // Disabled due to flakiness of site. - // See https://github.com/GoogleChrome/lighthouse/issues/1656 - // { - // initialUrl: 'https://shop.polymer-project.org/', - // url: 'https://shop.polymer-project.org/', - // audits: { - // 'is-on-https': { - // score: 1 - // }, - // 'redirects-http': { - // score: 1 - // }, - // 'service-worker': { - // score: 1 - // }, - // 'works-offline': { - // score: 1 - // }, - // 'viewport': { - // score: 1 - // }, - // 'without-javascript': { - // score: 1 - // }, - // 'load-fast-enough-for-pwa': { - // // Ignore speed test; just verify that it ran. - // }, - // 'webapp-install-banner': { - // score: 1, - // extendedInfo: { - // value: { - // manifestValues: { - // allChecks: [ - // {id: 'hasStartUrl', passing: true}, - // {id: 'hasIconsAtLeast192px', passing: true}, - // {id: 'hasIconsAtLeast512px', passing: true}, - // {id: 'hasPWADisplayValue', passing: true}, - // {id: 'hasBackgroundColor', passing: true}, - // {id: 'hasThemeColor', passing: true}, - // {id: 'hasShortName', passing: true}, - // {id: 'shortNameLength', passing: true}, - // {id: 'hasName', passing: true} - // ] - // } - // } - // } - // }, - // 'splash-screen': { - // score: 1, - // extendedInfo: { - // value: { - // manifestValues: { - // allChecks: [ - // {id: 'hasStartUrl', passing: true}, - // {id: 'hasIconsAtLeast192px', passing: true}, - // {id: 'hasIconsAtLeast512px', passing: true}, - // {id: 'hasPWADisplayValue', passing: true}, - // {id: 'hasBackgroundColor', passing: true}, - // {id: 'hasThemeColor', passing: true}, - // {id: 'hasShortName', passing: true}, - // {id: 'shortNameLength', passing: true}, - // {id: 'hasName', passing: true} - // ] - // } - // } - // } - // }, - // 'themed-omnibox': { - // score: 1, - // extendedInfo: { - // value: { - // manifestValues: { - // allChecks: [ - // {id: 'hasStartUrl', passing: true}, - // {id: 'hasIconsAtLeast192px', passing: true}, - // {id: 'hasIconsAtLeast512px', passing: true}, - // {id: 'hasPWADisplayValue', passing: true}, - // {id: 'hasBackgroundColor', passing: true}, - // {id: 'hasThemeColor', passing: true}, - // {id: 'hasShortName', passing: true}, - // {id: 'shortNameLength', passing: true}, - // {id: 'hasName', passing: true} - // ] - // } - // } - // } - // }, - // 'content-width': { - // score: 1 - // }, - - // // "manual" audits. Just verify in the results. - // 'pwa-cross-browser': { - // score: 0, - // manual: true - // }, - // 'pwa-page-transitions': { - // score: 0, - // manual: true - // }, - // 'pwa-each-page-has-url': { - // score: 0, - // manual: true - // } - // } - // }, - - { - initialUrl: 'https://pwa.rocks', - url: 'https://pwa.rocks/', - audits: { - 'is-on-https': { - score: 1, - }, - 'redirects-http': { - score: 1, - }, - 'service-worker': { - score: 1, - }, - 'works-offline': { - score: 1, - }, - 'viewport': { - score: 1, - }, - 'without-javascript': { - score: 1, - }, - 'load-fast-enough-for-pwa': { - // Ignore speed test; just verify that it ran . - }, - 'webapp-install-banner': { - score: 1, - extendedInfo: { - value: { - manifestValues: { - allChecks: [ - {id: 'hasStartUrl', passing: true}, - {id: 'hasIconsAtLeast192px', passing: true}, - {id: 'hasIconsAtLeast512px', passing: false}, - {id: 'hasPWADisplayValue', passing: true}, - {id: 'hasBackgroundColor', passing: true}, - {id: 'hasThemeColor', passing: true}, - {id: 'hasShortName', passing: true}, - {id: 'shortNameLength', passing: true}, - {id: 'hasName', passing: true}, - ], - }, - }, - }, - }, - 'splash-screen': { - score: 0, - extendedInfo: { - value: { - manifestValues: { - allChecks: [ - {id: 'hasStartUrl', passing: true}, - {id: 'hasIconsAtLeast192px', passing: true}, - {id: 'hasIconsAtLeast512px', passing: false}, - {id: 'hasPWADisplayValue', passing: true}, - {id: 'hasBackgroundColor', passing: true}, - {id: 'hasThemeColor', passing: true}, - {id: 'hasShortName', passing: true}, - {id: 'shortNameLength', passing: true}, - {id: 'hasName', passing: true}, - ], - }, - }, - }, - }, - 'themed-omnibox': { - score: 0, - extendedInfo: { - value: { - manifestValues: { - allChecks: [ - {id: 'hasStartUrl', passing: true}, - {id: 'hasIconsAtLeast192px', passing: true}, - {id: 'hasIconsAtLeast512px', passing: false}, - {id: 'hasPWADisplayValue', passing: true}, - {id: 'hasBackgroundColor', passing: true}, - {id: 'hasThemeColor', passing: true}, - {id: 'hasShortName', passing: true}, - {id: 'shortNameLength', passing: true}, - {id: 'hasName', passing: true}, - ], - }, - }, - }, - }, - 'content-width': { - score: 1, - }, - - // "manual" audits. Just verify in the results. - 'pwa-cross-browser': { - score: 0, - manual: true, - }, - 'pwa-page-transitions': { - score: 0, - manual: true, - }, - 'pwa-each-page-has-url': { - score: 0, - manual: true, - }, - }, - }, ]; diff --git a/lighthouse-cli/test/smokehouse/pwa2-expectations.js b/lighthouse-cli/test/smokehouse/pwa2-expectations.js new file mode 100644 index 000000000000..057200f06b95 --- /dev/null +++ b/lighthouse-cli/test/smokehouse/pwa2-expectations.js @@ -0,0 +1,335 @@ +/** + * @license Copyright 2016 Google Inc. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + */ +'use strict'; + +/** + * Expected Lighthouse audit values for various sites with stable(ish) PWA + * results. + */ +module.exports = [ + { + initialUrl: 'https://jakearchibald.github.io/svgomg/', + url: 'https://jakearchibald.github.io/svgomg/', + audits: { + 'is-on-https': { + score: 1, + }, + 'redirects-http': { + // Note: relies on JS redirect. + // see https://github.com/GoogleChrome/lighthouse/issues/2383 + score: 0, + }, + 'service-worker': { + score: 1, + }, + 'works-offline': { + score: 1, + }, + 'viewport': { + score: 1, + }, + 'without-javascript': { + score: 1, + }, + 'load-fast-enough-for-pwa': { + // Ignore speed test; just verify that it ran. + }, + 'webapp-install-banner': { + score: 0, + extendedInfo: { + value: { + manifestValues: { + allChecks: [ + {id: 'hasStartUrl', passing: true}, + {id: 'hasIconsAtLeast192px', passing: true}, + {id: 'hasIconsAtLeast512px', passing: true}, + {id: 'hasPWADisplayValue', passing: true}, + {id: 'hasBackgroundColor', passing: true}, + {id: 'hasThemeColor', passing: true}, + {id: 'hasShortName', passing: false}, + {id: 'shortNameLength', passing: false}, + {id: 'hasName', passing: true}, + ], + }, + }, + }, + }, + 'splash-screen': { + score: 1, + extendedInfo: { + value: { + manifestValues: { + allChecks: [ + {id: 'hasStartUrl', passing: true}, + {id: 'hasIconsAtLeast192px', passing: true}, + {id: 'hasIconsAtLeast512px', passing: true}, + {id: 'hasPWADisplayValue', passing: true}, + {id: 'hasBackgroundColor', passing: true}, + {id: 'hasThemeColor', passing: true}, + {id: 'hasShortName', passing: false}, + {id: 'shortNameLength', passing: false}, + {id: 'hasName', passing: true}, + ], + }, + }, + }, + }, + 'themed-omnibox': { + score: 1, + extendedInfo: { + value: { + manifestValues: { + allChecks: [ + {id: 'hasStartUrl', passing: true}, + {id: 'hasIconsAtLeast192px', passing: true}, + {id: 'hasIconsAtLeast512px', passing: true}, + {id: 'hasPWADisplayValue', passing: true}, + {id: 'hasBackgroundColor', passing: true}, + {id: 'hasThemeColor', passing: true}, + {id: 'hasShortName', passing: false}, + {id: 'shortNameLength', passing: false}, + {id: 'hasName', passing: true}, + ], + }, + }, + }, + }, + 'content-width': { + score: 1, + }, + + // "manual" audits. Just verify in the results. + 'pwa-cross-browser': { + score: 0, + manual: true, + }, + 'pwa-page-transitions': { + score: 0, + manual: true, + }, + 'pwa-each-page-has-url': { + score: 0, + manual: true, + }, + }, + }, + + { + initialUrl: 'https://shop.polymer-project.org/', + url: 'https://shop.polymer-project.org/', + audits: { + 'is-on-https': { + score: 1, + }, + 'redirects-http': { + score: 1, + }, + 'service-worker': { + score: 1, + }, + 'works-offline': { + score: 1, + }, + 'viewport': { + score: 1, + }, + 'without-javascript': { + score: 1, + }, + 'load-fast-enough-for-pwa': { + // Ignore speed test; just verify that it ran. + }, + 'webapp-install-banner': { + // FIXME(bckenny): This is a lie, the site should pass this. Issue #4898 + score: 0, + extendedInfo: { + value: { + // FIXME(bckenny): There should not be any failures Issue #4898 + failures: [ + 'Service worker does not successfully serve the manifest\'s start_url', + 'Unable to fetch start URL via service worker', + ], + manifestValues: { + allChecks: [ + {id: 'hasStartUrl', passing: true}, + {id: 'hasIconsAtLeast192px', passing: true}, + {id: 'hasIconsAtLeast512px', passing: true}, + {id: 'hasPWADisplayValue', passing: true}, + {id: 'hasBackgroundColor', passing: true}, + {id: 'hasThemeColor', passing: true}, + {id: 'hasShortName', passing: true}, + {id: 'shortNameLength', passing: true}, + {id: 'hasName', passing: true}, + ], + }, + }, + }, + }, + 'splash-screen': { + score: 1, + extendedInfo: { + value: { + manifestValues: { + allChecks: [ + {id: 'hasStartUrl', passing: true}, + {id: 'hasIconsAtLeast192px', passing: true}, + {id: 'hasIconsAtLeast512px', passing: true}, + {id: 'hasPWADisplayValue', passing: true}, + {id: 'hasBackgroundColor', passing: true}, + {id: 'hasThemeColor', passing: true}, + {id: 'hasShortName', passing: true}, + {id: 'shortNameLength', passing: true}, + {id: 'hasName', passing: true}, + ], + }, + }, + }, + }, + 'themed-omnibox': { + score: 1, + extendedInfo: { + value: { + manifestValues: { + allChecks: [ + {id: 'hasStartUrl', passing: true}, + {id: 'hasIconsAtLeast192px', passing: true}, + {id: 'hasIconsAtLeast512px', passing: true}, + {id: 'hasPWADisplayValue', passing: true}, + {id: 'hasBackgroundColor', passing: true}, + {id: 'hasThemeColor', passing: true}, + {id: 'hasShortName', passing: true}, + {id: 'shortNameLength', passing: true}, + {id: 'hasName', passing: true}, + ], + }, + }, + }, + }, + 'content-width': { + score: 1, + }, + + // "manual" audits. Just verify in the results. + 'pwa-cross-browser': { + score: 0, + manual: true, + }, + 'pwa-page-transitions': { + score: 0, + manual: true, + }, + 'pwa-each-page-has-url': { + score: 0, + manual: true, + }, + }, + }, + + { + initialUrl: 'https://pwa.rocks', + url: 'https://pwa.rocks/', + audits: { + 'is-on-https': { + score: 1, + }, + 'redirects-http': { + score: 1, + }, + 'service-worker': { + score: 1, + }, + 'works-offline': { + score: 1, + }, + 'viewport': { + score: 1, + }, + 'without-javascript': { + score: 1, + }, + 'load-fast-enough-for-pwa': { + // Ignore speed test; just verify that it ran . + }, + 'webapp-install-banner': { + score: 1, + extendedInfo: { + value: { + manifestValues: { + allChecks: [ + {id: 'hasStartUrl', passing: true}, + {id: 'hasIconsAtLeast192px', passing: true}, + {id: 'hasIconsAtLeast512px', passing: false}, + {id: 'hasPWADisplayValue', passing: true}, + {id: 'hasBackgroundColor', passing: true}, + {id: 'hasThemeColor', passing: true}, + {id: 'hasShortName', passing: true}, + {id: 'shortNameLength', passing: true}, + {id: 'hasName', passing: true}, + ], + }, + }, + }, + }, + 'splash-screen': { + score: 0, + extendedInfo: { + value: { + manifestValues: { + allChecks: [ + {id: 'hasStartUrl', passing: true}, + {id: 'hasIconsAtLeast192px', passing: true}, + {id: 'hasIconsAtLeast512px', passing: false}, + {id: 'hasPWADisplayValue', passing: true}, + {id: 'hasBackgroundColor', passing: true}, + {id: 'hasThemeColor', passing: true}, + {id: 'hasShortName', passing: true}, + {id: 'shortNameLength', passing: true}, + {id: 'hasName', passing: true}, + ], + }, + }, + }, + }, + 'themed-omnibox': { + score: 0, + extendedInfo: { + value: { + manifestValues: { + allChecks: [ + {id: 'hasStartUrl', passing: true}, + {id: 'hasIconsAtLeast192px', passing: true}, + {id: 'hasIconsAtLeast512px', passing: false}, + {id: 'hasPWADisplayValue', passing: true}, + {id: 'hasBackgroundColor', passing: true}, + {id: 'hasThemeColor', passing: true}, + {id: 'hasShortName', passing: true}, + {id: 'shortNameLength', passing: true}, + {id: 'hasName', passing: true}, + ], + }, + }, + }, + }, + 'content-width': { + score: 1, + }, + + // "manual" audits. Just verify in the results. + 'pwa-cross-browser': { + score: 0, + manual: true, + }, + 'pwa-page-transitions': { + score: 0, + manual: true, + }, + 'pwa-each-page-has-url': { + score: 0, + manual: true, + }, + }, + }, +]; diff --git a/lighthouse-cli/test/smokehouse/redirects/run-tests.sh b/lighthouse-cli/test/smokehouse/redirects/run-tests.sh deleted file mode 100755 index 89bdee363ae5..000000000000 --- a/lighthouse-cli/test/smokehouse/redirects/run-tests.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash - -node lighthouse-cli/test/fixtures/static-server.js & - -sleep 0.5s - -config="lighthouse-cli/test/smokehouse/redirects-config.js" -expectations="lighthouse-cli/test/smokehouse/redirects/expectations.js" - -# run smoketest, expecting results found in offline-expectations -yarn smokehouse --config-path=$config --expectations-path=$expectations -exit_code=$? - -# kill test servers -kill $(jobs -p) - -exit "$exit_code" diff --git a/lighthouse-cli/test/smokehouse/run-all-tests.sh b/lighthouse-cli/test/smokehouse/run-all-tests.sh deleted file mode 100644 index c4ed77028f5a..000000000000 --- a/lighthouse-cli/test/smokehouse/run-all-tests.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -for d in "$DIR"/*/ ; do - bash "${d}run-tests.sh" - if [ $? -ne 0 ] - then - echo "Error: $d smoke test failed" - exit 1 - fi -done diff --git a/lighthouse-cli/test/smokehouse/run-smoke.js b/lighthouse-cli/test/smokehouse/run-smoke.js new file mode 100644 index 000000000000..d8b81c1e15d4 --- /dev/null +++ b/lighthouse-cli/test/smokehouse/run-smoke.js @@ -0,0 +1,186 @@ +/** + * @license Copyright 2018 Google Inc. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + */ +'use strict'; + +/* eslint-disable no-console */ +const {promisify} = require('util'); +const execAsync = promisify(require('child_process').exec); + +const {server, serverForOffline} = require('../fixtures/static-server'); +const log = require('lighthouse-logger'); + +const purpleify = str => `${log.purple}${str}${log.reset}`; +const smokehouseDir = 'lighthouse-cli/test/smokehouse/'; + +/** + * @typedef {object} SmoketestDfn + * @property {string} id + * @property {string} expectations + * @property {string} config + * @property {string | undefined} batch + */ + +/** @type {Array} */ +const SMOKETESTS = [{ + id: 'ally', + config: smokehouseDir + 'a11y/a11y-config.js', + expectations: 'a11y/expectations.js', +}, { + id: 'pwa', + expectations: smokehouseDir + 'pwa-expectations.js', + config: smokehouseDir + 'pwa-config.js', +}, { + id: 'pwa2', + expectations: smokehouseDir + 'pwa2-expectations.js', + config: smokehouseDir + 'pwa-config.js', +}, { + id: 'dbw', + expectations: 'dobetterweb/dbw-expectations.js', + config: smokehouseDir + 'dbw-config.js', +}, { + id: 'redirects', + expectations: 'redirects/expectations.js', + config: smokehouseDir + 'redirects-config.js', +}, { + id: 'seo', + expectations: 'seo/expectations.js', + config: smokehouseDir + 'seo-config.js', +}, { + id: 'offline', + expectations: 'offline-local/offline-expectations.js', + config: smokehouseDir + 'offline-config.js', +}, { + id: 'byte', + expectations: 'byte-efficiency/expectations.js', + config: smokehouseDir + 'byte-config.js', + batch: 'perf', +}, { + id: 'perf', + expectations: 'perf/expectations.js', + config: 'lighthouse-core/config/perf-config.js', + batch: 'perf', +}, { + id: 'ttci', + expectations: 'tricky-ttci/expectations.js', + config: 'lighthouse-core/config/default-config.js', +}]; + +/** + * Display smokehouse output from child process + * @param {{id: string, process: NodeJS.Process} || {id: string, error: Error & {stdout : NodeJS.WriteStream, stderr: NodeJS.WriteStream}}} result + */ +function displaySmokehouseOutput(result) { + console.log(`\n${purpleify(result.id)} smoketest results:`); + if (result.error) { + console.log(result.error.message); + process.stdout.write(result.error.stdout); + process.stderr.write(result.error.stderr); + } else { + process.stdout.write(result.process.stdout); + process.stderr.write(result.process.stderr); + } + console.timeEnd(`smoketest-${result.id}`); + console.log(`${purpleify(result.id)} smoketest complete. \n`); +} + +/** + * Run smokehouse in child processes for selected smoketests + * Display output from each as soon as they finish, but resolve function when ALL are complete + * @param {Array} smokes + */ +async function runSmokehouse(smokes) { + const cmdPromises = []; + for (const {id, expectations, config} of smokes) { + console.log(`${purpleify(id)} smoketest starting…`); + console.time(`smoketest-${id}`); + const cmd = [ + 'node lighthouse-cli/test/smokehouse/smokehouse.js', + `--config-path=${config}`, + `--expectations-path=${expectations}`, + ].join(' '); + + // The promise ensures we output immediately, even if the process errors + const p = execAsync(cmd, {timeout: 3 * 60 * 1000, encoding: 'utf8'}) + .then(cp => ({id: id, process: cp})) + .catch(err => ({id: id, error: err})) + .then(result => displaySmokehouseOutput(result)); + + // If the machine is terribly slow, we'll run all smoketests in succession, not parallel + if (process.env.APPVEYOR) { + await p; + } + cmdPromises.push(p); + } + + return Promise.all(cmdPromises); +} + +/** + * Determine batches of smoketests to run, based on argv + * @param {string[]} argv + * @return {Map>} + */ +function getSmoketestBatches(argv) { + let smokes = []; + const usage = ` ${log.dim}yarn smoke ${SMOKETESTS.map(t => t.id).join(' ')}${log.reset}\n`; + + if (argv.length === 0) { + smokes = SMOKETESTS; + console.log('Running ALL smoketests. Equivalent to:'); + console.log(usage); + } else { + smokes = SMOKETESTS.filter(test => argv.includes(test.id)); + console.log(`Running ONLY smoketests for: ${smokes.map(t => t.id).join(' ')}\n`); + } + + const unmatchedIds = argv.filter(requestedId => !SMOKETESTS.map(t => t.id).includes(requestedId)); + if (unmatchedIds.length) { + console.log(log.redify(`Smoketests not found for: ${unmatchedIds.join(' ')}`)); + console.log(usage); + } + + // Split into serial batches that will run their tests concurrently + const batches = smokes.reduce((map, test) => { + const batch = map.get(test.batch) || []; + batch.push(test); + return map.set(test.batch, batch); + }, new Map()); + + return batches; +} + +/** + * Main function. Run webservers, smokehouse, then report on failures + */ +async function cli() { + server.listen(10200, 'localhost'); + serverForOffline.listen(10503, 'localhost'); + + const argv = process.argv.slice(2); + const batches = getSmoketestBatches(argv); + + const smokeResults = []; + for (const [batchName, batch] of batches) { + console.log(`Smoketest batch: ${batchName || 'default'}`); + const results = await runSmokehouse(batch); + smokeResults.push(...results); + } + + await new Promise(resolve => server.close(resolve)); + await new Promise(resolve => serverForOffline.close(resolve)); + + const failingTests = smokeResults.filter(result => !!result.error); + + if (failingTests.length) { + const testNames = failingTests.map(t => t.id).join(', '); + console.error(log.redify(`We have ${failingTests.length} failing smoketests: ${testNames}`)); + process.exit(1); + } + + process.exit(0); +} + +cli(); diff --git a/lighthouse-cli/test/smokehouse/seo/run-tests.sh b/lighthouse-cli/test/smokehouse/seo/run-tests.sh deleted file mode 100755 index 4d101bf23425..000000000000 --- a/lighthouse-cli/test/smokehouse/seo/run-tests.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash - -node lighthouse-cli/test/fixtures/static-server.js & - -sleep 0.5s - -config="lighthouse-cli/test/smokehouse/seo-config.js" -expectations="lighthouse-cli/test/smokehouse/seo/expectations.js" - -yarn smokehouse --config-path=$config --expectations-path=$expectations -exit_code=$? - -# kill test servers -kill $(jobs -p) - -exit "$exit_code" diff --git a/lighthouse-cli/test/smokehouse/smokehouse.js b/lighthouse-cli/test/smokehouse/smokehouse.js index c46bc5787f33..903dd09c8601 100755 --- a/lighthouse-cli/test/smokehouse/smokehouse.js +++ b/lighthouse-cli/test/smokehouse/smokehouse.js @@ -14,9 +14,6 @@ const spawnSync = require('child_process').spawnSync; const yargs = require('yargs'); const log = require('lighthouse-logger'); -const DEFAULT_CONFIG_PATH = 'pwa-config'; -const DEFAULT_EXPECTATIONS_PATH = 'pwa-expectations'; - const PROTOCOL_TIMEOUT_EXIT_CODE = 67; const RETRIES = 3; const NUMERICAL_EXPECTATION_REGEXP = /^(<=?|>=?)((\d|\.)+)$/; @@ -252,7 +249,9 @@ function reportAssertion(assertion) { ${log.redify(log.cross)} difference at ${log.bold}${diff.path}${log.reset} expected: ${JSON.stringify(diff.expected)} found: ${JSON.stringify(diff.actual)} - found result: ${log.redify(fullActual)} + + found result: + ${log.redify(fullActual)} `; console.log(msg); } else { @@ -288,7 +287,7 @@ function report(results) { }); const plural = correctCount === 1 ? '' : 's'; - const correctStr = `${correctCount} audit${plural}`; + const correctStr = `${correctCount} assertion${plural}`; const colorFn = correctCount === 0 ? log.redify : log.greenify; console.log(` Correctly passed ${colorFn(correctStr)}\n`); @@ -305,8 +304,8 @@ const cli = yargs 'expectations-path': 'The path to the expected audit results file', 'debug': 'Save the artifacts along with the output', }) - .default('config-path', DEFAULT_CONFIG_PATH) - .default('expectations-path', DEFAULT_EXPECTATIONS_PATH) + .require('config-path') + .require('expectations-path') .argv; const configPath = resolveLocalOrCwd(cli['config-path']); @@ -317,8 +316,10 @@ const expectations = require(resolveLocalOrCwd(cli['expectations-path'])); let passingCount = 0; let failingCount = 0; expectations.forEach(expected => { - console.log(`Checking '${expected.initialUrl}'...`); + console.log(`Doing a run of '${expected.initialUrl}'...`); const results = runLighthouse(expected.initialUrl, configPath, cli.debug); + + console.log(`Asserting expected results match those found. (${expected.initialUrl})`); const collated = collateResults(results, expected); const counts = report(collated); passingCount += counts.passed; diff --git a/lighthouse-cli/test/smokehouse/tricky-ttci/run-tests.sh b/lighthouse-cli/test/smokehouse/tricky-ttci/run-tests.sh deleted file mode 100644 index f2c016e87dda..000000000000 --- a/lighthouse-cli/test/smokehouse/tricky-ttci/run-tests.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash - -node lighthouse-cli/test/fixtures/static-server.js & - -sleep 0.5s - -config="lighthouse-core/config/default-config.js" -expectations="lighthouse-cli/test/smokehouse/tricky-ttci/expectations.js" - -yarn smokehouse --config-path=$config --expectations-path=$expectations -exit_code=$? - -# kill test servers -kill $(jobs -p) - -exit "$exit_code" diff --git a/package.json b/package.json index 70f7e3fb868b..650ef77529a0 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,9 @@ "build-viewer": "cd ./lighthouse-viewer && yarn build", "clean": "rimraf *.report.html *.report.dom.html *.report.json *.screenshots.html *.screenshots.json *.devtoolslog.json *.trace.json || true", "lint": "[ \"$CI\" = true ] && eslint --quiet -f codeframe . || eslint .", - "smoke": "bash lighthouse-cli/test/smokehouse/run-all-tests.sh", + + "smoke": "node lighthouse-cli/test/smokehouse/run-smoke.js", + "debug": "node --inspect-brk ./lighthouse-cli/index.js", "start": "node ./lighthouse-cli/index.js", "test": "yarn lint --quiet && yarn unit && yarn type-check && yarn closure",