From 543b422603d91be5f97db132be3b0837086ee61b Mon Sep 17 00:00:00 2001 From: Paul Irish Date: Fri, 19 Aug 2016 20:15:25 -0700 Subject: [PATCH] Revamp smoketest, include offline & SW test * Replace pythonsimplehttpserver with a configurable node-based server ### offline & sw details * use SW logic from from googlechrome.github.io/samples/service-worker/basic , but adapted * Simpler skipWaiting thanks to https://github.com/GoogleChrome/samples/commit/dbca5f70c8beec398b7bde257983ede212c4b25d#commitcomment-18601764 * run smoketests on diff ports so sw isn't shared. * Load offline-ready page twice so we dont have to worry about racing between cache population and lighthouse's gatherer. * make offline page take longer to load to allow enough time for SW to populate cache --- lighthouse-cli/scripts/run-smoke-tests.sh | 29 ++++--- .../test/fixtures/offline-ready-sw.js | 82 +++++++++++++++++++ .../test/fixtures/offline-ready.html | 55 +++++++++++++ ...fig.json => smoketest-offline-config.json} | 8 +- lighthouse-cli/test/fixtures/static-server.js | 72 ++++++++++++++++ lighthouse-core/config/index.js | 6 +- lighthouse-core/scripts/run-mocha.sh | 2 +- package.json | 2 +- 8 files changed, 236 insertions(+), 20 deletions(-) create mode 100644 lighthouse-cli/test/fixtures/offline-ready-sw.js create mode 100644 lighthouse-cli/test/fixtures/offline-ready.html rename lighthouse-cli/test/fixtures/{smoketest-config.json => smoketest-offline-config.json} (94%) create mode 100644 lighthouse-cli/test/fixtures/static-server.js diff --git a/lighthouse-cli/scripts/run-smoke-tests.sh b/lighthouse-cli/scripts/run-smoke-tests.sh index 000fde9273d6..25bfb0dffa81 100755 --- a/lighthouse-cli/scripts/run-smoke-tests.sh +++ b/lighthouse-cli/scripts/run-smoke-tests.sh @@ -1,10 +1,11 @@ #!/usr/bin/env bash -cd lighthouse-cli/test/fixtures && python -m SimpleHTTPServer 10200 & +node lighthouse-cli/test/fixtures/static-server.js & + +sleep 0.5s NODE=$([ $(node -v | grep -E "v4") ] && echo "node --harmony" || echo "node") -#config="$PWD/lighthouse-cli/test/fixtures/smoketest-config.json" -#flags="--config-path=$config" +config="$PWD/lighthouse-cli/test/fixtures/smoketest-offline-config.json" offline200result="URL responds with a 200 when offline" @@ -30,15 +31,13 @@ if ! grep -q "$offline200result: false" results; then exit 1 fi -# SKIP this test for now until the flakiness is addressed. -# sleep 1s -# -# # test mojibrush which should pass the offline test -# $NODE lighthouse-cli $flags https://www.moji-brush.com > results -# -# if ! grep -q "$offline200result: true" results; then -# echo "Fail! offline ready site did not work while offline" -# cat results -# exit 1 -# fi -# +sleep 0.5s + +# run minimal lighthouse run against a basic offline-sw page +$NODE lighthouse-cli --config-path=$config --quiet http://localhost:10503/offline-ready.html > results + +if ! grep -q "$offline200result: true" results; then + echo "Fail! offline ready site did not work while offline" + cat results + exit 1 +fi diff --git a/lighthouse-cli/test/fixtures/offline-ready-sw.js b/lighthouse-cli/test/fixtures/offline-ready-sw.js new file mode 100644 index 000000000000..5b3a75347faa --- /dev/null +++ b/lighthouse-cli/test/fixtures/offline-ready-sw.js @@ -0,0 +1,82 @@ +/** + * @license + * Copyright 2016 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* eslint-env worker, serviceworker */ + +// This service-worker courtesy of googlechrome.github.io/samples/service-worker/basic/index.html + +// A list of local resources we always want to be cached. +const PRECACHE_URLS = [ + './offline-ready.html', + './offline-ready-sw.js', + './smoketest-offline-config.json' +]; + +// Names of the two caches used in this version of the service worker. +// Change to v2, etc. when you update any of the local resources, which will +// in turn trigger the install event again. +const PRECACHE = 'precache-v1'; +const RUNTIME = 'runtime'; + +// The install handler takes care of precaching the resources we always need. +self.addEventListener('install', event => { + self.skipWaiting(); + + const populateCaches = caches.open(PRECACHE) + .then(cache => cache.addAll(PRECACHE_URLS)); + + event.waitUntil(populateCaches); +}); + +// The activate handler takes care of cleaning up old caches. +self.addEventListener('activate', event => { + const currentCaches = [PRECACHE, RUNTIME]; + event.waitUntil( + caches.keys().then(cacheNames => { + return cacheNames.filter(cacheName => !currentCaches.includes(cacheName)); + }).then(cachesToDelete => { + return Promise.all(cachesToDelete.map(cacheToDelete => { + return caches.delete(cacheToDelete); + })); + }).then(() => self.clients.claim()) + ); +}); + +// The fetch handler serves responses for same-origin resources from a cache. +// If no response is found, it populates the runtime cache with the response +// from the network before returning it to the page. +self.addEventListener('fetch', event => { + // Skip cross-origin requests, like those for Google Analytics. + if (!event.request.url.startsWith(self.location.origin)) { + return; + } + + event.respondWith( + caches.match(event.request).then(cachedResponse => { + if (cachedResponse) { + return cachedResponse; + } + + return caches.open(RUNTIME).then(cache => { + return fetch(event.request).then(response => { + // Put a copy of the response in the runtime cache. + return cache.put(event.request, response.clone()).then(_ => response); + }); + }); + }) + ); +}); diff --git a/lighthouse-cli/test/fixtures/offline-ready.html b/lighthouse-cli/test/fixtures/offline-ready.html new file mode 100644 index 000000000000..aadfcd8c179a --- /dev/null +++ b/lighthouse-cli/test/fixtures/offline-ready.html @@ -0,0 +1,55 @@ + + + +So offline-ready. The most. + + + + +

+ Whenever you call me, I'll be there. +

+

+ Whenever you want me, I'll be there. +

+

+ Whenever you need me, I'll be there. +

+

+ I'll be arounddddd. +

+ + + + + diff --git a/lighthouse-cli/test/fixtures/smoketest-config.json b/lighthouse-cli/test/fixtures/smoketest-offline-config.json similarity index 94% rename from lighthouse-cli/test/fixtures/smoketest-config.json rename to lighthouse-cli/test/fixtures/smoketest-offline-config.json index 47cc2f9cdeb1..2381769a838a 100644 --- a/lighthouse-cli/test/fixtures/smoketest-config.json +++ b/lighthouse-cli/test/fixtures/smoketest-offline-config.json @@ -2,10 +2,14 @@ "passes": [{ "loadPage": true, "gatherers": [ - "service-worker", - "offline", "viewport" ] + },{ + "loadPage": true, + "gatherers": [ + "service-worker", + "offline" + ] }], "audits": [ diff --git a/lighthouse-cli/test/fixtures/static-server.js b/lighthouse-cli/test/fixtures/static-server.js new file mode 100644 index 000000000000..e9eee0c2cacf --- /dev/null +++ b/lighthouse-cli/test/fixtures/static-server.js @@ -0,0 +1,72 @@ +/** + * @license + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +'use strict'; + +const http = require('http'); +const path = require('path'); +const fs = require('fs'); +const parseURL = require('url').parse; + +function requestHandler(request, response) { + const filePath = parseURL(request.url).pathname; + const queryString = parseURL(request.url).search; + const absoluteFilePath = path.join(__dirname, filePath); + + fs.exists(absoluteFilePath, fsExistsCallback); + + function fsExistsCallback(fileExists) { + if (!fileExists) { + return sendResponse(404, `404 - File not found. ${absoluteFilePath}`); + } + fs.readFile(absoluteFilePath, 'binary', readFileCallback); + } + + function readFileCallback(err, file) { + if (err) { + console.error(`Unable to read local file ${absoluteFilePath}:`, err); + return sendResponse(500, '500 - Internal Server Error'); + } + sendResponse(200, file); + } + + function sendResponse(statusCode, data) { + const headers = filePath.endsWith('.js') ? + {'Content-Type': 'text/javascript'} : undefined; + response.writeHead(statusCode, headers); + + if (queryString && queryString.includes('delay')) { + response.write(''); + return setTimeout(finishResponse, 2000, data); + } + finishResponse(data); + } + + function finishResponse(data) { + response.write(data, 'binary'); + response.end(); + } +} + +const serverForOnline = http.createServer(requestHandler); +const serverForOffline = http.createServer(requestHandler); + +serverForOnline.on('error', e => console.error(e.code, e)); +serverForOffline.on('error', e => console.error(e.code, e)); + +// Listen +serverForOnline.listen(10200); +serverForOffline.listen(10503); diff --git a/lighthouse-core/config/index.js b/lighthouse-core/config/index.js index 49b864ef018a..a08417218775 100644 --- a/lighthouse-core/config/index.js +++ b/lighthouse-core/config/index.js @@ -118,7 +118,11 @@ function filterPasses(passes, audits, rootPath) { freshPass.gatherers = freshPass.gatherers.filter(gatherer => { const GathererClass = GatherRunner.getGathererClass(gatherer, rootPath); - return requiredGatherers.has(GathererClass.name); + const isGatherRequiredByAudits = requiredGatherers.has(GathererClass.name); + if (isGatherRequiredByAudits === false) { + log.warn('config', `Skipping ${GathererClass.name} gatherer as no audit requires it.`); + } + return isGatherRequiredByAudits; }); return freshPass; diff --git a/lighthouse-core/scripts/run-mocha.sh b/lighthouse-core/scripts/run-mocha.sh index 968ddf0a4e95..52ec2946e25c 100755 --- a/lighthouse-core/scripts/run-mocha.sh +++ b/lighthouse-core/scripts/run-mocha.sh @@ -3,7 +3,7 @@ flag=$1 function _runmocha() { - mocha $2 $__node_harmony $(find $1/test -name '*.js') --timeout 60000; + mocha $2 $__node_harmony $(find $1/test -name '*.js' -not -path '*/fixtures/*') --timeout 60000; } if [ "$flag" == '--watch' ]; then diff --git a/package.json b/package.json index 3cb4a71889a3..0b73f3a64cfc 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,7 @@ "scripts": { "lint": "[ \"$CI\" = true ] && eslint --quiet . || eslint .", "smoke": "lighthouse-cli/scripts/run-smoke-tests.sh", - "coverage": "node $__node_harmony $(npm bin)/istanbul cover -x \"**/third_party/**\" _mocha -- $(find */test -name '*.js') --timeout 60000 --reporter progress", + "coverage": "node $__node_harmony $(npm bin)/istanbul cover -x \"**/third_party/**\" _mocha -- $(find */test -name '*.js' -not -path '*/fixtures/*') --timeout 60000 --reporter progress", "coveralls": "npm run coverage && cat ./coverage/lcov.info | coveralls", "start": "node ./lighthouse-cli/index.js", "test": "npm run lint --silent && npm run unit && npm run closure",