Revamp smoketest, include offline & SW test

* Replace pythonsimplehttpserver with a configurable node-based server

### offline & sw details
* use SW logic from from googlechrome.github.io/samples/service-worker/basic , but adapted
* Simpler skipWaiting thanks to GoogleChrome/samples@dbca5f7#commitcomment-18601764
* run smoketests on diff ports so sw isn't shared.
* Load offline-ready page twice so we dont have to worry about racing between cache population and lighthouse's gatherer.
* make offline page take longer to load to allow enough time for SW to populate cache

lighthouse-cli/scripts/run-smoke-tests.sh

@@ -1,10 +1,11 @@
 #!/usr/bin/env bash
 
-cd lighthouse-cli/test/fixtures && python -m SimpleHTTPServer 10200 &
+node lighthouse-cli/test/fixtures/static-server.js &
+
+sleep 0.5s
 
 NODE=$([ $(node -v | grep -E "v4") ] && echo "node --harmony" || echo "node")
-#config="$PWD/lighthouse-cli/test/fixtures/smoketest-config.json"
-#flags="--config-path=$config"
+config="$PWD/lighthouse-cli/test/fixtures/smoketest-offline-config.json"
 
 offline200result="URL responds with a 200 when offline"
 
@@ -30,15 +31,13 @@ if ! grep -q "$offline200result: false" results; then
   exit 1
 fi
 
-# SKIP this test for now until the flakiness is addressed.
-# sleep 1s
-#
-# # test mojibrush which should pass the offline test
-# $NODE lighthouse-cli $flags https://www.moji-brush.com > results
-#
-# if ! grep -q "$offline200result: true" results; then
-#   echo "Fail! offline ready site did not work while offline"
-#   cat results
-#   exit 1
-# fi
-#
+sleep 0.5s
+
+# run minimal lighthouse run against a basic offline-sw page
+$NODE lighthouse-cli --config-path=$config --quiet http://localhost:10503/offline-ready.html > results
+
+if ! grep -q "$offline200result: true" results; then
+  echo "Fail! offline ready site did not work while offline"
+  cat results
+  exit 1
+fi

lighthouse-cli/test/fixtures/offline-ready-sw.js

@@ -0,0 +1,82 @@
+/**
+ * @license
+ * Copyright 2016 Google Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* eslint-env worker, serviceworker */
+
+// This service-worker courtesy of googlechrome.github.io/samples/service-worker/basic/index.html
+
+// A list of local resources we always want to be cached.
+const PRECACHE_URLS = [
+  './offline-ready.html',
+  './offline-ready-sw.js',
+  './smoketest-offline-config.json'
+];
+
+// Names of the two caches used in this version of the service worker.
+// Change to v2, etc. when you update any of the local resources, which will
+// in turn trigger the install event again.
+const PRECACHE = 'precache-v1';
+const RUNTIME = 'runtime';
+
+// The install handler takes care of precaching the resources we always need.
+self.addEventListener('install', event => {
+  self.skipWaiting();
+
+  const populateCaches = caches.open(PRECACHE)
+      .then(cache => cache.addAll(PRECACHE_URLS));
+
+  event.waitUntil(populateCaches);
+});
+
+// The activate handler takes care of cleaning up old caches.
+self.addEventListener('activate', event => {
+  const currentCaches = [PRECACHE, RUNTIME];
+  event.waitUntil(
+    caches.keys().then(cacheNames => {
+      return cacheNames.filter(cacheName => !currentCaches.includes(cacheName));
+    }).then(cachesToDelete => {
+      return Promise.all(cachesToDelete.map(cacheToDelete => {
+        return caches.delete(cacheToDelete);
+      }));
+    }).then(() => self.clients.claim())
+  );
+});
+
+// The fetch handler serves responses for same-origin resources from a cache.
+// If no response is found, it populates the runtime cache with the response
+// from the network before returning it to the page.
+self.addEventListener('fetch', event => {
+  // Skip cross-origin requests, like those for Google Analytics.
+  if (!event.request.url.startsWith(self.location.origin)) {
+    return;
+  }
+
+  event.respondWith(
+    caches.match(event.request).then(cachedResponse => {
+      if (cachedResponse) {
+        return cachedResponse;
+      }
+
+      return caches.open(RUNTIME).then(cache => {
+        return fetch(event.request).then(response => {
+          // Put a copy of the response in the runtime cache.
+          return cache.put(event.request, response.clone()).then(_ => response);
+        });
+      });
+    })
+  );
+});

lighthouse-cli/test/fixtures/offline-ready.html

@@ -0,0 +1,55 @@
+<!doctype html>
+<!--
+ * Copyright 2016 Google Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+-->
+
+<title>So offline-ready. The most.</title>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1.0, minimum-scale=1.0">
+<!--<link rel="manifest" href="/manifest.json">-->
+
+<h1>
+  Whenever you call me, I'll be there.
+</h1>
+<h2>
+  Whenever you want me, I'll be there.
+</h2>
+<h3>
+  Whenever you need me, I'll be there.
+</h3>
+<h4>
+  I'll be arounddddd.
+</h4>
+
+<script>
+  if ('serviceWorker' in navigator) {
+    navigator.serviceWorker.register('/offline-ready-sw.js').then(function(registration) {
+      console.log('service worker registration complete');
+
+      registration.addEventListener('statechange',  e => {
+        console.log('sw registration is now', e.target.state);
+      });
+    }).catch(function(e) {
+      console.error('service worker is not so cool.', e);
+      throw e;
+    });
+  }
+</script>
+
+<!--
+Lighthouse will move on to the next thing at onload, however this will race with
+the cache population in the serviceworker's install phase.
+We use an image that takes 2-seconds to load to delay window onload. -->
+<img src="icon-128.png?delay">

lighthouse-cli/test/fixtures/smoketest-config.json → lighthouse-cli/test/fixtures/smoketest-offline-config.json

@@ -2,10 +2,14 @@
   "passes": [{
     "loadPage": true,
     "gatherers": [
-      "service-worker",
-      "offline",
       "viewport"
     ]
+  },{
+    "loadPage": true,
+    "gatherers": [
+      "service-worker",
+      "offline"
+    ]
   }],
 
   "audits": [

lighthouse-cli/test/fixtures/static-server.js

@@ -0,0 +1,72 @@
+/**
+ * @license
+ * Copyright 2016 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+'use strict';
+
+const http = require('http');
+const path = require('path');
+const fs = require('fs');
+const parseURL = require('url').parse;
+
+function requestHandler(request, response) {
+  const filePath = parseURL(request.url).pathname;
+  const queryString = parseURL(request.url).search;
+  const absoluteFilePath = path.join(__dirname, filePath);
+
+  fs.exists(absoluteFilePath, fsExistsCallback);
+
+  function fsExistsCallback(fileExists) {
+    if (!fileExists) {
+      return sendResponse(404, `404 - File not found. ${absoluteFilePath}`);
+    }
+    fs.readFile(absoluteFilePath, 'binary', readFileCallback);
+  }
+
+  function readFileCallback(err, file) {
+    if (err) {
+      console.error(`Unable to read local file ${absoluteFilePath}:`, err);
+      return sendResponse(500, '500 - Internal Server Error');
+    }
+    sendResponse(200, file);
+  }
+
+  function sendResponse(statusCode, data) {
+    const headers = filePath.endsWith('.js') ?
+      {'Content-Type': 'text/javascript'} : undefined;
+    response.writeHead(statusCode, headers);
+
+    if (queryString && queryString.includes('delay')) {
+      response.write('');
+      return setTimeout(finishResponse, 2000, data);
+    }
+    finishResponse(data);
+  }
+
+  function finishResponse(data) {
+    response.write(data, 'binary');
+    response.end();
+  }
+}
+
+const serverForOnline = http.createServer(requestHandler);
+const serverForOffline = http.createServer(requestHandler);
+
+serverForOnline.on('error', e => console.error(e.code, e));
+serverForOffline.on('error', e => console.error(e.code, e));
+
+// Listen
+serverForOnline.listen(10200);
+serverForOffline.listen(10503);

lighthouse-core/config/index.js

@@ -118,7 +118,11 @@ function filterPasses(passes, audits, rootPath) {
 
     freshPass.gatherers = freshPass.gatherers.filter(gatherer => {
       const GathererClass = GatherRunner.getGathererClass(gatherer, rootPath);
-      return requiredGatherers.has(GathererClass.name);
+      const isGatherRequiredByAudits = requiredGatherers.has(GathererClass.name);
+      if (isGatherRequiredByAudits === false) {
+        log.warn('config', `Skipping ${GathererClass.name} gatherer as no audit requires it.`);
+      }
+      return isGatherRequiredByAudits;
     });
 
     return freshPass;

lighthouse-core/scripts/run-mocha.sh

@@ -3,7 +3,7 @@
 flag=$1
 
 function _runmocha() {
-  mocha $2 $__node_harmony $(find $1/test -name '*.js') --timeout 60000;
+  mocha $2 $__node_harmony $(find $1/test -name '*.js' -not -path '*/fixtures/*') --timeout 60000;
 }
 
 if [ "$flag" == '--watch' ]; then

package.json

@@ -10,7 +10,7 @@
   "scripts": {
     "lint": "[ \"$CI\" = true ] && eslint --quiet . || eslint .",
     "smoke": "lighthouse-cli/scripts/run-smoke-tests.sh",
-    "coverage": "node $__node_harmony $(npm bin)/istanbul cover -x \"**/third_party/**\" _mocha -- $(find */test -name '*.js') --timeout 60000 --reporter progress",
+    "coverage": "node $__node_harmony $(npm bin)/istanbul cover -x \"**/third_party/**\" _mocha -- $(find */test -name '*.js' -not -path '*/fixtures/*') --timeout 60000 --reporter progress",
     "coveralls": "npm run coverage && cat ./coverage/lcov.info | coveralls",
     "start": "node ./lighthouse-cli/index.js",
     "test": "npm run lint --silent && npm run unit && npm run closure",