extension(tests): Add extension smoketest

.travis.yml

@@ -17,24 +17,28 @@ cache:
     - lighthouse-viewer/node_modules
     - /home/travis/.rvm/gems/
 install:
+  # if our e2e tests fail in the future it might be that we are not compatible
+  # with the latest puppeteer api so we probably need to run on chromimum
+  # @see https://github.com/GoogleChrome/lighthouse/pull/4640/files#r171425004
+  - export PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=1
   - yarn
   # travis can't handle the parallel install (without caches)
   - yarn run install-all:task:windows
 before_script:
   - export DISPLAY=:99.0
-  - export CHROME_PATH="$(pwd)/chrome-linux/chrome"
+  # see comment above about puppeteer
+  - export CHROME_PATH="$(which google-chrome-stable)"
   - sh -e /etc/init.d/xvfb start
   - yarn build-all
 script:
-  - echo $TRAVIS_EVENT_TYPE;
-  - echo $TRAVIS_BRANCH
   - yarn bundlesize
   - yarn lint
   - yarn unit
   - yarn type-check
   - yarn closure
   - yarn smoke
   - yarn smokehouse
+  - yarn test-extension
   # _JAVA_OPTIONS is breaking parsing of compiler output. See #3338.
   - unset _JAVA_OPTIONS
   - yarn compile-devtools

lighthouse-extension/package.json

@@ -6,7 +6,8 @@
   },
   "scripts": {
     "watch": "gulp watch",
-    "build": "gulp build:production"
+    "build": "gulp build:production",
+    "test": "mocha test/extension-test.js"
   },
   "devDependencies": {
     "brfs": "^1.5.0",

lighthouse-extension/test/extension-test.js

@@ -0,0 +1,159 @@
+/**
+ * @license Copyright 2018 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+/* eslint-env mocha */
+
+const path = require('path');
+const assert = require('assert');
+const fs = require('fs');
+const puppeteer = require('../../node_modules/puppeteer/index.js');
+
+const lighthouseExtensionPath = path.resolve(__dirname, '../dist');
+const config = require(path.resolve(__dirname, '../../lighthouse-core/config/default.js'));
+
+const getAuditsOfCategory = category => config.categories[category].audits;
+
+describe('Lighthouse chrome extension', function() {
+  const manifestLocation = path.join(lighthouseExtensionPath, 'manifest.json');
+  const lighthouseCategories = Object.keys(config.categories);
+  let browser;
+  let extensionPage;
+  let originalManifest;
+
+  function getAuditElementsCount({category, selector}) {
+    return extensionPage.evaluate(
+      ({category, selector}) =>
+        document.querySelector(`#${category}`).parentNode.querySelectorAll(selector).length,
+      {category, selector}
+    );
+  }
+
+  before(async function() {
+    // eslint-disable-next-line
+    this.timeout(90 * 1000);
+
+    // read original manifest
+    originalManifest = fs.readFileSync(manifestLocation);
+
+    const manifest = JSON.parse(originalManifest);
+    // add tabs permision to the manifest
+    manifest.permissions.push('tabs');
+    // write new file to document
+    fs.writeFileSync(manifestLocation, JSON.stringify(manifest, null, 2));
+
+    // start puppeteer
+    browser = await puppeteer.launch({
+      headless: false,
+      executablePath: process.env.CHROME_PATH,
+      args: [
+        `--disable-extensions-except=${lighthouseExtensionPath}`,
+        `--load-extension=${lighthouseExtensionPath}`,
+      ],
+    });
+
+    const page = await browser.newPage();
+    await page.goto('https://www.paulirish.com', {waitUntil: 'networkidle2'});
+    const targets = await browser.targets();
+    const extensionTarget = targets.find(({_targetInfo}) => {
+      return _targetInfo.title === 'Lighthouse' && _targetInfo.type === 'background_page';
+    });
+
+    if (!extensionTarget) {
+      return await browser.close();
+    }
+
+    const client = await extensionTarget.createCDPSession();
+    const lighthouseResult = await client.send('Runtime.evaluate', {
+      expression: `runLighthouseInExtension({
+          restoreCleanState: true,
+        }, ${JSON.stringify(lighthouseCategories)})`,
+      awaitPromise: true,
+      returnByValue: true,
+    });
+
+    if (lighthouseResult.exceptionDetails) {
+      if (lighthouseResult.exceptionDetails.exception) {
+        throw new Error(lighthouseResult.exceptionDetails.exception.description);
+      }
+
+      throw new Error(lighthouseResult.exceptionDetails.text);
+    }
+
+    extensionPage = (await browser.pages()).find(page =>
+      page.url().includes('blob:chrome-extension://')
+    );
+  });
+
+  after(async () => {
+    // put the default manifest back
+    fs.writeFileSync(manifestLocation, originalManifest);
+
+    if (browser) {
+      await browser.close();
+    }
+  });
+
+
+  const selectors = {
+    audits: '.lh-audit,.lh-timeline-metric,.lh-perf-hint',
+    titles: '.lh-score__title, .lh-perf-hint__title, .lh-timeline-metric__title',
+  };
+
+  it('should contain all categories', async () => {
+    const categories = await extensionPage.$$(`#${lighthouseCategories.join(',#')}`);
+    assert.equal(
+      categories.length,
+      lighthouseCategories.length,
+      `${categories.join(' ')} does not match ${lighthouseCategories.join(' ')}`
+    );
+  });
+
+  it('should contain audits of all categories', async () => {
+    for (const category of lighthouseCategories) {
+      let expected = getAuditsOfCategory(category).length;
+      if (category === 'performance') {
+        expected = getAuditsOfCategory(category).filter(a => !!a.group).length;
+      }
+
+      const elementCount = await getAuditElementsCount({category, selector: selectors.audits});
+
+      assert.equal(
+        expected,
+        elementCount,
+        `${category} does not have the correct amount of audits`
+      );
+    }
+  });
+
+  it('should contain a filmstrip', async () => {
+    const filmstrip = await extensionPage.$('.lh-filmstrip');
+
+    assert.ok(!!filmstrip, `filmstrip is not available`);
+  });
+
+  it('should not have any audit errors', async () => {
+    function getDebugStrings(elems, selectors) {
+      return elems.map(el => {
+        const audit = el.closest(selectors.audits);
+        const auditTitle = audit && audit.querySelector(selectors.titles);
+        return {
+          debugString: el.textContent,
+          title: auditTitle ? auditTitle.textContent : 'Audit title unvailable',
+        };
+      });
+    }
+
+    const auditErrors = await extensionPage.$$eval('.lh-debug', getDebugStrings, selectors);
+    const errors = auditErrors.filter(
+      item =>
+        item.debugString.includes('Audit error:') &&
+        // FIXME(phulce): fix timing failing on travis.
+        !item.debugString.includes('No timing information available')
+    );
+    assert.deepStrictEqual(errors, [], 'Audit errors found within the report');
+  });
+});

package.json

@@ -29,6 +29,7 @@
     "debug": "node --inspect-brk ./lighthouse-cli/index.js",
     "start": "node ./lighthouse-cli/index.js",
     "test": "yarn lint --quiet && yarn unit && yarn type-check && yarn closure",
+    "test-extension": "cd lighthouse-extension && yarn test",
     "unit-core": "bash lighthouse-core/scripts/run-mocha.sh --core",
     "unit-cli": "bash lighthouse-core/scripts/run-mocha.sh --cli",
     "unit-viewer": "bash lighthouse-core/scripts/run-mocha.sh --viewer",
@@ -78,6 +79,7 @@
     "mocha": "^3.2.0",
     "npm-run-posix-or-windows": "^2.0.2",
     "postinstall-prepare": "^1.0.1",
+    "puppeteer": "^1.1.1",
     "sinon": "^2.3.5",
     "typescript": "^2.8.0-rc",
     "vscode-chrome-debug-core": "^3.23.8",