misc(scripts): add lantern evaluation scripts

.gitignore

@@ -31,6 +31,7 @@ last-run-results.html
 !lighthouse-core/test/fixtures/artifacts/**/*.devtoolslog.json
 
 latest-run
+lantern-data
 
 closure-error.log
 yarn-error.log

.npmignore

@@ -31,6 +31,7 @@ lighthouse-cli/types/*.map
 
 node_modules/
 results/
+lantern-data/
 
 plots/
 

lighthouse-core/audits/predictive-perf.js

@@ -82,7 +82,7 @@ class PredictivePerf extends Audit {
       score,
       rawValue: values.roughEstimateOfTTI,
       displayValue: Util.formatMilliseconds(values.roughEstimateOfTTI),
-      extendedInfo: {value: values},
+      details: {items: [values]},
     };
   }
 }

lighthouse-core/scripts/lantern/download-traces.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+DIRNAME="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+LH_ROOT_PATH="$DIRNAME/../../.."
+cd $LH_ROOT_PATH
+
+# snapshot of ~100 traces with no throttling recorded 2017-12-06 on a HP z840 workstation
+TAR_URL="https://drive.google.com/a/chromium.org/uc?id=1_w2g6fQVLgHI62FApsyUDejZyHNXMLm0&export=download"
+curl -o lantern-traces.tar.gz -L $TAR_URL
+
+tar -xzf lantern-traces.tar.gz
+mv lantern-traces-subset lantern-data
+rm lantern-traces.tar.gz

lighthouse-core/scripts/lantern/evaluate-results.js

@@ -0,0 +1,153 @@
+#!/usr/bin/env node
+/**
+ * @license Copyright 2018 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+/* eslint-disable no-console */
+
+const path = require('path');
+
+const GOOD_ABSOLUTE_THRESHOLD = 0.2;
+const OK_ABSOLUTE_THRESHOLD = 0.5;
+
+const GOOD_RANK_THRESHOLD = 0.1;
+
+if (!process.argv[2]) throw new Error('Usage $0 <computed summary file>');
+
+const COMPUTATIONS_PATH = path.resolve(process.cwd(), process.argv[2]);
+/** @type {{sites: LanternSiteDefinition[]}} */
+const expectations = require(COMPUTATIONS_PATH);
+
+const entries = expectations.sites.filter(site => site.lantern);
+
+if (!entries.length) {
+  throw new Error('No lantern metrics available, did you run run-all-expectations.js');
+}
+
+/** @type {LanternSiteDefinition[]} */
+const totalGood = [];
+/** @type {LanternSiteDefinition[]} */
+const totalOk = [];
+/** @type {LanternSiteDefinition[]} */
+const totalBad = [];
+
+/**
+ * @param {string} metric
+ * @param {string} lanternMetric
+ */
+function evaluateBuckets(metric, lanternMetric) {
+  const good = [];
+  const ok = [];
+  const bad = [];
+
+  // @ts-ignore
+  const sortedByMetric = entries.slice().sort((a, b) => a[metric] - b[metric]);
+  const sortedByLanternMetric = entries
+    .slice()
+    .sort((a, b) => a.lantern[lanternMetric] - b.lantern[lanternMetric]);
+
+  const rankErrors = [];
+  const percentErrors = [];
+  for (let entry of entries) {
+    // @ts-ignore
+    const expected = Math.round(entry[metric]);
+    if (expected === 0) continue;
+
+    const expectedRank = sortedByMetric.indexOf(entry);
+    const actual = Math.round(entry.lantern[lanternMetric]);
+    const actualRank = sortedByLanternMetric.indexOf(entry);
+    const diff = Math.abs(actual - expected);
+    const diffAsPercent = diff / expected;
+    const rankDiff = Math.abs(expectedRank - actualRank);
+    const rankDiffAsPercent = rankDiff / entries.length;
+
+    rankErrors.push(rankDiffAsPercent);
+    percentErrors.push(diffAsPercent);
+    entry = {...entry, expected, actual, diff, rankDiff, rankDiffAsPercent, metric};
+    if (diffAsPercent < GOOD_ABSOLUTE_THRESHOLD || rankDiffAsPercent < GOOD_RANK_THRESHOLD) {
+      good.push(entry);
+    } else if (diffAsPercent < OK_ABSOLUTE_THRESHOLD) {
+      ok.push(entry);
+    } else bad.push(entry);
+  }
+
+  if (lanternMetric.includes('roughEstimate')) {
+    totalGood.push(...good);
+    totalOk.push(...ok);
+    totalBad.push(...bad);
+  }
+
+  const MAPE = Math.round(percentErrors.reduce((x, y) => x + y) / percentErrors.length * 1000) / 10;
+  const rank = Math.round(rankErrors.reduce((x, y) => x + y) / rankErrors.length * 1000) / 10;
+  const buckets = `${good.length}/${ok.length}/${bad.length}`;
+  console.log(
+    metric.padEnd(30),
+    lanternMetric.padEnd(25),
+    `${rank}%`.padEnd(12),
+    `${MAPE}%`.padEnd(10),
+    buckets.padEnd(15)
+  );
+}
+
+console.log('----    Metric Stats    ----');
+console.log(
+  'metric'.padEnd(30),
+  'estimate'.padEnd(25),
+  'rank error'.padEnd(12),
+  'MAPE'.padEnd(10),
+  'Good/OK/Bad'.padEnd(15)
+);
+evaluateBuckets('firstContentfulPaint', 'optimisticFCP');
+evaluateBuckets('firstContentfulPaint', 'pessimisticFCP');
+evaluateBuckets('firstContentfulPaint', 'roughEstimateOfFCP');
+
+evaluateBuckets('firstMeaningfulPaint', 'optimisticFMP');
+evaluateBuckets('firstMeaningfulPaint', 'pessimisticFMP');
+evaluateBuckets('firstMeaningfulPaint', 'roughEstimateOfFMP');
+
+evaluateBuckets('timeToFirstInteractive', 'optimisticTTFCPUI');
+evaluateBuckets('timeToFirstInteractive', 'pessimisticTTFCPUI');
+evaluateBuckets('timeToFirstInteractive', 'roughEstimateOfTTFCPUI');
+
+evaluateBuckets('timeToConsistentlyInteractive', 'optimisticTTI');
+evaluateBuckets('timeToConsistentlyInteractive', 'pessimisticTTI');
+evaluateBuckets('timeToConsistentlyInteractive', 'roughEstimateOfTTI');
+
+evaluateBuckets('speedIndex', 'optimisticSI');
+evaluateBuckets('speedIndex', 'pessimisticSI');
+evaluateBuckets('speedIndex', 'roughEstimateOfSI');
+
+const total = totalGood.length + totalOk.length + totalBad.length;
+console.log('\n----    Summary Stats    ----');
+console.log(`Good: ${Math.round(totalGood.length / total * 100)}%`);
+console.log(`OK: ${Math.round(totalOk.length / total * 100)}%`);
+console.log(`Bad: ${Math.round(totalBad.length / total * 100)}%`);
+
+console.log('\n----    Worst10 Sites    ----');
+for (const entry of totalBad.sort((a, b) => b.rankDiff - a.rankDiff).slice(0, 10)) {
+  console.log(
+    entry.actual < entry.expected ? 'underestimated' : 'overestimated',
+    entry.metric,
+    'by',
+    Math.round(entry.diff),
+    'on',
+    entry.url
+  );
+}
+
+/**
+ * @typedef LanternSiteDefinition
+ * @property {string} url
+ * @property {string} tracePath
+ * @property {string} devtoolsLogPath
+ * @property {*} lantern
+ * @property {string} [metric]
+ * @property {number} [expected]
+ * @property {number} [actual]
+ * @property {number} [diff]
+ * @property {number} [rankDiff]
+ * @property {number} [rankDiffAsPercent]
+ */

lighthouse-core/scripts/lantern/run-all-expectations.js

@@ -0,0 +1,37 @@
+#!/usr/bin/env node
+/**
+ * @license Copyright 2018 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+/* eslint-disable no-console */
+
+const fs = require('fs');
+const path = require('path');
+const execFileSync = require('child_process').execFileSync;
+
+if (!process.argv[2]) throw new Error('Usage $0 <expectations file>');
+
+const RUN_ONCE_PATH = path.join(__dirname, 'run-once.js');
+const EXPECTATIONS_PATH = path.resolve(process.cwd(), process.argv[2]);
+const EXPECTATIONS_DIR = path.dirname(EXPECTATIONS_PATH);
+const expectations = require(EXPECTATIONS_PATH);
+
+for (const site of expectations.sites) {
+  const trace = path.join(EXPECTATIONS_DIR, site.tracePath);
+  const log = path.join(EXPECTATIONS_DIR, site.devtoolsLogPath);
+
+  console.log('Running', site.url, '...');
+  const rawOutput = execFileSync(RUN_ONCE_PATH, [trace, log])
+    .toString()
+    .trim();
+  if (!rawOutput) console.log('ERROR EMPTY OUTPUT!');
+  const lantern = JSON.parse(rawOutput);
+
+  Object.assign(site, {lantern});
+}
+
+const computedSummaryPath = path.join(EXPECTATIONS_DIR, 'lantern-computed.json');
+fs.writeFileSync(computedSummaryPath, JSON.stringify(expectations, null, 2));

lighthouse-core/scripts/lantern/run-once.js

@@ -0,0 +1,29 @@
+#!/usr/bin/env node
+/**
+ * @license Copyright 2018 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+const path = require('path');
+const LH_ROOT_DIR = path.join(__dirname, '../../../');
+
+if (process.argv.length !== 4) throw new Error('Usage $0 <trace file> <devtools file>');
+
+async function run() {
+  const PredictivePerf = require(path.join(LH_ROOT_DIR, 'lighthouse-core/audits/predictive-perf'));
+  const Runner = require(path.join(LH_ROOT_DIR, 'lighthouse-core/runner'));
+
+  const traces = {defaultPass: require(process.argv[2])};
+  const devtoolsLogs = {defaultPass: require(process.argv[3])};
+  const artifacts = {traces, devtoolsLogs, ...Runner.instantiateComputedArtifacts()};
+
+  const result = await PredictivePerf.audit(artifacts);
+  process.stdout.write(JSON.stringify(result.details.items[0], null, 2));
+}
+
+run().catch(err => {
+  process.stderr.write(err.stack);
+  process.exit(1);
+});

lighthouse-core/test/audits/predictive-perf-test.js

@@ -29,7 +29,7 @@ describe('Performance: predictive performance audit', () => {
       assert.equal(Math.round(output.rawValue), 4309);
       assert.equal(output.displayValue, '4,310\xa0ms');
 
-      const valueOf = name => Math.round(output.extendedInfo.value[name]);
+      const valueOf = name => Math.round(output.details.items[0][name]);
       assert.equal(valueOf('roughEstimateOfFCP'), 1038);
       assert.equal(valueOf('optimisticFCP'), 611);
       assert.equal(valueOf('pessimisticFCP'), 611);