From 4d05ee1c0d7fa87e3f42e57a50a7923f953737a3 Mon Sep 17 00:00:00 2001 From: Patrick Hulce Date: Thu, 17 May 2018 17:28:42 -0700 Subject: [PATCH 1/5] misc(scripts): add lantern evaluation scripts --- .gitignore | 1 + .npmignore | 1 + lighthouse-core/audits/predictive-perf.js | 2 +- .../scripts/lantern/download-traces.sh | 10 ++ .../scripts/lantern/evaluate-results.js | 142 ++++++++++++++++++ .../scripts/lantern/run-all-expectations.js | 37 +++++ lighthouse-core/scripts/lantern/run-once.js | 29 ++++ .../test/audits/predictive-perf-test.js | 2 +- 8 files changed, 222 insertions(+), 2 deletions(-) create mode 100755 lighthouse-core/scripts/lantern/download-traces.sh create mode 100755 lighthouse-core/scripts/lantern/evaluate-results.js create mode 100755 lighthouse-core/scripts/lantern/run-all-expectations.js create mode 100755 lighthouse-core/scripts/lantern/run-once.js diff --git a/.gitignore b/.gitignore index 6bdba3308eb6..f5ed26d626a0 100644 --- a/.gitignore +++ b/.gitignore @@ -31,6 +31,7 @@ last-run-results.html !lighthouse-core/test/fixtures/artifacts/**/*.devtoolslog.json latest-run +lantern-data closure-error.log yarn-error.log diff --git a/.npmignore b/.npmignore index a7df47f9759a..e93a289c883c 100644 --- a/.npmignore +++ b/.npmignore @@ -31,6 +31,7 @@ lighthouse-cli/types/*.map node_modules/ results/ +lantern-data/ plots/ diff --git a/lighthouse-core/audits/predictive-perf.js b/lighthouse-core/audits/predictive-perf.js index abbed304b93b..41644194cf24 100644 --- a/lighthouse-core/audits/predictive-perf.js +++ b/lighthouse-core/audits/predictive-perf.js @@ -82,7 +82,7 @@ class PredictivePerf extends Audit { score, rawValue: values.roughEstimateOfTTI, displayValue: Util.formatMilliseconds(values.roughEstimateOfTTI), - extendedInfo: {value: values}, + details: {items: [values]}, }; } } diff --git a/lighthouse-core/scripts/lantern/download-traces.sh b/lighthouse-core/scripts/lantern/download-traces.sh new file mode 100755 index 000000000000..0a7eec34f134 --- /dev/null +++ b/lighthouse-core/scripts/lantern/download-traces.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +# THIS SCRIPT ASSUMES CWD IS ROOT PROJECT + +TAR_URL="https://drive.google.com/a/chromium.org/uc?id=1_w2g6fQVLgHI62FApsyUDejZyHNXMLm0&export=download" +curl -o lantern-traces.tar.gz -L $TAR_URL + +tar -xzf lantern-traces.tar.gz +mv lantern-traces-subset lantern-data +rm lantern-traces.tar.gz diff --git a/lighthouse-core/scripts/lantern/evaluate-results.js b/lighthouse-core/scripts/lantern/evaluate-results.js new file mode 100755 index 000000000000..e739704b9a29 --- /dev/null +++ b/lighthouse-core/scripts/lantern/evaluate-results.js @@ -0,0 +1,142 @@ +#!/usr/bin/env node +/** + * @license Copyright 2018 Google Inc. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + */ +'use strict'; + +/* eslint-disable no-console */ + +const path = require('path'); + +const GOOD_ABSOLUTE_THRESHOLD = 0.2; +const OK_ABSOLUTE_THRESHOLD = 0.5; + +const GOOD_RANK_THRESHOLD = 0.1; + +if (!process.argv[2]) throw new Error('Usage $0 '); + +const COMPUTATIONS_PATH = path.resolve(process.cwd(), process.argv[2]); +/** @type {{sites: LanternSiteDefinition[]}} */ +const expectations = require(COMPUTATIONS_PATH); + +const entries = expectations.sites.filter(site => site.lantern); + +if (!entries.length) { + throw new Error('No lantern metrics available, did you run run-all-expectations.js'); +} + +/** @type {LanternSiteDefinition[]} */ +const totalGood = []; +/** @type {LanternSiteDefinition[]} */ +const totalOk = []; +/** @type {LanternSiteDefinition[]} */ +const totalBad = []; + +/** + * @param {string} metric + * @param {string} lanternMetric + */ +function evaluateBuckets(metric, lanternMetric) { + const good = []; + const ok = []; + const bad = []; + + // @ts-ignore + const sortedByMetric = entries.slice().sort((a, b) => a[metric] - b[metric]); + const sortedByLanternMetric = entries + .slice() + .sort((a, b) => a.lantern[lanternMetric] - b.lantern[lanternMetric]); + + const rankErrors = []; + const percentErrors = []; + for (let entry of entries) { + // @ts-ignore + const expected = Math.round(entry[metric]); + if (expected === 0) continue; + + const expectedRank = sortedByMetric.indexOf(entry); + const actual = Math.round(entry.lantern[lanternMetric]); + const actualRank = sortedByLanternMetric.indexOf(entry); + const diff = Math.abs(actual - expected); + const diffAsPercent = diff / expected; + const rankDiff = Math.abs(expectedRank - actualRank); + const rankDiffAsPercent = rankDiff / entries.length; + + rankErrors.push(rankDiffAsPercent); + percentErrors.push(diffAsPercent); + entry = {...entry, expected, actual, diff, rankDiff, rankDiffAsPercent, metric}; + if (diffAsPercent < GOOD_ABSOLUTE_THRESHOLD || rankDiffAsPercent < GOOD_RANK_THRESHOLD) { + good.push(entry); + } else if (diffAsPercent < OK_ABSOLUTE_THRESHOLD) { + ok.push(entry); + } else bad.push(entry); + } + + if (lanternMetric.includes('roughEstimate')) { + totalGood.push(...good); + totalOk.push(...ok); + totalBad.push(...bad); + } + + const MAPE = Math.round(percentErrors.reduce((x, y) => x + y) / percentErrors.length * 1000) / 10; + const rank = Math.round(rankErrors.reduce((x, y) => x + y) / rankErrors.length * 1000) / 10; + const buckets = `${good.length}/${ok.length}/${bad.length}`; + console.log( + `Evaluating ${metric} vs. ${lanternMetric}: ${rank}% ${MAPE}% - ${buckets}` + ); +} + +console.log('---- Metric Stats ----'); +evaluateBuckets('firstContentfulPaint', 'optimisticFCP'); +evaluateBuckets('firstContentfulPaint', 'pessimisticFCP'); +evaluateBuckets('firstContentfulPaint', 'roughEstimateOfFCP'); + +evaluateBuckets('firstMeaningfulPaint', 'optimisticFMP'); +evaluateBuckets('firstMeaningfulPaint', 'pessimisticFMP'); +evaluateBuckets('firstMeaningfulPaint', 'roughEstimateOfFMP'); + +evaluateBuckets('timeToFirstInteractive', 'optimisticTTFCPUI'); +evaluateBuckets('timeToFirstInteractive', 'pessimisticTTFCPUI'); +evaluateBuckets('timeToFirstInteractive', 'roughEstimateOfTTFCPUI'); + +evaluateBuckets('timeToConsistentlyInteractive', 'optimisticTTI'); +evaluateBuckets('timeToConsistentlyInteractive', 'pessimisticTTI'); +evaluateBuckets('timeToConsistentlyInteractive', 'roughEstimateOfTTI'); + +evaluateBuckets('speedIndex', 'optimisticSI'); +evaluateBuckets('speedIndex', 'pessimisticSI'); +evaluateBuckets('speedIndex', 'roughEstimateOfSI'); + +const total = totalGood.length + totalOk.length + totalBad.length; +console.log('\n---- Summary Stats ----'); +console.log(`Good: ${Math.round(totalGood.length / total * 100)}%`); +console.log(`OK: ${Math.round(totalOk.length / total * 100)}%`); +console.log(`Bad: ${Math.round(totalBad.length / total * 100)}%`); + +console.log('\n---- Worst10 Sites ----'); +for (const entry of totalBad.sort((a, b) => b.rankDiff - a.rankDiff).slice(0, 10)) { + console.log( + entry.actual < entry.expected ? 'underestimated' : 'overestimated', + entry.metric, + 'by', + Math.round(entry.diff), + 'on', + entry.url + ); +} + +/** + * @typedef LanternSiteDefinition + * @property {string} url + * @property {string} tracePath + * @property {string} devtoolsLogPath + * @property {*} lantern + * @property {string} [metric] + * @property {number} [expected] + * @property {number} [actual] + * @property {number} [diff] + * @property {number} [rankDiff] + * @property {number} [rankDiffAsPercent] + */ diff --git a/lighthouse-core/scripts/lantern/run-all-expectations.js b/lighthouse-core/scripts/lantern/run-all-expectations.js new file mode 100755 index 000000000000..8a09df54faa7 --- /dev/null +++ b/lighthouse-core/scripts/lantern/run-all-expectations.js @@ -0,0 +1,37 @@ +#!/usr/bin/env node +/** + * @license Copyright 2018 Google Inc. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + */ +'use strict'; + +/* eslint-disable no-console */ + +const fs = require('fs'); +const path = require('path'); +const execFileSync = require('child_process').execFileSync; + +if (!process.argv[2]) throw new Error('Usage $0 '); + +const RUN_ONCE_PATH = path.join(__dirname, 'run-once.js'); +const EXPECTATIONS_PATH = path.resolve(process.cwd(), process.argv[2]); +const EXPECTATIONS_DIR = path.dirname(EXPECTATIONS_PATH); +const expectations = require(EXPECTATIONS_PATH); + +for (const site of expectations.sites) { + const trace = path.join(EXPECTATIONS_DIR, site.tracePath); + const log = path.join(EXPECTATIONS_DIR, site.devtoolsLogPath); + + console.log('Running', site.url, '...'); + const rawOutput = execFileSync(RUN_ONCE_PATH, [trace, log]) + .toString() + .trim(); + if (!rawOutput) console.log('ERROR EMPTY OUTPUT!'); + const lantern = JSON.parse(rawOutput); + + Object.assign(site, {lantern}); +} + +const computedSummaryPath = path.join(EXPECTATIONS_DIR, 'lantern-computed.json'); +fs.writeFileSync(computedSummaryPath, JSON.stringify(expectations, null, 2)); diff --git a/lighthouse-core/scripts/lantern/run-once.js b/lighthouse-core/scripts/lantern/run-once.js new file mode 100755 index 000000000000..b434a5212d5a --- /dev/null +++ b/lighthouse-core/scripts/lantern/run-once.js @@ -0,0 +1,29 @@ +#!/usr/bin/env node +/** + * @license Copyright 2018 Google Inc. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + */ +'use strict'; + +const path = require('path'); +const LH_ROOT_DIR = path.join(__dirname, '../../../'); + +if (process.argv.length !== 4) throw new Error('Usage $0 '); + +async function run() { + const PredictivePerf = require(path.join(LH_ROOT_DIR, 'lighthouse-core/audits/predictive-perf')); + const Runner = require(path.join(LH_ROOT_DIR, 'lighthouse-core/runner')); + + const traces = {defaultPass: require(process.argv[2])}; + const devtoolsLogs = {defaultPass: require(process.argv[3])}; + const artifacts = {traces, devtoolsLogs, ...Runner.instantiateComputedArtifacts()}; + + const result = await PredictivePerf.audit(artifacts); + process.stdout.write(JSON.stringify(result.details.items[0], null, 2)); +} + +run().catch(err => { + process.stderr.write(err.stack); + process.exit(1); +}); diff --git a/lighthouse-core/test/audits/predictive-perf-test.js b/lighthouse-core/test/audits/predictive-perf-test.js index 9896a17258c3..0b01c8749802 100644 --- a/lighthouse-core/test/audits/predictive-perf-test.js +++ b/lighthouse-core/test/audits/predictive-perf-test.js @@ -29,7 +29,7 @@ describe('Performance: predictive performance audit', () => { assert.equal(Math.round(output.rawValue), 4309); assert.equal(output.displayValue, '4,310\xa0ms'); - const valueOf = name => Math.round(output.extendedInfo.value[name]); + const valueOf = name => Math.round(output.details.items[0][name]); assert.equal(valueOf('roughEstimateOfFCP'), 1272); assert.equal(valueOf('optimisticFCP'), 611); assert.equal(valueOf('pessimisticFCP'), 611); From 611fe858e5580bc89221bb88d2f006c6d6196483 Mon Sep 17 00:00:00 2001 From: Patrick Hulce Date: Fri, 18 May 2018 15:29:21 -0700 Subject: [PATCH 2/5] add headers --- lighthouse-core/scripts/lantern/evaluate-results.js | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/lighthouse-core/scripts/lantern/evaluate-results.js b/lighthouse-core/scripts/lantern/evaluate-results.js index e739704b9a29..bc1ce538f070 100755 --- a/lighthouse-core/scripts/lantern/evaluate-results.js +++ b/lighthouse-core/scripts/lantern/evaluate-results.js @@ -84,11 +84,22 @@ function evaluateBuckets(metric, lanternMetric) { const rank = Math.round(rankErrors.reduce((x, y) => x + y) / rankErrors.length * 1000) / 10; const buckets = `${good.length}/${ok.length}/${bad.length}`; console.log( - `Evaluating ${metric} vs. ${lanternMetric}: ${rank}% ${MAPE}% - ${buckets}` + metric.padEnd(30), + lanternMetric.padEnd(25), + `${rank}%`.padEnd(12), + `${MAPE}%`.padEnd(10), + buckets.padEnd(15) ); } console.log('---- Metric Stats ----'); +console.log( + 'metric'.padEnd(30), + 'estimate'.padEnd(25), + 'rank error'.padEnd(12), + 'MAPE'.padEnd(10), + 'Good/OK/Bad'.padEnd(15) +); evaluateBuckets('firstContentfulPaint', 'optimisticFCP'); evaluateBuckets('firstContentfulPaint', 'pessimisticFCP'); evaluateBuckets('firstContentfulPaint', 'roughEstimateOfFCP'); From 070cadd0d336d38a77cd4314f5f8aee2668defd7 Mon Sep 17 00:00:00 2001 From: Patrick Hulce Date: Mon, 21 May 2018 15:33:19 -0700 Subject: [PATCH 3/5] more flexible download script --- lighthouse-core/scripts/lantern/download-traces.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lighthouse-core/scripts/lantern/download-traces.sh b/lighthouse-core/scripts/lantern/download-traces.sh index 0a7eec34f134..b47894541cdd 100755 --- a/lighthouse-core/scripts/lantern/download-traces.sh +++ b/lighthouse-core/scripts/lantern/download-traces.sh @@ -1,6 +1,8 @@ #!/bin/bash -# THIS SCRIPT ASSUMES CWD IS ROOT PROJECT +DIRNAME="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +LH_ROOT_PATH="$DIRNAME/../../.." +cd $LH_ROOT_PATH TAR_URL="https://drive.google.com/a/chromium.org/uc?id=1_w2g6fQVLgHI62FApsyUDejZyHNXMLm0&export=download" curl -o lantern-traces.tar.gz -L $TAR_URL From 93680a0ff28ccbd41c1f8b048cff3012672b306b Mon Sep 17 00:00:00 2001 From: Patrick Hulce Date: Wed, 23 May 2018 14:50:06 -0700 Subject: [PATCH 4/5] add comment to trace tar link --- lighthouse-core/scripts/lantern/download-traces.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/lighthouse-core/scripts/lantern/download-traces.sh b/lighthouse-core/scripts/lantern/download-traces.sh index b47894541cdd..b252b6b3a37e 100755 --- a/lighthouse-core/scripts/lantern/download-traces.sh +++ b/lighthouse-core/scripts/lantern/download-traces.sh @@ -4,6 +4,7 @@ DIRNAME="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" LH_ROOT_PATH="$DIRNAME/../../.." cd $LH_ROOT_PATH +# snapshot of ~100 traces with no throttling recorded 2017-12-06 on a HP z840 workstation TAR_URL="https://drive.google.com/a/chromium.org/uc?id=1_w2g6fQVLgHI62FApsyUDejZyHNXMLm0&export=download" curl -o lantern-traces.tar.gz -L $TAR_URL From 89d9251f3d7482f8f1c53cc2f07d799ceda5cdc7 Mon Sep 17 00:00:00 2001 From: Patrick Hulce Date: Wed, 23 May 2018 15:30:08 -0700 Subject: [PATCH 5/5] typedefs --- .../scripts/lantern/evaluate-results.js | 63 ++++++++++++++----- 1 file changed, 46 insertions(+), 17 deletions(-) diff --git a/lighthouse-core/scripts/lantern/evaluate-results.js b/lighthouse-core/scripts/lantern/evaluate-results.js index bc1ce538f070..58e1f109be32 100755 --- a/lighthouse-core/scripts/lantern/evaluate-results.js +++ b/lighthouse-core/scripts/lantern/evaluate-results.js @@ -27,16 +27,16 @@ if (!entries.length) { throw new Error('No lantern metrics available, did you run run-all-expectations.js'); } -/** @type {LanternSiteDefinition[]} */ +/** @type {LanternEvaluation[]} */ const totalGood = []; -/** @type {LanternSiteDefinition[]} */ +/** @type {LanternEvaluation[]} */ const totalOk = []; -/** @type {LanternSiteDefinition[]} */ +/** @type {LanternEvaluation[]} */ const totalBad = []; /** - * @param {string} metric - * @param {string} lanternMetric + * @param {keyof LanternSiteDefinition} metric + * @param {keyof LanternMetrics} lanternMetric */ function evaluateBuckets(metric, lanternMetric) { const good = []; @@ -51,7 +51,7 @@ function evaluateBuckets(metric, lanternMetric) { const rankErrors = []; const percentErrors = []; - for (let entry of entries) { + for (const entry of entries) { // @ts-ignore const expected = Math.round(entry[metric]); if (expected === 0) continue; @@ -66,12 +66,12 @@ function evaluateBuckets(metric, lanternMetric) { rankErrors.push(rankDiffAsPercent); percentErrors.push(diffAsPercent); - entry = {...entry, expected, actual, diff, rankDiff, rankDiffAsPercent, metric}; + const evaluation = {...entry, expected, actual, diff, rankDiff, rankDiffAsPercent, metric}; if (diffAsPercent < GOOD_ABSOLUTE_THRESHOLD || rankDiffAsPercent < GOOD_RANK_THRESHOLD) { - good.push(entry); + good.push(evaluation); } else if (diffAsPercent < OK_ABSOLUTE_THRESHOLD) { - ok.push(entry); - } else bad.push(entry); + ok.push(evaluation); + } else bad.push(evaluation); } if (lanternMetric.includes('roughEstimate')) { @@ -143,11 +143,40 @@ for (const entry of totalBad.sort((a, b) => b.rankDiff - a.rankDiff).slice(0, 10 * @property {string} url * @property {string} tracePath * @property {string} devtoolsLogPath - * @property {*} lantern - * @property {string} [metric] - * @property {number} [expected] - * @property {number} [actual] - * @property {number} [diff] - * @property {number} [rankDiff] - * @property {number} [rankDiffAsPercent] + * @property {LanternMetrics} lantern + * @property {number} [firstContentfulPaint] + * @property {number} [firstMeaningfulPaint] + * @property {number} [timeToFirstInteractive] + * @property {number} [timeToConsistentlyInteractive] + * @property {number} [speedIndex] */ + +/** + * @typedef LanternEvaluation + * @property {string} url + * @property {string} metric + * @property {number} expected + * @property {number} actual + * @property {number} diff + * @property {number} rankDiff + * @property {number} rankDiffAsPercent + */ + +/** + * @typedef LanternMetrics + * @property {number} optimisticFCP + * @property {number} optimisticFMP + * @property {number} optimisticSI + * @property {number} optimisticTTFCPUI + * @property {number} optimisticTTI + * @property {number} pessimisticFCP + * @property {number} pessimisticFMP + * @property {number} pessimisticSI + * @property {number} pessimisticTTFCPUI + * @property {number} pessimisticTTI + * @property {number} roughEstimateOfFCP + * @property {number} roughEstimateOfFMP + * @property {number} roughEstimateOfSI + * @property {number} roughEstimateOfTTFCPUI + * @property {number} roughEstimateOfTTI + */