From 6d73d9eb19f444202608f3415862885f98ba0cad Mon Sep 17 00:00:00 2001
From: Connor Clark <cjamcl@google.com>
Date: Thu, 19 Sep 2019 01:01:14 -0700
Subject: [PATCH 01/19] timings script

---
 .gitignore                         |   1 +
 lighthouse-core/scripts/timings.js | 141 +++++++++++++++++++++++++++++
 2 files changed, 142 insertions(+)
 create mode 100644 lighthouse-core/scripts/timings.js

diff --git a/.gitignore b/.gitignore
index 02f265f28640..aab2ee485d5c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -32,6 +32,7 @@ last-run-results.html
 
 latest-run
 lantern-data
+timings-data
 
 closure-error.log
 yarn-error.log
diff --git a/lighthouse-core/scripts/timings.js b/lighthouse-core/scripts/timings.js
new file mode 100644
index 000000000000..53876d60cf73
--- /dev/null
+++ b/lighthouse-core/scripts/timings.js
@@ -0,0 +1,141 @@
+/**
+ * @license Copyright 2019 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+// Example:
+//     node lighthouse-core/scripts/timings.js --name my-collection --collect -n 3 --lh-flags '--only-audits=unminified-javascript' --urls https://www.example.com https://www.nyt.com
+//     node lighthouse-core/scripts/timings.js --name my-collection --summarize --measure-filter 'loadPage|connect'
+
+const fs = require('fs');
+const {execSync} = require('child_process');
+const rimraf = require('rimraf');
+const yargs = require('yargs');
+
+const LH_ROOT = `${__dirname}/../..`;
+const ROOT_OUTPUT_DIR = `${LH_ROOT}/timings-data`;
+
+const argv = yargs
+  .help('help')
+  .describe({
+    'name': 'Unique identifier, makes the folder for storing LHRs. Not a path',
+    // --collect
+    'collect': 'Saves LHRs to disk',
+    'lh-flags': 'Lighthouse flags',
+    'urls': 'Urls to run',
+    'n': 'Number of times to run',
+    // --summarize
+    'summarize': 'Prints statistics report',
+    'measure-filter': 'Regex filter of measures to report. Optional',
+    'output': 'table, json',
+  })
+  .string('measure-filter')
+  .default('output', 'table')
+  .array('urls')
+  .string('lh-flags')
+  .default('lh-flags', '')
+  // Why is the printing for examples so awful?
+  // eslint-disable max-len
+  // .example("node lighthouse-core/scripts/timings.js --name my-collection --collect -n 3 --lh-flags '--only-audits=unminified-javascript' --urls https://www.example.com", 'Collect')
+  // .example("node lighthouse-core/scripts/timings.js --name my-collection --summarize --measure-filter 'loadPage|connect'", 'Summarize')
+  // eslint-enable max-len
+  .wrap(yargs.terminalWidth())
+  .argv;
+
+const outputDir = `${ROOT_OUTPUT_DIR}/${argv.name}`;
+
+if (argv.collect) {
+  if (!fs.existsSync(ROOT_OUTPUT_DIR)) fs.mkdirSync(ROOT_OUTPUT_DIR);
+  if (fs.existsSync(outputDir)) rimraf.sync(outputDir);
+  fs.mkdirSync(outputDir);
+
+  for (const url of argv.urls) {
+    for (let i = 0; i < argv.n; i++) {
+      // const cmd = `node ${LH_ROOT}/lighthouse-cli ${url} --output-path=${outputDir}/lhr${i}.json --output json ${argv['lh-flags']}`;
+      const cmd = [
+        'node',
+        `${LH_ROOT}/lighthouse-cli`,
+        url,
+        `--output-path=${outputDir}/lhr-${url.replace(/[^a-zA-Z0-9]/g, '_')}-${i}.json`,
+        '--output=json',
+        argv['lh-flags'],
+      ].join(' ');
+      execSync(cmd, {stdio: 'ignore'});
+    }
+  }
+}
+
+/**
+ * @param {number[]} values
+ */
+function average(values) {
+  return values.reduce((sum, value) => sum + value) / values.length;
+}
+
+/**
+ * Round to the tenth.
+ * @param {number} value
+ */
+function round(value) {
+  return Math.round(value * 10) / 10;
+}
+
+if (argv.summarize) {
+  /** @type {Map<string, number[]>} */
+  const measuresMap = new Map();
+  /** @type {RegExp|null} */
+  const measureFilter = argv['measure-filter'] ? new RegExp(argv['measure-filter'], 'i') : null;
+
+  for (const lhrPath of fs.readdirSync(outputDir)) {
+    const lhrJson = fs.readFileSync(`${outputDir}/${lhrPath}`, 'utf-8');
+    /** @type {LH.Result} */
+    const lhr = JSON.parse(lhrJson);
+
+    for (const measureName of lhr.timing.entries.map(entry => entry.name)) {
+      if (measureFilter && !measureFilter.test(measureName)) {
+        continue;
+      }
+
+      const measuresKey = `${lhr.requestedUrl}@@@${measureName}`;
+      let measures = measuresMap.get(measuresKey);
+      if (!measures) {
+        measures = [];
+        measuresMap.set(measuresKey, measures);
+      }
+
+      const measureEntry = lhr.timing.entries.find(measure => measure.name === measureName);
+      if (!measureEntry) throw new Error('missing measure');
+
+      measures.push(measureEntry.duration);
+    }
+  }
+
+  const results = [...measuresMap.entries()].map(([measuresKey, measures]) => {
+    const [url, measureName] = measuresKey.split('@@@');
+    const mean = average(measures);
+    const min = Math.min(...measures);
+    const max = Math.max(...measures);
+    const stdev = Math.sqrt(average(measures.map(measure => (measure - mean) ** 2)));
+    return {
+      measureName,
+      url,
+      n: measures.length,
+      mean: round(mean),
+      stdev: round(stdev),
+      min,
+      max,
+    };
+  }).sort((a, b) => {
+    return a.measureName.localeCompare(b.measureName);
+  });
+
+  if (argv.output === 'table') {
+    // eslint-disable-next-line no-console
+    console.table(results);
+  } else if (argv.output === 'json') {
+    // eslint-disable-next-line no-console
+    console.log(JSON.stringify(results, null, 2));
+  }
+}

From f7daeebaf2d5b49fdb8233c460e3667e6ccdea98 Mon Sep 17 00:00:00 2001
From: Connor Clark <cjamcl@google.com>
Date: Thu, 19 Sep 2019 12:07:27 -0700
Subject: [PATCH 02/19] dont stomp on data folders

---
 lighthouse-core/scripts/timings.js | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/lighthouse-core/scripts/timings.js b/lighthouse-core/scripts/timings.js
index 53876d60cf73..366ac7804f74 100644
--- a/lighthouse-core/scripts/timings.js
+++ b/lighthouse-core/scripts/timings.js
@@ -11,7 +11,6 @@
 
 const fs = require('fs');
 const {execSync} = require('child_process');
-const rimraf = require('rimraf');
 const yargs = require('yargs');
 
 const LH_ROOT = `${__dirname}/../..`;
@@ -48,7 +47,7 @@ const outputDir = `${ROOT_OUTPUT_DIR}/${argv.name}`;
 
 if (argv.collect) {
   if (!fs.existsSync(ROOT_OUTPUT_DIR)) fs.mkdirSync(ROOT_OUTPUT_DIR);
-  if (fs.existsSync(outputDir)) rimraf.sync(outputDir);
+  if (fs.existsSync(outputDir)) throw new Error(`folder already exists: ${outputDir}`);
   fs.mkdirSync(outputDir);
 
   for (const url of argv.urls) {

From 2be55abbecd844cdb0195760c7e30efdf7696f81 Mon Sep 17 00:00:00 2001
From: Connor Clark <cjamcl@google.com>
Date: Thu, 19 Sep 2019 12:20:09 -0700
Subject: [PATCH 03/19] measure

---
 lighthouse-core/scripts/timings.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lighthouse-core/scripts/timings.js b/lighthouse-core/scripts/timings.js
index 366ac7804f74..5192abb8d460 100644
--- a/lighthouse-core/scripts/timings.js
+++ b/lighthouse-core/scripts/timings.js
@@ -118,7 +118,7 @@ if (argv.summarize) {
     const max = Math.max(...measures);
     const stdev = Math.sqrt(average(measures.map(measure => (measure - mean) ** 2)));
     return {
-      measureName,
+      measure: measureName,
       url,
       n: measures.length,
       mean: round(mean),
@@ -127,7 +127,7 @@ if (argv.summarize) {
       max,
     };
   }).sort((a, b) => {
-    return a.measureName.localeCompare(b.measureName);
+    return a.measure.localeCompare(b.measure);
   });
 
   if (argv.output === 'table') {

From fece91a23b27703b54e60352be5e812bd2e5a498 Mon Sep 17 00:00:00 2001
From: Connor Clark <cjamcl@google.com>
Date: Thu, 19 Sep 2019 15:13:32 -0700
Subject: [PATCH 04/19] better

---
 lighthouse-core/scripts/timings.js | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/lighthouse-core/scripts/timings.js b/lighthouse-core/scripts/timings.js
index 5192abb8d460..0a693244692e 100644
--- a/lighthouse-core/scripts/timings.js
+++ b/lighthouse-core/scripts/timings.js
@@ -6,7 +6,7 @@
 'use strict';
 
 // Example:
-//     node lighthouse-core/scripts/timings.js --name my-collection --collect -n 3 --lh-flags '--only-audits=unminified-javascript' --urls https://www.example.com https://www.nyt.com
+//     node lighthouse-core/scripts/timings.js --name my-collection --collect -n 3 --lh-flags='--only-audits=unminified-javascript' --urls https://www.example.com https://www.nyt.com
 //     node lighthouse-core/scripts/timings.js --name my-collection --summarize --measure-filter 'loadPage|connect'
 
 const fs = require('fs');
@@ -37,7 +37,7 @@ const argv = yargs
   .default('lh-flags', '')
   // Why is the printing for examples so awful?
   // eslint-disable max-len
-  // .example("node lighthouse-core/scripts/timings.js --name my-collection --collect -n 3 --lh-flags '--only-audits=unminified-javascript' --urls https://www.example.com", 'Collect')
+  // .example("node lighthouse-core/scripts/timings.js --name my-collection --collect -n 3 --lh-flags='--only-audits=unminified-javascript' --urls https://www.example.com", 'Collect')
   // .example("node lighthouse-core/scripts/timings.js --name my-collection --summarize --measure-filter 'loadPage|connect'", 'Summarize')
   // eslint-enable max-len
   .wrap(yargs.terminalWidth())
@@ -59,7 +59,7 @@ if (argv.collect) {
         url,
         `--output-path=${outputDir}/lhr-${url.replace(/[^a-zA-Z0-9]/g, '_')}-${i}.json`,
         '--output=json',
-        argv['lh-flags'],
+        argv.lhFlags,
       ].join(' ');
       execSync(cmd, {stdio: 'ignore'});
     }
@@ -85,7 +85,7 @@ if (argv.summarize) {
   /** @type {Map<string, number[]>} */
   const measuresMap = new Map();
   /** @type {RegExp|null} */
-  const measureFilter = argv['measure-filter'] ? new RegExp(argv['measure-filter'], 'i') : null;
+  const measureFilter = argv.measureFilter ? new RegExp(argv.measureFilter, 'i') : null;
 
   for (const lhrPath of fs.readdirSync(outputDir)) {
     const lhrJson = fs.readFileSync(`${outputDir}/${lhrPath}`, 'utf-8');

From 62e12cb45eda8cef717d61cc2ac724b2490084c1 Mon Sep 17 00:00:00 2001
From: Connor Clark <cjamcl@google.com>
Date: Mon, 23 Sep 2019 11:48:53 -0700
Subject: [PATCH 05/19] remove dead code

---
 lighthouse-core/scripts/timings.js | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/lighthouse-core/scripts/timings.js b/lighthouse-core/scripts/timings.js
index 0a693244692e..0a11f6bb366e 100644
--- a/lighthouse-core/scripts/timings.js
+++ b/lighthouse-core/scripts/timings.js
@@ -37,8 +37,8 @@ const argv = yargs
   .default('lh-flags', '')
   // Why is the printing for examples so awful?
   // eslint-disable max-len
-  // .example("node lighthouse-core/scripts/timings.js --name my-collection --collect -n 3 --lh-flags='--only-audits=unminified-javascript' --urls https://www.example.com", 'Collect')
-  // .example("node lighthouse-core/scripts/timings.js --name my-collection --summarize --measure-filter 'loadPage|connect'", 'Summarize')
+  .example("node lighthouse-core/scripts/timings.js --name my-collection --collect -n 3 --lh-flags='--only-audits=unminified-javascript' --urls https://www.example.com", 'Collect')
+  .example("node lighthouse-core/scripts/timings.js --name my-collection --summarize --measure-filter 'loadPage|connect'", 'Summarize')
   // eslint-enable max-len
   .wrap(yargs.terminalWidth())
   .argv;
@@ -52,7 +52,6 @@ if (argv.collect) {
 
   for (const url of argv.urls) {
     for (let i = 0; i < argv.n; i++) {
-      // const cmd = `node ${LH_ROOT}/lighthouse-cli ${url} --output-path=${outputDir}/lhr${i}.json --output json ${argv['lh-flags']}`;
       const cmd = [
         'node',
         `${LH_ROOT}/lighthouse-cli`,

From f0911dd31455ec3756a8fdc5391045b87f0fa795 Mon Sep 17 00:00:00 2001
From: Connor Clark <cjamcl@google.com>
Date: Mon, 23 Sep 2019 11:50:51 -0700
Subject: [PATCH 06/19] comment out examples

---
 lighthouse-core/scripts/timings.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lighthouse-core/scripts/timings.js b/lighthouse-core/scripts/timings.js
index 0a11f6bb366e..182462c11153 100644
--- a/lighthouse-core/scripts/timings.js
+++ b/lighthouse-core/scripts/timings.js
@@ -37,8 +37,8 @@ const argv = yargs
   .default('lh-flags', '')
   // Why is the printing for examples so awful?
   // eslint-disable max-len
-  .example("node lighthouse-core/scripts/timings.js --name my-collection --collect -n 3 --lh-flags='--only-audits=unminified-javascript' --urls https://www.example.com", 'Collect')
-  .example("node lighthouse-core/scripts/timings.js --name my-collection --summarize --measure-filter 'loadPage|connect'", 'Summarize')
+  // .example('node lighthouse-core/scripts/timings.js --name my-collection --collect -n 3 --lh-flags=\'--only-audits=unminified-javascript\' --urls https://www.example.com', 'Collect')
+  // .example('node lighthouse-core/scripts/timings.js --name my-collection --summarize --measure-filter \'loadPage|connect\'', 'Summarize')
   // eslint-enable max-len
   .wrap(yargs.terminalWidth())
   .argv;

From e5527b438279029593de38a22431ca94b13d28a0 Mon Sep 17 00:00:00 2001
From: Connor Clark <cjamcl@google.com>
Date: Mon, 23 Sep 2019 16:36:14 -0700
Subject: [PATCH 07/19] fns. round

---
 lighthouse-core/scripts/timings.js | 61 +++++++++++++++++-------------
 1 file changed, 34 insertions(+), 27 deletions(-)

diff --git a/lighthouse-core/scripts/timings.js b/lighthouse-core/scripts/timings.js
index 182462c11153..082eae73e5f4 100644
--- a/lighthouse-core/scripts/timings.js
+++ b/lighthouse-core/scripts/timings.js
@@ -41,11 +41,26 @@ const argv = yargs
   // .example('node lighthouse-core/scripts/timings.js --name my-collection --summarize --measure-filter \'loadPage|connect\'', 'Summarize')
   // eslint-enable max-len
   .wrap(yargs.terminalWidth())
-  .argv;
+.argv;
 
 const outputDir = `${ROOT_OUTPUT_DIR}/${argv.name}`;
 
-if (argv.collect) {
+/**
+ * @param {number[]} values
+ */
+function average(values) {
+  return values.reduce((sum, value) => sum + value) / values.length;
+}
+
+/**
+ * Round to the tenth.
+ * @param {number} value
+ */
+function round(value) {
+  return Math.round(value * 10) / 10;
+}
+
+function collect() {
   if (!fs.existsSync(ROOT_OUTPUT_DIR)) fs.mkdirSync(ROOT_OUTPUT_DIR);
   if (fs.existsSync(outputDir)) throw new Error(`folder already exists: ${outputDir}`);
   fs.mkdirSync(outputDir);
@@ -65,51 +80,36 @@ if (argv.collect) {
   }
 }
 
-/**
- * @param {number[]} values
- */
-function average(values) {
-  return values.reduce((sum, value) => sum + value) / values.length;
-}
-
-/**
- * Round to the tenth.
- * @param {number} value
- */
-function round(value) {
-  return Math.round(value * 10) / 10;
-}
-
-if (argv.summarize) {
+function summarize() {
   /** @type {Map<string, number[]>} */
   const measuresMap = new Map();
   /** @type {RegExp|null} */
   const measureFilter = argv.measureFilter ? new RegExp(argv.measureFilter, 'i') : null;
-
+  
   for (const lhrPath of fs.readdirSync(outputDir)) {
     const lhrJson = fs.readFileSync(`${outputDir}/${lhrPath}`, 'utf-8');
     /** @type {LH.Result} */
     const lhr = JSON.parse(lhrJson);
-
+  
     for (const measureName of lhr.timing.entries.map(entry => entry.name)) {
       if (measureFilter && !measureFilter.test(measureName)) {
         continue;
       }
-
+  
       const measuresKey = `${lhr.requestedUrl}@@@${measureName}`;
       let measures = measuresMap.get(measuresKey);
       if (!measures) {
         measures = [];
         measuresMap.set(measuresKey, measures);
       }
-
+  
       const measureEntry = lhr.timing.entries.find(measure => measure.name === measureName);
       if (!measureEntry) throw new Error('missing measure');
-
+  
       measures.push(measureEntry.duration);
     }
   }
-
+  
   const results = [...measuresMap.entries()].map(([measuresKey, measures]) => {
     const [url, measureName] = measuresKey.split('@@@');
     const mean = average(measures);
@@ -122,13 +122,13 @@ if (argv.summarize) {
       n: measures.length,
       mean: round(mean),
       stdev: round(stdev),
-      min,
-      max,
+      min: round(min),
+      max: round(max),
     };
   }).sort((a, b) => {
     return a.measure.localeCompare(b.measure);
   });
-
+  
   if (argv.output === 'table') {
     // eslint-disable-next-line no-console
     console.table(results);
@@ -137,3 +137,10 @@ if (argv.summarize) {
     console.log(JSON.stringify(results, null, 2));
   }
 }
+
+function main() {
+  if (argv.collect) collect();
+  if (argv.summarize) summarize();
+}
+
+main();

From 115b340de72c1064026f25a672bf343cb52c97ff Mon Sep 17 00:00:00 2001
From: Connor Clark <cjamcl@google.com>
Date: Mon, 23 Sep 2019 16:38:01 -0700
Subject: [PATCH 08/19] sort by url

---
 lighthouse-core/scripts/timings.js | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/lighthouse-core/scripts/timings.js b/lighthouse-core/scripts/timings.js
index 082eae73e5f4..79ba5c83e745 100644
--- a/lighthouse-core/scripts/timings.js
+++ b/lighthouse-core/scripts/timings.js
@@ -126,7 +126,10 @@ function summarize() {
       max: round(max),
     };
   }).sort((a, b) => {
-    return a.measure.localeCompare(b.measure);
+    // sort by {measure, url}
+    const measureComp = a.measure.localeCompare(b.measure);
+    if (measureComp !== 0) return measureComp;
+    return a.url.localeCompare(b.url);
   });
   
   if (argv.output === 'table') {

From ba735a462f3abc428b3983b2668de31998ddbe07 Mon Sep 17 00:00:00 2001
From: Connor Clark <cjamcl@google.com>
Date: Mon, 23 Sep 2019 16:38:28 -0700
Subject: [PATCH 09/19] remove dead code

---
 lighthouse-core/scripts/timings.js | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/lighthouse-core/scripts/timings.js b/lighthouse-core/scripts/timings.js
index 79ba5c83e745..149c5b5c8eaa 100644
--- a/lighthouse-core/scripts/timings.js
+++ b/lighthouse-core/scripts/timings.js
@@ -35,11 +35,6 @@ const argv = yargs
   .array('urls')
   .string('lh-flags')
   .default('lh-flags', '')
-  // Why is the printing for examples so awful?
-  // eslint-disable max-len
-  // .example('node lighthouse-core/scripts/timings.js --name my-collection --collect -n 3 --lh-flags=\'--only-audits=unminified-javascript\' --urls https://www.example.com', 'Collect')
-  // .example('node lighthouse-core/scripts/timings.js --name my-collection --summarize --measure-filter \'loadPage|connect\'', 'Summarize')
-  // eslint-enable max-len
   .wrap(yargs.terminalWidth())
 .argv;
 
@@ -140,7 +135,7 @@ function summarize() {
     console.log(JSON.stringify(results, null, 2));
   }
 }
-
+comp
 function main() {
   if (argv.collect) collect();
   if (argv.summarize) summarize();

From 52f1a878deaf5084024fadd213b1aa9507c88445 Mon Sep 17 00:00:00 2001
From: Connor Clark <cjamcl@google.com>
Date: Mon, 23 Sep 2019 16:38:54 -0700
Subject: [PATCH 10/19] lint

---
 lighthouse-core/scripts/timings.js | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/lighthouse-core/scripts/timings.js b/lighthouse-core/scripts/timings.js
index 149c5b5c8eaa..7db661825894 100644
--- a/lighthouse-core/scripts/timings.js
+++ b/lighthouse-core/scripts/timings.js
@@ -80,31 +80,31 @@ function summarize() {
   const measuresMap = new Map();
   /** @type {RegExp|null} */
   const measureFilter = argv.measureFilter ? new RegExp(argv.measureFilter, 'i') : null;
-  
+
   for (const lhrPath of fs.readdirSync(outputDir)) {
     const lhrJson = fs.readFileSync(`${outputDir}/${lhrPath}`, 'utf-8');
     /** @type {LH.Result} */
     const lhr = JSON.parse(lhrJson);
-  
+
     for (const measureName of lhr.timing.entries.map(entry => entry.name)) {
       if (measureFilter && !measureFilter.test(measureName)) {
         continue;
       }
-  
+
       const measuresKey = `${lhr.requestedUrl}@@@${measureName}`;
       let measures = measuresMap.get(measuresKey);
       if (!measures) {
         measures = [];
         measuresMap.set(measuresKey, measures);
       }
-  
+
       const measureEntry = lhr.timing.entries.find(measure => measure.name === measureName);
       if (!measureEntry) throw new Error('missing measure');
-  
+
       measures.push(measureEntry.duration);
     }
   }
-  
+
   const results = [...measuresMap.entries()].map(([measuresKey, measures]) => {
     const [url, measureName] = measuresKey.split('@@@');
     const mean = average(measures);
@@ -126,7 +126,7 @@ function summarize() {
     if (measureComp !== 0) return measureComp;
     return a.url.localeCompare(b.url);
   });
-  
+
   if (argv.output === 'table') {
     // eslint-disable-next-line no-console
     console.table(results);
@@ -135,7 +135,7 @@ function summarize() {
     console.log(JSON.stringify(results, null, 2));
   }
 }
-comp
+
 function main() {
   if (argv.collect) collect();
   if (argv.summarize) summarize();

From c52b9a69f7adb70c04181f4e70bbc0284cc0421b Mon Sep 17 00:00:00 2001
From: Connor Clark <cjamcl@google.com>
Date: Fri, 27 Sep 2019 14:14:13 -0700
Subject: [PATCH 11/19] sample stdev

---
 lighthouse-core/scripts/timings.js | 20 ++++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/lighthouse-core/scripts/timings.js b/lighthouse-core/scripts/timings.js
index 7db661825894..ecefbd914164 100644
--- a/lighthouse-core/scripts/timings.js
+++ b/lighthouse-core/scripts/timings.js
@@ -40,11 +40,27 @@ const argv = yargs
 
 const outputDir = `${ROOT_OUTPUT_DIR}/${argv.name}`;
 
+/**
+ * @param {number[]} values
+ */
+function sum(values) {
+  return values.reduce((sum, value) => sum + value);
+}
+
 /**
  * @param {number[]} values
  */
 function average(values) {
-  return values.reduce((sum, value) => sum + value) / values.length;
+  return sum(values) / values.length;
+}
+
+/**
+ * @param {number[]} values
+ */
+function sampleStdev(values) {
+  const mean = average(values);
+  const variance = sum(values.map(value => (value - mean) ** 2)) / (values.length - 1);
+  return Math.sqrt(variance);
 }
 
 /**
@@ -110,7 +126,7 @@ function summarize() {
     const mean = average(measures);
     const min = Math.min(...measures);
     const max = Math.max(...measures);
-    const stdev = Math.sqrt(average(measures.map(measure => (measure - mean) ** 2)));
+    const stdev = sampleStdev(measures);
     return {
       measure: measureName,
       url,

From d69306f2d5cabac6f0f64d0cbad5653eca59ba21 Mon Sep 17 00:00:00 2001
From: Connor Clark <cjamcl@google.com>
Date: Fri, 27 Sep 2019 14:29:31 -0700
Subject: [PATCH 12/19] mean of measures

---
 lighthouse-core/scripts/timings.js | 24 ++++++++++++++----------
 1 file changed, 14 insertions(+), 10 deletions(-)

diff --git a/lighthouse-core/scripts/timings.js b/lighthouse-core/scripts/timings.js
index ecefbd914164..a2f8b3d17f14 100644
--- a/lighthouse-core/scripts/timings.js
+++ b/lighthouse-core/scripts/timings.js
@@ -102,26 +102,30 @@ function summarize() {
     /** @type {LH.Result} */
     const lhr = JSON.parse(lhrJson);
 
-    for (const measureName of lhr.timing.entries.map(entry => entry.name)) {
-      if (measureFilter && !measureFilter.test(measureName)) {
-        continue;
+    // Sum all measures of the same name.
+    const measuresSummed = lhr.timing.entries.reduce((acc, entry) => {
+      if (measureFilter && !measureFilter.test(entry.name)) {
+        return acc;
       }
 
-      const measuresKey = `${lhr.requestedUrl}@@@${measureName}`;
+      acc[entry.name] = acc[entry.name] || [];
+      acc[entry.name].push(entry.duration);
+      return acc;
+    }, {});
+
+    // Push the average of all the measures of each name.
+    for (const [name, durations] of Object.entries(measuresSummed)) {
+      const measuresKey = `${lhr.requestedUrl}@@@${name}`;
       let measures = measuresMap.get(measuresKey);
       if (!measures) {
         measures = [];
         measuresMap.set(measuresKey, measures);
       }
-
-      const measureEntry = lhr.timing.entries.find(measure => measure.name === measureName);
-      if (!measureEntry) throw new Error('missing measure');
-
-      measures.push(measureEntry.duration);
+      measures.push(average(durations));
     }
   }
 
-  const results = [...measuresMap.entries()].map(([measuresKey, measures]) => {
+  const results = [...measuresMap].map(([measuresKey, measures]) => {
     const [url, measureName] = measuresKey.split('@@@');
     const mean = average(measures);
     const min = Math.min(...measures);

From 2630ea6f0911f8a6b62b78d0d8695dc674aa2473 Mon Sep 17 00:00:00 2001
From: Connor Clark <cjamcl@google.com>
Date: Fri, 27 Sep 2019 14:36:58 -0700
Subject: [PATCH 13/19] naming

---
 lighthouse-core/scripts/timings.js | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/lighthouse-core/scripts/timings.js b/lighthouse-core/scripts/timings.js
index a2f8b3d17f14..d2db5e1e2e79 100644
--- a/lighthouse-core/scripts/timings.js
+++ b/lighthouse-core/scripts/timings.js
@@ -92,8 +92,9 @@ function collect() {
 }
 
 function summarize() {
+  // `${url}@@@${entry.name}` -> duration
   /** @type {Map<string, number[]>} */
-  const measuresMap = new Map();
+  const durationsMap = new Map();
   /** @type {RegExp|null} */
   const measureFilter = argv.measureFilter ? new RegExp(argv.measureFilter, 'i') : null;
 
@@ -114,18 +115,18 @@ function summarize() {
     }, {});
 
     // Push the average of all the measures of each name.
-    for (const [name, durations] of Object.entries(measuresSummed)) {
+    for (const [name, durationsForSingleRun] of Object.entries(measuresSummed)) {
       const measuresKey = `${lhr.requestedUrl}@@@${name}`;
-      let measures = measuresMap.get(measuresKey);
-      if (!measures) {
-        measures = [];
-        measuresMap.set(measuresKey, measures);
+      let durations = durationsMap.get(measuresKey);
+      if (!durations) {
+        durations = [];
+        durationsMap.set(measuresKey, durations);
       }
-      measures.push(average(durations));
+      durations.push(average(durationsForSingleRun));
     }
   }
 
-  const results = [...measuresMap].map(([measuresKey, measures]) => {
+  const results = [...durationsMap].map(([measuresKey, measures]) => {
     const [url, measureName] = measuresKey.split('@@@');
     const mean = average(measures);
     const min = Math.min(...measures);

From 597160eeb07d4667c720094ee37cda4c76c9a801 Mon Sep 17 00:00:00 2001
From: Connor Clark <cjamcl@google.com>
Date: Fri, 27 Sep 2019 14:38:37 -0700
Subject: [PATCH 14/19] naming

---
 lighthouse-core/scripts/timings.js | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/lighthouse-core/scripts/timings.js b/lighthouse-core/scripts/timings.js
index d2db5e1e2e79..a954420dbaf9 100644
--- a/lighthouse-core/scripts/timings.js
+++ b/lighthouse-core/scripts/timings.js
@@ -126,16 +126,16 @@ function summarize() {
     }
   }
 
-  const results = [...durationsMap].map(([measuresKey, measures]) => {
-    const [url, measureName] = measuresKey.split('@@@');
-    const mean = average(measures);
-    const min = Math.min(...measures);
-    const max = Math.max(...measures);
-    const stdev = sampleStdev(measures);
+  const results = [...durationsMap].map(([key, durations]) => {
+    const [url, entryName] = key.split('@@@');
+    const mean = average(durations);
+    const min = Math.min(...durations);
+    const max = Math.max(...durations);
+    const stdev = sampleStdev(durations);
     return {
-      measure: measureName,
+      measure: entryName,
       url,
-      n: measures.length,
+      n: durations.length,
       mean: round(mean),
       stdev: round(stdev),
       min: round(min),

From e4d786a487cb4aaa489c51372463badb0b6ad48c Mon Sep 17 00:00:00 2001
From: Connor Clark <cjamcl@google.com>
Date: Fri, 27 Sep 2019 14:42:50 -0700
Subject: [PATCH 15/19] mv

---
 lighthouse-core/scripts/{timings.js => compare-timings.js} | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
 rename lighthouse-core/scripts/{timings.js => compare-timings.js} (93%)

diff --git a/lighthouse-core/scripts/timings.js b/lighthouse-core/scripts/compare-timings.js
similarity index 93%
rename from lighthouse-core/scripts/timings.js
rename to lighthouse-core/scripts/compare-timings.js
index a954420dbaf9..ce8821d3f64b 100644
--- a/lighthouse-core/scripts/timings.js
+++ b/lighthouse-core/scripts/compare-timings.js
@@ -6,8 +6,8 @@
 'use strict';
 
 // Example:
-//     node lighthouse-core/scripts/timings.js --name my-collection --collect -n 3 --lh-flags='--only-audits=unminified-javascript' --urls https://www.example.com https://www.nyt.com
-//     node lighthouse-core/scripts/timings.js --name my-collection --summarize --measure-filter 'loadPage|connect'
+//     node lighthouse-core/scripts/compare-timings.js --name my-collection --collect -n 3 --lh-flags='--only-audits=unminified-javascript' --urls https://www.example.com https://www.nyt.com
+//     node lighthouse-core/scripts/compare-timings.js --name my-collection --summarize --measure-filter 'loadPage|connect'
 
 const fs = require('fs');
 const {execSync} = require('child_process');

From 3426452f22c9c144778f49b81ddd945a91489859 Mon Sep 17 00:00:00 2001
From: Connor Clark <cjamcl@google.com>
Date: Fri, 27 Sep 2019 16:40:33 -0700
Subject: [PATCH 16/19] tsc

---
 lighthouse-core/scripts/compare-timings.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lighthouse-core/scripts/compare-timings.js b/lighthouse-core/scripts/compare-timings.js
index ce8821d3f64b..f6a049cd87d1 100644
--- a/lighthouse-core/scripts/compare-timings.js
+++ b/lighthouse-core/scripts/compare-timings.js
@@ -112,7 +112,7 @@ function summarize() {
       acc[entry.name] = acc[entry.name] || [];
       acc[entry.name].push(entry.duration);
       return acc;
-    }, {});
+    }, /** @type {Record<string, number[]>} */ ({}));
 
     // Push the average of all the measures of each name.
     for (const [name, durationsForSingleRun] of Object.entries(measuresSummed)) {

From 2cbbd05ea135dc69c57fb8b913020d23f50c3b42 Mon Sep 17 00:00:00 2001
From: Connor Clark <cjamcl@google.com>
Date: Mon, 30 Sep 2019 18:00:15 -0700
Subject: [PATCH 17/19] pr

---
 lighthouse-core/scripts/compare-timings.js | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/lighthouse-core/scripts/compare-timings.js b/lighthouse-core/scripts/compare-timings.js
index f6a049cd87d1..a8ff735bd368 100644
--- a/lighthouse-core/scripts/compare-timings.js
+++ b/lighthouse-core/scripts/compare-timings.js
@@ -103,18 +103,19 @@ function summarize() {
     /** @type {LH.Result} */
     const lhr = JSON.parse(lhrJson);
 
-    // Sum all measures of the same name.
-    const measuresSummed = lhr.timing.entries.reduce((acc, entry) => {
+    // Group all measures of the same name.
+    /** @type {Record<string, number[]>} */
+    const measuresSummed = {};
+    for (const entry of lhr.timing.entries) {
       if (measureFilter && !measureFilter.test(entry.name)) {
-        return acc;
+        continue;
       }
 
-      acc[entry.name] = acc[entry.name] || [];
-      acc[entry.name].push(entry.duration);
-      return acc;
-    }, /** @type {Record<string, number[]>} */ ({}));
+      const durations = measuresSummed[entry.name] = measuresSummed[entry.name] || [];
+      durations.push(entry.duration);
+    }
 
-    // Push the average of all the measures of each name.
+    // Push the aggregate time of each unique measure.
     for (const [name, durationsForSingleRun] of Object.entries(measuresSummed)) {
       const measuresKey = `${lhr.requestedUrl}@@@${name}`;
       let durations = durationsMap.get(measuresKey);
@@ -122,7 +123,7 @@ function summarize() {
         durations = [];
         durationsMap.set(measuresKey, durations);
       }
-      durations.push(average(durationsForSingleRun));
+      durations.push(sum(durationsForSingleRun));
     }
   }
 

From 44e0abc4f5f5aba2ac4963e98ccfe64a474b2662 Mon Sep 17 00:00:00 2001
From: Connor Clark <cjamcl@google.com>
Date: Mon, 30 Sep 2019 18:05:14 -0700
Subject: [PATCH 18/19] rename

---
 lighthouse-core/scripts/compare-timings.js | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/lighthouse-core/scripts/compare-timings.js b/lighthouse-core/scripts/compare-timings.js
index a8ff735bd368..d44c4a357cc2 100644
--- a/lighthouse-core/scripts/compare-timings.js
+++ b/lighthouse-core/scripts/compare-timings.js
@@ -103,25 +103,25 @@ function summarize() {
     /** @type {LH.Result} */
     const lhr = JSON.parse(lhrJson);
 
-    // Group all measures of the same name.
+    // Group all entries of the same name.
     /** @type {Record<string, number[]>} */
-    const measuresSummed = {};
+    const entriesByName = {};
     for (const entry of lhr.timing.entries) {
       if (measureFilter && !measureFilter.test(entry.name)) {
         continue;
       }
 
-      const durations = measuresSummed[entry.name] = measuresSummed[entry.name] || [];
+      const durations = entriesByName[entry.name] = entriesByName[entry.name] || [];
       durations.push(entry.duration);
     }
 
-    // Push the aggregate time of each unique measure.
-    for (const [name, durationsForSingleRun] of Object.entries(measuresSummed)) {
-      const measuresKey = `${lhr.requestedUrl}@@@${name}`;
-      let durations = durationsMap.get(measuresKey);
+    // Push the aggregate time of each unique (by name) entry.
+    for (const [name, durationsForSingleRun] of Object.entries(entriesByName)) {
+      const key = `${lhr.requestedUrl}@@@${name}`;
+      let durations = durationsMap.get(key);
       if (!durations) {
         durations = [];
-        durationsMap.set(measuresKey, durations);
+        durationsMap.set(key, durations);
       }
       durations.push(sum(durationsForSingleRun));
     }

From 2460d9062b7344cf78c7133ab2e3492df7d9845e Mon Sep 17 00:00:00 2001
From: Connor Clark <cjamcl@google.com>
Date: Mon, 30 Sep 2019 18:06:02 -0700
Subject: [PATCH 19/19] rename

---
 lighthouse-core/scripts/compare-timings.js | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/lighthouse-core/scripts/compare-timings.js b/lighthouse-core/scripts/compare-timings.js
index d44c4a357cc2..f04b39911998 100644
--- a/lighthouse-core/scripts/compare-timings.js
+++ b/lighthouse-core/scripts/compare-timings.js
@@ -103,20 +103,20 @@ function summarize() {
     /** @type {LH.Result} */
     const lhr = JSON.parse(lhrJson);
 
-    // Group all entries of the same name.
+    // Group the durations of each entry of the same name.
     /** @type {Record<string, number[]>} */
-    const entriesByName = {};
+    const durationsByName = {};
     for (const entry of lhr.timing.entries) {
       if (measureFilter && !measureFilter.test(entry.name)) {
         continue;
       }
 
-      const durations = entriesByName[entry.name] = entriesByName[entry.name] || [];
+      const durations = durationsByName[entry.name] = durationsByName[entry.name] || [];
       durations.push(entry.duration);
     }
 
     // Push the aggregate time of each unique (by name) entry.
-    for (const [name, durationsForSingleRun] of Object.entries(entriesByName)) {
+    for (const [name, durationsForSingleRun] of Object.entries(durationsByName)) {
       const key = `${lhr.requestedUrl}@@@${name}`;
       let durations = durationsMap.get(key);
       if (!durations) {