Add support for custom measurement units

CHANGELOG.md

@@ -5,7 +5,9 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/) and this
 project adheres to [Semantic Versioning](http://semver.org/).
 
-<!-- ## Unreleased -->
+## Unreleased
+
+- Add support for defining the unit of a measurement.
 
 ## [0.5.5] 2020-09-21
 

README.md

@@ -90,8 +90,8 @@ If `measurement` is an array, then all of the given measurements will be
 retrieved from each page load. Each measurement from a page is treated as its
 own benchmark.
 
-A measurement can specify a `name` property that will be used to display its
-results.
+A measurement can specify a `name` and/or a `unit` property that will be used to
+display its results.
 
 #### Performance API
 

config.schema.json

@@ -13,6 +13,9 @@
                 },
                 "name": {
                     "type": "string"
+                },
+                "unit": {
+                    "type": "string"
                 }
             },
             "required": [
@@ -218,6 +221,9 @@
                 },
                 "name": {
                     "type": "string"
+                },
+                "unit": {
+                    "type": "string"
                 }
             },
             "required": [
@@ -361,6 +367,9 @@
                 },
                 "name": {
                     "type": "string"
+                },
+                "unit": {
+                    "type": "string"
                 }
             },
             "required": [

src/configfile.ts

@@ -362,6 +362,14 @@ async function parseBenchmark(benchmark: ConfigFileBenchmark, root: string):
     spec.measurement = [benchmark.measurement];
   }
 
+  if (spec.measurement) {
+    for (const measurement of spec.measurement) {
+      if (measurement.unit == null) {
+        measurement.unit = defaults.measurementUnit;
+      }
+    }
+  }
+
   const url = benchmark.url;
   if (url !== undefined) {
     if (isHttpUrl(url)) {

src/csv.ts

@@ -69,11 +69,11 @@ export function formatCsvRaw(results: ResultStatsWithDifferences[]): string {
   for (let r = 0; r < results.length; r++) {
     const {result} = results[r];
     headers.push(result.name);
-    for (let m = 0; m < result.millis.length; m++) {
+    for (let m = 0; m < result.rawData.length; m++) {
       if (rows[m] === undefined) {
         rows[m] = [];
       }
-      rows[m][r] = result.millis[m];
+      rows[m][r] = result.rawData[m];
     }
   }
   return csvStringify([headers, ...rows]);

src/defaults.ts

@@ -24,13 +24,15 @@ export const mode = 'automatic';
 export const resolveBareModules = true;
 export const forceCleanNpmInstall = false;
 export const measurementExpression = 'window.tachometerResult';
+export const measurementUnit = 'ms';
 
 export function measurement(url: LocalUrl|RemoteUrl): Measurement {
   if (url.kind === 'remote') {
     return {
       mode: 'performance',
       entryName: 'first-contentful-paint',
+      unit: measurementUnit
     };
   }
-  return {mode: 'callback'};
+  return {mode: 'callback', unit: measurementUnit};
 }

src/format.ts

@@ -16,7 +16,7 @@ import {UAParser} from 'ua-parser-js';
 import ansi = require('ansi-escape-sequences');
 
 import {Difference, ConfidenceInterval, ResultStats, ResultStatsWithDifferences} from './stats';
-import {BenchmarkSpec, BenchmarkResult} from './types';
+import {BenchmarkSpec, BenchmarkResult, Measurement} from './types';
 
 export const spinner = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏'].map(
     (frame) => ansi.format(`[blue]{${frame}}`));
@@ -93,7 +93,7 @@ export function automaticResultTable(results: ResultStats[]): AutomaticResults {
           if (diff === null) {
             return ansi.format('\n[gray]{-}       ');
           }
-          return formatDifference(diff);
+          return formatDifference(diff, r.result.measurement);
         },
       });
     }
@@ -262,7 +262,7 @@ const browserDimension: Dimension = {
 
 const sampleSizeDimension: Dimension = {
   label: 'Sample size',
-  format: (r: ResultStats) => r.result.millis.length.toString(),
+  format: (r: ResultStats) => r.result.rawData.length.toString(),
 };
 
 const bytesSentDimension: Dimension = {
@@ -275,25 +275,28 @@ const runtimeConfidenceIntervalDimension: Dimension = {
   tableConfig: {
     alignment: 'right',
   },
-  format: (r: ResultStats) => formatConfidenceInterval(r.stats.meanCI, milli),
+  format: (r: ResultStats) => formatConfidenceInterval(
+      r.stats.meanCI, formatMeasure(r.result.measurement)),
 };
 
-function formatDifference({absolute, relative}: Difference): string {
+function formatDifference(
+    {absolute, relative}: Difference, measurement: Measurement): string {
+  const format = formatMeasure(measurement);
   let word, rel, abs;
   if (absolute.low > 0 && relative.low > 0) {
     word = `[bold red]{slower}`;
     rel = formatConfidenceInterval(relative, percent);
-    abs = formatConfidenceInterval(absolute, milli);
+    abs = formatConfidenceInterval(absolute, format);
 
   } else if (absolute.high < 0 && relative.high < 0) {
     word = `[bold green]{faster}`;
     rel = formatConfidenceInterval(negate(relative), percent);
-    abs = formatConfidenceInterval(negate(absolute), milli);
+    abs = formatConfidenceInterval(negate(absolute), format);
 
   } else {
     word = `[bold blue]{unsure}`;
     rel = formatConfidenceInterval(relative, (n) => colorizeSign(n, percent));
-    abs = formatConfidenceInterval(absolute, (n) => colorizeSign(n, milli));
+    abs = formatConfidenceInterval(absolute, (n) => colorizeSign(n, format));
   }
 
   return ansi.format(`${word}\n${rel}\n${abs}`);
@@ -303,9 +306,9 @@ function percent(n: number): string {
   return (n * 100).toFixed(0) + '%';
 }
 
-function milli(n: number): string {
-  return n.toFixed(2) + 'ms';
-}
+const formatMeasure = (measurement: Measurement) => (n: number) => {
+  return n.toFixed(2) + measurement.unit;
+};
 
 function negate(ci: ConfidenceInterval): ConfidenceInterval {
   return {

src/json-output.ts

@@ -83,7 +83,7 @@ export function jsonOutput(results: ResultStatsWithDifferences[]):
         high: result.stats.meanCI.high,
       },
       differences,
-      samples: result.result.millis,
+      samples: result.result.rawData,
     });
   }
   return {benchmarks};

src/runner.ts

@@ -154,7 +154,7 @@ export class Runner {
       if (primary === undefined) {
         specResults[newResult.measurementIndex] = newResult;
       } else {
-        primary.millis.push(...newResult.millis);
+        primary.rawData.push(...newResult.rawData);
       }
     }
   }
@@ -320,7 +320,7 @@ export class Runner {
           version: spec.url.kind === 'local' && spec.url.version !== undefined ?
               spec.url.version.label :
               '',
-          millis: [measurementResults[measurementIndex]],
+          rawData: [measurementResults[measurementIndex]],
           bytesSent: session ? session.bytesSent : 0,
           browser: spec.browser,
           userAgent: session ? session.userAgent : '',
@@ -332,7 +332,7 @@ export class Runner {
     for (const results of this.results.values()) {
       for (let r = 0; r < results.length; r++) {
         const result = results[r];
-        resultStats.push({result, stats: summaryStats(result.millis)});
+        resultStats.push({result, stats: summaryStats(result.rawData)});
       }
     }
     return computeDifferences(resultStats);

src/specs.ts

@@ -73,17 +73,20 @@ export async function specsFromOpts(opts: Opts): Promise<BenchmarkSpec[]> {
   if (opts.measure === 'callback') {
     measurement = {
       mode: 'callback',
+      unit: defaults.measurementUnit,
     };
   } else if (opts.measure === 'fcp') {
     measurement = {
       mode: 'performance',
       entryName: 'first-contentful-paint',
+      unit: defaults.measurementUnit,
     };
   } else if (opts.measure === 'global') {
     measurement = {
       mode: 'expression',
       expression:
           opts['measurement-expression'] || defaults.measurementExpression,
+      unit: defaults.measurementUnit,
     };
   } else if (opts.measure !== undefined) {
     throwUnreachable(

src/test/config_test.ts

@@ -60,9 +60,7 @@ suite('makeConfig', function() {
               width: 1024,
             },
           },
-          measurement: [{
-            mode: 'callback',
-          }],
+          measurement: [{mode: 'callback', unit: 'ms'}],
           name: 'random-global.html',
           url: {
             kind: 'local',
@@ -103,9 +101,7 @@ suite('makeConfig', function() {
               width: 1024,
             },
           },
-          measurement: [{
-            mode: 'callback',
-          }],
+          measurement: [{mode: 'callback', unit: 'ms'}],
           // TODO(aomarks) Why does this have a forward-slash?
           name: '/random-global.html',
           url: {
@@ -146,9 +142,7 @@ suite('makeConfig', function() {
               width: 1024,
             },
           },
-          measurement: [{
-            mode: 'callback',
-          }],
+          measurement: [{mode: 'callback', unit: 'ms'}],
           // TODO(aomarks) Why does this have a forward-slash?
           name: '/random-global.html',
           url: {
@@ -196,9 +190,7 @@ suite('makeConfig', function() {
               width: 1024,
             },
           },
-          measurement: [{
-            mode: 'callback',
-          }],
+          measurement: [{mode: 'callback', unit: 'ms'}],
           // TODO(aomarks) Why does this have a forward-slash?
           name: '/random-global.html',
           url: {

src/test/configfile_test.ts

@@ -111,6 +111,7 @@ suite('config', () => {
               {
                 mode: 'performance',
                 entryName: 'first-contentful-paint',
+                unit: 'ms'
               },
             ],
             url: {
@@ -130,6 +131,7 @@ suite('config', () => {
             measurement: [
               {
                 mode: 'callback',
+                unit: 'ms',
               },
             ],
             url: {
@@ -156,6 +158,7 @@ suite('config', () => {
               {
                 mode: 'performance',
                 entryName: 'first-contentful-paint',
+                unit: 'ms'
               },
             ],
             url: {
@@ -197,6 +200,7 @@ suite('config', () => {
               {
                 mode: 'performance',
                 entryName: 'first-contentful-paint',
+                unit: 'ms'
               },
             ],
             browser: defaultBrowser,
@@ -209,9 +213,7 @@ suite('config', () => {
               queryString: '?foo=bar',
             },
             measurement: [
-              {
-                mode: 'callback',
-              },
+              {mode: 'callback', unit: 'ms'},
             ],
             browser: defaultBrowser,
           },
@@ -261,9 +263,7 @@ suite('config', () => {
               queryString: '?foo=a',
             },
             measurement: [
-              {
-                mode: 'callback',
-              },
+              {mode: 'callback', unit: 'ms'},
             ],
             browser: defaultBrowser,
           },
@@ -275,10 +275,7 @@ suite('config', () => {
               queryString: '?foo=b',
             },
             measurement: [
-              {
-                mode: 'expression',
-                expression: 'window.foo',
-              },
+              {mode: 'expression', expression: 'window.foo', unit: 'ms'},
             ],
             browser: defaultBrowser,
           },
@@ -290,10 +287,7 @@ suite('config', () => {
               queryString: '?foo=c',
             },
             measurement: [
-              {
-                mode: 'performance',
-                entryName: 'foo-measure',
-              },
+              {mode: 'performance', entryName: 'foo-measure', unit: 'ms'},
             ],
             browser: defaultBrowser,
           },
@@ -339,6 +333,7 @@ suite('config', () => {
               {
                 mode: 'performance',
                 entryName: 'first-contentful-paint',
+                unit: 'ms',
               },
             ],
             browser: defaultBrowser,
@@ -350,6 +345,7 @@ suite('config', () => {
               {
                 mode: 'performance',
                 entryName: 'first-contentful-paint',
+                unit: 'ms',
               },
             ],
             browser: {
@@ -363,6 +359,7 @@ suite('config', () => {
             measurement: [
               {
                 mode: 'callback',
+                unit: 'ms',
               },
             ],
             browser: defaultBrowser,
@@ -373,6 +370,7 @@ suite('config', () => {
             measurement: [
               {
                 mode: 'callback',
+                unit: 'ms',
               },
             ],
             browser: {
@@ -417,6 +415,7 @@ suite('config', () => {
               {
                 mode: 'performance',
                 entryName: 'first-contentful-paint',
+                unit: 'ms',
               },
             ],
             browser: {