diff --git a/package.json b/package.json index f6175be36168..42b5d8f4ebad 100644 --- a/package.json +++ b/package.json @@ -303,6 +303,7 @@ "stylelint-config-standard": "^18.2.0", "testem": "^2.16.0", "through2": "^2.0.3", + "ttest": "^2.1.1", "vinyl-buffer": "^1.0.1", "vinyl-source-stream": "^2.0.0", "watchify": "^3.11.1" diff --git a/test/e2e/benchmark.js b/test/e2e/benchmark.js index caaba39b386a..d89cd5666a69 100644 --- a/test/e2e/benchmark.js +++ b/test/e2e/benchmark.js @@ -2,6 +2,7 @@ const path = require('path') const { promises: fs, constants: fsConstants } = require('fs') +const ttest = require('ttest') const { By, Key } = require('selenium-webdriver') const { withFixtures } = require('./helpers') const { PAGES } = require('./webdriver/driver') @@ -41,6 +42,9 @@ const standardDeviationResult = calculateResult((array) => { const squareDiffs = array.map(value => Math.pow(value - average, 2)) return Math.sqrt(calculateAverage(squareDiffs)) }) +// 95% margin of error calculated using Student's t-distrbution +const calculateMarginOfError = (array) => ttest(array).confidence()[1] - calculateAverage(array) +const marginOfErrorResult = calculateResult((array) => calculateMarginOfError(array)) async function profilePageLoad (pages, numSamples) { const results = {} @@ -68,6 +72,7 @@ async function profilePageLoad (pages, numSamples) { max: maxResult(result), average: averageResult(result), standardDeviation: standardDeviationResult(result), + marginOfError: marginOfErrorResult(result), } } return results diff --git a/yarn.lock b/yarn.lock index 0ec33b66f7a4..e4f3734f3b7b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6606,6 +6606,11 @@ ccount@^1.0.0, ccount@^1.0.3: resolved "https://registry.yarnpkg.com/ccount/-/ccount-1.0.4.tgz#9cf2de494ca84060a2a8d2854edd6dfb0445f386" integrity sha512-fpZ81yYfzentuieinmGnphk0pLkOTMm6MZdVqwd77ROvhko6iujLNGrHH5E7utq3ygWklwfmwuG+A7P+NpqT6w== +cephes@^1.1.2: + version "1.2.0" + resolved "https://registry.yarnpkg.com/cephes/-/cephes-1.2.0.tgz#4d246b622bc488ab8c982a1f702f0c9f49705d96" + integrity sha512-twuUuJRrIrsELHz6foJtZlqrz6FC36zoHZJvvThsrM1UWPKxyoilw1Rka6Hk0AmPFKHKUoGwGfAtvNZNtNZu0g== + chai-checkmark@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/chai-checkmark/-/chai-checkmark-1.0.1.tgz#9fbb3c9ad9101f097ef288328d30f4227d74fffb" @@ -8970,6 +8975,13 @@ disposables@^1.0.1: resolved "https://registry.yarnpkg.com/disposables/-/disposables-1.0.2.tgz#36c6a674475f55a2d6913567a601444e487b4b6e" integrity sha1-NsamdEdfVaLWkTVnpgFETkh7S24= +distributions@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/distributions/-/distributions-2.1.0.tgz#7005f01898937fbe6404abda20fa7f4aaaa058e0" + integrity sha512-4eov7WXRip5Tu0K7Y51W1oVt+DcvkoE0EzsvbLzgedC/n4PUpYXHqk6vo/OH1HteBHp5Vm6cmfnInf9NAXM4qQ== + dependencies: + cephes "^1.1.2" + dlv@^1.1.2, dlv@^1.1.3: version "1.1.3" resolved "https://registry.yarnpkg.com/dlv/-/dlv-1.1.3.tgz#5c198a8a11453596e751494d49874bc7732f2e79" @@ -26071,6 +26083,11 @@ sumchecker@^2.0.2: dependencies: debug "^2.2.0" +summary@0.3.x: + version "0.3.2" + resolved "https://registry.yarnpkg.com/summary/-/summary-0.3.2.tgz#3d7439b30dab3449dfd66840a822838d4a4a8561" + integrity sha1-PXQ5sw2rNEnf1mhAqCKDjUpKhWE= + superagent-proxy@^1.0.3, superagent-proxy@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/superagent-proxy/-/superagent-proxy-2.0.0.tgz#9f57515cd660e2e9ce55c0e6bd70f92eb07c3ee0" @@ -27056,6 +27073,14 @@ tsscmp@1.0.6: resolved "https://registry.yarnpkg.com/tsscmp/-/tsscmp-1.0.6.tgz#85b99583ac3589ec4bfef825b5000aa911d605eb" integrity sha512-LxhtAkPDTkVCMQjt2h6eBVY28KCjikZqZfMcC15YBeNjkgUpdCfBu5HoiOTDu86v6smE8yOjyEktJ8hlbANHQA== +ttest@^2.1.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/ttest/-/ttest-2.1.1.tgz#ee0c9118fff6580a5c8d6cfe69845219e075277f" + integrity sha512-YlRYswipptMVPLil4bsblnp2HXmcAR0McnfnuCM26EYrBZTCdUc/BVPuqZoaMb9h06smAPhe+s+wFMtYOULVGQ== + dependencies: + distributions "^2.1.0" + summary "0.3.x" + tty-browserify@0.0.0: version "0.0.0" resolved "https://registry.yarnpkg.com/tty-browserify/-/tty-browserify-0.0.0.tgz#a157ba402da24e9bf957f9aa69d524eed42901a6"