From 4242df8a5a63135b2fdaed848a043340c8d26b6d Mon Sep 17 00:00:00 2001 From: Paul Irish Date: Thu, 27 May 2021 16:26:05 -0700 Subject: [PATCH 1/3] core(scoring): update TBT score curve --- lighthouse-core/audits/metrics/total-blocking-time.js | 8 ++++---- .../test/audits/metrics/total-blocking-time-test.js | 2 +- lighthouse-core/test/results/sample_v2.json | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/lighthouse-core/audits/metrics/total-blocking-time.js b/lighthouse-core/audits/metrics/total-blocking-time.js index 43a2126a84cd..c2bc67b97603 100644 --- a/lighthouse-core/audits/metrics/total-blocking-time.js +++ b/lighthouse-core/audits/metrics/total-blocking-time.js @@ -40,11 +40,11 @@ class TotalBlockingTime extends Audit { // According to a cluster telemetry run over top 10k sites on mobile, 5th percentile was 0ms, // 25th percentile was 270ms and median was 895ms. These numbers include 404 pages. Picking // thresholds according to our 25/75-th rule will be quite harsh scoring (a single 350ms task) - // after FCP will yield a score of .5. The following coefficients are semi-arbitrarily picked - // to give 600ms jank a score of .5 and 100ms jank a score of .999. We can tweak these numbers - // in the future. See https://www.desmos.com/calculator/bbsv8fedg5 + // after FCP will yield a score of .5. The following coefficients are semi-arbitrarily chosen, + // but start to approach the 25/75 numbers. + // See https://www.desmos.com/calculator/pwcgna1cvf go/lh8-tbt-curves scoring: { - p10: 287, + p10: 200, median: 600, }, }, diff --git a/lighthouse-core/test/audits/metrics/total-blocking-time-test.js b/lighthouse-core/test/audits/metrics/total-blocking-time-test.js index ab351c410f9e..d051edabd392 100644 --- a/lighthouse-core/test/audits/metrics/total-blocking-time-test.js +++ b/lighthouse-core/test/audits/metrics/total-blocking-time-test.js @@ -58,7 +58,7 @@ describe('Performance: total-blocking-time audit', () => { const outputMobile = await TBTAudit.audit(artifactsMobile, contextMobile); expect(outputMobile.numericValue).toBeCloseTo(333, 1); - expect(outputMobile.score).toBe(0.85); + expect(outputMobile.score).toBe(0.75); expect(outputMobile.displayValue).toBeDisplayString('330\xa0ms'); const artifactsDesktop = generateArtifacts({trace: lcpTrace, diff --git a/lighthouse-core/test/results/sample_v2.json b/lighthouse-core/test/results/sample_v2.json index cbac6f1ede1e..83e053ef15c0 100644 --- a/lighthouse-core/test/results/sample_v2.json +++ b/lighthouse-core/test/results/sample_v2.json @@ -185,7 +185,7 @@ "id": "total-blocking-time", "title": "Total Blocking Time", "description": "Sum of all time periods between FCP and Time to Interactive, when task length exceeded 50ms, expressed in milliseconds. [Learn more](https://web.dev/lighthouse-total-blocking-time/).", - "score": 1, + "score": 0.97, "scoreDisplayMode": "numeric", "numericValue": 116.79800000000023, "numericUnit": "millisecond", @@ -5276,7 +5276,7 @@ } ], "id": "performance", - "score": 0.6 + "score": 0.59 }, "accessibility": { "title": "Accessibility", From 5ae62e0f6731ec30ee8264073bc4ba74ce1524d1 Mon Sep 17 00:00:00 2001 From: Paul Irish Date: Thu, 27 May 2021 16:44:56 -0700 Subject: [PATCH 2/3] update comment --- .../audits/metrics/total-blocking-time.js | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/lighthouse-core/audits/metrics/total-blocking-time.js b/lighthouse-core/audits/metrics/total-blocking-time.js index c2bc67b97603..9806feee0254 100644 --- a/lighthouse-core/audits/metrics/total-blocking-time.js +++ b/lighthouse-core/audits/metrics/total-blocking-time.js @@ -37,11 +37,18 @@ class TotalBlockingTime extends Audit { static get defaultOptions() { return { mobile: { - // According to a cluster telemetry run over top 10k sites on mobile, 5th percentile was 0ms, - // 25th percentile was 270ms and median was 895ms. These numbers include 404 pages. Picking - // thresholds according to our 25/75-th rule will be quite harsh scoring (a single 350ms task) - // after FCP will yield a score of .5. The following coefficients are semi-arbitrarily chosen, - // but start to approach the 25/75 numbers. + // If determined from HTTP Archive data… + // SELECT + // APPROX_QUANTILES(tbt_value, 100)[OFFSET(8)] AS p08_tbt, + // APPROX_QUANTILES(tbt_value, 100)[OFFSET(25)] AS p25_tbt + // FROM ( + // SELECT CAST(JSON_EXTRACT_SCALAR(report, '$.audits.total-blocking-time.numericValue') AS FLOAT64) AS tbt_value + // FROM `httparchive.lighthouse.2021_05_01_mobile` + // WHERE report is not NULL + // ) + // …we'd use control points of 19 and 189, which leads to surprisingly harsh scoring. + // + // The following coefficients are semi-arbitrarily chosen, but start to approach the "correct" ones: // See https://www.desmos.com/calculator/pwcgna1cvf go/lh8-tbt-curves scoring: { p10: 200, From 949bcb91ef3463ab2172a352f8cc7edc17df1ec8 Mon Sep 17 00:00:00 2001 From: Paul Irish Date: Thu, 27 May 2021 16:53:32 -0700 Subject: [PATCH 3/3] test --- lighthouse-core/test/report/report-generator-test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lighthouse-core/test/report/report-generator-test.js b/lighthouse-core/test/report/report-generator-test.js index 90e4e715779e..98cdfc21d445 100644 --- a/lighthouse-core/test/report/report-generator-test.js +++ b/lighthouse-core/test/report/report-generator-test.js @@ -104,7 +104,7 @@ describe('ReportGenerator', () => { expect(lines.length).toBeGreaterThan(100); expect(lines.slice(0, 3).join('\n')).toMatchInlineSnapshot(` "requestedUrl,finalUrl,category,name,title,type,score - \\"http://localhost:10200/dobetterweb/dbw_tester.html\\",\\"http://localhost:10200/dobetterweb/dbw_tester.html\\",\\"Performance\\",\\"performance-score\\",\\"Overall Performance Category Score\\",\\"numeric\\",\\"0.6\\" + \\"http://localhost:10200/dobetterweb/dbw_tester.html\\",\\"http://localhost:10200/dobetterweb/dbw_tester.html\\",\\"Performance\\",\\"performance-score\\",\\"Overall Performance Category Score\\",\\"numeric\\",\\"0.59\\" \\"http://localhost:10200/dobetterweb/dbw_tester.html\\",\\"http://localhost:10200/dobetterweb/dbw_tester.html\\",\\"Performance\\",\\"first-contentful-paint\\",\\"First Contentful Paint\\",\\"numeric\\",\\"0.24\\" " `);