From ff505ba252687732dcdea0ef00bc8d2e3ef14105 Mon Sep 17 00:00:00 2001 From: Maksim Sadym Date: Thu, 19 Dec 2024 19:14:57 +0100 Subject: [PATCH 1/5] feat: provide proper network timings --- .../modules/network/NetworkRequest.ts | 74 ++++++++++++++----- .../modules/network/NetworkUtils.ts | 9 ++- tests/network/test_network.py | 13 +++- 3 files changed, 72 insertions(+), 24 deletions(-) diff --git a/src/bidiMapper/modules/network/NetworkRequest.ts b/src/bidiMapper/modules/network/NetworkRequest.ts index da40b1766..005754c49 100644 --- a/src/bidiMapper/modules/network/NetworkRequest.ts +++ b/src/bidiMapper/modules/network/NetworkRequest.ts @@ -23,27 +23,27 @@ import type {Protocol} from 'devtools-protocol'; import { - Network, type BrowsingContext, ChromiumBidi, + Network, type NetworkEvent, } from '../../../protocol/protocol.js'; import {assert} from '../../../utils/assert.js'; import {Deferred} from '../../../utils/Deferred.js'; -import {LogType, type LoggerFn} from '../../../utils/log.js'; +import {type LoggerFn, LogType} from '../../../utils/log.js'; import type {CdpTarget} from '../cdp/CdpTarget.js'; import type {EventManager} from '../session/EventManager.js'; import type {NetworkStorage} from './NetworkStorage.js'; import { - computeHeadersSize, + bidiBodySizeFromCdpPostDataEntries, bidiNetworkHeadersFromCdpNetworkHeaders, - cdpToBiDiCookie, - cdpFetchHeadersFromBidiNetworkHeaders, cdpAuthChallengeResponseFromBidiAuthContinueWithAuthAction, - bidiBodySizeFromCdpPostDataEntries, - networkHeaderFromCookieHeaders, + cdpFetchHeadersFromBidiNetworkHeaders, + cdpToBiDiCookie, + computeHeadersSize, getTiming, + networkHeaderFromCookieHeaders, } from './NetworkUtils.js'; const REALM_REGEX = /(?<=realm=").*(?=")/; @@ -304,26 +304,66 @@ export class NetworkRequest { } get #timings(): Network.FetchTimingInfo { + // The timing in the CDP events are provided relative to the event's baseline. + // However, the baseline can be different for different events, and the events have to + // be normalized throughout resource events. Normalize events timestamps by the + // request. + // TODO: Verify this is correct. + const responseTimeOffset = getTiming( + getTiming(this.#response.info?.timing?.requestTime) - + getTiming(this.#request.info?.timestamp), + ); + return { // TODO: Verify this is correct - timeOrigin: getTiming(this.#response.info?.timing?.requestTime), - requestTime: getTiming(this.#response.info?.timing?.requestTime), + timeOrigin: Math.round(getTiming(this.#request.info?.wallTime) * 1000), + // Timing baseline. + // TODO: Verify this is correct. + requestTime: 0, + // TODO: set if redirect detected. redirectStart: 0, + // TODO: set if redirect detected. redirectEnd: 0, // TODO: Verify this is correct // https://source.chromium.org/chromium/chromium/src/+/main:net/base/load_timing_info.h;l=145 - fetchStart: getTiming(this.#response.info?.timing?.requestTime), - dnsStart: getTiming(this.#response.info?.timing?.dnsStart), - dnsEnd: getTiming(this.#response.info?.timing?.dnsEnd), - connectStart: getTiming(this.#response.info?.timing?.connectStart), - connectEnd: getTiming(this.#response.info?.timing?.connectEnd), - tlsStart: getTiming(this.#response.info?.timing?.sslStart), - requestStart: getTiming(this.#response.info?.timing?.sendStart), + fetchStart: getTiming( + this.#response.info?.timing?.workerFetchStart, + responseTimeOffset, + ), + // fetchStart: 0, + dnsStart: getTiming( + this.#response.info?.timing?.dnsStart, + responseTimeOffset, + ), + dnsEnd: getTiming( + this.#response.info?.timing?.dnsEnd, + responseTimeOffset, + ), + connectStart: getTiming( + this.#response.info?.timing?.connectStart, + responseTimeOffset, + ), + connectEnd: getTiming( + this.#response.info?.timing?.connectEnd, + responseTimeOffset, + ), + tlsStart: getTiming( + this.#response.info?.timing?.sslStart, + responseTimeOffset, + ), + requestStart: getTiming( + this.#response.info?.timing?.sendStart, + responseTimeOffset, + ), // https://source.chromium.org/chromium/chromium/src/+/main:net/base/load_timing_info.h;l=196 responseStart: getTiming( this.#response.info?.timing?.receiveHeadersStart, + responseTimeOffset, + ), + responseEnd: getTiming( + this.#response.info?.timing?.receiveHeadersEnd, + responseTimeOffset, ), - responseEnd: getTiming(this.#response.info?.timing?.receiveHeadersEnd), }; } diff --git a/src/bidiMapper/modules/network/NetworkUtils.ts b/src/bidiMapper/modules/network/NetworkUtils.ts index fd8843d42..a668d84a6 100644 --- a/src/bidiMapper/modules/network/NetworkUtils.ts +++ b/src/bidiMapper/modules/network/NetworkUtils.ts @@ -366,13 +366,16 @@ export function bidiBodySizeFromCdpPostDataEntries( return size; } -export function getTiming(timing: number | undefined): number { +export function getTiming( + timing: number | undefined, + offset: number = 0, +): number { if (!timing) { return 0; } - if (timing < 0) { + if (timing <= 0 || timing + offset <= 0) { return 0; } - return timing; + return timing + offset; } diff --git a/tests/network/test_network.py b/tests/network/test_network.py index c66f8a819..ef14ad59a 100644 --- a/tests/network/test_network.py +++ b/tests/network/test_network.py @@ -422,10 +422,15 @@ async def test_network_before_request_sent_event_with_data_url_emitted( @pytest.mark.asyncio +@pytest.mark.parametrize('capabilities', [{ + 'acceptInsecureCerts': True +}], + indirect=True) async def test_network_sends_only_included_cookies(websocket, context_id, - url_base): + url_example_another_origin, + url_bad_ssl): - await goto_url(websocket, context_id, "https://example.com") + await goto_url(websocket, context_id, url_bad_ssl) await execute_command( websocket, { @@ -446,7 +451,7 @@ async def test_network_sends_only_included_cookies(websocket, context_id, websocket, { "method": "browsingContext.navigate", "params": { - "url": url_base, + "url": url_example_another_origin, "wait": "complete", "context": context_id } @@ -463,7 +468,7 @@ async def test_network_sends_only_included_cookies(websocket, context_id, "redirectCount": 0, "request": { "request": ANY_STR, - "url": url_base, + "url": url_example_another_origin, "method": "GET", "headers": ANY_LIST, "cookies": [], From f03e4247a39b73de50b29684aaf45db501cf2639 Mon Sep 17 00:00:00 2001 From: Maksim Sadym Date: Fri, 20 Dec 2024 10:27:50 +0100 Subject: [PATCH 2/5] AppArmor --- .github/workflows/e2e.yml | 4 ++++ .github/workflows/wpt.yml | 2 ++ 2 files changed, 6 insertions(+) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index d2c1db1f8..5588d0513 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -63,6 +63,10 @@ jobs: with: node-version-file: '.nvmrc' cache: npm + - name: Disable AppArmor + if: ${{ matrix.os == 'ubuntu-latest' }} + # https://chromium.googlesource.com/chromium/src/+/main/docs/security/apparmor-userns-restrictions.md + run: echo 0 | sudo tee /proc/sys/kernel/apparmor_restrict_unprivileged_userns - uses: google/wireit@83d7f8bed70b7bcfc40f4b9f54f4b7485753991b # setup-github-actions-caching/v2.0.1 - name: Install and build npm dependencies run: npm ci diff --git a/.github/workflows/wpt.yml b/.github/workflows/wpt.yml index 03c0d898a..d3583b1df 100644 --- a/.github/workflows/wpt.yml +++ b/.github/workflows/wpt.yml @@ -82,6 +82,8 @@ jobs: with: node-version-file: '.nvmrc' cache: npm + - name: Disable AppArmor + run: echo 0 | sudo tee /proc/sys/kernel/apparmor_restrict_unprivileged_userns - uses: google/wireit@83d7f8bed70b7bcfc40f4b9f54f4b7485753991b # setup-github-actions-caching/v2.0.1 - name: Install and build npm dependencies run: npm ci From 29cc13de721d4d46e7478c1828e49657a15d5497 Mon Sep 17 00:00:00 2001 From: Maksim Sadym Date: Fri, 20 Dec 2024 10:29:24 +0100 Subject: [PATCH 3/5] address eslint warnings --- src/bidiMapper/modules/context/NavigationTracker.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bidiMapper/modules/context/NavigationTracker.spec.ts b/src/bidiMapper/modules/context/NavigationTracker.spec.ts index 86b6f217d..54420e6a4 100644 --- a/src/bidiMapper/modules/context/NavigationTracker.spec.ts +++ b/src/bidiMapper/modules/context/NavigationTracker.spec.ts @@ -318,7 +318,7 @@ describe('NavigationTracker', () => { assertNoNavigationEvents(); }); - it('canceled by script-initiated navigation', async () => { + it('canceled by script-initiated navigation', () => { navigationTracker.frameRequestedNavigation(SOME_URL); assertNoNavigationEvents(); @@ -350,7 +350,7 @@ describe('NavigationTracker', () => { assert.equal(navigationTracker.url, INITIAL_URL); }); - it('canceled by command navigation', async () => { + it('canceled by command navigation', () => { navigationTracker.frameRequestedNavigation(SOME_URL); navigationTracker.frameStartedNavigating(ANOTHER_URL, LOADER_ID); From 659367f5f406928bf3e31722cd039fb2a855fdb6 Mon Sep 17 00:00:00 2001 From: browser-automation-bot <133232582+browser-automation-bot@users.noreply.github.com> Date: Fri, 20 Dec 2024 10:44:59 +0100 Subject: [PATCH 4/5] test: update the expectations for PR (#2920) Automatically generated by https://github.com/GoogleChromeLabs/chromium-bidi/blob/main/.github/workflows/wpt.yml --- .../before_request_sent/before_request_sent.py.ini | 3 --- .../network/response_completed/response_completed.py.ini | 9 --------- .../network/response_started/response_started.py.ini | 9 --------- .../before_request_sent/before_request_sent.py.ini | 3 --- .../network/response_completed/response_completed.py.ini | 9 --------- .../network/response_started/response_started.py.ini | 9 --------- .../before_request_sent/before_request_sent.py.ini | 3 --- .../network/response_completed/response_completed.py.ini | 9 --------- .../network/response_started/response_started.py.ini | 9 --------- 9 files changed, 63 deletions(-) diff --git a/wpt-metadata/chromedriver/headful/webdriver/tests/bidi/network/before_request_sent/before_request_sent.py.ini b/wpt-metadata/chromedriver/headful/webdriver/tests/bidi/network/before_request_sent/before_request_sent.py.ini index 0fb8bdb39..64ae78858 100644 --- a/wpt-metadata/chromedriver/headful/webdriver/tests/bidi/network/before_request_sent/before_request_sent.py.ini +++ b/wpt-metadata/chromedriver/headful/webdriver/tests/bidi/network/before_request_sent/before_request_sent.py.ini @@ -1,6 +1,3 @@ [before_request_sent.py] - [test_serviceworker_request] - expected: FAIL - [test_destination_initiator] expected: FAIL diff --git a/wpt-metadata/chromedriver/headful/webdriver/tests/bidi/network/response_completed/response_completed.py.ini b/wpt-metadata/chromedriver/headful/webdriver/tests/bidi/network/response_completed/response_completed.py.ini index 8537d8b99..71b5912ea 100644 --- a/wpt-metadata/chromedriver/headful/webdriver/tests/bidi/network/response_completed/response_completed.py.ini +++ b/wpt-metadata/chromedriver/headful/webdriver/tests/bidi/network/response_completed/response_completed.py.ini @@ -14,14 +14,5 @@ [test_response_status[407-Proxy Authentication Required\]] expected: FAIL - [test_request_timing_info] - expected: FAIL - - [test_serviceworker_request] - expected: FAIL - - [test_url_with_fragment] - expected: FAIL - [test_destination_initiator] expected: FAIL diff --git a/wpt-metadata/chromedriver/headful/webdriver/tests/bidi/network/response_started/response_started.py.ini b/wpt-metadata/chromedriver/headful/webdriver/tests/bidi/network/response_started/response_started.py.ini index 2b61e599e..1a3328b68 100644 --- a/wpt-metadata/chromedriver/headful/webdriver/tests/bidi/network/response_started/response_started.py.ini +++ b/wpt-metadata/chromedriver/headful/webdriver/tests/bidi/network/response_started/response_started.py.ini @@ -8,14 +8,5 @@ [test_www_authenticate] expected: FAIL - [test_request_timing_info] - expected: FAIL - - [test_serviceworker_request] - expected: FAIL - - [test_url_with_fragment] - expected: FAIL - [test_destination_initiator] expected: FAIL diff --git a/wpt-metadata/chromedriver/headless/webdriver/tests/bidi/network/before_request_sent/before_request_sent.py.ini b/wpt-metadata/chromedriver/headless/webdriver/tests/bidi/network/before_request_sent/before_request_sent.py.ini index 0fb8bdb39..64ae78858 100644 --- a/wpt-metadata/chromedriver/headless/webdriver/tests/bidi/network/before_request_sent/before_request_sent.py.ini +++ b/wpt-metadata/chromedriver/headless/webdriver/tests/bidi/network/before_request_sent/before_request_sent.py.ini @@ -1,6 +1,3 @@ [before_request_sent.py] - [test_serviceworker_request] - expected: FAIL - [test_destination_initiator] expected: FAIL diff --git a/wpt-metadata/chromedriver/headless/webdriver/tests/bidi/network/response_completed/response_completed.py.ini b/wpt-metadata/chromedriver/headless/webdriver/tests/bidi/network/response_completed/response_completed.py.ini index 8537d8b99..71b5912ea 100644 --- a/wpt-metadata/chromedriver/headless/webdriver/tests/bidi/network/response_completed/response_completed.py.ini +++ b/wpt-metadata/chromedriver/headless/webdriver/tests/bidi/network/response_completed/response_completed.py.ini @@ -14,14 +14,5 @@ [test_response_status[407-Proxy Authentication Required\]] expected: FAIL - [test_request_timing_info] - expected: FAIL - - [test_serviceworker_request] - expected: FAIL - - [test_url_with_fragment] - expected: FAIL - [test_destination_initiator] expected: FAIL diff --git a/wpt-metadata/chromedriver/headless/webdriver/tests/bidi/network/response_started/response_started.py.ini b/wpt-metadata/chromedriver/headless/webdriver/tests/bidi/network/response_started/response_started.py.ini index 2b61e599e..1a3328b68 100644 --- a/wpt-metadata/chromedriver/headless/webdriver/tests/bidi/network/response_started/response_started.py.ini +++ b/wpt-metadata/chromedriver/headless/webdriver/tests/bidi/network/response_started/response_started.py.ini @@ -8,14 +8,5 @@ [test_www_authenticate] expected: FAIL - [test_request_timing_info] - expected: FAIL - - [test_serviceworker_request] - expected: FAIL - - [test_url_with_fragment] - expected: FAIL - [test_destination_initiator] expected: FAIL diff --git a/wpt-metadata/mapper/headless/webdriver/tests/bidi/network/before_request_sent/before_request_sent.py.ini b/wpt-metadata/mapper/headless/webdriver/tests/bidi/network/before_request_sent/before_request_sent.py.ini index 0fb8bdb39..64ae78858 100644 --- a/wpt-metadata/mapper/headless/webdriver/tests/bidi/network/before_request_sent/before_request_sent.py.ini +++ b/wpt-metadata/mapper/headless/webdriver/tests/bidi/network/before_request_sent/before_request_sent.py.ini @@ -1,6 +1,3 @@ [before_request_sent.py] - [test_serviceworker_request] - expected: FAIL - [test_destination_initiator] expected: FAIL diff --git a/wpt-metadata/mapper/headless/webdriver/tests/bidi/network/response_completed/response_completed.py.ini b/wpt-metadata/mapper/headless/webdriver/tests/bidi/network/response_completed/response_completed.py.ini index 8537d8b99..71b5912ea 100644 --- a/wpt-metadata/mapper/headless/webdriver/tests/bidi/network/response_completed/response_completed.py.ini +++ b/wpt-metadata/mapper/headless/webdriver/tests/bidi/network/response_completed/response_completed.py.ini @@ -14,14 +14,5 @@ [test_response_status[407-Proxy Authentication Required\]] expected: FAIL - [test_request_timing_info] - expected: FAIL - - [test_serviceworker_request] - expected: FAIL - - [test_url_with_fragment] - expected: FAIL - [test_destination_initiator] expected: FAIL diff --git a/wpt-metadata/mapper/headless/webdriver/tests/bidi/network/response_started/response_started.py.ini b/wpt-metadata/mapper/headless/webdriver/tests/bidi/network/response_started/response_started.py.ini index 144a85b24..ac1f16956 100644 --- a/wpt-metadata/mapper/headless/webdriver/tests/bidi/network/response_started/response_started.py.ini +++ b/wpt-metadata/mapper/headless/webdriver/tests/bidi/network/response_started/response_started.py.ini @@ -5,14 +5,5 @@ [test_response_status[407-Proxy Authentication Required\]] expected: FAIL - [test_request_timing_info] - expected: FAIL - - [test_serviceworker_request] - expected: FAIL - - [test_url_with_fragment] - expected: FAIL - [test_destination_initiator] expected: FAIL From b1f07890e80bf486f684b0bb6b68066f5559578d Mon Sep 17 00:00:00 2001 From: Maksim Sadym Date: Fri, 20 Dec 2024 10:47:18 +0100 Subject: [PATCH 5/5] AppArmor --- .github/workflows/selenium.yml | 3 +++ .github/workflows/webdriverio.yml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/.github/workflows/selenium.yml b/.github/workflows/selenium.yml index fc7eb0577..7efa717bd 100644 --- a/.github/workflows/selenium.yml +++ b/.github/workflows/selenium.yml @@ -30,6 +30,9 @@ jobs: with: node-version-file: '.nvmrc' cache: npm + - name: Disable AppArmor + # https://chromium.googlesource.com/chromium/src/+/main/docs/security/apparmor-userns-restrictions.md + run: echo 0 | sudo tee /proc/sys/kernel/apparmor_restrict_unprivileged_userns - uses: google/wireit@83d7f8bed70b7bcfc40f4b9f54f4b7485753991b # setup-github-actions-caching/v2.0.1 - name: Install and build npm dependencies run: npm ci diff --git a/.github/workflows/webdriverio.yml b/.github/workflows/webdriverio.yml index c9381147c..dfa8254dc 100644 --- a/.github/workflows/webdriverio.yml +++ b/.github/workflows/webdriverio.yml @@ -30,6 +30,9 @@ jobs: with: node-version-file: '.nvmrc' cache: npm + - name: Disable AppArmor + # https://chromium.googlesource.com/chromium/src/+/main/docs/security/apparmor-userns-restrictions.md + run: echo 0 | sudo tee /proc/sys/kernel/apparmor_restrict_unprivileged_userns - uses: google/wireit@83d7f8bed70b7bcfc40f4b9f54f4b7485753991b # setup-github-actions-caching/v2.0.1 - name: Install and build npm dependencies run: npm ci