fix: implement less flaky network module

src/bidiMapper/CommandProcessor.ts

@@ -91,7 +91,11 @@ export class CommandProcessor extends EventEmitter<CommandProcessorEventsMap> {
     this.#parser = parser;
     this.#logger = logger;
 
-    const networkStorage = new NetworkStorage(eventManager, browserCdpClient);
+    const networkStorage = new NetworkStorage(
+      eventManager,
+      browserCdpClient,
+      logger
+    );
     const preloadScriptStorage = new PreloadScriptStorage();
 
     // keep-sorted start block=yes

src/bidiMapper/domains/network/NetworkModuleMocks.spec.ts

@@ -0,0 +1,347 @@
+/*
+ * Copyright 2024 Google LLC.
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+import type {Protocol} from 'devtools-protocol';
+
+import {
+  CloseError,
+  type CdpClient,
+  type CdpEvents,
+} from '../../../cdp/CdpClient.js';
+import {EventEmitter} from '../../../utils/EventEmitter.js';
+
+// TODO: Extend with Redirect
+export class MockCdpNetworkEvents {
+  static defaultFrameId = '099A5216AF03AAFEC988F214B024DF08';
+  static defaultUrl = 'http://www.google.com';
+
+  cdpClient: CdpClient;
+
+  url: string;
+  requestId: string;
+  fetchId: string;
+  private loaderId: string;
+  private frameId: string;
+  private type: Protocol.Network.ResourceType;
+
+  constructor(
+    cdpClient: CdpClient,
+    {
+      requestId,
+      fetchId,
+      loaderId,
+      url,
+      frameId,
+      type,
+    }: Partial<{
+      requestId: string;
+      fetchId: string;
+      loaderId: string;
+      url: string;
+      frameId: string;
+      type: Protocol.Network.ResourceType;
+    }> = {}
+  ) {
+    this.cdpClient = cdpClient;
+
+    this.requestId = requestId ?? '7151.2';
+    this.fetchId = fetchId ?? 'interception-job-1.0';
+    this.loaderId = loaderId ?? '7760711DEFCFA23132D98ABA6B4E175C';
+    this.url = url ?? MockCdpNetworkEvents.defaultUrl;
+    this.frameId = frameId ?? MockCdpNetworkEvents.defaultFrameId;
+    this.type = type ?? 'XHR';
+  }
+
+  requestWillBeSent() {
+    this.cdpClient.emit('Network.requestWillBeSent', {
+      requestId: this.requestId,
+      loaderId: this.loaderId,
+      documentURL: this.url,
+      request: {
+        url: this.url,
+        method: 'GET',
+        headers: {
+          'sec-ch-ua': '"Not-A.Brand";v="99", "Chromium";v="124"',
+          Referer: 'http://localhost:49243/',
+          'sec-ch-ua-mobile': '?0',
+          'User-Agent':
+            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
+          'sec-ch-ua-platform': '"macOS"',
+        },
+        mixedContentType: 'none',
+        initialPriority: 'High',
+        referrerPolicy: 'strict-origin-when-cross-origin',
+        isSameSite: true,
+      },
+      timestamp: 2111.55635,
+      wallTime: 1637315638.473634,
+      initiator: {
+        type: 'script',
+        stack: {
+          callFrames: [
+            {
+              functionName: '',
+              scriptId: '5',
+              url: '',
+              lineNumber: 0,
+              columnNumber: 0,
+            },
+          ],
+        },
+      },
+      redirectHasExtraInfo: false,
+      type: this.type,
+      frameId: this.frameId,
+      hasUserGesture: false,
+    });
+  }
+
+  requestWillBeSentExtraInfo() {
+    this.cdpClient.emit('Network.requestWillBeSentExtraInfo', {
+      requestId: this.requestId,
+      associatedCookies: [],
+      headers: {
+        Accept: '*/*',
+        'Accept-Encoding': 'gzip, deflate, br, zstd',
+        'Accept-Language': 'en-US,en;q=0.9',
+        Connection: 'keep-alive',
+        Host: 'localhost:49243',
+        Referer: 'http://localhost:49243/',
+        'Sec-Fetch-Dest': 'empty',
+        'Sec-Fetch-Mode': 'cors',
+        'Sec-Fetch-Site': 'same-origin',
+        'User-Agent':
+          'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
+        'sec-ch-ua': '"Not-A.Brand";v="99", "Chromium";v="124"',
+        'sec-ch-ua-mobile': '?0',
+        'sec-ch-ua-platform': '"macOS"',
+      },
+      connectTiming: {requestTime: 2111.557593},
+      clientSecurityState: {
+        initiatorIsSecureContext: true,
+        initiatorIPAddressSpace: 'Local',
+        privateNetworkRequestPolicy: 'PreflightWarn',
+      },
+      siteHasCookieInOtherPartition: false,
+    });
+  }
+
+  requestServedFromCache() {
+    this.cdpClient.emit('Network.requestServedFromCache', {
+      requestId: this.requestId,
+    });
+  }
+
+  responseReceivedExtraInfo() {
+    this.cdpClient.emit('Network.responseReceivedExtraInfo', {
+      requestId: this.requestId,
+      blockedCookies: [],
+      headers: {
+        'Cache-Control': 'no-cache, no-store',
+        'Content-Type': 'text/html; charset=utf-8',
+        Date: 'Fri, 19 Nov 2021 09:53:58 GMT',
+        Connection: 'keep-alive',
+        'Keep-Alive': 'timeout=5',
+        'Content-Length': '0',
+      },
+      resourceIPAddressSpace: 'Local',
+      statusCode: 200,
+      headersText:
+        'HTTP/1.1 200 OK\r\nCache-Control: no-cache, no-store\r\nContent-Type: text/html; charset=utf-8\r\nDate: Fri, 19 Nov 2021 09:53:58 GMT\r\nConnection: keep-alive\r\nKeep-Alive: timeout=5\r\nContent-Length: 0\r\n\r\n',
+    });
+  }
+
+  responseReceived(hasExtraInfo = false) {
+    this.cdpClient.emit('Network.responseReceived', {
+      requestId: this.requestId,
+      loaderId: this.loaderId,
+      timestamp: 2111.563565,
+      type: this.type,
+      response: {
+        url: this.url,
+        status: 200,
+        statusText: 'OK',
+        headers: {
+          'Cache-Control': 'no-cache, no-store',
+          'Content-Type': 'text/html; charset=utf-8',
+          Date: 'Fri, 19 Nov 2021 09:53:58 GMT',
+          Connection: 'keep-alive',
+          'Keep-Alive': 'timeout=5',
+          'Content-Length': '0',
+        },
+        // TODO: set to a correct one
+        charset: '',
+        mimeType: 'text/html',
+        connectionReused: true,
+        connectionId: 322,
+        remoteIPAddress: '[::1]',
+        remotePort: 8907,
+        fromDiskCache: false,
+        fromServiceWorker: false,
+        fromPrefetchCache: false,
+        encodedDataLength: 197,
+        timing: {
+          receiveHeadersStart: 0,
+          requestTime: 2111.561759,
+          proxyStart: -1,
+          proxyEnd: -1,
+          dnsStart: -1,
+          dnsEnd: -1,
+          connectStart: -1,
+          connectEnd: -1,
+          sslStart: -1,
+          sslEnd: -1,
+          workerStart: -1,
+          workerReady: -1,
+          workerFetchStart: -1,
+          workerRespondWithSettled: -1,
+          sendStart: 0.148,
+          sendEnd: 0.19,
+          pushStart: 0,
+          pushEnd: 0,
+          receiveHeadersEnd: 0.925,
+        },
+        responseTime: 1.637315638479928e12,
+        protocol: 'http/1.1',
+        securityState: 'secure',
+      },
+      hasExtraInfo,
+      frameId: this.frameId,
+    });
+  }
+
+  requestPaused() {
+    this.cdpClient.emit('Fetch.requestPaused', {
+      requestId: this.fetchId,
+      request: {
+        url: this.url,
+        method: 'GET',
+        headers: {
+          Accept: '*/*',
+          Referer: 'http://localhost:49243/',
+          'User-Agent':
+            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
+          'sec-ch-ua': '"Not-A.Brand";v="99", "Chromium";v="124"',
+          'sec-ch-ua-mobile': '?0',
+          'sec-ch-ua-platform': '"macOS"',
+        },
+        initialPriority: 'High',
+        referrerPolicy: 'strict-origin-when-cross-origin',
+      },
+      frameId: this.frameId,
+      resourceType: this.type,
+      networkId: this.requestId,
+    });
+  }
+
+  responsePaused() {
+    this.cdpClient.emit('Fetch.requestPaused', {
+      requestId: this.fetchId,
+      request: {
+        url: this.url,
+        method: 'GET',
+        headers: {},
+        initialPriority: 'VeryHigh',
+        referrerPolicy: 'strict-origin-when-cross-origin',
+      },
+      // TODO: fill for response correctly
+      responseStatusCode: 200,
+      responseStatusText: '',
+      responseHeaders: [],
+      frameId: this.frameId,
+      resourceType: this.type,
+      networkId: this.requestId,
+    });
+  }
+
+  authRequired() {
+    this.cdpClient.emit('Fetch.authRequired', {
+      requestId: this.fetchId,
+      request: {
+        url: this.url,
+        method: 'GET',
+        headers: {
+          Accept: '*/*',
+          Referer: 'http://localhost:49243/',
+          'User-Agent':
+            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
+          'sec-ch-ua': '"Not-A.Brand";v="99", "Chromium";v="124"',
+          'sec-ch-ua-mobile': '?0',
+          'sec-ch-ua-platform': '"macOS"',
+        },
+        initialPriority: 'High',
+        referrerPolicy: 'strict-origin-when-cross-origin',
+      },
+      frameId: this.frameId,
+      resourceType: this.type,
+      authChallenge: {
+        source: 'Server',
+        origin: new URL(this.url).origin,
+        scheme: 'basic',
+        realm: 'Access to staging site',
+      },
+    });
+  }
+
+  loadingFailed() {
+    this.cdpClient.emit('Network.loadingFailed', {
+      requestId: this.requestId,
+      timestamp: 279179.745291,
+      type: 'Fetch',
+      errorText: 'net::ERR_NAME_NOT_RESOLVED',
+      canceled: false,
+    });
+  }
+}
+
+export class MockCdpClient extends EventEmitter<CdpEvents> {
+  #logger: (...args: any[]) => void;
+
+  sessionId = '23E8E97ED43449740710991CD32AEFA3';
+
+  constructor(logger: (...args: any[]) => void) {
+    super();
+    this.#logger = logger;
+  }
+
+  sendCommand(...args: any[]) {
+    this.#logger(...args);
+
+    return new Promise((resolve) => {
+      setTimeout(resolve, 100);
+    });
+  }
+
+  isCloseError(error: unknown) {
+    return error instanceof CloseError;
+  }
+}
+
+export class MockCdpTarget {
+  cdpClient: CdpClient;
+  #logger: (...args: any[]) => void;
+
+  constructor(logger: (...args: any[]) => void) {
+    this.#logger = logger;
+    this.cdpClient = new MockCdpClient(logger) as CdpClient;
+  }
+
+  enableFetchIfNeeded() {
+    this.#logger('Fetch.enabled called');
+    return Promise.resolve();
+  }
+}