feat: add support for contexts in addInterception (#1945)

This PR move the logic to for when to enable the Fetch Domain to the
CdpTarget. That way we can manage interception more gradually.

Also fixes an issue where if we `network.addInterception` then
`network.subscribe`, the interception would have not been enabled.
Also fixes an issue if a CdpTarget is created later but network
interception was enable globally before that.

src/bidiMapper/domains/context/CdpTarget.ts

@@ -36,11 +36,16 @@ export class CdpTarget {
   readonly #eventManager: EventManager;
 
   readonly #preloadScriptStorage: PreloadScriptStorage;
+  readonly #networkStorage: NetworkStorage;
 
   readonly #unblocked = new Deferred<Result<void>>();
   readonly #acceptInsecureCerts: boolean;
   #networkDomainEnabled = false;
-  #fetchDomainEnabled = false;
+  #fetchDomainStages = {
+    request: false,
+    response: false,
+    auth: false,
+  };
 
   static create(
     targetId: Protocol.Target.TargetID,
@@ -59,6 +64,7 @@ export class CdpTarget {
       browserCdpClient,
       eventManager,
       preloadScriptStorage,
+      networkStorage,
       acceptInsecureCerts
     );
 
@@ -79,13 +85,15 @@ export class CdpTarget {
     browserCdpClient: CdpClient,
     eventManager: EventManager,
     preloadScriptStorage: PreloadScriptStorage,
+    networkStorage: NetworkStorage,
     acceptInsecureCerts: boolean
   ) {
     this.#id = targetId;
     this.#cdpClient = cdpClient;
+    this.#browserCdpClient = browserCdpClient;
     this.#eventManager = eventManager;
     this.#preloadScriptStorage = preloadScriptStorage;
-    this.#browserCdpClient = browserCdpClient;
+    this.#networkStorage = networkStorage;
     this.#acceptInsecureCerts = acceptInsecureCerts;
   }
 
@@ -122,6 +130,7 @@ export class CdpTarget {
         BiDiModule.Network,
         this.#id
       );
+    this.#networkDomainEnabled = enabledNetwork;
 
     try {
       await Promise.all([
@@ -138,6 +147,7 @@ export class CdpTarget {
         enabledNetwork
           ? this.#cdpClient.sendCommand('Network.enable')
           : undefined,
+        this.toggleFetchIfNeeded(),
         this.#cdpClient.sendCommand('Target.setAutoAttach', {
           autoAttach: true,
           waitForDebuggerOnStart: true,
@@ -163,29 +173,41 @@ export class CdpTarget {
     });
   }
 
-  async enableFetchIfNeeded(params: Protocol.Fetch.EnableRequest) {
-    if (!this.#networkDomainEnabled || this.#fetchDomainEnabled) {
+  async toggleFetchIfNeeded() {
+    const stages = this.#networkStorage.getInterceptionStages(this.id);
+
+    if (
+      // Only toggle interception when Network is enabled
+      !this.#networkDomainEnabled ||
+      (this.#fetchDomainStages.request === stages.request &&
+        this.#fetchDomainStages.response === stages.response &&
+        this.#fetchDomainStages.auth === stages.auth)
+    ) {
       return;
     }
+    const patterns: Protocol.Fetch.EnableRequest['patterns'] = [];
 
-    this.#fetchDomainEnabled = true;
-    try {
-      await this.#cdpClient.sendCommand('Fetch.enable', params);
-    } catch (err) {
-      this.#fetchDomainEnabled = false;
+    this.#fetchDomainStages = stages;
+    if (stages.request || stages.auth) {
+      // CDP quirk we need request interception when we intercept auth
+      patterns.push({
+        urlPattern: '*',
+        requestStage: 'Request',
+      });
     }
-  }
-
-  async disableFetchIfNeeded() {
-    if (!this.#fetchDomainEnabled) {
-      return;
+    if (stages.response) {
+      patterns.push({
+        urlPattern: '*',
+        requestStage: 'Response',
+      });
     }
-
-    this.#fetchDomainEnabled = false;
-    try {
+    if (patterns.length) {
+      await this.#cdpClient.sendCommand('Fetch.enable', {
+        patterns,
+        handleAuthRequests: stages.auth,
+      });
+    } else {
       await this.#cdpClient.sendCommand('Fetch.disable');
-    } catch (err) {
-      this.#fetchDomainEnabled = true;
     }
   }
 
@@ -196,9 +218,12 @@ export class CdpTarget {
 
     this.#networkDomainEnabled = enabled;
     try {
-      await this.#cdpClient.sendCommand(
-        this.#networkDomainEnabled ? 'Network.enable' : 'Network.disable'
-      );
+      await Promise.all([
+        this.#cdpClient.sendCommand(
+          enabled ? 'Network.enable' : 'Network.disable'
+        ),
+        this.toggleFetchIfNeeded(),
+      ]);
     } catch (err) {
       this.#networkDomainEnabled = !enabled;
     }

src/bidiMapper/domains/network/NetworkProcessor.ts

@@ -49,18 +49,24 @@ export class NetworkProcessor {
   async addIntercept(
     params: Network.AddInterceptParameters
   ): Promise<Network.AddInterceptResult> {
-    // TODO: Use in intercepts
     this.#browsingContextStorage.verifyContextsList(params.contexts);
 
     const urlPatterns: Network.UrlPattern[] = params.urlPatterns ?? [];
     const parsedUrlPatterns: Network.UrlPattern[] =
       NetworkProcessor.parseUrlPatterns(urlPatterns);
 
-    const intercept: Network.Intercept =
-      await this.#networkStorage.addIntercept({
-        urlPatterns: parsedUrlPatterns,
-        phases: params.phases,
-      });
+    const intercept: Network.Intercept = this.#networkStorage.addIntercept({
+      urlPatterns: parsedUrlPatterns,
+      phases: params.phases,
+      contexts: params.contexts,
+    });
+
+    await Promise.all(
+      // TODO: We should to this for other CDP targets as well
+      this.#browsingContextStorage.getTopLevelContexts().map((context) => {
+        return context.cdpTarget.toggleFetchIfNeeded();
+      })
+    );
 
     return {
       intercept,
@@ -213,7 +219,14 @@ export class NetworkProcessor {
   async removeIntercept(
     params: Network.RemoveInterceptParameters
   ): Promise<EmptyResult> {
-    await this.#networkStorage.removeIntercept(params.intercept);
+    this.#networkStorage.removeIntercept(params.intercept);
+
+    await Promise.all(
+      // TODO: We should to this for other CDP targets as well
+      this.#browsingContextStorage.getTopLevelContexts().map((context) => {
+        return context.cdpTarget.toggleFetchIfNeeded();
+      })
+    );
 
     return {};
   }

src/bidiMapper/domains/network/NetworkRequest.ts

@@ -150,6 +150,10 @@ export class NetworkRequest {
     return this.#redirectCount;
   }
 
+  get cdpTarget() {
+    return this.#cdpTarget;
+  }
+
   get cdpClient() {
     return this.#cdpTarget.cdpClient;
   }

src/bidiMapper/domains/network/NetworkStorage.spec.ts

@@ -125,7 +125,7 @@ describe('NetworkStorage', () => {
 
     it('should work interception', async () => {
       const request = new MockCdpNetworkEvents(cdpClient);
-      const interception = await networkStorage.addIntercept({
+      const interception = networkStorage.addIntercept({
         urlPatterns: [{type: 'string', pattern: request.url}],
         phases: [Network.InterceptPhase.BeforeRequestSent],
       });
@@ -144,7 +144,7 @@ describe('NetworkStorage', () => {
 
     it('should work interception pause first', async () => {
       const request = new MockCdpNetworkEvents(cdpClient);
-      const interception = await networkStorage.addIntercept({
+      const interception = networkStorage.addIntercept({
         urlPatterns: [{type: 'string', pattern: request.url}],
         phases: [Network.InterceptPhase.BeforeRequestSent],
       });
@@ -162,7 +162,7 @@ describe('NetworkStorage', () => {
     });
 
     it('should work non blocking interception', async () => {
-      await networkStorage.addIntercept({
+      networkStorage.addIntercept({
         urlPatterns: [{type: 'string', pattern: 'http://not.correct.com'}],
         phases: [Network.InterceptPhase.BeforeRequestSent],
       });
@@ -182,7 +182,7 @@ describe('NetworkStorage', () => {
 
     it('should work with non blocking interception and fail response', async () => {
       const request = new MockCdpNetworkEvents(cdpClient);
-      await networkStorage.addIntercept({
+      networkStorage.addIntercept({
         urlPatterns: [{type: 'string', pattern: 'http://not.correct.com'}],
         phases: [Network.InterceptPhase.BeforeRequestSent],
       });
@@ -238,7 +238,7 @@ describe('NetworkStorage', () => {
 
     it('should work interception', async () => {
       const request = new MockCdpNetworkEvents(cdpClient);
-      const interception = await networkStorage.addIntercept({
+      const interception = networkStorage.addIntercept({
         urlPatterns: [{type: 'string', pattern: request.url}],
         phases: [Network.InterceptPhase.ResponseStarted],
       });
@@ -259,7 +259,7 @@ describe('NetworkStorage', () => {
     });
 
     it('should work non blocking interception', async () => {
-      await networkStorage.addIntercept({
+      networkStorage.addIntercept({
         urlPatterns: [{type: 'string', pattern: 'http://not.correct.com'}],
         phases: [Network.InterceptPhase.ResponseStarted],
       });