[devtools] Add CDP method to fetch resources

This CL adds a method to the Network domain that DevTools can use to
fetch resources in a more secure way. Such resources are, for example,
source maps, or files referenced by source maps. The improvement over
the old implementation is that the fetch now is very similar to a fetch
in the page and uses the correct settings for SiteForCookies and NIK.

Initially, we had planed to change source map fetching to CORS, but this
would be a breaking change, so due to COVID-19 and the hold on breaking
changes, this CL uses a no-CORS fetch with CORB blocking disabled to
fetch source maps. CORB-blocking must be disabled because source maps
may be valid JSON.

The front-end can provide a frame_id (in DevTools terms, a devtools
frame token in Chromium terms) and the page will look for this frame
id in its sub resources. The fetch will then occur using a loader that
is very similar to the loaders for that frame.

Change-Id: Ideb36adbd79b9a36e6d3333299dcd036eb7a1332
Bug: chromium:1069378
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2027416
Commit-Queue: Sigurd Schneider <sigurds@chromium.org>
Reviewed-by: Mike West <mkwst@chromium.org>
Reviewed-by: Kinuko Yasuda <kinuko@chromium.org>
Reviewed-by: Andrey Kosyakov <caseq@chromium.org>
Reviewed-by: Matt Menke <mmenke@chromium.org>
Reviewed-by: Łukasz Anforowicz <lukasza@chromium.org>
Cr-Commit-Position: refs/heads/master@{#810177}
GitOrigin-RevId: 7c94a51362a4a2ff8dae039f0c193babe79f2a4b

blink/public/devtools_protocol/browser_protocol.pdl

@@ -5400,6 +5400,38 @@ domain Network
     returns
       SecurityIsolationStatus status
 
+  # An object providing the result of a network resource load.
+  experimental type LoadNetworkResourcePageResult extends object
+    properties
+      boolean success
+      # Optional values used for error reporting.
+      optional number netError
+      optional string netErrorName
+      optional number httpStatusCode
+      # If successful, one of the following two fields holds the result.
+      optional IO.StreamHandle stream
+      # Response headers.
+      optional Network.Headers headers
+
+  # An options object that may be extended later to better support CORS,
+  # CORB and streaming.
+  experimental type LoadNetworkResourceOptions extends object
+    properties
+      boolean disableCache
+      boolean includeCredentials
+
+  # Fetches the resource and returns the content.
+  experimental command loadNetworkResource
+    parameters
+      # Frame id to get the resource for.
+      Page.FrameId frameId
+      # URL of the resource to get content for.
+      string url
+      # Options for the request.
+      LoadNetworkResourceOptions options
+    returns
+      LoadNetworkResourcePageResult resource
+
 # This domain provides various functionality related to drawing atop the inspected page.
 experimental domain Overlay
   depends on DOM

blink/renderer/core/inspector/inspector_protocol_config.json

@@ -92,7 +92,7 @@
                 "domain": "Network",
                 "exclude": ["clearBrowserCache", "clearBrowserCookies", "getCookies", "getAllCookies", "deleteCookies", "setCookie", "setCookies", "canEmulateNetworkConditions",
                             "setRequestInterception", "continueInterceptedRequest", "getResponseBodyForInterception",
-                            "takeResponseBodyForInterceptionAsStream", "getSecurityIsolationStatus"],
+                            "takeResponseBodyForInterceptionAsStream", "getSecurityIsolationStatus", "loadNetworkResource"],
                 "async": ["getResponseBody", "getRequestPostData"]
             },
             {

blink/web_tests/flag-specific/disable-site-isolation-trials/http/tests/inspector-protocol/network/load-network-resource-errors-expected.txt

@@ -0,0 +1,39 @@
+Tests Page.loadNetworkResource for different frames with cookies
+Number of frames in page: 3
+Response for fetch: {
+    id : <number>
+    result : {
+        resource : {
+            headers : <object>
+            httpStatusCode : 200
+            stream : <string>
+            success : true
+        }
+    }
+    sessionId : <string>
+}
+Response for fetch: {
+    id : <number>
+    result : {
+        resource : {
+            headers : <object>
+            httpStatusCode : 200
+            stream : <string>
+            success : true
+        }
+    }
+    sessionId : <string>
+}
+Response for fetch: {
+    id : <number>
+    result : {
+        resource : {
+            headers : <object>
+            httpStatusCode : 200
+            stream : <string>
+            success : true
+        }
+    }
+    sessionId : <string>
+}
+

blink/web_tests/http/tests/inspector-protocol/network/load-network-resource-basic-expected.txt

@@ -0,0 +1,45 @@
+Tests for Network.loadNetworkResource on the same origin
+Response for fetch with existing resource with text content type: {
+    resource : {
+        headers : <object>
+        httpStatusCode : 200
+        stream : <string>
+        success : true
+    }
+}
+{
+    id : <number>
+    result : {
+        base64Encoded : false
+        data : {"version":3,"file":"source.js","sourceRoot":"","sources":["source.ts"],"names":[],"mappings":"AAAA,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA"}
+        eof : false
+    }
+    sessionId : <string>
+}
+Response for fetch with existing resource without content type: {
+    resource : {
+        headers : <object>
+        httpStatusCode : 200
+        stream : <string>
+        success : true
+    }
+}
+{
+    id : <number>
+    result : {
+        base64Encoded : true
+        data : eyJ2ZXJzaW9uIjozLCJmaWxlIjoic291cmNlLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsic291cmNlLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sQ0FBQyxHQUFHLENBQUMsT0FBTyxDQUFDLENBQUEifQo=
+        eof : false
+    }
+    sessionId : <string>
+}
+Response for fetch with non-existing resource: {
+    resource : {
+        headers : <object>
+        httpStatusCode : 404
+        netError : -379
+        netErrorName : net::ERR_HTTP_RESPONSE_CODE_FAILURE
+        success : false
+    }
+}
+

blink/web_tests/http/tests/inspector-protocol/network/load-network-resource-basic.js

@@ -0,0 +1,29 @@
+(async function(testRunner) {
+  const {page, session, dp} = await testRunner.startBlank(
+      `Tests for Network.loadNetworkResource on the same origin`);
+
+  await dp.Network.enable();
+
+  const frameId = (await dp.Target.getTargetInfo()).result.targetInfo.targetId;
+
+  async function requestSourceMap(frameId, testExplanation, url) {
+    const response = await dp.Network.loadNetworkResource({frameId, url, options: {disableCache:false, includeCredentials: false}});
+    testRunner.log(response.result, testExplanation, ["headers", "stream"]);
+    if (response.result.resource.success) {
+      let result = await dp.IO.read({handle: response.result.resource.stream, size: 1000*1000});
+      testRunner.log(result);
+      await dp.IO.close({handle: response.result.resource.stream});
+    }
+  }
+
+  const urlWithMimeType = `http://localhost:8000/inspector-protocol/network/resources/source.map.php`;
+  await requestSourceMap(frameId, `Response for fetch with existing resource with text content type: `, urlWithMimeType);
+
+  const urlWithoutMimeType = `http://localhost:8000/inspector-protocol/network/resources/source.map`;
+  await requestSourceMap(frameId, `Response for fetch with existing resource without content type: `, urlWithoutMimeType);
+
+  const nonExistentUrl = `http://localhost:8000/inspector-protocol/network/resources/source.map-DOES-NOT-EXIST`;
+  await requestSourceMap(frameId, `Response for fetch with non-existing resource: `, nonExistentUrl);
+
+  testRunner.completeTest();
+})

blink/web_tests/http/tests/inspector-protocol/network/load-network-resource-different-frame-cookie-expected.txt

@@ -0,0 +1,132 @@
+Tests Page.loadNetworkResource for different frames with cookies
+Number of frames in page: 3
+Response for fetch: {
+    resource : {
+        headers : <object>
+        httpStatusCode : 200
+        stream : <string>
+        success : true
+    }
+}
+Steam content:
+{"version":3,"file":"source.js","sourceRoot":"","sources":["source.ts"],"names":[],"mappings":"AAAA,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA"}
+
+Response for fetch: {
+    resource : {
+        headers : <object>
+        httpStatusCode : 200
+        stream : <string>
+        success : true
+    }
+}
+Steam content:
+{"version":3,"file":"source.js","sourceRoot":"","sources":["source.ts"],"names":[],"mappings":"AAAA,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA"}
+
+Response for fetch: {
+    resource : {
+        headers : <object>
+        httpStatusCode : 200
+        stream : <string>
+        success : true
+    }
+}
+Steam content:
+{"version":3,"file":"source.js","sourceRoot":"","sources":["source.ts"],"names":[],"mappings":"AAAA,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA"}
+
+Response for fetching
+https://127.0.0.1:8443/inspector-protocol/network/resources/echo-headers.php?headers=HTTP_COOKIE
+ from
+https://127.0.0.1:8443
+after setting cookie including credentials: {
+    resource : {
+        headers : <object>
+        httpStatusCode : 200
+        stream : <string>
+        success : true
+    }
+}
+Steam content:
+HTTP_COOKIE: <not set>
+
+
+Response for fetching
+https://127.0.0.1:8443/inspector-protocol/network/resources/echo-headers.php?headers=HTTP_COOKIE
+ from
+https://127.0.0.1:8443
+after setting cookie including only samesite credentials: {
+    resource : {
+        headers : <object>
+        httpStatusCode : 200
+        stream : <string>
+        success : true
+    }
+}
+Steam content:
+HTTP_COOKIE: <not set>
+
+
+Response for fetching
+https://devtools.oopif-a.test:8443/inspector-protocol/network/resources/echo-headers.php?headers=HTTP_COOKIE
+ from
+https://devtools.oopif-a.test:8443
+after setting cookie including credentials: {
+    resource : {
+        headers : <object>
+        httpStatusCode : 200
+        stream : <string>
+        success : true
+    }
+}
+Steam content:
+HTTP_COOKIE: name=value
+
+
+Response for fetching
+https://devtools.oopif-a.test:8443/inspector-protocol/network/resources/echo-headers.php?headers=HTTP_COOKIE
+ from
+https://devtools.oopif-a.test:8443
+after setting cookie including only samesite credentials: {
+    resource : {
+        headers : <object>
+        httpStatusCode : 200
+        stream : <string>
+        success : true
+    }
+}
+Steam content:
+HTTP_COOKIE: name=value
+
+
+Response for fetching
+https://devtools.oopif-b.test:8443/inspector-protocol/network/resources/echo-headers.php?headers=HTTP_COOKIE
+ from
+https://devtools.oopif-b.test:8443
+after setting cookie including credentials: {
+    resource : {
+        headers : <object>
+        httpStatusCode : 200
+        stream : <string>
+        success : true
+    }
+}
+Steam content:
+HTTP_COOKIE: <not set>
+
+
+Response for fetching
+https://devtools.oopif-b.test:8443/inspector-protocol/network/resources/echo-headers.php?headers=HTTP_COOKIE
+ from
+https://devtools.oopif-b.test:8443
+after setting cookie including only samesite credentials: {
+    resource : {
+        headers : <object>
+        httpStatusCode : 200
+        stream : <string>
+        success : true
+    }
+}
+Steam content:
+HTTP_COOKIE: <not set>
+
+
+

blink/web_tests/http/tests/inspector-protocol/network/load-network-resource-different-frame-cookie.js

@@ -0,0 +1,91 @@
+(async function(testRunner) {
+  const {page, session, dp} = await testRunner.startBlank(
+      `Tests Page.loadNetworkResource for different frames with cookies`);
+
+  await session.protocol.Network.clearBrowserCache();
+  await session.protocol.Network.setCacheDisabled({cacheDisabled: true});
+  await dp.Target.setAutoAttach({autoAttach: true, waitForDebuggerOnStart: true, flatten: true});
+
+
+  let loadCount = 5;
+  let loadCallback;
+  const loadPromise = new Promise(fulfill => loadCallback = fulfill);
+
+  const allTargets = [];
+  async function initalizeTarget(dp) {
+    allTargets.push(dp);
+    await dp.Page.enable();
+    await dp.Network.enable();
+    dp.Page.onFrameStoppedLoading(e => {
+      if (!--loadCount)
+        loadCallback();
+    });
+    await dp.Runtime.runIfWaitingForDebugger();
+  }
+
+  dp.Target.onAttachedToTarget(async e => {
+    const child = session.createChild(e.params.sessionId);
+    const targetProtocol = child.protocol;
+    await initalizeTarget(targetProtocol);
+  });
+  await initalizeTarget(dp);
+
+  await dp.Page.navigate({url: 'https://127.0.0.1:8443/inspector-protocol/resources/iframe-navigation-secure.html'});
+
+  const frames = new Map();
+  function getFrameIds(dp, frameTree) {
+    frames.set(frameTree.frame.securityOrigin, {frameId: frameTree.frame.id, dp});
+    (frameTree.childFrames || []).forEach(getFrameIds.bind(null, dp));
+  }
+
+  await loadPromise;
+  const frameTargetList = [];
+  for (const dp of allTargets) {
+    const {result} = await dp.Page.getFrameTree();
+    frameTargetList.push({dp, frameTree: result.frameTree});
+  }
+  frameTargetList.sort((a,b) => a.frameTree.frame.url.localeCompare(b.frameTree.frame.url));
+  frameTargetList.forEach(({dp, frameTree}) => getFrameIds(dp, frameTree));
+
+  testRunner.log(`Number of frames in page: ${frames.size}`);
+
+  async function requestSourceMap(dp, frameId, testExplanation, url, options) {
+    const response = await dp.Network.loadNetworkResource({frameId, url, options: {disableCache: false, ...options}});
+    testRunner.log(response.result, testExplanation, ["headers", "stream"]);
+    if (response.result.resource.success) {
+      let result = await dp.IO.read({handle: response.result.resource.stream, size: 1000*1000});
+      testRunner.log(`Steam content:`)
+      testRunner.log(result.result.data);
+      await dp.IO.close({handle: response.result.resource.stream});
+    }
+    testRunner.log(``);
+  }
+
+  for (const {frameId, dp} of frames.values()) {
+    const url = `https://localhost:8443/inspector-protocol/network/resources/source.map.php`;
+    await requestSourceMap(dp, frameId, `Response for fetch: `, url, {includeCredentials: true});
+  }
+
+  // Now test cookie behavior.
+  const setCookieUrl = 'https://devtools.oopif-a.test:8443/inspector-protocol/network/resources/set-cookie.php?cookie='
+  + encodeURIComponent('name=value; SameSite=None; Secure');
+  await session.evaluate(`fetch('${setCookieUrl}', {method: 'POST', credentials: 'include'})`);
+
+  const setCookieUrl2 = 'https://devtools.oopif-a.test:8443/inspector-protocol/network/resources/set-cookie.php?cookie='
+  + encodeURIComponent('nameStrict=value2; SameSite=Strict; Secure');
+  await session.evaluate(`fetch('${setCookieUrl2}', {method: 'POST', credentials: 'include'})`);
+
+  const setCookieUrl3 = 'https://devtools.oopif-a.test:8443/inspector-protocol/network/resources/set-cookie.php?cookie='
+  + encodeURIComponent('nameOther=value3; SameSite=Lax; Secure');
+  await session.evaluate(`fetch('${setCookieUrl3}', {method: 'POST', credentials: 'include'})`);
+
+  for (const [frameUrl, {dp, frameId}] of frames.entries()) {
+    const parsedURL = new URL(frameUrl);
+    const url = `${parsedURL.protocol}//${parsedURL.host}/inspector-protocol/network/resources/echo-headers.php?headers=HTTP_COOKIE`;
+    for (const includeCredentials of [true, false]) {
+      await requestSourceMap(dp, frameId, `Response for fetching\n${url}\n from\n${frameUrl}\nafter setting cookie ${includeCredentials?"including":"including only samesite"} credentials: `, url, {includeCredentials});
+    }
+  }
+
+  testRunner.completeTest();
+})