From e0d13fea5b2bc7d57accff9e3c4ac361dc0d62f3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fran=C3=A7ois=20Daoust?= <fd@w3.org>
Date: Tue, 30 May 2023 07:48:08 +0200
Subject: [PATCH] Crawl using Node.js fetch function directly (#1300)

Through `fetch-filecache-for-crawling`, Reffy used to depend on `node-fetch` to
send HTTP requests. The `fetch-filecache-for-crawling` library now uses the
native implementation of `fetch` in Node.js v18 and above. This update bumps the
version of `fetch-filecache-for-crawling`, making Reffy use the native
implementation of `fetch` in Node.js instead.

This update also removes the dependency on the `AbortController` polyfill, since
it is no longer needed.

On top of a couple of changes needed to account for slight differences in the
way headers are handled, main changes in this update are test-related, to
replace the `nock` library, which can no longer be used because it cannot
intercept `fetch` requests, with the mock functions of `undici` (which provides
the implementation of `fetch` in Node.js).

From a crawling perspective, this update is a no-op, although it should be noted
that, when certain specs are crawled (MathML Core in practice), Node.js may
report memory leak warnings such as:

```
MaxListenersExceededWarning: Possible EventTarget memory leak detected. 101
abort listeners added to [AbortSignal]. Use events.setMaxListeners() to increase
limit
```

These are due to the fact that the same `AbortController` is (rightly) connected
to all pending HTTP requests linked to the spec being crawled, and the MathML
Core draft references over 100 embedded resources. In other words, that's all
normal!

The `AbortSignal` implementation in Node.js does not directly inherit from
`EventEmitter`. As far as I can tell, there is no direct way to call
`setMaxListeners()` as a result.

If that proves annoying over time, we could rather adjust
`events.defaultMaxListeners` but that is a global setting:
https://nodejs.org/dist/latest-v18.x/docs/api/events.html#eventsdefaultmaxlisteners

(Alternatively, it may be worth digging into MathML Core to check whether we
actually need to download all of these resources).
---
 package-lock.json      | 758 +++++++++++++++++++++++++++++++++--------
 package.json           |   7 +-
 src/lib/mock-server.js | 186 ++++++++++
 src/lib/nock-server.js | 126 -------
 src/lib/util.js        |  18 +-
 tests/crawl.js         |   6 +-
 tests/reffy-package.js |   2 +-
 7 files changed, 821 insertions(+), 282 deletions(-)
 create mode 100644 src/lib/mock-server.js
 delete mode 100644 src/lib/nock-server.js

diff --git a/package-lock.json b/package-lock.json
index d9d20bff..9b3079b5 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,19 +1,18 @@
 {
   "name": "reffy",
-  "version": "13.0.2",
+  "version": "13.0.3",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "reffy",
-      "version": "13.0.2",
+      "version": "13.0.3",
       "license": "MIT",
       "dependencies": {
-        "abortcontroller-polyfill": "1.7.5",
         "ajv": "8.12.0",
         "ajv-formats": "2.1.1",
         "commander": "10.0.1",
-        "fetch-filecache-for-crawling": "4.1.0",
+        "fetch-filecache-for-crawling": "5.0.0",
         "puppeteer": "20.3.0",
         "semver": "^7.3.5",
         "web-specs": "2.59.0",
@@ -25,15 +24,18 @@
       "devDependencies": {
         "chai": "4.3.7",
         "mocha": "10.2.0",
-        "nock": "13.3.1",
         "respec": "34.1.1",
         "respec-hljs": "2.1.1",
-        "rollup": "3.23.0"
+        "rollup": "3.23.0",
+        "undici": "^5.22.1"
       },
       "engines": {
         "node": ">=18"
       }
     },
+    "../../fetch-filecache-for-crawling": {
+      "extraneous": true
+    },
     "node_modules/@babel/code-frame": {
       "version": "7.18.6",
       "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.18.6.tgz",
@@ -122,6 +124,104 @@
         "node": ">=4"
       }
     },
+    "node_modules/@isaacs/cliui": {
+      "version": "8.0.2",
+      "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz",
+      "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==",
+      "dependencies": {
+        "string-width": "^5.1.2",
+        "string-width-cjs": "npm:string-width@^4.2.0",
+        "strip-ansi": "^7.0.1",
+        "strip-ansi-cjs": "npm:strip-ansi@^6.0.1",
+        "wrap-ansi": "^8.1.0",
+        "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0"
+      },
+      "engines": {
+        "node": ">=12"
+      }
+    },
+    "node_modules/@isaacs/cliui/node_modules/ansi-regex": {
+      "version": "6.0.1",
+      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz",
+      "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==",
+      "engines": {
+        "node": ">=12"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/ansi-regex?sponsor=1"
+      }
+    },
+    "node_modules/@isaacs/cliui/node_modules/ansi-styles": {
+      "version": "6.2.1",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz",
+      "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==",
+      "engines": {
+        "node": ">=12"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
+      }
+    },
+    "node_modules/@isaacs/cliui/node_modules/emoji-regex": {
+      "version": "9.2.2",
+      "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz",
+      "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg=="
+    },
+    "node_modules/@isaacs/cliui/node_modules/string-width": {
+      "version": "5.1.2",
+      "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz",
+      "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==",
+      "dependencies": {
+        "eastasianwidth": "^0.2.0",
+        "emoji-regex": "^9.2.2",
+        "strip-ansi": "^7.0.1"
+      },
+      "engines": {
+        "node": ">=12"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
+    "node_modules/@isaacs/cliui/node_modules/strip-ansi": {
+      "version": "7.0.1",
+      "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.0.1.tgz",
+      "integrity": "sha512-cXNxvT8dFNRVfhVME3JAe98mkXDYN2O1l7jmcwMnOslDeESg1rF/OZMtK0nRAhiari1unG5cD4jG3rapUAkLbw==",
+      "dependencies": {
+        "ansi-regex": "^6.0.1"
+      },
+      "engines": {
+        "node": ">=12"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/strip-ansi?sponsor=1"
+      }
+    },
+    "node_modules/@isaacs/cliui/node_modules/wrap-ansi": {
+      "version": "8.1.0",
+      "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz",
+      "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==",
+      "dependencies": {
+        "ansi-styles": "^6.1.0",
+        "string-width": "^5.0.1",
+        "strip-ansi": "^7.0.1"
+      },
+      "engines": {
+        "node": ">=12"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/wrap-ansi?sponsor=1"
+      }
+    },
+    "node_modules/@pkgjs/parseargs": {
+      "version": "0.11.0",
+      "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz",
+      "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==",
+      "optional": true,
+      "engines": {
+        "node": ">=14"
+      }
+    },
     "node_modules/@puppeteer/browsers": {
       "version": "1.3.0",
       "resolved": "https://registry.npmjs.org/@puppeteer/browsers/-/browsers-1.3.0.tgz",
@@ -213,11 +313,6 @@
         "@types/node": "*"
       }
     },
-    "node_modules/abortcontroller-polyfill": {
-      "version": "1.7.5",
-      "resolved": "https://registry.npmjs.org/abortcontroller-polyfill/-/abortcontroller-polyfill-1.7.5.tgz",
-      "integrity": "sha512-JMJ5soJWP18htbbxJjG7bG6yuI6pRhgJ0scHHTfkUjf6wjP912xZWvM+A4sJK3gqd9E8fcPbDnOefbA9Th/FIQ=="
-    },
     "node_modules/agent-base": {
       "version": "6.0.2",
       "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz",
@@ -365,6 +460,7 @@
       "version": "1.1.11",
       "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
       "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
+      "dev": true,
       "dependencies": {
         "balanced-match": "^1.0.0",
         "concat-map": "0.0.1"
@@ -419,6 +515,18 @@
         "node": "*"
       }
     },
+    "node_modules/busboy": {
+      "version": "1.6.0",
+      "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz",
+      "integrity": "sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==",
+      "dev": true,
+      "dependencies": {
+        "streamsearch": "^1.1.0"
+      },
+      "engines": {
+        "node": ">=10.16.0"
+      }
+    },
     "node_modules/callsites": {
       "version": "3.1.0",
       "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz",
@@ -584,7 +692,8 @@
     "node_modules/concat-map": {
       "version": "0.0.1",
       "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
-      "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s="
+      "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=",
+      "dev": true
     },
     "node_modules/cosmiconfig": {
       "version": "8.1.3",
@@ -611,6 +720,19 @@
         "node-fetch": "^2.6.11"
       }
     },
+    "node_modules/cross-spawn": {
+      "version": "7.0.3",
+      "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
+      "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==",
+      "dependencies": {
+        "path-key": "^3.1.0",
+        "shebang-command": "^2.0.0",
+        "which": "^2.0.1"
+      },
+      "engines": {
+        "node": ">= 8"
+      }
+    },
     "node_modules/debug": {
       "version": "4.3.4",
       "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
@@ -689,6 +811,11 @@
         "node": ">=0.3.1"
       }
     },
+    "node_modules/eastasianwidth": {
+      "version": "0.2.0",
+      "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz",
+      "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA=="
+    },
     "node_modules/ee-first": {
       "version": "1.1.1",
       "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz",
@@ -789,22 +916,21 @@
       }
     },
     "node_modules/fetch-filecache-for-crawling": {
-      "version": "4.1.0",
-      "resolved": "https://registry.npmjs.org/fetch-filecache-for-crawling/-/fetch-filecache-for-crawling-4.1.0.tgz",
-      "integrity": "sha512-z7f5oCxQaDEST+4HqdPj7w0hX41hWXw36N9I9SQ5wVEWMy01Oz6ZRSKl0lQQ/zVpwrHp/inFXd8ojLojXnMDzQ==",
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/fetch-filecache-for-crawling/-/fetch-filecache-for-crawling-5.0.0.tgz",
+      "integrity": "sha512-+zIz6ARXTR8C+maoDJZTS8QgzVH3WWqBpngMeV2I+jm5z89XK/zmaPie18vli7z/zGORvUQAYbTtzfBMy16u4A==",
       "dependencies": {
         "filenamify-url": "^2.1.2",
-        "node-fetch": "^2.6.0",
-        "rimraf": "^3.0.2"
+        "rimraf": "^5.0.1"
       },
       "engines": {
-        "node": ">=12.0.0"
+        "node": ">=18.0.0"
       }
     },
     "node_modules/filename-reserved-regex": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/filename-reserved-regex/-/filename-reserved-regex-2.0.0.tgz",
-      "integrity": "sha1-q/c9+rc10EVECr/qLZHzieu/oik=",
+      "integrity": "sha512-lc1bnsSr4L4Bdif8Xb/qrtokGbq5zlsms/CYH8PP+WtCkGNF65DPiQY8vG3SakEdRn8Dlnm+gW/qWKKjS5sZzQ==",
       "engines": {
         "node": ">=4"
       }
@@ -894,6 +1020,21 @@
         "flat": "cli.js"
       }
     },
+    "node_modules/foreground-child": {
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.1.1.tgz",
+      "integrity": "sha512-TMKDUnIte6bfb5nWv7V/caI169OHgvwjb7V4WkeUvbQQdjr5rWKqHFiKWb/fcOwB+CzBT+qbWjvj+DVwRskpIg==",
+      "dependencies": {
+        "cross-spawn": "^7.0.0",
+        "signal-exit": "^4.0.1"
+      },
+      "engines": {
+        "node": ">=14"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
+      }
+    },
     "node_modules/fresh": {
       "version": "0.5.2",
       "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz",
@@ -911,7 +1052,8 @@
     "node_modules/fs.realpath": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
-      "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8="
+      "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=",
+      "dev": true
     },
     "node_modules/fsevents": {
       "version": "2.3.2",
@@ -959,19 +1101,24 @@
       }
     },
     "node_modules/glob": {
-      "version": "7.1.4",
-      "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.4.tgz",
-      "integrity": "sha512-hkLPepehmnKk41pUGm3sYxoFs/umurYfYJCerbXEyFIWcAzvpipAgVkBqqT9RBKMGjnq6kMuyYwha6csxbiM1A==",
+      "version": "10.2.6",
+      "resolved": "https://registry.npmjs.org/glob/-/glob-10.2.6.tgz",
+      "integrity": "sha512-U/rnDpXJGF414QQQZv5uVsabTVxMSwzS5CH0p3DRCIV6ownl4f7PzGnkGmvlum2wB+9RlJWJZ6ACU1INnBqiPA==",
       "dependencies": {
-        "fs.realpath": "^1.0.0",
-        "inflight": "^1.0.4",
-        "inherits": "2",
-        "minimatch": "^3.0.4",
-        "once": "^1.3.0",
-        "path-is-absolute": "^1.0.0"
+        "foreground-child": "^3.1.0",
+        "jackspeak": "^2.0.3",
+        "minimatch": "^9.0.1",
+        "minipass": "^5.0.0 || ^6.0.2",
+        "path-scurry": "^1.7.0"
+      },
+      "bin": {
+        "glob": "dist/cjs/src/bin.js"
       },
       "engines": {
-        "node": "*"
+        "node": ">=16 || 14 >=14.17"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
       }
     },
     "node_modules/glob-parent": {
@@ -1094,6 +1241,7 @@
       "version": "1.0.6",
       "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
       "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=",
+      "dev": true,
       "dependencies": {
         "once": "^1.3.0",
         "wrappy": "1"
@@ -1171,6 +1319,28 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
+    "node_modules/isexe": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
+      "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw=="
+    },
+    "node_modules/jackspeak": {
+      "version": "2.2.1",
+      "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-2.2.1.tgz",
+      "integrity": "sha512-MXbxovZ/Pm42f6cDIDkl3xpwv1AGwObKwfmjs2nQePiy85tP3fatofl3FC1aBsOtP/6fq5SbtgHwWcMsLP+bDw==",
+      "dependencies": {
+        "@isaacs/cliui": "^8.0.2"
+      },
+      "engines": {
+        "node": ">=14"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
+      },
+      "optionalDependencies": {
+        "@pkgjs/parseargs": "^0.11.0"
+      }
+    },
     "node_modules/js-tokens": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",
@@ -1197,23 +1367,11 @@
       "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz",
       "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug=="
     },
-    "node_modules/json-stringify-safe": {
-      "version": "5.0.1",
-      "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz",
-      "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=",
-      "dev": true
-    },
     "node_modules/lines-and-columns": {
       "version": "1.2.4",
       "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz",
       "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg=="
     },
-    "node_modules/lodash": {
-      "version": "4.17.21",
-      "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz",
-      "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==",
-      "dev": true
-    },
     "node_modules/log-symbols": {
       "version": "4.1.0",
       "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-4.1.0.tgz",
@@ -1275,14 +1433,33 @@
       }
     },
     "node_modules/minimatch": {
-      "version": "3.1.2",
-      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
-      "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
+      "version": "9.0.1",
+      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.1.tgz",
+      "integrity": "sha512-0jWhJpD/MdhPXwPuiRkCbfYfSKp2qnn2eOc279qI7f+osl/l+prKSrvhg157zSYvx/1nmgn2NqdT6k2Z7zSH9w==",
       "dependencies": {
-        "brace-expansion": "^1.1.7"
+        "brace-expansion": "^2.0.1"
       },
       "engines": {
-        "node": "*"
+        "node": ">=16 || 14 >=14.17"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
+      }
+    },
+    "node_modules/minimatch/node_modules/brace-expansion": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
+      "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
+      "dependencies": {
+        "balanced-match": "^1.0.0"
+      }
+    },
+    "node_modules/minipass": {
+      "version": "6.0.2",
+      "resolved": "https://registry.npmjs.org/minipass/-/minipass-6.0.2.tgz",
+      "integrity": "sha512-MzWSV5nYVT7mVyWCwn2o7JH13w2TBRmmSqSRCKzTw+lmft9X4z+3wjvs06Tzijo5z4W/kahUCDpRXTF+ZrmF/w==",
+      "engines": {
+        "node": ">=16 || 14 >=14.17"
       }
     },
     "node_modules/mitt": {
@@ -1488,21 +1665,6 @@
         "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1"
       }
     },
-    "node_modules/nock": {
-      "version": "13.3.1",
-      "resolved": "https://registry.npmjs.org/nock/-/nock-13.3.1.tgz",
-      "integrity": "sha512-vHnopocZuI93p2ccivFyGuUfzjq2fxNyNurp7816mlT5V5HF4SzXu8lvLrVzBbNqzs+ODooZ6OksuSUNM7Njkw==",
-      "dev": true,
-      "dependencies": {
-        "debug": "^4.1.0",
-        "json-stringify-safe": "^5.0.1",
-        "lodash": "^4.17.21",
-        "propagate": "^2.0.0"
-      },
-      "engines": {
-        "node": ">= 10.13"
-      }
-    },
     "node_modules/node-fetch": {
       "version": "2.6.11",
       "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.11.tgz",
@@ -1628,10 +1790,42 @@
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
       "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=",
+      "dev": true,
       "engines": {
         "node": ">=0.10.0"
       }
     },
+    "node_modules/path-key": {
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz",
+      "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==",
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/path-scurry": {
+      "version": "1.9.2",
+      "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.9.2.tgz",
+      "integrity": "sha512-qSDLy2aGFPm8i4rsbHd4MNyTcrzHFsLQykrtbuGRknZZCBBVXSv2tSCDN2Cg6Rt/GFRw8GoW9y9Ecw5rIPG1sg==",
+      "dependencies": {
+        "lru-cache": "^9.1.1",
+        "minipass": "^5.0.0 || ^6.0.2"
+      },
+      "engines": {
+        "node": ">=16 || 14 >=14.17"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
+      }
+    },
+    "node_modules/path-scurry/node_modules/lru-cache": {
+      "version": "9.1.1",
+      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-9.1.1.tgz",
+      "integrity": "sha512-65/Jky17UwSb0BuB9V+MyDpsOtXKmYwzhyl+cOa9XUiI4uV2Ouy/2voFP3+al0BjZbJgMBD8FojMpAf+Z+qn4A==",
+      "engines": {
+        "node": "14 || >=16.14"
+      }
+    },
     "node_modules/path-type": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz",
@@ -1674,15 +1868,6 @@
         "node": ">=0.4.0"
       }
     },
-    "node_modules/propagate": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/propagate/-/propagate-2.0.1.tgz",
-      "integrity": "sha512-vGrhOavPSTz4QVNuBNdcNXePNdNMaO1xj9yBeH1ScQPjk/rhg9sSlCXPhMkFuaNNW/syTvYqsnbIJxMBfRbbag==",
-      "dev": true,
-      "engines": {
-        "node": ">= 8"
-      }
-    },
     "node_modules/proxy-from-env": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz",
@@ -1839,14 +2024,17 @@
       "dev": true
     },
     "node_modules/rimraf": {
-      "version": "3.0.2",
-      "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz",
-      "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==",
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-5.0.1.tgz",
+      "integrity": "sha512-OfFZdwtd3lZ+XZzYP/6gTACubwFcHdLRqS9UX3UwpU2dnGQYkPFISRwvM3w9IiB2w7bW5qGo/uAwE4SmXXSKvg==",
       "dependencies": {
-        "glob": "^7.1.3"
+        "glob": "^10.2.5"
       },
       "bin": {
-        "rimraf": "bin.js"
+        "rimraf": "dist/cjs/src/bin.js"
+      },
+      "engines": {
+        "node": ">=14"
       },
       "funding": {
         "url": "https://github.com/sponsors/isaacs"
@@ -1982,6 +2170,36 @@
       "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==",
       "dev": true
     },
+    "node_modules/shebang-command": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz",
+      "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==",
+      "dependencies": {
+        "shebang-regex": "^3.0.0"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/shebang-regex": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz",
+      "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==",
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/signal-exit": {
+      "version": "4.0.2",
+      "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.0.2.tgz",
+      "integrity": "sha512-MY2/qGx4enyjprQnFaZsHib3Yadh3IXyV2C321GY0pjGfVBu4un0uDJkwgdxqO+Rdx8JMT8IfJIRwbYVz3Ob3Q==",
+      "engines": {
+        "node": ">=14"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
+      }
+    },
     "node_modules/statuses": {
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz",
@@ -1991,6 +2209,15 @@
         "node": ">= 0.8"
       }
     },
+    "node_modules/streamsearch": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz",
+      "integrity": "sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==",
+      "dev": true,
+      "engines": {
+        "node": ">=10.0.0"
+      }
+    },
     "node_modules/string_decoder": {
       "version": "1.3.0",
       "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz",
@@ -2012,6 +2239,20 @@
         "node": ">=8"
       }
     },
+    "node_modules/string-width-cjs": {
+      "name": "string-width",
+      "version": "4.2.3",
+      "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz",
+      "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==",
+      "dependencies": {
+        "emoji-regex": "^8.0.0",
+        "is-fullwidth-code-point": "^3.0.0",
+        "strip-ansi": "^6.0.1"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
     "node_modules/strip-ansi": {
       "version": "6.0.1",
       "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz",
@@ -2023,6 +2264,18 @@
         "node": ">=8"
       }
     },
+    "node_modules/strip-ansi-cjs": {
+      "name": "strip-ansi",
+      "version": "6.0.1",
+      "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz",
+      "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==",
+      "dependencies": {
+        "ansi-regex": "^5.0.1"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
     "node_modules/strip-json-comments": {
       "version": "3.1.1",
       "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz",
@@ -2116,7 +2369,7 @@
     "node_modules/trim-repeated": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/trim-repeated/-/trim-repeated-1.0.0.tgz",
-      "integrity": "sha1-42RqLqTokTEr9+rObPsFOAvAHCE=",
+      "integrity": "sha512-pkonvlKk8/ZuR0D5tLW8ljt5I8kmxp2XKymhepUeOdCEfKpZaktSArkLHZt76OB1ZvO9bssUsDty4SWhLvZpLg==",
       "dependencies": {
         "escape-string-regexp": "^1.0.2"
       },
@@ -2142,6 +2395,18 @@
         "through": "^2.3.8"
       }
     },
+    "node_modules/undici": {
+      "version": "5.22.1",
+      "resolved": "https://registry.npmjs.org/undici/-/undici-5.22.1.tgz",
+      "integrity": "sha512-Ji2IJhFXZY0x/0tVBXeQwgPlLWw13GVzpsWPQ3rV50IFMMof2I55PZZxtm4P6iNq+L5znYN9nSTAq0ZyE6lSJw==",
+      "dev": true,
+      "dependencies": {
+        "busboy": "^1.6.0"
+      },
+      "engines": {
+        "node": ">=14.0"
+      }
+    },
     "node_modules/unpipe": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz",
@@ -2177,6 +2442,20 @@
         "node": ">= 14"
       }
     },
+    "node_modules/which": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
+      "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==",
+      "dependencies": {
+        "isexe": "^2.0.0"
+      },
+      "bin": {
+        "node-which": "bin/node-which"
+      },
+      "engines": {
+        "node": ">= 8"
+      }
+    },
     "node_modules/workerpool": {
       "version": "6.2.1",
       "resolved": "https://registry.npmjs.org/workerpool/-/workerpool-6.2.1.tgz",
@@ -2199,6 +2478,23 @@
         "url": "https://github.com/chalk/wrap-ansi?sponsor=1"
       }
     },
+    "node_modules/wrap-ansi-cjs": {
+      "name": "wrap-ansi",
+      "version": "7.0.0",
+      "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz",
+      "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==",
+      "dependencies": {
+        "ansi-styles": "^4.0.0",
+        "string-width": "^4.1.0",
+        "strip-ansi": "^6.0.0"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/wrap-ansi?sponsor=1"
+      }
+    },
     "node_modules/wrappy": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
@@ -2380,6 +2676,70 @@
         }
       }
     },
+    "@isaacs/cliui": {
+      "version": "8.0.2",
+      "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz",
+      "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==",
+      "requires": {
+        "string-width": "^5.1.2",
+        "string-width-cjs": "npm:string-width@^4.2.0",
+        "strip-ansi": "^7.0.1",
+        "strip-ansi-cjs": "npm:strip-ansi@^6.0.1",
+        "wrap-ansi": "^8.1.0",
+        "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0"
+      },
+      "dependencies": {
+        "ansi-regex": {
+          "version": "6.0.1",
+          "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz",
+          "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA=="
+        },
+        "ansi-styles": {
+          "version": "6.2.1",
+          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz",
+          "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug=="
+        },
+        "emoji-regex": {
+          "version": "9.2.2",
+          "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz",
+          "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg=="
+        },
+        "string-width": {
+          "version": "5.1.2",
+          "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz",
+          "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==",
+          "requires": {
+            "eastasianwidth": "^0.2.0",
+            "emoji-regex": "^9.2.2",
+            "strip-ansi": "^7.0.1"
+          }
+        },
+        "strip-ansi": {
+          "version": "7.0.1",
+          "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.0.1.tgz",
+          "integrity": "sha512-cXNxvT8dFNRVfhVME3JAe98mkXDYN2O1l7jmcwMnOslDeESg1rF/OZMtK0nRAhiari1unG5cD4jG3rapUAkLbw==",
+          "requires": {
+            "ansi-regex": "^6.0.1"
+          }
+        },
+        "wrap-ansi": {
+          "version": "8.1.0",
+          "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz",
+          "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==",
+          "requires": {
+            "ansi-styles": "^6.1.0",
+            "string-width": "^5.0.1",
+            "strip-ansi": "^7.0.1"
+          }
+        }
+      }
+    },
+    "@pkgjs/parseargs": {
+      "version": "0.11.0",
+      "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz",
+      "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==",
+      "optional": true
+    },
     "@puppeteer/browsers": {
       "version": "1.3.0",
       "resolved": "https://registry.npmjs.org/@puppeteer/browsers/-/browsers-1.3.0.tgz",
@@ -2447,11 +2807,6 @@
         "@types/node": "*"
       }
     },
-    "abortcontroller-polyfill": {
-      "version": "1.7.5",
-      "resolved": "https://registry.npmjs.org/abortcontroller-polyfill/-/abortcontroller-polyfill-1.7.5.tgz",
-      "integrity": "sha512-JMJ5soJWP18htbbxJjG7bG6yuI6pRhgJ0scHHTfkUjf6wjP912xZWvM+A4sJK3gqd9E8fcPbDnOefbA9Th/FIQ=="
-    },
     "agent-base": {
       "version": "6.0.2",
       "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz",
@@ -2549,6 +2904,7 @@
       "version": "1.1.11",
       "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
       "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
+      "dev": true,
       "requires": {
         "balanced-match": "^1.0.0",
         "concat-map": "0.0.1"
@@ -2583,6 +2939,15 @@
       "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz",
       "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ=="
     },
+    "busboy": {
+      "version": "1.6.0",
+      "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz",
+      "integrity": "sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==",
+      "dev": true,
+      "requires": {
+        "streamsearch": "^1.1.0"
+      }
+    },
     "callsites": {
       "version": "3.1.0",
       "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz",
@@ -2703,7 +3068,8 @@
     "concat-map": {
       "version": "0.0.1",
       "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
-      "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s="
+      "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=",
+      "dev": true
     },
     "cosmiconfig": {
       "version": "8.1.3",
@@ -2724,6 +3090,16 @@
         "node-fetch": "^2.6.11"
       }
     },
+    "cross-spawn": {
+      "version": "7.0.3",
+      "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
+      "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==",
+      "requires": {
+        "path-key": "^3.1.0",
+        "shebang-command": "^2.0.0",
+        "which": "^2.0.1"
+      }
+    },
     "debug": {
       "version": "4.3.4",
       "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
@@ -2777,6 +3153,11 @@
       "integrity": "sha512-/VTCrvm5Z0JGty/BWHljh+BAiw3IK+2j87NGMu8Nwc/f48WoDAC395uomO9ZD117ZOBaHmkX1oyLvkVM/aIT3w==",
       "dev": true
     },
+    "eastasianwidth": {
+      "version": "0.2.0",
+      "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz",
+      "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA=="
+    },
     "ee-first": {
       "version": "1.1.1",
       "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz",
@@ -2857,19 +3238,18 @@
       }
     },
     "fetch-filecache-for-crawling": {
-      "version": "4.1.0",
-      "resolved": "https://registry.npmjs.org/fetch-filecache-for-crawling/-/fetch-filecache-for-crawling-4.1.0.tgz",
-      "integrity": "sha512-z7f5oCxQaDEST+4HqdPj7w0hX41hWXw36N9I9SQ5wVEWMy01Oz6ZRSKl0lQQ/zVpwrHp/inFXd8ojLojXnMDzQ==",
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/fetch-filecache-for-crawling/-/fetch-filecache-for-crawling-5.0.0.tgz",
+      "integrity": "sha512-+zIz6ARXTR8C+maoDJZTS8QgzVH3WWqBpngMeV2I+jm5z89XK/zmaPie18vli7z/zGORvUQAYbTtzfBMy16u4A==",
       "requires": {
         "filenamify-url": "^2.1.2",
-        "node-fetch": "^2.6.0",
-        "rimraf": "^3.0.2"
+        "rimraf": "^5.0.1"
       }
     },
     "filename-reserved-regex": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/filename-reserved-regex/-/filename-reserved-regex-2.0.0.tgz",
-      "integrity": "sha1-q/c9+rc10EVECr/qLZHzieu/oik="
+      "integrity": "sha512-lc1bnsSr4L4Bdif8Xb/qrtokGbq5zlsms/CYH8PP+WtCkGNF65DPiQY8vG3SakEdRn8Dlnm+gW/qWKKjS5sZzQ=="
     },
     "filenamify": {
       "version": "4.3.0",
@@ -2937,6 +3317,15 @@
       "integrity": "sha512-b6suED+5/3rTpUBdG1gupIl8MPFCAMA0QXwmljLhvCUKcUvdE4gWky9zpuGCcXHOsz4J9wPGNWq6OKpmIzz3hQ==",
       "dev": true
     },
+    "foreground-child": {
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.1.1.tgz",
+      "integrity": "sha512-TMKDUnIte6bfb5nWv7V/caI169OHgvwjb7V4WkeUvbQQdjr5rWKqHFiKWb/fcOwB+CzBT+qbWjvj+DVwRskpIg==",
+      "requires": {
+        "cross-spawn": "^7.0.0",
+        "signal-exit": "^4.0.1"
+      }
+    },
     "fresh": {
       "version": "0.5.2",
       "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz",
@@ -2951,7 +3340,8 @@
     "fs.realpath": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
-      "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8="
+      "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=",
+      "dev": true
     },
     "fsevents": {
       "version": "2.3.2",
@@ -2980,16 +3370,15 @@
       }
     },
     "glob": {
-      "version": "7.1.4",
-      "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.4.tgz",
-      "integrity": "sha512-hkLPepehmnKk41pUGm3sYxoFs/umurYfYJCerbXEyFIWcAzvpipAgVkBqqT9RBKMGjnq6kMuyYwha6csxbiM1A==",
+      "version": "10.2.6",
+      "resolved": "https://registry.npmjs.org/glob/-/glob-10.2.6.tgz",
+      "integrity": "sha512-U/rnDpXJGF414QQQZv5uVsabTVxMSwzS5CH0p3DRCIV6ownl4f7PzGnkGmvlum2wB+9RlJWJZ6ACU1INnBqiPA==",
       "requires": {
-        "fs.realpath": "^1.0.0",
-        "inflight": "^1.0.4",
-        "inherits": "2",
-        "minimatch": "^3.0.4",
-        "once": "^1.3.0",
-        "path-is-absolute": "^1.0.0"
+        "foreground-child": "^3.1.0",
+        "jackspeak": "^2.0.3",
+        "minimatch": "^9.0.1",
+        "minipass": "^5.0.0 || ^6.0.2",
+        "path-scurry": "^1.7.0"
       }
     },
     "glob-parent": {
@@ -3071,6 +3460,7 @@
       "version": "1.0.6",
       "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
       "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=",
+      "dev": true,
       "requires": {
         "once": "^1.3.0",
         "wrappy": "1"
@@ -3127,6 +3517,20 @@
       "integrity": "sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw==",
       "dev": true
     },
+    "isexe": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
+      "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw=="
+    },
+    "jackspeak": {
+      "version": "2.2.1",
+      "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-2.2.1.tgz",
+      "integrity": "sha512-MXbxovZ/Pm42f6cDIDkl3xpwv1AGwObKwfmjs2nQePiy85tP3fatofl3FC1aBsOtP/6fq5SbtgHwWcMsLP+bDw==",
+      "requires": {
+        "@isaacs/cliui": "^8.0.2",
+        "@pkgjs/parseargs": "^0.11.0"
+      }
+    },
     "js-tokens": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",
@@ -3150,23 +3554,11 @@
       "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz",
       "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug=="
     },
-    "json-stringify-safe": {
-      "version": "5.0.1",
-      "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz",
-      "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=",
-      "dev": true
-    },
     "lines-and-columns": {
       "version": "1.2.4",
       "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz",
       "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg=="
     },
-    "lodash": {
-      "version": "4.17.21",
-      "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz",
-      "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==",
-      "dev": true
-    },
     "log-symbols": {
       "version": "4.1.0",
       "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-4.1.0.tgz",
@@ -3207,13 +3599,28 @@
       "dev": true
     },
     "minimatch": {
-      "version": "3.1.2",
-      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
-      "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
+      "version": "9.0.1",
+      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.1.tgz",
+      "integrity": "sha512-0jWhJpD/MdhPXwPuiRkCbfYfSKp2qnn2eOc279qI7f+osl/l+prKSrvhg157zSYvx/1nmgn2NqdT6k2Z7zSH9w==",
       "requires": {
-        "brace-expansion": "^1.1.7"
+        "brace-expansion": "^2.0.1"
+      },
+      "dependencies": {
+        "brace-expansion": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
+          "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
+          "requires": {
+            "balanced-match": "^1.0.0"
+          }
+        }
       }
     },
+    "minipass": {
+      "version": "6.0.2",
+      "resolved": "https://registry.npmjs.org/minipass/-/minipass-6.0.2.tgz",
+      "integrity": "sha512-MzWSV5nYVT7mVyWCwn2o7JH13w2TBRmmSqSRCKzTw+lmft9X4z+3wjvs06Tzijo5z4W/kahUCDpRXTF+ZrmF/w=="
+    },
     "mitt": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/mitt/-/mitt-3.0.0.tgz",
@@ -3361,18 +3768,6 @@
       "integrity": "sha512-p1sjXuopFs0xg+fPASzQ28agW1oHD7xDsd9Xkf3T15H3c/cifrFHVwrh74PdoklAPi+i7MdRsE47vm2r6JoB+w==",
       "dev": true
     },
-    "nock": {
-      "version": "13.3.1",
-      "resolved": "https://registry.npmjs.org/nock/-/nock-13.3.1.tgz",
-      "integrity": "sha512-vHnopocZuI93p2ccivFyGuUfzjq2fxNyNurp7816mlT5V5HF4SzXu8lvLrVzBbNqzs+ODooZ6OksuSUNM7Njkw==",
-      "dev": true,
-      "requires": {
-        "debug": "^4.1.0",
-        "json-stringify-safe": "^5.0.1",
-        "lodash": "^4.17.21",
-        "propagate": "^2.0.0"
-      }
-    },
     "node-fetch": {
       "version": "2.6.11",
       "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.11.tgz",
@@ -3464,7 +3859,29 @@
     "path-is-absolute": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
-      "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18="
+      "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=",
+      "dev": true
+    },
+    "path-key": {
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz",
+      "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q=="
+    },
+    "path-scurry": {
+      "version": "1.9.2",
+      "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.9.2.tgz",
+      "integrity": "sha512-qSDLy2aGFPm8i4rsbHd4MNyTcrzHFsLQykrtbuGRknZZCBBVXSv2tSCDN2Cg6Rt/GFRw8GoW9y9Ecw5rIPG1sg==",
+      "requires": {
+        "lru-cache": "^9.1.1",
+        "minipass": "^5.0.0 || ^6.0.2"
+      },
+      "dependencies": {
+        "lru-cache": {
+          "version": "9.1.1",
+          "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-9.1.1.tgz",
+          "integrity": "sha512-65/Jky17UwSb0BuB9V+MyDpsOtXKmYwzhyl+cOa9XUiI4uV2Ouy/2voFP3+al0BjZbJgMBD8FojMpAf+Z+qn4A=="
+        }
+      }
     },
     "path-type": {
       "version": "4.0.0",
@@ -3493,12 +3910,6 @@
       "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz",
       "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA=="
     },
-    "propagate": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/propagate/-/propagate-2.0.1.tgz",
-      "integrity": "sha512-vGrhOavPSTz4QVNuBNdcNXePNdNMaO1xj9yBeH1ScQPjk/rhg9sSlCXPhMkFuaNNW/syTvYqsnbIJxMBfRbbag==",
-      "dev": true
-    },
     "proxy-from-env": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz",
@@ -3611,11 +4022,11 @@
       "dev": true
     },
     "rimraf": {
-      "version": "3.0.2",
-      "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz",
-      "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==",
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-5.0.1.tgz",
+      "integrity": "sha512-OfFZdwtd3lZ+XZzYP/6gTACubwFcHdLRqS9UX3UwpU2dnGQYkPFISRwvM3w9IiB2w7bW5qGo/uAwE4SmXXSKvg==",
       "requires": {
-        "glob": "^7.1.3"
+        "glob": "^10.2.5"
       }
     },
     "rollup": {
@@ -3716,12 +4127,36 @@
       "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==",
       "dev": true
     },
+    "shebang-command": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz",
+      "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==",
+      "requires": {
+        "shebang-regex": "^3.0.0"
+      }
+    },
+    "shebang-regex": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz",
+      "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A=="
+    },
+    "signal-exit": {
+      "version": "4.0.2",
+      "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.0.2.tgz",
+      "integrity": "sha512-MY2/qGx4enyjprQnFaZsHib3Yadh3IXyV2C321GY0pjGfVBu4un0uDJkwgdxqO+Rdx8JMT8IfJIRwbYVz3Ob3Q=="
+    },
     "statuses": {
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz",
       "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==",
       "dev": true
     },
+    "streamsearch": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz",
+      "integrity": "sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==",
+      "dev": true
+    },
     "string_decoder": {
       "version": "1.3.0",
       "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz",
@@ -3740,6 +4175,16 @@
         "strip-ansi": "^6.0.1"
       }
     },
+    "string-width-cjs": {
+      "version": "npm:string-width@4.2.3",
+      "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz",
+      "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==",
+      "requires": {
+        "emoji-regex": "^8.0.0",
+        "is-fullwidth-code-point": "^3.0.0",
+        "strip-ansi": "^6.0.1"
+      }
+    },
     "strip-ansi": {
       "version": "6.0.1",
       "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz",
@@ -3748,6 +4193,14 @@
         "ansi-regex": "^5.0.1"
       }
     },
+    "strip-ansi-cjs": {
+      "version": "npm:strip-ansi@6.0.1",
+      "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz",
+      "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==",
+      "requires": {
+        "ansi-regex": "^5.0.1"
+      }
+    },
     "strip-json-comments": {
       "version": "3.1.1",
       "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz",
@@ -3817,7 +4270,7 @@
     "trim-repeated": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/trim-repeated/-/trim-repeated-1.0.0.tgz",
-      "integrity": "sha1-42RqLqTokTEr9+rObPsFOAvAHCE=",
+      "integrity": "sha512-pkonvlKk8/ZuR0D5tLW8ljt5I8kmxp2XKymhepUeOdCEfKpZaktSArkLHZt76OB1ZvO9bssUsDty4SWhLvZpLg==",
       "requires": {
         "escape-string-regexp": "^1.0.2"
       }
@@ -3837,6 +4290,15 @@
         "through": "^2.3.8"
       }
     },
+    "undici": {
+      "version": "5.22.1",
+      "resolved": "https://registry.npmjs.org/undici/-/undici-5.22.1.tgz",
+      "integrity": "sha512-Ji2IJhFXZY0x/0tVBXeQwgPlLWw13GVzpsWPQ3rV50IFMMof2I55PZZxtm4P6iNq+L5znYN9nSTAq0ZyE6lSJw==",
+      "dev": true,
+      "requires": {
+        "busboy": "^1.6.0"
+      }
+    },
     "unpipe": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz",
@@ -3866,6 +4328,14 @@
       "resolved": "https://registry.npmjs.org/webidl2/-/webidl2-24.3.0.tgz",
       "integrity": "sha512-u42I31V2xQSBPCLtjFdk98cb06gTVnfPmq8eNDmcM3iaqtDdvY+GF5ZLPg3hEL/h6K50fsgPt+2YXFysSq8wcw=="
     },
+    "which": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
+      "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==",
+      "requires": {
+        "isexe": "^2.0.0"
+      }
+    },
     "workerpool": {
       "version": "6.2.1",
       "resolved": "https://registry.npmjs.org/workerpool/-/workerpool-6.2.1.tgz",
@@ -3882,6 +4352,16 @@
         "strip-ansi": "^6.0.0"
       }
     },
+    "wrap-ansi-cjs": {
+      "version": "npm:wrap-ansi@7.0.0",
+      "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz",
+      "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==",
+      "requires": {
+        "ansi-styles": "^4.0.0",
+        "string-width": "^4.1.0",
+        "strip-ansi": "^6.0.0"
+      }
+    },
     "wrappy": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
diff --git a/package.json b/package.json
index f9ae8642..b777923b 100644
--- a/package.json
+++ b/package.json
@@ -32,11 +32,10 @@
   "main": "index.js",
   "bin": "./reffy.js",
   "dependencies": {
-    "abortcontroller-polyfill": "1.7.5",
     "ajv": "8.12.0",
     "ajv-formats": "2.1.1",
     "commander": "10.0.1",
-    "fetch-filecache-for-crawling": "4.1.0",
+    "fetch-filecache-for-crawling": "5.0.0",
     "puppeteer": "20.3.0",
     "semver": "^7.3.5",
     "web-specs": "2.59.0",
@@ -45,10 +44,10 @@
   "devDependencies": {
     "chai": "4.3.7",
     "mocha": "10.2.0",
-    "nock": "13.3.1",
     "respec": "34.1.1",
     "respec-hljs": "2.1.1",
-    "rollup": "3.23.0"
+    "rollup": "3.23.0",
+    "undici": "^5.22.1"
   },
   "scripts": {
     "test": "mocha --recursive tests/"
diff --git a/src/lib/mock-server.js b/src/lib/mock-server.js
new file mode 100644
index 00000000..50600b33
--- /dev/null
+++ b/src/lib/mock-server.js
@@ -0,0 +1,186 @@
+/**
+ * Setup a proxy server that intercepts some network requests. To be used in
+ * tests not to hit the network.
+ *
+ * @module mock-server
+ */
+
+const { MockAgent, setGlobalDispatcher } = require('undici');
+const path = require("path");
+const { existsSync, readFileSync } = require('fs');
+
+/**
+ * Determine the path to the "node_modules" folder. The path depends on whether
+ * Reffy is run directly, or installed as a library.
+ *
+ * @function
+ * @return {String} Path to the node_modules folder.
+ */
+function getModulesFolder() {
+    const rootFolder = path.resolve(__dirname, '../..');
+    let folder = path.resolve(rootFolder, 'node_modules');
+    if (existsSync(folder)) {
+        return folder;
+    }
+    folder = path.resolve(rootFolder, '..');
+    return folder;
+}
+const modulesFolder = getModulesFolder();
+
+const mockSpecs = {
+  "/woff/woff2/": {
+    html: `
+      <title>WOFF2</title>
+      <body>
+        <dfn id='foo' data-dfn-type="dfn">Foo</dfn>
+        <a href="https://www.w3.org/TR/bar/#baz">bar</a>
+        <ul class='toc'><li><a href='page.html'>page</a></ul>`,
+    pages: {
+      "page.html": `<h2 id='bar'>Heading in subpage</h2>`
+    }
+  },
+  "/mediacapture-output/": `
+    <script>respecConfig = { shortName: 'test' };</script>
+    <script src='https://www.w3.org/Tools/respec/respec-w3c'></script>
+    <div id=abstract></div>
+    <pre class='idl'>[Exposed=Window] interface Foo { attribute DOMString bar; };</pre>`,
+  "/accelerometer/": `<html>
+    <h2>Normative references</h2>
+    <dl>
+      <dt>FOO</dt>
+      <dd><a href='https://www.w3.org/TR/Foo'>Foo</a></dd>
+    </dl>`,
+  "/pointerlock/": `<html>
+    <h1>Pointer Lock 2.0`,
+  "/TR/remote-playback/": {
+    html: `<title>Published version</title>
+      <body><h1>Published version</h1></body>`,
+    domain: 'https://www.w3.org'
+  }
+};
+
+const respecHiglight = readFileSync(
+  path.join(modulesFolder, "respec-hljs", "dist", "respec-highlight.js"),
+  'utf8'
+);
+const respecW3C = readFileSync(
+  path.join(modulesFolder, "respec", "builds", "respec-w3c.js"),
+  'utf8'
+);
+
+const mockAgent = new MockAgent();
+setGlobalDispatcher(mockAgent);
+mockAgent.disableNetConnect();
+// for chrome devtool protocol
+mockAgent.enableNetConnect('127.0.0.1');
+
+for (const [path, desc] of Object.entries(mockSpecs)) {
+  mockAgent.get(desc.domain || "https://w3c.github.io")
+    .intercept({ method: "GET", path })
+    .reply(200, desc.html || desc, {
+      headers: { "Content-Type": "text/html" }
+    })
+    .persist();
+
+  for (const [page, pageContent] of Object.entries(desc.pages || {})) {
+    mockAgent.get(desc.domain || "https://w3c.github.io")
+      .intercept({ method: "GET", path: path + page })
+      .reply(200, pageContent, {
+        headers: { "Content-Type": "text/html" }
+      })
+      .persist();
+  }
+}
+
+
+// Handling requests generated by ReSpec documents
+mockAgent
+  .get("https://api.specref.org")
+  .intercept({ method: "GET", path: "/bibrefs?refs=webidl" })
+  .reply(200, { webidl: { href: "https://webidl.spec.whatwg.org/" } }, {
+    headers: {
+      "Content-Type": "application/json",
+      "Access-Control-Allow-Origin": "*"
+    }
+  })
+  .persist();
+
+mockAgent
+  .get("https://www.w3.org")
+  .intercept({ method: "GET", path: "/scripts/TR/2021/fixup.js" })
+  .reply(200, '')
+  .persist();
+
+mockAgent
+  .get("https://www.w3.org")
+  .intercept({ method: "GET", path: "/StyleSheets/TR/2021/logos/W3C" })
+  .reply(200, '')
+  .persist();
+
+mockAgent
+  .get("https://www.w3.org")
+  .intercept({ method: "GET", path: "/StyleSheets/TR/2021/base.css" })
+  .reply(200, '')
+  .persist();
+
+mockAgent
+  .get("https://www.w3.org")
+  .intercept({ method: "GET", path: "/Tools/respec/respec-highlight" })
+  .reply(200, respecHiglight, {
+    headers: { "Content-Type": "application/js" }
+  })
+  .persist();
+
+mockAgent
+  .get("https://www.w3.org")
+  .intercept({ method: "GET", path: "/Tools/respec/respec-w3c" })
+  .reply(200, respecW3C, {
+    headers: { "Content-Type": "application/js" }
+  })
+  .persist();
+
+mockAgent
+  .get("https://www.w3.org")
+  .intercept({ method: "GET", path: "/TR/idontexist/" })
+  .reply(404, '');
+
+mockAgent
+  .get("https://www.w3.org")
+  .intercept({ method: "GET", path: "/TR/ididnotchange/" })
+  .reply(({ headers }) => {
+    // NB: the headers parameters is not an instance of Headers as suggested in
+    // examples, e.g.:
+    // https://github.com/nodejs/undici/blob/main/docs/api/MockPool.md#example---mocked-request-using-reply-options-callback
+    // It is rather an array that alternates header names and header values.
+    const pos = headers.findIndex(h => h === 'If-Modified-Since');
+    if (pos === -1) {
+      return { statusCode: 200, data: 'Unexpected If-Modified-Since header' };
+    }
+    const value = headers[pos+1];
+    if (value === "Fri, 11 Feb 2022 00:00:42 GMT") {
+      return { statusCode: 304 };
+    } else {
+      return { statusCode: 200, data: 'Unexpected If-Modified-Since header' };
+    }
+  });
+
+mockAgent
+  .get("https://drafts.csswg.org")
+  .intercept({ method: "GET", path: "/server-hiccup/" })
+  .reply(200,
+    `<html><title>Server hiccup</title>
+    <h1> Index of Server Hiccup Module Level 42 </h1>`,
+    { header: { "Content-Type": "text/html" } })
+  .persist();
+
+/*nock.emitter.on('error', function (err) {
+  console.error(err);
+});
+nock.emitter.on('no match', function(req, options, requestBody) {
+  // 127.0.0.1 is used by the devtool protocol, we ignore it
+  if (req && req.hostname !== '127.0.0.1') {
+    console.error("No match for nock request on " + (options ? options.href : req.href));
+  }
+});*/
+
+module.exports = mockAgent;
diff --git a/src/lib/nock-server.js b/src/lib/nock-server.js
deleted file mode 100644
index a3af5ae4..00000000
--- a/src/lib/nock-server.js
+++ /dev/null
@@ -1,126 +0,0 @@
-/**
- * Setup a proxy server that intercepts some network requests. To be used in
- * tests not to hit the network.
- *
- * @module nock-server
- */
-
-const nock = require("nock");
-const path = require("path");
-const { existsSync } = require('fs');
-
-/**
- * Determine the path to the "node_modules" folder. The path depends on whether
- * Reffy is run directly, or installed as a library.
- *
- * @function
- * @return {String} Path to the node_modules folder.
- */
-function getModulesFolder() {
-    const rootFolder = path.resolve(__dirname, '../..');
-    let folder = path.resolve(rootFolder, 'node_modules');
-    if (existsSync(folder)) {
-        return folder;
-    }
-    folder = path.resolve(rootFolder, '..');
-    return folder;
-}
-const modulesFolder = getModulesFolder();
-
-const mockSpecs = {
-  "/woff/woff2/": {
-    html: `
-      <title>WOFF2</title>
-      <body>
-        <dfn id='foo' data-dfn-type="dfn">Foo</dfn>
-        <a href="https://www.w3.org/TR/bar/#baz">bar</a>
-        <ul class='toc'><li><a href='page.html'>page</a></ul>`,
-    pages: {
-      "page.html": `<h2 id='bar'>Heading in subpage</h2>`
-    }
-  },
-  "/mediacapture-output/": `
-    <script>respecConfig = { shortName: 'test' };</script>
-    <script src='https://www.w3.org/Tools/respec/respec-w3c'></script>
-    <div id=abstract></div>
-    <pre class='idl'>[Exposed=Window] interface Foo { attribute DOMString bar; };</pre>`,
-  "/accelerometer/": `<html>
-    <h2>Normative references</h2>
-    <dl>
-      <dt>FOO</dt>
-      <dd><a href='https://www.w3.org/TR/Foo'>Foo</a></dd>
-    </dl>`,
-  "/pointerlock/": `<html>
-    <h1>Pointer Lock 2.0`,
-  "/TR/remote-playback/": {
-    html: `<title>Published version</title>
-      <body><h1>Published version</h1></body>`,
-    domain: 'https://www.w3.org'
-  }
-};
-
-nock.disableNetConnect();
-// for chrome devtool protocol
-nock.enableNetConnect('127.0.0.1');
-
-for (const [path, desc] of Object.entries(mockSpecs)) {
-  nock(desc.domain || "https://w3c.github.io")
-    .persist()
-    .get(path)
-    .reply(200, desc.html || desc, { 'Content-Type': 'text/html' });
-
-  for (const [page, pageContent] of Object.entries(desc.pages || {})) {
-    nock("https://w3c.github.io")
-      .persist()
-      .get(path + page)
-      .reply(200, pageContent, { 'Content-Type': 'text/html' });
-  }
-}
-
-
-// Handling requests generated by ReSpec documents
-nock("https://api.specref.org")
-  .persist()
-  .get("/bibrefs?refs=webidl").reply(200,
-    { webidl: { href: "https://webidl.spec.whatwg.org/" } },
-    { "Access-Control-Allow-Origin": "*" }
-  );
-
-nock("https://www.w3.org")
-  .persist()
-  .get("/scripts/TR/2021/fixup.js").reply(200, '')
-  .get("/StyleSheets/TR/2021/logos/W3C").reply(200, '')
-  .get("/StyleSheets/TR/2021/base.css").reply(200, '')
-  .get("/Tools/respec/respec-highlight").replyWithFile(200,
-    path.join(modulesFolder, "respec-hljs", "dist", "respec-highlight.js"),
-    { "Content-Type": "application/js" })
-  .get("/Tools/respec/respec-w3c").replyWithFile(200,
-    path.join(modulesFolder, "respec", "builds", "respec-w3c.js"),
-    { "Content-Type": "application/js" })
-  .get("/TR/idontexist/").reply(404, '')
-  .get("/TR/ididnotchange/").reply(function() {
-    if (this.req.headers['if-modified-since'][0] === "Fri, 11 Feb 2022 00:00:42 GMT") {
-      return [304, ''];
-    } else {
-      return [200, 'Unexpected path'];
-    }
-  });
-
-nock("https://drafts.csswg.org")
-  .persist()
-  .get("/server-hiccup/").reply(200,
-    `<html><title>Server hiccup</title>
-    <h1> Index of Server Hiccup Module Level 42 </h1>`,
-    { 'Content-Type': 'text/html' });
-
-nock.emitter.on('error', function (err) {
-  console.error(err);
-});
-nock.emitter.on('no match', function(req, options, requestBody) {
-  // 127.0.0.1 is used by the devtool protocol, we ignore it
-  if (req && req.hostname !== '127.0.0.1') {
-    console.error("No match for nock request on " + (options ? options.href : req.href));
-  }
-});
-
-module.exports = nock;
diff --git a/src/lib/util.js b/src/lib/util.js
index 9c35fbec..feb2871b 100644
--- a/src/lib/util.js
+++ b/src/lib/util.js
@@ -7,7 +7,7 @@ const { existsSync, readdirSync } = require('fs');
 const path = require('path');
 const puppeteer = require('puppeteer');
 const crypto = require('crypto');
-const { AbortController } = require('abortcontroller-polyfill/dist/cjs-ponyfill');
+const { Buffer } = require('buffer');
 const Ajv = require('ajv');
 const addFormats = require('ajv-formats');
 const commonSchema = require('../../schemas/common.json');
@@ -413,19 +413,19 @@ async function processSpecification(spec, processFunction, args, options) {
                         await cdp.send('Fetch.continueRequest', { requestId });
                         return;
                     }
-                    const response = prefetchedResponse[request.url] ?? await fetch(request.url, { signal: controller.signal, headers: request.headers });
+                    const response = prefetchedResponse[request.url] ??
+                        await fetch(request.url, { signal: controller.signal, headers: request.headers });
+                    const body = Buffer.from(await response.arrayBuffer());
 
-                    const body = await response.buffer();
+                    const headers = [];
+                    response.headers.forEach((value, name) => {
+                        headers.push({ name, value });
+                    });
 
                     await cdp.send('Fetch.fulfillRequest', {
                         requestId,
                         responseCode: response.status,
-                        responseHeaders: Object.keys(response.headers.raw()).map(header => {
-                          return {
-                            name: header,
-                            value: response.headers.raw()[header].join(',')
-                          };
-                        }),
+                        responseHeaders: headers,
                         body: body.toString('base64')
                     });
                 }
diff --git a/tests/crawl.js b/tests/crawl.js
index 9b0773e3..74e63fd4 100644
--- a/tests/crawl.js
+++ b/tests/crawl.js
@@ -1,5 +1,5 @@
 const { crawlSpecs } = require("../src/lib/specs-crawler");
-const nock = require('../src/lib/nock-server');
+const mockServer = require('../src/lib/mock-server');
 const fs = require("fs");
 const path = require("path");
 const os = require("os");
@@ -179,8 +179,8 @@ if (global.describe && describe instanceof Function) {
     });
 
     after(() => {
-      if (!nock.isDone()) {
-        throw new Error("Additional network requests expected: " + nock.pendingMocks());
+      if (mockServer.pendingInterceptors().length > 0) {
+        throw new Error("Additional network requests expected on:\n- " + mockServer.pendingInterceptors().map(miss => miss.origin + miss.path).join('\n- '));
       }
     });
   });
diff --git a/tests/reffy-package.js b/tests/reffy-package.js
index ee592bc1..32346cae 100644
--- a/tests/reffy-package.js
+++ b/tests/reffy-package.js
@@ -4,7 +4,7 @@ const fs = require("fs");
 const path = require("path");
 const util = require('util');
 const exec = util.promisify(require('child_process').exec);
-const nock = require('../src/lib/nock-server');
+const mock = require('../src/lib/mock-server');
 
 const specs = [
   {url: "https://www.w3.org/TR/WOFF2/", nightly: {url: "https://w3c.github.io/woff/woff2/", pages:["https://w3c.github.io/woff/woff2/page.html"]}},