From 9187f82bfb360e987dfa8648919c1534291c0cd5 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Wed, 19 Jul 2023 18:12:42 +0100 Subject: [PATCH 01/11] fix: this param for fetch calls --- .github/workflows/client.yml | 12 +++- packages/api/package.json | 2 +- packages/client/package.json | 2 +- packages/client/src/lib.js | 17 ++++- packages/client/test/lib.spec.js | 2 +- packages/client/test/mock-server.js | 27 ++++--- yarn.lock | 108 +++++++++++++++------------- 7 files changed, 96 insertions(+), 74 deletions(-) diff --git a/.github/workflows/client.yml b/.github/workflows/client.yml index e7bcd8680f..a2ffe38be8 100644 --- a/.github/workflows/client.yml +++ b/.github/workflows/client.yml @@ -23,7 +23,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-node@v2 with: - node-version: '16' + node-version: 18 - uses: bahmutov/npm-install@v1 - name: Typecheck uses: gozala/typescript-error-reporter-action@v1.0.8 @@ -33,11 +33,17 @@ jobs: test: name: Test runs-on: ubuntu-latest + strategy: + matrix: + node_version: + - 16 + - 18 + - 20 steps: - uses: actions/checkout@v2 - uses: actions/setup-node@v2 with: - node-version: '16' + node-version: { { matrix.node_version } } - uses: bahmutov/npm-install@v1 - name: Test (ES) run: yarn --cwd packages/client test:es @@ -69,7 +75,7 @@ jobs: - uses: actions/setup-node@v2 if: ${{ steps.tag-release.outputs.releases_created }} with: - node-version: '16' + node-version: 18 registry-url: https://registry.npmjs.org/ - uses: bahmutov/npm-install@v1 if: ${{ steps.tag-release.outputs.releases_created }} diff --git a/packages/api/package.json b/packages/api/package.json index 536453b397..ed14c58fcc 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -62,7 +62,7 @@ "git-rev-sync": "^3.0.1", "ipfs-unixfs-importer": "^9.0.3", "miniflare": "^2.10.0", - "minio": "^7.0.28", + "minio": "^7.1.1", "npm-run-all": "^4.1.5", "openapi-typescript": "^4.0.2", "pg": "^8.7.1", diff --git a/packages/client/package.json b/packages/client/package.json index f912e06e56..e28e346613 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -57,7 +57,7 @@ "@ipld/car": "^3.2.3", "@ipld/dag-cbor": "^6.0.13", "@web-std/blob": "^3.0.1", - "@web-std/fetch": "^3.0.3", + "@web-std/fetch": "^4.1.2", "@web-std/file": "^3.0.0", "@web-std/form-data": "^3.0.0", "carbites": "^1.0.6", diff --git a/packages/client/src/lib.js b/packages/client/src/lib.js index e04a3b881e..842fb12104 100644 --- a/packages/client/src/lib.js +++ b/packages/client/src/lib.js @@ -172,7 +172,10 @@ class NFTStorage { { onStoredChunk, maxRetries, maxChunkSize, decoders, signal } = {} ) { const url = new URL('upload/', endpoint) - const headers = NFTStorage.auth(token) + const headers = { + ...NFTStorage.auth(token), + 'Content-Type': 'application/car', + } const targetSize = maxChunkSize || MAX_CHUNK_SIZE const splitter = car instanceof Blob @@ -187,6 +190,15 @@ class NFTStorage { carParts.push(part) } const carFile = new Blob(carParts, { type: 'application/car' }) + /** @type {Blob|ArrayBuffer} */ + let body = carFile + // FIXME: should not be necessary to await arrayBuffer()! + // Node.js 20 hangs reading the stream (it never ends) but in + // older node versions and the browser it is fine to pass a blob. + /* c8 ignore next 3 */ + if (parseInt(globalThis.process?.versions?.node) > 18) { + body = await body.arrayBuffer() + } const cid = await pRetry( async () => { await rateLimiter() @@ -196,10 +208,11 @@ class NFTStorage { response = await fetch(url.toString(), { method: 'POST', headers, - body: carFile, + body, signal, }) } catch (/** @type {any} */ err) { + console.log(err) // TODO: remove me and test when client accepts custom fetch impl /* c8 ignore next 1 */ throw signal && signal.aborted ? new AbortError(err) : err diff --git a/packages/client/test/lib.spec.js b/packages/client/test/lib.spec.js index ecfa1759a5..42ad2eb968 100644 --- a/packages/client/test/lib.spec.js +++ b/packages/client/test/lib.spec.js @@ -173,7 +173,7 @@ describe('client', () => { it('upload large CAR with a CarReader', async function () { // @ts-ignore - this.timeout(130e3) + this.timeout(360e3) let uploadedChunks = 0 const client = new NFTStorage({ token, endpoint }) diff --git a/packages/client/test/mock-server.js b/packages/client/test/mock-server.js index 784a97ee6a..e729dc5bf2 100644 --- a/packages/client/test/mock-server.js +++ b/packages/client/test/mock-server.js @@ -21,21 +21,15 @@ export { fetch, Headers } const toReadableStream = (source) => new ReadableStream({ async pull(controller) { - try { - while (controller.desiredSize || 0 > 0) { - const chunk = await source.next() - if (chunk.done) { - controller.close() - } else { - const bytes = - typeof chunk.value === 'string' - ? encoder.encode(chunk.value) - : chunk.value - controller.enqueue(bytes) - } - } - } catch (error) { - controller.error(error) + const chunk = await source.next() + if (chunk.done) { + controller.close() + } else { + const bytes = + typeof chunk.value === 'string' + ? encoder.encode(chunk.value) + : chunk.value + controller.enqueue(bytes) } }, cancel(reason) { @@ -272,6 +266,8 @@ export class Service { // @ts-ignore - headers don't have right type headers: new Headers({ ...incoming.headers }), body: toBody(incoming), + // @ts-expect-error TypeError: RequestInit: duplex option is required when sending a body. + duplex: 'half', }) const response = await this.handler(request, this.state) @@ -285,6 +281,7 @@ export class Service { outgoing.end() } catch (err) { + console.error(err) const error = /**@type {Error & {status: number}} */ (err) if (!outgoing.hasHeader) { outgoing.writeHead(error.status || 500) diff --git a/yarn.lock b/yarn.lock index 3e77d7ff7c..be20d28c2f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5209,6 +5209,18 @@ data-uri-to-buffer "^3.0.1" mrmime "^1.0.0" +"@web-std/fetch@^4.1.2": + version "4.1.2" + resolved "https://registry.yarnpkg.com/@web-std/fetch/-/fetch-4.1.2.tgz#57feb1ae5a004cfeb5102441d35f49caf1dc2a83" + integrity sha512-NUX+nnCTjC6URLtFC2O9dX9FtzCS5nlbF/vZwkPlheq5h6+rQxluH/aO+ORbLjGY4z4iQOulfEGoHcXwx5GFUQ== + dependencies: + "@web-std/blob" "^3.0.3" + "@web-std/form-data" "^3.0.2" + "@web-std/stream" "^1.0.1" + "@web3-storage/multipart-parser" "^1.0.0" + data-uri-to-buffer "^3.0.1" + mrmime "^1.0.0" + "@web-std/file@^3.0.0": version "3.0.2" resolved "https://registry.yarnpkg.com/@web-std/file/-/file-3.0.2.tgz#b84cc9ed754608b18dcf78ac62c40dbcc6a94692" @@ -6054,7 +6066,7 @@ async-each@^1.0.1: resolved "https://registry.yarnpkg.com/async-each/-/async-each-1.0.3.tgz#b727dbf87d7651602f06f4d4ac387f47d91b0cbf" integrity sha512-z/WhQ5FPySLdvREByI2vZiTWwCnF0moMJ1hK9YQwDTHKh6I7/uSckMetoRGb5UBZPC1z0jlw+n/XCgjeH7y1AQ== -async@^3.1.0: +async@^3.2.4: version "3.2.4" resolved "https://registry.yarnpkg.com/async/-/async-3.2.4.tgz#2d22e00f8cddeb5fde5dd33522b56d1cf569a81c" integrity sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ== @@ -6414,7 +6426,7 @@ blob-to-it@^1.0.1: dependencies: browser-readablestream-to-it "^1.0.3" -block-stream2@^2.0.0: +block-stream2@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/block-stream2/-/block-stream2-2.1.0.tgz#ac0c5ef4298b3857796e05be8ebed72196fa054b" integrity sha512-suhjmLI57Ewpmq00qaygS8UgEq2ly2PCItenIyhMqVjo4t4pGzqMvfgJuX8iWTeSDdfSSqS6j38fL4ToNL7Pfg== @@ -6579,10 +6591,10 @@ brorand@^1.0.1, brorand@^1.1.0: resolved "https://registry.yarnpkg.com/brorand/-/brorand-1.1.0.tgz#12c25efe40a45e3c323eb8675a0a0ce57b22371f" integrity sha512-cKV8tMCEpQs4hK/ik71d6LrPOnpkpGBR0wzxqr68g2m/LB2GxVYQroAjMJZRVM1Y4BCjCKc3vAamxSzOY2RP+w== -browser-or-node@^1.3.0: - version "1.3.0" - resolved "https://registry.yarnpkg.com/browser-or-node/-/browser-or-node-1.3.0.tgz#f2a4e8568f60263050a6714b2cc236bb976647a7" - integrity sha512-0F2z/VSnLbmEeBcUrSuDH5l0HxTXdQQzLjkmBR4cYfvg1zJrKSlmIZFqyFR8oX0NrwPhy3c3HQ6i3OxMbew4Tg== +browser-or-node@^2.1.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/browser-or-node/-/browser-or-node-2.1.1.tgz#738790b3a86a8fc020193fa581273fbe65eaea0f" + integrity sha512-8CVjaLJGuSKMVTxJ2DpBl5XnlNDiT4cQFeuCJJrvJmts9YrTZDizTX7PjC2s6W4x+MBGZeEY6dGMrF04/6Hgqg== browser-readablestream-to-it@^1.0.1, browser-readablestream-to-it@^1.0.2, browser-readablestream-to-it@^1.0.3: version "1.0.3" @@ -7841,7 +7853,7 @@ cross-spawn@^7.0.0, cross-spawn@^7.0.2, cross-spawn@^7.0.3: shebang-command "^2.0.0" which "^2.0.1" -crypto-browserify@^3.11.0, crypto-browserify@^3.12.0: +crypto-browserify@^3.11.0: version "3.12.0" resolved "https://registry.yarnpkg.com/crypto-browserify/-/crypto-browserify-3.12.0.tgz#396cf9f3137f03e4b8e532c58f698254e00f80ec" integrity sha512-fz4spIh+znjO2VjL+IdhEpRJ3YN6sMzITSBijk6FK2UvTqruSQW+/cCZTSNsMiZNvUeq0CqurF+dAbyiGOY6Wg== @@ -8085,6 +8097,11 @@ decode-uri-component@^0.2.0: resolved "https://registry.yarnpkg.com/decode-uri-component/-/decode-uri-component-0.2.0.tgz#eb3913333458775cb84cd1a1fae062106bb87545" integrity sha512-hjf+xovcEn31w/EUYdTXQh/8smFL/dzYjohQGEIgjyNavaJfBY2p5F527Bo1VPATxv0VYTUC2bOcXvqFwk78Og== +decode-uri-component@^0.2.2: + version "0.2.2" + resolved "https://registry.yarnpkg.com/decode-uri-component/-/decode-uri-component-0.2.2.tgz#e69dbe25d37941171dd540e024c444cd5188e1e9" + integrity sha512-FqUYQ+8o158GyGTrMFJms9qh3CqTKvAqgqsTnkLI8sKu0028orqBhxNMFkFen0zGyg6epACD32pjVk58ngIErQ== + decompress-response@^3.3.0: version "3.3.0" resolved "https://registry.yarnpkg.com/decompress-response/-/decompress-response-3.3.0.tgz#80a4dd323748384bfa248083622aedec982adff3" @@ -8799,7 +8816,7 @@ es5-shim@^4.5.13: resolved "https://registry.yarnpkg.com/es5-shim/-/es5-shim-4.6.7.tgz#bc67ae0fc3dd520636e0a1601cc73b450ad3e955" integrity sha512-jg21/dmlrNQI7JyyA2w7n+yifSxBng0ZralnSfVZjoCawgNTCnS+yBCyVM9DL5itm7SUnDGgv7hcq2XCZX4iRQ== -es6-error@^4.0.1, es6-error@^4.1.1: +es6-error@^4.0.1: version "4.1.1" resolved "https://registry.yarnpkg.com/es6-error/-/es6-error-4.1.1.tgz#9e3af407459deed47e9a91f9b885a84eb05c561d" integrity sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg== @@ -9976,12 +9993,12 @@ fast-xml-parser@3.19.0: resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-3.19.0.tgz#cb637ec3f3999f51406dd8ff0e6fc4d83e520d01" integrity sha512-4pXwmBplsCPv8FOY1WRakF970TjNGnGnfbOnLqjlYvMiF1SR3yOHyxMR/YCXpPTOspNF5gwudqktIP4VsWkvBg== -fast-xml-parser@^3.17.5: - version "3.21.1" - resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-3.21.1.tgz#152a1d51d445380f7046b304672dd55d15c9e736" - integrity sha512-FTFVjYoBOZTJekiUsawGsSYV9QL0A+zDYCRj7y34IO6Jg+2IMYEtQa+bbictpdpV8dHxXywqU7C0gRDEOFtBFg== +fast-xml-parser@^4.2.2: + version "4.2.6" + resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-4.2.6.tgz#30ad37b014c16e31eec0e01fbf90a85cedb4eacf" + integrity sha512-Xo1qV++h/Y3Ng8dphjahnYe+rGHaaNdsYOBWL9Y9GCPKpNKilJtilvWkLcI9f9X2DoKTLsZsGYAls5+JL5jfLA== dependencies: - strnum "^1.0.4" + strnum "^1.0.5" fastq@^1.6.0: version "1.13.0" @@ -14407,7 +14424,7 @@ mime-db@1.52.0, "mime-db@>= 1.43.0 < 2", mime-db@^1.28.0: resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.52.0.tgz#bbabcdc02859f4987301c856e3387ce5ec43bf70" integrity sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg== -mime-types@^2.1.12, mime-types@^2.1.14, mime-types@^2.1.27, mime-types@~2.1.24, mime-types@~2.1.34: +mime-types@^2.1.12, mime-types@^2.1.27, mime-types@^2.1.35, mime-types@~2.1.24, mime-types@~2.1.34: version "2.1.35" resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.35.tgz#381a871b62a734450660ae3deee44813f70d959a" integrity sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw== @@ -14558,28 +14575,25 @@ minimist@^1.1.1, minimist@^1.1.3, minimist@^1.2.0, minimist@^1.2.3, minimist@^1. resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.6.tgz#8637a5b759ea0d6e98702cfb3a9283323c93af44" integrity sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q== -minio@^7.0.28: - version "7.0.32" - resolved "https://registry.yarnpkg.com/minio/-/minio-7.0.32.tgz#fed6a4679c5954d3efc6df47f73f7e7124446e2c" - integrity sha512-txa7Vr0N24MKzeAybP/wY1jxbLnfGHXwZYyfFXuMW55HX2+HOcKEIgH4hU6Qj/kiMgyXs/ozHjAuLIDrR8nwLg== +minio@^7.1.1: + version "7.1.1" + resolved "https://registry.yarnpkg.com/minio/-/minio-7.1.1.tgz#fbfc0c8cde023da262a94401db671d377550f9c9" + integrity sha512-HBLRFXs1CkNwAkahU+j1ilB9YS/Tmkdc6orpxVW1YN11NlEJyLjarIpBYu/inF+dj+tJIsA8PSKNnRmUNm+9qQ== dependencies: - async "^3.1.0" - block-stream2 "^2.0.0" - browser-or-node "^1.3.0" + async "^3.2.4" + block-stream2 "^2.1.0" + browser-or-node "^2.1.1" buffer-crc32 "^0.2.13" - crypto-browserify "^3.12.0" - es6-error "^4.1.1" - fast-xml-parser "^3.17.5" + fast-xml-parser "^4.2.2" ipaddr.js "^2.0.1" json-stream "^1.0.0" lodash "^4.17.21" - mime-types "^2.1.14" - mkdirp "^0.5.1" - query-string "^7.1.1" - through2 "^3.0.1" + mime-types "^2.1.35" + query-string "^7.1.3" + through2 "^4.0.2" web-encoding "^1.1.5" - xml "^1.0.0" - xml2js "^0.4.15" + xml "^1.0.1" + xml2js "^0.5.0" minipass-collect@^1.0.2: version "1.0.2" @@ -17108,12 +17122,12 @@ query-string@^5.0.1: object-assign "^4.1.0" strict-uri-encode "^1.0.0" -query-string@^7.1.1: - version "7.1.1" - resolved "https://registry.yarnpkg.com/query-string/-/query-string-7.1.1.tgz#754620669db978625a90f635f12617c271a088e1" - integrity sha512-MplouLRDHBZSG9z7fpuAAcI7aAYjDLhtsiVZsevsfaHWDS2IDdORKbSd1kWUA+V4zyva/HZoSfpwnYMMQDhb0w== +query-string@^7.1.3: + version "7.1.3" + resolved "https://registry.yarnpkg.com/query-string/-/query-string-7.1.3.tgz#a1cf90e994abb113a325804a972d98276fe02328" + integrity sha512-hh2WYhq4fi8+b+/2Kg9CEge4fDPvHS534aOOvOZeQ3+Vf2mCFsaFBYj0i+iXcAq6I9Vzp5fjMFBlONvayDC1qg== dependencies: - decode-uri-component "^0.2.0" + decode-uri-component "^0.2.2" filter-obj "^1.1.0" split-on-first "^1.0.0" strict-uri-encode "^2.0.0" @@ -17521,7 +17535,7 @@ readable-stream@1.1.x: isarray "0.0.1" string_decoder "~0.10.x" -"readable-stream@2 || 3", readable-stream@3, readable-stream@^3.1.1, readable-stream@^3.4.0, readable-stream@^3.5.0, readable-stream@^3.6.0: +readable-stream@3, readable-stream@^3.1.1, readable-stream@^3.4.0, readable-stream@^3.5.0, readable-stream@^3.6.0: version "3.6.0" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.6.0.tgz#337bbda3adc0706bd3e024426a286d4b4b2c9198" integrity sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA== @@ -19234,7 +19248,7 @@ strip-outer@^1.0.0: dependencies: escape-string-regexp "^1.0.2" -strnum@^1.0.4: +strnum@^1.0.5: version "1.0.5" resolved "https://registry.yarnpkg.com/strnum/-/strnum-1.0.5.tgz#5c4e829fe15ad4ff0d20c3db5ac97b73c9b072db" integrity sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA== @@ -19627,7 +19641,7 @@ throttled-queue@^2.1.2: resolved "https://registry.yarnpkg.com/throttled-queue/-/throttled-queue-2.1.4.tgz#4e2008c73ab3f72ba1bb09496c3cc9c5b745dbee" integrity sha512-YGdk8sdmr4ge3g+doFj/7RLF5kLM+Mi7DEciu9PHxnMJZMeVuZeTj31g4VE7ekUffx/IdbvrtOCiz62afg0mkg== -through2@4.0.2: +through2@4.0.2, through2@^4.0.2: version "4.0.2" resolved "https://registry.yarnpkg.com/through2/-/through2-4.0.2.tgz#a7ce3ac2a7a8b0b966c80e7c49f0484c3b239764" integrity sha512-iOqSav00cVxEEICeD7TjLB1sueEL+81Wpzp2bY17uZjZN0pWZPuo4suZ/61VujxmqSGFfgOcNuTZ85QJwNZQpw== @@ -19642,14 +19656,6 @@ through2@^2.0.0: readable-stream "~2.3.6" xtend "~4.0.1" -through2@^3.0.1: - version "3.0.2" - resolved "https://registry.yarnpkg.com/through2/-/through2-3.0.2.tgz#99f88931cfc761ec7678b41d5d7336b5b6a07bf4" - integrity sha512-enaDQ4MUyP2W6ZyT6EsMzqBPZaM/avg8iuo+l2d3QCs0J+6RaqkHV/2/lOwDTueBHeJ/2LG9lrLW3d5rWPucuQ== - dependencies: - inherits "^2.0.4" - readable-stream "2 || 3" - through@^2.3.6, through@^2.3.8, through@~2.3.4: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" @@ -21192,15 +21198,15 @@ xml-but-prettier@^1.0.1: dependencies: repeat-string "^1.5.2" -xml2js@^0.4.15: - version "0.4.23" - resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.23.tgz#a0c69516752421eb2ac758ee4d4ccf58843eac66" - integrity sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug== +xml2js@^0.5.0: + version "0.5.0" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.5.0.tgz#d9440631fbb2ed800203fad106f2724f62c493b7" + integrity sha512-drPFnkQJik/O+uPKpqSgr22mpuFHqKdbS835iAQrUC73L2F5WkboIRd63ai/2Yg6I1jzifPFKH2NTK+cfglkIA== dependencies: sax ">=0.6.0" xmlbuilder "~11.0.0" -xml@=1.0.1, xml@^1.0.0: +xml@=1.0.1, xml@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/xml/-/xml-1.0.1.tgz#78ba72020029c5bc87b8a81a3cfcd74b4a2fc1e5" integrity sha512-huCv9IH9Tcf95zuYCsQraZtWnJvBtLVE0QHMOs8bWyZAFZNDcYjsPq1nEx8jKA9y+Beo9v+7OBPRisQTjinQMw== From 12b7ef4a0c6833194ad47b75094afacd1642cf2c Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Wed, 19 Jul 2023 18:19:07 +0100 Subject: [PATCH 02/11] fix: workflow --- packages/client/src/lib.js | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/client/src/lib.js b/packages/client/src/lib.js index 842fb12104..8ef5380948 100644 --- a/packages/client/src/lib.js +++ b/packages/client/src/lib.js @@ -212,7 +212,6 @@ class NFTStorage { signal, }) } catch (/** @type {any} */ err) { - console.log(err) // TODO: remove me and test when client accepts custom fetch impl /* c8 ignore next 1 */ throw signal && signal.aborted ? new AbortError(err) : err From 575ba19903df0ce22db294d884e36f6dd1283fa1 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Wed, 19 Jul 2023 18:22:37 +0100 Subject: [PATCH 03/11] fix: workflow --- .github/workflows/client.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/client.yml b/.github/workflows/client.yml index a2ffe38be8..f8e1efda68 100644 --- a/.github/workflows/client.yml +++ b/.github/workflows/client.yml @@ -43,7 +43,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-node@v2 with: - node-version: { { matrix.node_version } } + node-version: ${{ matrix.node_version }} - uses: bahmutov/npm-install@v1 - name: Test (ES) run: yarn --cwd packages/client test:es From 28abe9cd484f0c8ddd3349dbeff4b4be2cb5a844 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Thu, 20 Jul 2023 14:37:13 +0100 Subject: [PATCH 04/11] fix: keepalive --- packages/client/src/lib.js | 16 ++-------------- packages/client/test/mock-server.js | 11 ++++++++--- 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/packages/client/src/lib.js b/packages/client/src/lib.js index 8ef5380948..e04a3b881e 100644 --- a/packages/client/src/lib.js +++ b/packages/client/src/lib.js @@ -172,10 +172,7 @@ class NFTStorage { { onStoredChunk, maxRetries, maxChunkSize, decoders, signal } = {} ) { const url = new URL('upload/', endpoint) - const headers = { - ...NFTStorage.auth(token), - 'Content-Type': 'application/car', - } + const headers = NFTStorage.auth(token) const targetSize = maxChunkSize || MAX_CHUNK_SIZE const splitter = car instanceof Blob @@ -190,15 +187,6 @@ class NFTStorage { carParts.push(part) } const carFile = new Blob(carParts, { type: 'application/car' }) - /** @type {Blob|ArrayBuffer} */ - let body = carFile - // FIXME: should not be necessary to await arrayBuffer()! - // Node.js 20 hangs reading the stream (it never ends) but in - // older node versions and the browser it is fine to pass a blob. - /* c8 ignore next 3 */ - if (parseInt(globalThis.process?.versions?.node) > 18) { - body = await body.arrayBuffer() - } const cid = await pRetry( async () => { await rateLimiter() @@ -208,7 +196,7 @@ class NFTStorage { response = await fetch(url.toString(), { method: 'POST', headers, - body, + body: carFile, signal, }) } catch (/** @type {any} */ err) { diff --git a/packages/client/test/mock-server.js b/packages/client/test/mock-server.js index e729dc5bf2..0f36613217 100644 --- a/packages/client/test/mock-server.js +++ b/packages/client/test/mock-server.js @@ -281,9 +281,8 @@ export class Service { outgoing.end() } catch (err) { - console.error(err) const error = /**@type {Error & {status: number}} */ (err) - if (!outgoing.hasHeader) { + if (!outgoing.headersSent) { outgoing.writeHead(error.status || 500) } outgoing.write(error.stack) @@ -323,7 +322,13 @@ export const listen = (service, port = 0) => * @param {(request:Request, state:State) => Promise} handler */ export const activate = async (state, handler) => { - const service = new Service(new http.Server(), state, handler) + const server = new http.Server() + // Server keepalive timeout is 5s by default but undici sets to 60s in + // Node.js 18 & 20. So when client tries to reuse a conenction after 5s it + // gets ECONNRESET from the server. + // https://connectreport.com/blog/tuning-http-keep-alive-in-node-js/ + server.keepAliveTimeout = 60_000 + const service = new Service(server, state, handler) await listen(service) return service } From 7534f93a0b9764c571aada3026be18041d0c7570 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Thu, 20 Jul 2023 15:00:48 +0100 Subject: [PATCH 05/11] chore: bump timeout for custom chunk size test --- packages/client/test/lib.spec.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/client/test/lib.spec.js b/packages/client/test/lib.spec.js index 42ad2eb968..073df73e12 100644 --- a/packages/client/test/lib.spec.js +++ b/packages/client/test/lib.spec.js @@ -197,7 +197,7 @@ describe('client', () => { it('upload CAR with custom chunk size', async function () { // @ts-ignore - this.timeout(10_000) + this.timeout(30_000) let uploadedChunks = 0 From daac82c37792a513700c93890e6fba1fddcc500f Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Thu, 20 Jul 2023 15:01:48 +0100 Subject: [PATCH 06/11] chore: unbump timeout for large CAR test (should be way enough) --- packages/client/test/lib.spec.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/client/test/lib.spec.js b/packages/client/test/lib.spec.js index 073df73e12..fc1a20b1b5 100644 --- a/packages/client/test/lib.spec.js +++ b/packages/client/test/lib.spec.js @@ -173,7 +173,7 @@ describe('client', () => { it('upload large CAR with a CarReader', async function () { // @ts-ignore - this.timeout(360e3) + this.timeout(130e3) let uploadedChunks = 0 const client = new NFTStorage({ token, endpoint }) From 1c2bd1a62844be0ea9933eba7c99254d15e7d8be Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Thu, 20 Jul 2023 15:02:09 +0100 Subject: [PATCH 07/11] chore: bump timeout for custom chunk size test --- packages/client/test/lib.spec.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/client/test/lib.spec.js b/packages/client/test/lib.spec.js index fc1a20b1b5..7de57164a6 100644 --- a/packages/client/test/lib.spec.js +++ b/packages/client/test/lib.spec.js @@ -197,7 +197,7 @@ describe('client', () => { it('upload CAR with custom chunk size', async function () { // @ts-ignore - this.timeout(30_000) + this.timeout(20_000) let uploadedChunks = 0 From 863dda3cfcb4b202510d43e92647ec254480a83e Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Thu, 20 Jul 2023 15:26:37 +0100 Subject: [PATCH 08/11] fix: add back nodejs 20 fix --- packages/client/src/lib.js | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/packages/client/src/lib.js b/packages/client/src/lib.js index e04a3b881e..d85a35a4ee 100644 --- a/packages/client/src/lib.js +++ b/packages/client/src/lib.js @@ -172,7 +172,10 @@ class NFTStorage { { onStoredChunk, maxRetries, maxChunkSize, decoders, signal } = {} ) { const url = new URL('upload/', endpoint) - const headers = NFTStorage.auth(token) + const headers = { + ...NFTStorage.auth(token), + 'Content-Type': 'application/car', + } const targetSize = maxChunkSize || MAX_CHUNK_SIZE const splitter = car instanceof Blob @@ -187,6 +190,15 @@ class NFTStorage { carParts.push(part) } const carFile = new Blob(carParts, { type: 'application/car' }) + /** @type {Blob|ArrayBuffer} */ + let body = carFile + // FIXME: should not be necessary to await arrayBuffer()! + // Node.js 20 hangs reading the stream (it never ends) but in + // older node versions and the browser it is fine to pass a blob. + /* c8 ignore next 3 */ + if (parseInt(globalThis.process?.versions?.node) > 18) { + // body = await body.arrayBuffer() + } const cid = await pRetry( async () => { await rateLimiter() @@ -196,7 +208,7 @@ class NFTStorage { response = await fetch(url.toString(), { method: 'POST', headers, - body: carFile, + body, signal, }) } catch (/** @type {any} */ err) { From dd2ad64534ecad9af0bfbc73e33a646085d54dc3 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Thu, 20 Jul 2023 15:28:49 +0100 Subject: [PATCH 09/11] fix: really add back nodejs 20 fix --- packages/client/src/lib.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/client/src/lib.js b/packages/client/src/lib.js index d85a35a4ee..8ef5380948 100644 --- a/packages/client/src/lib.js +++ b/packages/client/src/lib.js @@ -197,7 +197,7 @@ class NFTStorage { // older node versions and the browser it is fine to pass a blob. /* c8 ignore next 3 */ if (parseInt(globalThis.process?.versions?.node) > 18) { - // body = await body.arrayBuffer() + body = await body.arrayBuffer() } const cid = await pRetry( async () => { From 1eb2dbac81b1e88f66c8e66e1210802347d1a208 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Thu, 20 Jul 2023 15:53:49 +0100 Subject: [PATCH 10/11] fix: encode NFT in nodejs 20 --- packages/client/src/token.js | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/packages/client/src/token.js b/packages/client/src/token.js index 62f80ad491..7aee22077d 100644 --- a/packages/client/src/token.js +++ b/packages/client/src/token.js @@ -106,7 +106,16 @@ export class Token { // @ts-ignore blob may be a File! const name = blob.name || 'blob' /** @type {import('./platform.js').ReadableStream} */ - const content = blob.stream() + let content + // FIXME: should not be necessary to await arrayBuffer()! + // Node.js 20 hangs reading the stream (it never ends) but in + // older node versions and the browser it is fine to use blob.stream(). + /* c8 ignore next 3 */ + if (parseInt(globalThis.process?.versions?.node) > 18) { + content = new Uint8Array(await blob.arrayBuffer()) + } else { + content = blob.stream() + } const { root: cid } = await pack({ input: [{ path: name, content }], blockstore, From 52fe3a556298eecda5dda68575c7fc757e5812f0 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Thu, 20 Jul 2023 15:59:16 +0100 Subject: [PATCH 11/11] chore: ignore more --- packages/client/src/token.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/client/src/token.js b/packages/client/src/token.js index 7aee22077d..6aaa7f5dd3 100644 --- a/packages/client/src/token.js +++ b/packages/client/src/token.js @@ -110,7 +110,7 @@ export class Token { // FIXME: should not be necessary to await arrayBuffer()! // Node.js 20 hangs reading the stream (it never ends) but in // older node versions and the browser it is fine to use blob.stream(). - /* c8 ignore next 3 */ + /* c8 ignore next 5 */ if (parseInt(globalThis.process?.versions?.node) > 18) { content = new Uint8Array(await blob.arrayBuffer()) } else {