From 1466f0d2b9e16c1c89129608a945656d1e3648e9 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Wed, 13 Sep 2023 11:37:38 +0200 Subject: [PATCH] fix: limit max upload size (#224) Adds limit based on previous decisions, considering that S3 Put command has hard-limit of 5GiB. - By limiting CAR size to 127*(1<<25), we guarantee max-4GiB-padded Filecoin pieces and have better utilization of Fil sector space. - By receiving one more byte, we would immediatly get to 8GiB padded piece. Closes https://github.com/web3-storage/w3up/issues/827 --- package-lock.json | 16 ++++++++-------- upload-api/package.json | 2 +- upload-api/service.js | 8 ++++++-- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/package-lock.json b/package-lock.json index 0d62c7af..8035444c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4826,9 +4826,9 @@ } }, "node_modules/@web3-storage/upload-api": { - "version": "5.3.0", - "resolved": "https://registry.npmjs.org/@web3-storage/upload-api/-/upload-api-5.3.0.tgz", - "integrity": "sha512-Q/4a9YKR1pft14l832xi/AflEdRnQWoob4X/2e1AeOyQQ/RbRI+c3NlfgV4dQx+zy7wByHsCcMVtD/8QMFBu5w==", + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/@web3-storage/upload-api/-/upload-api-5.3.1.tgz", + "integrity": "sha512-awLKn4f9ADRnFRI1MREZLsnHIEhEqh9qMDIcf+tkndr/pGbC23e6HxEsZeTYJjKa0BaHJi1GH4OFrw7uSui2bQ==", "dependencies": { "@ucanto/client": "^8.0.0", "@ucanto/interface": "^8.0.0", @@ -15770,7 +15770,7 @@ "@web-std/fetch": "^4.1.0", "@web3-storage/access": "^14.0.0", "@web3-storage/capabilities": "^9.0.0", - "@web3-storage/upload-api": "^5.3.0", + "@web3-storage/upload-api": "^5.3.1", "@web3-storage/w3infra-ucan-invocation": "*", "multiformats": "^11.0.1", "nanoid": "^4.0.2", @@ -19786,9 +19786,9 @@ } }, "@web3-storage/upload-api": { - "version": "5.3.0", - "resolved": "https://registry.npmjs.org/@web3-storage/upload-api/-/upload-api-5.3.0.tgz", - "integrity": "sha512-Q/4a9YKR1pft14l832xi/AflEdRnQWoob4X/2e1AeOyQQ/RbRI+c3NlfgV4dQx+zy7wByHsCcMVtD/8QMFBu5w==", + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/@web3-storage/upload-api/-/upload-api-5.3.1.tgz", + "integrity": "sha512-awLKn4f9ADRnFRI1MREZLsnHIEhEqh9qMDIcf+tkndr/pGbC23e6HxEsZeTYJjKa0BaHJi1GH4OFrw7uSui2bQ==", "requires": { "@ucanto/client": "^8.0.0", "@ucanto/interface": "^8.0.0", @@ -20052,7 +20052,7 @@ "@web3-storage/access": "^14.0.0", "@web3-storage/capabilities": "^9.0.0", "@web3-storage/sigv4": "^1.0.2", - "@web3-storage/upload-api": "^5.3.0", + "@web3-storage/upload-api": "^5.3.1", "@web3-storage/w3infra-ucan-invocation": "*", "ava": "^4.3.3", "aws-lambda-test-utils": "^1.3.0", diff --git a/upload-api/package.json b/upload-api/package.json index bd24da4f..0c4c701c 100644 --- a/upload-api/package.json +++ b/upload-api/package.json @@ -24,7 +24,7 @@ "@web-std/fetch": "^4.1.0", "@web3-storage/access": "^14.0.0", "@web3-storage/capabilities": "^9.0.0", - "@web3-storage/upload-api": "^5.3.0", + "@web3-storage/upload-api": "^5.3.1", "@web3-storage/w3infra-ucan-invocation": "*", "multiformats": "^11.0.1", "nanoid": "^4.0.2", diff --git a/upload-api/service.js b/upload-api/service.js index 0deb3a50..828cb231 100644 --- a/upload-api/service.js +++ b/upload-api/service.js @@ -9,7 +9,11 @@ Sentry.AWSLambda.init({ export const createServiceRouter = UploadAPI.createService -export const MAX_S3_PUT_SIZE = 5_000_000_000 +// S3 Put command has hard-limit of 5GiB. +// By limiting CAR size to 127*(1<<25), we guarantee max-4GiB-padded Filecoin pieces +// and have better utilization of Fil sector space. +// By receiving one more byte, we would immediatly get to 8GiB padded piece. +export const MAX_UPLOAD_SIZE = 127*(1<<25) /** * @param {import('@ucanto/interface').Signer} servicePrincipal @@ -19,7 +23,7 @@ export const createUcantoServer = (servicePrincipal, context) => UploadAPI.createServer({ ...context, id: servicePrincipal, - maxUploadSize: MAX_S3_PUT_SIZE, + maxUploadSize: MAX_UPLOAD_SIZE, errorReporter: { catch: (/** @type {string | Error} */ err) => { console.warn(err)