From 67665f7c20dcc600484654f2a4e07065c4d752ae Mon Sep 17 00:00:00 2001 From: bengo <171782+gobengo@users.noreply.github.com> Date: Fri, 22 Mar 2024 13:32:03 -0700 Subject: [PATCH] mock w3up uses ucanto. POST /upload/ handler does w3up.uploadCAR --- packages/api/package.json | 1 + packages/api/src/routes/nfts-upload.js | 23 +++++-- packages/api/src/utils/context.js | 5 +- packages/api/test/nfts-upload.spec.js | 51 ++++++++++++-- packages/api/test/utils/w3up-testing.js | 88 +++++++++++++++++++++++++ packages/api/wrangler.toml | 6 +- yarn.lock | 19 +++++- 7 files changed, 178 insertions(+), 15 deletions(-) create mode 100644 packages/api/test/utils/w3up-testing.js diff --git a/packages/api/package.json b/packages/api/package.json index a41e176b1b..0d779a5ab2 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -29,6 +29,7 @@ "@supabase/postgrest-js": "^0.34.1", "@ucanto/core": "^9.0.1", "@ucanto/principal": "^9.0.0", + "@ucanto/server": "^9.0.1", "@web3-storage/access": "^18.2.0", "@web3-storage/car-block-validator": "^1.2.0", "@web3-storage/w3up-client": "^12.5.0", diff --git a/packages/api/src/routes/nfts-upload.js b/packages/api/src/routes/nfts-upload.js index e970c4717c..7f2b6ef233 100644 --- a/packages/api/src/routes/nfts-upload.js +++ b/packages/api/src/routes/nfts-upload.js @@ -31,6 +31,11 @@ export async function nftUpload(event, ctx) { /** @type {import('../utils/db-client-types').UploadOutput} */ let upload + /** + * blob that should be stored in w3up + * @type {Blob} + */ + let carBlobForW3up if (contentType.includes('multipart/form-data')) { const form = await event.request.formData() // Our API schema requires that all file parts be named `file` and @@ -49,7 +54,7 @@ export async function nftUpload(event, ctx) { input, wrapWithDirectory: true, }) - + carBlobForW3up = car upload = await uploadCar({ event, ctx, @@ -91,7 +96,7 @@ export async function nftUpload(event, ctx) { uploadType = 'Blob' structure = 'Complete' } - + carBlobForW3up = car upload = await uploadCar({ event, ctx, @@ -116,8 +121,18 @@ export async function nftUpload(event, ctx) { ...w3upConfig, w3up: ctx.w3up, }) - if (ctx.W3UP_URL) { - const w3upResponse = await fetch(ctx.W3UP_URL) + if (ctx.w3up) { + try { + await ctx.w3up.uploadCAR(carBlobForW3up) + console.warn('w3up.uploadCAR() succeeded in POST /upload/ handler') + } catch (error) { + // explicitly log so we can debug w/ cause + console.warn('error with w3up.uploadCAR', { + error, + cause: error?.cause, + }) + throw error + } } return new JSONResponse({ ok: true, value: toNFTResponse(upload) }) diff --git a/packages/api/src/utils/context.js b/packages/api/src/utils/context.js index 9ae34c2593..773362ac19 100644 --- a/packages/api/src/utils/context.js +++ b/packages/api/src/utils/context.js @@ -77,11 +77,14 @@ export async function getContext(event, params) { config.W3_NFTSTORAGE_PROOF ) { try { - w3up = await createW3upClientFromConfig({ + const w3upWIP = await createW3upClientFromConfig({ url: config.W3UP_URL, principal: config.W3_NFTSTORAGE_PRINCIPAL, proof: config.W3_NFTSTORAGE_PROOF, }) + // @ts-expect-error todo add DID check + w3upWIP.setCurrentSpace(config.W3_NFTSTORAGE_SPACE) + w3up = w3upWIP } catch (error) { console.error(`error creating w3up-client from config`, error) } diff --git a/packages/api/test/nfts-upload.spec.js b/packages/api/test/nfts-upload.spec.js index 330c9d20cf..76afabe637 100644 --- a/packages/api/test/nfts-upload.spec.js +++ b/packages/api/test/nfts-upload.spec.js @@ -31,19 +31,42 @@ import { ed25519 } from '@ucanto/principal' import { delegate } from '@ucanto/core' import { encodeDelegationAsCid } from '../src/utils/w3up.js' import { base64 } from 'multiformats/bases/base64' +import { createMockW3up, locate } from './utils/w3up-testing.js' const __dirname = path.dirname(fileURLToPath(import.meta.url)) const nftStorageSpace = ed25519.generate() const nftStorageApiPrincipal = ed25519.generate() -const mockW3up = createListeningMockW3up() +let mockW3upStoreAddCount = 0 +let mockW3upUploadAddCount = 0 +const mockW3up = Promise.resolve( + (async function () { + const server = createServer( + await createMockW3up({ + async onHandleStoreAdd(invocation) { + mockW3upStoreAddCount++ + }, + async onHandleUploadAdd(invocation) { + mockW3upUploadAddCount++ + }, + }) + ) + server.listen(0) + await new Promise((resolve) => + server.addListener('listening', () => resolve(undefined)) + ) + return { + server, + } + })() +) test.before(async (t) => { const linkdexUrl = 'http://fake.api.net' await setupMiniflareContext(t, { overrides: { LINKDEX_URL: linkdexUrl, - W3UP_URL: (await mockW3up).url.toString(), + W3UP_URL: locate((await mockW3up).server).url.toString(), W3_NFTSTORAGE_SPACE: (await nftStorageSpace).did(), W3_NFTSTORAGE_PRINCIPAL: ed25519.format(await nftStorageApiPrincipal), W3_NFTSTORAGE_PROOF: ( @@ -63,7 +86,7 @@ test.before(async (t) => { }) test.after(async (t) => { - ;(await mockW3up).close() + ;(await mockW3up).server.close() }) test.serial('should upload a single file', async (t) => { @@ -96,7 +119,8 @@ test.serial('should upload a single file', async (t) => { }) test.serial('should forward uploads to W3UP_URL', async (t) => { - const initialW3upRequestCount = (await mockW3up).requestCount + const initialW3upStoreAddCount = mockW3upStoreAddCount + const initialW3upUploadAddCount = mockW3upUploadAddCount const client = await createClientWithUser(t) const mf = getMiniflareContext(t) const file = new Blob(['hello world!'], { type: 'application/text' }) @@ -107,9 +131,22 @@ test.serial('should forward uploads to W3UP_URL', async (t) => { }) const { ok, value } = await res.json() t.truthy(ok, 'Server response payload has `ok` property') - const finalW3upRequestCount = (await mockW3up).requestCount - const w3upRequestCountDelta = finalW3upRequestCount - initialW3upRequestCount - t.is(w3upRequestCountDelta, 1, 'this upload sent one http request to w3up') + + const finalW3upStoreAddCount = mockW3upStoreAddCount + const storeAddCountDelta = finalW3upStoreAddCount - initialW3upStoreAddCount + t.is( + storeAddCountDelta, + 1, + 'this upload sent one valid store/add invocation to w3up' + ) + + const finalW3upUploadAddCount = mockW3upUploadAddCount + const uploadAddCountDelta = finalW3upUploadAddCount - initialW3upStoreAddCount + t.is( + storeAddCountDelta, + 1, + 'this upload sent one valid store/add invocation to w3up' + ) }) test.serial('should upload multiple blobs', async (t) => { diff --git a/packages/api/test/utils/w3up-testing.js b/packages/api/test/utils/w3up-testing.js new file mode 100644 index 0000000000..a6b97988a5 --- /dev/null +++ b/packages/api/test/utils/w3up-testing.js @@ -0,0 +1,88 @@ +import { Store, Upload, Filecoin } from '@web3-storage/capabilities' +import * as Server from '@ucanto/server' +import * as CAR from '@ucanto/transport/car' +import * as consumers from 'stream/consumers' +import { ed25519 } from '@ucanto/principal' + +/** + * create a RequestListener that can be a mock up.web3.storage + * @param {object} [options] - options + * @param {(invocation: import('@ucanto/server').ProviderInput>) => Promise} [options.onHandleStoreAdd] - called at start of store/add handler + * @param {(invocation: import('@ucanto/server').ProviderInput>) => Promise} [options.onHandleUploadAdd] - called at start of upload/add handler + */ +export async function createMockW3up(options = {}) { + const service = { + filecoin: { + offer: Server.provide(Filecoin.offer, async (invocation) => { + return {} + }), + }, + store: { + add: Server.provide(Store.add, async (invocation) => { + await options.onHandleStoreAdd?.(invocation) + /** @type {import('@web3-storage/access').StoreAddSuccessDone} */ + const success = { + status: 'done', + allocated: invocation.capability.nb.size, + link: invocation.capability.nb.link, + with: invocation.capability.with, + } + return { + ok: success, + } + }), + }, + upload: { + add: Server.provide(Upload.add, async (invocation) => { + await options.onHandleUploadAdd?.(invocation) + /** @type {import('@web3-storage/access').UploadAddSuccess} */ + const success = { + root: invocation.capability.nb.root, + } + return { + ok: success, + } + }), + }, + } + const serverId = (await ed25519.generate()).withDID('did:web:web3.storage') + const server = Server.create({ + id: serverId, + service, + codec: CAR.inbound, + validateAuthorization: () => ({ ok: {} }), + }) + /** @type {import('node:http').RequestListener} */ + const listener = async (req, res) => { + try { + const requestBody = new Uint8Array(await consumers.arrayBuffer(req)) + const response = await server.request({ + body: requestBody, + // @ts-expect-error slight mismatch. ignore like w3infra does + headers: req.headers, + }) + res.writeHead(200, response.headers) + res.write(response.body) + } catch (error) { + console.error('error in mock w3up', error) + res.writeHead(500) + res.write(JSON.stringify(error)) + } finally { + res.end() + } + } + return listener +} + +/** + * Get location of a server + * @param {import('http').Server} server - server that should be listening on the returned url + */ +export function locate(server) { + const address = server.address() + if (typeof address === 'string' || !address) + throw new Error(`unexpected address string`) + const { port } = address + const url = new URL(`http://localhost:${port}/`) + return { url } +} diff --git a/packages/api/wrangler.toml b/packages/api/wrangler.toml index 5ae7ac7a3d..18f43dd497 100644 --- a/packages/api/wrangler.toml +++ b/packages/api/wrangler.toml @@ -7,7 +7,11 @@ workers_dev = true # Compatibility flags https://github.com/cloudflare/wrangler/pull/2009 compatibility_date = "2021-08-23" -compatibility_flags = ["formdata_parser_supports_files"] +compatibility_flags = [ + "formdata_parser_supports_files", + "streams_enable_constructors", + "transformstream_enable_standard_constructor", +] no_bundle = true r2_buckets = [ # CAR files live here. diff --git a/yarn.lock b/yarn.lock index b08e8349c5..03b6903e59 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5396,6 +5396,16 @@ multiformats "^11.0.2" one-webcrypto "^1.0.3" +"@ucanto/server@^9.0.1": + version "9.0.1" + resolved "https://registry.yarnpkg.com/@ucanto/server/-/server-9.0.1.tgz#949d38bd2dad30991220f5839608434d400c0bfc" + integrity sha512-EGhgKLjPgvM39j86WxSD7UoR0rr7jpTMclCOcpOEVC9r91sob8BReW2i7cm1zPvhSNFqS8rLjlGEgUIAhdAxmg== + dependencies: + "@ucanto/core" "^9.0.0" + "@ucanto/interface" "^9.0.0" + "@ucanto/principal" "^9.0.0" + "@ucanto/validator" "^9.0.0" + "@ucanto/transport@^9.0.0", "@ucanto/transport@^9.1.0": version "9.1.0" resolved "https://registry.yarnpkg.com/@ucanto/transport/-/transport-9.1.0.tgz#e37026d0cd389604cf85f62d7dc165ca5de50f06" @@ -5404,7 +5414,7 @@ "@ucanto/core" "^9.0.1" "@ucanto/interface" "^9.0.0" -"@ucanto/validator@^9.0.1": +"@ucanto/validator@^9.0.0", "@ucanto/validator@^9.0.1": version "9.0.1" resolved "https://registry.yarnpkg.com/@ucanto/validator/-/validator-9.0.1.tgz#ab6458e4400365645119f1b843805fca80ea46b3" integrity sha512-H9GMOXHNW3vCv36eQZN1/h8zOXHEljRV5yNZ/huyOaJLVAKxt7Va1Ww8VBf2Ho/ac6P7jwvQRT7WgxaXx1/3Hg== @@ -18002,11 +18012,16 @@ prettier-linter-helpers@^1.0.0: dependencies: fast-diff "^1.1.2" -prettier@2.5.1, "prettier@>=2.2.1 <=2.3.0", prettier@^2.5.1: +prettier@2.5.1, prettier@^2.5.1: version "2.5.1" resolved "https://registry.yarnpkg.com/prettier/-/prettier-2.5.1.tgz#fff75fa9d519c54cf0fce328c1017d94546bc56a" integrity sha512-vBZcPRUR5MZJwoyi3ZoyQlc1rXeEck8KgeC9AwwOn+exuxLxq5toTRDTSaVrXHxelDMHy9zlicw8u66yxoSUFg== +"prettier@>=2.2.1 <=2.3.0": + version "2.3.0" + resolved "https://registry.yarnpkg.com/prettier/-/prettier-2.3.0.tgz#b6a5bf1284026ae640f17f7ff5658a7567fc0d18" + integrity sha512-kXtO4s0Lz/DW/IJ9QdWhAf7/NmPWQXkFr/r/WkR3vyI+0v8amTDxiaQSLzs8NBlytfLWX/7uQUMIW677yLKl4w== + pretty-error@^2.1.1: version "2.1.2" resolved "https://registry.yarnpkg.com/pretty-error/-/pretty-error-2.1.2.tgz#be89f82d81b1c86ec8fdfbc385045882727f93b6"