Skip to content

Commit

Permalink
mock w3up uses ucanto. POST /upload/ handler does w3up.uploadCAR
Browse files Browse the repository at this point in the history
  • Loading branch information
gobengo committed Mar 22, 2024
1 parent 05b65cb commit 67665f7
Show file tree
Hide file tree
Showing 7 changed files with 178 additions and 15 deletions.
1 change: 1 addition & 0 deletions packages/api/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
"@supabase/postgrest-js": "^0.34.1",
"@ucanto/core": "^9.0.1",
"@ucanto/principal": "^9.0.0",
"@ucanto/server": "^9.0.1",
"@web3-storage/access": "^18.2.0",
"@web3-storage/car-block-validator": "^1.2.0",
"@web3-storage/w3up-client": "^12.5.0",
Expand Down
23 changes: 19 additions & 4 deletions packages/api/src/routes/nfts-upload.js
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,11 @@ export async function nftUpload(event, ctx) {

/** @type {import('../utils/db-client-types').UploadOutput} */
let upload
/**
* blob that should be stored in w3up
* @type {Blob}
*/
let carBlobForW3up
if (contentType.includes('multipart/form-data')) {
const form = await event.request.formData()
// Our API schema requires that all file parts be named `file` and
Expand All @@ -49,7 +54,7 @@ export async function nftUpload(event, ctx) {
input,
wrapWithDirectory: true,
})

carBlobForW3up = car
upload = await uploadCar({
event,
ctx,
Expand Down Expand Up @@ -91,7 +96,7 @@ export async function nftUpload(event, ctx) {
uploadType = 'Blob'
structure = 'Complete'
}

carBlobForW3up = car
upload = await uploadCar({
event,
ctx,
Expand All @@ -116,8 +121,18 @@ export async function nftUpload(event, ctx) {
...w3upConfig,
w3up: ctx.w3up,
})
if (ctx.W3UP_URL) {
const w3upResponse = await fetch(ctx.W3UP_URL)
if (ctx.w3up) {
try {
await ctx.w3up.uploadCAR(carBlobForW3up)
console.warn('w3up.uploadCAR() succeeded in POST /upload/ handler')
} catch (error) {
// explicitly log so we can debug w/ cause
console.warn('error with w3up.uploadCAR', {
error,
cause: error?.cause,
})
throw error
}
}

return new JSONResponse({ ok: true, value: toNFTResponse(upload) })
Expand Down
5 changes: 4 additions & 1 deletion packages/api/src/utils/context.js
Original file line number Diff line number Diff line change
Expand Up @@ -77,11 +77,14 @@ export async function getContext(event, params) {
config.W3_NFTSTORAGE_PROOF
) {
try {
w3up = await createW3upClientFromConfig({
const w3upWIP = await createW3upClientFromConfig({
url: config.W3UP_URL,
principal: config.W3_NFTSTORAGE_PRINCIPAL,
proof: config.W3_NFTSTORAGE_PROOF,
})
// @ts-expect-error todo add DID check
w3upWIP.setCurrentSpace(config.W3_NFTSTORAGE_SPACE)
w3up = w3upWIP
} catch (error) {
console.error(`error creating w3up-client from config`, error)
}
Expand Down
51 changes: 44 additions & 7 deletions packages/api/test/nfts-upload.spec.js
Original file line number Diff line number Diff line change
Expand Up @@ -31,19 +31,42 @@ import { ed25519 } from '@ucanto/principal'
import { delegate } from '@ucanto/core'
import { encodeDelegationAsCid } from '../src/utils/w3up.js'
import { base64 } from 'multiformats/bases/base64'
import { createMockW3up, locate } from './utils/w3up-testing.js'

const __dirname = path.dirname(fileURLToPath(import.meta.url))

const nftStorageSpace = ed25519.generate()
const nftStorageApiPrincipal = ed25519.generate()
const mockW3up = createListeningMockW3up()
let mockW3upStoreAddCount = 0
let mockW3upUploadAddCount = 0
const mockW3up = Promise.resolve(
(async function () {
const server = createServer(
await createMockW3up({
async onHandleStoreAdd(invocation) {
mockW3upStoreAddCount++
},
async onHandleUploadAdd(invocation) {
mockW3upUploadAddCount++
},
})
)
server.listen(0)
await new Promise((resolve) =>
server.addListener('listening', () => resolve(undefined))
)
return {
server,
}
})()
)

test.before(async (t) => {
const linkdexUrl = 'http://fake.api.net'
await setupMiniflareContext(t, {
overrides: {
LINKDEX_URL: linkdexUrl,
W3UP_URL: (await mockW3up).url.toString(),
W3UP_URL: locate((await mockW3up).server).url.toString(),
W3_NFTSTORAGE_SPACE: (await nftStorageSpace).did(),
W3_NFTSTORAGE_PRINCIPAL: ed25519.format(await nftStorageApiPrincipal),
W3_NFTSTORAGE_PROOF: (
Expand All @@ -63,7 +86,7 @@ test.before(async (t) => {
})

test.after(async (t) => {
;(await mockW3up).close()
;(await mockW3up).server.close()
})

test.serial('should upload a single file', async (t) => {
Expand Down Expand Up @@ -96,7 +119,8 @@ test.serial('should upload a single file', async (t) => {
})

test.serial('should forward uploads to W3UP_URL', async (t) => {
const initialW3upRequestCount = (await mockW3up).requestCount
const initialW3upStoreAddCount = mockW3upStoreAddCount
const initialW3upUploadAddCount = mockW3upUploadAddCount
const client = await createClientWithUser(t)
const mf = getMiniflareContext(t)
const file = new Blob(['hello world!'], { type: 'application/text' })
Expand All @@ -107,9 +131,22 @@ test.serial('should forward uploads to W3UP_URL', async (t) => {
})
const { ok, value } = await res.json()
t.truthy(ok, 'Server response payload has `ok` property')
const finalW3upRequestCount = (await mockW3up).requestCount
const w3upRequestCountDelta = finalW3upRequestCount - initialW3upRequestCount
t.is(w3upRequestCountDelta, 1, 'this upload sent one http request to w3up')

const finalW3upStoreAddCount = mockW3upStoreAddCount
const storeAddCountDelta = finalW3upStoreAddCount - initialW3upStoreAddCount
t.is(
storeAddCountDelta,
1,
'this upload sent one valid store/add invocation to w3up'
)

const finalW3upUploadAddCount = mockW3upUploadAddCount
const uploadAddCountDelta = finalW3upUploadAddCount - initialW3upStoreAddCount
t.is(
storeAddCountDelta,
1,
'this upload sent one valid store/add invocation to w3up'
)
})

test.serial('should upload multiple blobs', async (t) => {
Expand Down
88 changes: 88 additions & 0 deletions packages/api/test/utils/w3up-testing.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
import { Store, Upload, Filecoin } from '@web3-storage/capabilities'
import * as Server from '@ucanto/server'
import * as CAR from '@ucanto/transport/car'
import * as consumers from 'stream/consumers'
import { ed25519 } from '@ucanto/principal'

/**
* create a RequestListener that can be a mock up.web3.storage
* @param {object} [options] - options
* @param {(invocation: import('@ucanto/server').ProviderInput<import('@ucanto/client').InferInvokedCapability<typeof Store.add>>) => Promise<void>} [options.onHandleStoreAdd] - called at start of store/add handler
* @param {(invocation: import('@ucanto/server').ProviderInput<import('@ucanto/client').InferInvokedCapability<typeof Upload.add>>) => Promise<void>} [options.onHandleUploadAdd] - called at start of upload/add handler
*/
export async function createMockW3up(options = {}) {
const service = {
filecoin: {
offer: Server.provide(Filecoin.offer, async (invocation) => {
return {}
}),
},
store: {
add: Server.provide(Store.add, async (invocation) => {
await options.onHandleStoreAdd?.(invocation)
/** @type {import('@web3-storage/access').StoreAddSuccessDone} */
const success = {
status: 'done',
allocated: invocation.capability.nb.size,
link: invocation.capability.nb.link,
with: invocation.capability.with,
}
return {
ok: success,
}
}),
},
upload: {
add: Server.provide(Upload.add, async (invocation) => {
await options.onHandleUploadAdd?.(invocation)
/** @type {import('@web3-storage/access').UploadAddSuccess} */
const success = {
root: invocation.capability.nb.root,
}
return {
ok: success,
}
}),
},
}
const serverId = (await ed25519.generate()).withDID('did:web:web3.storage')
const server = Server.create({
id: serverId,
service,
codec: CAR.inbound,
validateAuthorization: () => ({ ok: {} }),
})
/** @type {import('node:http').RequestListener} */
const listener = async (req, res) => {
try {
const requestBody = new Uint8Array(await consumers.arrayBuffer(req))
const response = await server.request({
body: requestBody,
// @ts-expect-error slight mismatch. ignore like w3infra does
headers: req.headers,
})
res.writeHead(200, response.headers)
res.write(response.body)
} catch (error) {
console.error('error in mock w3up', error)
res.writeHead(500)
res.write(JSON.stringify(error))
} finally {
res.end()
}
}
return listener
}

/**
* Get location of a server
* @param {import('http').Server} server - server that should be listening on the returned url
*/
export function locate(server) {
const address = server.address()
if (typeof address === 'string' || !address)
throw new Error(`unexpected address string`)
const { port } = address
const url = new URL(`http://localhost:${port}/`)
return { url }
}
6 changes: 5 additions & 1 deletion packages/api/wrangler.toml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,11 @@ workers_dev = true

# Compatibility flags https://github.com/cloudflare/wrangler/pull/2009
compatibility_date = "2021-08-23"
compatibility_flags = ["formdata_parser_supports_files"]
compatibility_flags = [
"formdata_parser_supports_files",
"streams_enable_constructors",
"transformstream_enable_standard_constructor",
]
no_bundle = true
r2_buckets = [
# CAR files live here.
Expand Down
19 changes: 17 additions & 2 deletions yarn.lock
Original file line number Diff line number Diff line change
Expand Up @@ -5396,6 +5396,16 @@
multiformats "^11.0.2"
one-webcrypto "^1.0.3"

"@ucanto/server@^9.0.1":
version "9.0.1"
resolved "https://registry.yarnpkg.com/@ucanto/server/-/server-9.0.1.tgz#949d38bd2dad30991220f5839608434d400c0bfc"
integrity sha512-EGhgKLjPgvM39j86WxSD7UoR0rr7jpTMclCOcpOEVC9r91sob8BReW2i7cm1zPvhSNFqS8rLjlGEgUIAhdAxmg==
dependencies:
"@ucanto/core" "^9.0.0"
"@ucanto/interface" "^9.0.0"
"@ucanto/principal" "^9.0.0"
"@ucanto/validator" "^9.0.0"

"@ucanto/transport@^9.0.0", "@ucanto/transport@^9.1.0":
version "9.1.0"
resolved "https://registry.yarnpkg.com/@ucanto/transport/-/transport-9.1.0.tgz#e37026d0cd389604cf85f62d7dc165ca5de50f06"
Expand All @@ -5404,7 +5414,7 @@
"@ucanto/core" "^9.0.1"
"@ucanto/interface" "^9.0.0"

"@ucanto/validator@^9.0.1":
"@ucanto/validator@^9.0.0", "@ucanto/validator@^9.0.1":
version "9.0.1"
resolved "https://registry.yarnpkg.com/@ucanto/validator/-/validator-9.0.1.tgz#ab6458e4400365645119f1b843805fca80ea46b3"
integrity sha512-H9GMOXHNW3vCv36eQZN1/h8zOXHEljRV5yNZ/huyOaJLVAKxt7Va1Ww8VBf2Ho/ac6P7jwvQRT7WgxaXx1/3Hg==
Expand Down Expand Up @@ -18002,11 +18012,16 @@ prettier-linter-helpers@^1.0.0:
dependencies:
fast-diff "^1.1.2"

prettier@2.5.1, "prettier@>=2.2.1 <=2.3.0", prettier@^2.5.1:
prettier@2.5.1, prettier@^2.5.1:
version "2.5.1"
resolved "https://registry.yarnpkg.com/prettier/-/prettier-2.5.1.tgz#fff75fa9d519c54cf0fce328c1017d94546bc56a"
integrity sha512-vBZcPRUR5MZJwoyi3ZoyQlc1rXeEck8KgeC9AwwOn+exuxLxq5toTRDTSaVrXHxelDMHy9zlicw8u66yxoSUFg==

"prettier@>=2.2.1 <=2.3.0":
version "2.3.0"
resolved "https://registry.yarnpkg.com/prettier/-/prettier-2.3.0.tgz#b6a5bf1284026ae640f17f7ff5658a7567fc0d18"
integrity sha512-kXtO4s0Lz/DW/IJ9QdWhAf7/NmPWQXkFr/r/WkR3vyI+0v8amTDxiaQSLzs8NBlytfLWX/7uQUMIW677yLKl4w==

pretty-error@^2.1.1:
version "2.1.2"
resolved "https://registry.yarnpkg.com/pretty-error/-/pretty-error-2.1.2.tgz#be89f82d81b1c86ec8fdfbc385045882727f93b6"
Expand Down

0 comments on commit 67665f7

Please sign in to comment.