From efca52f3319e42e77deba48452f1589e6de33313 Mon Sep 17 00:00:00 2001 From: Travis Vachon Date: Thu, 11 Apr 2024 16:24:02 -0700 Subject: [PATCH] feat: add test for w3up deal fetch Add a test for fetching filecoin deal info from w3up. This is definitely not my best work - ideally we'd abstract this w3up mocking logic in a way that makes it less stateful, but I'm hesitant to do that when we will likely not be making many changes here in the future. --- packages/api/package.json | 2 +- packages/api/src/bindings.d.ts | 6 ++ packages/api/src/routes/nfts-get.js | 4 +- packages/api/src/routes/nfts-upload.js | 1 - packages/api/src/utils/context.js | 2 + packages/api/src/utils/w3up.js | 29 ++++-- packages/api/test/nfts-get.spec.js | 127 ++++++++++++++++++++++-- packages/api/test/utils/w3up-testing.js | 33 ++++++ yarn.lock | 30 +----- 9 files changed, 190 insertions(+), 44 deletions(-) diff --git a/packages/api/package.json b/packages/api/package.json index fd0519cdb1..fdde6b9fa3 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -35,7 +35,7 @@ "@web3-storage/car-block-validator": "^1.2.0", "@web3-storage/content-claims": "^4.0.4", "@web3-storage/upload-client": "^13.2.0", - "@web3-storage/w3up-client": "^12.5.0", + "@web3-storage/w3up-client": "^12.5.1", "cardex": "^1.0.0", "ipfs-car": "^0.6.1", "it-last": "^2.0.0", diff --git a/packages/api/src/bindings.d.ts b/packages/api/src/bindings.d.ts index 04da3d1ec3..31ba46662d 100644 --- a/packages/api/src/bindings.d.ts +++ b/packages/api/src/bindings.d.ts @@ -8,6 +8,7 @@ import { DBClient } from './utils/db-client.js' import { LinkdexApi } from './utils/linkdex.js' import { Logging } from './utils/logs.js' import { Client as W3upClient } from '@web3-storage/w3up-client' +import * as contentClaims from '@web3-storage/content-claims/client' export type RuntimeEnvironmentName = 'test' | 'dev' | 'staging' | 'production' @@ -142,6 +143,10 @@ export interface AuthOptions { checkHasPsaAccess?: boolean } +export interface ContentClaimsClient { + read: typeof contentClaims.read +} + export interface RouteContext { params: Record db: DBClient @@ -158,6 +163,7 @@ export interface RouteContext { W3_NFTSTORAGE_SPACE?: string W3_NFTSTORAGE_ENABLE_W3UP_FOR_EMAILS?: string w3up?: W3upClient + contentClaims?: ContentClaimsClient } export type Handler = ( diff --git a/packages/api/src/routes/nfts-get.js b/packages/api/src/routes/nfts-get.js index 8b84a984bb..beb7afaee3 100644 --- a/packages/api/src/routes/nfts-get.js +++ b/packages/api/src/routes/nfts-get.js @@ -16,7 +16,9 @@ export const nftGet = async (event, ctx) => { const cid = parseCid(params.cid) const [nft, w3upDeals] = await Promise.all([ db.getUpload(cid.sourceCid, user.id), - ctx.w3up ? getW3upDeals(ctx.w3up, cid.contentCid) : [], + ctx.w3up && ctx.contentClaims + ? getW3upDeals(ctx.w3up, ctx.contentClaims, cid.contentCid) + : [], ]) if (nft) { // merge deals from dagcargo with deals from w3up diff --git a/packages/api/src/routes/nfts-upload.js b/packages/api/src/routes/nfts-upload.js index d13f42021a..6f36eaea09 100644 --- a/packages/api/src/routes/nfts-upload.js +++ b/packages/api/src/routes/nfts-upload.js @@ -189,7 +189,6 @@ export async function uploadCarWithStat( if (stat.structure === 'Partial') { checkDagStructureTask = async () => { - // @ts-expect-error - I'm not sure why this started failing TODO debug further const info = await w3up.capability.upload.get(stat.rootCid) if (info.shards && info.shards.length > 1) { const structure = await ctx.linkdexApi.getDagStructureForCars( diff --git a/packages/api/src/utils/context.js b/packages/api/src/utils/context.js index 4b43242e54..bd4f4726ad 100644 --- a/packages/api/src/utils/context.js +++ b/packages/api/src/utils/context.js @@ -9,6 +9,7 @@ import { Service } from 'ucan-storage/service' import { LinkdexApi } from './linkdex.js' import { createW3upClientFromConfig } from './w3up.js' import { DID } from '@ucanto/core' +import * as contentClaims from '@web3-storage/content-claims/client' /** * Obtains a route context object. @@ -105,6 +106,7 @@ export async function getContext(event, params) { r2Uploader, log, ucanService, + contentClaims, w3up, } } diff --git a/packages/api/src/utils/w3up.js b/packages/api/src/utils/w3up.js index 5e8a67abb2..00bc17b32e 100644 --- a/packages/api/src/utils/w3up.js +++ b/packages/api/src/utils/w3up.js @@ -1,7 +1,6 @@ import * as W3UP from '@web3-storage/w3up-client' import * as ed25519 from '@ucanto/principal/ed25519' import { StoreMemory } from '@web3-storage/access/stores/store-memory' -import * as contentClaims from '@web3-storage/content-claims/client' import { CID } from 'multiformats/cid' import { base64 } from 'multiformats/bases/base64' import { identity } from 'multiformats/hashes/identity' @@ -112,16 +111,17 @@ export async function createW3upClientFromConfig(options) { /** * * @param {W3upClient.Client} client + * @param {{read: typeof import('@web3-storage/content-claims/client').read}} contentClaimsClient * @param {import('@web3-storage/upload-client/types').UploadListItem} upload * @returns {Promise[]>} */ -async function getFilecoinInfos(client, upload) { +async function getFilecoinInfos(client, contentClaimsClient, upload) { return await Promise.all( // for each shard of the upload upload.shards ? upload.shards.map(async (shard) => { // find the equivalent piece link - const pieceClaims = await contentClaims.read(shard) + const pieceClaims = await contentClaimsClient.read(shard) const pieceClaim = /** @type {import('@web3-storage/content-claims/client/api').EqualsClaim} */ ( pieceClaims.find((c) => c.type === 'assert/equals') @@ -151,15 +151,26 @@ async function getFilecoinInfos(client, upload) { /** * * @param {W3upClient.Client | undefined} client + * @param {{read: typeof import('@web3-storage/content-claims/client').read}} contentClaimsClient * @param {string} contentCid * @returns {Promise} */ -export async function getW3upDeals(client, contentCid) { +export async function getW3upDeals(client, contentClaimsClient, contentCid) { if (client) { const link = parseLink(contentCid) // get the upload - const upload = await client.capability.upload.get(link) - const filecoinInfoResults = await getFilecoinInfos(client, upload) + let upload + try { + upload = await client.capability.upload.get(link) + } catch (e) { + console.error('error getting upload', e) + return [] + } + const filecoinInfoResults = await getFilecoinInfos( + client, + contentClaimsClient, + upload + ) /** * @type {import('../bindings').Deal[]} */ @@ -171,9 +182,11 @@ export async function getW3upDeals(client, contentCid) { filecoinInfos.push({ pieceCid: info.piece.toString(), status: 'published', - // TODO: figure these two out + batchRootCid: deal.aggregate.toString(), + miner: deal.provider, + chainDealID: Number(deal.aux.dataSource.dealID), + // TODO: figure this out datamodelSelector: '', - batchRootCid: deal.aggregate, }) } } else { diff --git a/packages/api/test/nfts-get.spec.js b/packages/api/test/nfts-get.spec.js index cff8987da0..9e375176b0 100644 --- a/packages/api/test/nfts-get.spec.js +++ b/packages/api/test/nfts-get.spec.js @@ -1,22 +1,133 @@ import test from 'ava' +import { createServer } from 'node:http' +import { ed25519 } from '@ucanto/principal' +import { delegate, parseLink } from '@ucanto/core' +import { base64 } from 'multiformats/bases/base64' import { createClientWithUser } from './scripts/helpers.js' import { fixtures } from './scripts/fixtures.js' import { getMiniflareContext, setupMiniflareContext, } from './scripts/test-context.js' -import { read } from '@web3-storage/content-claims/client' -import { parseLink } from '@ucanto/core' +import { + createMockW3up, + locate, + encodeDelegationAsCid, +} from './utils/w3up-testing.js' + +const nftStorageSpace = ed25519.generate() +const nftStorageApiPrincipal = ed25519.generate() +const nftStorageAccountEmailAllowListedForW3up = 'test+w3up@dev.nft.storage' +const mockW3upDID = 'did:web:test.web3.storage' +/** + * @type {import('@web3-storage/access').PieceLink} + */ +const mockPieceLink = parseLink( + 'bafkzcibeslzwmewd4pugjanyiayot5m76a67dvdir25v6ms6kbuozy2sxotplrrrce' +) +/** + * @type {import('@web3-storage/access').FilecoinInfoAcceptedDeal[]} + */ +const mockDeals = [ + { + aggregate: parseLink( + 'bafkzcibcaapen7lfjgljzi523a5rau2l5pwpwseita6uunqy5otrlxa2l2pouca' + ), + aux: { + dataSource: { + dealID: BigInt(1), + }, + dataType: BigInt(1), + }, + provider: 'f01240', + }, +] +const mockW3up = Promise.resolve( + (async function () { + const server = createServer( + await createMockW3up({ + did: mockW3upDID, + // @ts-expect-error not returning a full upload get response for now + async onHandleUploadGet(cid) { + return { + // grabbed this shard CID from staging, it should correspond to a piece named bafkzcibeslzwmewd4pugjanyiayot5m76a67dvdir25v6ms6kbuozy2sxotplrrrce + shards: [ + parseLink( + 'bagbaieragf62xatg3bqrfafdy3lpk2fte7526kvxnltqsnhjr45cz6jjk7mq' + ), + ], + } + }, + async onHandleFilecoinInfo(invocation) { + return { + deals: mockDeals, + aggregates: [], + piece: mockPieceLink, + } + }, + }) + ) + server.listen(0) + await new Promise((resolve) => + server.addListener('listening', () => resolve(undefined)) + ) + return { + server, + } + })() +) test.before(async (t) => { - await setupMiniflareContext(t) + await setupMiniflareContext(t, { + overrides: { + W3UP_URL: locate((await mockW3up).server).url.toString(), + W3UP_DID: mockW3upDID, + W3_NFTSTORAGE_SPACE: (await nftStorageSpace).did(), + W3_NFTSTORAGE_PRINCIPAL: ed25519.format(await nftStorageApiPrincipal), + W3_NFTSTORAGE_PROOF: ( + await encodeDelegationAsCid( + await delegate({ + issuer: await nftStorageSpace, + audience: await nftStorageApiPrincipal, + capabilities: [ + { can: 'upload/get', with: (await nftStorageSpace).did() }, + { can: 'filecoin/info', with: (await nftStorageSpace).did() }, + ], + }) + ) + ).toString(base64), + W3_NFTSTORAGE_ENABLE_W3UP_FOR_EMAILS: JSON.stringify([ + nftStorageAccountEmailAllowListedForW3up, + ]), + }, + }) }) -test.only('should fetch deal details from w3up', async (t) => { - const testCid = 'bafybeiccy35oi3gajocq5bbg7pnaxb3kv5ibtdz3tc3kari53qhbjotzey' - const link = parseLink(testCid) - const claims = await read(link) - console.log('CLAIMS', claims) +test.serial('should fetch deal details from w3up', async (t) => { + const cid = 'bafybeiccy35oi3gajocq5bbg7pnaxb3kv5ibtdz3tc3kari53qhbjotzey' + const client = await createClientWithUser(t) + const mf = getMiniflareContext(t) + await client.addPin({ + cid, + name: 'test-filecoin-info', + }) + + const res = await mf.dispatchFetch(`http://miniflare.test/${cid}`, { + headers: { Authorization: `Bearer ${client.token}` }, + }) + const { ok, value } = await res.json() + t.assert(ok) + t.deepEqual( + value.deals, + mockDeals.map((deal) => ({ + pieceCid: mockPieceLink.toString(), + status: 'published', + datamodelSelector: '', + batchRootCid: deal.aggregate.toString(), + miner: deal.provider, + chainDealID: Number(deal.aux.dataSource.dealID), + })) + ) }) test.serial('should return proper response for cid v1', async (t) => { diff --git a/packages/api/test/utils/w3up-testing.js b/packages/api/test/utils/w3up-testing.js index 0df210ff6f..d0a347c026 100644 --- a/packages/api/test/utils/w3up-testing.js +++ b/packages/api/test/utils/w3up-testing.js @@ -42,6 +42,8 @@ export async function encodeDelegationAsCid(delegation) { * create a RequestListener that can be a mock up.web3.storage * @param {object} [options] - options * @param {string} options.did + * @param {(invocation: import('@ucanto/server').ProviderInput>) => Promise} [options.onHandleFilecoinInfo] - called in the filecoin/info handler and the result is returned + * @param {(invocation: import('@ucanto/server').ProviderInput>) => Promise} [options.onHandleUploadGet] - called in the upload/get handler and the result is returned * @param {(invocation: import('@ucanto/server').ProviderInput>) => Promise} [options.onHandleStoreAdd] - called at start of store/add handler * @param {(invocation: import('@ucanto/server').ProviderInput>) => Promise} [options.onHandleUploadAdd] - called at start of upload/add handler */ @@ -55,6 +57,21 @@ export async function createMockW3up( ok: {}, } }), + info: Server.provide(Filecoin.info, async (invocation) => { + const result = await options.onHandleFilecoinInfo?.(invocation) + if (result) { + return { + ok: result, + } + } else { + return { + error: { + name: 'UnexpectedError', + message: `onUploadGet was not defined or return ${result}`, + }, + } + } + }), }, store: { add: Server.provide(Store.add, async (invocation) => { @@ -82,6 +99,22 @@ export async function createMockW3up( ok: success, } }), + + get: Server.provide(Upload.get, async (invocation) => { + const result = await options.onHandleUploadGet?.(invocation) + if (result) { + return { + ok: result, + } + } else { + return { + error: { + name: 'UnexpectedError', + message: `onUploadGet was not defined or return ${result}`, + }, + } + } + }), }, } const serverId = (await ed25519.generate()).withDID( diff --git a/yarn.lock b/yarn.lock index 51efdf4e8f..e17259135e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5903,26 +5903,6 @@ resolved "https://registry.yarnpkg.com/@web3-storage/multipart-parser/-/multipart-parser-1.0.0.tgz#6b69dc2a32a5b207ba43e556c25cc136a56659c4" integrity sha512-BEO6al7BYqcnfX15W2cnGR+Q566ACXAT9UQykORCWW80lmkpWsnEob6zJS1ZVBKsSJC8+7vJkHwlp+lXG1UCdw== -"@web3-storage/upload-client@^13.1.0": - version "13.1.0" - resolved "https://registry.yarnpkg.com/@web3-storage/upload-client/-/upload-client-13.1.0.tgz#e29beb5ab0991682c28bcfe8c318aca42e43041c" - integrity sha512-RK67hUFviFG7KdupTwbMJCPdIsGEBSzpllybIOzbip3FKVH4fKDq4Sb2kXLptXeeqQfPJ86uRTmFtHTAVGVbZw== - dependencies: - "@ipld/car" "^5.2.2" - "@ipld/dag-cbor" "^9.0.6" - "@ipld/dag-ucan" "^3.4.0" - "@ipld/unixfs" "^2.1.1" - "@ucanto/client" "^9.0.0" - "@ucanto/interface" "^9.0.0" - "@ucanto/transport" "^9.1.0" - "@web3-storage/capabilities" "^13.2.0" - "@web3-storage/data-segment" "^5.1.0" - "@web3-storage/filecoin-client" "^3.3.0" - ipfs-utils "^9.0.14" - multiformats "^12.1.2" - p-retry "^5.1.2" - varint "^6.0.0" - "@web3-storage/upload-client@^13.2.0": version "13.2.0" resolved "https://registry.yarnpkg.com/@web3-storage/upload-client/-/upload-client-13.2.0.tgz#b6781344f405d84a6575d4880c3abe73e20d8e67" @@ -5943,10 +5923,10 @@ p-retry "^5.1.2" varint "^6.0.0" -"@web3-storage/w3up-client@^12.5.0": - version "12.5.0" - resolved "https://registry.yarnpkg.com/@web3-storage/w3up-client/-/w3up-client-12.5.0.tgz#67663f6c024bb7d198b2030f7752b139e5d3f9ae" - integrity sha512-SLpXXgA0TZJNSGtLHeq2kF+uwaHYfsH5068utikeRccCXJRrKQnCN1y2FpCe01H4SjLMTIQK13vga6DgSfJiuA== +"@web3-storage/w3up-client@^12.5.1": + version "12.5.1" + resolved "https://registry.yarnpkg.com/@web3-storage/w3up-client/-/w3up-client-12.5.1.tgz#a5722c9b8ca0e1ea2c1a2b7c7749512938eda16d" + integrity sha512-fv53VEWOcDxNi2qsE5uHvOWDXbXstlYQ505uMN5vcpdetxo3FcxIkVqGBIIQDpsXLyloRNA8+ZXOy8+rKeOPVw== dependencies: "@ipld/dag-ucan" "^3.4.0" "@ucanto/client" "^9.0.0" @@ -5958,7 +5938,7 @@ "@web3-storage/capabilities" "^13.2.0" "@web3-storage/did-mailto" "^2.1.0" "@web3-storage/filecoin-client" "^3.3.0" - "@web3-storage/upload-client" "^13.1.0" + "@web3-storage/upload-client" "^13.2.0" "@webassemblyjs/ast@1.11.1": version "1.11.1"