feat: add test for w3up deal fetch

Add a test for fetching filecoin deal info from w3up.

This is definitely not my best work - ideally we'd abstract this w3up mocking logic in a way that makes it less stateful, but I'm hesitant to do that when we will likely not be making many changes here in the future.

packages/api/package.json

@@ -35,7 +35,7 @@
     "@web3-storage/car-block-validator": "^1.2.0",
     "@web3-storage/content-claims": "^4.0.4",
     "@web3-storage/upload-client": "^13.2.0",
-    "@web3-storage/w3up-client": "^12.5.0",
+    "@web3-storage/w3up-client": "^12.5.1",
     "cardex": "^1.0.0",
     "ipfs-car": "^0.6.1",
     "it-last": "^2.0.0",

packages/api/src/bindings.d.ts

@@ -8,6 +8,7 @@ import { DBClient } from './utils/db-client.js'
 import { LinkdexApi } from './utils/linkdex.js'
 import { Logging } from './utils/logs.js'
 import { Client as W3upClient } from '@web3-storage/w3up-client'
+import * as contentClaims from '@web3-storage/content-claims/client'
 
 export type RuntimeEnvironmentName = 'test' | 'dev' | 'staging' | 'production'
 
@@ -142,6 +143,10 @@ export interface AuthOptions {
   checkHasPsaAccess?: boolean
 }
 
+export interface ContentClaimsClient {
+  read: typeof contentClaims.read
+}
+
 export interface RouteContext {
   params: Record<string, string>
   db: DBClient
@@ -158,6 +163,7 @@ export interface RouteContext {
   W3_NFTSTORAGE_SPACE?: string
   W3_NFTSTORAGE_ENABLE_W3UP_FOR_EMAILS?: string
   w3up?: W3upClient
+  contentClaims?: ContentClaimsClient
 }
 
 export type Handler = (

packages/api/src/routes/nfts-get.js

@@ -16,7 +16,9 @@ export const nftGet = async (event, ctx) => {
   const cid = parseCid(params.cid)
   const [nft, w3upDeals] = await Promise.all([
     db.getUpload(cid.sourceCid, user.id),
-    ctx.w3up ? getW3upDeals(ctx.w3up, cid.contentCid) : [],
+    ctx.w3up && ctx.contentClaims
+      ? getW3upDeals(ctx.w3up, ctx.contentClaims, cid.contentCid)
+      : [],
   ])
   if (nft) {
     // merge deals from dagcargo with deals from w3up

packages/api/src/routes/nfts-upload.js

@@ -189,7 +189,6 @@ export async function uploadCarWithStat(
 
     if (stat.structure === 'Partial') {
       checkDagStructureTask = async () => {
-        // @ts-expect-error - I'm not sure why this started failing TODO debug further
         const info = await w3up.capability.upload.get(stat.rootCid)
         if (info.shards && info.shards.length > 1) {
           const structure = await ctx.linkdexApi.getDagStructureForCars(

packages/api/src/utils/context.js

@@ -9,6 +9,7 @@ import { Service } from 'ucan-storage/service'
 import { LinkdexApi } from './linkdex.js'
 import { createW3upClientFromConfig } from './w3up.js'
 import { DID } from '@ucanto/core'
+import * as contentClaims from '@web3-storage/content-claims/client'
 
 /**
  * Obtains a route context object.
@@ -105,6 +106,7 @@ export async function getContext(event, params) {
     r2Uploader,
     log,
     ucanService,
+    contentClaims,
     w3up,
   }
 }

packages/api/src/utils/w3up.js

@@ -1,7 +1,6 @@
 import * as W3UP from '@web3-storage/w3up-client'
 import * as ed25519 from '@ucanto/principal/ed25519'
 import { StoreMemory } from '@web3-storage/access/stores/store-memory'
-import * as contentClaims from '@web3-storage/content-claims/client'
 import { CID } from 'multiformats/cid'
 import { base64 } from 'multiformats/bases/base64'
 import { identity } from 'multiformats/hashes/identity'
@@ -112,16 +111,17 @@ export async function createW3upClientFromConfig(options) {
 /**
  *
  * @param {W3upClient.Client} client
+ * @param {{read: typeof import('@web3-storage/content-claims/client').read}} contentClaimsClient
  * @param {import('@web3-storage/upload-client/types').UploadListItem} upload
  * @returns {Promise<import('@web3-storage/access').Result<import('@web3-storage/access').FilecoinInfoSuccess>[]>}
  */
-async function getFilecoinInfos(client, upload) {
+async function getFilecoinInfos(client, contentClaimsClient, upload) {
   return await Promise.all(
     // for each shard of the upload
     upload.shards
       ? upload.shards.map(async (shard) => {
           // find the equivalent piece link
-          const pieceClaims = await contentClaims.read(shard)
+          const pieceClaims = await contentClaimsClient.read(shard)
           const pieceClaim =
             /** @type {import('@web3-storage/content-claims/client/api').EqualsClaim} */ (
               pieceClaims.find((c) => c.type === 'assert/equals')
@@ -151,15 +151,26 @@ async function getFilecoinInfos(client, upload) {
 /**
  *
  * @param {W3upClient.Client | undefined} client
+ * @param {{read: typeof import('@web3-storage/content-claims/client').read}} contentClaimsClient
  * @param {string} contentCid
  * @returns {Promise<import('../bindings').Deal[]>}
  */
-export async function getW3upDeals(client, contentCid) {
+export async function getW3upDeals(client, contentClaimsClient, contentCid) {
   if (client) {
     const link = parseLink(contentCid)
     // get the upload
-    const upload = await client.capability.upload.get(link)
-    const filecoinInfoResults = await getFilecoinInfos(client, upload)
+    let upload
+    try {
+      upload = await client.capability.upload.get(link)
+    } catch (e) {
+      console.error('error getting upload', e)
+      return []
+    }
+    const filecoinInfoResults = await getFilecoinInfos(
+      client,
+      contentClaimsClient,
+      upload
+    )
     /**
      * @type {import('../bindings').Deal[]}
      */
@@ -171,9 +182,11 @@ export async function getW3upDeals(client, contentCid) {
           filecoinInfos.push({
             pieceCid: info.piece.toString(),
             status: 'published',
-            // TODO: figure these two out
+            batchRootCid: deal.aggregate.toString(),
+            miner: deal.provider,
+            chainDealID: Number(deal.aux.dataSource.dealID),
+            // TODO: figure this out
             datamodelSelector: '',
-            batchRootCid: deal.aggregate,
           })
         }
       } else {

packages/api/test/nfts-get.spec.js

@@ -1,22 +1,133 @@
 import test from 'ava'
+import { createServer } from 'node:http'
+import { ed25519 } from '@ucanto/principal'
+import { delegate, parseLink } from '@ucanto/core'
+import { base64 } from 'multiformats/bases/base64'
 import { createClientWithUser } from './scripts/helpers.js'
 import { fixtures } from './scripts/fixtures.js'
 import {
   getMiniflareContext,
   setupMiniflareContext,
 } from './scripts/test-context.js'
-import { read } from '@web3-storage/content-claims/client'
-import { parseLink } from '@ucanto/core'
+import {
+  createMockW3up,
+  locate,
+  encodeDelegationAsCid,
+} from './utils/w3up-testing.js'
+
+const nftStorageSpace = ed25519.generate()
+const nftStorageApiPrincipal = ed25519.generate()
+const nftStorageAccountEmailAllowListedForW3up = 'test+w3up@dev.nft.storage'
+const mockW3upDID = 'did:web:test.web3.storage'
+/**
+ * @type {import('@web3-storage/access').PieceLink}
+ */
+const mockPieceLink = parseLink(
+  'bafkzcibeslzwmewd4pugjanyiayot5m76a67dvdir25v6ms6kbuozy2sxotplrrrce'
+)
+/**
+ * @type {import('@web3-storage/access').FilecoinInfoAcceptedDeal[]}
+ */
+const mockDeals = [
+  {
+    aggregate: parseLink(
+      'bafkzcibcaapen7lfjgljzi523a5rau2l5pwpwseita6uunqy5otrlxa2l2pouca'
+    ),
+    aux: {
+      dataSource: {
+        dealID: BigInt(1),
+      },
+      dataType: BigInt(1),
+    },
+    provider: 'f01240',
+  },
+]
+const mockW3up = Promise.resolve(
+  (async function () {
+    const server = createServer(
+      await createMockW3up({
+        did: mockW3upDID,
+        // @ts-expect-error not returning a full upload get response for now
+        async onHandleUploadGet(cid) {
+          return {
+            // grabbed this shard CID from staging, it should correspond to a piece named bafkzcibeslzwmewd4pugjanyiayot5m76a67dvdir25v6ms6kbuozy2sxotplrrrce
+            shards: [
+              parseLink(
+                'bagbaieragf62xatg3bqrfafdy3lpk2fte7526kvxnltqsnhjr45cz6jjk7mq'
+              ),
+            ],
+          }
+        },
+        async onHandleFilecoinInfo(invocation) {
+          return {
+            deals: mockDeals,
+            aggregates: [],
+            piece: mockPieceLink,
+          }
+        },
+      })
+    )
+    server.listen(0)
+    await new Promise((resolve) =>
+      server.addListener('listening', () => resolve(undefined))
+    )
+    return {
+      server,
+    }
+  })()
+)
 
 test.before(async (t) => {
-  await setupMiniflareContext(t)
+  await setupMiniflareContext(t, {
+    overrides: {
+      W3UP_URL: locate((await mockW3up).server).url.toString(),
+      W3UP_DID: mockW3upDID,
+      W3_NFTSTORAGE_SPACE: (await nftStorageSpace).did(),
+      W3_NFTSTORAGE_PRINCIPAL: ed25519.format(await nftStorageApiPrincipal),
+      W3_NFTSTORAGE_PROOF: (
+        await encodeDelegationAsCid(
+          await delegate({
+            issuer: await nftStorageSpace,
+            audience: await nftStorageApiPrincipal,
+            capabilities: [
+              { can: 'upload/get', with: (await nftStorageSpace).did() },
+              { can: 'filecoin/info', with: (await nftStorageSpace).did() },
+            ],
+          })
+        )
+      ).toString(base64),
+      W3_NFTSTORAGE_ENABLE_W3UP_FOR_EMAILS: JSON.stringify([
+        nftStorageAccountEmailAllowListedForW3up,
+      ]),
+    },
+  })
 })
 
-test.only('should fetch deal details from w3up', async (t) => {
-  const testCid = 'bafybeiccy35oi3gajocq5bbg7pnaxb3kv5ibtdz3tc3kari53qhbjotzey'
-  const link = parseLink(testCid)
-  const claims = await read(link)
-  console.log('CLAIMS', claims)
+test.serial('should fetch deal details from w3up', async (t) => {
+  const cid = 'bafybeiccy35oi3gajocq5bbg7pnaxb3kv5ibtdz3tc3kari53qhbjotzey'
+  const client = await createClientWithUser(t)
+  const mf = getMiniflareContext(t)
+  await client.addPin({
+    cid,
+    name: 'test-filecoin-info',
+  })
+
+  const res = await mf.dispatchFetch(`http://miniflare.test/${cid}`, {
+    headers: { Authorization: `Bearer ${client.token}` },
+  })
+  const { ok, value } = await res.json()
+  t.assert(ok)
+  t.deepEqual(
+    value.deals,
+    mockDeals.map((deal) => ({
+      pieceCid: mockPieceLink.toString(),
+      status: 'published',
+      datamodelSelector: '',
+      batchRootCid: deal.aggregate.toString(),
+      miner: deal.provider,
+      chainDealID: Number(deal.aux.dataSource.dealID),
+    }))
+  )
 })
 
 test.serial('should return proper response for cid v1', async (t) => {

packages/api/test/utils/w3up-testing.js

@@ -42,6 +42,8 @@ export async function encodeDelegationAsCid(delegation) {
  * create a RequestListener that can be a mock up.web3.storage
  * @param {object} [options] - options
  * @param {string} options.did
+ * @param {(invocation: import('@ucanto/server').ProviderInput<import('@ucanto/client').InferInvokedCapability<typeof Filecoin.info>>) => Promise<import('@web3-storage/capabilities/types').FilecoinInfoSuccess | undefined>} [options.onHandleFilecoinInfo] - called in the filecoin/info handler and the result is returned
+ * @param {(invocation: import('@ucanto/server').ProviderInput<import('@ucanto/client').InferInvokedCapability<typeof Upload.get>>) => Promise<import('@web3-storage/upload-client/types').UploadGetSuccess | undefined>} [options.onHandleUploadGet] - called in the upload/get handler and the result is returned
  * @param {(invocation: import('@ucanto/server').ProviderInput<import('@ucanto/client').InferInvokedCapability<typeof Store.add>>) => Promise<void>} [options.onHandleStoreAdd] - called at start of store/add handler
  * @param {(invocation: import('@ucanto/server').ProviderInput<import('@ucanto/client').InferInvokedCapability<typeof Upload.add>>) => Promise<void>} [options.onHandleUploadAdd] - called at start of upload/add handler
  */
@@ -55,6 +57,21 @@ export async function createMockW3up(
           ok: {},
         }
       }),
+      info: Server.provide(Filecoin.info, async (invocation) => {
+        const result = await options.onHandleFilecoinInfo?.(invocation)
+        if (result) {
+          return {
+            ok: result,
+          }
+        } else {
+          return {
+            error: {
+              name: 'UnexpectedError',
+              message: `onUploadGet was not defined or return ${result}`,
+            },
+          }
+        }
+      }),
     },
     store: {
       add: Server.provide(Store.add, async (invocation) => {
@@ -82,6 +99,22 @@ export async function createMockW3up(
           ok: success,
         }
       }),
+
+      get: Server.provide(Upload.get, async (invocation) => {
+        const result = await options.onHandleUploadGet?.(invocation)
+        if (result) {
+          return {
+            ok: result,
+          }
+        } else {
+          return {
+            error: {
+              name: 'UnexpectedError',
+              message: `onUploadGet was not defined or return ${result}`,
+            },
+          }
+        }
+      }),
     },
   }
   const serverId = (await ed25519.generate()).withDID(

yarn.lock

@@ -5903,26 +5903,6 @@
   resolved "https://registry.yarnpkg.com/@web3-storage/multipart-parser/-/multipart-parser-1.0.0.tgz#6b69dc2a32a5b207ba43e556c25cc136a56659c4"
   integrity sha512-BEO6al7BYqcnfX15W2cnGR+Q566ACXAT9UQykORCWW80lmkpWsnEob6zJS1ZVBKsSJC8+7vJkHwlp+lXG1UCdw==
 
-"@web3-storage/upload-client@^13.1.0":
-  version "13.1.0"
-  resolved "https://registry.yarnpkg.com/@web3-storage/upload-client/-/upload-client-13.1.0.tgz#e29beb5ab0991682c28bcfe8c318aca42e43041c"
-  integrity sha512-RK67hUFviFG7KdupTwbMJCPdIsGEBSzpllybIOzbip3FKVH4fKDq4Sb2kXLptXeeqQfPJ86uRTmFtHTAVGVbZw==
-  dependencies:
-    "@ipld/car" "^5.2.2"
-    "@ipld/dag-cbor" "^9.0.6"
-    "@ipld/dag-ucan" "^3.4.0"
-    "@ipld/unixfs" "^2.1.1"
-    "@ucanto/client" "^9.0.0"
-    "@ucanto/interface" "^9.0.0"
-    "@ucanto/transport" "^9.1.0"
-    "@web3-storage/capabilities" "^13.2.0"
-    "@web3-storage/data-segment" "^5.1.0"
-    "@web3-storage/filecoin-client" "^3.3.0"
-    ipfs-utils "^9.0.14"
-    multiformats "^12.1.2"
-    p-retry "^5.1.2"
-    varint "^6.0.0"
-
 "@web3-storage/upload-client@^13.2.0":
   version "13.2.0"
   resolved "https://registry.yarnpkg.com/@web3-storage/upload-client/-/upload-client-13.2.0.tgz#b6781344f405d84a6575d4880c3abe73e20d8e67"
@@ -5943,10 +5923,10 @@
     p-retry "^5.1.2"
     varint "^6.0.0"
 
-"@web3-storage/w3up-client@^12.5.0":
-  version "12.5.0"
-  resolved "https://registry.yarnpkg.com/@web3-storage/w3up-client/-/w3up-client-12.5.0.tgz#67663f6c024bb7d198b2030f7752b139e5d3f9ae"
-  integrity sha512-SLpXXgA0TZJNSGtLHeq2kF+uwaHYfsH5068utikeRccCXJRrKQnCN1y2FpCe01H4SjLMTIQK13vga6DgSfJiuA==
+"@web3-storage/w3up-client@^12.5.1":
+  version "12.5.1"
+  resolved "https://registry.yarnpkg.com/@web3-storage/w3up-client/-/w3up-client-12.5.1.tgz#a5722c9b8ca0e1ea2c1a2b7c7749512938eda16d"
+  integrity sha512-fv53VEWOcDxNi2qsE5uHvOWDXbXstlYQ505uMN5vcpdetxo3FcxIkVqGBIIQDpsXLyloRNA8+ZXOy8+rKeOPVw==
   dependencies:
     "@ipld/dag-ucan" "^3.4.0"
     "@ucanto/client" "^9.0.0"
@@ -5958,7 +5938,7 @@
     "@web3-storage/capabilities" "^13.2.0"
     "@web3-storage/did-mailto" "^2.1.0"
     "@web3-storage/filecoin-client" "^3.3.0"
-    "@web3-storage/upload-client" "^13.1.0"
+    "@web3-storage/upload-client" "^13.2.0"
 
 "@webassemblyjs/ast@1.11.1":
   version "1.11.1"