feat: show deal info from w3up

packages/api/package.json

@@ -33,8 +33,9 @@
     "@ucanto/server": "^9.0.1",
     "@web3-storage/access": "^18.2.0",
     "@web3-storage/car-block-validator": "^1.2.0",
+    "@web3-storage/content-claims": "^4.0.4",
     "@web3-storage/upload-client": "^13.2.0",
-    "@web3-storage/w3up-client": "^12.5.0",
+    "@web3-storage/w3up-client": "^12.5.1",
     "cardex": "^1.0.0",
     "ipfs-car": "^0.6.1",
     "it-last": "^2.0.0",
@@ -47,7 +48,7 @@
     "regexparam": "^2.0.0",
     "toucan-js": "^2.7.0",
     "ucan-storage": "^1.3.0",
-    "uint8arrays": "^3.0.0"
+    "uint8arrays": "5.0.2"
   },
   "devDependencies": {
     "@cloudflare/workers-types": "^3.17.0",
@@ -67,7 +68,7 @@
     "carbites": "^1.0.6",
     "delay": "^5.0.0",
     "dotenv": "^10.0.0",
-    "esbuild": "^0.13.13",
+    "esbuild": "^0.20.2",
     "execa": "^5.1.1",
     "git-rev-sync": "^3.0.1",
     "ipfs-unixfs-importer": "^9.0.3",

packages/api/src/bindings.d.ts

@@ -8,6 +8,7 @@ import { DBClient } from './utils/db-client.js'
 import { LinkdexApi } from './utils/linkdex.js'
 import { Logging } from './utils/logs.js'
 import { Client as W3upClient } from '@web3-storage/w3up-client'
+import * as contentClaims from '@web3-storage/content-claims/client'
 
 export type RuntimeEnvironmentName = 'test' | 'dev' | 'staging' | 'production'
 
@@ -142,6 +143,10 @@ export interface AuthOptions {
   checkHasPsaAccess?: boolean
 }
 
+export interface ContentClaimsClient {
+  read: typeof contentClaims.read
+}
+
 export interface RouteContext {
   params: Record<string, string>
   db: DBClient
@@ -158,6 +163,7 @@ export interface RouteContext {
   W3_NFTSTORAGE_SPACE?: string
   W3_NFTSTORAGE_ENABLE_W3UP_FOR_EMAILS?: string
   w3up?: W3upClient
+  contentClaims?: ContentClaimsClient
 }
 
 export type Handler = (

packages/api/src/routes/nfts-get.js

@@ -3,6 +3,7 @@ import { JSONResponse } from '../utils/json-response.js'
 import { checkAuth, validate } from '../utils/auth.js'
 import { parseCid } from '../utils/utils.js'
 import { toNFTResponse } from '../utils/db-transforms.js'
+import { getW3upDeals } from '../utils/w3up.js'
 
 /**
  * @typedef {import('../bindings').Deal} Deal
@@ -13,8 +14,15 @@ export const nftGet = async (event, ctx) => {
   const { params, db } = ctx
   const { user } = checkAuth(ctx)
   const cid = parseCid(params.cid)
-  const nft = await db.getUpload(cid.sourceCid, user.id)
+  const [nft, w3upDeals] = await Promise.all([
+    db.getUpload(cid.sourceCid, user.id),
+    ctx.w3up && ctx.contentClaims
+      ? getW3upDeals(ctx.w3up, ctx.contentClaims, cid.contentCid)
+      : [],
+  ])
   if (nft) {
+    // merge deals from dagcargo with deals from w3up
+    nft.deals = [...nft?.deals, ...(w3upDeals || [])]
     return new JSONResponse({
       ok: true,
       value: toNFTResponse(nft, cid.sourceCid),

packages/api/src/routes/nfts-upload.js

@@ -189,7 +189,6 @@ export async function uploadCarWithStat(
 
     if (stat.structure === 'Partial') {
       checkDagStructureTask = async () => {
-        // @ts-expect-error - I'm not sure why this started failing TODO debug further
         const info = await w3up.capability.upload.get(stat.rootCid)
         if (info.shards && info.shards.length > 1) {
           const structure = await ctx.linkdexApi.getDagStructureForCars(

packages/api/src/utils/context.js

@@ -9,6 +9,7 @@ import { Service } from 'ucan-storage/service'
 import { LinkdexApi } from './linkdex.js'
 import { createW3upClientFromConfig } from './w3up.js'
 import { DID } from '@ucanto/core'
+import * as contentClaims from '@web3-storage/content-claims/client'
 
 /**
  * Obtains a route context object.
@@ -105,6 +106,7 @@ export async function getContext(event, params) {
     r2Uploader,
     log,
     ucanService,
+    contentClaims,
     w3up,
   }
 }

packages/api/src/utils/db-client.js

@@ -429,7 +429,7 @@ export class DBClient {
 
     const cids = uploads?.map((u) => u.content_cid)
 
-    const deals = await this.getDealsForCids(cids)
+    const deals = await this.getDealsFromDagcargoFDW(cids)
 
     return uploads?.map((u) => {
       return {
@@ -515,7 +515,7 @@ export class DBClient {
    * @returns {Promise<import('./../bindings').Deal[]>}
    */
   async getDeals(cid) {
-    const deals = await this.getDealsForCids([cid])
+    const deals = await this.getDealsFromDagcargoFDW([cid])
 
     return deals[cid] ? deals[cid] : []
   }
@@ -527,7 +527,7 @@ export class DBClient {
    *
    * @param {string[]} cids
    */
-  async getDealsForCids(cids = []) {
+  async getDealsFromDagcargoFDW(cids = []) {
     try {
       const rsp = await this.client.rpc('find_deals_by_content_cids', {
         cids,

packages/api/src/utils/w3up.js

@@ -7,6 +7,7 @@ import { identity } from 'multiformats/hashes/identity'
 import { CarReader } from '@ipld/car'
 import { importDAG } from '@ucanto/core/delegation'
 import * as W3upClient from '@web3-storage/w3up-client'
+import { parseLink } from '@ucanto/core'
 import { connect } from '@ucanto/client'
 import { CAR, HTTP } from '@ucanto/transport'
 
@@ -106,3 +107,95 @@ export async function createW3upClientFromConfig(options) {
   await w3up.addSpace(await parseW3Proof(options.proof))
   return w3up
 }
+
+/**
+ *
+ * @param {W3upClient.Client} client
+ * @param {{read: typeof import('@web3-storage/content-claims/client').read}} contentClaimsClient
+ * @param {import('@web3-storage/upload-client/types').UploadListItem} upload
+ * @returns {Promise<import('@web3-storage/access').Result<import('@web3-storage/access').FilecoinInfoSuccess>[]>}
+ */
+async function getFilecoinInfos(client, contentClaimsClient, upload) {
+  return await Promise.all(
+    // for each shard of the upload
+    upload.shards
+      ? upload.shards.map(async (shard) => {
+          // find the equivalent piece link
+          const pieceClaims = await contentClaimsClient.read(shard)
+          const pieceClaim =
+            /** @type {import('@web3-storage/content-claims/client/api').EqualsClaim} */ (
+              pieceClaims.find((c) => c.type === 'assert/equals')
+            )
+          if (pieceClaim) {
+            const pieceLink = pieceClaim.equals
+            // and get filecoin info for it
+            const filecoinInfo = await client.capability.filecoin.info(
+              /** @type {import('@web3-storage/access').PieceLink} */ (
+                pieceLink
+              )
+            )
+            return filecoinInfo.out
+          } else {
+            return {
+              error: {
+                name: 'PieceLinkClaimNotFound',
+                message: `could not find piece link equivalent of ${shard}`,
+              },
+            }
+          }
+        })
+      : []
+  )
+}
+
+/**
+ *
+ * @param {W3upClient.Client | undefined} client
+ * @param {{read: typeof import('@web3-storage/content-claims/client').read}} contentClaimsClient
+ * @param {string} contentCid
+ * @returns {Promise<import('../bindings').Deal[]>}
+ */
+export async function getW3upDeals(client, contentClaimsClient, contentCid) {
+  if (client) {
+    const link = parseLink(contentCid)
+    // get the upload
+    let upload
+    try {
+      upload = await client.capability.upload.get(link)
+    } catch (e) {
+      console.error('error getting upload', e)
+      return []
+    }
+    const filecoinInfoResults = await getFilecoinInfos(
+      client,
+      contentClaimsClient,
+      upload
+    )
+    /**
+     * @type {import('../bindings').Deal[]}
+     */
+    const filecoinInfos = []
+    for (const result of filecoinInfoResults) {
+      if (result.ok) {
+        const info = result.ok
+        for (const deal of info.deals) {
+          filecoinInfos.push({
+            pieceCid: info.piece.toString(),
+            status: 'published',
+            batchRootCid: deal.aggregate.toString(),
+            miner: deal.provider,
+            chainDealID: Number(deal.aux.dataSource.dealID),
+            // TODO: figure this out
+            datamodelSelector: '',
+          })
+        }
+      } else {
+        // @ts-expect-error - in practice this will just be undefined if message doesn't exist
+        console.warn(`error getting filecoininfo: ${result.error.message}`)
+      }
+    }
+    return filecoinInfos
+  } else {
+    return []
+  }
+}

packages/api/test/nfts-get.spec.js

@@ -1,13 +1,146 @@
 import test from 'ava'
+import { createServer } from 'node:http'
+import { ed25519 } from '@ucanto/principal'
+import { delegate, parseLink } from '@ucanto/core'
+import { base64 } from 'multiformats/bases/base64'
 import { createClientWithUser } from './scripts/helpers.js'
 import { fixtures } from './scripts/fixtures.js'
 import {
   getMiniflareContext,
   setupMiniflareContext,
 } from './scripts/test-context.js'
+import {
+  createMockW3up,
+  locate,
+  encodeDelegationAsCid,
+} from './utils/w3up-testing.js'
+
+const nftStorageSpace = ed25519.generate()
+const nftStorageApiPrincipal = ed25519.generate()
+const nftStorageAccountEmailAllowListedForW3up = 'test+w3up@dev.nft.storage'
+const mockW3upDID = 'did:web:test.web3.storage'
+/**
+ * @type {import('@web3-storage/access').PieceLink}
+ */
+const mockPieceLink = parseLink(
+  'bafkzcibeslzwmewd4pugjanyiayot5m76a67dvdir25v6ms6kbuozy2sxotplrrrce'
+)
+/**
+ * @type {import('@web3-storage/access').FilecoinInfoAcceptedDeal[]}
+ */
+const mockDeals = [
+  {
+    aggregate: parseLink(
+      'bafkzcibcaapen7lfjgljzi523a5rau2l5pwpwseita6uunqy5otrlxa2l2pouca'
+    ),
+    aux: {
+      dataSource: {
+        dealID: BigInt(1),
+      },
+      dataType: BigInt(1),
+    },
+    provider: 'f01240',
+  },
+]
+const cidWithShards = parseLink(
+  'bafybeiccy35oi3gajocq5bbg7pnaxb3kv5ibtdz3tc3kari53qhbjotzey'
+)
+const mockW3up = Promise.resolve(
+  (async function () {
+    const server = createServer(
+      await createMockW3up({
+        did: mockW3upDID,
+        // @ts-expect-error not returning a full upload get response for now
+        async onHandleUploadGet(invocation) {
+          if (invocation.capability.nb.root?.equals(cidWithShards)) {
+            return {
+              // grabbed this shard CID from staging, it should correspond to a piece named bafkzcibeslzwmewd4pugjanyiayot5m76a67dvdir25v6ms6kbuozy2sxotplrrrce
+              shards: [
+                parseLink(
+                  'bagbaieragf62xatg3bqrfafdy3lpk2fte7526kvxnltqsnhjr45cz6jjk7mq'
+                ),
+              ],
+            }
+          } else {
+            return {
+              shards: [],
+            }
+          }
+        },
+        async onHandleFilecoinInfo(invocation) {
+          if (invocation.capability.nb.piece.equals(mockPieceLink)) {
+            return {
+              deals: mockDeals,
+              aggregates: [],
+              piece: mockPieceLink,
+            }
+          } else {
+            return undefined
+          }
+        },
+      })
+    )
+    server.listen(0)
+    await new Promise((resolve) =>
+      server.addListener('listening', () => resolve(undefined))
+    )
+    return {
+      server,
+    }
+  })()
+)
 
 test.before(async (t) => {
-  await setupMiniflareContext(t)
+  await setupMiniflareContext(t, {
+    overrides: {
+      W3UP_URL: locate((await mockW3up).server).url.toString(),
+      W3UP_DID: mockW3upDID,
+      W3_NFTSTORAGE_SPACE: (await nftStorageSpace).did(),
+      W3_NFTSTORAGE_PRINCIPAL: ed25519.format(await nftStorageApiPrincipal),
+      W3_NFTSTORAGE_PROOF: (
+        await encodeDelegationAsCid(
+          await delegate({
+            issuer: await nftStorageSpace,
+            audience: await nftStorageApiPrincipal,
+            capabilities: [
+              { can: 'upload/get', with: (await nftStorageSpace).did() },
+              { can: 'filecoin/info', with: (await nftStorageSpace).did() },
+            ],
+          })
+        )
+      ).toString(base64),
+      W3_NFTSTORAGE_ENABLE_W3UP_FOR_EMAILS: JSON.stringify([
+        nftStorageAccountEmailAllowListedForW3up,
+      ]),
+    },
+  })
+})
+
+test.serial('should fetch deal details from w3up', async (t) => {
+  const cid = cidWithShards.toString()
+  const client = await createClientWithUser(t)
+  const mf = getMiniflareContext(t)
+  await client.addPin({
+    cid,
+    name: 'test-filecoin-info',
+  })
+
+  const res = await mf.dispatchFetch(`http://miniflare.test/${cid}`, {
+    headers: { Authorization: `Bearer ${client.token}` },
+  })
+  const { ok, value } = await res.json()
+  t.assert(ok)
+  t.deepEqual(
+    value.deals,
+    mockDeals.map((deal) => ({
+      pieceCid: mockPieceLink.toString(),
+      status: 'published',
+      datamodelSelector: '',
+      batchRootCid: deal.aggregate.toString(),
+      miner: deal.provider,
+      chainDealID: Number(deal.aux.dataSource.dealID),
+    }))
+  )
 })
 
 test.serial('should return proper response for cid v1', async (t) => {