chore: remove legacy upload codepath

decisions/20240313-try-w3up.md

@@ -86,14 +86,6 @@ configures how nft.storage will authenticate to web3.storage when sending invoca
 configures the capabilities that nft.storage has access to when interacting with web3.storage to store nfts. These capabilities will usually be UCAN delegations whose audience is the identifier of `W3_NFTSTORAGE_PRINCIPAL`.
 W3_NFTSTORAGE_PROOF needs to have proof rooted in W3_NFTSTORAGE_SPACE that authorize W3_NFTSTORAGE_PRINCIPAL to store in W3_NFTSTORAGE_SPACE.
 
-##### `W3_NFTSTORAGE_ENABLE_W3UP_FOR_EMAILS` Environment Variable
-
-configures feature switch for which nftstorage accounts will have new uploads stored in web3.storage.
-
-Note: this environment variable may not be a permanent addition to the codebase. It's only meant to be used as a feature switch that decouples enabling the new functionality from deploying the new code. After testing, we may remove or change this feature switch when it is no longer useful.
-
-Format: JSON Array of email address strings.
-
 #### UI Changes
 
 None. But the existing UI workflow of uploading via https://nft.storage/files/ and form should behave just like they do now. But after this change, there should be a new side effect, which is that the upload should appear in the listing of uploads for the configured `W3_NFTSTORAGE_SPACE` (e.g. via w3cli `w3 ls` or in console.web3.storage).

packages/api/package.json

@@ -33,8 +33,9 @@
     "@ucanto/server": "^9.0.1",
     "@web3-storage/access": "^18.2.0",
     "@web3-storage/car-block-validator": "^1.2.0",
+    "@web3-storage/content-claims": "^4.0.4",
     "@web3-storage/upload-client": "^13.2.0",
-    "@web3-storage/w3up-client": "^12.5.0",
+    "@web3-storage/w3up-client": "^12.5.1",
     "cardex": "^1.0.0",
     "ipfs-car": "^0.6.1",
     "it-last": "^2.0.0",
@@ -47,7 +48,7 @@
     "regexparam": "^2.0.0",
     "toucan-js": "^2.7.0",
     "ucan-storage": "^1.3.0",
-    "uint8arrays": "^3.0.0"
+    "uint8arrays": "5.0.2"
   },
   "devDependencies": {
     "@cloudflare/workers-types": "^3.17.0",
@@ -67,7 +68,7 @@
     "carbites": "^1.0.6",
     "delay": "^5.0.0",
     "dotenv": "^10.0.0",
-    "esbuild": "^0.13.13",
+    "esbuild": "^0.20.2",
     "execa": "^5.1.1",
     "git-rev-sync": "^3.0.1",
     "ipfs-unixfs-importer": "^9.0.3",

packages/api/src/bindings.d.ts

@@ -8,6 +8,7 @@ import { DBClient } from './utils/db-client.js'
 import { LinkdexApi } from './utils/linkdex.js'
 import { Logging } from './utils/logs.js'
 import { Client as W3upClient } from '@web3-storage/w3up-client'
+import * as contentClaims from '@web3-storage/content-claims/client'
 
 export type RuntimeEnvironmentName = 'test' | 'dev' | 'staging' | 'production'
 
@@ -112,13 +113,6 @@ export interface ServiceConfiguration {
 
   /** did:key of the w3up space in which to store NFTs */
   W3_NFTSTORAGE_SPACE?: string
-
-  /**
-   * JSON array of strings that are emails whose uploads should be uploaded via w3up.
-   * This is meant as a feature switch to test new functionality,
-   * and this configuration may be removed once the feature switch isn't needed to limit access.
-   */
-  W3_NFTSTORAGE_ENABLE_W3UP_FOR_EMAILS?: string
 }
 
 export interface Ucan {
@@ -142,6 +136,10 @@ export interface AuthOptions {
   checkHasPsaAccess?: boolean
 }
 
+export interface ContentClaimsClient {
+  read: typeof contentClaims.read
+}
+
 export interface RouteContext {
   params: Record<string, string>
   db: DBClient
@@ -156,8 +154,8 @@ export interface RouteContext {
   W3_NFTSTORAGE_PRINCIPAL?: string
   W3_NFTSTORAGE_PROOF?: string
   W3_NFTSTORAGE_SPACE?: string
-  W3_NFTSTORAGE_ENABLE_W3UP_FOR_EMAILS?: string
   w3up?: W3upClient
+  contentClaims?: ContentClaimsClient
 }
 
 export type Handler = (

packages/api/src/config.js

@@ -67,8 +67,6 @@ export function serviceConfigFromVariables(vars) {
     W3_NFTSTORAGE_PRINCIPAL: vars.W3_NFTSTORAGE_PRINCIPAL,
     W3_NFTSTORAGE_PROOF: vars.W3_NFTSTORAGE_PROOF,
     W3_NFTSTORAGE_SPACE: vars.W3_NFTSTORAGE_SPACE,
-    W3_NFTSTORAGE_ENABLE_W3UP_FOR_EMAILS:
-      vars.W3_NFTSTORAGE_ENABLE_W3UP_FOR_EMAILS,
   }
 }
 
@@ -136,7 +134,6 @@ export function loadConfigVariables() {
     'W3_NFTSTORAGE_SPACE',
     'W3_NFTSTORAGE_PRINCIPAL',
     'W3_NFTSTORAGE_PROOF',
-    'W3_NFTSTORAGE_ENABLE_W3UP_FOR_EMAILS',
   ]
 
   for (const name of optional) {

packages/api/src/routes/nfts-get.js

@@ -3,6 +3,7 @@ import { JSONResponse } from '../utils/json-response.js'
 import { checkAuth, validate } from '../utils/auth.js'
 import { parseCid } from '../utils/utils.js'
 import { toNFTResponse } from '../utils/db-transforms.js'
+import { getW3upDeals } from '../utils/w3up.js'
 
 /**
  * @typedef {import('../bindings').Deal} Deal
@@ -13,8 +14,15 @@ export const nftGet = async (event, ctx) => {
   const { params, db } = ctx
   const { user } = checkAuth(ctx)
   const cid = parseCid(params.cid)
-  const nft = await db.getUpload(cid.sourceCid, user.id)
+  const [nft, w3upDeals] = await Promise.all([
+    db.getUpload(cid.sourceCid, user.id),
+    ctx.w3up && ctx.contentClaims
+      ? getW3upDeals(ctx.w3up, ctx.contentClaims, cid.contentCid)
+      : [],
+  ])
   if (nft) {
+    // merge deals from dagcargo with deals from w3up
+    nft.deals = [...nft?.deals, ...(w3upDeals || [])]
     return new JSONResponse({
       ok: true,
       value: toNFTResponse(nft, cid.sourceCid),

packages/api/src/routes/nfts-upload.js

@@ -108,24 +108,6 @@ export async function nftUpload(event, ctx) {
   return new JSONResponse({ ok: true, value: toNFTResponse(upload) })
 }
 
-/**
- * returns whether w3up uploading feature is enabled given context + event
- * @param {object} context - context of server operation, e.g. including configuration of feature switch
- * @param {string} [context.W3_NFTSTORAGE_ENABLE_W3UP_FOR_EMAILS] - JSON array of allowed emails
- * @param {object} event - specific event for which we should determine whether w3up feature is enabled
- * @param {object} event.user
- * @param {string} event.user.email - email address of user associated with event
- */
-function w3upFeatureSwitchEnabled(context, event) {
-  // const { W3_NFTSTORAGE_ENABLE_W3UP_FOR_EMAILS = '[]' } = context
-  // const allowedEmails = JSON.parse(W3_NFTSTORAGE_ENABLE_W3UP_FOR_EMAILS)
-  // if (!Array.isArray(allowedEmails)) return false
-  // const eventHasAllowedEmail = allowedEmails.find(
-  //   (allowed) => allowed === event.user.email
-  // )
-  return true
-}
-
 /**
  * @typedef {{
  *   event: FetchEvent,
@@ -170,10 +152,8 @@ export async function uploadCarWithStat(
   /** @type {(() => Promise<void>)|undefined} */
   let checkDagStructureTask
   const backupUrls = []
-  // @ts-expect-error email is not expected in types
-  if (ctx.w3up && w3upFeatureSwitchEnabled(ctx, { user })) {
-    const { w3up } = ctx
-
+  const { w3up } = ctx
+  if (w3up) {
     // we perform store/add and upload/add concurrently to save time.
     await Promise.all([
       w3up.capability.store.add(car),
@@ -189,7 +169,6 @@ export async function uploadCarWithStat(
 
     if (stat.structure === 'Partial') {
       checkDagStructureTask = async () => {
-        // @ts-expect-error - I'm not sure why this started failing TODO debug further
         const info = await w3up.capability.upload.get(stat.rootCid)
         if (info.shards && info.shards.length > 1) {
           const structure = await ctx.linkdexApi.getDagStructureForCars(
@@ -205,32 +184,7 @@ export async function uploadCarWithStat(
       }
     }
   } else {
-    const carBytes = new Uint8Array(await car.arrayBuffer())
-    const [s3Backup, r2Backup] = await Promise.all([
-      ctx.s3Uploader.uploadCar(carBytes, stat.cid, user.id, metadata),
-      ctx.r2Uploader.uploadCar(carBytes, stat.cid, user.id, metadata),
-    ])
-    backupUrls.push(s3Backup.url, r2Backup.url)
-
-    // no need to ask linkdex if it's Complete or Unknown
-    if (stat.structure === 'Partial') {
-      // ask linkdex for the dag structure across the set of CARs in S3 for this upload.
-      checkDagStructureTask = async () => {
-        try {
-          const structure = await ctx.linkdexApi.getDagStructure(s3Backup.key)
-          if (structure === 'Complete') {
-            return ctx.db.updatePinStatus(
-              upload.content_cid,
-              elasticPin(structure)
-            )
-          }
-        } catch (/** @type {any} */ err) {
-          if (err.code !== MissingApiUrlCode) {
-            throw err
-          }
-        }
-      }
-    }
+    throw new Error('w3up not defined, cannot upload')
   }
   const xName = event.request.headers.get('x-name')
   let name = xName && decodeURIComponent(xName)

packages/api/src/utils/context.js

@@ -9,6 +9,7 @@ import { Service } from 'ucan-storage/service'
 import { LinkdexApi } from './linkdex.js'
 import { createW3upClientFromConfig } from './w3up.js'
 import { DID } from '@ucanto/core'
+import * as contentClaims from '@web3-storage/content-claims/client'
 
 /**
  * Obtains a route context object.
@@ -72,8 +73,6 @@ export async function getContext(event, params) {
     W3_NFTSTORAGE_PRINCIPAL: config.W3_NFTSTORAGE_PRINCIPAL,
     W3_NFTSTORAGE_PROOF: config.W3_NFTSTORAGE_PROOF,
     W3_NFTSTORAGE_SPACE: config.W3_NFTSTORAGE_SPACE,
-    W3_NFTSTORAGE_ENABLE_W3UP_FOR_EMAILS:
-      config.W3_NFTSTORAGE_ENABLE_W3UP_FOR_EMAILS,
   }
   let w3up
   if (
@@ -105,6 +104,7 @@ export async function getContext(event, params) {
     r2Uploader,
     log,
     ucanService,
+    contentClaims,
     w3up,
   }
 }

packages/api/src/utils/db-client.js

@@ -429,7 +429,7 @@ export class DBClient {
 
     const cids = uploads?.map((u) => u.content_cid)
 
-    const deals = await this.getDealsForCids(cids)
+    const deals = await this.getDealsFromDagcargoFDW(cids)
 
     return uploads?.map((u) => {
       return {
@@ -515,7 +515,7 @@ export class DBClient {
    * @returns {Promise<import('./../bindings').Deal[]>}
    */
   async getDeals(cid) {
-    const deals = await this.getDealsForCids([cid])
+    const deals = await this.getDealsFromDagcargoFDW([cid])
 
     return deals[cid] ? deals[cid] : []
   }
@@ -527,7 +527,7 @@ export class DBClient {
    *
    * @param {string[]} cids
    */
-  async getDealsForCids(cids = []) {
+  async getDealsFromDagcargoFDW(cids = []) {
     try {
       const rsp = await this.client.rpc('find_deals_by_content_cids', {
         cids,

packages/api/src/utils/w3up.js

@@ -7,6 +7,7 @@ import { identity } from 'multiformats/hashes/identity'
 import { CarReader } from '@ipld/car'
 import { importDAG } from '@ucanto/core/delegation'
 import * as W3upClient from '@web3-storage/w3up-client'
+import { parseLink } from '@ucanto/core'
 import { connect } from '@ucanto/client'
 import { CAR, HTTP } from '@ucanto/transport'
 
@@ -106,3 +107,95 @@ export async function createW3upClientFromConfig(options) {
   await w3up.addSpace(await parseW3Proof(options.proof))
   return w3up
 }
+
+/**
+ *
+ * @param {W3upClient.Client} client
+ * @param {{read: typeof import('@web3-storage/content-claims/client').read}} contentClaimsClient
+ * @param {import('@web3-storage/upload-client/types').UploadListItem} upload
+ * @returns {Promise<import('@web3-storage/access').Result<import('@web3-storage/access').FilecoinInfoSuccess>[]>}
+ */
+async function getFilecoinInfos(client, contentClaimsClient, upload) {
+  return await Promise.all(
+    // for each shard of the upload
+    upload.shards
+      ? upload.shards.map(async (shard) => {
+          // find the equivalent piece link
+          const pieceClaims = await contentClaimsClient.read(shard)
+          const pieceClaim =
+            /** @type {import('@web3-storage/content-claims/client/api').EqualsClaim} */ (
+              pieceClaims.find((c) => c.type === 'assert/equals')
+            )
+          if (pieceClaim) {
+            const pieceLink = pieceClaim.equals
+            // and get filecoin info for it
+            const filecoinInfo = await client.capability.filecoin.info(
+              /** @type {import('@web3-storage/access').PieceLink} */ (
+                pieceLink
+              )
+            )
+            return filecoinInfo.out
+          } else {
+            return {
+              error: {
+                name: 'PieceLinkClaimNotFound',
+                message: `could not find piece link equivalent of ${shard}`,
+              },
+            }
+          }
+        })
+      : []
+  )
+}
+
+/**
+ *
+ * @param {W3upClient.Client | undefined} client
+ * @param {{read: typeof import('@web3-storage/content-claims/client').read}} contentClaimsClient
+ * @param {string} contentCid
+ * @returns {Promise<import('../bindings').Deal[]>}
+ */
+export async function getW3upDeals(client, contentClaimsClient, contentCid) {
+  if (client) {
+    const link = parseLink(contentCid)
+    // get the upload
+    let upload
+    try {
+      upload = await client.capability.upload.get(link)
+    } catch (e) {
+      console.error('error getting upload', e)
+      return []
+    }
+    const filecoinInfoResults = await getFilecoinInfos(
+      client,
+      contentClaimsClient,
+      upload
+    )
+    /**
+     * @type {import('../bindings').Deal[]}
+     */
+    const filecoinInfos = []
+    for (const result of filecoinInfoResults) {
+      if (result.ok) {
+        const info = result.ok
+        for (const deal of info.deals) {
+          filecoinInfos.push({
+            pieceCid: info.piece.toString(),
+            status: 'published',
+            batchRootCid: deal.aggregate.toString(),
+            miner: deal.provider,
+            chainDealID: Number(deal.aux.dataSource.dealID),
+            // TODO: figure this out
+            datamodelSelector: '',
+          })
+        }
+      } else {
+        // @ts-expect-error - in practice this will just be undefined if message doesn't exist
+        console.warn(`error getting filecoininfo: ${result.error.message}`)
+      }
+    }
+    return filecoinInfos
+  } else {
+    return []
+  }
+}

packages/api/test/bindings.d.ts

@@ -0,0 +1,26 @@
+import { ProviderInput } from '@ucanto/server'
+import { InferInvokedCapability } from '@ucanto/client'
+import { Store, Upload, Filecoin } from '@web3-storage/capabilities'
+import { FilecoinInfoSuccess } from '@web3-storage/capabilities/types'
+import { Server as HttpServer } from 'http'
+
+interface MockW3upOptions {
+  did?: string
+  onHandleFilecoinInfo?: (
+    invocation: ProviderInput<InferInvokedCapability<typeof Filecoin.info>>
+  ) => Promise<FilecoinInfoSuccess | undefined>
+  onHandleUploadGet?: (
+    invocation: ProviderInput<InferInvokedCapability<typeof Upload.get>>
+  ) => Promise<UploadGetSuccess | undefined>
+  onHandleStoreAdd?: (
+    invocation: ProviderInput<InferInvokedCapability<typeof Store.add>>
+  ) => Promise<void>
+  onHandleUploadAdd?: (
+    invocation: ProviderInput<InferInvokedCapability<typeof Upload.add>>
+  ) => Promise<void>
+}
+
+interface MockW3up {
+  server: HttpServer
+  did: string
+}