feat: POST /uploads can write uploads to w3up for web3 serving and da…

…gcargoless filecoin deal-making (#2522)

Motivation:
* part of storacha/project-tracking#7
* implement #2518

Note
* many files I modified only because the upgrade to new typescript
compiler or eslint required it
* w3up-client requires node 18 so this drops support for node 16 from
the api package
* I think the client will still work on node 16, but I couldn't find a
way of getting the ci for client to, using node 16, install just the
client package deps and not the api package deps (which tries to install
w3up-client, and then `yarn` errored because that requies node 18)
* w3up-client requires node 18, but everything had been running on node
16
* [ ] if we can't find a way of continuing to use v16, then when this
merges to main, we probably need to go into cloudflare workers settings
and change NODE_VERSION from 16 to 18+

---------

Co-authored-by: Vasco Santos <santos.vasco10@gmail.com>
Co-authored-by: Travis Vachon <travis.vachon@gmail.com>
Co-authored-by: Alan Shaw <alan.shaw@protocol.ai>

.github/workflows/client.yml

@@ -36,7 +36,6 @@ jobs:
     strategy:
       matrix:
         node_version:
-          - 16
           - 18
           - 20
     steps:

.github/workflows/cron-dagcargo-sizes.yml

@@ -23,7 +23,7 @@ jobs:
           git checkout $LATEST_TAG
       - uses: actions/setup-node@v2
         with:
-          node-version: '16'
+          node-version: '18'
       - uses: bahmutov/npm-install@v1
       - name: Run job
         env:

.github/workflows/cron-metrics.yml

@@ -23,7 +23,7 @@ jobs:
           git checkout $LATEST_TAG
       - uses: actions/setup-node@v2
         with:
-          node-version: '16'
+          node-version: '18'
       - uses: bahmutov/npm-install@v1
       - name: Run job
         env:

.github/workflows/cron-nft-ttr.yml

@@ -20,7 +20,7 @@ jobs:
           git checkout $LATEST_TAG
       - uses: actions/setup-node@v2
         with:
-          node-version: '16'
+          node-version: '18'
       - uses: bahmutov/npm-install@v1
       - name: Run job
         env:

.github/workflows/cron-pins-failed.yml

@@ -27,7 +27,7 @@ jobs:
           git checkout $LATEST_TAG
       - uses: actions/setup-node@v2
         with:
-          node-version: '16'
+          node-version: '18'
       - uses: bahmutov/npm-install@v1
       - name: Run job
         env:

.github/workflows/cron-pins.yml

@@ -23,7 +23,7 @@ jobs:
           git checkout $LATEST_TAG
       - uses: actions/setup-node@v2
         with:
-          node-version: '16'
+          node-version: '18'
       - uses: bahmutov/npm-install@v1
       - name: Run job
         env:

.github/workflows/cron.yml

@@ -22,7 +22,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
-          node-version: '16'
+          node-version: '18'
       - uses: bahmutov/npm-install@v1
       - run: yarn workspace nft.storage prepare
       - run: yarn workspace cron lint
@@ -34,7 +34,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
-          node-version: '16'
+          node-version: '18'
       - uses: bahmutov/npm-install@v1
       - run: yarn workspace nft.storage prepare
       - run: yarn workspace cron typecheck
@@ -46,7 +46,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
-          node-version: '16'
+          node-version: '18'
       - uses: bahmutov/npm-install@v1
       - name: Test (ES)
         run: yarn --cwd packages/cron test

.github/workflows/deploy.yml

@@ -16,7 +16,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
-          node-version: '16'
+          node-version: '18'
       - uses: bahmutov/npm-install@v1
       - name: Publish app
         uses: cloudflare/wrangler-action@1.3.0
@@ -43,29 +43,29 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
-          node-version: '16'
+          node-version: '18'
       - uses: bahmutov/npm-install@v1
-      - run: ./packages/tools/cli.js dns --name nft.storage --token ${{ secrets.CF_API_TOKEN }} --zone ${{ secrets.CF_ZONE }} --content ${{ github.event.inputs.frontend_cname }}
-      - run: echo "::warning::https://nft.storage"
+      - run: ./packages/tools/cli.js dns --name classic.nft.storage --token ${{ secrets.CF_API_TOKEN }} --zone ${{ secrets.CF_ZONE }} --content ${{ github.event.inputs.frontend_cname }}
+      - run: echo "::warning::https://classic.nft.storage"
   deploy-frontend-staging:
     if: github.event.inputs.environment == 'staging'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
-          node-version: '16'
+          node-version: '18'
       - uses: bahmutov/npm-install@v1
-      - run: ./packages/tools/cli.js dns --name staging.nft.storage --token ${{ secrets.CF_API_TOKEN }} --zone ${{ secrets.CF_ZONE }} --content ${{ github.event.inputs.frontend_cname }}
-      - run: echo "::warning::https://staging.nft.storage"
+      - run: ./packages/tools/cli.js dns --name classic-staging.nft.storage --token ${{ secrets.CF_API_TOKEN }} --zone ${{ secrets.CF_ZONE }} --content ${{ github.event.inputs.frontend_cname }}
+      - run: echo "::warning::https://classic-staging.nft.storage"
   deploy-frontend-dev:
     if: github.event.inputs.environment == ''
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
-          node-version: '16'
+          node-version: '18'
       - uses: bahmutov/npm-install@v1
       - run: ./packages/tools/cli.js dns --name dev.nft.storage --token ${{ secrets.CF_API_TOKEN }} --zone ${{ secrets.CF_ZONE }} --content ${{ github.event.inputs.frontend_cname }}
       - run: echo "::warning::https://dev.nft.storage"

.github/workflows/website.yml

@@ -17,11 +17,17 @@ jobs:
   check:
     name: Test
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        node-version:
+          - 18
+          - 20
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
-          node-version: '16'
+          node-version: ${{ matrix.node-version }}
       - uses: bahmutov/npm-install@v1
       - name: Run build
         env:
@@ -42,7 +48,8 @@ jobs:
         os:
           - ubuntu-20.04
         node-version:
-          - '16'
+          - 18
+          - 20
         test_results_path:
           # corresponds to playwright invocation/configuration
           - packages/website/test-results
@@ -97,7 +104,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
-          node-version: '16'
+          node-version: 20
           registry-url: https://registry.npmjs.org/
       - uses: bahmutov/npm-install@v1
       - name: Run build

decisions/20240313-try-w3up.md

@@ -48,13 +48,29 @@ Rationale
 - when nft.storage api handles request POST /upload/ (e.g. from the nft.storage ui), the uploaded file should be stored using up.web3.storage in a space associated with an web3.storage account controlled by an administrator of nft.storage
   - initially, this will only happen when a feature switch is enabled
 
+##### Open Architectural Decisions
+
+###### Should write to w3up happen sync or async?
+
+i.e. if the write to w3up fails, should the POST /uploads request that triggered it also fail ('sync'=yes)? Or should we only do the write to w3up after a successful response to POST /uploads ('async')?
+
+Initially, it is simplest to get something working in a sync way, so we'll do that to make sure the w3up-client creation, configuration, authorization, accounting, etc. is set up correctly.
+
+Once that works, we can look at deferring the write to w3up. We'd need to use some kind of job queue (e.g. [Cloudflare Queues](https://developers.cloudflare.com/queues/) or perhaps using the mysql db as a queue and nftstorage/cron as a worker).
+
 #### Data Model Changes
 
 Minimal. Only new configuration.
 
 We'll need to decide which storage space that uploads will be added to.
 To start, we'll just making everything store in a configured storage space, and the app will read that from configuration passed in environment variables, similar to env vars in [add-to-web3](https://github.com/web3-storage/add-to-web3?tab=readme-ov-file#generating-a-secret_key-and-proof).
 
+##### `W3UP_URL` Environment Variable
+
+configures the URL for connecting to w3up.
+
+optional. If not set, no URL will be passed to `@web3-storage/w3up-client` constructors. [Currently that means](https://github.com/web3-storage/w3up/blob/7a6385bef1dd424d5eb952528ae5d86a83837c80/packages/upload-client/src/service.js#L5) the default will be `up.web3.storage`.
+
 ##### `W3_NFTSTORAGE_SPACE` Environment Variable
 
 configures the web3.storage space that nfts will be stored in.
@@ -70,13 +86,13 @@ configures how nft.storage will authenticate to web3.storage when sending invoca
 configures the capabilities that nft.storage has access to when interacting with web3.storage to store nfts. These capabilities will usually be UCAN delegations whose audience is the identifier of `W3_NFTSTORAGE_PRINCIPAL`.
 W3_NFTSTORAGE_PROOF needs to have proof rooted in W3_NFTSTORAGE_SPACE that authorize W3_NFTSTORAGE_PRINCIPAL to store in W3_NFTSTORAGE_SPACE.
 
-##### `NFTSTORAGE_W3S_ALLOW_ACCOUNTS` Environment Variable
+##### `W3_NFTSTORAGE_ENABLE_W3UP_FOR_EMAILS` Environment Variable
 
 configures feature switch for which nftstorage accounts will have new uploads stored in web3.storage.
 
 Note: this environment variable may not be a permanent addition to the codebase. It's only meant to be used as a feature switch that decouples enabling the new functionality from deploying the new code. After testing, we may remove or change this feature switch when it is no longer useful.
 
-Format: JSON Array of Account Identifiers
+Format: JSON Array of email address strings.
 
 #### UI Changes
 
@@ -134,9 +150,12 @@ If we determine it is not desirable for nft.storage to use the entirety of w3up'
 Prototype Happy path
 
 - [x] @gobengo set up nft.storage localdev to be able to hack on packages/api
-- [ ] nft.storage/api: can be configured with `W3_NFTSTORAGE_PRINCIPAL` and `W3_NFTSTORAGE_PROOF` and use them to get a w3up-client at runtime
-- [ ] nft.storage/api: modify POST /upload/ handler to store using w3up-client@12
-- [ ] nft.storage/api: add feature switch requiring allowlist for new uploads to store in web3.storage
+- [x] nft.storage/api: can be configured with W3_URL
+- [x] add deps on w3up-client, ucanto
+  - [ x] upgrade typescript past 5.x and node past 16 as is required by w3up-client+ucanto
+- [x] nft.storage/api: can be configured with `W3_NFTSTORAGE_PRINCIPAL` and `W3_NFTSTORAGE_PROOF` and use them to get a w3up-client at runtime
+- [x] nft.storage/api: modify POST /upload/ handler to store using w3up-client@12
+- [x] nft.storage/api: add feature switch requiring allowlist for new uploads to store in web3.storage
 
 Rollout
 

package.json

@@ -39,14 +39,13 @@
     "prettier": "2.5.1",
     "rimraf": "^3.0.2",
     "simple-git-hooks": "^2.3.1",
-    "typescript": "4.5.3",
+    "typescript": "5.2.2",
     "webpack": "^5.72.0",
     "wrangler": "^2.0.23"
   },
   "resolutions": {
     "prettier": "2.5.1",
-    "@types/react": "^17.0.34",
-    "typescript": "4.5.3"
+    "@types/react": "^17.0.34"
   },
   "engines": {
     "node": ">= 16.0"

packages/api/package.json

@@ -27,10 +27,16 @@
     "@nftstorage/ipfs-cluster": "^5.0.1",
     "@noble/ed25519": "^1.6.1",
     "@supabase/postgrest-js": "^0.34.1",
+    "@ucanto/core": "^9.0.1",
+    "@ucanto/principal": "^9.0.0",
+    "@ucanto/server": "^9.0.1",
+    "@web3-storage/access": "^18.2.0",
     "@web3-storage/car-block-validator": "^1.2.0",
+    "@web3-storage/w3up-client": "^12.5.0",
     "cardex": "^1.0.0",
     "ipfs-car": "^0.6.1",
     "it-last": "^2.0.0",
+    "linkdex": "^3.0.0",
     "merge-options": "^3.0.4",
     "multiformats": "^9.6.4",
     "nanoid": "^3.1.30",

packages/api/src/bindings.d.ts

@@ -7,6 +7,7 @@ import { UserOutput, UserOutputKey } from './utils/db-client-types.js'
 import { DBClient } from './utils/db-client.js'
 import { LinkdexApi } from './utils/linkdex.js'
 import { Logging } from './utils/logs.js'
+import { Client as W3upClient } from '@web3-storage/w3up-client'
 
 export type RuntimeEnvironmentName = 'test' | 'dev' | 'staging' | 'production'
 
@@ -96,6 +97,28 @@ export interface ServiceConfiguration {
 
   /** Slack webhook url */
   SLACK_USER_REQUEST_WEBHOOK_URL: string
+
+  /** w3up connection URL (e.g. https://up.web3.storage) */
+  W3UP_URL?: string
+
+  /** w3up service DID (e.g. did:web:web3.storage) */
+  W3UP_DID?: string
+
+  /** base64 encoded multiformats ed25519 secretKey */
+  W3_NFTSTORAGE_PRINCIPAL?: string
+
+  /** CID (identity codec) of CAR-encoded UCAN DAG */
+  W3_NFTSTORAGE_PROOF?: string
+
+  /** did:key of the w3up space in which to store NFTs */
+  W3_NFTSTORAGE_SPACE?: string
+
+  /**
+   * JSON array of strings that are emails whose uploads should be uploaded via w3up.
+   * This is meant as a feature switch to test new functionality,
+   * and this configuration may be removed once the feature switch isn't needed to limit access.
+   */
+  W3_NFTSTORAGE_ENABLE_W3UP_FOR_EMAILS?: string
 }
 
 export interface Ucan {
@@ -123,11 +146,18 @@ export interface RouteContext {
   params: Record<string, string>
   db: DBClient
   log: Logging
-  linkdexApi?: LinkdexApi
+  linkdexApi: LinkdexApi
   s3Uploader: Uploader
   r2Uploader: Uploader
   ucanService: Service
   auth?: Auth
+  W3UP_DID?: string
+  W3UP_URL?: string
+  W3_NFTSTORAGE_PRINCIPAL?: string
+  W3_NFTSTORAGE_PROOF?: string
+  W3_NFTSTORAGE_SPACE?: string
+  W3_NFTSTORAGE_ENABLE_W3UP_FOR_EMAILS?: string
+  w3up?: W3upClient
 }
 
 export type Handler = (

packages/api/src/config.js

@@ -61,6 +61,14 @@ export function serviceConfigFromVariables(vars) {
     VERSION: vars.NFT_STORAGE_VERSION || NFT_STORAGE_VERSION,
     // @ts-ignore
     COMMITHASH: vars.NFT_STORAGE_COMMITHASH || NFT_STORAGE_COMMITHASH,
+
+    W3UP_URL: vars.W3UP_URL,
+    W3UP_DID: vars.W3UP_DID,
+    W3_NFTSTORAGE_PRINCIPAL: vars.W3_NFTSTORAGE_PRINCIPAL,
+    W3_NFTSTORAGE_PROOF: vars.W3_NFTSTORAGE_PROOF,
+    W3_NFTSTORAGE_SPACE: vars.W3_NFTSTORAGE_SPACE,
+    W3_NFTSTORAGE_ENABLE_W3UP_FOR_EMAILS:
+      vars.W3_NFTSTORAGE_ENABLE_W3UP_FOR_EMAILS,
   }
 }
 
@@ -123,6 +131,12 @@ export function loadConfigVariables() {
     'LINKDEX_URL',
     'S3_ENDPOINT',
     'SLACK_USER_REQUEST_WEBHOOK_URL',
+    'W3UP_URL',
+    'W3UP_DID',
+    'W3_NFTSTORAGE_SPACE',
+    'W3_NFTSTORAGE_PRINCIPAL',
+    'W3_NFTSTORAGE_PROOF',
+    'W3_NFTSTORAGE_ENABLE_W3UP_FOR_EMAILS',
   ]
 
   for (const name of optional) {
@@ -150,6 +164,12 @@ function parseRuntimeEnv(s) {
     return 'test'
   }
 
+  if (typeof s !== 'string') {
+    throw new Error(
+      `Unable to parse non-string (${typeof s}) environment name: ${s}`
+    )
+  }
+
   switch (s) {
     case 'test':
     case 'dev':