feat: support CAR file uploads (#178)

A first pass at handling CAR uploads, smaller than 100MB. (no car-chunking here). The aim was the smallest change that woud allow users to create and upload their own CARs and have nft.storage handle it correctly, to unblock users who want to pre-compute the root CID for their files. Turns out it was almost supported already. Good work Spark!

- api: check for Content-Type: application/car and handle cluster size vs bytes difference and test it.
- api: add a `smoke` mock for posting small car files to cluster.
- client: add isCar flag to storeBlob to force application/car content type on upload.
- website: add is CAR? check box to file upload form to try it out, and an explainer.
- website: update openAPI schema

Fixes #172 

| upload form | details expanded  |
|-------------|-------------------|
| <img width="904" alt="Screenshot 2021-06-07 at 14 34 39" src="https://user-images.githubusercontent.com/58871/121255774-b6618180-c8a3-11eb-9119-29b32bd0902f.png"> |  <img width="904" alt="Screenshot 2021-06-07 at 14 34 50" src="https://user-images.githubusercontent.com/58871/121255797-bfeae980-c8a3-11eb-8b40-d79104e03ecc.png">

License: MIT
Signed-off-by: Oli Evans <oli@tableflip.io>

packages/api/src/routes/nfts-upload.js

@@ -49,15 +49,20 @@ export async function upload(event, ctx) {
     if (blob.size === 0) {
       throw new HTTPError('Empty payload', 400)
     }
-    const { cid, size } = await cluster.add(blob)
+    // Ensure car blob.type is set; it is used by the cluster client to set the foramt=car flag on the /add call.
+    const content = contentType.includes('application/car')
+      ? blob.slice(0, blob.size, 'application/car')
+      : blob
+    // cluster returns `bytes` rather than `size` when upload is a CAR.
+    const { cid, size, bytes } = await cluster.add(content)
     nft = {
       cid,
       created,
-      type: blob.type,
+      type: content.type,
       scope: tokenName,
       files: [],
     }
-    nftSize = size
+    nftSize = size || bytes
   }
 
   let pin = await pins.get(nft.cid)

packages/api/test/mocks/cluster/post_add$format=car.js

@@ -0,0 +1,26 @@
+const { CarReader } = require('@ipld/car')
+
+/**
+ * https://github.com/sinedied/smoke#javascript-mocks
+ * @typedef {{ buffer: Buffer, originalname: string }} MultrFile
+ * @param {{ query: Record<string, string>, files: MultrFile[] }} request
+ */
+module.exports = async ({ query, files }) => {
+  const car = await CarReader.fromBytes(files[0].buffer)
+  const roots = await car.getRoots()
+  // @ts-ignore
+  const { cid, bytes } = await car.get(roots[0])
+  const result = {
+    cid: {
+      '/': cid.toString(),
+    },
+    name: files[0].originalname,
+    // car uploads may not be unixfs, so get a bytes property instead of `size` https://github.com/ipfs/ipfs-cluster/issues/1362
+    bytes: bytes.length,
+  }
+  return {
+    statusCode: 200,
+    headers: { 'Content-Type': 'application/json' },
+    body: query['stream-channels'] === 'false' ? [result] : result,
+  }
+}

packages/api/test/nfts-upload.spec.js

@@ -4,38 +4,46 @@ import stores from './scripts/stores.js'
 import { endpoint } from './scripts/constants.js'
 import { signJWT } from '../src/utils/jwt.js'
 import { SALT } from './scripts/worker-globals.js'
+import { createCar } from './scripts/car.js'
+
+/**
+ * @param {{publicAddress?: string, issuer?: string, name?: string}} userInfo
+ */
+async function createTestUser({
+  publicAddress = `0x73573r${Date.now()}`,
+  issuer = `did:eth:${publicAddress}`,
+  name = 'A Tester',
+} = {}) {
+  const token = await signJWT(
+    {
+      sub: issuer,
+      iss: 'nft-storage',
+      iat: Date.now(),
+      name: 'test',
+    },
+    SALT
+  )
+  await stores.users.put(
+    issuer,
+    JSON.stringify({
+      sub: issuer,
+      nickname: 'testymctestface',
+      name,
+      email: 'a.tester@example.org',
+      picture: 'http://example.org/avatar.png',
+      issuer,
+      publicAddress,
+      tokens: { test: token },
+    })
+  )
+  return { token, issuer }
+}
 
 describe('/upload', () => {
   beforeEach(clearStores)
 
   it('should upload a single file', async () => {
-    const publicAddress = `0x73573r${Date.now()}`
-    const issuer = `did:eth:${publicAddress}`
-    const name = 'A Tester'
-
-    const token = await signJWT(
-      {
-        sub: issuer,
-        iss: 'nft-storage',
-        iat: Date.now(),
-        name: 'test',
-      },
-      SALT
-    )
-
-    await stores.users.put(
-      issuer,
-      JSON.stringify({
-        sub: issuer,
-        nickname: 'testymctestface',
-        name,
-        email: 'a.tester@example.org',
-        picture: 'http://example.org/avatar.png',
-        issuer,
-        publicAddress,
-        tokens: { test: token },
-      })
-    )
+    const { token, issuer } = await createTestUser()
 
     const file = new Blob(['hello world!'])
     // expected CID for the above data
@@ -45,18 +53,61 @@ describe('/upload', () => {
       headers: { Authorization: `Bearer ${token}` },
       body: file,
     })
-    assert(res)
-    assert(res.ok)
+    assert(res, 'Server responded')
+    assert(res.ok, 'Server response ok')
+    const { ok, value } = await res.json()
+    assert(ok, 'Server response payload has `ok` property')
+    assert.strictEqual(value.cid, cid, 'Server responded with expected CID')
+
+    const nftData = await stores.nfts.get(`${issuer}:${cid}`)
+    assert(nftData, 'nft data was stored')
+
+    const pinData = await stores.pins.getWithMetadata(cid)
+    assert(pinData.metadata, 'pin metadata was stored')
+    assert.strictEqual(
+      // @ts-ignore
+      pinData.metadata.status,
+      'pinned',
+      'pin status is "pinned"'
+    )
+  })
+
+  it('should upload a single CAR file', async () => {
+    const { token, issuer } = await createTestUser()
+
+    const { root, car } = await createCar('hello world!')
+    // expected CID for the above data
+    const cid = 'bafkreidvbhs33ighmljlvr7zbv2ywwzcmp5adtf4kqvlly67cy56bdtmve'
+    assert.strictEqual(root.toString(), cid, 'car file has correct root')
+    const res = await fetch(new URL('upload', endpoint).toString(), {
+      method: 'POST',
+      headers: {
+        Authorization: `Bearer ${token}`,
+        'Content-Type': 'application/car',
+      },
+      body: car,
+    })
+
+    assert(res, 'Server responded')
+    assert(res.ok, 'Server response ok')
     const { ok, value } = await res.json()
-    assert(ok)
-    assert.strictEqual(value.cid, cid)
+    assert(ok, 'Server response payload has `ok` property')
+    assert.strictEqual(value.cid, cid, 'Server responded with expected CID')
 
     const nftData = await stores.nfts.get(`${issuer}:${cid}`)
-    assert(nftData)
+    assert(nftData, 'nft data was stored')
 
     const pinData = await stores.pins.getWithMetadata(cid)
-    assert(pinData.metadata)
+    assert(pinData.metadata, 'pin metadata was stored')
+    // ipfs dag stat bafkreidvbhs33ighmljlvr7zbv2ywwzcmp5adtf4kqvlly67cy56bdtmve
+    // Size: 12, NumBlocks: 1
     // @ts-ignore
-    assert.strictEqual(pinData.metadata.status, 'pinned')
+    assert.strictEqual(pinData.metadata.size, 12, 'pin size correct')
+    assert.strictEqual(
+      // @ts-ignore
+      pinData.metadata.status,
+      'pinned',
+      'pin status is "pinned"'
+    )
   })
 })

packages/api/test/scripts/car.js

@@ -0,0 +1,19 @@
+import * as Block from 'multiformats/block'
+import * as Raw from 'multiformats/codecs/raw'
+import { sha256 } from 'multiformats/hashes/sha2'
+import * as CAR from '../../src/utils/car.js'
+
+/**
+ * @param {string} str Data to encode into CAR file.
+ */
+export async function createCar(str) {
+  const value = new TextEncoder().encode(str)
+  const block = await Block.encode({
+    value,
+    codec: Raw,
+    hasher: sha256,
+  })
+  const root = block.cid
+  const car = await CAR.encode([root], [block])
+  return { root, car }
+}

packages/client/examples/browser/README.md

@@ -1,11 +1,18 @@
 # nft.storage browser examples
 
-Here are two examples for using the nft.storage client from the browser.
+Examples using the nft.storage client from the browser.
 
 - `store.html` - creates ERC-1155 compatible metadata
 - `storeBlob.html` - uploads a single file using `storeBlob`
 - `storeDirectory.html` - uploads a multiple files using `storeDirectory`
 
+These demos use https://skypack.dev to bundle the deps up.
+
+**Note**
+
+At time of writing skypack's bundle of `ipfs-car` is incorrect, so a CAR upload demo is provided in /car that
+the CAR upload demo converts it's deps to local ESM modules with [`vite`](https://vitejs.dev/), in the /car folder.
+
 ## Setup
 
 Register an account at https://nft.storage and create a new API key.

packages/client/examples/browser/car/.gitignore

@@ -0,0 +1,5 @@
+node_modules
+.DS_Store
+dist
+dist-ssr
+*.local

packages/client/examples/browser/car/README.md

@@ -0,0 +1,16 @@
+# CAR upload demo - nft.storage
+
+A demo using ipfs-car in the browser to create a CAR file and pre-calculate the CID for an asset then storing it on nft.storage and confirming that it uses the exact same CID for the asset.
+
+## Getting started
+
+```console
+npm install
+npm run dev
+
+# or
+yarn
+yarn dev
+```
+
+Then visit `http://localhost:3000?key=<your nft.storage API KEY here>`

packages/client/examples/browser/car/index.html

@@ -0,0 +1,11 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <title>CAR upload - nft.storage</title>
+  </head>
+  <body>
+    <pre id="out"></pre>
+    <script type="module" src="/main.js"></script>
+  </body>
+</html>

packages/client/examples/browser/car/main.js

@@ -0,0 +1,35 @@
+import { NFTStorage } from 'nft.storage'
+import { packToBlob } from 'ipfs-car/pack/blob'
+
+const endpoint = 'http://api.nft.storage' // the default
+const token =
+  new URLSearchParams(window.location.search).get('key') || 'API_KEY' // your API key from https://nft.storage/manage
+
+function log(msg) {
+  msg = JSON.stringify(msg, null, 2)
+  document.getElementById('out').innerHTML += `${msg}\n`
+}
+
+async function main() {
+  const store = new NFTStorage({ endpoint, token })
+  const data = 'Hello nft.storage!'
+
+  // locally chunk'n'hash the data to get the CID and pack the blocks in to a CAR
+  const { root, car } = await packToBlob({
+    input: [new TextEncoder().encode(data)],
+  })
+  const expectedCid = root.toString()
+  console.log({ expectedCid })
+
+  // send the CAR to nft.storage, the returned CID will match the one we created above.
+  const cid = await store.storeCar(car)
+
+  // verify the service is storing the CID we expect
+  const cidsMatch = expectedCid === cid
+  log({ data, cid, expectedCid, cidsMatch })
+
+  // check that the CID is pinned
+  const status = await store.status(cid)
+  log(status)
+}
+main()

packages/client/examples/browser/car/package.json

@@ -0,0 +1,15 @@
+{
+  "version": "0.0.0",
+  "scripts": {
+    "dev": "vite",
+    "build": "vite build",
+    "serve": "vite preview"
+  },
+  "devDependencies": {
+    "vite": "^2.3.7"
+  },
+  "dependencies": {
+    "ipfs-car": "^0.2.4",
+    "nft.storage": "../../../"
+  }
+}

packages/client/examples/node.js/README.md

@@ -1,9 +1,10 @@
 # nft.storage Node.js examples
 
-Here are two examples for using the nft.storage client from Node.js.
+Examples for using the nft.storage client from Node.js.
 
 - `store.js` - creates ERC-1155 compatible metadata
 - `storeBlob.js` - uploads a single file using `storeBlob`
+- `storeCar.js` - uploads a Content Addressed Archive using `storeCar`
 - `storeDirectory.js` - uploads a multiple files using `storeDirectory`
 
 ## Setup
@@ -16,7 +17,7 @@ npm install
 
 Register an account at https://nft.storage and create a new API key.
 
-Open `store.js`/`storeBlob.js`/`storeDirectory.js` and replace `API_KEY` in the code with your key.
+Open `store.js`/`storeBlob.js`/`storeCar.js`/`storeDirectory.js` and replace `API_KEY` in the code with your key.
 
 ## Running
 
@@ -25,5 +26,7 @@ node store.js
 # or
 node storeBlob.js
 # or
+node storeCar.js
+# or
 node storeDirectory.js
 ```

packages/client/examples/node.js/package.json

@@ -11,6 +11,7 @@
   "author": "Alan Shaw",
   "license": "(Apache-2.0 AND MIT)",
   "dependencies": {
+    "ipfs-car": "^0.2.4",
     "nft.storage": "^0.3.9"
   }
 }

packages/client/examples/node.js/storeCar.js

@@ -0,0 +1,30 @@
+import fs from 'fs'
+import { packToBlob } from 'ipfs-car/pack/blob'
+import { NFTStorage } from '../../src/lib.js'
+
+const endpoint = 'https://api.nft.storage' // the default
+const token = 'API_KEY' // your API key from https://nft.storage/manage
+
+async function main() {
+  const storage = new NFTStorage({ endpoint, token })
+
+  // locally chunk'n'hash the file to get the CID and pack the blocks in to a CAR
+  const { root, car } = await packToBlob({
+    input: fs.createReadStream('pinpie.jpg'),
+  })
+
+  const expectedCid = root.toString()
+  console.log({ expectedCid })
+
+  // send the CAR to nft.storage, setting isCar to true
+  const cid = await storage.storeCar(car)
+
+  // verify the service stored the CID we expected
+  const cidsMatch = expectedCid === cid
+  console.log({ cid, expectedCid, cidsMatch })
+
+  // check that the CID is pinned
+  const status = await storage.status(cid)
+  console.log(status)
+}
+main()

packages/client/package.json

@@ -45,6 +45,7 @@
     "@ssttevee/multipart-parser": "0.1.8",
     "@types/mocha": "8.2.2",
     "hundreds": "0.0.9",
+    "ipfs-car": "^0.2.4",
     "ipfs-unixfs-importer": "6.0.1",
     "ipld": "0.29.0",
     "ipld-dag-pb": "0.22.2",