feat: nft.storage naive gateway implementation (#908)

Co-authored-by: Alan Shaw <alan.shaw@protocol.ai>

.github/workflows/gateway.yml

@@ -0,0 +1,41 @@
+name: Gateway
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - 'packages/gateway/**'
+      - '.github/workflows/gateway.yml'
+  pull_request:
+    paths:
+      - 'packages/gateway/**'
+      - '.github/workflows/gateway.yml'
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    name: Test
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v2
+        with:
+          node-version: '16'
+      - uses: bahmutov/npm-install@v1
+      - run: npx playwright install-deps
+      - run: yarn test:gateway
+  deploy-staging:
+    name: Deploy Staging
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+    runs-on: ubuntu-latest
+    needs: test
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v2
+        with:
+          node-version: '16'
+      - uses: bahmutov/npm-install@v1
+      - name: Publish app
+        uses: cloudflare/wrangler-action@1.3.0
+        with:
+          apiToken: ${{secrets.CF_API_TOKEN }}
+          workingDirectory: 'packages/gateway'
+          environment: 'staging'

.github/workflows/release.yml

@@ -117,6 +117,13 @@ jobs:
           apiToken: ${{ secrets.CF_API_TOKEN }}
           workingDirectory: 'packages/api'
           environment: 'production'
+      - name: Gateway - Deploy
+        if: ${{ steps.tag-release.outputs.release_created && matrix.package == 'gateway' }}
+        uses: cloudflare/wrangler-action@1.3.0
+        with:
+          apiToken: ${{ secrets.CF_API_TOKEN }}
+          workingDirectory: 'packages/gateway'
+          environment: 'production'
       - name: Website - Deploy
         if: ${{ steps.tag-release.outputs.release_created && matrix.package == 'website' }}
         run: ./packages/tools/cli.js deploy-website --email ${{ secrets.CF_EMAIL }} --key ${{secrets.CF_KEY}} --zone ${{ secrets.CF_ZONE }} --account ${{secrets.CF_ACCOUNT}}

package.json

@@ -7,6 +7,7 @@
     "test": "run-s test:*",
     "test:client": "yarn --cwd packages/client test",
     "test:api": "yarn --cwd packages/api test",
+    "test:gateway": "yarn --cwd packages/gateway test",
     "test:website": "yarn --cwd packages/website test",
     "build:client:docs": "yarn --cwd packages/client typedoc",
     "build:website": "yarn --cwd packages/website build",

packages/gateway/README.md

@@ -0,0 +1,31 @@
+# gateway.nft.storage
+
+> The IPFS gateway for nft.storage.
+
+## Getting started
+
+One time set up of your cloudflare worker subdomain for dev:
+
+- `npm install` - Install the project dependencies
+- Sign up to Cloudflare and log in with your default browser.
+- `npm i @cloudflare/wrangler -g` - Install the Cloudflare wrangler CLI
+- `wrangler login` - Authenticate your wrangler cli; it'll open your browser.
+- Copy your cloudflare account id from `wrangler whoami`
+- Update `wrangler.toml` with a new `env`. Set your env name to be the value of `whoami` on your system you can use `npm start` to run the worker in dev mode for you.
+
+  [**wrangler.toml**](./wrangler.toml)
+
+  ```toml
+  [env.bobbytables]
+  workers_dev = true
+  account_id = "<what does the `wrangler whoami` say>"
+  ```
+
+- `npm run publish` - Publish the worker under your env. An alias for `wrangler publish --env $(whoami)`
+- `npm start` - Run the worker in dev mode. An alias for `wrangler dev --env $(whoami)`
+
+You only need to `npm start` for subsequent runs. PR your env config to the `wrangler.toml` to celebrate 🎉
+
+## API
+
+TODO

packages/gateway/ava.config.js

@@ -0,0 +1,8 @@
+export default {
+  nonSemVerExperiments: {
+    configurableModuleFormat: true,
+  },
+  files: ['test/*.spec.js'],
+  timeout: '5m',
+  nodeArguments: ['--experimental-vm-modules'],
+}

packages/gateway/package.json

@@ -0,0 +1,31 @@
+{
+  "name": "gateway-nft-storage",
+  "version": "0.0.0",
+  "description": "IPFS gateway for nft.storage",
+  "private": true,
+  "type": "module",
+  "module": "./dist/index.mjs",
+  "scripts": {
+    "build": "node scripts/cli.js build",
+    "dev": "miniflare --watch --debug",
+    "deploy": "wrangler publish --env production",
+    "pretest": "npm run build",
+    "test": "npm-run-all -p -r mock:ipfs.io test:worker",
+    "test:worker": "ava --verbose test/*.spec.js",
+    "mock:ipfs.io": "smoke -p 9081 test/mocks/ipfs.io"
+  },
+  "dependencies": {
+    "multiformats": "^9.5.2"
+  },
+  "devDependencies": {
+    "ava": "^3.15.0",
+    "esbuild": "^0.14.2",
+    "git-rev-sync": "^3.0.1",
+    "miniflare": "^2.0.0-rc.2",
+    "npm-run-all": "^4.1.5",
+    "sade": "^1.7.4",
+    "smoke": "^3.1.1"
+  },
+  "author": "Vasco Santos <santos.vasco10@gmail.com>",
+  "license": "Apache-2.0 OR MIT"
+}

packages/gateway/scripts/cli.js

@@ -0,0 +1,39 @@
+#!/usr/bin/env node
+import fs from 'fs'
+import path from 'path'
+import { fileURLToPath } from 'url'
+import sade from 'sade'
+import { build } from 'esbuild'
+import git from 'git-rev-sync'
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url))
+const pkg = JSON.parse(
+  fs.readFileSync(path.join(__dirname, '..', 'package.json'), 'utf8')
+)
+
+const prog = sade('gateway')
+
+prog
+  .command('build')
+  .describe('Build the worker.')
+  .option('--env', 'Environment', 'dev')
+  .action(async (opts) => {
+    const version = `${pkg.name}@${pkg.version}-${opts.env}+${git.short(
+      __dirname
+    )}`
+
+    await build({
+      entryPoints: [path.join(__dirname, '../src/index.js')],
+      bundle: true,
+      format: 'esm',
+      outfile: 'dist/index.mjs',
+      legalComments: 'external',
+      define: {
+        global: 'globalThis',
+      },
+      minify: opts.env === 'dev' ? false : true,
+      sourcemap: true,
+    })
+  })
+
+prog.parse(process.argv)

packages/gateway/src/cors.js

@@ -0,0 +1,20 @@
+/* eslint-env serviceworker */
+
+/**
+ * @param {Request} request
+ * @param {Response} response
+ * @returns {Response}
+ */
+export function addCorsHeaders(request, response) {
+  // Clone the response so that it's no longer immutable (like if it comes from cache or fetch)
+  response = new Response(response.body, response)
+  const origin = request.headers.get('origin')
+  if (origin) {
+    response.headers.set('Access-Control-Allow-Origin', origin)
+    response.headers.set('Vary', 'Origin')
+  } else {
+    response.headers.set('Access-Control-Allow-Origin', '*')
+  }
+  response.headers.set('Access-Control-Expose-Headers', 'Link')
+  return response
+}

packages/gateway/src/error-handler.js

@@ -0,0 +1,11 @@
+/**
+ * @param {Error & {status?: number;code?: string;}} err
+ */
+export function errorHandler(err) {
+  // TODO: setup sentry
+  console.error(err.stack)
+
+  const status = err.status || 500
+
+  return new Response(err.message || 'Server Error', { status })
+}

packages/gateway/src/errors.js

@@ -0,0 +1,12 @@
+export class InvalidUrlError extends Error {
+  /**
+   * @param {string} message
+   */
+  constructor(message = 'invalid URL') {
+    super(message)
+    this.name = 'InvalidUrlError'
+    this.status = 400
+    this.code = InvalidUrlError.CODE
+  }
+}
+InvalidUrlError.CODE = 'ERROR_INVALID_URL'

packages/gateway/src/index.js

@@ -0,0 +1,43 @@
+import { addCorsHeaders } from './cors.js'
+import { errorHandler } from './error-handler.js'
+import { getCidFromSubdomainUrl } from './utils/cid.js'
+
+/**
+ * @typedef {Object} Env
+ * @property {string} IPFS_GATEWAY
+ */
+
+/**
+ * Handle gateway request
+ * @param {Request} request
+ * @param {Env} env
+ */
+async function handleRequest(request, env) {
+  const publicGatewayUrl = new URL('ipfs', env.IPFS_GATEWAY)
+  const url = new URL(request.url)
+  const cid = getCidFromSubdomainUrl(url)
+  const response = await fetch(
+    `${publicGatewayUrl}/${cid}${url.pathname || ''}`
+  )
+
+  // forward gateway response
+  return addCorsHeaders(request, response)
+}
+
+/**
+ * @param {Error} error
+ * @param {Request} request
+ */
+function serverError(error, request) {
+  return addCorsHeaders(request, errorHandler(error))
+}
+
+export default {
+  async fetch(request, env) {
+    try {
+      return await handleRequest(request, env)
+    } catch (error) {
+      return serverError(error, request)
+    }
+  },
+}

packages/gateway/src/utils/cid.js

@@ -0,0 +1,34 @@
+import { CID } from 'multiformats/cid'
+
+import { InvalidUrlError } from '../errors.js'
+
+/**
+ * Parse subdomain URL and return cid
+ *
+ * @param {URL} url
+ */
+export function getCidFromSubdomainUrl(url) {
+  // Replace "ipfs-staging" by "ipfs" if needed
+  const host = url.hostname.replace('ipfs-staging', 'ipfs')
+  const splitHost = host.split('.ipfs.')
+
+  if (!splitHost.length) {
+    throw new InvalidUrlError(url.hostname)
+  }
+
+  try {
+    return normalizeCid(splitHost[0])
+  } catch (err) {
+    throw new InvalidUrlError(`invalid CID: ${splitHost[0]}: ${err.message}`)
+  }
+}
+
+/**
+ * Parse CID and return normalized b32 v1
+ *
+ * @param {string} cid
+ */
+export function normalizeCid(cid) {
+  const c = CID.parse(cid)
+  return c.toV1().toString()
+}

packages/gateway/test/index.spec.js

@@ -0,0 +1,52 @@
+import test from 'ava'
+import { Miniflare } from 'miniflare'
+
+test.beforeEach((t) => {
+  // Create a new Miniflare environment for each test
+  const mf = new Miniflare({
+    // Autoload configuration from `.env`, `package.json` and `wrangler.toml`
+    envPath: true,
+    packagePath: true,
+    wranglerConfigPath: true,
+    // We don't want to rebuild our worker for each test, we're already doing
+    // it once before we run all tests in package.json, so disable it here.
+    // This will override the option in wrangler.toml.
+    buildCommand: undefined,
+    wranglerConfigEnv: 'test',
+  })
+
+  t.context = {
+    mf,
+  }
+})
+
+test('Fails when invalid cid is provided', async (t) => {
+  const { mf } = t.context
+
+  const invalidCid = 'bafy'
+  const response = await mf.dispatchFetch(
+    `https://${invalidCid}.ipfs.localhost:8787`
+  )
+  t.is(response.status, 400)
+
+  const textResponse = await response.text()
+  t.is(textResponse, `invalid CID: ${invalidCid}: Unexpected end of data`)
+})
+
+test('Gets content', async (t) => {
+  const { mf } = t.context
+
+  const response = await mf.dispatchFetch(
+    'https://bafkreidchi5c4c3kwr5rpkvvwnjz3lh44xi2y2lnbldehwmpplgynigidm.ipfs.localhost:8787'
+  )
+  t.is(await response.text(), 'Hello gateway.nft.storage!')
+})
+
+test('Gets content with path', async (t) => {
+  const { mf } = t.context
+
+  const response = await mf.dispatchFetch(
+    'https://bafkreidchi5c4c3kwr5rpkvvwnjz3lh44xi2y2lnbldehwmpplgynigidm.ipfs.localhost:8787/path'
+  )
+  t.is(await response.text(), 'Hello gateway.nft.storage resource!')
+})

packages/gateway/test/mocks/ipfs.io/get_ipfs#@cid#path.js

@@ -0,0 +1,10 @@
+/**
+ * https://github.com/sinedied/smoke#javascript-mocks
+ */
+module.exports = () => {
+  return {
+    statusCode: 200,
+    headers: { 'Content-Type': 'text/plain' },
+    body: 'Hello gateway.nft.storage resource!',
+  }
+}

packages/gateway/test/mocks/ipfs.io/get_ipfs#@cid.js

@@ -0,0 +1,10 @@
+/**
+ * https://github.com/sinedied/smoke#javascript-mocks
+ */
+module.exports = () => {
+  return {
+    statusCode: 200,
+    headers: { 'Content-Type': 'text/plain' },
+    body: 'Hello gateway.nft.storage!',
+  }
+}

packages/gateway/test/mocks/package.json

@@ -0,0 +1,5 @@
+{
+  "name": "mocks",
+  "version": "1.0.0",
+  "description": "just here to fix cjs loading"
+}