feat: http perma cache post (#28)

.env.tpl

@@ -0,0 +1,20 @@
+# Wrangler overrides
+DEBUG=true
+
+# API Secrets
+# the salt is literally the word 'secret', not a random string
+SALT=secret
+
+LOGTAIL_TOKEN=secret
+
+## API Sentry
+SENTRY_DSN=https://000000@0000000.ingest.sentry.io/00000
+SENTRY_TOKEN=secret
+SENTRY_UPLOAD=false
+
+## API PostgREST
+DATABASE_URL=http://localhost:3000
+DATABASE_TOKEN=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJzdXBhYmFzZSIsImlhdCI6MTYwMzk2ODgzNCwiZXhwIjoyNTUwNjUzNjM0LCJyb2xlIjoic2VydmljZV9yb2xlIn0.necIJaiP7X2T2QjGeV-FhpkizcNTX8HjDDBAxpgQTEI
+
+# Postgres Database
+DATABASE_CONNECTION=postgresql://postgres:postgres@localhost:5432/postgres

.github/workflows/api.yml

@@ -0,0 +1,98 @@
+name: api
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - 'packages/api/**'
+      - '.github/workflows/api.yml'
+  pull_request:
+    paths:
+      - 'packages/api/**'
+      - '.github/workflows/api.yml'
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    name: Test
+    steps:
+      - uses: actions/checkout@v2
+      - uses: pnpm/action-setup@v2.0.1
+        with:
+          version: 6.32.x
+      - uses: actions/setup-node@v2
+        with:
+          node-version: 16
+      - run: pnpm install
+      - run: npx playwright install-deps
+      - run: pnpm test:api
+        env:
+          DATABASE_URL: http://localhost:3000
+          DATABASE_TOKEN: eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJzdXBhYmFzZSIsImlhdCI6MTYwMzk2ODgzNCwiZXhwIjoyNTUwNjUzNjM0LCJyb2xlIjoic2VydmljZV9yb2xlIn0.necIJaiP7X2T2QjGeV-FhpkizcNTX8HjDDBAxpgQTEI
+          DATABASE_CONNECTION: postgresql://postgres:postgres@localhost:5432/postgres
+  deploy-staging:
+    name: Deploy Staging
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+    runs-on: ubuntu-latest
+    needs: test
+    steps:
+      - uses: actions/checkout@v2
+      - uses: pnpm/action-setup@v2.0.1
+        with:
+          version: 6.32.x
+      - uses: actions/setup-node@v2
+        with:
+          cache: 'pnpm'
+      - run: pnpm install
+      - name: Publish app
+        # FIXME: uses Node.js 16 not 12 (upcoming release).
+        # FIXME: update to tag > 1.3.0 when released.
+        uses: cloudflare/wrangler-action@6f62debcf8abf8e33e41343df9d7ab49612c324d
+        env:
+          ENV: 'staging' # inform the build process what the env is
+          SENTRY_TOKEN: ${{secrets.SENTRY_TOKEN}}
+          SENTRY_UPLOAD: ${{ secrets.SENTRY_UPLOAD }}
+        with:
+          apiToken: ${{secrets.CF_GATEWAY_TOKEN }}
+          workingDirectory: 'packages/api'
+          environment: 'staging'
+  release:
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+    name: Release
+    runs-on: ubuntu-latest
+    needs: test
+    steps:
+      - uses: GoogleCloudPlatform/release-please-action@v3
+        id: tag-release
+        with:
+          path: packages/api
+          token: ${{ secrets.GITHUB_TOKEN }}
+          release-type: node
+          monorepo-tags: true
+          package-name: api
+      - uses: actions/checkout@v2
+        if: ${{ steps.tag-release.outputs.releases_created }}
+      - uses: pnpm/action-setup@v2.0.1
+        if: ${{ steps.tag-release.outputs.releases_created }}
+        with:
+          version: 6.32.x
+      - uses: actions/setup-node@v2
+        if: ${{ steps.tag-release.outputs.releases_created }}
+        with:
+          cache: 'pnpm'
+          node-version: 16
+          registry-url: 'https://registry.npmjs.org'
+      - run: pnpm install
+        if: ${{ steps.tag-release.outputs.releases_created }}
+      - name: Deploy
+        if: ${{ steps.tag-release.outputs.releases_created }}
+        # FIXME: uses Node.js 16 not 12 (upcoming release).
+        # FIXME: update to tag > 1.3.0 when released.
+        uses: cloudflare/wrangler-action@6f62debcf8abf8e33e41343df9d7ab49612c324d
+        env:
+          ENV: 'production' # inform the build process what the env is
+          SENTRY_TOKEN: ${{ secrets.SENTRY_TOKEN }}
+          SENTRY_UPLOAD: ${{ secrets.SENTRY_UPLOAD }}
+        with:
+          apiToken: ${{ secrets.CF_GATEWAY_TOKEN }}
+          workingDirectory: 'packages/api'
+          environment: 'production'

package.json

@@ -5,6 +5,7 @@
   ],
   "scripts": {
     "test": "run-s test:*",
+    "test:api": "pnpm --filter api test",
     "test:website": "pnpm --filter website test",
     "test:edge-gateway": "pnpm --filter edge-gateway test",
     "build:website": "pnpm --filter website build",

packages/api/README.md

@@ -0,0 +1,71 @@
+# nftstorage.link API
+
+> The API for nftstorage.link provides a perma-cache layer for super fast content addressing retrieval.
+
+## Getting started
+
+One time set up of your cloudflare worker subdomain for dev:
+
+- `pnpm install` - Install the project dependencies from the monorepo root directory
+- Sign up to Cloudflare and log in with your default browser.
+- `npm i @cloudflare/wrangler -g` - Install the Cloudflare wrangler CLI
+- `wrangler login` - Authenticate your wrangler cli; it'll open your browser.
+- Copy your cloudflare account id from `wrangler whoami`
+- Update `wrangler.toml` with a new `env`. Set your env name to be the value of `whoami` on your system you can use `pnpm start` to run the worker in dev mode for you.
+
+  [**wrangler.toml**](./wrangler.toml)
+
+  ```toml
+  [env.bobbytables]
+  workers_dev = true
+  account_id = "<what does the `wrangler whoami` say>"
+  ```
+
+- Add secrets
+
+  ```sh
+    wrangler secret put SALT --env production # open `https://csprng.xyz/v1/api` in the browser and use the value of `Data`
+    wrangler secret put SENTRY_DSN --env $(whoami) # Get from Sentry (not required for dev)
+    wrangler secret put LOGTAIL_TOKEN --env $(whoami) # Get from Logtail
+    wrangler secret put DATABASE_TOKEN --env $(whoami) # Get from database account
+  ```
+
+- Add KV namespaces
+
+  ```sh
+  wrangler kv:namespace create PERMACACHE --preview --env $(whoami)
+  # Outputs something like: `{ binding = "PERMACACHE", preview_id = "7e441603d1bc4d5a87f6cecb959018e4" }`
+  # but you need to put `{ binding = "PERMACACHE", preview_id = "7e441603d1bc4d5a87f6cecb959018e4", id = "7e441603d1bc4d5a87f6cecb959018e4" }` inside the `kv_namespaces`.
+  # for production: wrangler kv:namespace create PERMACACHE --env production
+  wrangler kv:namespace create PERMACACHE_HISTORY --preview --env $(whoami)
+  # Outputs something like: `{ binding = "PERMACACHE_HISTORY", preview_id = "bac8069051ee4796a305b4d3f366b930" }`
+  # but you need to put `{ binding = "PERMACACHE_HISTORY", preview_id = "bac8069051ee4796a305b4d3f366b930", id = "bac8069051ee4796a305b4d3f366b930" }` inside the `kv_namespaces`.
+  # for production: wrangler kv:namespace create PERMACACHE_HISTORY --env production
+  ```
+
+- Add R2 bucket (Note that it is only available as Private Beta at the time of writing)
+
+  ```sh
+  wrangler r2 bucket create super-hot --env $(whoami)
+  # 🌀  Creating bucket "super-hot"
+  # ✨  Success!
+  ```
+
+- `pnpm run publish` - Publish the worker under your env. An alias for `wrangler publish --env $(whoami)`
+- `pnpm start` - Run the worker in dev mode. An alias for `wrangler dev --env $(whoami)`
+
+You only need to `pnpm start` for subsequent runs. PR your env config to the `wrangler.toml` to celebrate 🎉
+
+## High level architecture
+
+TODO
+
+![High level Architecture](./nftstorage.link-super-hot.jpg)
+
+## Usage
+
+TODO
+
+### Rate limiting
+
+nft.storage Gateway is currently rate limited at 200 requests per minute to a given IP Address. In the event of a rate limit, the IP will be blocked for 30 seconds.

packages/api/ava.config.cjs

@@ -0,0 +1,21 @@
+const fs = require('fs')
+const path = require('path')
+const dotenv = require('dotenv')
+
+const envPath = path.join(__dirname, '../../.env')
+if (fs.statSync(envPath, { throwIfNoEntry: false })) {
+  dotenv.config({
+    path: envPath,
+  })
+}
+
+module.exports = {
+  nonSemVerExperiments: {
+    configurableModuleFormat: true,
+  },
+  files: ['test/*.spec.js'],
+  timeout: '5m',
+  concurrency: 1,
+  nodeArguments: ['--experimental-vm-modules'],
+  require: ['dotenv/config', './test/_setup-browser-env.js'],
+}

packages/api/docker/docker-compose.yml

@@ -0,0 +1,9 @@
+version: '3.6'
+
+services:
+  ipfs1:
+    container_name: ipfs1
+    image: ipfs/go-ipfs:v0.12.0
+    ports:
+      - '9089:5001' # ipfs api - expose if needed/wanted
+      - '9081:8080' # ipfs gateway - expose if needed/wanted

packages/api/package.json

@@ -0,0 +1,49 @@
+{
+  "name": "api",
+  "version": "1.0.0",
+  "description": "API for nftstorage.link IPFS edge gateway",
+  "private": true,
+  "type": "module",
+  "main": "./dist/worker.mjs",
+  "scripts": {
+    "build": "node scripts/cli.js build",
+    "dev": "miniflare --watch --debug",
+    "deploy": "wrangler publish --env production",
+    "test": "npm run test:setup && npm run test:worker && npm run test:teardown",
+    "test:setup": "npm run build && npm run docker:start",
+    "test:worker": "ava --verbose test/*.spec.js",
+    "test:teardown": "npm run docker:stop",
+    "docker:start": "npm run docker:db:start && npm run docker:ipfs:start",
+    "docker:stop": "npm run docker:db:stop && npm run docker:ipfs:stop",
+    "docker:db:start": "node scripts/cli.js db --start && node scripts/cli.js db-sql --cargo --testing --reset",
+    "docker:db:stop": "node scripts/cli.js db --clean",
+    "docker:ipfs:start": "node scripts/cli.js ipfs --start",
+    "docker:ipfs:stop": "node scripts/cli.js ipfs --stop"
+  },
+  "dependencies": {
+    "@supabase/postgrest-js": "^0.37.2",
+    "itty-router": "^2.4.5",
+    "multiformats": "^9.6.4",
+    "nanoid": "^3.1.30",
+    "toucan-js": "^2.5.0"
+  },
+  "devDependencies": {
+    "@sentry/cli": "^1.71.0",
+    "@web-std/fetch": "^4.1.0",
+    "ava": "^3.15.0",
+    "browser-env": "^3.3.0",
+    "dotenv": "^16.0.0",
+    "edge-gateway": "^1.5.5",
+    "esbuild": "^0.14.2",
+    "execa": "^5.1.1",
+    "git-rev-sync": "^3.0.1",
+    "miniflare": "^2.2.0",
+    "nft.storage-api": "https://gitpkg.now.sh/nftstorage/nft.storage/packages/api?019a505e8f4bb93a24b8c480646779f5e4b66326",
+    "npm-run-all": "^4.1.5",
+    "pg": "^8.7.3",
+    "sade": "^1.7.4",
+    "uint8arrays": "^3.0.0"
+  },
+  "author": "Vasco Santos <santos.vasco10@gmail.com>",
+  "license": "Apache-2.0 OR MIT"
+}

packages/api/scripts/build.js

@@ -0,0 +1,61 @@
+import fs from 'fs'
+import path from 'path'
+import { fileURLToPath } from 'url'
+import { build } from 'esbuild'
+import git from 'git-rev-sync'
+import Sentry from '@sentry/cli'
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url))
+
+const pkg = JSON.parse(
+  fs.readFileSync(path.join(__dirname, '..', 'package.json'), 'utf8')
+)
+
+export async function buildCmd(opts) {
+  const sentryRelease = `nftstorage.link-api@${pkg.version}-${
+    opts.env || 'dev'
+  }+${git.short(__dirname)}`
+  console.log(`Building ${sentryRelease}`)
+
+  await build({
+    entryPoints: [path.join(__dirname, '..', 'src', 'index.js')],
+    bundle: true,
+    format: 'esm',
+    outfile: path.join(__dirname, '..', 'dist', 'worker.mjs'),
+    legalComments: 'external',
+    define: {
+      SENTRY_RELEASE: JSON.stringify(sentryRelease),
+      VERSION: JSON.stringify(pkg.version),
+      COMMITHASH: JSON.stringify(git.long(__dirname)),
+      BRANCH: JSON.stringify(git.branch(__dirname)),
+      global: 'globalThis',
+    },
+    minify: opts.env === 'dev' ? false : true,
+    sourcemap: 'external',
+  })
+
+  // Sentry release and sourcemap upload
+  if (process.env.SENTRY_UPLOAD === 'true') {
+    const cli = new Sentry(undefined, {
+      authToken: process.env.SENTRY_TOKEN,
+      org: 'protocol-labs-it',
+      project: 'nftstorage.link-api',
+      dist: git.short(__dirname),
+    })
+
+    await cli.releases.new(sentryRelease)
+    await cli.releases.setCommits(sentryRelease, {
+      auto: true,
+      ignoreEmpty: true,
+      ignoreMissing: true,
+    })
+    await cli.releases.uploadSourceMaps(sentryRelease, {
+      include: ['./dist'],
+      ext: ['map', 'mjs'],
+    })
+    await cli.releases.finalize(sentryRelease)
+    await cli.releases.newDeploy(sentryRelease, {
+      env: opts.env,
+    })
+  }
+}

packages/api/scripts/cli.js

@@ -0,0 +1,50 @@
+#!/usr/bin/env node
+import path from 'path'
+import dotenv from 'dotenv'
+import { fileURLToPath } from 'url'
+import sade from 'sade'
+
+import { buildCmd } from './build.js'
+import { dbCmd } from '../node_modules/nft.storage-api/scripts/cmds/db.js'
+import { dbSqlCmd } from '../node_modules/nft.storage-api/scripts/cmds/db-sql.js'
+import { ipfsCmd } from '../node_modules/edge-gateway/scripts/ipfs.js'
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url))
+dotenv.config({
+  path: path.join(__dirname, '..', '..', '..', '.env'),
+})
+
+const prog = sade('nftstorage.link-api')
+
+prog
+  .command('build')
+  .describe('Build the worker.')
+  .option('--env', 'Environment', process.env.ENV)
+  .action(buildCmd)
+  .command('ipfs')
+  .describe('Run ipfs node')
+  .option('--start', 'Start docker container', false)
+  .option('--stop', 'Stop and clean all dockers artifacts', false)
+  .action((opts) =>
+    ipfsCmd({
+      ...opts,
+      composePath: path.join(__dirname, '../docker/docker-compose.yml'),
+      containerName: 'ipfs1',
+    })
+  )
+  .command('db')
+  .describe('Run docker compose to setup pg and pgrest')
+  .option('--init', 'Init docker container', false)
+  .option('--start', 'Start docker container', false)
+  .option('--stop', 'Stop docker container', false)
+  .option('--project', 'Project name', 'nftstorage.link-api')
+  .option('--clean', 'Clean all dockers artifacts', false)
+  .action(dbCmd)
+  .command('db-sql')
+  .describe('Database scripts')
+  .option('--reset', 'Reset db before running SQL.', false)
+  .option('--cargo', 'Import cargo data.', false)
+  .option('--testing', 'Tweak schema for testing.', false)
+  .action(dbSqlCmd)
+
+prog.parse(process.argv)