feat: use multiple gateways and track metrics (#961)

packages/gateway/README.md

@@ -32,6 +32,12 @@ One time set up of your cloudflare worker subdomain for dev:
 
 You only need to `npm start` for subsequent runs. PR your env config to the `wrangler.toml` to celebrate 🎉
 
-## API
+## High level architecture
 
-TODO
+![High level Architecture](./gateway.nft.storage.jpg)
+
+The IPFS gateway for nft.storage is not "another gateway", but a caching layer for NFT’s that sits on top of existing IPFS public gateways.
+
+## Persistence
+
+Several metrics per gateway are persisted to track the performance of each public gateway over time. Moreover, the list of gateways that have previously fetched successfully a given CID are also persisted.

packages/gateway/package.json

@@ -10,12 +10,16 @@
     "dev": "miniflare --watch --debug",
     "deploy": "wrangler publish --env production",
     "pretest": "npm run build",
-    "test": "npm-run-all -p -r mock:ipfs.io test:worker",
+    "test": "npm-run-all -p -r mock:ipfs.io mock:cf-ipfs.com test:worker",
     "test:worker": "ava --verbose test/*.spec.js",
-    "mock:ipfs.io": "smoke -p 9081 test/mocks/ipfs.io"
+    "mock:ipfs.io": "smoke -p 9081 test/mocks/ipfs.io",
+    "mock:cf-ipfs.com": "smoke -p 9082 test/mocks/cf-ipfs.com"
   },
   "dependencies": {
+    "itty-router": "^2.4.5",
     "multiformats": "^9.5.2",
+    "p-any": "^4.0.0",
+    "p-settle": "^5.0.0",
     "toucan-js": "^2.5.0"
   },
   "devDependencies": {
@@ -25,6 +29,7 @@
     "git-rev-sync": "^3.0.1",
     "miniflare": "^2.0.0-rc.2",
     "npm-run-all": "^4.1.5",
+    "p-retry": "^5.0.0",
     "sade": "^1.7.4",
     "smoke": "^3.1.1"
   },

packages/gateway/src/constants.js

@@ -0,0 +1 @@
+export const METRICS_CACHE_MAX_AGE = 10 * 60 // in seconds (10 minutes)

packages/gateway/src/cors.js

@@ -1,5 +1,20 @@
 /* eslint-env serviceworker */
 
+/**
+ * @param {import('itty-router').RouteHandler} handler
+ * @returns {import('itty-router').RouteHandler}
+ */
+export function withCorsHeaders(handler) {
+  /**
+   * @param {Request} request
+   * @returns {Promise<Response>}
+   */
+  return async (request, ...rest) => {
+    const response = await handler(request, ...rest)
+    return addCorsHeaders(request, response)
+  }
+}
+
 /**
  * @param {Request} request
  * @param {Response} response

packages/gateway/src/durable-objects/cids.js

@@ -0,0 +1,44 @@
+import { normalizeCid } from '../utils/cid.js'
+
+/**
+ * @typedef {Object} CidUpdateRequest
+ * @property {string} cid
+ * @property {string[]} urls gateway URLs
+ */
+
+/**
+ * Durable Object for tracking CIDs fetching state.
+ * For each CID requested, a list of the gateways that successfully fetched it are stored.
+ */
+export class CidsTracker0 {
+  constructor(state) {
+    this.state = state
+  }
+
+  // Handle HTTP requests from clients.
+  async fetch(request) {
+    // Apply requested action.
+    let url = new URL(request.url)
+
+    if (url.pathname === '/update') {
+      /** @type {CidUpdateRequest} */
+      const data = await request.json()
+
+      await this.state.storage.put(data.cid, data.urls)
+      return new Response()
+    } else if (url.pathname.includes('/status')) {
+      const cid = url.pathname.split('/status/')[1]
+      const nCid = normalizeCid(cid)
+
+      const stored = await this.state.storage.get(nCid)
+
+      if (stored) {
+        return new Response(JSON.stringify(stored))
+      }
+
+      return new Response('Not found', { status: 404 })
+    }
+
+    return new Response('Not found', { status: 404 })
+  }
+}

packages/gateway/src/durable-objects/metrics.js

@@ -0,0 +1,143 @@
+/**
+ * @typedef {Object} GatewayMetrics
+ * @property {number} totalRequests total number of performed requests
+ * @property {number} totalResponseTime total response time of the requests
+ * @property {number} totalFailedRequests total number of requests failed
+ * @property {number} totalWinnerRequests number of performed requests where faster
+ * @property {Record<string, number>} responseTimeHistogram
+ *
+ * @typedef {Object} ResponseStats
+ * @property {string} url gateway URL
+ * @property {boolean} ok request was successful
+ * @property {number} [responseTime] number of milliseconds to get response
+ * @property {boolean} [faster]
+ */
+
+// Key to track total time for fast gateway to respond
+const TOTAL_FAST_RESPONSE_TIME = 'totalFastResponseTime'
+
+/**
+ * Durable Object for keeping Metrics state.
+ */
+export class Metrics13 {
+  constructor(state, env) {
+    this.state = state
+    /** @type {Array<string>} */
+    this.ipfsGateways = JSON.parse(env.IPFS_GATEWAYS)
+
+    // `blockConcurrencyWhile()` ensures no requests are delivered until initialization completes.
+    this.state.blockConcurrencyWhile(async () => {
+      // Get state and initialize if not existing
+      /** @type {Map<string, GatewayMetrics>} */
+      this.gatewayMetrics = new Map()
+
+      // Gateway related metrics
+      const storedMetricsPerGw = await Promise.all(
+        this.ipfsGateways.map(async (gw) => {
+          /** @type {GatewayMetrics} */
+          const value =
+            (await this.state.storage.get(gw)) || createMetricsTracker()
+
+          return {
+            gw,
+            value: { ...value },
+          }
+        })
+      )
+
+      storedMetricsPerGw.forEach(({ gw, value }) => {
+        this.gatewayMetrics.set(gw, value)
+      })
+
+      // Total response time
+      this.totalFastResponseTime =
+        (await this.state.storage.get(TOTAL_FAST_RESPONSE_TIME)) || 0
+    })
+  }
+
+  // Handle HTTP requests from clients.
+  async fetch(request) {
+    // Apply requested action.
+    let url = new URL(request.url)
+    switch (url.pathname) {
+      case '/update':
+        const data = await request.json()
+        // Updated Metrics
+        this._updateMetrics(data)
+        // Save updated Metrics
+        await Promise.all([
+          ...this.ipfsGateways.map((gw) =>
+            this.state.storage.put(gw, this.gatewayMetrics.get(gw))
+          ),
+          this.state.storage.put(
+            TOTAL_FAST_RESPONSE_TIME,
+            this.totalFastResponseTime
+          ),
+        ])
+        return new Response()
+      case '/metrics':
+        const resp = {
+          totalFastResponseTime: this.totalFastResponseTime,
+          ipfsGateways: {},
+        }
+        this.ipfsGateways.forEach((url) => {
+          resp.ipfsGateways[url] = this.gatewayMetrics.get(url)
+        })
+
+        return new Response(JSON.stringify(resp))
+      default:
+        return new Response('Not found', { status: 404 })
+    }
+  }
+
+  /**
+   * @param {ResponseStats[]} responseStats
+   */
+  _updateMetrics(responseStats) {
+    responseStats.forEach((stats) => {
+      const gwMetrics = this.gatewayMetrics.get(stats.url)
+
+      if (!stats.ok) {
+        // Update request count
+        gwMetrics.totalRequests += 1
+        gwMetrics.totalFailedRequests += 1
+        return
+      }
+
+      // Update request count and response time sum
+      gwMetrics.totalRequests += 1
+      gwMetrics.totalResponseTime += stats.responseTime
+
+      // Update faster count if appropriate
+      if (stats.faster) {
+        gwMetrics.totalWinnerRequests += 1
+        this.totalFastResponseTime += stats.responseTime
+      }
+
+      // Update histogram
+      const gwHistogram = {
+        ...gwMetrics.responseTimeHistogram,
+      }
+
+      const histogramCandidate =
+        histogram.find((h) => stats.responseTime <= h) ||
+        histogram[histogram.length - 1]
+      gwHistogram[histogramCandidate] += 1
+      gwMetrics.responseTimeHistogram = gwHistogram
+    })
+  }
+}
+
+export const histogram = [300, 500, 750, 1000, 1500, 2000, 3000, 5000, 10000]
+
+function createMetricsTracker() {
+  const h = histogram.map((h) => [h, 0])
+
+  return {
+    totalRequests: 0,
+    totalResponseTime: 0,
+    totalFailedRequests: 0,
+    totalWinnerRequests: 0,
+    responseTimeHistogram: Object.fromEntries(h),
+  }
+}

packages/gateway/src/env.js

@@ -0,0 +1,61 @@
+import Toucan from 'toucan-js'
+import pkg from '../package.json'
+
+// TODO: Get Durable object typedef
+/**
+ * @typedef {Object} EnvInput
+ * @property {string} IPFS_GATEWAYS
+ * @property {string} VERSION
+ * @property {string} ENV
+ * @property {string} [SENTRY_DSN]
+ * @property {number} [REQUEST_TIMEOUT]
+ * @property {Object} METRICS
+ * @property {Object} CIDSTRACKER
+ *
+ * @typedef {Object} EnvTransformed
+ * @property {Array<string>} ipfsGateways
+ * @property {Object} metricsDurable
+ * @property {Object} cidsTrackerDurable
+ * @property {number} REQUEST_TIMEOUT
+ * @property {Toucan} [sentry]
+ *
+ * @typedef {EnvInput & EnvTransformed} Env
+ */
+
+/**
+ * @param {Request} request
+ * @param {Env} env
+ */
+export function envAll(request, env) {
+  env.sentry = getSentry(request, env)
+  env.ipfsGateways = JSON.parse(env.IPFS_GATEWAYS)
+  env.metricsDurable = env.METRICS
+  env.cidsTrackerDurable = env.CIDSTRACKER
+  env.REQUEST_TIMEOUT = env.REQUEST_TIMEOUT || 20000
+}
+
+/**
+ * Get sentry instance if configured
+ *
+ * @param {Request} request
+ * @param {Env} env
+ */
+function getSentry(request, env) {
+  if (!env.SENTRY_DSN) {
+    return
+  }
+
+  return new Toucan({
+    request,
+    dsn: env.SENTRY_DSN,
+    allowedHeaders: ['user-agent'],
+    allowedSearchParams: /(.*)/,
+    debug: false,
+    environment: env.ENV || 'dev',
+    rewriteFrames: {
+      root: '/',
+    },
+    release: env.VERSION,
+    pkg,
+  })
+}

packages/gateway/src/error-handler.js

@@ -1,46 +1,15 @@
-import Toucan from 'toucan-js'
-import pkg from '../package.json'
-
 /**
  * @param {Error & {status?: number;code?: string;}} err
- * @param {Request} request
- * @param {import('./index').Env} env
+ * @param {import('./env').Env} env
  */
-export function errorHandler(err, request, env) {
+export function errorHandler(err, env) {
   console.error(err.stack)
 
   const status = err.status || 500
 
-  const sentry = getSentry(request, env)
-  if (sentry && status >= 500) {
-    sentry.captureException(err)
+  if (env.sentry && status >= 500) {
+    env.sentry.captureException(err)
   }
 
   return new Response(err.message || 'Server Error', { status })
 }
-
-/**
- * Get sentry instance if configured
- *
- * @param {Request} request
- * @param {import('./index').Env} env
- */
-function getSentry(request, env) {
-  if (!env.SENTRY_DSN) {
-    return
-  }
-
-  return new Toucan({
-    request,
-    dsn: env.SENTRY_DSN,
-    allowedHeaders: ['user-agent', 'x-client'],
-    allowedSearchParams: /(.*)/,
-    debug: false,
-    environment: env.ENV || 'dev',
-    rewriteFrames: {
-      root: '/',
-    },
-    release: env.VERSION,
-    pkg,
-  })
-}