feat: add support for gateway race configuration via wrangler secret

packages/edge-gateway/README.md

@@ -15,6 +15,8 @@
     wrangler secret put SENTRY_DSN --env $(whoami) # Get from Sentry
     wrangler secret put LOKI_URL --env $(whoami) # Get from Loki
     wrangler secret put LOKI_TOKEN --env $(whoami) # Get from Loki
+    wrangler secret put IPFS_GATEWAYS_RACE_L1 --env $(whoami) # JSON String with array of IPFS Gateways URLs (eg. [\"https://ipfs.io\"])
+    wrangler secret put IPFS_GATEWAYS_RACE_L2 --env $(whoami) # JSON String with array of IPFS Gateways URLs (eg. [\"https://cf.dag.haus\", \"https://w3link.mypinata.cloud\"])
   ```
 
 - `pnpm run publish` - Publish the worker under desired env. An alias for `wrangler publish --env $(whoami)`

packages/edge-gateway/src/constants.js

@@ -13,3 +13,12 @@ export const RESOLUTION_IDENTIFIERS = {
   CACHE_ZONE: 'cache-zone',
   PERMA_CACHE: 'perma-cache'
 }
+
+export const DEFAULT_RACE_L1_GATEWAYS = [
+  'https://ipfs.io'
+]
+
+export const DEFAULT_RACE_L2_GATEWAYS = [
+  'https://cf.dag.haus',
+  'https://w3link.mypinata.cloud'
+]

packages/edge-gateway/src/env.js

@@ -5,6 +5,10 @@ import { Logging } from '@web3-storage/worker-utils/loki'
 import { createGatewayRacer } from 'ipfs-gateway-race'
 
 import pkg from '../package.json'
+import {
+  DEFAULT_RACE_L1_GATEWAYS,
+  DEFAULT_RACE_L2_GATEWAYS
+} from './constants.js'
 
 /**
  * @typedef {import('./bindings').Env} Env
@@ -27,15 +31,8 @@ export function envAll (request, env, ctx) {
   env.COMMITHASH = COMMITHASH
   env.SENTRY_RELEASE = SENTRY_RELEASE
 
+  setGatewayRace(env)
   env.sentry = getSentry(request, env, ctx)
-  env.ipfsGatewaysL1 = JSON.parse(env.IPFS_GATEWAYS_RACE_L1)
-  env.ipfsGatewaysL2 = JSON.parse(env.IPFS_GATEWAYS_RACE_L2)
-  env.gwRacerL1 = createGatewayRacer(env.ipfsGatewaysL1, {
-    timeout: env.REQUEST_TIMEOUT
-  })
-  env.gwRacerL2 = createGatewayRacer(env.ipfsGatewaysL2, {
-    timeout: env.REQUEST_TIMEOUT
-  })
   env.startTime = Date.now()
 
   env.isCidVerifierEnabled = env.CID_VERIFIER_ENABLED === 'true'
@@ -55,6 +52,49 @@ export function envAll (request, env, ctx) {
   env.log.time('request')
 }
 
+/**
+ * @param {Env} env
+ */
+function setGatewayRace (env) {
+  /**
+   * @param {string} input
+   */
+  function getListFromInput (input) {
+    const list = JSON.parse(input)
+    if (!Array.isArray(list)) {
+      throw new Error('invalid environment variable')
+    }
+    list.map(gwUrl => new URL(gwUrl))
+    return list
+  }
+
+  // Set Layer 1
+  let l1Gateways
+  try {
+    l1Gateways = getListFromInput(env.IPFS_GATEWAYS_RACE_L1)
+  } catch (_) {
+    env.log && env.log.warn(`Invalid JSON string with race L1 Gateways: ${env.IPFS_GATEWAYS_RACE_L1}`)
+    l1Gateways = DEFAULT_RACE_L1_GATEWAYS
+  }
+  env.ipfsGatewaysL1 = l1Gateways
+  env.gwRacerL1 = createGatewayRacer(l1Gateways, {
+    timeout: env.REQUEST_TIMEOUT
+  })
+
+  // Set Layer 2
+  let l2Gateways
+  try {
+    l2Gateways = getListFromInput(env.IPFS_GATEWAYS_RACE_L2)
+  } catch (_) {
+    env.log && env.log.warn(`Invalid JSON string with race L2 Gateways: ${env.IPFS_GATEWAYS_RACE_L2}`)
+    l2Gateways = DEFAULT_RACE_L2_GATEWAYS
+  }
+  env.ipfsGatewaysL2 = l2Gateways
+  env.gwRacerL2 = createGatewayRacer(l2Gateways, {
+    timeout: env.REQUEST_TIMEOUT
+  })
+}
+
 /**
  * Get sentry instance if configured
  *

packages/edge-gateway/src/version.js

@@ -10,6 +10,8 @@ export async function versionGet (request, env) {
   return new JSONResponse({
     version: env.VERSION,
     commit: env.COMMITHASH,
-    branch: env.BRANCH
+    branch: env.BRANCH,
+    raceGatewaysL1: env.ipfsGatewaysL1,
+    raceGatewaysL2: env.ipfsGatewaysL2
   })
 }

packages/edge-gateway/test/env.spec.js

@@ -0,0 +1,38 @@
+import fs from 'fs'
+import path from 'path'
+import { fileURLToPath } from 'url'
+import git from 'git-rev-sync'
+
+import { 
+  DEFAULT_RACE_L1_GATEWAYS,
+  DEFAULT_RACE_L2_GATEWAYS
+} from '../src/constants.js'
+import { test, getMiniflare } from './utils/setup.js'
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url))
+const pkg = JSON.parse(
+  fs.readFileSync(path.join(__dirname, '..', 'package.json'), 'utf8')
+)
+
+// Create a new Miniflare environment for each test
+test.before((t) => {
+  t.context = {
+    mf: getMiniflare()
+  }
+})
+
+test('Defaults to hardcoded gateways if invalid secrets are set', async (t) => {
+  const mf = getMiniflare({
+    IPFS_GATEWAYS_RACE_L1: 'invalid gateways value',
+    IPFS_GATEWAYS_RACE_L2: '[\"no-url\"]'
+  })
+
+  const response = await mf.dispatchFetch('http://localhost:8787/version')
+  const { version, commit, branch, raceGatewaysL1, raceGatewaysL2 } = await response.json()
+
+  t.is(version, pkg.version)
+  t.is(commit, git.long(__dirname))
+  t.is(branch, git.branch(__dirname))
+  t.deepEqual(raceGatewaysL1, DEFAULT_RACE_L1_GATEWAYS)
+  t.deepEqual(raceGatewaysL2, DEFAULT_RACE_L2_GATEWAYS)
+})

packages/edge-gateway/test/utils/miniflare.js

@@ -2,6 +2,11 @@ import fs from 'fs'
 import path from 'path'
 import { Miniflare } from 'miniflare'
 
+export const secrets = {
+  IPFS_GATEWAYS_RACE_L1: '[\"http://127.0.0.1:9081\"]',
+  IPFS_GATEWAYS_RACE_L2: '[\"http://localhost:9082\", \"http://localhost:9083\"]'
+}
+
 export function getMiniflare (bindings = {}) {
   let envPath = path.join(process.cwd(), '../../.env')
   if (!fs.statSync(envPath, { throwIfNoEntry: false })) {
@@ -41,6 +46,7 @@ export function getMiniflare (bindings = {}) {
       PUBLIC_RACE_TTFB: createAnalyticsEngine(),
       PUBLIC_RACE_STATUS_CODE: createAnalyticsEngine(),
       REQUEST_TIMEOUT: 3000,
+      ...secrets,
       ...bindings
     }
   })

packages/edge-gateway/test/version.spec.js

@@ -3,7 +3,7 @@ import path from 'path'
 import { fileURLToPath } from 'url'
 import git from 'git-rev-sync'
 
-import { test, getMiniflare } from './utils/setup.js'
+import { test, getMiniflare, secrets } from './utils/setup.js'
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url))
 const pkg = JSON.parse(
@@ -21,9 +21,11 @@ test('Gets Version', async (t) => {
   const { mf } = t.context
 
   const response = await mf.dispatchFetch('http://localhost:8787/version')
-  const { version, commit, branch } = await response.json()
+  const { version, commit, branch, raceGatewaysL1, raceGatewaysL2 } = await response.json()
 
   t.is(version, pkg.version)
   t.is(commit, git.long(__dirname))
   t.is(branch, git.branch(__dirname))
+  t.deepEqual(raceGatewaysL1, JSON.parse(secrets.IPFS_GATEWAYS_RACE_L1))
+  t.deepEqual(raceGatewaysL2, JSON.parse(secrets.IPFS_GATEWAYS_RACE_L2))
 })

packages/edge-gateway/wrangler.toml

@@ -34,8 +34,6 @@ kv_namespaces = [
 ]
 
 [env.production.vars]
-IPFS_GATEWAYS_RACE_L1 = "[\"https://ipfs.io\"]"
-IPFS_GATEWAYS_RACE_L2 = "[\"https://cf.dag.haus\", \"https://w3link.mypinata.cloud\"]"
 GATEWAY_HOSTNAME = 'ipfs.dag.haus'
 CID_VERIFIER_URL = 'https://cid-verifier.dag.haus'
 EDGE_GATEWAY_API_URL = 'https://api.nftstorage.link'
@@ -87,8 +85,6 @@ kv_namespaces = [
 ]
 
 [env.staging.vars]
-IPFS_GATEWAYS_RACE_L1 = "[\"https://ipfs.io\"]"
-IPFS_GATEWAYS_RACE_L2 = "[\"https://cf.dag.haus\", \"https://w3link.mypinata.cloud\"]"
 GATEWAY_HOSTNAME = 'ipfs-staging.dag.haus'
 CID_VERIFIER_URL = 'https://cid-verifier-staging.dag.haus'
 EDGE_GATEWAY_API_URL = 'https://api.nftstorage.link'
@@ -129,8 +125,6 @@ name = "PUBLIC_RACE_STATUS_CODE"
 workers_dev = true
 
 [env.test.vars]
-IPFS_GATEWAYS_RACE_L1 = "[\"http://127.0.0.1:9081\"]"
-IPFS_GATEWAYS_RACE_L2 = "[\"http://localhost:9082\", \"http://localhost:9083\"]"
 GATEWAY_HOSTNAME = 'ipfs.localhost:8787'
 CID_VERIFIER_URL = 'http://cid-verifier.localhost:8787'
 EDGE_GATEWAY_API_URL = 'http://localhost:8787'