feat: dotstorage hosts resolution layer

packages/edge-gateway/README.md

@@ -15,6 +15,7 @@
     wrangler secret put SENTRY_DSN --env $(whoami) # Get from Sentry
     wrangler secret put LOKI_URL --env $(whoami) # Get from Loki
     wrangler secret put LOKI_TOKEN --env $(whoami) # Get from Loki
+    wrangler secret put CDN_GATEWAYS_RACE --env $(whoami) # JSON String with array of CDN Gateways URLs (eg. echo -e '["https://freeway.dag.haus"]' | wrangler secret ...)
     wrangler secret put IPFS_GATEWAYS_RACE_L1 --env $(whoami) # JSON String with array of IPFS Gateways URLs (eg. echo -e '["https://ipfs.io","https://dagula.dag.haus"]' | wrangler secret ...)
     wrangler secret put IPFS_GATEWAYS_RACE_L2 --env $(whoami) # JSON String with array of IPFS Gateways URLs (eg. echo -e '["https://cf.dag.haus","https://w3link.mypinata.cloud"]' | wrangler secret ...)
   ```

packages/edge-gateway/src/bindings.d.ts

@@ -20,6 +20,7 @@ export interface EnvInput {
   CID_VERIFIER_ENABLED: string
   CID_VERIFIER_URL: string
   CID_VERIFIER: Fetcher
+  CDN_GATEWAYS_RACE: string
   IPFS_GATEWAYS_RACE_L1: string
   IPFS_GATEWAYS_RACE_L2: string
   GATEWAY_HOSTNAME: string
@@ -43,6 +44,7 @@ export interface EnvTransformed {
   REQUEST_TIMEOUT: number
   IPFS_GATEWAY_HOSTNAME: string
   IPNS_GATEWAY_HOSTNAME: string
+  cdnGateways: Array<string>
   ipfsGatewaysL1: Array<string>
   ipfsGatewaysL2: Array<string>
   sentry?: Toucan

packages/edge-gateway/src/constants.js

@@ -15,6 +15,9 @@ export const RESOLUTION_IDENTIFIERS = {
   PERMA_CACHE: 'perma-cache'
 }
 
+/** @type {string[]} */
+export const DEFAULT_CDN_GATEWAYS = []
+
 export const DEFAULT_RACE_L1_GATEWAYS = [
   'https://ipfs.io'
 ]

packages/edge-gateway/src/env.js

@@ -7,7 +7,8 @@ import { createGatewayRacer } from 'ipfs-gateway-race'
 import pkg from '../package.json'
 import {
   DEFAULT_RACE_L1_GATEWAYS,
-  DEFAULT_RACE_L2_GATEWAYS
+  DEFAULT_RACE_L2_GATEWAYS,
+  DEFAULT_CDN_GATEWAYS
 } from './constants.js'
 
 /**
@@ -33,6 +34,9 @@ export function envAll (request, env, ctx) {
 
   env.sentry = getSentry(request, env, ctx)
 
+  // Set CDN Gateways
+  env.cdnGateways = parseGatewayUrls(env.CDN_GATEWAYS_RACE, DEFAULT_CDN_GATEWAYS, env)
+
   // Set Layer 1 racer
   env.ipfsGatewaysL1 = parseGatewayUrls(env.IPFS_GATEWAYS_RACE_L1, DEFAULT_RACE_L1_GATEWAYS, env)
   env.gwRacerL1 = createGatewayRacer(env.ipfsGatewaysL1, {

packages/edge-gateway/src/gateway.js

@@ -5,6 +5,7 @@ import pAny, { AggregateError } from 'p-any'
 import pDefer from 'p-defer'
 import pSettle from 'p-settle'
 import { FilterError } from 'p-some'
+import { gatewayFetch } from 'ipfs-gateway-race'
 
 import { getCidFromSubdomainUrl } from './utils/cid.js'
 import { getHeaders } from './utils/headers.js'
@@ -75,7 +76,7 @@ export async function gatewayGet (request, env, ctx) {
   }
 
   // 2nd layer
-  const dotstorageRes = await getFromDotstorage(request, cid, { pathname })
+  const dotstorageRes = await getFromDotstorage(request, env, cid, { pathname })
   if (dotstorageRes) {
     return getResponseWithCustomHeaders(
       dotstorageRes.response,
@@ -163,16 +164,17 @@ async function getFromCdn (request, env, cache) {
 
 /**
  * @param {Request} request
+ * @param {Env} env
  * @param {string} cid
  * @param {{ pathname?: string}} [options]
  * @return {Promise<ProxiedCDNResponse | undefined>}
  */
-async function getFromDotstorage (request, cid, options = {}) {
+async function getFromDotstorage (request, env, cid, options = {}) {
   const pathname = options.pathname || ''
   try {
     // Get onlyIfCached hosts provided
     /** @type {string[]} */
-    const hosts = []
+    const onlyIfCachedHosts = []
     // @ts-ignore custom entry in cf object
     if (request.cf?.onlyIfCachedGateways) {
       /** @type {URL[]} */
@@ -181,15 +183,15 @@ async function getFromDotstorage (request, cid, options = {}) {
         request.cf?.onlyIfCachedGateways
       ).map((/** @type {string} */ gw) => new URL(gw))
 
-      onlyIfCachedGateways.forEach((gw) => hosts.push(gw.host))
+      onlyIfCachedGateways.forEach((gw) => onlyIfCachedHosts.push(gw.host))
     }
 
-    // Add only if cached header
-    const headers = getHeaders(request)
-    headers.set('Cache-Control', 'only-if-cached')
+    const proxiedCDNResponse = await pAny([
+      ...onlyIfCachedHosts.map(async (host) => {
+        // Add only if cached header
+        const headers = getHeaders(request)
+        headers.set('Cache-Control', 'only-if-cached')
 
-    const proxiedCDNResponse = await pAny(
-      hosts.map(async (host) => {
         const response = await fetch(`https://${cid}.ipfs.${host}${pathname}`, {
           headers
         })
@@ -202,8 +204,24 @@ async function getFromDotstorage (request, cid, options = {}) {
           response,
           resolutionIdentifier: host
         }
+      }),
+      ...env.cdnGateways.map(async (host) => {
+        const gwResponse = await gatewayFetch(host, cid, pathname, {
+          timeout: env.REQUEST_TIMEOUT
+        })
+
+        // @ts-ignore 'response' does not exist on type 'GatewayResponseFailure'
+        if (!gwResponse?.response.ok) {
+          throw new Error()
+        }
+
+        return {
+          // @ts-ignore 'response' does not exist on type 'GatewayResponseFailure'
+          response: gwResponse?.response,
+          resolutionIdentifier: host
+        }
       })
-    )
+    ])
 
     return proxiedCDNResponse
   } catch (_) {}

packages/edge-gateway/test/cdn.spec.js

@@ -148,3 +148,27 @@ test('Should not get from cache if no-cache cache control header is provided', a
     clearTimeout(timer)
   }
 })
+
+test('should get from cdn gateways race if they can resolve', async (t) => {
+  const url =
+    'https://bafkreihl44bu5rqxctfvl3ahcln7gnjgmjqi7v5wfwojqwriqnq7wo4n7u.ipfs.localhost:8787'
+
+  const mf = getMiniflare({
+    CDN_GATEWAYS_RACE: '["http://localhost:9082"]',
+    IPFS_GATEWAYS_RACE_L1: '["http://localhost:9083"]',
+    IPFS_GATEWAYS_RACE_L2: '["http://localhost:9081"]'
+  })
+
+  const response = await mf.dispatchFetch(
+    url
+  )
+  await response.waitUntil()
+  t.is(await response.text(), 'Hello dot.storage! 😎')
+
+  // Validate content headers
+  t.is(response.headers.get('content-length'), '23')
+
+  // Validate x-dotstorage headers
+  t.is(response.headers.get('x-dotstorage-resolution-layer'), 'dotstorage-race')
+  t.is(response.headers.get('x-dotstorage-resolution-id'), 'http://localhost:9082')
+})

packages/ipfs-gateway-race/lib/index.js

@@ -137,7 +137,7 @@ export function createGatewayRacer (ipfsGateways, options = {}) {
  * @param {number} [options.timeout]
  * @param {AbortSignal} [options.signal]
  */
-async function gatewayFetch (
+export async function gatewayFetch (
   gwUrl,
   cid,
   pathname,