fix: gateway use cf dns to prevent rate limit instead of durable obje…

…ct (#36)

Co-authored-by: Oli Evans <oli.evans@gmail.com>

packages/edge-gateway/src/env.js

@@ -19,7 +19,6 @@ import { Logging } from './logs.js'
  * @property {Object} GATEWAYMETRICS
  * @property {Object} SUMMARYMETRICS
  * @property {Object} CIDSTRACKER
- * @property {Object} GATEWAYRATELIMITS
  * @property {Object} GATEWAYREDIRECTCOUNTER
  * @property {KVNamespace} DENYLIST
  *
@@ -30,7 +29,6 @@ import { Logging } from './logs.js'
  * @property {DurableObjectNamespace} gatewayMetricsDurable
  * @property {DurableObjectNamespace} summaryMetricsDurable
  * @property {DurableObjectNamespace} cidsTrackerDurable
- * @property {DurableObjectNamespace} gatewayRateLimitsDurable
  * @property {DurableObjectNamespace} gatewayRedirectCounter
  * @property {number} REQUEST_TIMEOUT
  * @property {Toucan} [sentry]
@@ -50,7 +48,6 @@ export function envAll(request, env, ctx) {
   env.gatewayMetricsDurable = env.GATEWAYMETRICS
   env.summaryMetricsDurable = env.SUMMARYMETRICS
   env.cidsTrackerDurable = env.CIDSTRACKER
-  env.gatewayRateLimitsDurable = env.GATEWAYRATELIMITS
   env.gatewayRedirectCounter = env.GATEWAYREDIRECTCOUNTER
   env.REQUEST_TIMEOUT = env.REQUEST_TIMEOUT || 20000
   env.IPFS_GATEWAY_HOSTNAME = env.GATEWAY_HOSTNAME

packages/edge-gateway/src/gateway.js

@@ -81,12 +81,10 @@ export async function gatewayGet(request, env, ctx) {
   }
 
   // Prepare IPFS gateway requests
-  const shouldPreventRateLimit = await getGatewayRateLimitState(request, env)
   const gatewayReqs = env.ipfsGateways.map((gwUrl) =>
     gatewayFetch(gwUrl, cid, request, {
       pathname,
       timeout: env.REQUEST_TIMEOUT,
-      shouldPreventRateLimit: shouldPreventRateLimit[gwUrl],
     })
   )
   try {
@@ -201,24 +199,13 @@ async function storeWinnerGwResponse(request, env, winnerGwResponse) {
  * @param {Object} [options]
  * @param {string} [options.pathname]
  * @param {number} [options.timeout]
- * @param {boolean} [options.shouldPreventRateLimit]
  */
 async function gatewayFetch(
   gwUrl,
   cid,
   request,
-  { pathname = '', timeout = 60000, shouldPreventRateLimit = false } = {}
+  { pathname = '', timeout = 60000 } = {}
 ) {
-  // Block before hitting rate limit if needed
-  if (shouldPreventRateLimit) {
-    /** @type {GatewayResponse} */
-    return {
-      url: gwUrl,
-      aborted: true,
-      reason: REQUEST_PREVENTED_RATE_LIMIT_CODE,
-    }
-  }
-
   const ipfsUrl = new URL('ipfs', gwUrl)
   const controller = new AbortController()
   const startTs = Date.now()
@@ -300,34 +287,6 @@ async function updateGatewayRedirectCounter(request, env) {
   await stub.fetch(getDurableRequestUrl(request, 'update'))
 }
 
-/**
- * @param {Request} request
- * @param {import('./env').Env} env
- */
-async function getGatewayRateLimitState(request, env) {
-  // Get durable object for gateway rate limits
-  const id = env.gatewayRateLimitsDurable.idFromName(GATEWAY_RATE_LIMIT_ID)
-  const stub = env.gatewayRateLimitsDurable.get(id)
-
-  try {
-    const stubResponse = await stub.fetch(
-      getDurableRequestUrl(request, 'request')
-    )
-
-    /** @type {import('./durable-objects/gateway-rate-limits').RateLimitResponse} */
-    const rateLimitResponse = await stubResponse.json()
-    return rateLimitResponse
-  } catch (err) {
-    env.log.log(err, 'error')
-    // Just force no prevention of rate limit
-    const shouldPreventRateLimit = {}
-    env.ipfsGateways.forEach((gwUrl) => {
-      shouldPreventRateLimit[gwUrl] = false
-    })
-    return shouldPreventRateLimit
-  }
-}
-
 /**
  * @param {Request} request
  * @param {import('./env').Env} env

packages/edge-gateway/src/index.js

@@ -11,7 +11,6 @@ import { metricsGet } from './metrics.js'
 export { GatewayMetrics0 } from './durable-objects/gateway-metrics.js'
 export { SummaryMetrics0 } from './durable-objects/summary-metrics.js'
 export { CidsTracker0 } from './durable-objects/cids.js'
-export { GatewayRateLimits4 } from './durable-objects/gateway-rate-limits.js'
 export { GatewayRedirectCounter0 } from './durable-objects/gateway-redirect-counter.js'
 
 import { addCorsHeaders, withCorsHeaders } from './cors.js'

packages/edge-gateway/wrangler.toml

@@ -21,7 +21,6 @@ bindings = [
   {name = "GATEWAYMETRICS", class_name = "GatewayMetrics0"},
   {name = "SUMMARYMETRICS", class_name = "SummaryMetrics0"},
   {name = "CIDSTRACKER", class_name = "CidsTracker0"},
-  {name = "GATEWAYRATELIMITS", class_name = "GatewayRateLimits4"},
   {name = "GATEWAYREDIRECTCOUNTER", class_name = "GatewayRedirectCounter0"}
 ]
 
@@ -34,7 +33,7 @@ route = "*.ipfs.nftstorage.link/*"
 kv_namespaces = [{ binding = "DENYLIST", id = "785cf627e913468ca5319523ae929def" }]
 
 [env.production.vars]
-IPFS_GATEWAYS = "[\"https://ipfs.io\", \"https://cf.nftstorage.link\", \"https://nft-storage.mypinata.cloud/\"]"
+IPFS_GATEWAYS = "[\"https://ipfs.io\", \"https://cf.nftstorage.link\", \"https://pinata.nftstorage.link\"]"
 GATEWAY_HOSTNAME = 'ipfs.nftstorage.link'
 DEBUG = "false"
 ENV = "production"
@@ -44,7 +43,6 @@ bindings = [
   {name = "GATEWAYMETRICS", class_name = "GatewayMetrics0"},
   {name = "SUMMARYMETRICS", class_name = "SummaryMetrics0"},
   {name = "CIDSTRACKER", class_name = "CidsTracker0"},
-  {name = "GATEWAYRATELIMITS", class_name = "GatewayRateLimits4"},
   {name = "GATEWAYREDIRECTCOUNTER", class_name = "GatewayRedirectCounter0"}
 ]
 
@@ -57,7 +55,7 @@ route = "*.ipfs-staging.nftstorage.link/*"
 kv_namespaces = [{ binding = "DENYLIST", id = "f4eb0eca32e14e28b643604a82e00cb3" }]
 
 [env.staging.vars]
-IPFS_GATEWAYS = "[\"https://ipfs.io\", \"https://cf.nftstorage.link\", \"https://nft-storage.mypinata.cloud/\"]"
+IPFS_GATEWAYS = "[\"https://ipfs.io\", \"https://cf.nftstorage.link\", \"https://pinata.nftstorage.link\"]"
 GATEWAY_HOSTNAME = 'ipfs-staging.nftstorage.link'
 DEBUG = "true"
 ENV = "staging"
@@ -67,7 +65,6 @@ bindings = [
   {name = "GATEWAYMETRICS", class_name = "GatewayMetrics0"},
   {name = "SUMMARYMETRICS", class_name = "SummaryMetrics0"},
   {name = "CIDSTRACKER", class_name = "CidsTracker0"},
-  {name = "GATEWAYRATELIMITS", class_name = "GatewayRateLimits4"},
   {name = "GATEWAYREDIRECTCOUNTER", class_name = "GatewayRedirectCounter0"}
 ]
 
@@ -87,7 +84,6 @@ bindings = [
   {name = "GATEWAYMETRICS", class_name = "GatewayMetrics0"},
   {name = "SUMMARYMETRICS", class_name = "SummaryMetrics0"},
   {name = "CIDSTRACKER", class_name = "CidsTracker0"},
-  {name = "GATEWAYRATELIMITS", class_name = "GatewayRateLimits4"},
   {name = "GATEWAYREDIRECTCOUNTER", class_name = "GatewayRedirectCounter0"}
 ]
 
@@ -106,7 +102,6 @@ bindings = [
   {name = "GATEWAYMETRICS", class_name = "GatewayMetrics0"},
   {name = "SUMMARYMETRICS", class_name = "SummaryMetrics0"},
   {name = "CIDSTRACKER", class_name = "CidsTracker0"},
-  {name = "GATEWAYRATELIMITS", class_name = "GatewayRateLimits4"},
   {name = "GATEWAYREDIRECTCOUNTER", class_name = "GatewayRedirectCounter0"}
 ]
 
@@ -124,7 +119,6 @@ bindings = [
   {name = "GATEWAYMETRICS", class_name = "GatewayMetrics0"},
   {name = "SUMMARYMETRICS", class_name = "SummaryMetrics0"},
   {name = "CIDSTRACKER", class_name = "CidsTracker0"},
-  {name = "GATEWAYRATELIMITS", class_name = "GatewayRateLimits4"},
   {name = "GATEWAYREDIRECTCOUNTER", class_name = "GatewayRedirectCounter0"}
 ]
 
@@ -148,4 +142,8 @@ deleted_classes = ["GatewayRateLimits2"]
 [[migrations]]
 tag = "v5" # Should be unique for each entry
 new_classes = ["GatewayRateLimits4"]
-deleted_classes = ["GatewayRateLimits3"]
+deleted_classes = ["GatewayRateLimits3"]
+
+[[migrations]]
+tag = "v6" # Should be unique for each entry
+deleted_classes = ["GatewayRateLimits4"]