feat: range header support

packages/api/package.json

@@ -23,6 +23,7 @@
   },
   "dependencies": {
     "@supabase/postgrest-js": "^0.37.2",
+    "http-range-parse": "^1.0.0",
     "itty-router": "^2.4.5",
     "multiformats": "^9.6.4",
     "nanoid": "^3.1.30",

packages/api/src/env.js

@@ -126,9 +126,16 @@ function getSentry(request, env, ctx) {
  * @property {Headers} [httpMetadata]
  * @property {Record<string, string>} [customMetadata]
  *
+ * @typedef {{ offset: number, length?: number } | { offset?: number, length: number } | { suffix: number }} R2Range
+ *
+ * @typedef {Object} R2GetOptions
+ * @property {R2Range} [range]
+ *
  * @typedef {Object} R2Object
  * @property {Date} uploaded
  * @property {number} size
+ * @property {string} httpEtag
+ * @property {(headers: Headers) => void} writeHttpMetadata
  * @property {Headers} [httpMetadata]
  * @property {Record<string, string>} [customMetadata]
  *
@@ -144,7 +151,7 @@ function getSentry(request, env, ctx) {
  *
  * @typedef {Object} R2Bucket
  * @property {(key: string) => Promise<R2Object | null>} head
- * @property {(key: string) => Promise<R2ObjectBody | null>} get
+ * @property {(key: string, options?: R2GetOptions) => Promise<Response & R2Object | null>} get
  * @property {(key: string, value: ReadableStream | ArrayBuffer | ArrayBufferView | string | null, options?: R2PutOptions) => Promise<R2Object>} put
  * @property {(key: string) => Promise<void>} delete
  */

packages/api/src/errors.js

@@ -45,6 +45,20 @@ export class TimeoutError extends Error {
 }
 TimeoutError.CODE = 'ERROR_TIMEOUT'
 
+export class InvalidRangeError extends Error {
+  /**
+   * @param {string} message
+   */
+  constructor(message = 'invalid Range') {
+    const status = 400
+    super(message)
+    this.name = 'InvalidRangeError'
+    this.status = status
+    this.code = InvalidRangeError.CODE
+  }
+}
+InvalidRangeError.CODE = 'ERROR_INVALID_RANGE'
+
 export class HTTPError extends Error {
   /**
    *

packages/api/src/perma-cache/get.js

@@ -1,11 +1,14 @@
 /* eslint-env serviceworker, browser */
 /* global Response */
 
+import parseRange from 'http-range-parse'
+
 import { getSourceUrl, getNormalizedUrl } from '../utils/url.js'
-import { UrlNotFoundError } from '../errors.js'
+import { UrlNotFoundError, InvalidRangeError } from '../errors.js'
 
 /**
  * @typedef {import('../env').Env} Env
+ * @typedef {import('../env').R2Range} R2Range
  */
 
 /**
@@ -19,10 +22,60 @@ export async function permaCacheGet(request, env) {
   const normalizedUrl = getNormalizedUrl(sourceUrl, env)
   const r2Key = normalizedUrl.toString()
 
-  const r2Object = await env.SUPERHOT.get(r2Key)
-  if (r2Object) {
-    return new Response(r2Object.body)
+  // Get range header
+  const range = toR2Range(request.headers.get('range'))
+  const r2Object = await env.SUPERHOT.get(r2Key, {
+    range,
+  })
+
+  if (!r2Object || !r2Object.body) {
+    throw new UrlNotFoundError()
+  }
+
+  const headers = new Headers()
+  headers.set('etag', r2Object.httpEtag)
+  r2Object.writeHttpMetadata(headers)
+
+  if (range) {
+    headers.set('status', '206')
+    let first, last
+    if (range.suffix != null) {
+      first = r2Object.size - range.suffix
+      last = r2Object.size - 1
+    } else {
+      first = range.offset || 0
+      last = range.length != null ? first + range.length - 1 : r2Object.size - 1
+    }
+    headers.set('content-range', `bytes ${first}-${last}/${r2Object.size}`)
+    headers.set('content-length', `${last - first + 1}`)
+  } else {
+    headers.set('status', '200')
+    headers.set('content-length', `${r2Object.size}`)
+  }
+
+  return new Response(r2Object.body, {
+    status: range ? 206 : 200,
+  })
+}
+
+/**
+ * Convert a HTTP Range header to an R2 range object.
+ *
+ * @param {string|null} encoded
+ * @returns {R2Range|undefined}
+ */
+function toR2Range(encoded) {
+  if (encoded === null) {
+    return
   }
 
-  throw new UrlNotFoundError()
+  const result = parseRange(encoded)
+  if (result.ranges)
+    throw new InvalidRangeError('Multiple ranges not supported')
+  const { unit, first, last, suffix } = result
+  if (unit !== 'bytes')
+    throw new InvalidRangeError(`Unsupported range unit: ${unit}`)
+  return suffix != null
+    ? { suffix }
+    : { offset: first, length: last != null ? last - first + 1 : undefined }
 }

packages/api/test/perma-cache-get.spec.js

@@ -43,6 +43,36 @@ test('Gets content from perma cache by URL', async (t) => {
   t.deepEqual(await responseGet.text(), gatewayTxtResponse)
 })
 
+test('Gets range content from perma cache by URL', async (t) => {
+  const { mf, user } = t.context
+
+  const url =
+    'http://localhost:9081/ipfs/bafkreidyeivj7adnnac6ljvzj2e3rd5xdw3revw4da7mx2ckrstapoupoq'
+  const gatewayTxtResponse = 'Hello nft.storage! 😎'
+
+  // Post URL content to perma cache
+  const responsePost = await mf.dispatchFetch(getPermaCachePutUrl(url), {
+    method: 'POST',
+    headers: { Authorization: `Bearer ${user.token}` },
+  })
+  t.is(responsePost.status, 200)
+
+  // GET URL content from perma cache
+  const { normalizedUrl } = getParsedUrl(url)
+
+  const responseGet = await mf.dispatchFetch(
+    getPermaCachePutUrl(normalizedUrl),
+    {
+      method: 'GET',
+      headers: {
+        Range: 'bytes=0-2',
+      },
+    }
+  )
+  t.is(responseGet.status, 206)
+  t.deepEqual(await responseGet.text(), gatewayTxtResponse.slice(0, 2 + 1)) // range includes
+})
+
 test('Gets 404 response from perma cache by URL when url not perma cached', async (t) => {
   const { mf } = t.context
   const url =

packages/api/test/scripts/mocks/r2.js

@@ -32,19 +32,25 @@ export function createR2Bucket() {
         size: data.length,
       })
     },
-    get: async (key) => {
+    get: async (key, options = {}) => {
       const value = bucket.get(key)
       if (!value) {
         return undefined
       }
 
-      const response = new Response(value.body, { status: 200 })
+      let body = value.body
+      if (options.range) {
+        body = value.body.slice(options.range.offset, options.range.length)
+      }
+
+      const response = new Response(body, { status: 200 })
 
       return Promise.resolve(
         Object.assign(response, {
           httpMetadata: value.httpMetadata || {},
           customMetadata: value.customMetadata || {},
           size: value.body.length,
+          writeHttpMetadata: () => {},
         })
       )
     },

packages/edge-gateway/src/gateway.js

@@ -159,9 +159,9 @@ export async function gatewayIpfs(request, env, ctx, options = {}) {
           winnerGwResponse.response.headers.get('content-length')
         )
 
-        // Cache request URL in Cloudflare CDN if smaller than CF_CACHE_MAX_OBJECT_SIZE
+        // Cache request in Cloudflare CDN if smaller than CF_CACHE_MAX_OBJECT_SIZE
         if (contentLengthMb <= CF_CACHE_MAX_OBJECT_SIZE) {
-          await cache.put(request.url, winnerGwResponse.response.clone())
+          await cache.put(request, winnerGwResponse.response.clone())
         }
       })()
     )
@@ -273,7 +273,11 @@ async function cdnResolution(request, env, cache) {
 
   try {
     const res = await pAny(
-      [cache.match(request.url), getFromPermaCache(request, env)],
+      [
+        cache.match(request), // Request from cache API
+        cache.match(request.url), // Request URL from cache API - To be deprecated
+        getFromPermaCache(request, env),
+      ],
       {
         filter: (res) => !!res,
       }