diff --git a/packages/api/package.json b/packages/api/package.json index 5b6b0748..55a18b09 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -23,6 +23,7 @@ }, "dependencies": { "@supabase/postgrest-js": "^0.37.2", + "http-range-parse": "^1.0.0", "itty-router": "^2.4.5", "multiformats": "^9.6.4", "nanoid": "^3.1.30", diff --git a/packages/api/src/env.js b/packages/api/src/env.js index 294e02a9..5142009c 100644 --- a/packages/api/src/env.js +++ b/packages/api/src/env.js @@ -126,9 +126,16 @@ function getSentry(request, env, ctx) { * @property {Headers} [httpMetadata] * @property {Record} [customMetadata] * + * @typedef {{ offset: number, length?: number } | { offset?: number, length: number } | { suffix: number }} R2Range + * + * @typedef {Object} R2GetOptions + * @property {R2Range} [range] + * * @typedef {Object} R2Object * @property {Date} uploaded * @property {number} size + * @property {string} httpEtag + * @property {(headers: Headers) => void} writeHttpMetadata * @property {Headers} [httpMetadata] * @property {Record} [customMetadata] * @@ -144,7 +151,7 @@ function getSentry(request, env, ctx) { * * @typedef {Object} R2Bucket * @property {(key: string) => Promise} head - * @property {(key: string) => Promise} get + * @property {(key: string, options?: R2GetOptions) => Promise} get * @property {(key: string, value: ReadableStream | ArrayBuffer | ArrayBufferView | string | null, options?: R2PutOptions) => Promise} put * @property {(key: string) => Promise} delete */ diff --git a/packages/api/src/errors.js b/packages/api/src/errors.js index 4d5fbea2..7877e93f 100644 --- a/packages/api/src/errors.js +++ b/packages/api/src/errors.js @@ -45,6 +45,20 @@ export class TimeoutError extends Error { } TimeoutError.CODE = 'ERROR_TIMEOUT' +export class InvalidRangeError extends Error { + /** + * @param {string} message + */ + constructor(message = 'invalid Range') { + const status = 400 + super(message) + this.name = 'InvalidRangeError' + this.status = status + this.code = InvalidRangeError.CODE + } +} +InvalidRangeError.CODE = 'ERROR_INVALID_RANGE' + export class HTTPError extends Error { /** * diff --git a/packages/api/src/perma-cache/get.js b/packages/api/src/perma-cache/get.js index 2b723e10..2cf9d294 100644 --- a/packages/api/src/perma-cache/get.js +++ b/packages/api/src/perma-cache/get.js @@ -1,11 +1,14 @@ /* eslint-env serviceworker, browser */ /* global Response */ +import parseRange from 'http-range-parse' + import { getSourceUrl, getNormalizedUrl } from '../utils/url.js' -import { UrlNotFoundError } from '../errors.js' +import { UrlNotFoundError, InvalidRangeError } from '../errors.js' /** * @typedef {import('../env').Env} Env + * @typedef {import('../env').R2Range} R2Range */ /** @@ -19,10 +22,60 @@ export async function permaCacheGet(request, env) { const normalizedUrl = getNormalizedUrl(sourceUrl, env) const r2Key = normalizedUrl.toString() - const r2Object = await env.SUPERHOT.get(r2Key) - if (r2Object) { - return new Response(r2Object.body) + // Get range header + const range = toR2Range(request.headers.get('range')) + const r2Object = await env.SUPERHOT.get(r2Key, { + range, + }) + + if (!r2Object || !r2Object.body) { + throw new UrlNotFoundError() + } + + const headers = new Headers() + headers.set('etag', r2Object.httpEtag) + r2Object.writeHttpMetadata(headers) + + if (range) { + headers.set('status', '206') + let first, last + if (range.suffix != null) { + first = r2Object.size - range.suffix + last = r2Object.size - 1 + } else { + first = range.offset || 0 + last = range.length != null ? first + range.length - 1 : r2Object.size - 1 + } + headers.set('content-range', `bytes ${first}-${last}/${r2Object.size}`) + headers.set('content-length', `${last - first + 1}`) + } else { + headers.set('status', '200') + headers.set('content-length', `${r2Object.size}`) + } + + return new Response(r2Object.body, { + status: range ? 206 : 200, + }) +} + +/** + * Convert a HTTP Range header to an R2 range object. + * + * @param {string|null} encoded + * @returns {R2Range|undefined} + */ +function toR2Range(encoded) { + if (encoded === null) { + return } - throw new UrlNotFoundError() + const result = parseRange(encoded) + if (result.ranges) + throw new InvalidRangeError('Multiple ranges not supported') + const { unit, first, last, suffix } = result + if (unit !== 'bytes') + throw new InvalidRangeError(`Unsupported range unit: ${unit}`) + return suffix != null + ? { suffix } + : { offset: first, length: last != null ? last - first + 1 : undefined } } diff --git a/packages/api/test/perma-cache-get.spec.js b/packages/api/test/perma-cache-get.spec.js index 397d546d..4d16c3b9 100644 --- a/packages/api/test/perma-cache-get.spec.js +++ b/packages/api/test/perma-cache-get.spec.js @@ -43,6 +43,36 @@ test('Gets content from perma cache by URL', async (t) => { t.deepEqual(await responseGet.text(), gatewayTxtResponse) }) +test('Gets range content from perma cache by URL', async (t) => { + const { mf, user } = t.context + + const url = + 'http://localhost:9081/ipfs/bafkreidyeivj7adnnac6ljvzj2e3rd5xdw3revw4da7mx2ckrstapoupoq' + const gatewayTxtResponse = 'Hello nft.storage! 😎' + + // Post URL content to perma cache + const responsePost = await mf.dispatchFetch(getPermaCachePutUrl(url), { + method: 'POST', + headers: { Authorization: `Bearer ${user.token}` }, + }) + t.is(responsePost.status, 200) + + // GET URL content from perma cache + const { normalizedUrl } = getParsedUrl(url) + + const responseGet = await mf.dispatchFetch( + getPermaCachePutUrl(normalizedUrl), + { + method: 'GET', + headers: { + Range: 'bytes=0-2', + }, + } + ) + t.is(responseGet.status, 206) + t.deepEqual(await responseGet.text(), gatewayTxtResponse.slice(0, 2 + 1)) // range includes +}) + test('Gets 404 response from perma cache by URL when url not perma cached', async (t) => { const { mf } = t.context const url = diff --git a/packages/api/test/scripts/mocks/r2.js b/packages/api/test/scripts/mocks/r2.js index 06c6d664..aacf4573 100644 --- a/packages/api/test/scripts/mocks/r2.js +++ b/packages/api/test/scripts/mocks/r2.js @@ -32,19 +32,25 @@ export function createR2Bucket() { size: data.length, }) }, - get: async (key) => { + get: async (key, options = {}) => { const value = bucket.get(key) if (!value) { return undefined } - const response = new Response(value.body, { status: 200 }) + let body = value.body + if (options.range) { + body = value.body.slice(options.range.offset, options.range.length) + } + + const response = new Response(body, { status: 200 }) return Promise.resolve( Object.assign(response, { httpMetadata: value.httpMetadata || {}, customMetadata: value.customMetadata || {}, size: value.body.length, + writeHttpMetadata: () => {}, }) ) }, diff --git a/packages/edge-gateway/src/gateway.js b/packages/edge-gateway/src/gateway.js index 6ed7b031..60a58021 100644 --- a/packages/edge-gateway/src/gateway.js +++ b/packages/edge-gateway/src/gateway.js @@ -159,9 +159,9 @@ export async function gatewayIpfs(request, env, ctx, options = {}) { winnerGwResponse.response.headers.get('content-length') ) - // Cache request URL in Cloudflare CDN if smaller than CF_CACHE_MAX_OBJECT_SIZE + // Cache request in Cloudflare CDN if smaller than CF_CACHE_MAX_OBJECT_SIZE if (contentLengthMb <= CF_CACHE_MAX_OBJECT_SIZE) { - await cache.put(request.url, winnerGwResponse.response.clone()) + await cache.put(request, winnerGwResponse.response.clone()) } })() ) @@ -273,7 +273,11 @@ async function cdnResolution(request, env, cache) { try { const res = await pAny( - [cache.match(request.url), getFromPermaCache(request, env)], + [ + cache.match(request), // Request from cache API + cache.match(request.url), // Request URL from cache API - To be deprecated + getFromPermaCache(request, env), + ], { filter: (res) => !!res, } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a6b98f31..f9f1f350 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -34,6 +34,7 @@ importers: esbuild: ^0.14.38 execa: ^5.1.1 git-rev-sync: ^3.0.1 + http-range-parse: ^1.0.0 itty-router: ^2.4.5 miniflare: ^2.5.0 multiformats: ^9.6.4 @@ -50,6 +51,7 @@ importers: uint8arrays: ^3.0.0 dependencies: '@supabase/postgrest-js': 0.37.2 + http-range-parse: 1.0.0 itty-router: 2.6.1 multiformats: 9.6.4 nanoid: 3.3.3 @@ -11001,6 +11003,13 @@ packages: toidentifier: 1.0.1 dev: true + /http-range-parse/1.0.0: + resolution: + { + integrity: sha512-8xfObVjVGz3VGdiMbbsxoMhxgZrcLOagvzf5rcTfwU2OmmyWJib6Y9rtyjjIFFJ8J/UUEaik4U6AmvnC70YLiw==, + } + dev: false + /http-signature/1.2.0: resolution: { integrity: sha1-muzZJRFHcvPZW2WmCruPfBj7rOE= } engines: { node: '>=0.8', npm: '>=1.3.7' }