From 2ffcba29c5515494c7259bb572431fda2660a8ff Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Wed, 14 Feb 2024 10:10:17 -0800 Subject: [PATCH 1/4] fix: verified-fetch etag header --- .../verified-fetch/src/utils/get-e-tag.ts | 35 +++++++++++++++++++ .../src/utils/parse-url-string.ts | 7 +++- packages/verified-fetch/src/verified-fetch.ts | 8 +++-- .../verified-fetch/test/get-e-tag.spec.ts | 33 +++++++++++++++++ 4 files changed, 79 insertions(+), 4 deletions(-) create mode 100644 packages/verified-fetch/src/utils/get-e-tag.ts create mode 100644 packages/verified-fetch/test/get-e-tag.spec.ts diff --git a/packages/verified-fetch/src/utils/get-e-tag.ts b/packages/verified-fetch/src/utils/get-e-tag.ts new file mode 100644 index 00000000..34a0cd0a --- /dev/null +++ b/packages/verified-fetch/src/utils/get-e-tag.ts @@ -0,0 +1,35 @@ +import type { RequestFormatShorthand } from '../types.js' +import type { CID } from 'multiformats/cid' + +interface GetETagArg { + cid: CID + reqFormat?: RequestFormatShorthand + rangeStart?: number + rangeEnd?: number + /** + * If non-determinictic, weak should be true. Some examples: + * - IPNS requests + * - CAR streamed with blocks in non-deterministic order + * - TAR streamed with files in non-deterministic order + */ + weak?: boolean +} + +/** + * etag + * you need to wrap cid with "" + * we use strong Etags for immutable responses and weak one (prefixed with W/ ) for mutable/generated ones (ipns and generated HTML). + * block and car responses should have different etag than deserialized one, so you can add some prefix like we do in existing gateway + * + * @see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag + * @see https://specs.ipfs.tech/http-gateways/path-gateway/#etag-response-header + */ +export function getETag ({ cid, reqFormat, weak, rangeStart, rangeEnd }: GetETagArg): string { + const prefix = weak === true ? 'W/' : '' + let suffix = reqFormat == null ? '' : `.${reqFormat}` + if (rangeStart != null || rangeEnd != null) { + suffix += `.${rangeStart ?? '0'}-${rangeEnd ?? 'N'}` + } + + return `${prefix}"${cid.toString()}${suffix}"` +} diff --git a/packages/verified-fetch/src/utils/parse-url-string.ts b/packages/verified-fetch/src/utils/parse-url-string.ts index 5846c413..6c64f7e0 100644 --- a/packages/verified-fetch/src/utils/parse-url-string.ts +++ b/packages/verified-fetch/src/utils/parse-url-string.ts @@ -1,6 +1,7 @@ import { peerIdFromString } from '@libp2p/peer-id' import { CID } from 'multiformats/cid' import { TLRU } from './tlru.js' +import type { RequestFormatShorthand } from '../types' import type { IPNS, IPNSRoutingEvents, ResolveDnsLinkProgressEvents, ResolveProgressEvents, ResolveResult } from '@helia/ipns' import type { ComponentLogger } from '@libp2p/interface' import type { ProgressOptions } from 'progress-events' @@ -16,11 +17,15 @@ export interface ParseUrlStringOptions extends ProgressOptions { + format?: RequestFormatShorthand +} + export interface ParsedUrlStringResults { protocol: string path: string cid: CID - query: Record + query: ParsedUrlQuery } const URL_REGEX = /^(?ip[fn]s):\/\/(?[^/$?]+)\/?(?[^$?]*)\??(?.*)$/ diff --git a/packages/verified-fetch/src/verified-fetch.ts b/packages/verified-fetch/src/verified-fetch.ts index 4dc3a925..510982c1 100644 --- a/packages/verified-fetch/src/verified-fetch.ts +++ b/packages/verified-fetch/src/verified-fetch.ts @@ -11,10 +11,12 @@ import { code as jsonCode } from 'multiformats/codecs/json' import { decode, code as rawCode } from 'multiformats/codecs/raw' import { identity } from 'multiformats/hashes/identity' import { CustomProgressEvent } from 'progress-events' +import { getETag } from './utils/get-e-tag.js' import { getStreamFromAsyncIterable } from './utils/get-stream-from-async-iterable.js' import { parseResource } from './utils/parse-resource.js' import { walkPath, type PathWalkerFn } from './utils/walk-path.js' import type { CIDDetail, ContentTypeParser, Resource, VerifiedFetchInit as VerifiedFetchOptions } from './index.js' +import type { RequestFormatShorthand } from './types.js' import type { Helia } from '@helia/interface' import type { AbortOptions, Logger } from '@libp2p/interface' import type { UnixFSEntry } from 'ipfs-unixfs-exporter' @@ -246,8 +248,8 @@ export class VerifiedFetch { * @see https://specs.ipfs.tech/http-gateways/path-gateway/#format-request-query-parameter * @default 'raw' */ - private getFormat ({ headerFormat, queryFormat }: { headerFormat: string | null, queryFormat: string | null }): string | null { - const formatMap: Record = { + private getFormat ({ headerFormat, queryFormat }: { headerFormat: string | null, queryFormat: RequestFormatShorthand | null }): RequestFormatShorthand | null { + const formatMap: Record = { 'vnd.ipld.raw': 'raw', 'vnd.ipld.car': 'car', 'application/x-tar': 'tar', @@ -338,7 +340,7 @@ export class VerifiedFetch { } } - response.headers.set('etag', cid.toString()) // https://specs.ipfs.tech/http-gateways/path-gateway/#etag-response-header + response.headers.set('etag', getETag({ cid, reqFormat: format ?? undefined, weak: false })) response.headers.set('cache-control', 'public, max-age=29030400, immutable') response.headers.set('X-Ipfs-Path', resource.toString()) // https://specs.ipfs.tech/http-gateways/path-gateway/#x-ipfs-path-response-header diff --git a/packages/verified-fetch/test/get-e-tag.spec.ts b/packages/verified-fetch/test/get-e-tag.spec.ts new file mode 100644 index 00000000..a0908d58 --- /dev/null +++ b/packages/verified-fetch/test/get-e-tag.spec.ts @@ -0,0 +1,33 @@ +import { expect } from 'aegir/chai' +import { CID } from 'multiformats/cid' +import { getETag } from '../src/utils/get-e-tag.js' + +const cidString = 'QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr' +const testCID = CID.parse(cidString) + +describe('getETag', () => { + it('CID eTag', () => { + expect(getETag({ cid: testCID, weak: true })).to.equal(`W/"${cidString}"`) + expect(getETag({ cid: testCID, weak: false })).to.equal(`"${cidString}"`) + }) + + it('should return ETag with CID and format suffix', () => { + expect(getETag({ cid: testCID, reqFormat: 'raw' })).to.equal(`"${cidString}.raw"`) + expect(getETag({ cid: testCID, reqFormat: 'json' })).to.equal(`"${cidString}.json"`) + }) + + it('should return ETag with CID and range suffix', () => { + expect(getETag({ cid: testCID, weak: true, reqFormat: 'car', rangeStart: 10, rangeEnd: 20 })).to.equal(`W/"${cidString}.car.10-20"`) + expect(getETag({ cid: testCID, weak: false, reqFormat: 'car', rangeStart: 10, rangeEnd: 20 })).to.equal(`"${cidString}.car.10-20"`) + }) + + it('should return ETag with CID, format and range suffix', () => { + expect(getETag({ cid: testCID, reqFormat: 'raw', weak: false, rangeStart: 10, rangeEnd: 20 })).to.equal(`"${cidString}.raw.10-20"`) + }) + + it('should handle undefined rangeStart and rangeEnd', () => { + expect(getETag({ cid: testCID, reqFormat: 'raw', weak: false, rangeStart: undefined, rangeEnd: undefined })).to.equal(`"${cidString}.raw"`) + expect(getETag({ cid: testCID, reqFormat: 'raw', weak: false, rangeStart: 55, rangeEnd: undefined })).to.equal(`"${cidString}.raw.55-N"`) + expect(getETag({ cid: testCID, reqFormat: 'raw', weak: false, rangeStart: undefined, rangeEnd: 77 })).to.equal(`"${cidString}.raw.0-77"`) + }) +}) From 7562bfaa83547a4f8f2300272361bb4d6d9cd6df Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Wed, 14 Feb 2024 10:14:28 -0800 Subject: [PATCH 2/4] chore: src/types.ts --- packages/verified-fetch/src/types.ts | 1 + 1 file changed, 1 insertion(+) create mode 100644 packages/verified-fetch/src/types.ts diff --git a/packages/verified-fetch/src/types.ts b/packages/verified-fetch/src/types.ts new file mode 100644 index 00000000..4a235e1a --- /dev/null +++ b/packages/verified-fetch/src/types.ts @@ -0,0 +1 @@ +export type RequestFormatShorthand = 'raw' | 'car' | 'tar' | 'ipns-record' | 'dag-json' | 'dag-cbor' | 'json' | 'cbor' From caa3530e03948e83eb566cb497a1e370c60fecec Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Wed, 14 Feb 2024 10:32:29 -0800 Subject: [PATCH 3/4] chore: jsdoc comment clarification Co-authored-by: Marcin Rataj --- packages/verified-fetch/src/utils/get-e-tag.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/verified-fetch/src/utils/get-e-tag.ts b/packages/verified-fetch/src/utils/get-e-tag.ts index 34a0cd0a..74ab7303 100644 --- a/packages/verified-fetch/src/utils/get-e-tag.ts +++ b/packages/verified-fetch/src/utils/get-e-tag.ts @@ -7,7 +7,8 @@ interface GetETagArg { rangeStart?: number rangeEnd?: number /** - * If non-determinictic, weak should be true. Some examples: + * Weak Etag is used when we can't guarantee byte-for-byte-determinism (generated, or mutable content). + * Some examples: * - IPNS requests * - CAR streamed with blocks in non-deterministic order * - TAR streamed with files in non-deterministic order From 4328559b1a0981bcb8b4d4b74b710d7f03117c96 Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Thu, 15 Feb 2024 11:33:16 -0800 Subject: [PATCH 4/4] chore: import extension Co-authored-by: Alex Potsides --- packages/verified-fetch/src/utils/parse-url-string.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/verified-fetch/src/utils/parse-url-string.ts b/packages/verified-fetch/src/utils/parse-url-string.ts index 6c64f7e0..4c078973 100644 --- a/packages/verified-fetch/src/utils/parse-url-string.ts +++ b/packages/verified-fetch/src/utils/parse-url-string.ts @@ -1,7 +1,7 @@ import { peerIdFromString } from '@libp2p/peer-id' import { CID } from 'multiformats/cid' import { TLRU } from './tlru.js' -import type { RequestFormatShorthand } from '../types' +import type { RequestFormatShorthand } from '../types.js' import type { IPNS, IPNSRoutingEvents, ResolveDnsLinkProgressEvents, ResolveProgressEvents, ResolveResult } from '@helia/ipns' import type { ComponentLogger } from '@libp2p/interface' import type { ProgressOptions } from 'progress-events'