diff --git a/package.json b/package.json index 471e0e3..f7d640e 100644 --- a/package.json +++ b/package.json @@ -122,7 +122,7 @@ "docs:no-publish": "aegir docs --publish false" }, "devDependencies": { - "aegir": "^42.2.5", + "aegir": "^42.2.11", "npm-run-all": "^4.1.5" }, "type": "module", diff --git a/packages/gateway-conformance/.aegir.js b/packages/gateway-conformance/.aegir.js index c5e6076..12b43e9 100644 --- a/packages/gateway-conformance/.aegir.js +++ b/packages/gateway-conformance/.aegir.js @@ -1,5 +1,7 @@ // @ts-check import getPort from 'aegir/get-port' +import { logger } from '@libp2p/logger' +const log = logger('aegir') /** @type {import('aegir').PartialOptions} */ export default { @@ -12,38 +14,32 @@ export default { const { createKuboNode } = await import('./dist/src/fixtures/create-kubo.js') const KUBO_PORT = await getPort(3440) + const SERVER_PORT = await getPort(3441) + // The Kubo gateway will be passed to the VerifiedFetch config const { node: controller, gatewayUrl, repoPath } = await createKuboNode(KUBO_PORT) await controller.start() const { loadKuboFixtures } = await import('./dist/src/fixtures/kubo-mgmt.js') const IPFS_NS_MAP = await loadKuboFixtures(repoPath) const kuboGateway = gatewayUrl - const { startBasicServer } = await import('./dist/src/fixtures/basic-server.js') - const SERVER_PORT = await getPort(3441) - const stopBasicServer = await startBasicServer({ + const { startVerifiedFetchGateway } = await import('./dist/src/fixtures/basic-server.js') + const stopBasicServer = await startVerifiedFetchGateway({ serverPort: SERVER_PORT, - kuboGateway - }) - - const { startReverseProxy } = await import('./dist/src/fixtures/reverse-proxy.js') - const PROXY_PORT = await getPort(3442) - const stopReverseProxy = await startReverseProxy({ - backendPort: SERVER_PORT, - targetHost: 'localhost', - proxyPort: PROXY_PORT + kuboGateway, + IPFS_NS_MAP + }).catch((err) => { + log.error(err) }) const CONFORMANCE_HOST = 'localhost' return { controller, - stopReverseProxy, stopBasicServer, env: { IPFS_NS_MAP, CONFORMANCE_HOST, KUBO_PORT: `${KUBO_PORT}`, - PROXY_PORT: `${PROXY_PORT}`, SERVER_PORT: `${SERVER_PORT}`, KUBO_GATEWAY: kuboGateway } @@ -51,11 +47,13 @@ export default { }, after: async (options, beforeResult) => { // @ts-expect-error - broken aegir types - await beforeResult.stopReverseProxy() + await beforeResult.controller.stop() + log('controller stopped') + // @ts-expect-error - broken aegir types await beforeResult.stopBasicServer() - // @ts-expect-error - broken aegir types - await beforeResult.controller.stop() + log('basic server stopped') + } } } diff --git a/packages/gateway-conformance/package.json b/packages/gateway-conformance/package.json index 9e8d2fa..a836bc5 100644 --- a/packages/gateway-conformance/package.json +++ b/packages/gateway-conformance/package.json @@ -52,17 +52,29 @@ "test": "aegir test -t node" }, "dependencies": { + "@helia/block-brokers": "^3.0.1", + "@helia/http": "^1.0.8", "@helia/interface": "^4.3.0", + "@helia/routers": "^1.1.0", "@helia/verified-fetch": "1.4.2", - "@libp2p/logger": "^4.0.11", + "@libp2p/kad-dht": "^12.0.17", + "@libp2p/logger": "^4.0.13", + "@libp2p/peer-id": "^4.1.2", + "@multiformats/dns": "^1.0.6", "@sgtpooki/file-type": "^1.0.1", - "aegir": "^42.2.5", - "execa": "^8.0.1", + "aegir": "^42.2.11", + "blockstore-core": "^4.4.1", + "datastore-core": "^9.2.9", + "execa": "^9.1.0", "fast-glob": "^3.3.2", + "interface-blockstore": "^5.2.10", + "interface-datastore": "^8.2.11", "ipfsd-ctl": "^14.1.0", - "kubo": "^0.27.0", + "ipns": "^9.1.0", + "kubo": "^0.28.0", "kubo-rpc-client": "^4.1.1", - "undici": "^6.15.0" + "uint8arrays": "^5.1.0", + "undici": "^6.18.1" }, "browser": { "./dist/src/fixtures/create-kubo.js": "./dist/src/fixtures/create-kubo.browser.js", diff --git a/packages/gateway-conformance/src/conformance.spec.ts b/packages/gateway-conformance/src/conformance.spec.ts index 97f5ee8..c59d0db 100644 --- a/packages/gateway-conformance/src/conformance.spec.ts +++ b/packages/gateway-conformance/src/conformance.spec.ts @@ -1,4 +1,3 @@ -/* eslint-disable @typescript-eslint/no-unused-vars */ /* eslint-env mocha */ import { readFile } from 'node:fs/promises' import { homedir } from 'node:os' @@ -8,7 +7,7 @@ import { expect } from 'aegir/chai' import { execa } from 'execa' import { Agent, setGlobalDispatcher } from 'undici' -const logger = prefixLogger('conformance-tests') +const logger = prefixLogger('gateway-conformance') interface TestConfig { name: string @@ -16,6 +15,7 @@ interface TestConfig { skip?: string[] run?: string[] successRate: number + timeout?: number } function getGatewayConformanceBinaryPath (): string { @@ -29,10 +29,8 @@ function getGatewayConformanceBinaryPath (): string { function getConformanceTestArgs (name: string, gwcArgs: string[] = [], goTestArgs: string[] = []): string[] { return [ 'test', - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - `--gateway-url=http://${process.env.CONFORMANCE_HOST!}:${process.env.PROXY_PORT!}`, - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - `--subdomain-url=http://${process.env.CONFORMANCE_HOST!}:${process.env.PROXY_PORT!}`, + `--gateway-url=http://127.0.0.1:${process.env.SERVER_PORT}`, + `--subdomain-url=http://${process.env.CONFORMANCE_HOST}:${process.env.SERVER_PORT}`, '--verbose', '--json', `gwc-report-${name}.json`, ...gwcArgs, @@ -58,7 +56,7 @@ const tests: TestConfig[] = [ { name: 'TestDagPbConversion', run: ['TestDagPbConversion'], - successRate: 35.38 + successRate: 26.15 }, { name: 'TestPlainCodec', @@ -68,11 +66,14 @@ const tests: TestConfig[] = [ { name: 'TestPathing', run: ['TestPathing'], - successRate: 23.53 + successRate: 40 }, { name: 'TestDNSLinkGatewayUnixFSDirectoryListing', run: ['TestDNSLinkGatewayUnixFSDirectoryListing'], + skip: [ + 'TestDNSLinkGatewayUnixFSDirectoryListing/.*TODO:_cleanup_Kubo-specifics' + ], successRate: 0 }, { @@ -83,23 +84,22 @@ const tests: TestConfig[] = [ { name: 'TestGatewayJsonCbor', run: ['TestGatewayJsonCbor'], - successRate: 44.44 + successRate: 22.22 + }, + { + name: 'TestNativeDag', + run: ['TestNativeDag'], + successRate: 60.71 }, - // currently results in an infinite loop without verified-fetch stopping the request whether sessions are enabled or not. - // { - // name: 'TestNativeDag', - // run: ['TestNativeDag'], - // successRate: 100 - // }, { name: 'TestGatewayJSONCborAndIPNS', run: ['TestGatewayJSONCborAndIPNS'], - successRate: 24.24 + successRate: 51.52 }, { name: 'TestGatewayIPNSPath', run: ['TestGatewayIPNSPath'], - successRate: 27.27 + successRate: 100 }, { name: 'TestRedirectCanonicalIPNS', @@ -109,7 +109,7 @@ const tests: TestConfig[] = [ { name: 'TestGatewayBlock', run: ['TestGatewayBlock'], - successRate: 37.93 + successRate: 20.69 }, { name: 'TestTrustlessRawRanges', @@ -119,20 +119,21 @@ const tests: TestConfig[] = [ { name: 'TestTrustlessRaw', run: ['TestTrustlessRaw'], - successRate: 55.56 + skip: ['TestTrustlessRawRanges'], + successRate: 70.83 }, { name: 'TestGatewayIPNSRecord', run: ['TestGatewayIPNSRecord'], - successRate: 0 + successRate: 17.39 }, { name: 'TestTrustlessCarOrderAndDuplicates', run: ['TestTrustlessCarOrderAndDuplicates'], - successRate: 13.79 + successRate: 44.83 }, - // times out // { + // // currently timing out // name: 'TestTrustlessCarEntityBytes', // run: ['TestTrustlessCarEntityBytes'], // successRate: 100 @@ -140,44 +141,54 @@ const tests: TestConfig[] = [ { name: 'TestTrustlessCarDagScopeAll', run: ['TestTrustlessCarDagScopeAll'], - successRate: 36.36 - }, - { - name: 'TestTrustlessCarDagScopeEntity', - run: ['TestTrustlessCarDagScopeEntity'], - successRate: 34.57 - }, - { - name: 'TestTrustlessCarDagScopeBlock', - run: ['TestTrustlessCarDagScopeBlock'], - successRate: 34.69 - }, - { - name: 'TestTrustlessCarPathing', - run: ['TestTrustlessCarPathing'], - successRate: 33.85 - }, - { - name: 'TestSubdomainGatewayDNSLinkInlining', - run: ['TestSubdomainGatewayDNSLinkInlining'], - successRate: 0 + successRate: 54.55 }, + // { + // // currently timing out + // name: 'TestTrustlessCarDagScopeEntity', + // run: ['TestTrustlessCarDagScopeEntity'], + // successRate: 34.57 + // }, + // { + // // currently timing out + // name: 'TestTrustlessCarDagScopeBlock', + // run: ['TestTrustlessCarDagScopeBlock'], + // successRate: 34.69 + // }, + // { + // // passes at the set successRate, but takes incredibly long (consistently ~2m).. disabling for now. + // name: 'TestTrustlessCarPathing', + // run: ['TestTrustlessCarPathing'], + // successRate: 35, + // timeout: 130000 + // }, + // { + // // currently timing out + // name: 'TestSubdomainGatewayDNSLinkInlining', + // run: ['TestSubdomainGatewayDNSLinkInlining'], + // successRate: 100 + // }, { name: 'TestGatewaySubdomainAndIPNS', run: ['TestGatewaySubdomainAndIPNS'], - successRate: 0 + successRate: 31.58 }, { + // TODO: add directory listing support to verified-fetch name: 'TestGatewaySubdomains', - run: ['TestGatewaySubdomains'], - successRate: 7.17 + run: [ + 'TestGatewaySubdomains' + ], + skip: [ + 'TestGatewaySubdomains/.*HTTP_proxy_tunneling_via_CONNECT' // verified fetch should not be doing HTTP proxy tunneling. + ], + successRate: 41.35 + }, + { + name: 'TestUnixFSDirectoryListingOnSubdomainGateway', + run: ['TestUnixFSDirectoryListingOnSubdomainGateway'], + successRate: 10.26 }, - // times out - // { - // name: 'TestUnixFSDirectoryListingOnSubdomainGateway', - // run: ['TestUnixFSDirectoryListingOnSubdomainGateway'], - // successRate: 100 - // }, { name: 'TestRedirectsFileWithIfNoneMatchHeader', run: ['TestRedirectsFileWithIfNoneMatchHeader'], @@ -191,7 +202,8 @@ const tests: TestConfig[] = [ { name: 'TestRedirectsFileSupport', run: ['TestRedirectsFileSupport'], - successRate: 2.33 + skip: ['TestRedirectsFileSupportWithDNSLink'], + successRate: 0 }, { name: 'TestPathGatewayMiscellaneous', @@ -201,7 +213,7 @@ const tests: TestConfig[] = [ { name: 'TestGatewayUnixFSFileRanges', run: ['TestGatewayUnixFSFileRanges'], - successRate: 40 + successRate: 46.67 }, { name: 'TestGatewaySymlink', @@ -211,24 +223,29 @@ const tests: TestConfig[] = [ { name: 'TestGatewayCacheWithIPNS', run: ['TestGatewayCacheWithIPNS'], - successRate: 35.71 + successRate: 66.67 }, - // times out // { + // // passes at the set successRate, but takes incredibly long (consistently ~2m).. disabling for now. // name: 'TestGatewayCache', // run: ['TestGatewayCache'], - // successRate: 100 - // }, - // times out - // { - // name: 'TestUnixFSDirectoryListing', - // run: ['TestUnixFSDirectoryListing'], - // successRate: 100 + // skip: ['TestGatewayCacheWithIPNS'], + // successRate: 59.38, + // timeout: 1200000 // }, + { + name: 'TestUnixFSDirectoryListing', + run: ['TestUnixFSDirectoryListing'], + skip: [ + 'TestUnixFSDirectoryListingOnSubdomainGateway', + 'TestUnixFSDirectoryListing/.*TODO:_cleanup_Kubo-specifics' + ], + successRate: 50 + }, { name: 'TestTar', run: ['TestTar'], - successRate: 50 + successRate: 62.5 } ] @@ -260,9 +277,6 @@ describe('@helia/verified-fetch - gateway conformance', function () { if (process.env.KUBO_GATEWAY == null) { throw new Error('KUBO_GATEWAY env var is required') } - if (process.env.PROXY_PORT == null) { - throw new Error('PROXY_PORT env var is required') - } if (process.env.SERVER_PORT == null) { throw new Error('SERVER_PORT env var is required') } @@ -270,7 +284,7 @@ describe('@helia/verified-fetch - gateway conformance', function () { throw new Error('CONFORMANCE_HOST env var is required') } // see https://stackoverflow.com/questions/71074255/use-custom-dns-resolver-for-any-request-in-nodejs - // EVERY undici/fetch request host resolves to local IP. Node.js does not resolve reverse-proxy requests properly + // EVERY undici/fetch request host resolves to local IP. Without this, Node.js does not resolve subdomain requests properly const staticDnsAgent = new Agent({ connect: { lookup: (_hostname, _options, callback) => { callback(null, [{ address: '0.0.0.0', family: 4 }]) } @@ -282,9 +296,7 @@ describe('@helia/verified-fetch - gateway conformance', function () { describe('smokeTests', () => { [ ['basic server path request works', `http://localhost:${process.env.SERVER_PORT}/ipfs/bafkqabtimvwgy3yk`], - ['proxy server path request works', `http://localhost:${process.env.PROXY_PORT}/ipfs/bafkqabtimvwgy3yk`], - ['basic server subdomain request works', `http://bafkqabtimvwgy3yk.ipfs.localhost:${process.env.SERVER_PORT}`], - ['proxy server subdomain request works', `http://bafkqabtimvwgy3yk.ipfs.localhost:${process.env.PROXY_PORT}`] + ['basic server subdomain request works', `http://bafkqabtimvwgy3yk.ipfs.localhost:${process.env.SERVER_PORT}`] ].forEach(([name, url]) => { it(name, async () => { const resp = await fetch(url) @@ -311,19 +323,28 @@ describe('@helia/verified-fetch - gateway conformance', function () { after(async () => { const log = logger.forComponent('after') - try { - await execa('rm', [binaryPath]) - log('gateway-conformance binary successfully uninstalled.') - } catch (error) { - log.error(`Error removing "${binaryPath}"`, error) + + if (process.env.GATEWAY_CONFORMANCE_BINARY == null) { + try { + await execa('rm', [binaryPath]) + log('gateway-conformance binary successfully uninstalled.') + } catch (error) { + log.error(`Error removing "${binaryPath}"`, error) + } + } else { + log('Not removing custom gateway-conformance binary at %s', binaryPath) } }) - tests.forEach(({ name, spec, skip, run, successRate: minSuccessRate }) => { - const log = logger.forComponent(name) + tests.forEach(({ name, spec, skip, run, timeout, successRate: minSuccessRate }) => { + const log = logger.forComponent(`output:${name}`) const expectedSuccessRate = process.env.SUCCESS_RATE != null ? Number.parseFloat(process.env.SUCCESS_RATE) : minSuccessRate it(`${name} has a success rate of at least ${expectedSuccessRate}%`, async function () { + if (timeout != null) { + this.timeout(timeout) + } + const { stderr, stdout } = await execa(binaryPath, getConformanceTestArgs(name, [ ...(spec != null ? ['--specs', spec] : []) @@ -332,7 +353,7 @@ describe('@helia/verified-fetch - gateway conformance', function () { ...((skip != null) ? ['-skip', `${skip.join('|')}`] : []), ...((run != null) ? ['-run', `${run.join('|')}`] : []) ] - ), { reject: false }) + ), { reject: false, cancelSignal: timeout != null ? AbortSignal.timeout(timeout) : undefined }) log(stdout) log.error(stderr) @@ -348,27 +369,20 @@ describe('@helia/verified-fetch - gateway conformance', function () { * as this test does. */ it('has expected total failures and successes', async function () { - const log = logger.forComponent('all') - - // TODO: unskip when verified-fetch is no longer infinitely looping on requests. - const toSkip = [ - 'TestNativeDag', - 'TestTrustlessCarEntityBytes', - 'TestUnixFSDirectoryListingOnSubdomainGateway', - 'TestGatewayCache', - 'TestUnixFSDirectoryListing' - ] + this.timeout(200000) + const log = logger.forComponent('output:all') - const { stderr, stdout } = await execa(binaryPath, getConformanceTestArgs('all', [], ['-skip', toSkip.join('|')]), { reject: false }) + const { stderr, stdout } = await execa(binaryPath, getConformanceTestArgs('all', [], []), { reject: false, cancelSignal: AbortSignal.timeout(200000) }) log(stdout) log.error(stderr) - const { failureCount, successCount, successRate } = await getReportDetails('gwc-report-all.json') + const { successRate } = await getReportDetails('gwc-report-all.json') + const knownSuccessRate = 42.47 + // check latest success rate with `SUCCESS_RATE=100 npm run test -- -g 'total'` + const expectedSuccessRate = process.env.SUCCESS_RATE != null ? Number.parseFloat(process.env.SUCCESS_RATE) : knownSuccessRate - expect(failureCount).to.be.lessThanOrEqual(1134) - expect(successCount).to.be.greaterThanOrEqual(262) - expect(successRate).to.be.greaterThanOrEqual(18.77) + expect(successRate).to.be.greaterThanOrEqual(expectedSuccessRate) }) }) }) diff --git a/packages/gateway-conformance/src/demo-server.ts b/packages/gateway-conformance/src/demo-server.ts index 39698a3..9c5e44e 100644 --- a/packages/gateway-conformance/src/demo-server.ts +++ b/packages/gateway-conformance/src/demo-server.ts @@ -3,33 +3,30 @@ */ import { logger } from '@libp2p/logger' import getPort from 'aegir/get-port' -import { startBasicServer } from './fixtures/basic-server.js' +import { startVerifiedFetchGateway } from './fixtures/basic-server.js' import { createKuboNode } from './fixtures/create-kubo.js' import { loadKuboFixtures } from './fixtures/kubo-mgmt.js' -import { startReverseProxy } from './fixtures/reverse-proxy.js' const log = logger('demo-server') -const { node: controller, gatewayUrl, repoPath } = await createKuboNode(await getPort(3440)) +const KUBO_GATEWAY_PORT = await getPort(3440) +const SERVER_PORT = await getPort(3441) +const { node: controller, gatewayUrl, repoPath } = await createKuboNode(KUBO_GATEWAY_PORT) const kuboGateway = gatewayUrl await controller.start() -await loadKuboFixtures(repoPath) +const IPFS_NS_MAP = await loadKuboFixtures(repoPath) -const SERVER_PORT = await getPort(3441) -await startBasicServer({ +const stopServer = await startVerifiedFetchGateway({ serverPort: SERVER_PORT, - kuboGateway -}) - -const PROXY_PORT = await getPort(3442) -await startReverseProxy({ - backendPort: SERVER_PORT, - targetHost: 'localhost', - proxyPort: PROXY_PORT + kuboGateway, + IPFS_NS_MAP }) process.on('exit', () => { + stopServer().catch((err) => { + log.error('Failed to stop server', err) + }) controller.stop().catch((err) => { log.error('Failed to stop controller', err) process.exit(1) diff --git a/packages/gateway-conformance/src/fixtures/basic-server.ts b/packages/gateway-conformance/src/fixtures/basic-server.ts index 45ca9b3..4f94958 100644 --- a/packages/gateway-conformance/src/fixtures/basic-server.ts +++ b/packages/gateway-conformance/src/fixtures/basic-server.ts @@ -1,7 +1,19 @@ -import { createServer } from 'node:http' +import { createServer, type IncomingMessage, type ServerResponse } from 'node:http' +import { trustlessGateway } from '@helia/block-brokers' +import { createHeliaHTTP } from '@helia/http' +import { httpGatewayRouting } from '@helia/routers' import { logger } from '@libp2p/logger' +import { dns } from '@multiformats/dns' +import { MemoryBlockstore } from 'blockstore-core' +import { Agent, setGlobalDispatcher } from 'undici' import { contentTypeParser } from './content-type-parser.js' import { createVerifiedFetch } from './create-verified-fetch.js' +import { getLocalDnsResolver } from './get-local-dns-resolver.js' +import { convertFetchHeadersToNodeJsHeaders, convertNodeJsHeadersToFetchHeaders } from './header-utils.js' +import { getIpnsRecordDatastore } from './ipns-record-datastore.js' +import type { DNSResolver } from '@multiformats/dns/resolvers' +import type { Blockstore } from 'interface-blockstore' +import type { Datastore } from 'interface-datastore' const log = logger('basic-server') /** @@ -13,9 +25,165 @@ const log = logger('basic-server') export interface BasicServerOptions { kuboGateway?: string serverPort: number + + /** + * @see https://github.com/ipfs/kubo/blob/5de5b77168be347186dbc9f1586c2deb485ca2ef/docs/environment-variables.md#ipfs_ns_map + */ + IPFS_NS_MAP: string +} + +type Response = ServerResponse & { + req: IncomingMessage +} + +interface CreateHeliaOptions { + gateways: string[] + dnsResolvers: DNSResolver[] + blockstore: Blockstore + datastore: Datastore +} + +/** + * We need to create helia manually so we can stub some of the things... + */ +async function createHelia (init: CreateHeliaOptions): Promise> { + return createHeliaHTTP({ + blockstore: init.blockstore, + datastore: init.datastore, + blockBrokers: [ + trustlessGateway({ + allowInsecure: true, + allowLocal: true + }) + ], + routers: [ + httpGatewayRouting({ + gateways: init.gateways + }) + ], + dns: dns({ + resolvers: { + '.': init.dnsResolvers + } + }) + }) +} + +interface CallVerifiedFetchOptions { + serverPort: number + useSessions: boolean + verifiedFetch: Awaited> } -export async function startBasicServer ({ kuboGateway, serverPort }: BasicServerOptions): Promise<() => Promise> { +async function callVerifiedFetch (req: IncomingMessage, res: Response, { serverPort, useSessions, verifiedFetch }: CallVerifiedFetchOptions): Promise { + const log = logger('basic-server:request') + if (req.method === 'OPTIONS') { + res.writeHead(200) + res.end() + return + } + if (req.method === 'HEAD') { + res.writeHead(200) + res.end() + return + } + + if (req.url == null) { + // this should never happen + log.error('No URL provided, returning 400 Bad Request') + res.writeHead(400) + res.end('Bad Request') + return + } + + // @see https://github.com/ipfs/gateway-conformance/issues/185#issuecomment-2123708150 + let fixingGwcAnnoyance = false + if (req.headers.host != null && req.headers.host === 'localhost') { + log.trace('set fixingGwcAnnoyance to true for %s', new URL(req.url, `http://${req.headers.host}`).href) + fixingGwcAnnoyance = true + req.headers.host = `localhost:${serverPort}` + } + + const fullUrlHref = new URL(req.url, `http://${req.headers.host}`) + + const urlLog = logger(`basic-server:request:${fullUrlHref}`) + urlLog('configuring request') + urlLog.trace('req.headers: %O', req.headers) + let requestController: AbortController | null = new AbortController() + // we need to abort the request if the client disconnects + const onReqEnd = (): void => { + urlLog('client disconnected, aborting request') + requestController?.abort() + } + req.on('end', onReqEnd) + + const reqTimeout = setTimeout(() => { + /** + * Abort the request because it's taking too long. + * This is only needed for when @helia/verified-fetch is not correctly + * handling a request and should not be needed once we have 100% gateway + * conformance coverage. + */ + urlLog.error('timing out request') + requestController?.abort() + }, 2000) + reqTimeout.unref() // don't keep the process alive just for this timeout + + const onResFinish = (): void => { + urlLog.trace('response finished, aborting signal') + requestController?.abort() + } + res.on('finish', onResFinish) + + try { + urlLog.trace('calling verified-fetch') + const resp = await verifiedFetch(fullUrlHref.toString(), { redirect: 'manual', signal: requestController.signal, session: useSessions, allowInsecure: true, allowLocal: true, headers: convertNodeJsHeadersToFetchHeaders(req.headers) }) + urlLog.trace('verified-fetch response status: %d', resp.status) + + const headers = convertFetchHeadersToNodeJsHeaders({ resp, log: urlLog, fixingGwcAnnoyance, serverPort }) + + res.writeHead(resp.status, headers) + if (resp.body == null) { + // need to convert ArrayBuffer to Buffer or Uint8Array + res.write(Buffer.from(await resp.arrayBuffer())) + urlLog.trace('wrote response') + } else { + // read the body of the response and write it to the response from the server + const reader = resp.body.getReader() + while (true) { + const { done, value } = await reader.read() + if (done) { + urlLog.trace('response stream finished') + break + } + + res.write(Buffer.from(value)) + } + } + res.end() + } catch (e: any) { + urlLog.error('Problem with request: %s', e.message, e) + if (!res.headersSent) { + res.writeHead(500) + } + res.end(`Internal Server Error: ${e.message}`) + } finally { + urlLog.trace('Cleaning up request') + clearTimeout(reqTimeout) + requestController.abort() + requestController = null + req.off('end', onReqEnd) + res.off('finish', onResFinish) + } +} + +export async function startVerifiedFetchGateway ({ kuboGateway, serverPort, IPFS_NS_MAP }: BasicServerOptions): Promise<() => Promise> { + const staticDnsAgent = new Agent({ + connect: { + lookup: (_hostname, _options, callback) => { callback(null, [{ address: '0.0.0.0', family: 4 }]) } + } + }) + setGlobalDispatcher(staticDnsAgent) kuboGateway = kuboGateway ?? process.env.KUBO_GATEWAY const useSessions = process.env.USE_SESSIONS !== 'false' @@ -24,77 +192,35 @@ export async function startBasicServer ({ kuboGateway, serverPort }: BasicServer if (kuboGateway == null) { throw new Error('options.kuboGateway or KUBO_GATEWAY env var is required') } - const verifiedFetch = await createVerifiedFetch({ - gateways: [kuboGateway], - routers: [], - allowInsecure: true, - allowLocal: true - }, { - contentTypeParser - }) - const server = createServer((req, res) => { - if (req.method === 'OPTIONS') { - res.writeHead(200) - res.end() - return - } - - if (req.url == null) { - // this should never happen - res.writeHead(400) - res.end('Bad Request') - return - } - - log.trace('req.headers: %O', req.headers) - const hostname = req.headers.host?.split(':')[0] - const host = req.headers['x-forwarded-for'] ?? `${hostname}:${serverPort}` + const blockstore = new MemoryBlockstore() + const datastore = getIpnsRecordDatastore() + const localDnsResolver = getLocalDnsResolver(IPFS_NS_MAP, kuboGateway) - const fullUrlHref = req.headers.referer ?? `http://${host}${req.url}` - log('fetching %s', fullUrlHref) + const helia = await createHelia({ gateways: [kuboGateway], dnsResolvers: [localDnsResolver], blockstore, datastore }) - const requestController = new AbortController() - // we need to abort the request if the client disconnects - req.on('close', () => { - log('client disconnected, aborting request') - requestController.abort() - }) + const verifiedFetch = await createVerifiedFetch(helia, { + contentTypeParser + }) - void verifiedFetch(fullUrlHref, { redirect: 'manual', signal: requestController.signal, session: useSessions, allowInsecure: true, allowLocal: true }).then(async (resp) => { - // loop over headers and set them on the response - const headers: Record = {} - for (const [key, value] of resp.headers.entries()) { - headers[key] = value - } + const server = createServer((req, res) => { + try { + void callVerifiedFetch(req, res, { serverPort, useSessions, verifiedFetch }).catch((err) => { + log.error('Error in callVerifiedFetch', err) - res.writeHead(resp.status, headers) - if (resp.body == null) { - // need to convert ArrayBuffer to Buffer or Uint8Array - res.write(Buffer.from(await resp.arrayBuffer())) - } else { - // read the body of the response and write it to the response from the server - const reader = resp.body.getReader() - while (true) { - const { done, value } = await reader.read() - if (done) { - break - } - log('typeof value: %s', typeof value) - - res.write(Buffer.from(value)) + if (!res.headersSent) { + res.writeHead(500) } - } - res.end() - }).catch((e) => { - log.error('Problem with request: %s', e.message, e) + res.end('Internal Server Error') + }) + } catch (err) { + log.error('Error in createServer', err) + if (!res.headersSent) { res.writeHead(500) } - res.end(`Internal Server Error: ${e.message}`) - }).finally(() => { - requestController.abort() - }) + res.end('Internal Server Error') + } }) server.listen(serverPort, () => { @@ -102,7 +228,11 @@ export async function startBasicServer ({ kuboGateway, serverPort }: BasicServer }) return async () => { + log('Stopping...') await new Promise((resolve, reject) => { + // no matter what happens, we need to kill the server + server.closeAllConnections() + log('Closed all connections') server.close((err: any) => { if (err != null) { reject(err) diff --git a/packages/gateway-conformance/src/fixtures/get-local-dns-resolver.ts b/packages/gateway-conformance/src/fixtures/get-local-dns-resolver.ts new file mode 100644 index 0000000..cebc5ff --- /dev/null +++ b/packages/gateway-conformance/src/fixtures/get-local-dns-resolver.ts @@ -0,0 +1,68 @@ +import { logger } from '@libp2p/logger' +import { type Answer, type Question } from '@multiformats/dns' +import { type DNSResolver } from '@multiformats/dns/resolvers' + +export function getLocalDnsResolver (ipfsNsMap: string, kuboGateway: string): DNSResolver { + const log = logger('basic-server:dns') + const nsMap = new Map() + const keyVals = ipfsNsMap.split(',') + for (const keyVal of keyVals) { + const [key, val] = keyVal.split(':') + log('Setting entry: %s="%s"', key, val) + nsMap.set(key, val) + } + + return async (domain, options) => { + const questions: Question[] = [] + const answers: Answer[] = [] + + if (Array.isArray(options?.types)) { + options?.types?.forEach?.((type) => { + questions.push({ name: domain, type }) + }) + } else { + questions.push({ name: domain, type: options?.types ?? 16 }) + } + // TODO: do we need to do anything with CNAME resolution...? + // if (questions.some((q) => q.type === 5)) { + // answers.push({ + // name: domain, + // type: 5, + // TTL: 180, + // data: '' + // }) + // } + if (questions.some((q) => q.type === 16)) { + log.trace('Querying "%s" for types %O', domain, options?.types) + const actualDomainKey = domain.replace('_dnslink.', '') + const nsValue = nsMap.get(actualDomainKey) + if (nsValue == null) { + log.error('No IPFS_NS_MAP entry for domain "%s"', actualDomainKey) + + throw new Error('No IPFS_NS_MAP entry for domain') + } + const data = `dnslink=${nsValue}` + answers.push({ + name: domain, + type: 16, + TTL: 180, + data // should be in the format 'dnslink=/ipfs/bafkqac3imvwgy3zao5xxe3de' + }) + } + + const dnsResponse = { + Status: 0, + TC: false, + RD: false, + RA: false, + AD: true, + CD: true, + Question: questions, + Answer: answers + } + + log.trace('Returning DNS response for %s: %O', domain, dnsResponse) + + return dnsResponse + } +} diff --git a/packages/gateway-conformance/src/fixtures/header-utils.ts b/packages/gateway-conformance/src/fixtures/header-utils.ts new file mode 100644 index 0000000..6b11ea4 --- /dev/null +++ b/packages/gateway-conformance/src/fixtures/header-utils.ts @@ -0,0 +1,47 @@ +import type { Logger } from '@libp2p/logger' +import type { IncomingHttpHeaders } from 'undici/types/header' + +export function convertNodeJsHeadersToFetchHeaders (headers: IncomingHttpHeaders): HeadersInit { + const fetchHeaders = new Headers() + for (const [key, value] of Object.entries(headers)) { + if (value == null) { + continue + } + if (Array.isArray(value)) { + for (const v of value) { + fetchHeaders.append(key, v) + } + } else { + fetchHeaders.append(key, value) + } + } + return fetchHeaders +} + +export interface ConvertFetchHeadersToNodeJsHeadersOptions { + resp: Response + log: Logger + fixingGwcAnnoyance: boolean + serverPort: number +} + +export function convertFetchHeadersToNodeJsHeaders ({ resp, log, fixingGwcAnnoyance, serverPort }: ConvertFetchHeadersToNodeJsHeadersOptions): IncomingHttpHeaders { + const headers: Record = {} + for (const [key, value] of resp.headers.entries()) { + if (fixingGwcAnnoyance) { + log.trace('need to fix GWC annoyance.') + if (value.includes(`localhost:${serverPort}`)) { + const newValue = value.replace(`localhost:${serverPort}`, 'localhost') + log.trace('fixing GWC annoyance. Replacing Header[%s] value of "%s" with "%s"', key, value, newValue) + // we need to fix any Location, or other headers that have localhost without port in them. + headers[key] = newValue + } else { + log.trace('NOT fixing GWC annoyance. Setting Header[%s] value of "%s"', key, value) + headers[key] = value + } + } else { + headers[key] = value + } + } + return headers +} diff --git a/packages/gateway-conformance/src/fixtures/ipns-record-datastore.ts b/packages/gateway-conformance/src/fixtures/ipns-record-datastore.ts new file mode 100644 index 0000000..37dc134 --- /dev/null +++ b/packages/gateway-conformance/src/fixtures/ipns-record-datastore.ts @@ -0,0 +1,11 @@ +import { MemoryDatastore } from 'datastore-core' +import type { Datastore } from 'interface-datastore' + +const datastore = new MemoryDatastore() +/** + * We need a normalized datastore so we can set custom records + * from the IPFS_NS_MAP like kubo does. + */ +export function getIpnsRecordDatastore (): Datastore { + return datastore +} diff --git a/packages/gateway-conformance/src/fixtures/kubo-mgmt.ts b/packages/gateway-conformance/src/fixtures/kubo-mgmt.ts index 1423da5..8a88c6b 100644 --- a/packages/gateway-conformance/src/fixtures/kubo-mgmt.ts +++ b/packages/gateway-conformance/src/fixtures/kubo-mgmt.ts @@ -10,11 +10,17 @@ import { readFile } from 'node:fs/promises' import { dirname, relative, posix, basename } from 'node:path' import { fileURLToPath } from 'node:url' +import { Record as DhtRecord } from '@libp2p/kad-dht' import { logger } from '@libp2p/logger' +import { peerIdFromString } from '@libp2p/peer-id' import { $ } from 'execa' import fg from 'fast-glob' +import { Key } from 'interface-datastore' +import { peerIdToRoutingKey } from 'ipns' import { path } from 'kubo' +import { toString as uint8ArrayToString } from 'uint8arrays/to-string' import { GWC_IMAGE } from '../constants.js' +import { getIpnsRecordDatastore } from './ipns-record-datastore.js' // eslint-disable-next-line @typescript-eslint/naming-convention const __dirname = dirname(fileURLToPath(import.meta.url)) @@ -85,20 +91,24 @@ export async function loadFixtures (kuboRepoDir: string): Promise { throw new Error('No *.car fixtures found') } - // TODO: fix in CI. See https://github.com/ipfs/helia-verified-fetch/actions/runs/9022946675/job/24793649918?pr=67#step:7:19 - if (process.env.CI == null) { - for (const ipnsRecord of await fg.glob([`${GWC_FIXTURES_PATH}/**/*.ipns-record`])) { - const key = basename(ipnsRecord, '.ipns-record') - const relativePath = relative(GWC_FIXTURES_PATH, ipnsRecord) - log('Loading *.ipns-record fixture %s', relativePath) - const { stdout } = await $(({ ...execaOptions }))`cd ${GWC_FIXTURES_PATH} && ${kuboBinary} routing put --allow-offline "/ipns/${key}" "${relativePath}"` - stdout.split('\n').forEach(log) - } + const datastore = getIpnsRecordDatastore() + + for (const fsIpnsRecord of await fg.glob([`${GWC_FIXTURES_PATH}/**/*.ipns-record`])) { + const peerIdString = basename(fsIpnsRecord, '.ipns-record').split('_')[0] + const relativePath = relative(GWC_FIXTURES_PATH, fsIpnsRecord) + log('Loading *.ipns-record fixture %s', relativePath) + const key = peerIdFromString(peerIdString) + const customRoutingKey = peerIdToRoutingKey(key) + const dhtKey = new Key('/dht/record/' + uint8ArrayToString(customRoutingKey, 'base32'), false) + + const dhtRecord = new DhtRecord(customRoutingKey, await readFile(fsIpnsRecord, null), new Date(Date.now() + 9999999)) + + await datastore.put(dhtKey, dhtRecord.serialize()) } const json = await readFile(`${GWC_FIXTURES_PATH}/dnslinks.json`, 'utf-8') const { subdomains, domains } = JSON.parse(json) - const subdomainDnsLinks = Object.entries(subdomains).map(([key, value]) => `${key}.example.com:${value}`).join(',') + const subdomainDnsLinks = Object.entries(subdomains).map(([key, value]) => `${key}.localhost%3A${3441}:${value}`).join(',') const domainDnsLinks = Object.entries(domains).map(([key, value]) => `${key}:${value}`).join(',') const ipfsNsMap = `${domainDnsLinks},${subdomainDnsLinks}` diff --git a/packages/gateway-conformance/src/fixtures/reverse-proxy.ts b/packages/gateway-conformance/src/fixtures/reverse-proxy.ts deleted file mode 100644 index d942883..0000000 --- a/packages/gateway-conformance/src/fixtures/reverse-proxy.ts +++ /dev/null @@ -1,113 +0,0 @@ -import { request, createServer, type RequestOptions, type IncomingMessage, type ServerResponse } from 'node:http' -import { logger } from '@libp2p/logger' - -const log = logger('reverse-proxy') - -let TARGET_HOST: string -let backendPort: number -let proxyPort: number -let subdomain: undefined | string -let prefixPath: undefined | string -let disableTryFiles: boolean -let X_FORWARDED_HOST: undefined | string - -const makeRequest = (options: RequestOptions, req: IncomingMessage, res: ServerResponse & { req: IncomingMessage }, attemptRootFallback = false): void => { - options.headers = options.headers ?? {} - options.headers.Host = TARGET_HOST - const clientIp = req.socket.remoteAddress - options.headers['X-Forwarded-For'] = req.headers.host ?? clientIp - - // override path to include prefixPath if set - if (prefixPath != null) { - options.path = `${prefixPath}${options.path}` - } - if (subdomain != null) { - options.headers.Host = `${subdomain}.${TARGET_HOST}` - } - if (X_FORWARDED_HOST != null) { - options.headers['X-Forwarded-Host'] = X_FORWARDED_HOST - } - - // log where we're making the request to - log('Proxying request to %s:%s%s', options.headers.Host, options.port, options.path) - - const proxyReq = request(options, (proxyRes) => { - if (!disableTryFiles && proxyRes.statusCode === 404) { // poor mans attempt to implement nginx style try_files - if (!attemptRootFallback) { - // Split the path and pop the last segment - const pathSegments = options.path?.split('/') ?? [] - const lastSegment = pathSegments.pop() ?? '' - - // Attempt to request the last segment at the root - makeRequest({ ...options, path: `/${lastSegment}` }, req, res, true) - } else { - // If already attempted a root fallback, serve index.html - makeRequest({ ...options, path: '/index.html' }, req, res) - } - } else { - // setCommonHeaders(res) - if (proxyRes.statusCode == null) { - log.error('No status code received from proxy') - res.writeHead(500) - res.end('Internal Server Error') - return - } - res.writeHead(proxyRes.statusCode, proxyRes.headers) - proxyRes.pipe(res, { end: true }) - } - }) - - req.pipe(proxyReq, { end: true }) - - proxyReq.on('error', (e) => { - log.error(`Problem with request: ${e.message}`) - res.writeHead(500) - res.end(`Internal Server Error: ${e.message}`) - }) -} - -export interface ReverseProxyOptions { - targetHost?: string - backendPort?: number - proxyPort?: number - subdomain?: string - prefixPath?: string - disableTryFiles?: boolean - xForwardedHost?: string -} -export async function startReverseProxy (options?: ReverseProxyOptions): Promise<() => Promise> { - TARGET_HOST = options?.targetHost ?? process.env.TARGET_HOST ?? 'localhost' - backendPort = options?.backendPort ?? Number(process.env.BACKEND_PORT ?? 3000) - proxyPort = options?.proxyPort ?? Number(process.env.PROXY_PORT ?? 3333) - subdomain = options?.subdomain ?? process.env.SUBDOMAIN - prefixPath = options?.prefixPath ?? process.env.PREFIX_PATH - disableTryFiles = options?.disableTryFiles ?? process.env.DISABLE_TRY_FILES === 'true' - X_FORWARDED_HOST = options?.xForwardedHost ?? process.env.X_FORWARDED_HOST - - const proxyServer = createServer((req, res) => { - if (req.method === 'OPTIONS') { - res.writeHead(200) - res.end() - return - } - log('req.headers: %O', req.headers) - - const options: RequestOptions = { - hostname: TARGET_HOST, - port: backendPort, - path: req.url, - method: req.method, - headers: { ...req.headers } - } - - makeRequest(options, req, res) - }) - - proxyServer.listen(proxyPort, () => { - log(`Proxy server listening on port ${proxyPort}`) - }) - - return async function stopReverseProxy (): Promise { - proxyServer?.close() - } -} diff --git a/packages/interop/package.json b/packages/interop/package.json index 702910c..112bec3 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -58,8 +58,8 @@ }, "dependencies": { "@helia/verified-fetch": "1.4.2", - "aegir": "^42.2.5", - "execa": "^8.0.1", + "aegir": "^42.2.11", + "execa": "^9.1.0", "fast-glob": "^3.3.2", "ipfsd-ctl": "^14.1.0", "kubo": "^0.28.0", diff --git a/packages/verified-fetch/package.json b/packages/verified-fetch/package.json index 4400856..b67e225 100644 --- a/packages/verified-fetch/package.json +++ b/packages/verified-fetch/package.json @@ -57,59 +57,59 @@ "release": "aegir release" }, "dependencies": { - "@helia/block-brokers": "^3.0.0", + "@helia/block-brokers": "^3.0.1", "@helia/car": "^3.1.5", - "@helia/http": "^1.0.7", + "@helia/http": "^1.0.8", "@helia/interface": "^4.3.0", "@helia/ipns": "^7.2.2", "@helia/routers": "^1.1.0", "@ipld/dag-cbor": "^9.2.0", "@ipld/dag-json": "^10.2.0", "@ipld/dag-pb": "^4.1.0", - "@libp2p/interface": "^1.1.6", - "@libp2p/kad-dht": "^12.0.11", - "@libp2p/peer-id": "^4.0.9", + "@libp2p/interface": "^1.4.0", + "@libp2p/kad-dht": "^12.0.17", + "@libp2p/peer-id": "^4.1.2", "@multiformats/dns": "^1.0.6", "cborg": "^4.2.0", "hashlru": "^2.3.0", "interface-blockstore": "^5.2.10", "interface-datastore": "^8.2.11", "ipfs-unixfs-exporter": "^13.5.0", - "it-map": "^3.0.5", + "it-map": "^3.1.0", "it-pipe": "^3.0.1", "it-tar": "^6.0.5", - "it-to-browser-readablestream": "^2.0.6", - "lru-cache": "^10.2.0", + "it-to-browser-readablestream": "^2.0.9", + "lru-cache": "^10.2.2", "multiformats": "^13.1.0", "progress-events": "^1.0.0", - "uint8arrays": "^5.0.3" + "uint8arrays": "^5.1.0" }, "devDependencies": { "@helia/dag-cbor": "^3.0.4", "@helia/dag-json": "^3.0.4", "@helia/json": "^3.0.4", "@helia/unixfs": "^3.0.6", - "@helia/utils": "^0.3.0", + "@helia/utils": "^0.3.1", "@ipld/car": "^5.3.0", - "@libp2p/interface-compliance-tests": "^5.3.4", - "@libp2p/logger": "^4.0.9", - "@libp2p/peer-id-factory": "^4.0.9", + "@libp2p/interface-compliance-tests": "^5.4.5", + "@libp2p/logger": "^4.0.13", + "@libp2p/peer-id-factory": "^4.1.2", "@sgtpooki/file-type": "^1.0.1", "@types/sinon": "^17.0.3", - "aegir": "^42.2.5", + "aegir": "^42.2.11", "blockstore-core": "^4.4.1", - "browser-readablestream-to-it": "^2.0.5", + "browser-readablestream-to-it": "^2.0.7", "datastore-core": "^9.2.9", - "helia": "^4.2.1", + "helia": "^4.2.2", "ipfs-unixfs-importer": "^15.2.5", "ipns": "^9.1.0", - "it-all": "^3.0.4", - "it-drain": "^3.0.5", - "it-last": "^3.0.4", - "it-to-buffer": "^4.0.5", + "it-all": "^3.0.6", + "it-drain": "^3.0.7", + "it-last": "^3.0.6", + "it-to-buffer": "^4.0.7", "magic-bytes.js": "^1.10.0", "p-defer": "^4.0.1", - "sinon": "^17.0.1", + "sinon": "^18.0.0", "sinon-ts": "^2.0.0" }, "sideEffects": false diff --git a/packages/verified-fetch/src/utils/handle-redirects.ts b/packages/verified-fetch/src/utils/handle-redirects.ts new file mode 100644 index 0000000..b123444 --- /dev/null +++ b/packages/verified-fetch/src/utils/handle-redirects.ts @@ -0,0 +1,101 @@ +import { type AbortOptions, type ComponentLogger } from '@libp2p/interface' +import { type VerifiedFetchInit, type Resource } from '../index.js' +import { matchURLString } from './parse-url-string.js' +import { movedPermanentlyResponse } from './responses.js' +import type { CID } from 'multiformats/cid' + +interface GetRedirectResponse { + cid: CID + resource: Resource + options?: Omit & AbortOptions + logger: ComponentLogger + + /** + * Only used in testing. + */ + fetch?: typeof globalThis.fetch +} + +function maybeAddTraillingSlash (path: string): string { + // if it has an extension-like ending, don't add a trailing slash + if (path.match(/\.[a-zA-Z0-9]{1,4}$/) != null) { + return path + } + return path.endsWith('/') ? path : `${path}/` +} + +// See https://specs.ipfs.tech/http-gateways/path-gateway/#location-response-header +export async function getRedirectResponse ({ resource, options, logger, cid, fetch = globalThis.fetch }: GetRedirectResponse): Promise { + const log = logger.forComponent('helia:verified-fetch:get-redirect-response') + + if (typeof resource !== 'string' || options == null || ['ipfs://', 'ipns://'].some((prefix) => resource.startsWith(prefix))) { + return null + } + + const headers = new Headers(options?.headers) + const forwardedHost = headers.get('x-forwarded-host') + const headerHost = headers.get('host') + const forwardedFor = headers.get('x-forwarded-for') + if (forwardedFor == null && forwardedHost == null && headerHost == null) { + log.trace('no redirect info found in headers') + return null + } + + log.trace('checking for redirect info') + // if x-forwarded-host is passed, we need to set the location header to the subdomain + // so that the browser can redirect to the correct subdomain + try { + const urlParts = matchURLString(resource) + const reqUrl = new URL(resource) + const actualHost = forwardedHost ?? reqUrl.host + const subdomainUrl = new URL(reqUrl) + if (urlParts.protocol === 'ipfs' && cid.version === 0) { + subdomainUrl.host = `${cid.toV1()}.ipfs.${actualHost}` + } else { + subdomainUrl.host = `${urlParts.cidOrPeerIdOrDnsLink}.${urlParts.protocol}.${actualHost}` + } + + if (headerHost?.includes(urlParts.protocol) === true && subdomainUrl.host.includes(headerHost)) { + log.trace('request was for a subdomain already, not setting location header') + return null + } + + if (headerHost != null && !subdomainUrl.host.includes(headerHost)) { + log.trace('host header is not the same as the subdomain url host, not setting location header') + return null + } + if (reqUrl.host === subdomainUrl.host) { + log.trace('req url is the same as the subdomain url, not setting location header') + return null + } + + subdomainUrl.pathname = maybeAddTraillingSlash(reqUrl.pathname.replace(`/${urlParts.cidOrPeerIdOrDnsLink}`, '').replace(`/${urlParts.protocol}`, '')) + log.trace('subdomain url %s', subdomainUrl.href) + const pathUrl = new URL(reqUrl, `${reqUrl.protocol}//${actualHost}`) + pathUrl.pathname = maybeAddTraillingSlash(reqUrl.pathname) + log.trace('path url %s', pathUrl.href) + // try to query subdomain with HEAD request to see if it's supported + try { + const subdomainTest = await fetch(subdomainUrl, { method: 'HEAD' }) + if (subdomainTest.ok) { + log('subdomain supported, redirecting to subdomain') + return movedPermanentlyResponse(resource.toString(), subdomainUrl.href) + } else { + log('subdomain not supported, subdomain failed with status %s %s', subdomainTest.status, subdomainTest.statusText) + throw new Error('subdomain not supported') + } + } catch (err: any) { + log('subdomain not supported', err) + if (pathUrl.href === reqUrl.href) { + log('path url is the same as the request url, not setting location header') + return null + } + // pathUrl is different from request URL (maybe even with just a trailing slash) + return movedPermanentlyResponse(resource.toString(), pathUrl.href) + } + } catch (e) { + // if it's not a full URL, we have nothing left to do. + log.error('error setting location header for x-forwarded-host', e) + } + return null +} diff --git a/packages/verified-fetch/src/utils/parse-url-string.ts b/packages/verified-fetch/src/utils/parse-url-string.ts index 63e761d..65e652b 100644 --- a/packages/verified-fetch/src/utils/parse-url-string.ts +++ b/packages/verified-fetch/src/utils/parse-url-string.ts @@ -72,7 +72,7 @@ function matchUrlGroupsGuard (groups?: null | { [key in string]: string; } | Mat } export function matchURLString (urlString: string): MatchUrlGroups { - for (const pattern of [URL_REGEX, PATH_REGEX, PATH_GATEWAY_REGEX, SUBDOMAIN_GATEWAY_REGEX]) { + for (const pattern of [SUBDOMAIN_GATEWAY_REGEX, URL_REGEX, PATH_GATEWAY_REGEX, PATH_REGEX]) { const match = urlString.match(pattern) if (matchUrlGroupsGuard(match?.groups)) { diff --git a/packages/verified-fetch/src/verified-fetch.ts b/packages/verified-fetch/src/verified-fetch.ts index dc61bce..2ca9939 100644 --- a/packages/verified-fetch/src/verified-fetch.ts +++ b/packages/verified-fetch/src/verified-fetch.ts @@ -24,15 +24,16 @@ import { getETag } from './utils/get-e-tag.js' import { getResolvedAcceptHeader } from './utils/get-resolved-accept-header.js' import { getStreamFromAsyncIterable } from './utils/get-stream-from-async-iterable.js' import { tarStream } from './utils/get-tar-stream.js' +import { getRedirectResponse } from './utils/handle-redirects.js' import { parseResource } from './utils/parse-resource.js' +import { type ParsedUrlStringResults } from './utils/parse-url-string.js' import { resourceToSessionCacheKey } from './utils/resource-to-cache-key.js' import { setCacheControlHeader, setIpfsRoots } from './utils/response-headers.js' -import { badRequestResponse, movedPermanentlyResponse, notAcceptableResponse, notSupportedResponse, okResponse, badRangeResponse, okRangeResponse, badGatewayResponse } from './utils/responses.js' +import { badRequestResponse, movedPermanentlyResponse, notAcceptableResponse, notSupportedResponse, okResponse, badRangeResponse, okRangeResponse, badGatewayResponse, notFoundResponse } from './utils/responses.js' import { selectOutputType } from './utils/select-output-type.js' import { handlePathWalking, isObjectNode } from './utils/walk-path.js' import type { CIDDetail, ContentTypeParser, CreateVerifiedFetchOptions, Resource, VerifiedFetchInit as VerifiedFetchOptions } from './index.js' import type { FetchHandlerFunctionArg, RequestFormatShorthand } from './types.js' -import type { ParsedUrlStringResults } from './utils/parse-url-string' import type { Helia, SessionBlockstore } from '@helia/interface' import type { Blockstore } from 'interface-blockstore' import type { ObjectNode } from 'ipfs-unixfs-exporter' @@ -403,6 +404,16 @@ export class VerifiedFetch { } private async handleRaw ({ resource, cid, path, session, options, accept }: FetchHandlerFunctionArg): Promise { + /** + * if we have a path, we can't walk it, so we need to return a 404. + * + * @see https://github.com/ipfs/gateway-conformance/blob/26994cfb056b717a23bf694ce4e94386728748dd/tests/subdomain_gateway_ipfs_test.go#L198-L204 + */ + if (path !== '') { + this.log.trace('404-ing raw codec request for %c/%s', cid, path) + return notFoundResponse(resource, 'Raw codec does not support paths') + } + const byteRangeContext = new ByteRangeContext(this.helia.logger, options?.headers) const blockstore = this.getBlockstore(cid, resource, session, options) const result = await blockstore.get(cid, options) @@ -508,6 +519,11 @@ export class VerifiedFetch { let response: Response let reqFormat: RequestFormatShorthand | undefined + const redirectResponse = await getRedirectResponse({ resource, options, logger: this.helia.logger, cid }) + if (redirectResponse != null) { + return redirectResponse + } + const handlerArgs: FetchHandlerFunctionArg = { resource: resource.toString(), cid, path, accept, session: options?.session ?? true, options } if (accept === 'application/vnd.ipfs.ipns-record') { diff --git a/packages/verified-fetch/test/content-type-parser.spec.ts b/packages/verified-fetch/test/content-type-parser.spec.ts index 54deb03..fb6f787 100644 --- a/packages/verified-fetch/test/content-type-parser.spec.ts +++ b/packages/verified-fetch/test/content-type-parser.spec.ts @@ -84,10 +84,14 @@ describe('content-type-parser', () => { it('is passed a filename from a deep traversal if it is available', async () => { const fs = unixfs(helia) - const deepDirCid = await fs.addFile({ - path: 'foo/bar/a-file.html', - content: uint8ArrayFromString('Hello world') - }) + + let barDir = await fs.addDirectory({ path: './bar' }) + const aFileHtml = await fs.addFile({ path: './bar/a-file.html', content: uint8ArrayFromString('Hello world') }) + barDir = await fs.cp(aFileHtml, barDir, 'a-file.html') + let fooDir = await fs.addDirectory({ path: './foo' }) + fooDir = await fs.cp(barDir, fooDir, 'bar') + let deepDirCid = await fs.addDirectory() + deepDirCid = await fs.cp(fooDir, deepDirCid, 'foo') verifiedFetch = new VerifiedFetch({ helia @@ -95,6 +99,7 @@ describe('content-type-parser', () => { contentTypeParser: async (data, fileName) => fileName }) const resp = await verifiedFetch.fetch(`ipfs://${deepDirCid}/foo/bar/a-file.html`) + expect(resp.status).to.equal(200) expect(resp.headers.get('content-type')).to.equal('a-file.html') }) diff --git a/packages/verified-fetch/test/utils/handle-redirects.spec.ts b/packages/verified-fetch/test/utils/handle-redirects.spec.ts new file mode 100644 index 0000000..173cf4c --- /dev/null +++ b/packages/verified-fetch/test/utils/handle-redirects.spec.ts @@ -0,0 +1,84 @@ +import { prefixLogger } from '@libp2p/logger' +import { expect } from 'aegir/chai' +import { CID } from 'multiformats/cid' +import Sinon from 'sinon' +import { getRedirectResponse } from '../../src/utils/handle-redirects.js' + +const logger = prefixLogger('test:handle-redirects') +describe('handle-redirects', () => { + describe('getRedirectResponse', () => { + const sandbox = Sinon.createSandbox() + const cid = CID.parse('bafkqabtimvwgy3yk') + + let fetchStub: Sinon.SinonStub + + beforeEach(() => { + fetchStub = sandbox.stub(globalThis, 'fetch') + }) + + afterEach(() => { + sandbox.restore() + }) + + const nullResponses = [ + { resource: cid, options: {}, logger, cid, testTitle: 'should return null if resource is not a string' }, + { resource: 'http://ipfs.io/ipfs/bafkqabtimvwgy3yk', options: undefined, logger, cid, testTitle: 'should return null if options is undefined' }, + { resource: 'ipfs://', options: {}, logger, cid, testTitle: 'should return null for ipfs:// protocol urls' }, + { resource: 'ipns://', options: {}, logger, cid, testTitle: 'should return null for ipns:// protocol urls' } + ] + + nullResponses.forEach(({ resource, options, logger, cid, testTitle }) => { + it(testTitle, async () => { + const response = await getRedirectResponse({ resource, options, logger, cid }) + expect(response).to.be.null() + }) + }) + + it('should attempt to get the current host from the headers', async () => { + const resource = 'http://ipfs.io/ipfs/bafkqabtimvwgy3yk' + const options = { headers: new Headers({ 'x-forwarded-host': 'localhost:3931' }) } + fetchStub.returns(Promise.resolve(new Response(null, { status: 200 }))) + + const response = await getRedirectResponse({ resource, options, logger, cid, fetch: fetchStub }) + expect(fetchStub.calledOnce).to.be.true() + expect(response).to.not.be.null() + expect(response).to.have.property('status', 301) + const location = response?.headers.get('location') + expect(location).to.equal('http://bafkqabtimvwgy3yk.ipfs.localhost:3931/') + }) + + it('should return redirect response to requested host with trailing slash when HEAD fetch fails', async () => { + const resource = 'http://ipfs.io/ipfs/bafkqabtimvwgy3yk' + const options = { headers: new Headers({ 'x-forwarded-host': 'localhost:3931' }) } + fetchStub.returns(Promise.reject(new Response(null, { status: 404 }))) + + const response = await getRedirectResponse({ resource, options, logger, cid, fetch: fetchStub }) + expect(fetchStub.calledOnce).to.be.true() + expect(response).to.not.be.null() + expect(response).to.have.property('status', 301) + const location = response?.headers.get('location') + // note that the URL returned in location header has trailing slash. + expect(location).to.equal('http://ipfs.io/ipfs/bafkqabtimvwgy3yk/') + }) + + it('should not return redirect response to x-forwarded-host if HEAD fetch fails', async () => { + const resource = 'http://ipfs.io/ipfs/bafkqabtimvwgy3yk/file.txt' + const options = { headers: new Headers({ 'x-forwarded-host': 'localhost:3931' }) } + fetchStub.returns(Promise.reject(new Response(null, { status: 404 }))) + + const response = await getRedirectResponse({ resource, options, logger, cid, fetch: fetchStub }) + expect(fetchStub.calledOnce).to.be.true() + expect(response).to.be.null() + }) + + it('should not return redirect response to x-forwarded-host when HEAD fetch fails and trailing slash already exists', async () => { + const resource = 'http://ipfs.io/ipfs/bafkqabtimvwgy3yk/' + const options = { headers: new Headers({ 'x-forwarded-host': 'localhost:3931' }) } + fetchStub.returns(Promise.reject(new Response(null, { status: 404 }))) + + const response = await getRedirectResponse({ resource, options, logger, cid, fetch: fetchStub }) + expect(fetchStub.calledOnce).to.be.true() + expect(response).to.be.null() + }) + }) +}) diff --git a/packages/verified-fetch/test/utils/parse-url-string.spec.ts b/packages/verified-fetch/test/utils/parse-url-string.spec.ts index c541862..08e2a49 100644 --- a/packages/verified-fetch/test/utils/parse-url-string.spec.ts +++ b/packages/verified-fetch/test/utils/parse-url-string.spec.ts @@ -931,4 +931,18 @@ describe('parseUrlString', () => { }) }) }) + + describe('subdomainURLs with paths', () => { + it('should correctly parse a subdomain that also has /ipfs in the path', async () => { + // straight from gateway-conformance test: http://bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.localhost:3441/ipfs/bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am + await assertMatchUrl( + 'http://bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.localhost:3441/ipfs/bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am', { + protocol: 'ipfs', + cid: 'bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am', + path: 'ipfs/bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am', + query: {} + } + ) + }) + }) })