-
Notifications
You must be signed in to change notification settings - Fork 106
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: configurable block brokers (#280)
Adds configurable block providers to allow using bitswap but also other methods such as trustless gateways and any yet-to-be-invented way of resolving a CID to a block..
- Loading branch information
1 parent
fdda692
commit 0749cbf
Showing
12 changed files
with
386 additions
and
72 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,58 @@ | ||
| import { createBitswap } from 'ipfs-bitswap' | ||
| import type { BlockAnnouncer, BlockRetriever } from '@helia/interface/blocks' | ||
| import type { Libp2p } from '@libp2p/interface' | ||
| import type { Startable } from '@libp2p/interface/startable' | ||
| import type { Blockstore } from 'interface-blockstore' | ||
| import type { AbortOptions } from 'interface-store' | ||
| import type { Bitswap, BitswapNotifyProgressEvents, BitswapWantBlockProgressEvents } from 'ipfs-bitswap' | ||
| import type { CID } from 'multiformats/cid' | ||
| import type { MultihashHasher } from 'multiformats/hashes/interface' | ||
| import type { ProgressOptions } from 'progress-events' | ||
|
|
||
| export class BitswapBlockBroker implements BlockAnnouncer<ProgressOptions<BitswapNotifyProgressEvents>>, BlockRetriever< | ||
| ProgressOptions<BitswapWantBlockProgressEvents> | ||
| >, Startable { | ||
| private readonly bitswap: Bitswap | ||
| private started: boolean | ||
|
|
||
| constructor (libp2p: Libp2p, blockstore: Blockstore, hashers: MultihashHasher[]) { | ||
| this.bitswap = createBitswap(libp2p, blockstore, { | ||
| hashLoader: { | ||
| getHasher: async (codecOrName: string | number): Promise<MultihashHasher<number>> => { | ||
| const hasher = hashers.find(hasher => { | ||
| return hasher.code === codecOrName || hasher.name === codecOrName | ||
| }) | ||
|
|
||
| if (hasher != null) { | ||
| return hasher | ||
| } | ||
|
|
||
| throw new Error(`Could not load hasher for code/name "${codecOrName}"`) | ||
| } | ||
| } | ||
| }) | ||
| this.started = false | ||
| } | ||
|
|
||
| isStarted (): boolean { | ||
| return this.started | ||
| } | ||
|
|
||
| async start (): Promise<void> { | ||
| await this.bitswap.start() | ||
| this.started = true | ||
| } | ||
|
|
||
| async stop (): Promise<void> { | ||
| await this.bitswap.stop() | ||
| this.started = false | ||
| } | ||
|
|
||
| announce (cid: CID, block: Uint8Array, options?: ProgressOptions<BitswapNotifyProgressEvents>): void { | ||
| this.bitswap.notify(cid, block, options) | ||
| } | ||
|
|
||
| async retrieve (cid: CID, options?: AbortOptions & ProgressOptions<BitswapWantBlockProgressEvents>): Promise<Uint8Array> { | ||
| return this.bitswap.want(cid, options) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,2 @@ | ||
| export { BitswapBlockBroker } from './bitswap-block-broker.js' | ||
| export { TrustedGatewayBlockBroker } from './trustless-gateway-block-broker.js' |
78 changes: 78 additions & 0 deletions
78
packages/helia/src/block-brokers/trustless-gateway-block-broker.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,78 @@ | ||
| import { logger } from '@libp2p/logger' | ||
| import type { BlockRetriever } from '@helia/interface/blocks' | ||
| import type { AbortOptions } from 'interface-store' | ||
| import type { CID } from 'multiformats/cid' | ||
| import type { ProgressEvent, ProgressOptions } from 'progress-events' | ||
|
|
||
| const log = logger('helia:trustless-gateway-block-provider') | ||
|
|
||
| export type TrustlessGatewayGetBlockProgressEvents = | ||
| ProgressEvent<'trustless-gateway:get-block:fetch', URL> | ||
|
|
||
| /** | ||
| * A class that accepts a list of trustless gateways that are queried | ||
| * for blocks. | ||
| */ | ||
| export class TrustedGatewayBlockBroker implements BlockRetriever< | ||
| ProgressOptions<TrustlessGatewayGetBlockProgressEvents> | ||
| > { | ||
| private readonly gateways: URL[] | ||
|
|
||
| constructor (urls: Array<string | URL>) { | ||
| this.gateways = urls.map(url => new URL(url.toString())) | ||
| } | ||
|
|
||
| async retrieve (cid: CID, options: AbortOptions & ProgressOptions<TrustlessGatewayGetBlockProgressEvents> = {}): Promise<Uint8Array> { | ||
| // choose a gateway | ||
| const url = this.gateways[Math.floor(Math.random() * this.gateways.length)] | ||
|
|
||
| log('getting block for %c from %s', cid, url) | ||
|
|
||
| try { | ||
| const block = await getRawBlockFromGateway(url, cid, options.signal) | ||
| log('got block for %c from %s', cid, url) | ||
|
|
||
| return block | ||
| } catch (err: any) { | ||
| log.error('failed to get block for %c from %s', cid, url, err) | ||
|
|
||
| throw err | ||
| } | ||
| } | ||
| } | ||
|
|
||
| async function getRawBlockFromGateway (url: URL, cid: CID, signal?: AbortSignal): Promise<Uint8Array> { | ||
| const gwUrl = new URL(url) | ||
| gwUrl.pathname = `/ipfs/${cid.toString()}` | ||
|
|
||
| // necessary as not every gateway supports dag-cbor, but every should support | ||
| // sending raw block as-is | ||
| gwUrl.search = '?format=raw' | ||
|
|
||
| if (signal?.aborted === true) { | ||
| throw new Error(`Signal to fetch raw block for CID ${cid} from gateway ${gwUrl.toString()} was aborted prior to fetch`) | ||
| } | ||
|
|
||
| try { | ||
| const res = await fetch(gwUrl.toString(), { | ||
| signal, | ||
| headers: { | ||
| // also set header, just in case ?format= is filtered out by some | ||
| // reverse proxy | ||
| Accept: 'application/vnd.ipld.raw' | ||
| }, | ||
| cache: 'force-cache' | ||
| }) | ||
| if (!res.ok) { | ||
| throw new Error(`unable to fetch raw block for CID ${cid} from gateway ${gwUrl.toString()}`) | ||
| } | ||
| return new Uint8Array(await res.arrayBuffer()) | ||
| } catch (cause) { | ||
| // @ts-expect-error - TS thinks signal?.aborted can only be false now | ||
| // because it was checked for true above. | ||
| if (signal?.aborted === true) { | ||
| throw new Error(`fetching raw block for CID ${cid} from gateway ${gwUrl.toString()} was aborted`) | ||
| } | ||
| throw new Error(`unable to fetch raw block for CID ${cid}`) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,12 @@ | ||
| import { identity } from 'multiformats/hashes/identity' | ||
| import { sha256, sha512 } from 'multiformats/hashes/sha2' | ||
| import type { MultihashHasher } from 'multiformats/hashes/interface' | ||
|
|
||
| export function defaultHashers (hashers: MultihashHasher[] = []): MultihashHasher[] { | ||
| return [ | ||
| sha256, | ||
| sha512, | ||
| identity, | ||
| ...hashers | ||
| ] | ||
| } |
Oops, something went wrong.