From 76220cd5adf45af7fa61fd0a1321de4722b744d6 Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Wed, 17 Jan 2024 08:06:52 +0100 Subject: [PATCH] feat!: add @helia/http to monorepo (#372) Adds code from https://github.com/meandavejustice/helia-http to the monorepo. Fixes: https://github.com/ipfs/helia/issues/289 and https://github.com/ipfs/helia/issues/344 BREAKING CHANGE: the `libp2p` property has been removed from the `Helia` interface in `@helia/interface` - it is still present on the return type of `createHelia` from the `helia` module --------- Co-authored-by: David Justice Co-authored-by: Daniel Norman <1992255+2color@users.noreply.github.com> Co-authored-by: Daniel N <2color@users.noreply.github.com> --- package.json | 4 - packages/block-brokers/package.json | 13 +- packages/block-brokers/src/index.ts | 1 - packages/car/package.json | 4 +- packages/core/CHANGELOG.md | 388 ++++++++++++++++++ packages/core/LICENSE | 4 + packages/core/LICENSE-APACHE | 5 + packages/core/LICENSE-MIT | 19 + packages/core/README.md | 66 +++ packages/core/package.json | 91 ++++ packages/core/src/index.ts | 214 ++++++++++ packages/{helia => core}/src/pins.ts | 3 +- packages/core/src/routing.ts | 169 ++++++++ packages/{helia => core}/src/storage.ts | 0 .../{helia => core}/src/utils/dag-walkers.ts | 12 +- .../src/utils/datastore-version.ts | 0 .../src/utils/default-hashers.ts | 0 .../src/utils/networked-storage.ts | 0 packages/core/src/version.ts | 2 + .../test/block-broker.spec.ts | 7 + .../test/fixtures/create-block.ts | 0 .../test/fixtures/create-dag.ts | 0 packages/core/test/fixtures/create-helia.ts | 25 ++ .../test/fixtures/dag-walker.ts | 2 +- packages/{helia => core}/test/gc.spec.ts | 25 +- packages/core/test/index.spec.ts | 52 +++ .../test/pins.depth-limited.spec.ts | 21 +- .../test/pins.recursive.spec.ts | 22 +- packages/core/test/pins.spec.ts | 207 ++++++++++ packages/{helia => core}/test/storage.spec.ts | 0 .../test/utils/networked-storage.spec.ts | 0 packages/core/tsconfig.json | 15 + packages/core/typedoc.json | 5 + packages/dag-cbor/package.json | 4 +- packages/dag-json/package.json | 4 +- packages/helia/package.json | 35 +- packages/helia/src/helia-p2p.ts | 37 ++ packages/helia/src/helia.ts | 107 ----- packages/helia/src/index.ts | 29 +- packages/helia/src/utils/default-hashers.ts | 12 - packages/helia/test/factory.spec.ts | 5 +- packages/helia/test/fixtures/create-helia.ts | 34 -- packages/helia/test/index.spec.ts | 5 +- packages/helia/test/libp2p.spec.ts | 12 +- packages/helia/test/pins.spec.ts | 210 +--------- packages/helia/tsconfig.json | 6 + packages/http/.aegir.js | 8 + packages/http/LICENSE | 4 + packages/http/LICENSE-APACHE | 5 + packages/http/LICENSE-MIT | 19 + packages/http/README.md | 96 +++++ packages/http/package.json | 71 ++++ packages/http/src/index.ts | 89 ++++ packages/http/test/factory.spec.ts | 34 ++ packages/http/test/index.spec.ts | 52 +++ packages/http/tsconfig.json | 24 ++ packages/http/typedoc.json | 5 + packages/interface/package.json | 3 +- packages/interface/src/blocks.ts | 16 +- packages/interface/src/index.ts | 30 +- packages/interface/src/routing.ts | 130 ++++++ packages/interop/package.json | 6 +- packages/interop/test/car.spec.ts | 4 +- packages/interop/test/dag-cbor.spec.ts | 4 +- packages/interop/test/dag-json.spec.ts | 4 +- packages/interop/test/fixtures/connect.ts | 4 +- .../test/fixtures/create-helia.browser.ts | 5 +- .../interop/test/fixtures/create-helia.ts | 5 +- .../interop/test/helia-blockstore.spec.ts | 4 +- packages/interop/test/helia-hashes.spec.ts | 4 +- packages/interop/test/helia-pins.spec.ts | 4 +- packages/interop/test/ipns-libp2p.spec.ts | 8 +- packages/interop/test/ipns-pubsub.spec.ts | 6 +- packages/interop/test/json.spec.ts | 4 +- packages/interop/test/mfs.spec.ts | 4 +- packages/interop/test/strings.spec.ts | 4 +- packages/interop/test/unixfs-bitswap.spec.ts | 5 +- packages/interop/test/unixfs-files.spec.ts | 5 +- packages/ipns/package.json | 4 +- packages/json/package.json | 2 +- packages/mfs/package.json | 6 +- packages/routers/LICENSE | 4 + packages/routers/LICENSE-APACHE | 5 + packages/routers/LICENSE-MIT | 19 + packages/routers/README.md | 53 +++ packages/routers/package.json | 73 ++++ .../routers/src/delegated-http-routing.ts | 101 +++++ packages/routers/src/index.ts | 7 + packages/routers/src/libp2p-routing.ts | 39 ++ .../test/delegated-http-routing.spec.ts | 117 ++++++ packages/routers/test/libp2p-routing.spec.ts | 85 ++++ packages/routers/tsconfig.json | 15 + packages/routers/typedoc.json | 5 + packages/strings/package.json | 2 +- packages/unixfs/package.json | 4 +- 95 files changed, 2535 insertions(+), 548 deletions(-) create mode 100644 packages/core/CHANGELOG.md create mode 100644 packages/core/LICENSE create mode 100644 packages/core/LICENSE-APACHE create mode 100644 packages/core/LICENSE-MIT create mode 100644 packages/core/README.md create mode 100644 packages/core/package.json create mode 100644 packages/core/src/index.ts rename packages/{helia => core}/src/pins.ts (98%) create mode 100644 packages/core/src/routing.ts rename packages/{helia => core}/src/storage.ts (100%) rename packages/{helia => core}/src/utils/dag-walkers.ts (95%) rename packages/{helia => core}/src/utils/datastore-version.ts (100%) rename packages/{block-brokers => core}/src/utils/default-hashers.ts (100%) rename packages/{block-brokers => core}/src/utils/networked-storage.ts (100%) create mode 100644 packages/core/src/version.ts rename packages/{block-brokers => core}/test/block-broker.spec.ts (97%) rename packages/{helia => core}/test/fixtures/create-block.ts (100%) rename packages/{helia => core}/test/fixtures/create-dag.ts (100%) create mode 100644 packages/core/test/fixtures/create-helia.ts rename packages/{helia => core}/test/fixtures/dag-walker.ts (87%) rename packages/{helia => core}/test/gc.spec.ts (90%) create mode 100644 packages/core/test/index.spec.ts rename packages/{helia => core}/test/pins.depth-limited.spec.ts (85%) rename packages/{helia => core}/test/pins.recursive.spec.ts (88%) create mode 100644 packages/core/test/pins.spec.ts rename packages/{helia => core}/test/storage.spec.ts (100%) rename packages/{block-brokers => core}/test/utils/networked-storage.spec.ts (100%) create mode 100644 packages/core/tsconfig.json create mode 100644 packages/core/typedoc.json create mode 100644 packages/helia/src/helia-p2p.ts delete mode 100644 packages/helia/src/helia.ts delete mode 100644 packages/helia/src/utils/default-hashers.ts delete mode 100644 packages/helia/test/fixtures/create-helia.ts create mode 100644 packages/http/.aegir.js create mode 100644 packages/http/LICENSE create mode 100644 packages/http/LICENSE-APACHE create mode 100644 packages/http/LICENSE-MIT create mode 100644 packages/http/README.md create mode 100644 packages/http/package.json create mode 100644 packages/http/src/index.ts create mode 100644 packages/http/test/factory.spec.ts create mode 100644 packages/http/test/index.spec.ts create mode 100644 packages/http/tsconfig.json create mode 100644 packages/http/typedoc.json create mode 100644 packages/interface/src/routing.ts create mode 100644 packages/routers/LICENSE create mode 100644 packages/routers/LICENSE-APACHE create mode 100644 packages/routers/LICENSE-MIT create mode 100644 packages/routers/README.md create mode 100644 packages/routers/package.json create mode 100644 packages/routers/src/delegated-http-routing.ts create mode 100644 packages/routers/src/index.ts create mode 100644 packages/routers/src/libp2p-routing.ts create mode 100644 packages/routers/test/delegated-http-routing.spec.ts create mode 100644 packages/routers/test/libp2p-routing.spec.ts create mode 100644 packages/routers/tsconfig.json create mode 100644 packages/routers/typedoc.json diff --git a/package.json b/package.json index efbd6095..9f023df3 100644 --- a/package.json +++ b/package.json @@ -11,10 +11,6 @@ "bugs": { "url": "https://github.com/ipfs/helia/issues" }, - "publishConfig": { - "access": "public", - "provenance": true - }, "keywords": [ "ipfs" ], diff --git a/packages/block-brokers/package.json b/packages/block-brokers/package.json index e21a2894..6a79a054 100644 --- a/packages/block-brokers/package.json +++ b/packages/block-brokers/package.json @@ -55,24 +55,15 @@ "dependencies": { "@helia/interface": "^3.0.1", "@libp2p/interface": "^1.1.1", - "any-signal": "^4.1.1", "interface-blockstore": "^5.2.7", - "interface-store": "^5.1.5", "ipfs-bitswap": "^20.0.0", - "it-filter": "^3.0.4", - "it-foreach": "^2.0.6", "multiformats": "^13.0.0", - "progress-events": "^1.0.0", - "uint8arrays": "^5.0.1" + "progress-events": "^1.0.0" }, "devDependencies": { - "@libp2p/logger": "^4.0.1", + "@libp2p/logger": "^4.0.4", "@types/sinon": "^17.0.2", "aegir": "^42.1.0", - "blockstore-core": "^4.3.8", - "delay": "^6.0.0", - "it-all": "^3.0.4", - "it-drain": "^3.0.5", "sinon": "^17.0.1", "sinon-ts": "^2.0.0" } diff --git a/packages/block-brokers/src/index.ts b/packages/block-brokers/src/index.ts index 004e98f0..e7f503c9 100644 --- a/packages/block-brokers/src/index.ts +++ b/packages/block-brokers/src/index.ts @@ -1,3 +1,2 @@ export { bitswap } from './bitswap.js' export { trustlessGateway } from './trustless-gateway/index.js' -export { NetworkedStorage } from './utils/networked-storage.js' diff --git a/packages/car/package.json b/packages/car/package.json index dbc95111..24737316 100644 --- a/packages/car/package.json +++ b/packages/car/package.json @@ -141,7 +141,7 @@ "dependencies": { "@helia/interface": "^3.0.1", "@ipld/car": "^5.1.1", - "@ipld/dag-pb": "^4.0.3", + "@ipld/dag-pb": "^4.0.6", "@libp2p/interfaces": "^3.3.1", "cborg": "^4.0.3", "it-drain": "^3.0.5", @@ -154,7 +154,7 @@ "devDependencies": { "@helia/unixfs": "^2.0.1", "aegir": "^42.1.0", - "blockstore-core": "^4.3.8", + "blockstore-core": "^4.3.10", "interface-blockstore": "^5.2.9", "ipfs-unixfs-importer": "^15.2.3", "it-to-buffer": "^4.0.2" diff --git a/packages/core/CHANGELOG.md b/packages/core/CHANGELOG.md new file mode 100644 index 00000000..998659a3 --- /dev/null +++ b/packages/core/CHANGELOG.md @@ -0,0 +1,388 @@ +## [helia-v1.3.12](https://github.com/ipfs/helia/compare/helia-v1.3.11...helia-v1.3.12) (2023-08-05) + + +### Dependencies + +* **dev:** bump aegir from 39.0.13 to 40.0.8 ([#198](https://github.com/ipfs/helia/issues/198)) ([4d75ecf](https://github.com/ipfs/helia/commit/4d75ecffb79e5177da35d3106e42dac7bc63153a)) +* update sibling dependencies ([beb10b5](https://github.com/ipfs/helia/commit/beb10b5590d66d1d5bef9b5e890b888263df2c92)) + +## [3.0.0](https://github.com/ipfs/helia/compare/helia-v2.1.0...helia-v3.0.0) (2024-01-07) + + +### ⚠ BREAKING CHANGES + +* `helia.pin.add` and `helia.pin.rm` now return `AsyncGenerator` +* The libp2p API has changed in a couple of places - please see the [upgrade guide](https://github.com/libp2p/js-libp2p/blob/main/doc/migrations/v0.46-v1.0.0.md) + +### deps + +* updates to libp2p v1 ([#320](https://github.com/ipfs/helia/issues/320)) ([635d7a2](https://github.com/ipfs/helia/commit/635d7a2938111ccc53f8defbd9b8f8f8ea3e8e6a)) + + +### Features + +* iterable pinning ([#231](https://github.com/ipfs/helia/issues/231)) ([c15c774](https://github.com/ipfs/helia/commit/c15c7749294d3d4aea5aef70544d088250336798)) + + +### Dependencies + +* The following workspace dependencies were updated + * dependencies + * @helia/interface bumped from ^2.1.0 to ^3.0.0 + +## [2.1.0](https://www.github.com/ipfs/helia/compare/helia-v2.0.3...helia-v2.1.0) (2023-11-06) + + +### Features + +* configurable block brokers ([#280](https://www.github.com/ipfs/helia/issues/280)) ([0749cbf](https://www.github.com/ipfs/helia/commit/0749cbf99745ea6ab4363f1b5d635634ca0ddcfa)) +* GatewayBlockBroker prioritizes & tries all gateways ([#281](https://www.github.com/ipfs/helia/issues/281)) ([9bad21b](https://www.github.com/ipfs/helia/commit/9bad21bd59fe6d1ba4a137db5a46bd2ead5238c3)) +* use trustless-gateway.link by default ([#299](https://www.github.com/ipfs/helia/issues/299)) ([bf11efa](https://www.github.com/ipfs/helia/commit/bf11efa4875f3b8f844511d70122983fc46b4f88)) + + +### Bug Fixes + +* listen on ip6 addresses ([#271](https://www.github.com/ipfs/helia/issues/271)) ([7ef5e79](https://www.github.com/ipfs/helia/commit/7ef5e79620f043522ff0dacc260af1fe83e5d77e)) +* remove trustless-gateway.link ([#301](https://www.github.com/ipfs/helia/issues/301)) ([0343725](https://www.github.com/ipfs/helia/commit/03437255213b14f5931aed91e8555d7fb7f92926)) +* replace IPNI gateway with delegated routing client ([#297](https://www.github.com/ipfs/helia/issues/297)) ([57d580d](https://www.github.com/ipfs/helia/commit/57d580da26c5e28852cc9fe4d0d80adb36699ece)) + + +### Dependencies + +* The following workspace dependencies were updated + * dependencies + * @helia/interface bumped from ^2.0.0 to ^2.1.0 + +### [2.0.3](https://www.github.com/ipfs/helia/compare/helia-v2.0.2...helia-v2.0.3) (2023-09-18) + + +### Bug Fixes + +* export libp2p service return type ([#263](https://www.github.com/ipfs/helia/issues/263)) ([76769cf](https://www.github.com/ipfs/helia/commit/76769cf33e06746f998b4f16b52d3e2a6a7a20a8)) +* try circuit relay transport first ([#267](https://www.github.com/ipfs/helia/issues/267)) ([d5e9c3c](https://www.github.com/ipfs/helia/commit/d5e9c3c45c8dc3e63969105b785f6a836820a1f8)) +* update attempt to add helia to identify agent version ([#268](https://www.github.com/ipfs/helia/issues/268)) ([6dc7d55](https://www.github.com/ipfs/helia/commit/6dc7d55cd3099785417a7a2c99db755e856bd59a)) + +### [2.0.2](https://www.github.com/ipfs/helia/compare/helia-v2.0.1...helia-v2.0.2) (2023-09-14) + + +### Bug Fixes + +* add dag walker for json codec ([#247](https://www.github.com/ipfs/helia/issues/247)) ([5c4b570](https://www.github.com/ipfs/helia/commit/5c4b5709e6b98de5efc9bed388942e367f5874e7)), closes [#246](https://www.github.com/ipfs/helia/issues/246) + +### [2.0.1](https://www.github.com/ipfs/helia/compare/helia-v2.0.0...helia-v2.0.1) (2023-08-16) + + +### Bug Fixes + +* enable dcutr by default ([#239](https://www.github.com/ipfs/helia/issues/239)) ([7431f09](https://www.github.com/ipfs/helia/commit/7431f09aef332dc142a5f7c2c59c9410e4529a92)) + +## [2.0.0](https://www.github.com/ipfs/helia/compare/helia-v1.3.12...helia-v2.0.0) (2023-08-16) + + +### ⚠ BREAKING CHANGES + +* libp2p has been updated to 0.46.x + +### Features + +* re-export types from @helia/interface ([#232](https://www.github.com/ipfs/helia/issues/232)) ([09c1e47](https://www.github.com/ipfs/helia/commit/09c1e4787a506d34a00d9ce7852d73471d47db1b)) + + +### Dependencies + +* bump @libp2p/ipni-content-routing from 1.0.2 to 2.0.0 ([#227](https://www.github.com/ipfs/helia/issues/227)) ([a33cb3e](https://www.github.com/ipfs/helia/commit/a33cb3ef2dd21a55b598f206e8d4295935ea2bcc)) +* update libp2p to 0.46.x ([#215](https://www.github.com/ipfs/helia/issues/215)) ([65b68f0](https://www.github.com/ipfs/helia/commit/65b68f071d04d2f6f0fcf35938b146706b1a3cd0)) +* update sibling dependencies ([07847bb](https://www.github.com/ipfs/helia/commit/07847bb60b9ebd26497080373e45871abb4b82dd)) + + +### Dependencies + +* The following workspace dependencies were updated + * dependencies + * @helia/interface bumped from ^1.0.0 to ^2.0.0 + +## [helia-v1.3.11](https://github.com/ipfs/helia/compare/helia-v1.3.10...helia-v1.3.11) (2023-08-04) + + +### Dependencies + +* update sibling dependencies ([aa249bc](https://github.com/ipfs/helia/commit/aa249bca021ca513c7847331970219e4a36dee97)) + +## [helia-v1.3.10](https://github.com/ipfs/helia/compare/helia-v1.3.9...helia-v1.3.10) (2023-08-04) + + +### Dependencies + +* update sibling dependencies ([89df3fe](https://github.com/ipfs/helia/commit/89df3fe803daa3228290bef105ce5d0b769dc3a0)) + +## [helia-v1.3.9](https://github.com/ipfs/helia/compare/helia-v1.3.8...helia-v1.3.9) (2023-08-01) + + +### Dependencies + +* update sibling dependencies ([0970da7](https://github.com/ipfs/helia/commit/0970da79e974a4c172e8fdfb7c207d5ba8152a83)) + +## [helia-v1.3.8](https://github.com/ipfs/helia/compare/helia-v1.3.7...helia-v1.3.8) (2023-07-14) + + +### Dependencies + +* update sibling dependencies ([5850e51](https://github.com/ipfs/helia/commit/5850e513c486f6d20e23c04936bbf843653cb5e4)) + +## [helia-v1.3.7](https://github.com/ipfs/helia/compare/helia-v1.3.6...helia-v1.3.7) (2023-07-11) + + +### Dependencies + +* update sibling dependencies ([2c52da3](https://github.com/ipfs/helia/commit/2c52da3957d56fe4e3ff6f161f9bec814abd5d8c)) + +## [helia-v1.3.6](https://github.com/ipfs/helia/compare/helia-v1.3.5...helia-v1.3.6) (2023-07-10) + + +### Dependencies + +* update sibling dependencies ([9139f30](https://github.com/ipfs/helia/commit/9139f30e857f4e247202e0d113027190a04892ba)) + +## [helia-v1.3.5](https://github.com/ipfs/helia/compare/helia-v1.3.4...helia-v1.3.5) (2023-07-04) + + +### Dependencies + +* update sibling dependencies ([99a5115](https://github.com/ipfs/helia/commit/99a5115713d2f17f17820f661dd22a87262c654b)) + + +### Trivial Changes + +* update project config ([#175](https://github.com/ipfs/helia/issues/175)) ([f185a72](https://github.com/ipfs/helia/commit/f185a7220a62f7fc0c025aa5c0be5a981c63cc48)) + +## [helia-v1.3.4](https://github.com/ipfs/helia/compare/helia-v1.3.3...helia-v1.3.4) (2023-06-26) + + +### Dependencies + +* update sibling dependencies ([64e300c](https://github.com/ipfs/helia/commit/64e300c289f4bfe4b72607d86ab9e83a1ac3c8d3)) + +## [helia-v1.3.3](https://github.com/ipfs/helia/compare/helia-v1.3.2...helia-v1.3.3) (2023-06-21) + + +### Dependencies + +* update sibling dependencies ([f7cb076](https://github.com/ipfs/helia/commit/f7cb076e9356535164812229eff22c5c0e052674)) + +## [helia-v1.3.2](https://github.com/ipfs/helia/compare/helia-v1.3.1...helia-v1.3.2) (2023-06-10) + + +### Dependencies + +* update sibling dependencies ([634ca4f](https://github.com/ipfs/helia/commit/634ca4faf5caf448bd068a78101ac0070145518e)) + +## [helia-v1.3.1](https://github.com/ipfs/helia/compare/helia-v1.3.0...helia-v1.3.1) (2023-06-07) + + +### Bug Fixes + +* pass options to blockstore.get during pin.add ([#148](https://github.com/ipfs/helia/issues/148)) ([3a5234e](https://github.com/ipfs/helia/commit/3a5234e3c2f88f9910678b0cbbac5fd340117cc9)) +* remove extra interface ([d577c61](https://github.com/ipfs/helia/commit/d577c61bcc6e4805d214b3ec4a39d78ee752a21e)) + + +### Dependencies + +* update sibling dependencies ([3323a5c](https://github.com/ipfs/helia/commit/3323a5cd518c63cb67e8eaef0cb64c542982b603)) + +## [helia-v1.3.0](https://github.com/ipfs/helia/compare/helia-v1.2.2...helia-v1.3.0) (2023-06-07) + + +### Features + +* add offline option to blockstore get ([#145](https://github.com/ipfs/helia/issues/145)) ([71c5f6b](https://github.com/ipfs/helia/commit/71c5f6bc32b324ee237e56c2c5a1ce903b3bdbef)) + + +### Dependencies + +* update sibling dependencies ([671ec87](https://github.com/ipfs/helia/commit/671ec874e90fbdcaf79d9d8253822fd85cee8bc5)) + +## [helia-v1.2.2](https://github.com/ipfs/helia/compare/helia-v1.2.1...helia-v1.2.2) (2023-06-07) + + +### Dependencies + +* update sibling dependencies ([a349576](https://github.com/ipfs/helia/commit/a34957650715efc45382dc005feea6162398b8f9)) + + +### Trivial Changes + +* update changelogs ([#142](https://github.com/ipfs/helia/issues/142)) ([fefd374](https://github.com/ipfs/helia/commit/fefd3744c0a6d8471de31762ece6ec59b65496c1)) + +## [helia-v1.2.1](https://github.com/ipfs/helia/compare/helia-v1.2.0...helia-v1.2.1) (2023-06-01) + + +### Bug Fixes + +* ensure pinned blocks are present ([#141](https://github.com/ipfs/helia/issues/141)) ([271c403](https://github.com/ipfs/helia/commit/271c403009d378a35375a9468e41388ebb978f54)) + + +### Dependencies + +* update sibling dependencies ([fcee11e](https://github.com/ipfs/helia/commit/fcee11eadb7edfa327e3f0bd586e20ea5dc06c8a)) + +## [helia-v1.2.0](https://github.com/ipfs/helia/compare/helia-v1.1.5...helia-v1.2.0) (2023-06-01) + + +### Features + +* allow passing partial libp2p config to helia factory ([#140](https://github.com/ipfs/helia/issues/140)) ([33a75d5](https://github.com/ipfs/helia/commit/33a75d5f80e2f211440c087806f463525de910d9)) + + +### Dependencies + +* update sibling dependencies ([c936ba6](https://github.com/ipfs/helia/commit/c936ba63a75276e206d804cf0ef35c3f9bf67f10)) + +## [helia-v1.1.5](https://github.com/ipfs/helia/compare/helia-v1.1.4...helia-v1.1.5) (2023-05-25) + + +### Bug Fixes + +* add dht validators/selectors for ipns ([#135](https://github.com/ipfs/helia/issues/135)) ([2c8e6b5](https://github.com/ipfs/helia/commit/2c8e6b51b3c401a0472a024b8dac3d3ba735d74c)) + + +### Dependencies + +* update sibling dependencies ([f565ffd](https://github.com/ipfs/helia/commit/f565ffdcf6923b78326ed4cb00be93083b45ccca)) + +## [helia-v1.1.4](https://github.com/ipfs/helia/compare/helia-v1.1.3...helia-v1.1.4) (2023-05-24) + + +### Dependencies + +* update sibling dependencies ([1ac389c](https://github.com/ipfs/helia/commit/1ac389c6fd8f276daf33c8a61849f3657cf88a10)) + +## [helia-v1.1.3](https://github.com/ipfs/helia/compare/helia-v1.1.2...helia-v1.1.3) (2023-05-24) + + +### Dependencies + +* **dev:** bump delay from 5.0.0 to 6.0.0 ([#130](https://github.com/ipfs/helia/issues/130)) ([d087ffc](https://github.com/ipfs/helia/commit/d087ffcb8074b41781346d09101b2b7bc64768d2)) +* update sibling dependencies ([ed49856](https://github.com/ipfs/helia/commit/ed4985677b62021f76541354ad06b70bd53e929a)) + +## [helia-v1.1.2](https://github.com/ipfs/helia/compare/helia-v1.1.1...helia-v1.1.2) (2023-05-19) + + +### Bug Fixes + +* dedupe bootstrap list ([#129](https://github.com/ipfs/helia/issues/129)) ([bb5d1e9](https://github.com/ipfs/helia/commit/bb5d1e91daae9f6c399e0fdf974318a4a7353fb9)), closes [/github.com/ipfs/helia/pull/127#discussion_r1199152374](https://github.com/ipfs//github.com/ipfs/helia/pull/127/issues/discussion_r1199152374) + + +### Dependencies + +* update sibling dependencies ([d33c843](https://github.com/ipfs/helia/commit/d33c84378c02f34277178e6553090b92b0eabe0b)) + +## [helia-v1.1.1](https://github.com/ipfs/helia/compare/helia-v1.1.0...helia-v1.1.1) (2023-05-19) + + +### Bug Fixes + +* add helia version to agent version ([#128](https://github.com/ipfs/helia/issues/128)) ([48e19ec](https://github.com/ipfs/helia/commit/48e19ec545cc67157e14ae59054fa377a583cb01)), closes [#122](https://github.com/ipfs/helia/issues/122) + +## [helia-v1.1.0](https://github.com/ipfs/helia/compare/helia-v1.0.4...helia-v1.1.0) (2023-05-19) + + +### Features + +* provide default libp2p instance ([#127](https://github.com/ipfs/helia/issues/127)) ([45c9d89](https://github.com/ipfs/helia/commit/45c9d896afa27f5ea043cc5f576d50fc4fa556e9)), closes [#121](https://github.com/ipfs/helia/issues/121) + +## [helia-v1.0.4](https://github.com/ipfs/helia/compare/helia-v1.0.3...helia-v1.0.4) (2023-05-04) + + +### Bug Fixes + +* **types:** Add missing types ([#95](https://github.com/ipfs/helia/issues/95)) ([e858b8d](https://github.com/ipfs/helia/commit/e858b8dbbff548b42dde225db674f0edd1990ed3)) + + +### Dependencies + +* **dev:** bump libp2p from 0.43.4 to 0.44.0 ([#96](https://github.com/ipfs/helia/issues/96)) ([6e37d9f](https://github.com/ipfs/helia/commit/6e37d9f8be58955c5ddc5472fe3adb4bd9a0459c)) + + +### Trivial Changes + +* bump aegir from 38.1.8 to 39.0.4 ([#111](https://github.com/ipfs/helia/issues/111)) ([2156568](https://github.com/ipfs/helia/commit/215656870cb821dd6be2f8054dc39932ba25af14)) + +## [helia-v1.0.3](https://github.com/ipfs/helia/compare/helia-v1.0.2...helia-v1.0.3) (2023-04-05) + + +### Dependencies + +* bump it-filter from 2.0.2 to 3.0.1 ([#74](https://github.com/ipfs/helia/issues/74)) ([3402724](https://github.com/ipfs/helia/commit/340272484df47d2f70f870d375ebb4235fb165a0)) + +## [helia-v1.0.2](https://github.com/ipfs/helia/compare/helia-v1.0.1...helia-v1.0.2) (2023-04-05) + + +### Dependencies + +* bump it-drain from 2.0.1 to 3.0.1 ([#71](https://github.com/ipfs/helia/issues/71)) ([c6eaca1](https://github.com/ipfs/helia/commit/c6eaca1d21cf16527851fffc2411a8e3bd651f34)) + +## [helia-v1.0.1](https://github.com/ipfs/helia/compare/helia-v1.0.0...helia-v1.0.1) (2023-04-05) + + +### Dependencies + +* bump it-all from 2.0.1 to 3.0.1 ([#72](https://github.com/ipfs/helia/issues/72)) ([e7ce5bc](https://github.com/ipfs/helia/commit/e7ce5bc0e0db0a6b41920a3c36b95eeea1863183)) +* bump it-foreach from 1.0.1 to 2.0.2 ([#75](https://github.com/ipfs/helia/issues/75)) ([6f5f059](https://github.com/ipfs/helia/commit/6f5f0592edd44257092d0b70dd364096864495bf)) + +## helia-v1.0.0 (2023-03-23) + + +### Features + +* add bitswap progress events ([#50](https://github.com/ipfs/helia/issues/50)) ([7460719](https://github.com/ipfs/helia/commit/7460719be44b4ff9bad629654efa29c56242e03a)), closes [#27](https://github.com/ipfs/helia/issues/27) +* add pinning API ([#36](https://github.com/ipfs/helia/issues/36)) ([270bb98](https://github.com/ipfs/helia/commit/270bb988eb8aefc8afe68e3580c3ef18960b3188)), closes [#28](https://github.com/ipfs/helia/issues/28) [/github.com/ipfs/helia/pull/36#issuecomment-1441403221](https://github.com/ipfs//github.com/ipfs/helia/pull/36/issues/issuecomment-1441403221) [#28](https://github.com/ipfs/helia/issues/28) +* initial implementation ([#17](https://github.com/ipfs/helia/issues/17)) ([343d360](https://github.com/ipfs/helia/commit/343d36016b164ed45cec4eb670d7f74860166ce4)) + + +### Bug Fixes + +* make all helia args optional ([#37](https://github.com/ipfs/helia/issues/37)) ([d15d76c](https://github.com/ipfs/helia/commit/d15d76cdc40a31bd1e47ca09583cc685583243b9)) +* survive a cid causing an error during gc ([#38](https://github.com/ipfs/helia/issues/38)) ([5330188](https://github.com/ipfs/helia/commit/53301881dc6226ea3fc6823fd6e298e4d4796408)) +* update blocks interface to align with interface-blockstore ([#54](https://github.com/ipfs/helia/issues/54)) ([202b966](https://github.com/ipfs/helia/commit/202b966df3866d449751f775ed3edc9c92e32f6a)) +* use release version of libp2p ([#59](https://github.com/ipfs/helia/issues/59)) ([a3a7c9c](https://github.com/ipfs/helia/commit/a3a7c9c2d81f2068fee85eeeca7425919f09e182)) + + +### Trivial Changes + +* add release config ([a1c7ed0](https://github.com/ipfs/helia/commit/a1c7ed0560aaab032b641a78c9a76fc05a691a10)) +* fix ci badge ([50929c0](https://github.com/ipfs/helia/commit/50929c01a38317f2f580609cc1b9c4c5485f32c8)) +* release main ([#62](https://github.com/ipfs/helia/issues/62)) ([2bce77c](https://github.com/ipfs/helia/commit/2bce77c7d68735efca6ba602c215f432ba9b722d)) +* update logo ([654a70c](https://github.com/ipfs/helia/commit/654a70cff9c222e4029ddd183d609514afc852ed)) +* update publish config ([913ab6a](https://github.com/ipfs/helia/commit/913ab6ae9a2970c4b908de04b8b6a236b931a3b0)) +* update release please config ([b52d5e3](https://github.com/ipfs/helia/commit/b52d5e3eecce41b10640426c339c99ad14ce874e)) +* use wildcards for interop test deps ([29b4fb0](https://github.com/ipfs/helia/commit/29b4fb0ef58f53e6f7e1cf6fcb78fbb699f7b2a7)) + + +### Dependencies + +* update interface-store to 5.x.x ([#63](https://github.com/ipfs/helia/issues/63)) ([5bf11d6](https://github.com/ipfs/helia/commit/5bf11d638eee423624ac49af97757d730744f384)) +* update sibling dependencies ([ac28d38](https://github.com/ipfs/helia/commit/ac28d3878f98a780fc57702921924fa92bd592a0)) + +## helia-v0.1.0 (2023-03-22) + + +### Features + +* add bitswap progress events ([#50](https://www.github.com/ipfs/helia/issues/50)) ([7460719](https://www.github.com/ipfs/helia/commit/7460719be44b4ff9bad629654efa29c56242e03a)), closes [#27](https://www.github.com/ipfs/helia/issues/27) +* add pinning API ([#36](https://www.github.com/ipfs/helia/issues/36)) ([270bb98](https://www.github.com/ipfs/helia/commit/270bb988eb8aefc8afe68e3580c3ef18960b3188)), closes [#28](https://www.github.com/ipfs/helia/issues/28) +* initial implementation ([#17](https://www.github.com/ipfs/helia/issues/17)) ([343d360](https://www.github.com/ipfs/helia/commit/343d36016b164ed45cec4eb670d7f74860166ce4)) + + +### Bug Fixes + +* make all helia args optional ([#37](https://www.github.com/ipfs/helia/issues/37)) ([d15d76c](https://www.github.com/ipfs/helia/commit/d15d76cdc40a31bd1e47ca09583cc685583243b9)) +* survive a cid causing an error during gc ([#38](https://www.github.com/ipfs/helia/issues/38)) ([5330188](https://www.github.com/ipfs/helia/commit/53301881dc6226ea3fc6823fd6e298e4d4796408)) +* update blocks interface to align with interface-blockstore ([#54](https://www.github.com/ipfs/helia/issues/54)) ([202b966](https://www.github.com/ipfs/helia/commit/202b966df3866d449751f775ed3edc9c92e32f6a)) +* use release version of libp2p ([#59](https://www.github.com/ipfs/helia/issues/59)) ([a3a7c9c](https://www.github.com/ipfs/helia/commit/a3a7c9c2d81f2068fee85eeeca7425919f09e182)) + + +### Dependencies + +* The following workspace dependencies were updated + * dependencies + * @helia/interface bumped from ~0.0.0 to ^0.1.0 diff --git a/packages/core/LICENSE b/packages/core/LICENSE new file mode 100644 index 00000000..20ce483c --- /dev/null +++ b/packages/core/LICENSE @@ -0,0 +1,4 @@ +This project is dual licensed under MIT and Apache-2.0. + +MIT: https://www.opensource.org/licenses/mit +Apache-2.0: https://www.apache.org/licenses/license-2.0 diff --git a/packages/core/LICENSE-APACHE b/packages/core/LICENSE-APACHE new file mode 100644 index 00000000..14478a3b --- /dev/null +++ b/packages/core/LICENSE-APACHE @@ -0,0 +1,5 @@ +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. diff --git a/packages/core/LICENSE-MIT b/packages/core/LICENSE-MIT new file mode 100644 index 00000000..72dc60d8 --- /dev/null +++ b/packages/core/LICENSE-MIT @@ -0,0 +1,19 @@ +The MIT License (MIT) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/packages/core/README.md b/packages/core/README.md new file mode 100644 index 00000000..50941102 --- /dev/null +++ b/packages/core/README.md @@ -0,0 +1,66 @@ +

+ + Helia logo + +

+ +[![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) +[![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) +[![codecov](https://img.shields.io/codecov/c/github/ipfs/helia.svg?style=flat-square)](https://codecov.io/gh/ipfs/helia) +[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/helia/main.yml?branch=main\&style=flat-square)](https://github.com/ipfs/helia/actions/workflows/main.yml?query=branch%3Amain) + +> Shared code that implements the Helia API + +# About + +Exports a `Helia` class that implements the HeliaInterface API. + +In general you should use the `helia` or `@helia/http` modules instead which +pre-configure Helia for certain use-cases (p2p or pure-HTTP). + +## Example + +```typescript +import { Helia } from '@helia/core' + +const node = new Helia({ + // ...options +}) +``` + +# Install + +```console +$ npm i @helia/core +``` + +## Browser ` +``` + +# API Docs + +- + +# License + +Licensed under either of + +- Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) +- MIT ([LICENSE-MIT](LICENSE-MIT) / ) + +# Contribute + +Contributions welcome! Please check out [the issues](https://github.com/ipfs/helia/issues). + +Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. + +Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). + +Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. + +[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) diff --git a/packages/core/package.json b/packages/core/package.json new file mode 100644 index 00000000..87726770 --- /dev/null +++ b/packages/core/package.json @@ -0,0 +1,91 @@ +{ + "name": "@helia/core", + "version": "0.0.0", + "description": "Shared code that implements the Helia API", + "license": "Apache-2.0 OR MIT", + "homepage": "https://github.com/ipfs/helia/tree/main/packages/core#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/ipfs/helia.git" + }, + "bugs": { + "url": "https://github.com/ipfs/helia/issues" + }, + "publishConfig": { + "access": "public", + "provenance": true + }, + "keywords": [ + "IPFS" + ], + "type": "module", + "types": "./dist/src/index.d.ts", + "files": [ + "src", + "dist", + "!dist/test", + "!**/*.tsbuildinfo" + ], + "exports": { + ".": { + "types": "./dist/src/index.d.ts", + "import": "./dist/src/index.js" + } + }, + "eslintConfig": { + "extends": "ipfs", + "parserOptions": { + "project": true, + "sourceType": "module" + } + }, + "scripts": { + "clean": "aegir clean", + "lint": "aegir lint", + "dep-check": "aegir dep-check", + "build": "aegir build", + "test": "aegir test", + "test:chrome": "aegir test -t browser --cov", + "test:chrome-webworker": "aegir test -t webworker", + "test:firefox": "aegir test -t browser -- --browser firefox", + "test:firefox-webworker": "aegir test -t webworker -- --browser firefox", + "test:node": "aegir test -t node --cov", + "test:electron-main": "aegir test -t electron-main" + }, + "dependencies": { + "@helia/interface": "^3.0.1", + "@ipld/dag-cbor": "^9.0.6", + "@ipld/dag-json": "^10.1.5", + "@ipld/dag-pb": "^4.0.6", + "@libp2p/interface": "^1.1.1", + "@libp2p/logger": "^4.0.4", + "@libp2p/peer-collections": "^5.1.4", + "@libp2p/utils": "^5.2.0", + "any-signal": "^4.1.1", + "cborg": "^4.0.3", + "interface-blockstore": "^5.2.7", + "interface-datastore": "^8.2.9", + "interface-store": "^5.1.5", + "it-drain": "^3.0.5", + "it-filter": "^3.0.4", + "it-foreach": "^2.0.6", + "it-merge": "^3.0.3", + "mortice": "^3.0.1", + "multiformats": "^13.0.0", + "progress-events": "^1.0.0", + "uint8arrays": "^5.0.1" + }, + "devDependencies": { + "@types/sinon": "^17.0.2", + "aegir": "^42.1.0", + "blockstore-core": "^4.3.10", + "datastore-core": "^9.2.7", + "delay": "^6.0.0", + "it-all": "^3.0.4", + "sinon": "^17.0.1", + "sinon-ts": "^2.0.0" + }, + "browser": { + "./dist/src/utils/libp2p-defaults.js": "./dist/src/utils/libp2p-defaults.browser.js" + } +} diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts new file mode 100644 index 00000000..e3ecf978 --- /dev/null +++ b/packages/core/src/index.ts @@ -0,0 +1,214 @@ +/** + * @packageDocumentation + * + * Exports a `Helia` class that implements the {@link HeliaInterface} API. + * + * In general you should use the `helia` or `@helia/http` modules instead which + * pre-configure Helia for certain use-cases (p2p or pure-HTTP). + * + * @example + * + * ```typescript + * import { Helia } from '@helia/core' + * + * const node = new Helia({ + * // ...options + * }) + * ``` + */ + +import { contentRoutingSymbol, peerRoutingSymbol, start, stop } from '@libp2p/interface' +import { defaultLogger } from '@libp2p/logger' +import drain from 'it-drain' +import { CustomProgressEvent } from 'progress-events' +import { PinsImpl } from './pins.js' +import { Routing as RoutingClass } from './routing.js' +import { BlockStorage } from './storage.js' +import { defaultDagWalkers } from './utils/dag-walkers.js' +import { assertDatastoreVersionIsCurrent } from './utils/datastore-version.js' +import { defaultHashers } from './utils/default-hashers.js' +import { NetworkedStorage } from './utils/networked-storage.js' +import type { DAGWalker, GCOptions, Helia as HeliaInterface, Routing } from '@helia/interface' +import type { BlockBroker } from '@helia/interface/blocks' +import type { Pins } from '@helia/interface/pins' +import type { ComponentLogger, Logger } from '@libp2p/interface' +import type { Blockstore } from 'interface-blockstore' +import type { Datastore } from 'interface-datastore' +import type { CID } from 'multiformats/cid' +import type { MultihashHasher } from 'multiformats/hashes/interface' + +/** + * Options used to create a Helia node. + */ +export interface HeliaInit { + /** + * The blockstore is where blocks are stored + */ + blockstore: Blockstore + + /** + * The datastore is where data is stored + */ + datastore: Datastore + + /** + * By default sha256, sha512 and identity hashes are supported for + * bitswap operations. To bitswap blocks with CIDs using other hashes + * pass appropriate MultihashHashers here. + */ + hashers?: MultihashHasher[] + + /** + * In order to pin CIDs that correspond to a DAG, it's necessary to know + * how to traverse that DAG. DAGWalkers take a block and yield any CIDs + * encoded within that block. + */ + dagWalkers?: DAGWalker[] + + /** + * A list of strategies used to fetch blocks when they are not present in + * the local blockstore + */ + blockBrokers: Array<(components: any) => BlockBroker> + + /** + * Garbage collection requires preventing blockstore writes during searches + * for unpinned blocks as DAGs are typically pinned after they've been + * imported - without locking this could lead to the deletion of blocks while + * they are being added to the blockstore. + * + * By default this lock is held on the current process and other processes + * will contact this process for access. + * + * If Helia is being run in multiple processes, one process must hold the GC + * lock so use this option to control which process that is. + * + * @default true + */ + holdGcLock?: boolean + + /** + * An optional logging component to pass to libp2p. If not specified the + * default implementation from libp2p will be used. + */ + logger?: ComponentLogger + + /** + * Routers perform operations such as looking up content providers, + * information about network peers or getting/putting records. + */ + routers?: Array> + + /** + * Components used by subclasses + */ + components?: Record +} + +export class Helia implements HeliaInterface { + public blockstore: BlockStorage + public datastore: Datastore + public pins: Pins + public logger: ComponentLogger + public routing: Routing + private readonly log: Logger + + constructor (init: HeliaInit) { + this.logger = init.logger ?? defaultLogger() + this.log = this.logger.forComponent('helia') + const hashers = defaultHashers(init.hashers) + + const components = { + blockstore: init.blockstore, + datastore: init.datastore, + hashers, + logger: this.logger, + ...(init.components ?? {}) + } + + const blockBrokers = init.blockBrokers.map((fn) => { + return fn(components) + }) + + const networkedStorage = new NetworkedStorage(components, { + blockBrokers, + hashers + }) + + this.pins = new PinsImpl(init.datastore, networkedStorage, defaultDagWalkers(init.dagWalkers)) + + this.blockstore = new BlockStorage(networkedStorage, this.pins, { + holdGcLock: init.holdGcLock ?? true + }) + this.datastore = init.datastore + this.routing = new RoutingClass(components, { + routers: (init.routers ?? []).flatMap((router: any) => { + // if the router itself is a router + const routers = [ + router + ] + + // if the router provides a libp2p-style ContentRouter + if (router[contentRoutingSymbol] != null) { + routers.push(router[contentRoutingSymbol]) + } + + // if the router provides a libp2p-style PeerRouter + if (router[peerRoutingSymbol] != null) { + routers.push(router[peerRoutingSymbol]) + } + + return routers + }) + }) + } + + async start (): Promise { + await assertDatastoreVersionIsCurrent(this.datastore) + await start( + this.blockstore, + this.datastore, + this.routing + ) + } + + async stop (): Promise { + await stop( + this.blockstore, + this.datastore, + this.routing + ) + } + + async gc (options: GCOptions = {}): Promise { + const releaseLock = await this.blockstore.lock.writeLock() + + try { + const helia = this + const blockstore = this.blockstore.unwrap() + + this.log('gc start') + + await drain(blockstore.deleteMany((async function * (): AsyncGenerator { + for await (const { cid } of blockstore.getAll()) { + try { + if (await helia.pins.isPinned(cid, options)) { + continue + } + + yield cid + + options.onProgress?.(new CustomProgressEvent('helia:gc:deleted', cid)) + } catch (err) { + helia.log.error('Error during gc', err) + options.onProgress?.(new CustomProgressEvent('helia:gc:error', err)) + } + } + }()))) + } finally { + releaseLock() + } + + this.log('gc finished') + } +} diff --git a/packages/helia/src/pins.ts b/packages/core/src/pins.ts similarity index 98% rename from packages/helia/src/pins.ts rename to packages/core/src/pins.ts index acca953e..2187bfa0 100644 --- a/packages/helia/src/pins.ts +++ b/packages/core/src/pins.ts @@ -6,7 +6,8 @@ import { CID, type Version } from 'multiformats/cid' import { CustomProgressEvent, type ProgressOptions } from 'progress-events' import { equals as uint8ArrayEquals } from 'uint8arrays/equals' import { dagCborWalker, dagJsonWalker, dagPbWalker, jsonWalker, rawWalker } from './utils/dag-walkers.js' -import type { DAGWalker, GetBlockProgressEvents } from './index.js' +import type { DAGWalker } from '@helia/interface' +import type { GetBlockProgressEvents } from '@helia/interface/blocks' import type { AddOptions, AddPinEvents, IsPinnedOptions, LsOptions, Pin, Pins, RmOptions } from '@helia/interface/pins' import type { AbortOptions } from '@libp2p/interface' import type { Blockstore } from 'interface-blockstore' diff --git a/packages/core/src/routing.ts b/packages/core/src/routing.ts new file mode 100644 index 00000000..c0b6e26c --- /dev/null +++ b/packages/core/src/routing.ts @@ -0,0 +1,169 @@ +import { CodeError, start, stop } from '@libp2p/interface' +import { PeerSet } from '@libp2p/peer-collections' +import merge from 'it-merge' +import type { Routing as RoutingInterface, Provider, RoutingOptions } from '@helia/interface' +import type { AbortOptions, ComponentLogger, Logger, PeerId, PeerInfo, Startable } from '@libp2p/interface' +import type { CID } from 'multiformats/cid' + +export interface RoutingInit { + routers: Array> +} + +export interface RoutingComponents { + logger: ComponentLogger +} + +export class Routing implements RoutingInterface, Startable { + private readonly log: Logger + private readonly routers: Array> + + constructor (components: RoutingComponents, init: RoutingInit) { + this.log = components.logger.forComponent('helia:routing') + this.routers = init.routers ?? [] + } + + async start (): Promise { + await start(...this.routers) + } + + async stop (): Promise { + await stop(...this.routers) + } + + /** + * Iterates over all content routers in parallel to find providers of the given key + */ + async * findProviders (key: CID, options: RoutingOptions = {}): AsyncIterable { + if (this.routers.length === 0) { + throw new CodeError('No content routers available', 'ERR_NO_ROUTERS_AVAILABLE') + } + + const seen = new PeerSet() + + for await (const peer of merge( + ...supports(this.routers, 'findProviders') + .map(router => router.findProviders(key, options)) + )) { + // the peer was yielded by a content router without multiaddrs and we + // failed to load them + if (peer == null) { + continue + } + + // deduplicate peers + if (seen.has(peer.id)) { + continue + } + + seen.add(peer.id) + + yield peer + } + } + + /** + * Iterates over all content routers in parallel to notify it is + * a provider of the given key + */ + async provide (key: CID, options: AbortOptions = {}): Promise { + if (this.routers.length === 0) { + throw new CodeError('No content routers available', 'ERR_NO_ROUTERS_AVAILABLE') + } + + await Promise.all( + supports(this.routers, 'provide') + .map(async (router) => { + await router.provide(key, options) + }) + ) + } + + /** + * Store the given key/value pair in the available content routings + */ + async put (key: Uint8Array, value: Uint8Array, options?: AbortOptions): Promise { + await Promise.all( + supports(this.routers, 'put') + .map(async (router) => { + await router.put(key, value, options) + }) + ) + } + + /** + * Get the value to the given key. + * Times out after 1 minute by default. + */ + async get (key: Uint8Array, options?: AbortOptions): Promise { + return Promise.any( + supports(this.routers, 'get') + .map(async (router) => { + return router.get(key, options) + }) + ) + } + + /** + * Iterates over all peer routers in parallel to find the given peer + */ + async findPeer (id: PeerId, options?: RoutingOptions): Promise { + if (this.routers.length === 0) { + throw new CodeError('No peer routers available', 'ERR_NO_ROUTERS_AVAILABLE') + } + + const self = this + const source = merge( + ...supports(this.routers, 'findPeer') + .map(router => (async function * () { + try { + yield await router.findPeer(id, options) + } catch (err) { + self.log.error(err) + } + })()) + ) + + for await (const peer of source) { + if (peer == null) { + continue + } + + return peer + } + + throw new CodeError('Could not find peer in routing', 'ERR_NOT_FOUND') + } + + /** + * Attempt to find the closest peers on the network to the given key + */ + async * getClosestPeers (key: Uint8Array, options: RoutingOptions = {}): AsyncIterable { + if (this.routers.length === 0) { + throw new CodeError('No peer routers available', 'ERR_NO_ROUTERS_AVAILABLE') + } + + const seen = new PeerSet() + + for await (const peer of merge( + ...supports(this.routers, 'getClosestPeers') + .map(router => router.getClosestPeers(key, options)) + )) { + if (peer == null) { + continue + } + + // deduplicate peers + if (seen.has(peer.id)) { + continue + } + + seen.add(peer.id) + + yield peer + } + } +} + +function supports (routers: any[], key: Operation): Array> { + return routers.filter(router => router[key] != null) +} diff --git a/packages/helia/src/storage.ts b/packages/core/src/storage.ts similarity index 100% rename from packages/helia/src/storage.ts rename to packages/core/src/storage.ts diff --git a/packages/helia/src/utils/dag-walkers.ts b/packages/core/src/utils/dag-walkers.ts similarity index 95% rename from packages/helia/src/utils/dag-walkers.ts rename to packages/core/src/utils/dag-walkers.ts index cc770c25..966be534 100644 --- a/packages/helia/src/utils/dag-walkers.ts +++ b/packages/core/src/utils/dag-walkers.ts @@ -10,7 +10,7 @@ import { CID } from 'multiformats' import { base64 } from 'multiformats/bases/base64' import * as json from 'multiformats/codecs/json' import * as raw from 'multiformats/codecs/raw' -import type { DAGWalker } from '../index.js' +import type { DAGWalker } from '@helia/interface' /** * Dag walker for dag-pb CIDs @@ -179,3 +179,13 @@ export const jsonWalker: DAGWalker = { codec: json.code, * walk () {} } + +export function defaultDagWalkers (walkers: DAGWalker[] = []): DAGWalker[] { + return [ + dagPbWalker, + rawWalker, + dagCborWalker, + dagJsonWalker, + ...walkers + ] +} diff --git a/packages/helia/src/utils/datastore-version.ts b/packages/core/src/utils/datastore-version.ts similarity index 100% rename from packages/helia/src/utils/datastore-version.ts rename to packages/core/src/utils/datastore-version.ts diff --git a/packages/block-brokers/src/utils/default-hashers.ts b/packages/core/src/utils/default-hashers.ts similarity index 100% rename from packages/block-brokers/src/utils/default-hashers.ts rename to packages/core/src/utils/default-hashers.ts diff --git a/packages/block-brokers/src/utils/networked-storage.ts b/packages/core/src/utils/networked-storage.ts similarity index 100% rename from packages/block-brokers/src/utils/networked-storage.ts rename to packages/core/src/utils/networked-storage.ts diff --git a/packages/core/src/version.ts b/packages/core/src/version.ts new file mode 100644 index 00000000..4224ec2f --- /dev/null +++ b/packages/core/src/version.ts @@ -0,0 +1,2 @@ +export const version = '2.0.0' +export const name = 'helia' diff --git a/packages/block-brokers/test/block-broker.spec.ts b/packages/core/test/block-broker.spec.ts similarity index 97% rename from packages/block-brokers/test/block-broker.spec.ts rename to packages/core/test/block-broker.spec.ts index 605790fe..55887d32 100644 --- a/packages/block-brokers/test/block-broker.spec.ts +++ b/packages/core/test/block-broker.spec.ts @@ -1,5 +1,6 @@ /* eslint-env mocha */ +import { stop } from '@libp2p/interface' import { defaultLogger } from '@libp2p/logger' import { expect } from 'aegir/chai' import { MemoryBlockstore } from 'blockstore-core' @@ -44,6 +45,12 @@ describe('block-broker', () => { }) }) + afterEach(async () => { + await stop( + storage, blockstore + ) + }) + it('gets a block from the gatewayBlockBroker when it is not in the blockstore', async () => { const { cid, block } = blocks[0] diff --git a/packages/helia/test/fixtures/create-block.ts b/packages/core/test/fixtures/create-block.ts similarity index 100% rename from packages/helia/test/fixtures/create-block.ts rename to packages/core/test/fixtures/create-block.ts diff --git a/packages/helia/test/fixtures/create-dag.ts b/packages/core/test/fixtures/create-dag.ts similarity index 100% rename from packages/helia/test/fixtures/create-dag.ts rename to packages/core/test/fixtures/create-dag.ts diff --git a/packages/core/test/fixtures/create-helia.ts b/packages/core/test/fixtures/create-helia.ts new file mode 100644 index 00000000..f5547916 --- /dev/null +++ b/packages/core/test/fixtures/create-helia.ts @@ -0,0 +1,25 @@ +import { MemoryBlockstore } from 'blockstore-core' +import { MemoryDatastore } from 'datastore-core' +import { Helia as HeliaClass, type HeliaInit } from '../../src/index.js' +import type { Helia } from '@helia/interface' + +export async function createHelia (opts: Partial = {}): Promise { + const datastore = new MemoryDatastore() + const blockstore = new MemoryBlockstore() + + const init: HeliaInit = { + datastore, + blockstore, + blockBrokers: [], + holdGcLock: true, + ...opts + } + + const node = new HeliaClass(init) + + if (opts.start !== false) { + await node.start() + } + + return node +} diff --git a/packages/helia/test/fixtures/dag-walker.ts b/packages/core/test/fixtures/dag-walker.ts similarity index 87% rename from packages/helia/test/fixtures/dag-walker.ts rename to packages/core/test/fixtures/dag-walker.ts index 0d3a2e34..39090254 100644 --- a/packages/helia/test/fixtures/dag-walker.ts +++ b/packages/core/test/fixtures/dag-walker.ts @@ -1,6 +1,6 @@ import { toString as uint8ArrayToString } from 'uint8arrays/to-string' import type { DAGNode } from './create-dag.js' -import type { DAGWalker } from '../../src/index.js' +import type { DAGWalker } from '@helia/interface' export function dagWalker (codec: number, dag: Record): DAGWalker { return { diff --git a/packages/helia/test/gc.spec.ts b/packages/core/test/gc.spec.ts similarity index 90% rename from packages/helia/test/gc.spec.ts rename to packages/core/test/gc.spec.ts index 643d65bf..016757aa 100644 --- a/packages/helia/test/gc.spec.ts +++ b/packages/core/test/gc.spec.ts @@ -1,39 +1,19 @@ /* eslint-env mocha */ -import { noise } from '@chainsafe/libp2p-noise' -import { yamux } from '@chainsafe/libp2p-yamux' import * as dagCbor from '@ipld/dag-cbor' import * as dagJson from '@ipld/dag-json' import * as dagPb from '@ipld/dag-pb' -import { webSockets } from '@libp2p/websockets' import { expect } from 'aegir/chai' -import { MemoryBlockstore } from 'blockstore-core' -import { MemoryDatastore } from 'datastore-core' import drain from 'it-drain' -import { createLibp2p } from 'libp2p' import * as raw from 'multiformats/codecs/raw' -import { createHelia } from '../src/index.js' import { createAndPutBlock } from './fixtures/create-block.js' +import { createHelia } from './fixtures/create-helia.js' import type { GcEvents, Helia } from '@helia/interface' describe('gc', () => { let helia: Helia beforeEach(async () => { - helia = await createHelia({ - datastore: new MemoryDatastore(), - blockstore: new MemoryBlockstore(), - libp2p: await createLibp2p({ - transports: [ - webSockets() - ], - connectionEncryption: [ - noise() - ], - streamMuxers: [ - yamux() - ] - }) - }) + helia = await createHelia() }) afterEach(async () => { @@ -47,6 +27,7 @@ describe('gc', () => { Data: Uint8Array.from([0, 1, 2, 3]), Links: [] }), helia.blockstore) + const child2 = await createAndPutBlock(dagPb.code, dagPb.encode({ Data: Uint8Array.from([4, 5, 6, 7]), Links: [] diff --git a/packages/core/test/index.spec.ts b/packages/core/test/index.spec.ts new file mode 100644 index 00000000..17b69d34 --- /dev/null +++ b/packages/core/test/index.spec.ts @@ -0,0 +1,52 @@ +/* eslint-env mocha */ +import { expect } from 'aegir/chai' +import Sinon from 'sinon' +import { stubInterface } from 'sinon-ts' +import { createHelia } from './fixtures/create-helia.js' +import type { Helia, Routing } from '@helia/interface' +import type { Startable } from '@libp2p/interface' + +describe('helia', () => { + let helia: Helia + let routing: Routing + + beforeEach(async () => { + routing = stubInterface({ + start: Sinon.stub(), + stop: Sinon.stub() + }) + helia = await createHelia({ + start: false, + routers: [ + routing + ] + }) + }) + + afterEach(async () => { + if (helia != null) { + await helia.stop() + } + }) + + it('stops and starts', async () => { + expect(routing).to.have.nested.property('start.called', false) + + await helia.start() + + expect(routing).to.have.nested.property('start.called', true) + expect(routing).to.have.nested.property('stop.called', false) + + await helia.stop() + + expect(routing).to.have.nested.property('stop.called', true) + }) + + it('should have a blockstore', async () => { + expect(helia).to.have.property('blockstore').that.is.ok() + }) + + it('should have a datastore', async () => { + expect(helia).to.have.property('datastore').that.is.ok() + }) +}) diff --git a/packages/helia/test/pins.depth-limited.spec.ts b/packages/core/test/pins.depth-limited.spec.ts similarity index 85% rename from packages/helia/test/pins.depth-limited.spec.ts rename to packages/core/test/pins.depth-limited.spec.ts index c516ead3..2ac034c4 100644 --- a/packages/helia/test/pins.depth-limited.spec.ts +++ b/packages/core/test/pins.depth-limited.spec.ts @@ -1,14 +1,9 @@ /* eslint-env mocha */ -import { noise } from '@chainsafe/libp2p-noise' -import { yamux } from '@chainsafe/libp2p-yamux' -import { webSockets } from '@libp2p/websockets' import { expect } from 'aegir/chai' import { MemoryBlockstore } from 'blockstore-core' -import { MemoryDatastore } from 'datastore-core' import drain from 'it-drain' -import { createLibp2p } from 'libp2p' -import { createHelia } from '../src/index.js' import { createDag, type DAGNode } from './fixtures/create-dag.js' +import { createHelia } from './fixtures/create-helia.js' import { dagWalker } from './fixtures/dag-walker.js' import type { Helia } from '@helia/interface' @@ -28,20 +23,8 @@ describe('pins (depth limited)', () => { dag = await createDag(codec, blockstore, MAX_DEPTH, 3) helia = await createHelia({ - blockBrokers: [], - datastore: new MemoryDatastore(), blockstore, - libp2p: await createLibp2p({ - transports: [ - webSockets() - ], - connectionEncryption: [ - noise() - ], - streamMuxers: [ - yamux() - ] - }), + blockBrokers: [], dagWalkers: [ dagWalker(codec, dag) ] diff --git a/packages/helia/test/pins.recursive.spec.ts b/packages/core/test/pins.recursive.spec.ts similarity index 88% rename from packages/helia/test/pins.recursive.spec.ts rename to packages/core/test/pins.recursive.spec.ts index 63d3c260..712af13f 100644 --- a/packages/helia/test/pins.recursive.spec.ts +++ b/packages/core/test/pins.recursive.spec.ts @@ -1,17 +1,13 @@ /* eslint-env mocha */ -import { noise } from '@chainsafe/libp2p-noise' -import { yamux } from '@chainsafe/libp2p-yamux' -import { webSockets } from '@libp2p/websockets' import { expect } from 'aegir/chai' import { MemoryBlockstore } from 'blockstore-core' -import { MemoryDatastore } from 'datastore-core' import all from 'it-all' import drain from 'it-drain' -import { createLibp2p } from 'libp2p' -import { type AddPinEvents, createHelia } from '../src/index.js' import { createDag, type DAGNode } from './fixtures/create-dag.js' +import { createHelia } from './fixtures/create-helia.js' import { dagWalker } from './fixtures/dag-walker.js' import type { Helia } from '@helia/interface' +import type { AddPinEvents } from '@helia/interface/pins' describe('pins (recursive)', () => { let helia: Helia @@ -27,20 +23,8 @@ describe('pins (recursive)', () => { dag = await createDag(codec, blockstore, 2, 3) helia = await createHelia({ - blockBrokers: [], - datastore: new MemoryDatastore(), blockstore, - libp2p: await createLibp2p({ - transports: [ - webSockets() - ], - connectionEncryption: [ - noise() - ], - streamMuxers: [ - yamux() - ] - }), + blockBrokers: [], dagWalkers: [ dagWalker(codec, dag) ] diff --git a/packages/core/test/pins.spec.ts b/packages/core/test/pins.spec.ts new file mode 100644 index 00000000..a41c3458 --- /dev/null +++ b/packages/core/test/pins.spec.ts @@ -0,0 +1,207 @@ +/* eslint-env mocha */ + +import * as dagCbor from '@ipld/dag-cbor' +import * as dagJson from '@ipld/dag-json' +import * as dagPb from '@ipld/dag-pb' +import { expect } from 'aegir/chai' +import all from 'it-all' +import drain from 'it-drain' +import { CID } from 'multiformats/cid' +import * as json from 'multiformats/codecs/json' +import * as raw from 'multiformats/codecs/raw' +import { createAndPutBlock } from './fixtures/create-block.js' +import { createHelia } from './fixtures/create-helia.js' +import type { Helia } from '@helia/interface' +import type { ProgressEvent } from 'progress-events' + +describe('pins', () => { + let helia: Helia + + beforeEach(async () => { + helia = await createHelia() + }) + + afterEach(async () => { + if (helia != null) { + await helia.stop() + } + }) + + it('pins a block', async () => { + const cidV1 = await createAndPutBlock(raw.code, Uint8Array.from([0, 1, 2, 3]), helia.blockstore) + const cidV0 = CID.createV0(cidV1.multihash) + + await drain(helia.pins.add(cidV1)) + + await expect(helia.pins.isPinned(cidV1)).to.eventually.be.true('did not pin v1 CID') + await expect(helia.pins.isPinned(cidV0)).to.eventually.be.true('did not pin v0 CID') + }) + + it('pins a block with progress events', async () => { + const cidV1 = await createAndPutBlock(raw.code, Uint8Array.from([0, 1, 2, 3]), helia.blockstore) + + const events: ProgressEvent[] = [] + + await drain(helia.pins.add(cidV1, { + onProgress: (evt) => { + events.push(evt) + } + })) + + expect(events.map(e => e.type)).to.include.members([ + 'blocks:get:blockstore:get', + 'helia:pin:add' + ]) + }) + + it('unpins a block', async () => { + const cidV1 = await createAndPutBlock(raw.code, Uint8Array.from([0, 1, 2, 3]), helia.blockstore) + const cidV0 = CID.createV0(cidV1.multihash) + + await drain(helia.pins.add(cidV1)) + + await expect(helia.pins.isPinned(cidV1)).to.eventually.be.true('did not pin v1 CID') + await expect(helia.pins.isPinned(cidV0)).to.eventually.be.true('did not pin v0 CID') + + await drain(helia.pins.rm(cidV1)) + + await expect(helia.pins.isPinned(cidV1)).to.eventually.be.false('did not unpin v1 CID') + await expect(helia.pins.isPinned(cidV0)).to.eventually.be.false('did not unpin v0 CID') + }) + + it('does not delete a pinned block', async () => { + const cid = await createAndPutBlock(raw.code, Uint8Array.from([0, 1, 2, 3]), helia.blockstore) + + await drain(helia.pins.add(cid)) + + await expect(helia.blockstore.delete(cid)).to.eventually.be.rejected + .with.property('message', 'CID was pinned') + }) + + it('lists pins created with default args', async () => { + const cidV1 = await createAndPutBlock(raw.code, Uint8Array.from([0, 1, 2, 3]), helia.blockstore) + + await drain(helia.pins.add(cidV1)) + + const pins = await all(helia.pins.ls()) + + expect(pins).to.have.lengthOf(1) + expect(pins).to.have.nested.property('[0].cid').that.eql(cidV1) + expect(pins).to.have.nested.property('[0].depth', Infinity) + expect(pins).to.have.nested.property('[0].metadata').that.eql({}) + }) + + it('lists pins with depth', async () => { + const cidV1 = await createAndPutBlock(raw.code, Uint8Array.from([0, 1, 2, 3]), helia.blockstore) + + await drain(helia.pins.add(cidV1, { + depth: 5 + })) + + const pins = await all(helia.pins.ls()) + + expect(pins).to.have.lengthOf(1) + expect(pins).to.have.nested.property('[0].cid').that.eql(cidV1) + expect(pins).to.have.nested.property('[0].depth', 5) + }) + + it('lists pins with metadata', async () => { + const cidV1 = await createAndPutBlock(raw.code, Uint8Array.from([0, 1, 2, 3]), helia.blockstore) + const metadata = { + foo: 'bar', + baz: 5, + qux: false + } + + await drain(helia.pins.add(cidV1, { + metadata + })) + + const pins = await all(helia.pins.ls()) + + expect(pins).to.have.lengthOf(1) + expect(pins).to.have.nested.property('[0].cid').that.eql(cidV1) + expect(pins).to.have.nested.property('[0].metadata').that.eql(metadata) + }) + + it('lists pins directly', async () => { + const cid1 = await createAndPutBlock(raw.code, Uint8Array.from([0, 1, 2, 3]), helia.blockstore) + const cid2 = await createAndPutBlock(raw.code, Uint8Array.from([4, 5, 6, 7]), helia.blockstore) + + await drain(helia.pins.add(cid1)) + await drain(helia.pins.add(cid2)) + + const pins = await all(helia.pins.ls({ + cid: cid1 + })) + + expect(pins).to.have.lengthOf(1) + expect(pins).to.have.nested.property('[0].cid').that.eql(cid1) + expect(pins).to.have.nested.property('[0].depth', Infinity) + expect(pins).to.have.nested.property('[0].metadata').that.eql({}) + }) + + it('pins a json block', async () => { + const cid1 = await createAndPutBlock(json.code, json.encode({ hello: 'world' }), helia.blockstore) + + await drain(helia.pins.add(cid1)) + + const pins = await all(helia.pins.ls()) + + expect(pins).to.have.lengthOf(1) + expect(pins).to.have.nested.property('[0].cid').that.eql(cid1) + expect(pins).to.have.nested.property('[0].depth', Infinity) + expect(pins).to.have.nested.property('[0].metadata').that.eql({}) + }) + + it('pins a dag-json block', async () => { + const cid1 = await createAndPutBlock(dagJson.code, dagJson.encode({ hello: 'world' }), helia.blockstore) + const cid2 = await createAndPutBlock(dagJson.code, dagJson.encode({ hello: 'world', linked: cid1 }), helia.blockstore) + + await drain(helia.pins.add(cid2)) + + const pins = await all(helia.pins.ls()) + + expect(pins).to.have.lengthOf(1) + expect(pins).to.have.nested.property('[0].cid').that.eql(cid2) + expect(pins).to.have.nested.property('[0].depth', Infinity) + expect(pins).to.have.nested.property('[0].metadata').that.eql({}) + + await expect(helia.pins.isPinned(cid1)).to.eventually.be.true() + await expect(helia.pins.isPinned(cid2)).to.eventually.be.true() + }) + + it('pins a dag-cbor block', async () => { + const cid1 = await createAndPutBlock(dagCbor.code, dagCbor.encode({ hello: 'world' }), helia.blockstore) + const cid2 = await createAndPutBlock(dagCbor.code, dagCbor.encode({ hello: 'world', linked: cid1 }), helia.blockstore) + + await drain(helia.pins.add(cid2)) + + const pins = await all(helia.pins.ls()) + + expect(pins).to.have.lengthOf(1) + expect(pins).to.have.nested.property('[0].cid').that.eql(cid2) + expect(pins).to.have.nested.property('[0].depth', Infinity) + expect(pins).to.have.nested.property('[0].metadata').that.eql({}) + + await expect(helia.pins.isPinned(cid1)).to.eventually.be.true() + await expect(helia.pins.isPinned(cid2)).to.eventually.be.true() + }) + + it('pins a dag-pb block', async () => { + const cid1 = await createAndPutBlock(dagPb.code, dagPb.encode({ Data: Uint8Array.from([0, 1, 2, 3, 4]), Links: [] }), helia.blockstore) + const cid2 = await createAndPutBlock(dagPb.code, dagPb.encode({ Links: [{ Name: '', Hash: cid1, Tsize: 100 }] }), helia.blockstore) + + await drain(helia.pins.add(cid2)) + + const pins = await all(helia.pins.ls()) + + expect(pins).to.have.lengthOf(1) + expect(pins).to.have.nested.property('[0].cid').that.eql(cid2) + expect(pins).to.have.nested.property('[0].depth', Infinity) + expect(pins).to.have.nested.property('[0].metadata').that.eql({}) + + await expect(helia.pins.isPinned(cid1)).to.eventually.be.true() + await expect(helia.pins.isPinned(cid2)).to.eventually.be.true() + }) +}) diff --git a/packages/helia/test/storage.spec.ts b/packages/core/test/storage.spec.ts similarity index 100% rename from packages/helia/test/storage.spec.ts rename to packages/core/test/storage.spec.ts diff --git a/packages/block-brokers/test/utils/networked-storage.spec.ts b/packages/core/test/utils/networked-storage.spec.ts similarity index 100% rename from packages/block-brokers/test/utils/networked-storage.spec.ts rename to packages/core/test/utils/networked-storage.spec.ts diff --git a/packages/core/tsconfig.json b/packages/core/tsconfig.json new file mode 100644 index 00000000..4c0bdf77 --- /dev/null +++ b/packages/core/tsconfig.json @@ -0,0 +1,15 @@ +{ + "extends": "aegir/src/config/tsconfig.aegir.json", + "compilerOptions": { + "outDir": "dist" + }, + "include": [ + "src", + "test" + ], + "references": [ + { + "path": "../interface" + } + ] +} diff --git a/packages/core/typedoc.json b/packages/core/typedoc.json new file mode 100644 index 00000000..f599dc72 --- /dev/null +++ b/packages/core/typedoc.json @@ -0,0 +1,5 @@ +{ + "entryPoints": [ + "./src/index.ts" + ] +} diff --git a/packages/dag-cbor/package.json b/packages/dag-cbor/package.json index c97a7d60..44f9fdd0 100644 --- a/packages/dag-cbor/package.json +++ b/packages/dag-cbor/package.json @@ -141,14 +141,14 @@ }, "dependencies": { "@helia/interface": "^3.0.1", - "@ipld/dag-cbor": "^9.0.0", + "@ipld/dag-cbor": "^9.0.6", "@libp2p/interfaces": "^3.3.1", "multiformats": "^13.0.0", "progress-events": "^1.0.0" }, "devDependencies": { "aegir": "^42.1.0", - "blockstore-core": "^4.3.8", + "blockstore-core": "^4.3.10", "interface-blockstore": "^5.2.9" } } diff --git a/packages/dag-json/package.json b/packages/dag-json/package.json index ea184520..e81e35ca 100644 --- a/packages/dag-json/package.json +++ b/packages/dag-json/package.json @@ -140,14 +140,14 @@ }, "dependencies": { "@helia/interface": "^3.0.1", - "@ipld/dag-json": "^10.0.1", + "@ipld/dag-json": "^10.1.5", "@libp2p/interfaces": "^3.3.1", "multiformats": "^13.0.0", "progress-events": "^1.0.0" }, "devDependencies": { "aegir": "^42.1.0", - "blockstore-core": "^4.3.8", + "blockstore-core": "^4.3.10", "interface-blockstore": "^5.2.9" } } diff --git a/packages/helia/package.json b/packages/helia/package.json index 32c2b2f7..ffcf7033 100644 --- a/packages/helia/package.json +++ b/packages/helia/package.json @@ -57,12 +57,11 @@ "@chainsafe/libp2p-gossipsub": "^11.0.0", "@chainsafe/libp2p-noise": "^14.0.0", "@chainsafe/libp2p-yamux": "^6.0.1", - "@helia/block-brokers": "~1.0.0", - "@helia/delegated-routing-v1-http-api-client": "^1.1.0", + "@helia/block-brokers": "^1.0.0", + "@helia/core": "^0.0.0", + "@helia/delegated-routing-v1-http-api-client": "^2.0.2", "@helia/interface": "^3.0.1", - "@ipld/dag-cbor": "^9.0.0", - "@ipld/dag-json": "^10.0.1", - "@ipld/dag-pb": "^4.0.3", + "@helia/routers": "^0.0.0", "@libp2p/autonat": "^1.0.1", "@libp2p/bootstrap": "^10.0.2", "@libp2p/circuit-relay-v2": "^1.0.2", @@ -77,31 +76,23 @@ "@libp2p/ping": "^1.0.1", "@libp2p/tcp": "^9.0.2", "@libp2p/upnp-nat": "^1.0.1", - "@libp2p/utils": "^5.2.0", "@libp2p/webrtc": "^4.0.3", "@libp2p/websockets": "^8.0.2", "@libp2p/webtransport": "^4.0.3", - "blockstore-core": "^4.0.0", - "cborg": "^4.0.3", - "datastore-core": "^9.0.0", + "blockstore-core": "^4.3.8", + "datastore-core": "^9.2.6", "interface-blockstore": "^5.2.7", - "interface-datastore": "^8.2.2", - "interface-store": "^5.1.5", - "ipns": "^8.0.0", - "it-drain": "^3.0.5", - "libp2p": "^1.0.3", - "mortice": "^3.0.1", - "multiformats": "^13.0.0", - "progress-events": "^1.0.0", - "uint8arrays": "^5.0.1" + "interface-datastore": "^8.2.9", + "ipns": "^8.0.3", + "libp2p": "^1.1.1", + "multiformats": "^13.0.0" }, "devDependencies": { "@multiformats/mafmt": "^12.1.5", - "@multiformats/multiaddr": "^12.1.7", - "@types/sinon": "^17.0.2", + "@multiformats/multiaddr": "^12.1.12", "aegir": "^42.1.0", - "delay": "^6.0.0", - "it-all": "^3.0.4" + "it-all": "^3.0.4", + "it-drain": "^3.0.5" }, "browser": { "./dist/src/utils/libp2p-defaults.js": "./dist/src/utils/libp2p-defaults.browser.js" diff --git a/packages/helia/src/helia-p2p.ts b/packages/helia/src/helia-p2p.ts new file mode 100644 index 00000000..65eb227c --- /dev/null +++ b/packages/helia/src/helia-p2p.ts @@ -0,0 +1,37 @@ +import { Helia, type HeliaInit } from '@helia/core' +import type { BlockBroker } from './index.js' +import type { Libp2p } from '@libp2p/interface' +import type { Blockstore } from 'interface-blockstore' +import type { Datastore } from 'interface-datastore' + +export interface HeliaP2PInit extends HeliaInit { + libp2p: T + blockstore: Blockstore + datastore: Datastore + blockBrokers: Array<(components: any) => BlockBroker> +} + +export class HeliaP2P extends Helia { + public libp2p: T + + constructor (init: HeliaP2PInit) { + super({ + ...init, + components: { + libp2p: init.libp2p + } + }) + + this.libp2p = init.libp2p + } + + async start (): Promise { + await super.start() + await this.libp2p.start() + } + + async stop (): Promise { + await super.stop() + await this.libp2p.stop() + } +} diff --git a/packages/helia/src/helia.ts b/packages/helia/src/helia.ts deleted file mode 100644 index 010cb7e9..00000000 --- a/packages/helia/src/helia.ts +++ /dev/null @@ -1,107 +0,0 @@ -import { bitswap, trustlessGateway, NetworkedStorage } from '@helia/block-brokers' -import { start, stop } from '@libp2p/interface' -import drain from 'it-drain' -import { CustomProgressEvent } from 'progress-events' -import { PinsImpl } from './pins.js' -import { BlockStorage } from './storage.js' -import { assertDatastoreVersionIsCurrent } from './utils/datastore-version.js' -import { defaultHashers } from './utils/default-hashers.js' -import type { HeliaInit } from '.' -import type { GCOptions, Helia } from '@helia/interface' -import type { Pins } from '@helia/interface/pins' -import type { ComponentLogger, Libp2p, Logger } from '@libp2p/interface' -import type { Blockstore } from 'interface-blockstore' -import type { Datastore } from 'interface-datastore' -import type { CID } from 'multiformats/cid' - -interface HeliaImplInit extends HeliaInit { - libp2p: T - blockstore: Blockstore - datastore: Datastore -} - -export class HeliaImpl implements Helia { - public libp2p: Libp2p - public blockstore: BlockStorage - public datastore: Datastore - public pins: Pins - public logger: ComponentLogger - private readonly log: Logger - - constructor (init: HeliaImplInit) { - this.logger = init.libp2p.logger - this.log = this.logger.forComponent('helia') - const hashers = defaultHashers(init.hashers) - - const components = { - blockstore: init.blockstore, - datastore: init.datastore, - libp2p: init.libp2p, - hashers, - logger: init.libp2p.logger - } - - const blockBrokers = init.blockBrokers?.map((fn) => { - return fn(components) - }) ?? [ - bitswap()(components), - trustlessGateway()(components) - ] - - const networkedStorage = new NetworkedStorage(components, { - blockBrokers, - hashers - }) - - this.pins = new PinsImpl(init.datastore, networkedStorage, init.dagWalkers ?? []) - - this.libp2p = init.libp2p - this.blockstore = new BlockStorage(networkedStorage, this.pins, { - holdGcLock: init.holdGcLock - }) - this.datastore = init.datastore - } - - async start (): Promise { - await assertDatastoreVersionIsCurrent(this.datastore) - await start(this.blockstore) - await this.libp2p.start() - } - - async stop (): Promise { - await this.libp2p.stop() - await stop(this.blockstore) - } - - async gc (options: GCOptions = {}): Promise { - const releaseLock = await this.blockstore.lock.writeLock() - - try { - const helia = this - const blockstore = this.blockstore.unwrap() - - this.log('gc start') - - await drain(blockstore.deleteMany((async function * (): AsyncGenerator { - for await (const { cid } of blockstore.getAll()) { - try { - if (await helia.pins.isPinned(cid, options)) { - continue - } - - yield cid - - options.onProgress?.(new CustomProgressEvent('helia:gc:deleted', cid)) - } catch (err) { - helia.log.error('Error during gc', err) - options.onProgress?.(new CustomProgressEvent('helia:gc:error', err)) - } - } - }()))) - } finally { - releaseLock() - } - - this.log('gc finished') - } -} diff --git a/packages/helia/src/index.ts b/packages/helia/src/index.ts index c5a47f47..06fe4dd2 100644 --- a/packages/helia/src/index.ts +++ b/packages/helia/src/index.ts @@ -19,13 +19,15 @@ * ``` */ +import { bitswap, trustlessGateway } from '@helia/block-brokers' +import { libp2pRouting } from '@helia/routers' import { MemoryBlockstore } from 'blockstore-core' import { MemoryDatastore } from 'datastore-core' -import { HeliaImpl } from './helia.js' +import { HeliaP2P } from './helia-p2p.js' import { libp2pDefaults } from './utils/libp2p-defaults.js' import { createLibp2p } from './utils/libp2p.js' import type { DefaultLibp2pServices } from './utils/libp2p-defaults.js' -import type { Helia } from '@helia/interface' +import type { Helia } from '@helia/core' import type { BlockBroker } from '@helia/interface/blocks' import type { ComponentLogger, Libp2p } from '@libp2p/interface' import type { KeychainInit } from '@libp2p/keychain' @@ -38,8 +40,6 @@ import type { MultihashHasher } from 'multiformats/hashes/interface' // re-export interface types so people don't have to depend on @helia/interface // if they don't want to export * from '@helia/interface' -export * from '@helia/interface/blocks' -export * from '@helia/interface/pins' export type { DefaultLibp2pServices } export { libp2pDefaults } @@ -132,12 +132,16 @@ export interface HeliaInit { keychain?: KeychainInit } +export interface HeliaLibp2p> extends Helia { + libp2p: T +} + /** * Create and return a Helia node */ -export async function createHelia (init: HeliaInit): Promise> -export async function createHelia (init?: HeliaInit>): Promise>> -export async function createHelia (init: HeliaInit = {}): Promise> { +export async function createHelia (init: HeliaInit): Promise> +export async function createHelia (init?: HeliaInit>): Promise>> +export async function createHelia (init: HeliaInit = {}): Promise { const datastore = init.datastore ?? new MemoryDatastore() const blockstore = init.blockstore ?? new MemoryBlockstore() @@ -153,11 +157,18 @@ export async function createHelia (init: HeliaInit = {}): Promise }) } - const helia = new HeliaImpl({ + const helia = new HeliaP2P({ ...init, libp2p, datastore, - blockstore + blockstore, + blockBrokers: init.blockBrokers ?? [ + trustlessGateway(), + bitswap() + ], + routers: [ + libp2pRouting(libp2p) + ] }) if (init.start !== false) { diff --git a/packages/helia/src/utils/default-hashers.ts b/packages/helia/src/utils/default-hashers.ts deleted file mode 100644 index 9352e870..00000000 --- a/packages/helia/src/utils/default-hashers.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { identity } from 'multiformats/hashes/identity' -import { sha256, sha512 } from 'multiformats/hashes/sha2' -import type { MultihashHasher } from 'multiformats/hashes/interface' - -export function defaultHashers (hashers: MultihashHasher[] = []): MultihashHasher[] { - return [ - sha256, - sha512, - identity, - ...hashers - ] -} diff --git a/packages/helia/test/factory.spec.ts b/packages/helia/test/factory.spec.ts index a7de6740..b6cc48da 100644 --- a/packages/helia/test/factory.spec.ts +++ b/packages/helia/test/factory.spec.ts @@ -7,11 +7,10 @@ import { MemoryDatastore } from 'datastore-core' import { Key } from 'interface-datastore' import { createLibp2p } from 'libp2p' import { CID } from 'multiformats/cid' -import { createHelia } from '../src/index.js' -import type { Helia } from '@helia/interface' +import { createHelia, type HeliaLibp2p } from '../src/index.js' describe('helia factory', () => { - let helia: Helia + let helia: HeliaLibp2p afterEach(async () => { if (helia != null) { diff --git a/packages/helia/test/fixtures/create-helia.ts b/packages/helia/test/fixtures/create-helia.ts deleted file mode 100644 index bff75f87..00000000 --- a/packages/helia/test/fixtures/create-helia.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { bitswap } from '@helia/block-brokers' -import { circuitRelayTransport } from '@libp2p/circuit-relay-v2' -import { identify } from '@libp2p/identify' -import { webSockets } from '@libp2p/websockets' -import * as Filters from '@libp2p/websockets/filters' -import { createHelia as createNode } from '../../src/index.js' -import type { Helia } from '@helia/interface' - -export async function createHelia (): Promise { - return createNode({ - blockBrokers: [ - bitswap() - ], - libp2p: { - addresses: { - listen: [ - `${process.env.RELAY_SERVER}/p2p-circuit` - ] - }, - transports: [ - webSockets({ - filter: Filters.all - }), - circuitRelayTransport() - ], - connectionGater: { - denyDialMultiaddr: async () => false - }, - services: { - identify: identify() - } - } - }) -} diff --git a/packages/helia/test/index.spec.ts b/packages/helia/test/index.spec.ts index 56dfd190..5d3a22cb 100644 --- a/packages/helia/test/index.spec.ts +++ b/packages/helia/test/index.spec.ts @@ -6,11 +6,10 @@ import { expect } from 'aegir/chai' import { MemoryBlockstore } from 'blockstore-core' import { MemoryDatastore } from 'datastore-core' import { createLibp2p } from 'libp2p' -import { createHelia } from '../src/index.js' -import type { Helia } from '@helia/interface' +import { createHelia, type HeliaLibp2p } from '../src/index.js' describe('helia', () => { - let helia: Helia + let helia: HeliaLibp2p beforeEach(async () => { helia = await createHelia({ diff --git a/packages/helia/test/libp2p.spec.ts b/packages/helia/test/libp2p.spec.ts index 930d49dd..57aff7c0 100644 --- a/packages/helia/test/libp2p.spec.ts +++ b/packages/helia/test/libp2p.spec.ts @@ -1,13 +1,11 @@ /* eslint-env mocha */ -import { webSockets } from '@libp2p/websockets' import { expect } from 'aegir/chai' import { createLibp2p } from 'libp2p' -import { createHelia } from '../src/index.js' -import type { Helia } from '@helia/interface' +import { createHelia, type HeliaLibp2p } from '../src/index.js' describe('libp2p', () => { - let helia: Helia + let helia: HeliaLibp2p afterEach(async () => { if (helia != null) { @@ -38,11 +36,7 @@ describe('libp2p', () => { }) it('allows passing a libp2p node', async () => { - const libp2p = await createLibp2p({ - transports: [ - webSockets() - ] - }) + const libp2p = await createLibp2p() helia = await createHelia({ libp2p diff --git a/packages/helia/test/pins.spec.ts b/packages/helia/test/pins.spec.ts index 92e616d1..c53e8362 100644 --- a/packages/helia/test/pins.spec.ts +++ b/packages/helia/test/pins.spec.ts @@ -1,25 +1,35 @@ /* eslint-env mocha */ -import * as dagCbor from '@ipld/dag-cbor' -import * as dagJson from '@ipld/dag-json' -import * as dagPb from '@ipld/dag-pb' +import { webSockets } from '@libp2p/websockets' +import * as Filters from '@libp2p/websockets/filters' import { multiaddr } from '@multiformats/multiaddr' import { expect } from 'aegir/chai' import all from 'it-all' import drain from 'it-drain' import { CID } from 'multiformats/cid' -import * as json from 'multiformats/codecs/json' -import * as raw from 'multiformats/codecs/raw' -import { createAndPutBlock } from './fixtures/create-block.js' -import { createHelia } from './fixtures/create-helia.js' -import type { Helia } from '@helia/interface' -import type { ProgressEvent } from 'progress-events' +import { createHelia } from '../src/index.js' +import type { HeliaLibp2p } from '../src/index.js' +import type { Libp2p } from '@libp2p/interface' describe('pins', () => { - let helia: Helia + let helia: HeliaLibp2p beforeEach(async () => { - helia = await createHelia() + helia = await createHelia({ + libp2p: { + addresses: { + listen: [] + }, + connectionGater: { + denyDialMultiaddr: () => false + }, + transports: [ + webSockets({ + filter: Filters.all + }) + ] + } + }) }) afterEach(async () => { @@ -28,120 +38,6 @@ describe('pins', () => { } }) - it('pins a block', async () => { - const cidV1 = await createAndPutBlock(raw.code, Uint8Array.from([0, 1, 2, 3]), helia.blockstore) - const cidV0 = CID.createV0(cidV1.multihash) - - await drain(helia.pins.add(cidV1)) - - await expect(helia.pins.isPinned(cidV1)).to.eventually.be.true('did not pin v1 CID') - await expect(helia.pins.isPinned(cidV0)).to.eventually.be.true('did not pin v0 CID') - }) - - it('pins a block with progress events', async () => { - const cidV1 = await createAndPutBlock(raw.code, Uint8Array.from([0, 1, 2, 3]), helia.blockstore) - - const events: ProgressEvent[] = [] - - await drain(helia.pins.add(cidV1, { - onProgress: (evt) => { - events.push(evt) - } - })) - - expect(events.map(e => e.type)).to.include.members([ - 'blocks:get:blockstore:get', - 'helia:pin:add' - ]) - }) - - it('unpins a block', async () => { - const cidV1 = await createAndPutBlock(raw.code, Uint8Array.from([0, 1, 2, 3]), helia.blockstore) - const cidV0 = CID.createV0(cidV1.multihash) - - await drain(helia.pins.add(cidV1)) - - await expect(helia.pins.isPinned(cidV1)).to.eventually.be.true('did not pin v1 CID') - await expect(helia.pins.isPinned(cidV0)).to.eventually.be.true('did not pin v0 CID') - - await drain(helia.pins.rm(cidV1)) - - await expect(helia.pins.isPinned(cidV1)).to.eventually.be.false('did not unpin v1 CID') - await expect(helia.pins.isPinned(cidV0)).to.eventually.be.false('did not unpin v0 CID') - }) - - it('does not delete a pinned block', async () => { - const cid = await createAndPutBlock(raw.code, Uint8Array.from([0, 1, 2, 3]), helia.blockstore) - - await drain(helia.pins.add(cid)) - - await expect(helia.blockstore.delete(cid)).to.eventually.be.rejected - .with.property('message', 'CID was pinned') - }) - - it('lists pins created with default args', async () => { - const cidV1 = await createAndPutBlock(raw.code, Uint8Array.from([0, 1, 2, 3]), helia.blockstore) - - await drain(helia.pins.add(cidV1)) - - const pins = await all(helia.pins.ls()) - - expect(pins).to.have.lengthOf(1) - expect(pins).to.have.nested.property('[0].cid').that.eql(cidV1) - expect(pins).to.have.nested.property('[0].depth', Infinity) - expect(pins).to.have.nested.property('[0].metadata').that.eql({}) - }) - - it('lists pins with depth', async () => { - const cidV1 = await createAndPutBlock(raw.code, Uint8Array.from([0, 1, 2, 3]), helia.blockstore) - - await drain(helia.pins.add(cidV1, { - depth: 5 - })) - - const pins = await all(helia.pins.ls()) - - expect(pins).to.have.lengthOf(1) - expect(pins).to.have.nested.property('[0].cid').that.eql(cidV1) - expect(pins).to.have.nested.property('[0].depth', 5) - }) - - it('lists pins with metadata', async () => { - const cidV1 = await createAndPutBlock(raw.code, Uint8Array.from([0, 1, 2, 3]), helia.blockstore) - const metadata = { - foo: 'bar', - baz: 5, - qux: false - } - - await drain(helia.pins.add(cidV1, { - metadata - })) - - const pins = await all(helia.pins.ls()) - - expect(pins).to.have.lengthOf(1) - expect(pins).to.have.nested.property('[0].cid').that.eql(cidV1) - expect(pins).to.have.nested.property('[0].metadata').that.eql(metadata) - }) - - it('lists pins directly', async () => { - const cid1 = await createAndPutBlock(raw.code, Uint8Array.from([0, 1, 2, 3]), helia.blockstore) - const cid2 = await createAndPutBlock(raw.code, Uint8Array.from([4, 5, 6, 7]), helia.blockstore) - - await drain(helia.pins.add(cid1)) - await drain(helia.pins.add(cid2)) - - const pins = await all(helia.pins.ls({ - cid: cid1 - })) - - expect(pins).to.have.lengthOf(1) - expect(pins).to.have.nested.property('[0].cid').that.eql(cid1) - expect(pins).to.have.nested.property('[0].depth', Infinity) - expect(pins).to.have.nested.property('[0].metadata').that.eql({}) - }) - it('pins a block from another node', async () => { const cid = CID.parse(process.env.BLOCK_CID ?? '') await helia.libp2p.dial(multiaddr(process.env.RELAY_SERVER)) @@ -154,68 +50,4 @@ describe('pins', () => { expect(pins).to.have.nested.property('[0].depth', Infinity) expect(pins).to.have.nested.property('[0].metadata').that.eql({}) }) - - it('pins a json block', async () => { - const cid1 = await createAndPutBlock(json.code, json.encode({ hello: 'world' }), helia.blockstore) - - await drain(helia.pins.add(cid1)) - - const pins = await all(helia.pins.ls()) - - expect(pins).to.have.lengthOf(1) - expect(pins).to.have.nested.property('[0].cid').that.eql(cid1) - expect(pins).to.have.nested.property('[0].depth', Infinity) - expect(pins).to.have.nested.property('[0].metadata').that.eql({}) - }) - - it('pins a dag-json block', async () => { - const cid1 = await createAndPutBlock(dagJson.code, dagJson.encode({ hello: 'world' }), helia.blockstore) - const cid2 = await createAndPutBlock(dagJson.code, dagJson.encode({ hello: 'world', linked: cid1 }), helia.blockstore) - - await drain(helia.pins.add(cid2)) - - const pins = await all(helia.pins.ls()) - - expect(pins).to.have.lengthOf(1) - expect(pins).to.have.nested.property('[0].cid').that.eql(cid2) - expect(pins).to.have.nested.property('[0].depth', Infinity) - expect(pins).to.have.nested.property('[0].metadata').that.eql({}) - - await expect(helia.pins.isPinned(cid1)).to.eventually.be.true() - await expect(helia.pins.isPinned(cid2)).to.eventually.be.true() - }) - - it('pins a dag-cbor block', async () => { - const cid1 = await createAndPutBlock(dagCbor.code, dagCbor.encode({ hello: 'world' }), helia.blockstore) - const cid2 = await createAndPutBlock(dagCbor.code, dagCbor.encode({ hello: 'world', linked: cid1 }), helia.blockstore) - - await drain(helia.pins.add(cid2)) - - const pins = await all(helia.pins.ls()) - - expect(pins).to.have.lengthOf(1) - expect(pins).to.have.nested.property('[0].cid').that.eql(cid2) - expect(pins).to.have.nested.property('[0].depth', Infinity) - expect(pins).to.have.nested.property('[0].metadata').that.eql({}) - - await expect(helia.pins.isPinned(cid1)).to.eventually.be.true() - await expect(helia.pins.isPinned(cid2)).to.eventually.be.true() - }) - - it('pins a dag-pb block', async () => { - const cid1 = await createAndPutBlock(dagPb.code, dagPb.encode({ Data: Uint8Array.from([0, 1, 2, 3, 4]), Links: [] }), helia.blockstore) - const cid2 = await createAndPutBlock(dagPb.code, dagPb.encode({ Links: [{ Name: '', Hash: cid1, Tsize: 100 }] }), helia.blockstore) - - await drain(helia.pins.add(cid2)) - - const pins = await all(helia.pins.ls()) - - expect(pins).to.have.lengthOf(1) - expect(pins).to.have.nested.property('[0].cid').that.eql(cid2) - expect(pins).to.have.nested.property('[0].depth', Infinity) - expect(pins).to.have.nested.property('[0].metadata').that.eql({}) - - await expect(helia.pins.isPinned(cid1)).to.eventually.be.true() - await expect(helia.pins.isPinned(cid2)).to.eventually.be.true() - }) }) diff --git a/packages/helia/tsconfig.json b/packages/helia/tsconfig.json index b70ddfff..976a4ebf 100644 --- a/packages/helia/tsconfig.json +++ b/packages/helia/tsconfig.json @@ -11,8 +11,14 @@ { "path": "../block-brokers" }, + { + "path": "../core" + }, { "path": "../interface" + }, + { + "path": "../routers" } ] } diff --git a/packages/http/.aegir.js b/packages/http/.aegir.js new file mode 100644 index 00000000..f3c0948a --- /dev/null +++ b/packages/http/.aegir.js @@ -0,0 +1,8 @@ +/** @type {import('aegir').PartialOptions} */ +const options = { + build: { + bundlesizeMax: '80KB' + } +} + +export default options diff --git a/packages/http/LICENSE b/packages/http/LICENSE new file mode 100644 index 00000000..20ce483c --- /dev/null +++ b/packages/http/LICENSE @@ -0,0 +1,4 @@ +This project is dual licensed under MIT and Apache-2.0. + +MIT: https://www.opensource.org/licenses/mit +Apache-2.0: https://www.apache.org/licenses/license-2.0 diff --git a/packages/http/LICENSE-APACHE b/packages/http/LICENSE-APACHE new file mode 100644 index 00000000..14478a3b --- /dev/null +++ b/packages/http/LICENSE-APACHE @@ -0,0 +1,5 @@ +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. diff --git a/packages/http/LICENSE-MIT b/packages/http/LICENSE-MIT new file mode 100644 index 00000000..72dc60d8 --- /dev/null +++ b/packages/http/LICENSE-MIT @@ -0,0 +1,19 @@ +The MIT License (MIT) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/packages/http/README.md b/packages/http/README.md new file mode 100644 index 00000000..c28f1eeb --- /dev/null +++ b/packages/http/README.md @@ -0,0 +1,96 @@ +

+ + Helia logo + +

+ +[![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) +[![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) +[![codecov](https://img.shields.io/codecov/c/github/ipfs/helia.svg?style=flat-square)](https://codecov.io/gh/ipfs/helia) +[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/helia/main.yml?branch=main\&style=flat-square)](https://github.com/ipfs/helia/actions/workflows/main.yml?query=branch%3Amain) + +> A lightweight implementation of IPFS over HTTP in JavaScript + +# About + +Exports a `createHeliaHTTP` function that returns an object that implements a lightweight version of the Helia API that functions only over HTTP. + +By default, content and peer routing are requests are resolved using the [Delegated HTTP Routing API](https://specs.ipfs.tech/routing/http-routing-v1/) and blocks are fetched from [Trustless Gateways](https://specs.ipfs.tech/http-gateways/trustless-gateway/). + +Pass it to other modules like @helia/unixfs to fetch files from the distributed web. + +## Example + +```typescript +import { createHeliaHTTP } from '@helia/http' +import { unixfs } from '@helia/unixfs' +import { CID } from 'multiformats/cid' + +const helia = await createHeliaHTTP() + +const fs = unixfs(helia) +fs.cat(CID.parse('bafyFoo')) +``` + +## Example - with custom gateways and delegated routing endpoints + +```typescript +import { createHeliaHTTP } from '@helia/http' +import { trustlessGateway } from '@helia/block-brokers' +import { delegatedHTTPRouting } from '@helia/routers' +import { unixfs } from '@helia/unixfs' +import { CID } from 'multiformats/cid' + +const helia = await createHeliaHTTP({ + blockBrokers: [ + trustlessGateway({ + gateways: ['https://cloudflare-ipfs.com', 'https://ipfs.io'], + }), + ], + routers: [ + delegatedHTTPRouting('https://delegated-ipfs.dev') + ] +}) + +const fs = unixfs(helia) +fs.cat(CID.parse('bafyFoo')) +``` + +# Install + +```console +$ npm i @helia/http +``` + +## Browser ` +``` + +# Helia-http + +# API Docs + +- + +# License + +Licensed under either of + +- Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) +- MIT ([LICENSE-MIT](LICENSE-MIT) / ) + +# Contribute + +Contributions welcome! Please check out [the issues](https://github.com/ipfs/helia/issues). + +Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. + +Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). + +Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. + +[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) diff --git a/packages/http/package.json b/packages/http/package.json new file mode 100644 index 00000000..4d55b3b9 --- /dev/null +++ b/packages/http/package.json @@ -0,0 +1,71 @@ +{ + "name": "@helia/http", + "version": "0.9.0", + "description": "A lightweight implementation of IPFS over HTTP in JavaScript", + "license": "Apache-2.0 OR MIT", + "homepage": "https://github.com/ipfs/helia/tree/main/packages/http#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/ipfs/helia.git" + }, + "bugs": { + "url": "https://github.com/ipfs/helia/issues" + }, + "publishConfig": { + "access": "public", + "provenance": true + }, + "keywords": [ + "IPFS" + ], + "type": "module", + "types": "./dist/src/index.d.ts", + "files": [ + "src", + "dist", + "!dist/test", + "!**/*.tsbuildinfo" + ], + "exports": { + ".": { + "types": "./dist/src/index.d.ts", + "import": "./dist/src/index.js" + } + }, + "eslintConfig": { + "extends": "ipfs", + "parserOptions": { + "project": true, + "sourceType": "module" + } + }, + "scripts": { + "clean": "aegir clean", + "lint": "aegir lint", + "dep-check": "aegir dep-check", + "build": "aegir build", + "test": "aegir test", + "test:chrome": "aegir test -t browser --cov", + "test:chrome-webworker": "aegir test -t webworker", + "test:firefox": "aegir test -t browser -- --browser firefox", + "test:firefox-webworker": "aegir test -t webworker -- --browser firefox", + "test:node": "aegir test -t node --cov", + "test:electron-main": "aegir test -t electron-main" + }, + "dependencies": { + "@helia/block-brokers": "^1.0.0", + "@helia/core": "^0.0.0", + "@helia/interface": "^3.0.1", + "@helia/routers": "^0.0.0", + "blockstore-core": "^4.3.8", + "datastore-core": "^9.2.6" + }, + "devDependencies": { + "@libp2p/interface": "^1.1.1", + "aegir": "^42.1.0", + "interface-datastore": "^8.2.10", + "multiformats": "^13.0.1", + "sinon": "^17.0.1", + "sinon-ts": "^2.0.0" + } +} diff --git a/packages/http/src/index.ts b/packages/http/src/index.ts new file mode 100644 index 00000000..7fa44e3b --- /dev/null +++ b/packages/http/src/index.ts @@ -0,0 +1,89 @@ +/** + * @packageDocumentation + * + * Exports a `createHeliaHTTP` function that returns an object that implements a lightweight version of the {@link Helia} API that functions only over HTTP. + * + * By default, content and peer routing are requests are resolved using the [Delegated HTTP Routing API](https://specs.ipfs.tech/routing/http-routing-v1/) and blocks are fetched from [Trustless Gateways](https://specs.ipfs.tech/http-gateways/trustless-gateway/). + * + * Pass it to other modules like {@link https://www.npmjs.com/package/@helia/unixfs | @helia/unixfs} to fetch files from the distributed web. + * + * @example + * + * ```typescript + * import { createHeliaHTTP } from '@helia/http' + * import { unixfs } from '@helia/unixfs' + * import { CID } from 'multiformats/cid' + * + * const helia = await createHeliaHTTP() + * + * const fs = unixfs(helia) + * fs.cat(CID.parse('bafyFoo')) + * ``` + * @example with custom gateways and delegated routing endpoints + * ```typescript + * import { createHeliaHTTP } from '@helia/http' + * import { trustlessGateway } from '@helia/block-brokers' + * import { delegatedHTTPRouting } from '@helia/routers' + * import { unixfs } from '@helia/unixfs' + * import { CID } from 'multiformats/cid' + * + * const helia = await createHeliaHTTP({ + * blockBrokers: [ + * trustlessGateway({ + * gateways: ['https://cloudflare-ipfs.com', 'https://ipfs.io'], + * }), + * ], + * routers: [ + * delegatedHTTPRouting('https://delegated-ipfs.dev') + * ] + * }) + * + * const fs = unixfs(helia) + * fs.cat(CID.parse('bafyFoo')) + * ``` + */ + +import { trustlessGateway } from '@helia/block-brokers' +import { Helia as HeliaClass, type HeliaInit } from '@helia/core' +import { delegatedHTTPRouting } from '@helia/routers' +import { MemoryBlockstore } from 'blockstore-core' +import { MemoryDatastore } from 'datastore-core' +import type { Helia } from '@helia/interface' + +// re-export interface types so people don't have to depend on @helia/interface +// if they don't want to +export * from '@helia/interface' + +export interface HeliaHTTPInit extends HeliaInit { + /** + * Whether to start the Helia node + */ + start?: boolean +} + +/** +/** + * Create and return a Helia node + */ +export async function createHeliaHTTP (init: Partial = {}): Promise { + const datastore = init.datastore ?? new MemoryDatastore() + const blockstore = init.blockstore ?? new MemoryBlockstore() + + const helia = new HeliaClass({ + ...init, + datastore, + blockstore, + blockBrokers: init.blockBrokers ?? [ + trustlessGateway() + ], + routers: init.routers ?? [ + delegatedHTTPRouting('https://delegated-ipfs.dev') + ] + }) + + if (init.start !== false) { + await helia.start() + } + + return helia +} diff --git a/packages/http/test/factory.spec.ts b/packages/http/test/factory.spec.ts new file mode 100644 index 00000000..265a3358 --- /dev/null +++ b/packages/http/test/factory.spec.ts @@ -0,0 +1,34 @@ +/* eslint-env mocha */ + +import { expect } from 'aegir/chai' +import { Key } from 'interface-datastore' +import { CID } from 'multiformats/cid' +import { createHeliaHTTP } from '../src/index.js' +import type { Helia } from '@helia/interface' + +describe('helia factory', () => { + let heliaHTTP: Helia + + afterEach(async () => { + if (heliaHTTP != null) { + await heliaHTTP.stop() + } + }) + + it('does not require any constructor args', async () => { + heliaHTTP = await createHeliaHTTP() + + const cid = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') + const block = Uint8Array.from([0, 1, 2, 3]) + + await heliaHTTP.blockstore.put(cid, block) + const blockIsStored = await heliaHTTP.blockstore.has(cid) + + const key = new Key(`/${cid.toString()}`) + await heliaHTTP.datastore.put(key, block) + const dataIsStored = await heliaHTTP.datastore.has(key) + + expect(blockIsStored).to.be.true() + expect(dataIsStored).to.be.true() + }) +}) diff --git a/packages/http/test/index.spec.ts b/packages/http/test/index.spec.ts new file mode 100644 index 00000000..aad3c60d --- /dev/null +++ b/packages/http/test/index.spec.ts @@ -0,0 +1,52 @@ +/* eslint-env mocha */ +import { expect } from 'aegir/chai' +import Sinon from 'sinon' +import { stubInterface } from 'sinon-ts' +import { createHeliaHTTP } from '../src/index.js' +import type { Helia, Routing } from '@helia/interface' +import type { Startable } from '@libp2p/interface' + +describe('@helia/http', () => { + let helia: Helia + let routing: Routing + + beforeEach(async () => { + routing = stubInterface({ + start: Sinon.stub(), + stop: Sinon.stub() + }) + helia = await createHeliaHTTP({ + start: false, + routers: [ + routing + ] + }) + }) + + afterEach(async () => { + if (helia != null) { + await helia.stop() + } + }) + + it('stops and starts', async () => { + expect(routing).to.have.nested.property('start.called', false) + + await helia.start() + + expect(routing).to.have.nested.property('start.called', true) + expect(routing).to.have.nested.property('stop.called', false) + + await helia.stop() + + expect(routing).to.have.nested.property('stop.called', true) + }) + + it('should have a blockstore', async () => { + expect(helia).to.have.property('blockstore').that.is.ok() + }) + + it('should have a datastore', async () => { + expect(helia).to.have.property('datastore').that.is.ok() + }) +}) diff --git a/packages/http/tsconfig.json b/packages/http/tsconfig.json new file mode 100644 index 00000000..976a4ebf --- /dev/null +++ b/packages/http/tsconfig.json @@ -0,0 +1,24 @@ +{ + "extends": "aegir/src/config/tsconfig.aegir.json", + "compilerOptions": { + "outDir": "dist" + }, + "include": [ + "src", + "test" + ], + "references": [ + { + "path": "../block-brokers" + }, + { + "path": "../core" + }, + { + "path": "../interface" + }, + { + "path": "../routers" + } + ] +} diff --git a/packages/http/typedoc.json b/packages/http/typedoc.json new file mode 100644 index 00000000..f599dc72 --- /dev/null +++ b/packages/http/typedoc.json @@ -0,0 +1,5 @@ +{ + "entryPoints": [ + "./src/index.ts" + ] +} diff --git a/packages/interface/package.json b/packages/interface/package.json index 6526f228..29437890 100644 --- a/packages/interface/package.json +++ b/packages/interface/package.json @@ -72,9 +72,8 @@ "dependencies": { "@libp2p/interface": "^1.1.1", "interface-blockstore": "^5.2.7", - "interface-datastore": "^8.2.2", + "interface-datastore": "^8.2.9", "interface-store": "^5.1.5", - "ipfs-bitswap": "^20.0.0", "multiformats": "^13.0.0", "progress-events": "^1.0.0" }, diff --git a/packages/interface/src/blocks.ts b/packages/interface/src/blocks.ts index 0b8356ea..ecd23012 100644 --- a/packages/interface/src/blocks.ts +++ b/packages/interface/src/blocks.ts @@ -1,6 +1,5 @@ import type { Blockstore } from 'interface-blockstore' import type { AbortOptions } from 'interface-store' -import type { BitswapNotifyProgressEvents, BitswapWantProgressEvents } from 'ipfs-bitswap' import type { CID } from 'multiformats/cid' import type { ProgressEvent, ProgressOptions } from 'progress-events' @@ -12,32 +11,27 @@ export interface Pair { export type HasBlockProgressEvents = ProgressEvent<'blocks:put:duplicate', CID> | ProgressEvent<'blocks:put:providers:notify', CID> | - ProgressEvent<'blocks:put:blockstore:put', CID> | - BitswapNotifyProgressEvents + ProgressEvent<'blocks:put:blockstore:put', CID> export type PutBlockProgressEvents = ProgressEvent<'blocks:put:duplicate', CID> | ProgressEvent<'blocks:put:providers:notify', CID> | - ProgressEvent<'blocks:put:blockstore:put', CID> | - BitswapNotifyProgressEvents + ProgressEvent<'blocks:put:blockstore:put', CID> export type PutManyBlocksProgressEvents = ProgressEvent<'blocks:put-many:duplicate', CID> | ProgressEvent<'blocks:put-many:providers:notify', CID> | - ProgressEvent<'blocks:put-many:blockstore:put-many'> | - BitswapNotifyProgressEvents + ProgressEvent<'blocks:put-many:blockstore:put-many'> export type GetBlockProgressEvents = ProgressEvent<'blocks:get:providers:want', CID> | ProgressEvent<'blocks:get:blockstore:get', CID> | - ProgressEvent<'blocks:get:blockstore:put', CID> | - BitswapWantProgressEvents + ProgressEvent<'blocks:get:blockstore:put', CID> export type GetManyBlocksProgressEvents = ProgressEvent<'blocks:get-many:blockstore:get-many'> | ProgressEvent<'blocks:get-many:providers:want', CID> | - ProgressEvent<'blocks:get-many:blockstore:put', CID> | - BitswapWantProgressEvents + ProgressEvent<'blocks:get-many:blockstore:put', CID> export type GetAllBlocksProgressEvents = ProgressEvent<'blocks:get-all:blockstore:get-many'> diff --git a/packages/interface/src/index.ts b/packages/interface/src/index.ts index 1173ff06..3e62158a 100644 --- a/packages/interface/src/index.ts +++ b/packages/interface/src/index.ts @@ -16,7 +16,8 @@ import type { Blocks } from './blocks.js' import type { Pins } from './pins.js' -import type { Libp2p, AbortOptions, ComponentLogger } from '@libp2p/interface' +import type { Routing } from './routing.js' +import type { AbortOptions, ComponentLogger } from '@libp2p/interface' import type { Datastore } from 'interface-datastore' import type { CID } from 'multiformats/cid' import type { ProgressEvent, ProgressOptions } from 'progress-events' @@ -24,14 +25,9 @@ import type { ProgressEvent, ProgressOptions } from 'progress-events' export type { Await, AwaitIterable } from 'interface-store' /** - * The API presented by a Helia node. + * The API presented by a Helia node */ -export interface Helia { - /** - * The underlying libp2p node - */ - libp2p: T - +export interface Helia { /** * Where the blocks are stored */ @@ -52,6 +48,12 @@ export interface Helia { */ logger: ComponentLogger + /** + * The routing component allows performing operations such as looking up + * content providers, information about peers, etc. + */ + routing: Routing + /** * Starts the Helia node */ @@ -75,3 +77,15 @@ export type GcEvents = export interface GCOptions extends AbortOptions, ProgressOptions { } + +/** + * DAGWalkers take a block and yield CIDs encoded in that block + */ +export interface DAGWalker { + codec: number + walk(block: Uint8Array): Generator +} + +export * from './blocks.js' +export * from './pins.js' +export * from './routing.js' diff --git a/packages/interface/src/routing.ts b/packages/interface/src/routing.ts new file mode 100644 index 00000000..622f49b4 --- /dev/null +++ b/packages/interface/src/routing.ts @@ -0,0 +1,130 @@ +import type { AbortOptions, PeerId, PeerInfo } from '@libp2p/interface' +import type { CID } from 'multiformats/cid' +import type { ProgressOptions } from 'progress-events' + +/** + * When a routing operation involves reading values, these options allow + * controlling where the values are read from. Some implementations support a + * local cache that may be used in preference over network calls, for example + * when a record has a TTL. + */ +export interface RoutingOptions extends AbortOptions, ProgressOptions { + /** + * Pass `false` to not use the network + * + * @default true + */ + useNetwork?: boolean + + /** + * Pass `false` to not use cached values + * + * @default true + */ + useCache?: boolean +} + +/** + * A provider can supply the content for a CID + */ +export interface Provider extends PeerInfo { + /** + * If present these are the methods that the peer can supply the content via. + * + * If not present the caller should attempt to dial the remote peer and run + * the identify protocol to discover how to retrieve the content. + * + * Example values are (but not limited to): + * + * - transport-graphsync-filecoinv1 + * - transport-ipfs-gateway-http + * - transport-bitswap + */ + protocols?: string[] +} + +export interface Routing { + /** + * The implementation of this method should ensure that network peers know the + * caller can provide content that corresponds to the passed CID. + * + * @example + * + * ```js + * // ... + * await contentRouting.provide(cid) + * ``` + */ + provide(cid: CID, options?: RoutingOptions): Promise + + /** + * Find the providers of the passed CID. + * + * @example + * + * ```js + * // Iterate over the providers found for the given cid + * for await (const provider of contentRouting.findProviders(cid)) { + * console.log(provider.id, provider.multiaddrs) + * } + * ``` + */ + findProviders(cid: CID, options?: RoutingOptions): AsyncIterable + + /** + * Puts a value corresponding to the passed key in a way that can later be + * retrieved by another network peer using the get method. + * + * @example + * + * ```js + * // ... + * const key = '/key' + * const value = uint8ArrayFromString('oh hello there') + * + * await contentRouting.put(key, value) + * ``` + */ + put(key: Uint8Array, value: Uint8Array, options?: RoutingOptions): Promise + + /** + * Retrieves a value from the network corresponding to the passed key. + * + * @example + * + * ```js + * // ... + * + * const key = '/key' + * const value = await contentRouting.get(key) + * ``` + */ + get(key: Uint8Array, options?: RoutingOptions): Promise + + /** + * Searches the network for peer info corresponding to the passed peer id. + * + * @example + * + * ```js + * // ... + * const peer = await peerRouting.findPeer(peerId, options) + * ``` + */ + findPeer(peerId: PeerId, options?: RoutingOptions): Promise + + /** + * Search the network for peers that are closer to the passed key. Peer + * info should be yielded in ever-increasing closeness to the key. + * + * @example + * + * ```js + * // Iterate over the closest peers found for the given key + * for await (const peer of peerRouting.getClosestPeers(key)) { + * console.log(peer.id, peer.multiaddrs) + * } + * ``` + */ + getClosestPeers(key: Uint8Array, options?: RoutingOptions): AsyncIterable +} diff --git a/packages/interop/package.json b/packages/interop/package.json index e1c54776..f6f16880 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -53,11 +53,10 @@ "test:electron-main": "aegir test -t electron-main" }, "devDependencies": { - "@helia/block-brokers": "~1.0.0", + "@helia/block-brokers": "^1.0.0", "@helia/car": "^2.0.1", "@helia/dag-cbor": "^2.0.1", "@helia/dag-json": "^2.0.1", - "@helia/interface": "^3.0.1", "@helia/ipns": "^4.0.0", "@helia/json": "^2.0.1", "@helia/mfs": "^2.0.1", @@ -67,10 +66,9 @@ "@ipld/dag-cbor": "^9.0.7", "@libp2p/interface": "^1.1.1", "@libp2p/kad-dht": "^12.0.2", - "@libp2p/keychain": "^4.0.5", "@libp2p/peer-id": "^4.0.3", "@libp2p/peer-id-factory": "^4.0.3", - "@libp2p/websockets": "^8.0.1", + "@libp2p/websockets": "^8.0.10", "@multiformats/sha3": "^3.0.0", "aegir": "^42.1.0", "helia": "^3.0.1", diff --git a/packages/interop/test/car.spec.ts b/packages/interop/test/car.spec.ts index c31cf00d..a3123658 100644 --- a/packages/interop/test/car.spec.ts +++ b/packages/interop/test/car.spec.ts @@ -11,12 +11,12 @@ import { createHeliaNode } from './fixtures/create-helia.js' import { createKuboNode } from './fixtures/create-kubo.js' import { memoryCarWriter } from './fixtures/memory-car.js' import type { Car } from '@helia/car' -import type { Helia } from '@helia/interface' +import type { HeliaLibp2p } from 'helia' import type { FileCandidate } from 'ipfs-unixfs-importer' import type { Controller } from 'ipfsd-ctl' describe('@helia/car', () => { - let helia: Helia + let helia: HeliaLibp2p let c: Car let u: UnixFS let kubo: Controller diff --git a/packages/interop/test/dag-cbor.spec.ts b/packages/interop/test/dag-cbor.spec.ts index f6aa223b..8ef14105 100644 --- a/packages/interop/test/dag-cbor.spec.ts +++ b/packages/interop/test/dag-cbor.spec.ts @@ -6,12 +6,12 @@ import { expect } from 'aegir/chai' import { CID } from 'multiformats/cid' import { createHeliaNode } from './fixtures/create-helia.js' import { createKuboNode } from './fixtures/create-kubo.js' -import type { Helia } from '@helia/interface' +import type { HeliaLibp2p } from 'helia' import type { PutOptions as KuboAddOptions } from 'ipfs-core-types/src/block/index.js' import type { Controller } from 'ipfsd-ctl' describe('@helia/dag-cbor', () => { - let helia: Helia + let helia: HeliaLibp2p let d: DAGCBOR let kubo: Controller diff --git a/packages/interop/test/dag-json.spec.ts b/packages/interop/test/dag-json.spec.ts index 45b870b1..3ebc8dbe 100644 --- a/packages/interop/test/dag-json.spec.ts +++ b/packages/interop/test/dag-json.spec.ts @@ -6,12 +6,12 @@ import { CID } from 'multiformats/cid' import * as codec from 'multiformats/codecs/json' import { createHeliaNode } from './fixtures/create-helia.js' import { createKuboNode } from './fixtures/create-kubo.js' -import type { Helia } from '@helia/interface' +import type { HeliaLibp2p } from 'helia' import type { PutOptions as KuboAddOptions } from 'ipfs-core-types/src/block/index.js' import type { Controller } from 'ipfsd-ctl' describe('@helia/dag-json', () => { - let helia: Helia + let helia: HeliaLibp2p let d: DAGJSON let kubo: Controller diff --git a/packages/interop/test/fixtures/connect.ts b/packages/interop/test/fixtures/connect.ts index 38b5f0fe..2852bccc 100644 --- a/packages/interop/test/fixtures/connect.ts +++ b/packages/interop/test/fixtures/connect.ts @@ -1,11 +1,11 @@ import { expect } from 'aegir/chai' -import type { Helia } from '@helia/interface' +import type { HeliaLibp2p } from 'helia' import type { Controller } from 'ipfsd-ctl' /** * Connect the two nodes by dialing a protocol stream */ -export async function connect (helia: Helia, kubo: Controller, protocol: string): Promise { +export async function connect (helia: HeliaLibp2p, kubo: Controller, protocol: string): Promise { let connected = false for (const addr of kubo.peer.addresses) { try { diff --git a/packages/interop/test/fixtures/create-helia.browser.ts b/packages/interop/test/fixtures/create-helia.browser.ts index 49adc191..147739ee 100644 --- a/packages/interop/test/fixtures/create-helia.browser.ts +++ b/packages/interop/test/fixtures/create-helia.browser.ts @@ -5,11 +5,10 @@ import { webSockets } from '@libp2p/websockets' import { all } from '@libp2p/websockets/filters' import { sha3512 } from '@multiformats/sha3' import { createHelia, libp2pDefaults } from 'helia' -import type { Helia } from '@helia/interface' import type { Libp2p } from '@libp2p/interface' -import type { DefaultLibp2pServices } from 'helia' +import type { DefaultLibp2pServices, HeliaLibp2p } from 'helia' -export async function createHeliaNode (): Promise>> { +export async function createHeliaNode (): Promise>> { const defaults = libp2pDefaults() // allow dialing insecure WebSockets diff --git a/packages/interop/test/fixtures/create-helia.ts b/packages/interop/test/fixtures/create-helia.ts index b24a6d46..e7d43859 100644 --- a/packages/interop/test/fixtures/create-helia.ts +++ b/packages/interop/test/fixtures/create-helia.ts @@ -3,11 +3,10 @@ import { ipnsValidator, ipnsSelector } from '@helia/ipns' import { kadDHT, removePublicAddressesMapper } from '@libp2p/kad-dht' import { sha3512 } from '@multiformats/sha3' import { createHelia, libp2pDefaults } from 'helia' -import type { Helia } from '@helia/interface' import type { Libp2p } from '@libp2p/interface' -import type { DefaultLibp2pServices } from 'helia' +import type { DefaultLibp2pServices, HeliaLibp2p } from 'helia' -export async function createHeliaNode (): Promise>> { +export async function createHeliaNode (): Promise>> { const defaults = libp2pDefaults() defaults.addresses = { listen: [ diff --git a/packages/interop/test/helia-blockstore.spec.ts b/packages/interop/test/helia-blockstore.spec.ts index 401e4c73..1832d77c 100644 --- a/packages/interop/test/helia-blockstore.spec.ts +++ b/packages/interop/test/helia-blockstore.spec.ts @@ -8,11 +8,11 @@ import * as raw from 'multiformats/codecs/raw' import { sha256 } from 'multiformats/hashes/sha2' import { createHeliaNode } from './fixtures/create-helia.js' import { createKuboNode } from './fixtures/create-kubo.js' -import type { Helia } from '@helia/interface' +import type { HeliaLibp2p } from 'helia' import type { Controller } from 'ipfsd-ctl' describe('helia - blockstore', () => { - let helia: Helia + let helia: HeliaLibp2p let kubo: Controller beforeEach(async () => { diff --git a/packages/interop/test/helia-hashes.spec.ts b/packages/interop/test/helia-hashes.spec.ts index ef1ff42d..640e13e1 100644 --- a/packages/interop/test/helia-hashes.spec.ts +++ b/packages/interop/test/helia-hashes.spec.ts @@ -8,11 +8,11 @@ import { CID } from 'multiformats/cid' import * as raw from 'multiformats/codecs/raw' import { createHeliaNode } from './fixtures/create-helia.js' import { createKuboNode } from './fixtures/create-kubo.js' -import type { Helia } from '@helia/interface' +import type { HeliaLibp2p } from 'helia' import type { Controller } from 'ipfsd-ctl' describe('helia - hashes', () => { - let helia: Helia + let helia: HeliaLibp2p let kubo: Controller beforeEach(async () => { diff --git a/packages/interop/test/helia-pins.spec.ts b/packages/interop/test/helia-pins.spec.ts index f798ea06..57972fae 100644 --- a/packages/interop/test/helia-pins.spec.ts +++ b/packages/interop/test/helia-pins.spec.ts @@ -8,11 +8,11 @@ import * as raw from 'multiformats/codecs/raw' import { sha256 } from 'multiformats/hashes/sha2' import { createHeliaNode } from './fixtures/create-helia.js' import { createKuboNode } from './fixtures/create-kubo.js' -import type { Helia } from '@helia/interface' +import type { HeliaLibp2p } from 'helia' import type { Controller } from 'ipfsd-ctl' describe('helia - pins', () => { - let helia: Helia + let helia: HeliaLibp2p let kubo: Controller beforeEach(async () => { diff --git a/packages/interop/test/ipns-libp2p.spec.ts b/packages/interop/test/ipns-libp2p.spec.ts index a71c8055..d333371c 100644 --- a/packages/interop/test/ipns-libp2p.spec.ts +++ b/packages/interop/test/ipns-libp2p.spec.ts @@ -18,16 +18,14 @@ import { createKuboNode } from './fixtures/create-kubo.js' import { sortClosestPeers } from './fixtures/create-peer-ids.js' import { keyTypes } from './fixtures/key-types.js' import { waitFor } from './fixtures/wait-for.js' -import type { Helia } from '@helia/interface' import type { IPNS } from '@helia/ipns' -import type { Libp2p, PeerId } from '@libp2p/interface' -import type { KadDHT } from '@libp2p/kad-dht' -import type { Keychain } from '@libp2p/keychain' +import type { PeerId } from '@libp2p/interface' +import type { HeliaLibp2p } from 'helia' import type { Controller } from 'ipfsd-ctl' keyTypes.forEach(type => { describe(`@helia/ipns - libp2p routing with ${type} keys`, () => { - let helia: Helia> + let helia: HeliaLibp2p let kubo: Controller let name: IPNS diff --git a/packages/interop/test/ipns-pubsub.spec.ts b/packages/interop/test/ipns-pubsub.spec.ts index 488df145..5c66962e 100644 --- a/packages/interop/test/ipns-pubsub.spec.ts +++ b/packages/interop/test/ipns-pubsub.spec.ts @@ -19,10 +19,8 @@ import { createHeliaNode } from './fixtures/create-helia.js' import { createKuboNode } from './fixtures/create-kubo.js' import { keyTypes } from './fixtures/key-types.js' import { waitFor } from './fixtures/wait-for.js' -import type { Helia } from '@helia/interface' import type { IPNS } from '@helia/ipns' -import type { Libp2p, PubSub } from '@libp2p/interface' -import type { Keychain } from '@libp2p/keychain' +import type { HeliaLibp2p } from 'helia' import type { Controller } from 'ipfsd-ctl' const LIBP2P_KEY_CODEC = 0x72 @@ -32,7 +30,7 @@ const LIBP2P_KEY_CODEC = 0x72 // resolution because Kubo will use the DHT as well keyTypes.filter(keyType => keyType !== 'RSA').forEach(keyType => { describe(`@helia/ipns - pubsub routing with ${keyType} keys`, () => { - let helia: Helia> + let helia: HeliaLibp2p let kubo: Controller let name: IPNS diff --git a/packages/interop/test/json.spec.ts b/packages/interop/test/json.spec.ts index 9cb702c0..7f65af5f 100644 --- a/packages/interop/test/json.spec.ts +++ b/packages/interop/test/json.spec.ts @@ -6,12 +6,12 @@ import { CID } from 'multiformats/cid' import * as jsonCodec from 'multiformats/codecs/json' import { createHeliaNode } from './fixtures/create-helia.js' import { createKuboNode } from './fixtures/create-kubo.js' -import type { Helia } from '@helia/interface' +import type { HeliaLibp2p } from 'helia' import type { PutOptions as KuboAddOptions } from 'ipfs-core-types/src/block/index.js' import type { Controller } from 'ipfsd-ctl' describe('@helia/json', () => { - let helia: Helia + let helia: HeliaLibp2p let j: JSON let kubo: Controller diff --git a/packages/interop/test/mfs.spec.ts b/packages/interop/test/mfs.spec.ts index 58b48d78..5e4c9097 100644 --- a/packages/interop/test/mfs.spec.ts +++ b/packages/interop/test/mfs.spec.ts @@ -4,11 +4,11 @@ import { type MFS, mfs } from '@helia/mfs' import { expect } from 'aegir/chai' import { createHeliaNode } from './fixtures/create-helia.js' import { createKuboNode } from './fixtures/create-kubo.js' -import type { Helia } from '@helia/interface' +import type { HeliaLibp2p } from 'helia' import type { Controller } from 'ipfsd-ctl' describe('@helia/mfs', () => { - let helia: Helia + let helia: HeliaLibp2p let fs: MFS let kubo: Controller diff --git a/packages/interop/test/strings.spec.ts b/packages/interop/test/strings.spec.ts index 66f8bcdb..6580b67e 100644 --- a/packages/interop/test/strings.spec.ts +++ b/packages/interop/test/strings.spec.ts @@ -7,12 +7,12 @@ import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' import { toString as uint8ArrayToString } from 'uint8arrays/to-string' import { createHeliaNode } from './fixtures/create-helia.js' import { createKuboNode } from './fixtures/create-kubo.js' -import type { Helia } from '@helia/interface' +import type { HeliaLibp2p } from 'helia' import type { PutOptions as KuboAddOptions } from 'ipfs-core-types/src/block/index.js' import type { Controller } from 'ipfsd-ctl' describe('@helia/strings', () => { - let helia: Helia + let helia: HeliaLibp2p let str: Strings let kubo: Controller diff --git a/packages/interop/test/unixfs-bitswap.spec.ts b/packages/interop/test/unixfs-bitswap.spec.ts index 2e13a529..e25206a0 100644 --- a/packages/interop/test/unixfs-bitswap.spec.ts +++ b/packages/interop/test/unixfs-bitswap.spec.ts @@ -6,13 +6,12 @@ import toBuffer from 'it-to-buffer' import { CID } from 'multiformats/cid' import { createHeliaNode } from './fixtures/create-helia.js' import { createKuboNode } from './fixtures/create-kubo.js' -import type { Helia } from '@helia/interface' -import type { Libp2p } from '@libp2p/interface' +import type { HeliaLibp2p } from 'helia' import type { FileCandidate } from 'ipfs-unixfs-importer' import type { Controller } from 'ipfsd-ctl' describe('@helia/unixfs - bitswap', () => { - let helia: Helia + let helia: HeliaLibp2p let unixFs: UnixFS let kubo: Controller diff --git a/packages/interop/test/unixfs-files.spec.ts b/packages/interop/test/unixfs-files.spec.ts index 45dd3d83..8bc433ad 100644 --- a/packages/interop/test/unixfs-files.spec.ts +++ b/packages/interop/test/unixfs-files.spec.ts @@ -7,14 +7,13 @@ import { balanced } from 'ipfs-unixfs-importer/layout' import { CID } from 'multiformats/cid' import { createHeliaNode } from './fixtures/create-helia.js' import { createKuboNode } from './fixtures/create-kubo.js' -import type { Helia } from '@helia/interface' -import type { Libp2p } from '@libp2p/interface' +import type { HeliaLibp2p } from 'helia' import type { AddOptions as KuboAddOptions } from 'ipfs-core-types/src/root.js' import type { FileCandidate } from 'ipfs-unixfs-importer' import type { Controller } from 'ipfsd-ctl' describe('@helia/unixfs - files', () => { - let helia: Helia + let helia: HeliaLibp2p let unixFs: UnixFS let kubo: Controller diff --git a/packages/ipns/package.json b/packages/ipns/package.json index 0f3865ef..b170da4b 100644 --- a/packages/ipns/package.json +++ b/packages/ipns/package.json @@ -170,7 +170,7 @@ "dns-over-http-resolver": "^3.0.0", "dns-packet": "^5.6.0", "hashlru": "^2.3.0", - "interface-datastore": "^8.2.2", + "interface-datastore": "^8.2.9", "ipns": "^8.0.0", "is-ipfs": "^8.0.1", "multiformats": "^13.0.0", @@ -182,7 +182,7 @@ "@libp2p/peer-id-factory": "^4.0.3", "@types/dns-packet": "^5.6.4", "aegir": "^42.1.0", - "datastore-core": "^9.2.0", + "datastore-core": "^9.2.7", "sinon": "^17.0.1", "sinon-ts": "^2.0.0" }, diff --git a/packages/json/package.json b/packages/json/package.json index aec71584..7fe203d5 100644 --- a/packages/json/package.json +++ b/packages/json/package.json @@ -146,7 +146,7 @@ }, "devDependencies": { "aegir": "^42.1.0", - "blockstore-core": "^4.3.8", + "blockstore-core": "^4.3.10", "interface-blockstore": "^5.2.9" } } diff --git a/packages/mfs/package.json b/packages/mfs/package.json index 5a950d85..4d6f456e 100644 --- a/packages/mfs/package.json +++ b/packages/mfs/package.json @@ -143,7 +143,7 @@ "@helia/unixfs": "^2.0.1", "@libp2p/interfaces": "^3.3.1", "@libp2p/logger": "^4.0.4", - "interface-datastore": "^8.2.2", + "interface-datastore": "^8.2.9", "ipfs-unixfs": "^11.0.0", "ipfs-unixfs-exporter": "^13.1.0", "ipfs-unixfs-importer": "^15.1.0", @@ -152,8 +152,8 @@ "devDependencies": { "@ipld/dag-pb": "^4.0.7", "aegir": "^42.1.0", - "blockstore-core": "^4.3.8", - "datastore-core": "^9.2.0", + "blockstore-core": "^4.3.10", + "datastore-core": "^9.2.7", "delay": "^6.0.0", "interface-blockstore": "^5.2.9", "it-all": "^3.0.4", diff --git a/packages/routers/LICENSE b/packages/routers/LICENSE new file mode 100644 index 00000000..20ce483c --- /dev/null +++ b/packages/routers/LICENSE @@ -0,0 +1,4 @@ +This project is dual licensed under MIT and Apache-2.0. + +MIT: https://www.opensource.org/licenses/mit +Apache-2.0: https://www.apache.org/licenses/license-2.0 diff --git a/packages/routers/LICENSE-APACHE b/packages/routers/LICENSE-APACHE new file mode 100644 index 00000000..14478a3b --- /dev/null +++ b/packages/routers/LICENSE-APACHE @@ -0,0 +1,5 @@ +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. diff --git a/packages/routers/LICENSE-MIT b/packages/routers/LICENSE-MIT new file mode 100644 index 00000000..72dc60d8 --- /dev/null +++ b/packages/routers/LICENSE-MIT @@ -0,0 +1,19 @@ +The MIT License (MIT) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/packages/routers/README.md b/packages/routers/README.md new file mode 100644 index 00000000..22378715 --- /dev/null +++ b/packages/routers/README.md @@ -0,0 +1,53 @@ +

+ + Helia logo + +

+ +[![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) +[![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) +[![codecov](https://img.shields.io/codecov/c/github/ipfs/helia.svg?style=flat-square)](https://codecov.io/gh/ipfs/helia) +[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/helia/main.yml?branch=main\&style=flat-square)](https://github.com/ipfs/helia/actions/workflows/main.yml?query=branch%3Amain) + +> Routers for Helia + +# About + +Abstraction layer over different content and peer routing mechanisms. + +# Install + +```console +$ npm i @helia/routers +``` + +## Browser ` +``` + +# API Docs + +- + +# License + +Licensed under either of + +- Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) +- MIT ([LICENSE-MIT](LICENSE-MIT) / ) + +# Contribute + +Contributions welcome! Please check out [the issues](https://github.com/ipfs/helia/issues). + +Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. + +Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). + +Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. + +[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) diff --git a/packages/routers/package.json b/packages/routers/package.json new file mode 100644 index 00000000..54908e24 --- /dev/null +++ b/packages/routers/package.json @@ -0,0 +1,73 @@ +{ + "name": "@helia/routers", + "version": "0.0.0", + "description": "Routers for Helia", + "license": "Apache-2.0 OR MIT", + "homepage": "https://github.com/ipfs/helia/tree/main/packages/routers#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/ipfs/helia.git" + }, + "bugs": { + "url": "https://github.com/ipfs/helia/issues" + }, + "publishConfig": { + "access": "public", + "provenance": true + }, + "keywords": [ + "IPFS" + ], + "type": "module", + "types": "./dist/src/index.d.ts", + "files": [ + "src", + "dist", + "!dist/test", + "!**/*.tsbuildinfo" + ], + "exports": { + ".": { + "types": "./dist/src/index.d.ts", + "import": "./dist/src/index.js" + } + }, + "eslintConfig": { + "extends": "ipfs", + "parserOptions": { + "project": true, + "sourceType": "module" + } + }, + "scripts": { + "clean": "aegir clean", + "lint": "aegir lint", + "dep-check": "aegir dep-check", + "build": "aegir build", + "test": "aegir test", + "test:chrome": "aegir test -t browser --cov", + "test:chrome-webworker": "aegir test -t webworker", + "test:firefox": "aegir test -t browser -- --browser firefox", + "test:firefox-webworker": "aegir test -t webworker -- --browser firefox", + "test:node": "aegir test -t node --cov", + "test:electron-main": "aegir test -t electron-main" + }, + "dependencies": { + "@helia/delegated-routing-v1-http-api-client": "^2.0.2", + "@helia/interface": "^3.0.1", + "@libp2p/interface": "^1.1.1", + "@libp2p/peer-id": "^4.0.5", + "ipns": "^8.0.3", + "it-first": "^3.0.4", + "it-map": "^3.0.5", + "multiformats": "^13.0.0", + "uint8arrays": "^5.0.1" + }, + "devDependencies": { + "@libp2p/peer-id-factory": "^4.0.5", + "aegir": "^42.1.0", + "it-drain": "^3.0.5", + "sinon-ts": "^2.0.0" + }, + "sideEffects": false +} diff --git a/packages/routers/src/delegated-http-routing.ts b/packages/routers/src/delegated-http-routing.ts new file mode 100644 index 00000000..1c76fdf9 --- /dev/null +++ b/packages/routers/src/delegated-http-routing.ts @@ -0,0 +1,101 @@ +import { createDelegatedRoutingV1HttpApiClient } from '@helia/delegated-routing-v1-http-api-client' +import { CodeError } from '@libp2p/interface' +import { peerIdFromBytes } from '@libp2p/peer-id' +import { marshal, unmarshal } from 'ipns' +import first from 'it-first' +import map from 'it-map' +import { equals as uint8ArrayEquals } from 'uint8arrays/equals' +import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' +import type { DelegatedRoutingV1HttpApiClient } from '@helia/delegated-routing-v1-http-api-client' +import type { Provider, Routing, RoutingOptions } from '@helia/interface' +import type { PeerId, PeerInfo } from '@libp2p/interface' +import type { CID, Version } from 'multiformats' + +const IPNS_PREFIX = uint8ArrayFromString('/ipns/') + +function isIPNSKey (key: Uint8Array): boolean { + return uint8ArrayEquals(key.subarray(0, IPNS_PREFIX.byteLength), IPNS_PREFIX) +} + +const peerIdFromRoutingKey = (key: Uint8Array): PeerId => { + return peerIdFromBytes(key.slice(IPNS_PREFIX.length)) +} + +class DelegatedHTTPRouter implements Routing { + private readonly client: DelegatedRoutingV1HttpApiClient + + constructor (url: URL) { + this.client = createDelegatedRoutingV1HttpApiClient(url) + } + + async provide (cid: CID, options?: RoutingOptions | undefined): Promise { + // noop + } + + async * findProviders (cid: CID, options?: RoutingOptions | undefined): AsyncIterable { + yield * map(this.client.getProviders(cid, options), (record) => { + return { + id: record.ID, + multiaddrs: record.Addrs, + protocols: record.Protocols + } + }) + } + + async put (key: Uint8Array, value: Uint8Array, options?: RoutingOptions | undefined): Promise { + if (!isIPNSKey(key)) { + return + } + + const peerId = peerIdFromRoutingKey(key) + const record = unmarshal(value) + + await this.client.putIPNS(peerId, record, options) + } + + async get (key: Uint8Array, options?: RoutingOptions | undefined): Promise { + if (!isIPNSKey(key)) { + throw new CodeError('Not found', 'ERR_NOT_FOUND') + } + + const peerId = peerIdFromRoutingKey(key) + + try { + const record = await this.client.getIPNS(peerId, options) + + return marshal(record) + } catch (err: any) { + // ERR_BAD_RESPONSE is thrown when the response had no body, which means + // the record couldn't be found + if (err.code === 'ERR_BAD_RESPONSE') { + throw new CodeError('Not found', 'ERR_NOT_FOUND') + } + + throw err + } + } + + async findPeer (peerId: PeerId, options?: RoutingOptions | undefined): Promise { + const peer = await first(this.client.getPeers(peerId, options)) + + if (peer != null) { + return { + id: peer.ID, + multiaddrs: peer.Addrs ?? [] + } + } + + throw new CodeError('Not found', 'ERR_NOT_FOUND') + } + + async * getClosestPeers (key: Uint8Array, options?: RoutingOptions | undefined): AsyncIterable { + // noop + } +} + +/** + * Creates a Helia Router that connects to an endpoint that supports the [Delegated Routing V1 HTTP API](https://specs.ipfs.tech/routing/http-routing-v1/) spec. + */ +export function delegatedHTTPRouting (url: string | URL): Routing { + return new DelegatedHTTPRouter(new URL(url)) +} diff --git a/packages/routers/src/index.ts b/packages/routers/src/index.ts new file mode 100644 index 00000000..c82e8cbd --- /dev/null +++ b/packages/routers/src/index.ts @@ -0,0 +1,7 @@ +/** + * @packageDocumentation + * + * Abstraction layer over different content and peer routing mechanisms. + */ +export { delegatedHTTPRouting } from './delegated-http-routing.js' +export { libp2pRouting } from './libp2p-routing.js' diff --git a/packages/routers/src/libp2p-routing.ts b/packages/routers/src/libp2p-routing.ts new file mode 100644 index 00000000..935c86d4 --- /dev/null +++ b/packages/routers/src/libp2p-routing.ts @@ -0,0 +1,39 @@ +import type { Provider, Routing, RoutingOptions } from '@helia/interface' +import type { Libp2p, PeerId, PeerInfo } from '@libp2p/interface' +import type { CID } from 'multiformats' + +class Libp2pRouter implements Routing { + private readonly libp2p: Libp2p + + constructor (libp2p: Libp2p) { + this.libp2p = libp2p + } + + async provide (cid: CID, options?: RoutingOptions): Promise { + await this.libp2p.contentRouting.provide(cid, options) + } + + async * findProviders (cid: CID, options?: RoutingOptions): AsyncIterable { + yield * this.libp2p.contentRouting.findProviders(cid, options) + } + + async put (key: Uint8Array, value: Uint8Array, options?: RoutingOptions): Promise { + await this.libp2p.contentRouting.put(key, value, options) + } + + async get (key: Uint8Array, options?: RoutingOptions): Promise { + return this.libp2p.contentRouting.get(key, options) + } + + async findPeer (peerId: PeerId, options?: RoutingOptions): Promise { + return this.libp2p.peerRouting.findPeer(peerId, options) + } + + async * getClosestPeers (key: Uint8Array, options?: RoutingOptions): AsyncIterable { + yield * this.libp2p.peerRouting.getClosestPeers(key, options) + } +} + +export function libp2pRouting (libp2p: Libp2p): Routing { + return new Libp2pRouter(libp2p) +} diff --git a/packages/routers/test/delegated-http-routing.spec.ts b/packages/routers/test/delegated-http-routing.spec.ts new file mode 100644 index 00000000..c8443777 --- /dev/null +++ b/packages/routers/test/delegated-http-routing.spec.ts @@ -0,0 +1,117 @@ +import { peerIdFromString } from '@libp2p/peer-id' +import { createEd25519PeerId } from '@libp2p/peer-id-factory' +import { expect } from 'aegir/chai' +import { peerIdToRoutingKey, create, marshal } from 'ipns' +import drain from 'it-drain' +import { CID } from 'multiformats' +import { stubInterface } from 'sinon-ts' +import { delegatedHTTPRouting } from '../src/index.js' +import type { DelegatedRoutingV1HttpApiClient, PeerRecord } from '@helia/delegated-routing-v1-http-api-client' +import type { Routing } from '@helia/interface' +import type { StubbedInstance } from 'sinon-ts' + +describe('delegated-http-routing', () => { + let client: StubbedInstance + let router: Routing + + beforeEach(() => { + client = stubInterface() + + router = delegatedHTTPRouting('http://127.0.0.1') + // @ts-expect-error private field + router.client = client + }) + + it.skip('should provide', async () => { + // this is a no-op + }) + + it('should find providers', async () => { + const cid = CID.parse('bafyreidykglsfhoixmivffc5uwhcgshx4j465xwqntbmu43nb2dzqwfvae') + const options = {} + + const providers: PeerRecord[] = [{ + ID: peerIdFromString('12D3KooWPPMkhpoWGA7WRUL8jDGduGT486aE3hHEf6sfDq8hTaFJ'), + Schema: 'peer', + Protocols: ['transport-bitswap'], + Addrs: [] + }] + + client.getProviders.returns(async function * () { + yield * providers + }()) + + await drain(router.findProviders(cid, options)) + + expect(client.getProviders.calledWith(cid, options)).to.be.true() + }) + + it('should put a IPNS record value', async () => { + const peerId = await createEd25519PeerId() + const key = peerIdToRoutingKey(peerId) + const record = await create(peerId, '/hello world', 0, 100) + const value = marshal(record) + const options = {} + + await router.put(key, value, options) + + expect(client.putIPNS.called).to.be.true() + }) + + it('should not put a non-IPNS record value', async () => { + const key = Uint8Array.from([0, 1, 2, 3, 4]) + const value = Uint8Array.from([5, 6, 7, 8, 9]) + const options = {} + + await router.put(key, value, options) + + expect(client.putIPNS.called).to.be.false() + }) + + it('should get a IPNS record value', async () => { + const peerId = await createEd25519PeerId() + const key = peerIdToRoutingKey(peerId) + const record = await create(peerId, '/hello world', 0, 100) + const options = {} + + client.getIPNS.resolves(record) + + await router.get(key, options) + + expect(client.getIPNS.called).to.be.true() + }) + + it('should not get a non-IPNS record value', async () => { + const key = Uint8Array.from([0, 1, 2, 3, 4]) + const options = {} + + await expect(router.get(key, options)).to.eventually.be.rejected + .with.property('code', 'ERR_NOT_FOUND') + + expect(client.getIPNS.called).to.be.false() + }) + + it('should find a peer', async () => { + const peerId = peerIdFromString('12D3KooWPPMkhpoWGA7WRUL8jDGduGT486aE3hHEf6sfDq8hTaFJ') + const options = {} + + const peers: PeerRecord[] = [{ + ID: peerId, + Schema: 'peer', + Protocols: ['transport-bitswap'], + Addrs: [] + }] + + client.getPeers.returns(async function * () { + yield * peers + }()) + + await router.findPeer(peerId, options) + + expect(client.getPeers.calledWith(peerId, options)).to.be.true() + }) + + it.skip('should get closest peers', async () => { + // this is a no-op + }) +}) diff --git a/packages/routers/test/libp2p-routing.spec.ts b/packages/routers/test/libp2p-routing.spec.ts new file mode 100644 index 00000000..d7c916d8 --- /dev/null +++ b/packages/routers/test/libp2p-routing.spec.ts @@ -0,0 +1,85 @@ +import { peerIdFromString } from '@libp2p/peer-id' +import { expect } from 'aegir/chai' +import drain from 'it-drain' +import { CID } from 'multiformats' +import { stubInterface, type StubbedInstance } from 'sinon-ts' +import { libp2pRouting } from '../src/index.js' +import type { Routing } from '@helia/interface' +import type { ContentRouting, Libp2p, PeerRouting } from '@libp2p/interface' + +describe('libp2p-routing', () => { + let libp2p: StubbedInstance + let contentRouting: StubbedInstance + let peerRouting: StubbedInstance + let router: Routing + + beforeEach(() => { + contentRouting = stubInterface() + peerRouting = stubInterface() + libp2p = stubInterface({ + contentRouting, + peerRouting + }) + + router = libp2pRouting(libp2p) + }) + + it('should call through to contentRouting.provide', async () => { + const cid = CID.parse('bafyreidykglsfhoixmivffc5uwhcgshx4j465xwqntbmu43nb2dzqwfvae') + const options = {} + + await router.provide(cid, options) + + expect(contentRouting.provide.calledWith(cid, options)).to.be.true() + }) + + it('should call through to contentRouting.findProviders', async () => { + contentRouting.findProviders.returns(async function * () {}()) + + const cid = CID.parse('bafyreidykglsfhoixmivffc5uwhcgshx4j465xwqntbmu43nb2dzqwfvae') + const options = {} + + await drain(router.findProviders(cid, options)) + + expect(contentRouting.findProviders.calledWith(cid, options)).to.be.true() + }) + + it('should call through to contentRouting.put', async () => { + const key = Uint8Array.from([0, 1, 2, 3, 4]) + const value = Uint8Array.from([5, 6, 7, 8, 9]) + const options = {} + + await router.put(key, value, options) + + expect(contentRouting.put.calledWith(key, value, options)).to.be.true() + }) + + it('should call through to contentRouting.get', async () => { + const key = Uint8Array.from([0, 1, 2, 3, 4]) + const options = {} + + await router.get(key, options) + + expect(contentRouting.get.calledWith(key, options)).to.be.true() + }) + + it('should call through to peerRouting.findPeer', async () => { + const peerId = peerIdFromString('12D3KooWPPMkhpoWGA7WRUL8jDGduGT486aE3hHEf6sfDq8hTaFJ') + const options = {} + + await router.findPeer(peerId, options) + + expect(peerRouting.findPeer.calledWith(peerId, options)).to.be.true() + }) + + it('should call through to peerRouting.getClosestPeers', async () => { + peerRouting.getClosestPeers.returns(async function * () {}()) + + const key = Uint8Array.from([0, 1, 2, 3, 4]) + const options = {} + + await drain(router.getClosestPeers(key, options)) + + expect(peerRouting.getClosestPeers.calledWith(key, options)).to.be.true() + }) +}) diff --git a/packages/routers/tsconfig.json b/packages/routers/tsconfig.json new file mode 100644 index 00000000..4c0bdf77 --- /dev/null +++ b/packages/routers/tsconfig.json @@ -0,0 +1,15 @@ +{ + "extends": "aegir/src/config/tsconfig.aegir.json", + "compilerOptions": { + "outDir": "dist" + }, + "include": [ + "src", + "test" + ], + "references": [ + { + "path": "../interface" + } + ] +} diff --git a/packages/routers/typedoc.json b/packages/routers/typedoc.json new file mode 100644 index 00000000..f599dc72 --- /dev/null +++ b/packages/routers/typedoc.json @@ -0,0 +1,5 @@ +{ + "entryPoints": [ + "./src/index.ts" + ] +} diff --git a/packages/strings/package.json b/packages/strings/package.json index 8ec8f1e4..b33323d4 100644 --- a/packages/strings/package.json +++ b/packages/strings/package.json @@ -149,7 +149,7 @@ }, "devDependencies": { "aegir": "^42.1.0", - "blockstore-core": "^4.3.8", + "blockstore-core": "^4.3.10", "interface-blockstore": "^5.2.9" } } diff --git a/packages/unixfs/package.json b/packages/unixfs/package.json index 99db98aa..1edfd93b 100644 --- a/packages/unixfs/package.json +++ b/packages/unixfs/package.json @@ -160,7 +160,7 @@ }, "dependencies": { "@helia/interface": "^3.0.1", - "@ipld/dag-pb": "^4.0.3", + "@ipld/dag-pb": "^4.0.6", "@libp2p/interface": "^1.1.1", "@libp2p/logger": "^4.0.4", "@multiformats/murmur3": "^2.1.2", @@ -179,7 +179,7 @@ }, "devDependencies": { "aegir": "^42.1.0", - "blockstore-core": "^4.3.8", + "blockstore-core": "^4.3.10", "delay": "^6.0.0", "interface-blockstore": "^5.2.9", "iso-url": "^1.2.1",