From 9c6090975521ae517b570f9479f696acf2d5371b Mon Sep 17 00:00:00 2001 From: David Dias Date: Mon, 2 Jul 2018 14:38:14 +0100 Subject: [PATCH] feat: (BREAKING CHANGE) new libp2p configuration (#1401) BREAKING CHANGE: libp2p configuration has changed * old: `libp2p.modules.discovery` * new: `libp2p.modules.peerDiscovery` License: MIT Signed-off-by: David Dias License: MIT Signed-off-by: Alan Shaw --- README.md | 20 +++++--- examples/circuit-relaying/README.md | 40 +++++++-------- package.json | 21 ++++---- src/core/components/libp2p.js | 55 ++++++++++++++------- src/core/config.js | 7 +++ src/core/index.js | 4 -- src/core/runtime/libp2p-browser.js | 68 +++++++++++++++---------- src/core/runtime/libp2p-nodejs.js | 77 ++++++++++++++++------------- src/http/index.js | 8 ++- test/core/circuit-relay.js | 14 +++--- test/core/config.spec.js | 24 +++++++++ 11 files changed, 208 insertions(+), 130 deletions(-) diff --git a/README.md b/README.md index af89084433..cf4d930802 100644 --- a/README.md +++ b/README.md @@ -224,22 +224,28 @@ Creates and returns an instance of an IPFS node. Use the `options` argument to s - `pass` (string): A passphrase to encrypt/decrypt your keys. +- `relay` (object): Configure circuit relay (see the [circuit relay tutorial](https://github.com/ipfs/js-ipfs/tree/master/examples/circuit-relaying) to learn more). + - `enabled` (boolean): Enable circuit relay dialer and listener. (Default: `false`) + - `hop` (object) + - `enabled` (boolean): Make this node a relay (other nodes can connect *through* it). (Default: `false`) + - `active` (boolean): Make this an *active* relay node. Active relay nodes will attempt to dial a destination peer even if that peer is not yet connected to the relay. (Default: `false`) + - `EXPERIMENTAL` (object): Enable and configure experimental features. - `pubsub` (boolean): Enable libp2p pub-sub. (Default: `false`) - `sharding` (boolean): Enable directory sharding. Directories that have many child objects will be represented by multiple DAG nodes instead of just one. It can improve lookup performance when a directory has several thousand files or more. (Default: `false`) - `dht` (boolean): Enable KadDHT. **This is currently not interopable with `go-ipfs`.** - - `relay` (object): Configure circuit relay (see the [circuit relay tutorial](https://github.com/ipfs/js-ipfs/tree/master/examples/circuit-relaying) to learn more). - - `enabled` (boolean): Enable circuit relay dialer and listener. (Default: `false`) - - `hop` (object) - - `enabled` (boolean): Make this node a relay (other nodes can connect *through* it). (Default: `false`) - - `active` (boolean): Make this an *active* relay node. Active relay nodes will attempt to dial a destination peer even if that peer is not yet connected to the relay. (Default: `false`) - `config` (object) Modify the default IPFS node config. Find the Node.js defaults at [`src/core/runtime/config-nodejs.js`](https://github.com/ipfs/js-ipfs/tree/master/src/core/runtime/config-nodejs.js) and the browser defaults at [`src/core/runtime/config-browser.js`](https://github.com/ipfs/js-ipfs/tree/master/src/core/runtime/config-browser.js). This object will be *merged* with the default config; it will not replace it. - `libp2p` (object) add custom modules to the libp2p stack of your node - `modules` (object): - - `transport` (Array<[libp2p.Transport](https://github.com/libp2p/interface-transport)>): An array of additional Libp2p transport instances to use. See [libp2p/interface-transport](https://github.com/libp2p/interface-transport) for details. - - `discovery` (Array<[libp2p.PeerDiscovery](https://github.com/libp2p/interface-peer-discovery)>): An array of additional Libp2p peer discovery instances to use. See [libp2p/peer-discovery](https://github.com/libp2p/interface-peer-discovery) for details. + - `transport` (Array<[libp2p.Transport](https://github.com/libp2p/interface-transport)>): An array of Libp2p transport classes/instances to use _instead_ of the defaults. See [libp2p/interface-transport](https://github.com/libp2p/interface-transport) for details. + - `peerDiscovery` (Array<[libp2p.PeerDiscovery](https://github.com/libp2p/interface-peer-discovery)>): An array of Libp2p peer discovery classes/instances to use _instead_ of the defaults. See [libp2p/peer-discovery](https://github.com/libp2p/interface-peer-discovery) for details. If passing a class, configuration can be passed using the config section below under the key corresponding to you module's unique `tag` (a static property on the class) + - `config` (object): + - `peerDiscovery` (object): + - `[PeerDiscovery.tag]` (object): configuration for a peer discovery module + - `enabled` (boolean): whether this module is enabled or disabled + - `[custom config]` (any): other keys are specific to the module #### Events diff --git a/examples/circuit-relaying/README.md b/examples/circuit-relaying/README.md index e04faf456e..aaf6fd592f 100644 --- a/examples/circuit-relaying/README.md +++ b/examples/circuit-relaying/README.md @@ -22,7 +22,7 @@ Here is a simple diagram depicting how a typical circuit-relay connection might +---------------------+ ``` -`Node A` tries to connect to `Node B` but, UH-OH! There is a firewall in between that's preventing it from happening. If both `Node A` and `Node B` know about a relay, they can use it to establish the connection. +`Node A` tries to connect to `Node B` but, UH-OH! There is a firewall in between that's preventing it from happening. If both `Node A` and `Node B` know about a relay, they can use it to establish the connection. This is what it looks like, in simplified steps: @@ -60,7 +60,7 @@ There are a couple of caveats and features to be aware of: #### A word on circuit relay addresses -A circuit relay address is a [multiaddress](https://multiformats.io/multiaddr/) that describes how to either connect to a peer over a relay (or relays), or allow a peer to announce it is reachable over a particular relay or any relay it is already connected to. +A circuit relay address is a [multiaddress](https://multiformats.io/multiaddr/) that describes how to either connect to a peer over a relay (or relays), or allow a peer to announce it is reachable over a particular relay or any relay it is already connected to. Circuit relay addresses are very flexible and can describe many different aspects of how to esablish the relayed connection. In its simplest form, it looks something like this: @@ -78,8 +78,8 @@ We can take it a step further and encode the same information for the destinatio - `/ip4/127.0.0.1/tcp/65000/ipfs/QmRelay/p2p-circuit` -If a node is configured with this address, it will use the specified host (`/ip4/127.0.0.1/tcp/65000/ipfs/QmRelay`) as a relay and it will be reachable over this relay. - - There could multiple addresses of this sort specified in the config, in which case the node will be reachable over all of them. +If a node is configured with this address, it will use the specified host (`/ip4/127.0.0.1/tcp/65000/ipfs/QmRelay`) as a relay and it will be reachable over this relay. + - There could multiple addresses of this sort specified in the config, in which case the node will be reachable over all of them. - This is useful if, for example, the node is behind a firewall but wants to be reachable from the outside over a specific relay. Other use-cases are also supported by this scheme, e.g. we can have multiple hops (circuit-relay nodes) encoded in the address, something planed for future releases. @@ -153,7 +153,7 @@ In order to enable the relay functionality in `go-ipfs` we need to edit it's con } ``` -The two options we're looking for are `DisableRelay` and `EnableRelayHop`. We want the former (`DisableRelay`) set to `false` and the latter (`EnableRelayHop`) to `true`, just like in the example above. That should set our go node as a relay. +The two options we're looking for are `DisableRelay` and `EnableRelayHop`. We want the former (`DisableRelay`) set to `false` and the latter (`EnableRelayHop`) to `true`, just like in the example above. That should set our go node as a relay. We also need to make sure our go node can be dialed from the browser. For that, we need to enable a transport that both the browser and the go node can communicate over. We will use the web sockets transport, although there are others that can be used, such as `webrtc-star` and `websocket-star`. To enable the transport and set the interface and port we need to edit the `~/.ipfs/config` one more time. Let's find the `Swarm` array and add our desired address there. I picked `/ip4/0.0.0.0/tcp/4004/ws` because it is a port I know is not being used by anything on my machine, but we can also use port `0` so that the OS chooses a random available port for us — either one should work. @@ -173,15 +173,11 @@ We need to go through similar steps to enable circuit relay in `jsipfs`. However Just as we did with `go-ipfs`, go ahead and edit `js-ipfs` config file located under `~/.jsipfs/config`. Let's add the following config: -(Note that the "EXPERIMENTAL" section might be missing from the config file. In that case, just go ahead and add it) - ```js - "EXPERIMENTAL": { - "relay": { - "enabled": true, - "hop": { - "enabled": true - } + "relay": { + "enabled": true, + "hop": { + "enabled": true } } ``` @@ -247,7 +243,7 @@ Gateway (readonly) is listening on: /ip4/127.0.0.1/tcp/9090 Daemon is ready ``` -Look out for an address similar to `/ip4/127.0.0.1/tcp/4003/ws/ipfs/Qm...`. Note it down somewhere, and let's move on to the next step. +Look out for an address similar to `/ip4/127.0.0.1/tcp/4003/ws/ipfs/Qm...`. Note it down somewhere, and let's move on to the next step. ### 2. Configure and run the bundled example @@ -270,7 +266,7 @@ The bundled example is a simple chat app that uses another cool ipfs feature - [ ### 3. Connect the two browser nodes to the circuit relay -In order for our browser nodes to be able to messages each other, we need to get them connected. But to do that, we need to use a relay - browser nodes can't be connected directly because of lack of socket support. +In order for our browser nodes to be able to messages each other, we need to get them connected. But to do that, we need to use a relay - browser nodes can't be connected directly because of lack of socket support. Remember the caveat above `Currently a Relay will only work if it already has a connection to the STOP node`? This means that we need to connect our browser nodes to the relay node first. @@ -304,21 +300,19 @@ Thats it! ### So what just happened? -Good question! +Good question! - We used [js-ipfs](htpps://github.com/ipfs/js-ipfs) running in the browser with circuit relay enabled: - - _Notice the `EXPERIMENTAL.relay.enabled` below_ + - _Notice the `relay.enabled` below_ you can find it in [src/app.js](src/app.js) ```js const ipfs = new IPFS({ repo: repo(), - EXPERIMENTAL: { - relay: { - enabled: true, - hop: { - enabled: true - } + relay: { + enabled: true, + hop: { + enabled: true } }, config: { diff --git a/package.json b/package.json index dc8940affe..3747d22a78 100644 --- a/package.json +++ b/package.json @@ -61,7 +61,7 @@ }, "homepage": "https://github.com/ipfs/js-ipfs#readme", "devDependencies": { - "aegir": "^13.1.0", + "aegir": "^14.0.0", "buffer-loader": "~0.0.1", "chai": "^4.1.2", "delay": "^2.0.0", @@ -73,7 +73,7 @@ "expose-loader": "~0.7.5", "form-data": "^2.3.2", "hat": "0.0.3", - "interface-ipfs-core": "~0.69.0", + "interface-ipfs-core": "~0.69.2", "ipfsd-ctl": "~0.37.3", "mocha": "^5.1.1", "ncp": "^2.0.0", @@ -86,6 +86,7 @@ "transform-loader": "~0.2.4" }, "dependencies": { + "@nodeutils/defaults-deep": "^1.1.0", "async": "^2.6.0", "big.js": "^5.1.2", "binary-querystring": "~0.1.2", @@ -105,12 +106,12 @@ "hapi-set-header": "^1.0.2", "hoek": "^5.0.3", "human-to-milliseconds": "^1.0.0", - "interface-datastore": "^0.4.1", + "interface-datastore": "~0.4.1", "ipfs-api": "^22.1.1", "ipfs-bitswap": "~0.20.0", "ipfs-block": "~0.7.1", "ipfs-block-service": "~0.14.0", - "ipfs-http-response": "^0.1.2", + "ipfs-http-response": "~0.1.2", "ipfs-multipart": "~0.1.0", "ipfs-repo": "~0.22.1", "ipfs-unixfs": "~0.1.15", @@ -124,18 +125,18 @@ "joi": "^13.2.0", "joi-browser": "^13.0.1", "joi-multiaddr": "^2.0.0", - "libp2p": "~0.20.4", + "libp2p": "~0.22.0", "libp2p-circuit": "~0.2.0", "libp2p-floodsub": "~0.15.0", "libp2p-kad-dht": "~0.10.0", "libp2p-keychain": "~0.3.1", - "libp2p-mdns": "~0.11.0", - "libp2p-mplex": "~0.7.0", - "libp2p-railing": "~0.8.1", + "libp2p-mdns": "~0.12.0", + "libp2p-mplex": "~0.8.0", + "libp2p-railing": "~0.9.2", "libp2p-secio": "~0.10.0", "libp2p-tcp": "~0.12.0", - "libp2p-webrtc-star": "~0.15.0", - "libp2p-websocket-star": "~0.8.0", + "libp2p-webrtc-star": "~0.15.3", + "libp2p-websocket-star": "~0.8.1", "libp2p-websockets": "~0.12.0", "lodash": "^4.17.10", "mafmt": "^6.0.0", diff --git a/src/core/components/libp2p.js b/src/core/components/libp2p.js index c815a20df3..0ed6847d2f 100644 --- a/src/core/components/libp2p.js +++ b/src/core/components/libp2p.js @@ -4,6 +4,7 @@ const Node = require('../runtime/libp2p-nodejs') const promisify = require('promisify-es6') const get = require('lodash/get') +const defaultsDeep = require('@nodeutils/defaults-deep') module.exports = function libp2p (self) { return { @@ -15,27 +16,47 @@ module.exports = function libp2p (self) { return callback(err) } - const options = { - mdns: get(config, 'Discovery.MDNS.Enabled'), - webRTCStar: get(config, 'Discovery.webRTCStar.Enabled'), - bootstrap: get(config, 'Bootstrap'), - modules: self._libp2pModules, - // EXPERIMENTAL - pubsub: get(self._options, 'EXPERIMENTAL.pubsub', false), - dht: get(self._options, 'EXPERIMENTAL.dht', false), - relay: { - enabled: get(self._options, 'EXPERIMENTAL.relay.enabled', - get(config, 'EXPERIMENTAL.relay.enabled', false)), - hop: { - enabled: get(self._options, 'EXPERIMENTAL.relay.hop.enabled', - get(config, 'EXPERIMENTAL.relay.hop.enabled', false)), - active: get(self._options, 'EXPERIMENTAL.relay.hop.active', - get(config, 'EXPERIMENTAL.relay.hop.active', false)) + const libp2pDefaults = { + peerInfo: self._peerInfo, + peerBook: self._peerInfoBook, + config: { + peerDiscovery: { + mdns: { + enabled: get(self._options, 'config.Discovery.MDNS.Enabled', + get(config, 'Discovery.MDNS.Enabled', true)) + }, + webRTCStar: { + enabled: get(self._options, 'config.Discovery.webRTCStar.Enabled', + get(config, 'Discovery.webRTCStar.Enabled', true)) + }, + bootstrap: { + list: get(self._options, 'config.Bootstrap', + get(config, 'Bootstrap', [])) + } + }, + relay: { + enabled: get(self._options, 'relay.enabled', + get(config, 'relay.enabled', false)), + hop: { + enabled: get(self._options, 'relay.hop.enabled', + get(config, 'relay.hop.enabled', false)), + active: get(self._options, 'relay.hop.active', + get(config, 'relay.hop.active', false)) + } + }, + EXPERIMENTAL: { + dht: get(self._options, 'EXPERIMENTAL.dht', false), + pubsub: get(self._options, 'EXPERIMENTAL.pubsub', false) } } } - self._libp2pNode = new Node(self._peerInfo, self._peerInfoBook, options) + const libp2pOptions = defaultsDeep( + get(self._options, 'libp2p', {}), + libp2pDefaults + ) + + self._libp2pNode = new Node(libp2pOptions) self._libp2pNode.on('peer:discovery', (peerInfo) => { const dial = () => { diff --git a/src/core/config.js b/src/core/config.js index 493a30e5eb..9051dd9816 100644 --- a/src/core/config.js +++ b/src/core/config.js @@ -13,6 +13,13 @@ const schema = Joi.object().keys({ ).allow(null), start: Joi.boolean(), pass: Joi.string().allow(''), + relay: Joi.object().keys({ + enabled: Joi.boolean(), + hop: Joi.object().keys({ + enabled: Joi.boolean(), + active: Joi.boolean() + }).allow(null) + }).allow(null), EXPERIMENTAL: Joi.object().keys({ pubsub: Joi.boolean(), sharding: Joi.boolean(), diff --git a/src/core/index.js b/src/core/index.js index 57efa58385..1b561f96c6 100644 --- a/src/core/index.js +++ b/src/core/index.js @@ -34,7 +34,6 @@ class IPFS extends EventEmitter { } options = config.validate(options || {}) - this._libp2pModules = options.libp2p && options.libp2p.modules extend(this._options, options) @@ -120,9 +119,6 @@ class IPFS extends EventEmitter { if (this._options.EXPERIMENTAL.dht) { this.log('EXPERIMENTAL Kademlia DHT is enabled') } - if (this._options.EXPERIMENTAL.relay) { - this.log('EXPERIMENTAL Relay is enabled') - } this.state = require('./state')(this) diff --git a/src/core/runtime/libp2p-browser.js b/src/core/runtime/libp2p-browser.js index 6e015a9e13..167a147209 100644 --- a/src/core/runtime/libp2p-browser.js +++ b/src/core/runtime/libp2p-browser.js @@ -5,38 +5,54 @@ const WebRTCStar = require('libp2p-webrtc-star') const WebSocketStar = require('libp2p-websocket-star') const Multiplex = require('libp2p-mplex') const SECIO = require('libp2p-secio') -const Railing = require('libp2p-railing') +const Bootstrap = require('libp2p-railing') const libp2p = require('libp2p') +const defaultsDeep = require('@nodeutils/defaults-deep') class Node extends libp2p { - constructor (peerInfo, peerBook, options) { - options = options || {} - const wrtcstar = new WebRTCStar({id: peerInfo.id}) - const wsstar = new WebSocketStar({id: peerInfo.id}) - - const modules = { - transport: [new WS(), wrtcstar, wsstar], - connection: { - muxer: [Multiplex], - crypto: [SECIO] + constructor (_options) { + const wrtcstar = new WebRTCStar({id: _options.peerInfo.id}) + const wsstar = new WebSocketStar({id: _options.peerInfo.id}) + + const defaults = { + modules: { + transport: [ + WS, + wrtcstar, + wsstar + ], + streamMuxer: [ + Multiplex + ], + connEncryption: [ + SECIO + ], + peerDiscovery: [ + wrtcstar.discovery, + wsstar.discovery, + Bootstrap + ] }, - discovery: [wrtcstar.discovery, wsstar.discovery] - } - - if (options.bootstrap) { - const r = new Railing(options.bootstrap) - modules.discovery.push(r) - } - - if (options.modules && options.modules.transport) { - options.modules.transport.forEach((t) => modules.transport.push(t)) - } - - if (options.modules && options.modules.discovery) { - options.modules.discovery.forEach((d) => modules.discovery.push(d)) + config: { + peerDiscovery: { + bootstrap: { + enabled: true + }, + webRTCStar: { + enabled: true + }, + websocketStar: { + enabled: true + } + }, + EXPERIMENTAL: { + dht: false, + pubsub: false + } + } } - super(modules, peerInfo, peerBook, options) + super(defaultsDeep(_options, defaults)) } } diff --git a/src/core/runtime/libp2p-nodejs.js b/src/core/runtime/libp2p-nodejs.js index 0776ca5f97..90759001cd 100644 --- a/src/core/runtime/libp2p-nodejs.js +++ b/src/core/runtime/libp2p-nodejs.js @@ -4,49 +4,60 @@ const TCP = require('libp2p-tcp') const MulticastDNS = require('libp2p-mdns') const WS = require('libp2p-websockets') const WebSocketStar = require('libp2p-websocket-star') -const Railing = require('libp2p-railing') +const Bootstrap = require('libp2p-railing') const KadDHT = require('libp2p-kad-dht') const Multiplex = require('libp2p-mplex') const SECIO = require('libp2p-secio') const libp2p = require('libp2p') +const defaultsDeep = require('@nodeutils/defaults-deep') class Node extends libp2p { - constructor (peerInfo, peerBook, options) { - options = options || {} - const wsstar = new WebSocketStar({id: peerInfo.id}) + constructor (_options) { + const wsstar = new WebSocketStar({id: _options.peerInfo.id}) - const modules = { - transport: [new TCP(), new WS(), wsstar], - connection: { - muxer: [Multiplex], - crypto: [SECIO] + const defaults = { + modules: { + transport: [ + TCP, + WS, + wsstar + ], + streamMuxer: [ + Multiplex + ], + connEncryption: [ + SECIO + ], + peerDiscovery: [ + MulticastDNS, + Bootstrap, + wsstar.discovery + ], + dht: KadDHT }, - discovery: [wsstar.discovery] + config: { + peerDiscovery: { + mdns: { + enabled: true + }, + bootstrap: { + enabled: true + }, + websocketStar: { + enabled: true + } + }, + dht: { + kBucketSize: 20 + }, + EXPERIMENTAL: { + dht: false, + pubsub: false + } + } } - if (options.dht) { - modules.DHT = KadDHT - } - - if (options.mdns) { - const mdns = new MulticastDNS(peerInfo, 'ipfs.local') - modules.discovery.push(mdns) - } - - if (options.bootstrap) { - const r = new Railing(options.bootstrap) - modules.discovery.push(r) - } - - if (options.modules && options.modules.transport) { - options.modules.transport.forEach((t) => modules.transport.push(t)) - } - - if (options.modules && options.modules.discovery) { - options.modules.discovery.forEach((d) => modules.discovery.push(d)) - } - - super(modules, peerInfo, peerBook, options) + super(defaultsDeep(_options, defaults)) } } diff --git a/src/http/index.js b/src/http/index.js index d15f1375b0..84cd3d7975 100644 --- a/src/http/index.js +++ b/src/http/index.js @@ -9,6 +9,10 @@ const once = require('once') const IPFS = require('../core') const WStar = require('libp2p-webrtc-star') +const TCP = require('libp2p-tcp') +const MulticastDNS = require('libp2p-mdns') +const WS = require('libp2p-websockets') +const Bootstrap = require('libp2p-railing') const errorHandler = require('./error-handler') function uriToMultiaddr (uri) { @@ -55,8 +59,8 @@ function HttpApi (repo, config, cliArgs) { const using = wrtc ? 'wrtc' : 'electron-webrtc' console.log(`Using ${using} for webrtc support`) const wstar = new WStar({ wrtc: (wrtc || electronWebRTC) }) - libp2p.modules.transport = [wstar] - libp2p.modules.discovery = [wstar.discovery] + libp2p.modules.transport = [TCP, WS, wstar] + libp2p.modules.peerDiscovery = [MulticastDNS, Bootstrap, wstar.discovery] } // try-catch so that programmer errors are not swallowed during testing diff --git a/test/core/circuit-relay.js b/test/core/circuit-relay.js index e837641773..73f653ca72 100644 --- a/test/core/circuit-relay.js +++ b/test/core/circuit-relay.js @@ -38,17 +38,15 @@ function setupInProcNode (addrs, hop, callback) { } procDf.spawn({ + relay: { + enabled: true, + hop: { + enabled: hop + } + }, config: Object.assign({}, baseConf, { Addresses: { Swarm: addrs - }, - EXPERIMENTAL: { - relay: { - enabled: true, - hop: { - enabled: hop - } - } } }) }, (err, ipfsd) => { diff --git a/test/core/config.spec.js b/test/core/config.spec.js index c90e9454c6..1946801dde 100644 --- a/test/core/config.spec.js +++ b/test/core/config.spec.js @@ -104,6 +104,30 @@ describe('config', () => { cfgs.forEach(cfg => expect(() => config.validate(cfg)).to.throw()) }) + it('should validate valid relay', () => { + const cfgs = [ + { relay: { enabled: true, hop: { enabled: true } } }, + { relay: { enabled: false, hop: { enabled: false } } }, + { relay: { enabled: false, hop: null } }, + { relay: { enabled: false } }, + { relay: null }, + { relay: undefined } + ] + + cfgs.forEach(cfg => expect(() => config.validate(cfg)).to.not.throw()) + }) + + it('should validate invalid relay', () => { + const cfgs = [ + { relay: 138 }, + { relay: { enabled: 138 } }, + { relay: { enabled: true, hop: 138 } }, + { relay: { enabled: true, hop: { enabled: 138 } } } + ] + + cfgs.forEach(cfg => expect(() => config.validate(cfg)).to.throw()) + }) + it('should validate valid EXPERIMENTAL', () => { const cfgs = [ { EXPERIMENTAL: { pubsub: true, dht: true, sharding: true } },