feat!: use single DHT only by default (#2322)

We have a weird dual-DHT setup whereby we have a DHT for public addresses (amino, `/ipfs/kad/1.0.0`) and a DHT for private addresess (`/ipfs/lan/kad/1.0.0`). This is an artefact of the previous libp2p setup whereby there could only be a single DHT implementation at once. Now we have services we can configure an amino DHT service and if desired an additional lan-only DHT, or the user can just use amino and not pay the overhead of running an extra DHT. Most content is resolved via the public DHT so running the lan-only DHT gives the user no benefit. BREAKING CHANGE: the `kadDHT` function returns a single DHT - see the readme for how to configure amino/lan as before
libp2p · Dec 20, 2023 · c003789 · c003789
1 parent 83dfc7d
commit c003789
Show file tree

Hide file tree

Showing 52 changed files with 2,318 additions and 1,423 deletions.
diff --git a/packages/integration-tests/test/interop.ts b/packages/integration-tests/test/interop.ts
@@ -8,9 +8,8 @@ import { createClient } from '@libp2p/daemon-client'
 import { createServer } from '@libp2p/daemon-server'
 import { floodsub } from '@libp2p/floodsub'
 import { identify } from '@libp2p/identify'
-import { contentRoutingSymbol, peerDiscoverySymbol, peerRoutingSymbol } from '@libp2p/interface'
 import { interopTests } from '@libp2p/interop'
-import { kadDHT } from '@libp2p/kad-dht'
+import { kadDHT, passthroughMapper } from '@libp2p/kad-dht'
 import { logger } from '@libp2p/logger'
 import { mplex } from '@libp2p/mplex'
 import { peerIdFromKeys } from '@libp2p/peer-id'
@@ -155,41 +154,11 @@ async function createJsPeer (options: SpawnOptions): Promise<Daemon> {
   }
 
   if (options.dht === true) {
-    services.dht = (components: any) => {
-      const dht: any = kadDHT({
-        clientMode: false
-      })(components)
-
-      // go-libp2p-daemon only has the older single-table DHT instead of the dual
-      // lan/wan version found in recent go-ipfs versions. unfortunately it's been
-      // abandoned so here we simulate the older config with the js implementation
-      const lan: any = dht.lan
-
-      const protocol = '/ipfs/kad/1.0.0'
-      lan.protocol = protocol
-      lan.network.protocol = protocol
-      lan.topologyListener.protocol = protocol
-
-      Object.defineProperties(lan, {
-        [contentRoutingSymbol]: {
-          get () {
-            return dht[contentRoutingSymbol]
-          }
-        },
-        [peerRoutingSymbol]: {
-          get () {
-            return dht[peerRoutingSymbol]
-          }
-        },
-        [peerDiscoverySymbol]: {
-          get () {
-            return dht[peerDiscoverySymbol]
-          }
-        }
-      })
-
-      return lan
-    }
+    services.dht = kadDHT({
+      protocol: '/ipfs/kad/1.0.0',
+      peerInfoMapper: passthroughMapper,
+      clientMode: false
+    })
   }
 
   const node: any = await createLibp2p({

diff --git a/packages/kad-dht/README.md b/packages/kad-dht/README.md
@@ -5,6 +5,84 @@
 
 > JavaScript implementation of the Kad-DHT for libp2p
 
+# About
+
+This module implements the [libp2p Kademlia spec](https://github.com/libp2p/specs/blob/master/kad-dht/README.md) in TypeScript.
+
+The Kademlia DHT allow for several operations such as finding peers, searching for providers of DHT records, etc.
+
+## Example - Using with libp2p
+
+```TypeScript
+import { kadDHT } from '@libp2p/kad-dht'
+import { createLibp2p } from 'libp2p'
+import { peerIdFromString } from '@libp2p/peer-id'
+
+const node = await createLibp2p({
+  services: {
+    dht: kadDHT()
+  }
+})
+
+const peerId = peerIdFromString('QmFoo')
+const peerInfo = await libp2p.peerRouting.findPeer(peerId)
+
+console.info(peerInfo) // peer id, multiaddrs
+```
+
+## Example - Connecting to the IPFS Amino DHT
+
+The [Amino DHT](https://blog.ipfs.tech/2023-09-amino-refactoring/) is a public-good DHT used by IPFS to fetch content, find peers, etc.
+
+If you are trying to access content on the public internet, this is the implementation you want.
+
+```TypeScript
+import { kadDHT, removePrivateAddressesMapper } from '@libp2p/kad-dht'
+import { createLibp2p } from 'libp2p'
+import { peerIdFromString } from '@libp2p/peer-id'
+
+const node = await createLibp2p({
+  services: {
+    aminoDHT: kadDHT({
+      protocol: '/ipfs/kad/1.0.0',
+      addressFilter: removePrivateAddressesMapper
+    })
+  }
+})
+
+const peerId = peerIdFromString('QmFoo')
+const peerInfo = await libp2p.peerRouting.findPeer(peerId)
+
+console.info(peerInfo) // peer id, multiaddrs
+```
+
+## Example - Connecting to a LAN-only DHT
+
+This DHT only works with privately dialable peers.
+
+This is for use when peers are on the local area network.
+
+```TypeScript
+import { kadDHT, removePublicAddressesMapper } from '@libp2p/kad-dht'
+import { createLibp2p } from 'libp2p'
+import { peerIdFromString } from '@libp2p/peer-id'
+
+const node = await createLibp2p({
+  services: {
+    lanDHT: kadDHT({
+      protocol: '/ipfs/lan/kad/1.0.0',
+      addressFilter: removePublicAddressesMapper,
+      clientMode: false
+    })
+  }
+})
+
+const peerId = peerIdFromString('QmFoo')
+const peerInfo = await libp2p.peerRouting.findPeer(peerId)
+
+console.info(peerInfo) // peer id, multiaddrs
+```
+
 # Install
 
 ```console

diff --git a/packages/kad-dht/package.json b/packages/kad-dht/package.json
@@ -61,6 +61,7 @@
     "@libp2p/interface-internal": "^1.0.3",
     "@libp2p/peer-collections": "^5.1.1",
     "@libp2p/peer-id": "^4.0.2",
+    "@libp2p/utils": "^5.0.3",
     "@multiformats/multiaddr": "^12.1.10",
     "@types/sinon": "^17.0.0",
     "any-signal": "^4.1.1",
@@ -104,6 +105,7 @@
     "execa": "^8.0.1",
     "it-filter": "^3.0.1",
     "it-last": "^3.0.3",
+    "it-pair": "^2.0.6",
     "lodash.random": "^3.2.0",
     "lodash.range": "^3.2.0",
     "p-retry": "^6.1.0",

diff --git a/packages/kad-dht/src/constants.ts b/packages/kad-dht/src/constants.ts
@@ -11,11 +11,7 @@ export const hour = 60 * minute
 
 export const MAX_RECORD_AGE = 36 * hour
 
-export const LAN_PREFIX = '/lan'
-
-export const PROTOCOL_PREFIX = '/ipfs'
-
-export const PROTOCOL_DHT = '/kad/1.0.0'
+export const PROTOCOL = '/ipfs/kad/1.0.0'
 
 export const RECORD_KEY_PREFIX = '/dht/record'
 
@@ -39,19 +35,19 @@ export const K = 20
 export const ALPHA = 3
 
 // How often we look for our closest DHT neighbours
-export const QUERY_SELF_INTERVAL = Number(5 * minute)
+export const QUERY_SELF_INTERVAL = 5 * minute
 
 // How often we look for the first set of our closest DHT neighbours
-export const QUERY_SELF_INITIAL_INTERVAL = Number(Number(second))
+export const QUERY_SELF_INITIAL_INTERVAL = second
 
 // How long to look for our closest DHT neighbours for
-export const QUERY_SELF_TIMEOUT = Number(5 * second)
+export const QUERY_SELF_TIMEOUT = 5 * second
 
 // How often we try to find new peers
-export const TABLE_REFRESH_INTERVAL = Number(5 * minute)
+export const TABLE_REFRESH_INTERVAL = 5 * minute
 
 // How how long to look for new peers for
-export const TABLE_REFRESH_QUERY_TIMEOUT = Number(30 * second)
+export const TABLE_REFRESH_QUERY_TIMEOUT = 30 * second
 
 // When a timeout is not specified, run a query for this long
-export const DEFAULT_QUERY_TIMEOUT = Number(30 * second)
+export const DEFAULT_QUERY_TIMEOUT = 30 * second
diff --git a/packages/kad-dht/src/content-fetching/index.ts b/packages/kad-dht/src/content-fetching/index.ts
@@ -6,7 +6,7 @@ import { equals as uint8ArrayEquals } from 'uint8arrays/equals'
 import {
   ALPHA
 } from '../constants.js'
-import { Message, MESSAGE_TYPE } from '../message/index.js'
+import { MessageType } from '../message/dht.js'
 import {
   valueEvent,
   queryErrorEvent
@@ -15,20 +15,21 @@ import { Libp2pRecord } from '../record/index.js'
 import { bestRecord } from '../record/selectors.js'
 import { verifyRecord } from '../record/validators.js'
 import { createPutRecord, bufferToRecordKey } from '../utils.js'
-import type { KadDHTComponents, Validators, Selectors, ValueEvent, QueryOptions, QueryEvent } from '../index.js'
+import type { KadDHTComponents, Validators, Selectors, ValueEvent, QueryEvent } from '../index.js'
+import type { Message } from '../message/dht.js'
 import type { Network } from '../network.js'
 import type { PeerRouting } from '../peer-routing/index.js'
 import type { QueryManager } from '../query/manager.js'
 import type { QueryFunc } from '../query/types.js'
-import type { AbortOptions, Logger } from '@libp2p/interface'
+import type { Logger, RoutingOptions } from '@libp2p/interface'
 
 export interface ContentFetchingInit {
   validators: Validators
   selectors: Selectors
   peerRouting: PeerRouting
   queryManager: QueryManager
   network: Network
-  lan: boolean
+  logPrefix: string
 }
 
 export class ContentFetching {
@@ -41,10 +42,10 @@ export class ContentFetching {
   private readonly network: Network
 
   constructor (components: KadDHTComponents, init: ContentFetchingInit) {
-    const { validators, selectors, peerRouting, queryManager, network, lan } = init
+    const { validators, selectors, peerRouting, queryManager, network, logPrefix } = init
 
     this.components = components
-    this.log = components.logger.forComponent(`libp2p:kad-dht:${lan ? 'lan' : 'wan'}:content-fetching`)
+    this.log = components.logger.forComponent(`${logPrefix}:content-fetching`)
     this.validators = validators
     this.selectors = selectors
     this.peerRouting = peerRouting
@@ -81,7 +82,7 @@ export class ContentFetching {
   /**
    * Send the best record found to any peers that have an out of date record
    */
-  async * sendCorrectionRecord (key: Uint8Array, vals: ValueEvent[], best: Uint8Array, options: AbortOptions = {}): AsyncGenerator<QueryEvent> {
+  async * sendCorrectionRecord (key: Uint8Array, vals: ValueEvent[], best: Uint8Array, options: RoutingOptions = {}): AsyncGenerator<QueryEvent> {
     this.log('sendCorrection for %b', key)
     const fixupRec = createPutRecord(key, best)
 
@@ -107,8 +108,11 @@ export class ContentFetching {
 
       // send correction
       let sentCorrection = false
-      const request = new Message(MESSAGE_TYPE.PUT_VALUE, key, 0)
-      request.record = Libp2pRecord.deserialize(fixupRec)
+      const request: Partial<Message> = {
+        type: MessageType.PUT_VALUE,
+        key,
+        record: fixupRec
+      }
 
       for await (const event of this.network.sendRequest(from, request, options)) {
         if (event.name === 'PEER_RESPONSE' && (event.record != null) && uint8ArrayEquals(event.record.value, Libp2pRecord.deserialize(fixupRec).value)) {
@@ -129,7 +133,7 @@ export class ContentFetching {
   /**
    * Store the given key/value pair in the DHT
    */
-  async * put (key: Uint8Array, value: Uint8Array, options: AbortOptions = {}): AsyncGenerator<unknown, void, undefined> {
+  async * put (key: Uint8Array, value: Uint8Array, options: RoutingOptions = {}): AsyncGenerator<unknown, void, undefined> {
     this.log('put key %b value %b', key, value)
 
     // create record in the dht format
@@ -151,8 +155,11 @@ export class ContentFetching {
 
           const events = []
 
-          const msg = new Message(MESSAGE_TYPE.PUT_VALUE, key, 0)
-          msg.record = Libp2pRecord.deserialize(record)
+          const msg: Partial<Message> = {
+            type: MessageType.PUT_VALUE,
+            key,
+            record
+          }
 
           this.log('send put to %p', event.peer.id)
           for await (const putEvent of this.network.sendRequest(event.peer.id, msg, options)) {
@@ -185,7 +192,7 @@ export class ContentFetching {
   /**
    * Get the value to the given key
    */
-  async * get (key: Uint8Array, options: QueryOptions = {}): AsyncGenerator<QueryEvent | ValueEvent> {
+  async * get (key: Uint8Array, options: RoutingOptions = {}): AsyncGenerator<QueryEvent | ValueEvent> {
     this.log('get %b', key)
 
     const vals: ValueEvent[] = []
@@ -229,7 +236,7 @@ export class ContentFetching {
   /**
    * Get the `n` values to the given key without sorting
    */
-  async * getMany (key: Uint8Array, options: QueryOptions = {}): AsyncGenerator<QueryEvent> {
+  async * getMany (key: Uint8Array, options: RoutingOptions = {}): AsyncGenerator<QueryEvent> {
     this.log('getMany values for %b', key)
 
     try {