Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

go-libp2p-kad-dht version 2 #864

Closed
wants to merge 65 commits into from
Closed
Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
65 commits
Select commit Hold shift + click to select a range
7d18e89
Start go-libp2p-kad-dht v2 package
dennis-tra Aug 14, 2023
5bec109
improve DHT struct field documentation
dennis-tra Aug 14, 2023
311f0d3
add: default bootstrap peers method
dennis-tra Aug 14, 2023
dfe3a81
go mod tidy
dennis-tra Aug 14, 2023
57c4988
use slog for logging
dennis-tra Aug 14, 2023
984f28e
handle reachability change events
dennis-tra Aug 14, 2023
35a977f
Add network event subscription
dennis-tra Aug 15, 2023
e8e5be2
Add stream handler implementation
dennis-tra Aug 15, 2023
027c8d2
Add FIND_NODE and PING handlers
dennis-tra Aug 16, 2023
3171894
add handlers and tests
dennis-tra Aug 18, 2023
9f8f4a7
mark triage and gokad parts
dennis-tra Aug 18, 2023
3cdfb34
test put value handler
dennis-tra Aug 18, 2023
c584b02
add get value tests
dennis-tra Aug 18, 2023
4dfcd8b
remove unused methods
dennis-tra Aug 18, 2023
0af762c
update handlers
dennis-tra Aug 21, 2023
741843b
introduce backend concept
dennis-tra Aug 23, 2023
5e09e95
improve backend documentation
dennis-tra Aug 23, 2023
619429c
introduce protocol constants
dennis-tra Aug 23, 2023
3648184
simplify key handling
dennis-tra Aug 23, 2023
015617e
improve documentation
dennis-tra Aug 23, 2023
38c2d32
add add providers tests
dennis-tra Aug 23, 2023
7a0e26c
improve telemetry
dennis-tra Aug 23, 2023
a8a222b
Add AddressFilter feature
dennis-tra Aug 24, 2023
6bdd7ec
WIP
dennis-tra Aug 24, 2023
14c478c
test backend provider garbage collection
dennis-tra Aug 24, 2023
2643b72
refactor provider backend tests
dennis-tra Aug 24, 2023
890d4ec
add stream tests
dennis-tra Aug 24, 2023
bdac291
add logErr helper method
dennis-tra Aug 24, 2023
50f8d00
babysteps towards routing.Routing and kademlia.Router
dennis-tra Aug 24, 2023
5a30d89
update deps
dennis-tra Aug 24, 2023
4660bbc
add test for unregistered message type
dennis-tra Aug 25, 2023
9f88f20
add comments
dennis-tra Aug 25, 2023
9559064
clean up backends in DHT close
dennis-tra Aug 25, 2023
4418cfb
add notifee test
dennis-tra Aug 25, 2023
1cbb419
moved two handler tests from v1
dennis-tra Aug 28, 2023
57c5df3
Move kademlia types to own package
dennis-tra Aug 30, 2023
99dbdc6
Add backend tracing
dennis-tra Sep 1, 2023
f16d79e
improve kadt package doc
dennis-tra Sep 1, 2023
2610f89
document tracedBackend
dennis-tra Sep 1, 2023
54f20b5
Integrate Zikade/go-kademlia in v2 (#880)
dennis-tra Sep 7, 2023
722e958
Simplify usage of kadtest.CtxShort
iand Sep 8, 2023
add1d60
Merge pull request #885 from libp2p/v2-ctx-short
iand Sep 8, 2023
7d838c5
Give dht and coordinator their own telemetry instances (#891)
iand Sep 11, 2023
77dbff0
Migrate go-kademlia state machines (#893)
iand Sep 13, 2023
97e4e02
v2: upgrade build to include go1.21 (#890)
iand Sep 16, 2023
1d35505
Test query interactions with routing table (#887)
iand Sep 18, 2023
3723b8a
remove jaeger dependency (#900)
dennis-tra Sep 18, 2023
156aab2
fix: avoid panic when node is re-added to probe list (#902)
dennis-tra Sep 18, 2023
1220ddd
Decouple coord package from addressing (#903)
iand Sep 19, 2023
e86381e
refactor: v2 simplify tracing (#924)
dennis-tra Sep 19, 2023
83329a4
Clean up DHT test helpers (#928)
iand Sep 19, 2023
2da54ab
Improve query capabilities (#932)
iand Sep 21, 2023
74ffa67
Add broadcast state machine for storing records in the DHT (#930)
dennis-tra Sep 22, 2023
ae5a094
Add explore state machine to expand population of routing table (#934)
iand Sep 25, 2023
09dd7b0
Expose behaviour and state machine configs (#937)
iand Sep 25, 2023
dd5e537
feat: findProvidersAsync (#938)
dennis-tra Sep 26, 2023
dedca86
Add metrics to routing state machines (#939)
iand Sep 27, 2023
e4b1034
add provide routing tests (#940)
dennis-tra Sep 27, 2023
4fa560f
fix: bootstrap state machine goes idle after completion (#943)
iand Sep 27, 2023
6a4249c
Logging improvements (#941)
iand Sep 28, 2023
03adce6
Implement SearchValue/GetValue (#942)
dennis-tra Sep 28, 2023
90d748b
fix: missing QueryMessage return value
dennis-tra Sep 28, 2023
0e628c0
fix: use correct err log method
dennis-tra Sep 28, 2023
509eee4
fix: avoid panic in bootstrap when late messages arrive (#949)
iand Sep 28, 2023
1d1fe93
Use go-libdht (#952)
iand Sep 29, 2023
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 34 additions & 0 deletions v2/bootstrap.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
package dht

import (
"github.com/libp2p/go-libp2p/core/peer"
)

// defaultBootstrapPeers is a set of hard-coded public DHT bootstrap peers
// operated by Protocol Labs. This slice is filled in the init() method.
var defaultBootstrapPeers []peer.AddrInfo

func init() {
for _, s := range []string{
"/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN",
"/dnsaddr/bootstrap.libp2p.io/p2p/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa",
"/dnsaddr/bootstrap.libp2p.io/p2p/QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb",
"/dnsaddr/bootstrap.libp2p.io/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt",
"/ip4/104.131.131.82/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ", // mars.i.ipfs.io
} {
addrInfo, err := peer.AddrInfoFromString(s)
if err != nil {
panic(err)
}
defaultBootstrapPeers = append(defaultBootstrapPeers, *addrInfo)
}
}

// DefaultBootstrapPeers returns hard-coded public DHT bootstrap peers operated
// by Protocol Labs. You can configure your own set of bootstrap peers by
// overwriting the corresponding Config field.
func DefaultBootstrapPeers() []peer.AddrInfo {
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

perhaps we should rename this to <NETWORK_NAME>BootstrapPeers(), e.g., AminoBootstrapPeers

peers := make([]peer.AddrInfo, len(defaultBootstrapPeers))
copy(peers, defaultBootstrapPeers)
return peers
}
23 changes: 23 additions & 0 deletions v2/bootstrap_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
package dht

import (
"testing"

"github.com/libp2p/go-libp2p/core/peer"
)

func TestDefaultBootstrapPeers(t *testing.T) {
bps := DefaultBootstrapPeers()
if len(bps) != len(defaultBootstrapPeers) {
t.Errorf("len(DefaultBootstrapPeers()) = %d, want %v", len(bps), len(defaultBootstrapPeers))
}

bpmap := make(map[peer.ID]peer.AddrInfo)
for _, info := range bps {
bpmap[info.ID] = info
}

if len(bpmap) != len(defaultBootstrapPeers) {
t.Errorf("unique DefaultBootstrapPeers() = %d, want %v", len(bpmap), len(defaultBootstrapPeers))
}
}
105 changes: 105 additions & 0 deletions v2/config.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
package dht

import (
"fmt"

"github.com/plprobelab/go-kademlia/coord"
"github.com/plprobelab/go-kademlia/kad"
"github.com/plprobelab/go-kademlia/key"
)

type (
// ModeOpt describes in which mode this DHT process should operate in.
// Possible options are client, server, and any variant that switches
// between both automatically based on public reachability. The DHT receives
// reachability updates from libp2p via the EvtLocalReachabilityChanged
// event. A DHT that operates in client mode won't register a stream handler
// for incoming requests and therefore won't store, e.g., any provider or
// IPNS records. A DHT in server mode, on the other hand, does all of that.
//
// The `mode` type, on the other hand, captures the current state that the
// DHT is in. This can either be client or server.
ModeOpt string

// mode describes in which mode the DHT currently operates. Because the ModeOpt
// type has options that automatically switch between client and server mode
// based on public connectivity, the DHT mode at any point in time can differ
// from the desired mode. Therefore, we define this second mode type that
// only has the two forms: client or server.
mode string
)

const (
// ModeOptClient configures the DHT to only operate in client mode
// regardless of potential public reachability.
ModeOptClient ModeOpt = "client"

// ModeOptServer configures the DHT to always operate in server mode
// regardless of potentially not being publicly reachable.
ModeOptServer ModeOpt = "server"

// ModeOptAutoClient configures the DHT to start operating in client mode
// and if publicly reachability is detected to switch to server mode.
ModeOptAutoClient ModeOpt = "auto-client"

// ModeOptAutoServer configures the DHT to start operating in server mode,
// and if it is detected that we don't have public reachability switch
// to client mode.
ModeOptAutoServer ModeOpt = "auto-server"

// modeClient means that the DHT is currently operating in client mode.
// For more information, check ModeOpt documentation.
modeClient mode = "client"

// modeServer means that the DHT is currently operating in server mode.
// For more information, check ModeOpt documentation.
modeServer mode = "server"
)

// Config contains all the configuration options for a DHT. Use DefaultConfig
// to build up your own configuration struct. The DHT constructor New uses the
// below method Validate to test for violations of configuration invariants.
type Config struct {
// Mode defines if the DHT should operate as a server or client or switch
// between both automatically (see ModeOpt).
Mode ModeOpt

// Kademlia holds the configuration of the underlying Kademlia implementation.
Kademlia *coord.Config

// RoutingTable holds a reference to the specific routing table
// implementation that this DHT should use. If this field is nil, the
// triert.TrieRT routing table will be used.
RoutingTable kad.RoutingTable[key.Key256, kad.NodeID[key.Key256]]
}

// DefaultConfig returns a configuration struct that can be used as-is to
// instantiate a fully functional DHT client.
func DefaultConfig() *Config {
return &Config{
Mode: ModeOptAutoClient,
Kademlia: coord.DefaultConfig(),
RoutingTable: nil,
}
}

// Validate validates the configuration struct it is called on. It returns
// an error if any configuration issue was detected and nil if this is
// a valid configuration.
func (c *Config) Validate() error {
switch c.Mode {
case ModeOptClient, ModeOptServer, ModeOptAutoClient, ModeOptAutoServer:
default:
return fmt.Errorf("invalid mode option: %s", c.Mode)
}

if c.Kademlia == nil {
return fmt.Errorf("kademlia configuration must not be nil")
}

if err := c.Kademlia.Validate(); err != nil {
return fmt.Errorf("invalid kademlia configuration: %w", err)
}

return nil
}
52 changes: 52 additions & 0 deletions v2/config_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
package dht

import (
"testing"
)

func TestConfig_Validate(t *testing.T) {
tests := []struct {
name string
mutate func(*Config) *Config
wantErr bool
}{
{
name: "happy path",
wantErr: false,
mutate: func(c *Config) *Config { return c },
},
{
name: "invalid mode",
wantErr: true,
mutate: func(c *Config) *Config {
c.Mode = "invalid"
return c
},
},
{
name: "nil Kademlia configuration",
wantErr: true,
mutate: func(c *Config) *Config {
c.Kademlia = nil
return c
},
},
{
name: "invalid Kademlia configuration",
wantErr: true,
mutate: func(c *Config) *Config {
c.Kademlia.Clock = nil
return c
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
c := DefaultConfig()
c = tt.mutate(c)
if err := c.Validate(); (err != nil) != tt.wantErr {
t.Errorf("Validate() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}
82 changes: 82 additions & 0 deletions v2/dht.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
package dht

import (
"fmt"

"github.com/libp2p/go-libp2p/core/host"
ma "github.com/multiformats/go-multiaddr"
"github.com/plprobelab/go-kademlia/coord"
"github.com/plprobelab/go-kademlia/kad"
"github.com/plprobelab/go-kademlia/key"
"github.com/plprobelab/go-kademlia/routing/triert"
)

// DHT is an implementation of Kademlia with S/Kademlia modifications.
// It is used to implement the base Routing module.
type DHT struct {
// host holds a reference to the underlying libp2p host
host host.Host

// cfg holds a reference to the DHT configuration struct
cfg *Config

// mode indicates the current mode the DHT operates in. This can differ from
// the desired mode if set to auto-client or auto-server. The desired mode
// can be configured via the Config struct.
mode mode

// kad is a reference to the go-kademlia coordinator
kad *coord.Coordinator[key.Key256, ma.Multiaddr]

// rt holds a reference to the routing table implementation. This can be
// configured via the Config struct.
rt kad.RoutingTable[key.Key256, kad.NodeID[key.Key256]]
}

// New constructs a new DHT for the given underlying host and with the given
// configuration. Use DefaultConfig() to construct a configuration.
func New(h host.Host, cfg *Config) (*DHT, error) {
var err error

// check if the configuration is valid
if err = cfg.Validate(); err != nil {
return nil, fmt.Errorf("validate DHT config: %w", err)
}

d := &DHT{
host: h,
cfg: cfg,
}

nid := nodeID(d.host.ID())

// Use the configured routing table if it was provided
if cfg.RoutingTable != nil {
d.rt = cfg.RoutingTable
} else {
rtCfg := triert.DefaultConfig[key.Key256, kad.NodeID[key.Key256]]()
d.rt, err = triert.New[key.Key256, kad.NodeID[key.Key256]](nid, rtCfg)
if err != nil {
return nil, fmt.Errorf("new trie routing table: %w", err)
}
}

// instantiate a new Kademlia DHT coordinator.
d.kad, err = coord.NewCoordinator[key.Key256, ma.Multiaddr](nid, nil, d.rt, cfg.Kademlia)
if err != nil {
return nil, fmt.Errorf("new coordinator: %w", err)
}

// determine mode to start in
switch cfg.Mode {
case ModeOptClient, ModeOptAutoClient:
d.mode = modeClient
case ModeOptServer, ModeOptAutoServer:
d.mode = modeServer
default:
// should never happen because of the configuration validation above
return nil, fmt.Errorf("invalid dht mode %s", cfg.Mode)
}

return d, nil
}
78 changes: 78 additions & 0 deletions v2/dht_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
package dht

import (
"reflect"
"testing"

"github.com/libp2p/go-libp2p"
)

func TestNew(t *testing.T) {
h, err := libp2p.New(libp2p.NoListenAddrs)
if err != nil {
t.Fatal(err)
}

tests := []struct {
name string
cfgBuilder func(*Config) *Config
wantBuilder func(*DHT) *DHT
wantErr bool
}{
{
name: "happy path",
cfgBuilder: func(c *Config) *Config { return c },
wantBuilder: func(dht *DHT) *DHT { return dht },
wantErr: false,
},
{
name: "mode set to server",
cfgBuilder: func(c *Config) *Config {
c.Mode = ModeOptServer
return c
},
wantBuilder: func(dht *DHT) *DHT {
dht.mode = modeServer
return dht
},
wantErr: false,
},
{
name: "mode set to auto client",
cfgBuilder: func(c *Config) *Config {
c.Mode = ModeOptAutoClient
return c
},
wantBuilder: func(dht *DHT) *DHT {
dht.mode = modeClient
return dht
},
wantErr: false,
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
c := DefaultConfig()
d, err := New(h, c)
if err != nil {
t.Fatal(err)
}

got, err := New(h, tt.cfgBuilder(c))
if (err != nil) != tt.wantErr {
t.Errorf("New() error = %v, wantErr %v", err, tt.wantErr)
return
}

want := tt.wantBuilder(d)

want.kad = nil
got.kad = nil

if !reflect.DeepEqual(got, want) {
t.Errorf("New() got = %v, want %v", got, want)
}
})
}
}
Loading
Loading