From 28cd95b6398efcf9312ad11d7da7c845b591678b Mon Sep 17 00:00:00 2001 From: Jorropo Date: Fri, 19 May 2023 16:53:10 +0200 Subject: [PATCH] feat: warn users who are falling behind reprovides Fixes: #9704 Fixes: #9702 Fixes: #9482 Depends on: ipfs/boxo#273 --- cmd/ipfs/daemon.go | 3 + config/experiments.go | 2 +- config/routing.go | 2 + config/types.go | 24 +++ core/commands/stat_provide.go | 17 +-- core/coreapi/coreapi.go | 2 +- core/node/groups.go | 8 +- core/node/libp2p/routing.go | 2 +- core/node/provider.go | 204 +++++++++++++------------ docs/changelogs/v0.21.md | 11 ++ docs/config.md | 42 ++++- docs/examples/kubo-as-a-library/go.mod | 3 +- docs/examples/kubo-as-a-library/go.sum | 6 +- docs/experimental-features.md | 57 +------ go.mod | 5 +- go.sum | 6 +- repo/fsrepo/fsrepo.go | 2 +- repo/fsrepo/migrations/fetcher.go | 2 +- 18 files changed, 206 insertions(+), 192 deletions(-) diff --git a/cmd/ipfs/daemon.go b/cmd/ipfs/daemon.go index 1e03a8264a8e..f43f1da242aa 100644 --- a/cmd/ipfs/daemon.go +++ b/cmd/ipfs/daemon.go @@ -425,6 +425,9 @@ func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment case routingOptionNoneKwd: ncfg.Routing = libp2p.NilRouterOption case routingOptionCustomKwd: + if cfg.Routing.AcceleratedDHTClient { + return fmt.Errorf("Routing.AcceleratedDHTClient option is set even tho Routing.Type is custom, using custom .AcceleratedDHTClient needs to be set on DHT routers individually") + } ncfg.Routing = libp2p.ConstructDelegatedRouting( cfg.Routing.Routers, cfg.Routing.Methods, diff --git a/config/experiments.go b/config/experiments.go index 072dcd0ddd62..f5ecf4be6292 100644 --- a/config/experiments.go +++ b/config/experiments.go @@ -8,7 +8,7 @@ type Experiments struct { Libp2pStreamMounting bool P2pHttpProxy bool //nolint StrategicProviding bool - AcceleratedDHTClient bool + AcceleratedDHTClient experimentalAcceleratedDHTClient `json:",omitempty"` OptimisticProvide bool OptimisticProvideJobsPoolSize int } diff --git a/config/routing.go b/config/routing.go index 1210bb3cecc3..7f5e48aa262b 100644 --- a/config/routing.go +++ b/config/routing.go @@ -15,6 +15,8 @@ type Routing struct { // When "custom" is set, user-provided Routing.Routers is used. Type *OptionalString `json:",omitempty"` + AcceleratedDHTClient bool + Routers Routers Methods Methods diff --git a/config/types.go b/config/types.go index 3a0d4f4b3b2d..a781f023af91 100644 --- a/config/types.go +++ b/config/types.go @@ -438,3 +438,27 @@ func (swarmLimits) UnmarshalJSON(b []byte) error { } } } + +type experimentalAcceleratedDHTClient struct{} + +var _ json.Unmarshaler = experimentalAcceleratedDHTClient{} + +func (experimentalAcceleratedDHTClient) UnmarshalJSON(b []byte) error { + d := json.NewDecoder(bytes.NewReader(b)) + for { + switch tok, err := d.Token(); err { + case io.EOF: + return nil + case nil: + switch tok { + case json.Delim('{'), json.Delim('}'): + // accept empty objects + continue + } + //nolint + return fmt.Errorf("The Experimental.AcceleratedDHTClient key has been moved to Routing.AcceleratedDHTClient in Kubo 0.21, please use this new key and remove the old one.") + default: + return err + } + } +} diff --git a/core/commands/stat_provide.go b/core/commands/stat_provide.go index f93f73cb4b8b..6ee51e516fa0 100644 --- a/core/commands/stat_provide.go +++ b/core/commands/stat_provide.go @@ -7,10 +7,10 @@ import ( "time" humanize "github.com/dustin/go-humanize" + "github.com/ipfs/boxo/provider" cmds "github.com/ipfs/go-ipfs-cmds" "github.com/ipfs/kubo/core/commands/cmdenv" - - "github.com/ipfs/boxo/provider/batched" + "golang.org/x/exp/constraints" ) var statProvideCmd = &cmds.Command{ @@ -34,12 +34,7 @@ This interface is not stable and may change from release to release. return ErrNotOnline } - sys, ok := nd.Provider.(*batched.BatchProvidingSystem) - if !ok { - return fmt.Errorf("can only return stats if Experimental.AcceleratedDHTClient is enabled") - } - - stats, err := sys.Stat(req.Context) + stats, err := nd.Provider.Stat() if err != nil { return err } @@ -51,7 +46,7 @@ This interface is not stable and may change from release to release. return nil }, Encoders: cmds.EncoderMap{ - cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, s *batched.BatchedProviderStats) error { + cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, s *provider.ReproviderStats) error { wtr := tabwriter.NewWriter(w, 1, 2, 1, ' ', 0) defer wtr.Flush() @@ -62,14 +57,14 @@ This interface is not stable and may change from release to release. return nil }), }, - Type: batched.BatchedProviderStats{}, + Type: provider.ReproviderStats{}, } func humanDuration(val time.Duration) string { return val.Truncate(time.Microsecond).String() } -func humanNumber(n int) string { +func humanNumber[T constraints.Float | constraints.Integer](n T) string { nf := float64(n) str := humanSI(nf, 0) fullStr := humanFull(nf, 0) diff --git a/core/coreapi/coreapi.go b/core/coreapi/coreapi.go index 6c97800a3384..5a7e321a9c6e 100644 --- a/core/coreapi/coreapi.go +++ b/core/coreapi/coreapi.go @@ -238,7 +238,7 @@ func (api *CoreAPI) WithOptions(opts ...options.ApiOption) (coreiface.CoreAPI, e return nil, fmt.Errorf("error constructing namesys: %w", err) } - subAPI.provider = provider.NewOfflineProvider() + subAPI.provider = provider.NewNoopProvider() subAPI.peerstore = nil subAPI.peerHost = nil diff --git a/core/node/groups.go b/core/node/groups.go index 5486788358cc..e6a570ab8d06 100644 --- a/core/node/groups.go +++ b/core/node/groups.go @@ -304,7 +304,6 @@ func Online(bcfg *BuildCfg, cfg *config.Config, userResourceOverrides rcmgr.Part LibP2P(bcfg, cfg, userResourceOverrides), OnlineProviders( cfg.Experimental.StrategicProviding, - cfg.Experimental.AcceleratedDHTClient, cfg.Reprovider.Strategy.WithDefault(config.DefaultReproviderStrategy), cfg.Reprovider.Interval.WithDefault(config.DefaultReproviderInterval), ), @@ -320,12 +319,7 @@ func Offline(cfg *config.Config) fx.Option { fx.Provide(libp2p.Routing), fx.Provide(libp2p.ContentRouting), fx.Provide(libp2p.OfflineRouting), - OfflineProviders( - cfg.Experimental.StrategicProviding, - cfg.Experimental.AcceleratedDHTClient, - cfg.Reprovider.Strategy.WithDefault(config.DefaultReproviderStrategy), - cfg.Reprovider.Interval.WithDefault(config.DefaultReproviderInterval), - ), + OfflineProviders(), ) } diff --git a/core/node/libp2p/routing.go b/core/node/libp2p/routing.go index 2e356e447f5d..0b642ed8ce18 100644 --- a/core/node/libp2p/routing.go +++ b/core/node/libp2p/routing.go @@ -90,7 +90,7 @@ func BaseRouting(cfg *config.Config) interface{} { } } - if dualDHT != nil && cfg.Experimental.AcceleratedDHTClient { + if dualDHT != nil && cfg.Routing.AcceleratedDHTClient { cfg, err := in.Repo.Config() if err != nil { return out, err diff --git a/core/node/provider.go b/core/node/provider.go index 58e74bdb3f89..63beeb4018b0 100644 --- a/core/node/provider.go +++ b/core/node/provider.go @@ -5,145 +5,151 @@ import ( "fmt" "time" + "github.com/ipfs/boxo/blockstore" "github.com/ipfs/boxo/fetcher" pin "github.com/ipfs/boxo/pinning/pinner" provider "github.com/ipfs/boxo/provider" - "github.com/ipfs/boxo/provider/batched" - q "github.com/ipfs/boxo/provider/queue" - "github.com/ipfs/boxo/provider/simple" - "go.uber.org/fx" - - "github.com/ipfs/kubo/core/node/helpers" "github.com/ipfs/kubo/repo" irouting "github.com/ipfs/kubo/routing" + "go.uber.org/fx" ) -// SIMPLE - -// ProviderQueue creates new datastore backed provider queue -func ProviderQueue(mctx helpers.MetricsCtx, lc fx.Lifecycle, repo repo.Repo) (*q.Queue, error) { - return q.NewQueue(helpers.LifecycleCtx(mctx, lc), "provider-v1", repo.Datastore()) -} - -// SimpleProvider creates new record provider -func SimpleProvider(mctx helpers.MetricsCtx, lc fx.Lifecycle, queue *q.Queue, rt irouting.ProvideManyRouter) provider.Provider { - return simple.NewProvider(helpers.LifecycleCtx(mctx, lc), queue, rt) -} - -// SimpleReprovider creates new reprovider -func SimpleReprovider(reproviderInterval time.Duration) interface{} { - return func(mctx helpers.MetricsCtx, lc fx.Lifecycle, rt irouting.ProvideManyRouter, keyProvider simple.KeyChanFunc) (provider.Reprovider, error) { - return simple.NewReprovider(helpers.LifecycleCtx(mctx, lc), reproviderInterval, rt, keyProvider), nil - } -} - -// SimpleProviderSys creates new provider system -func SimpleProviderSys(isOnline bool) interface{} { - return func(lc fx.Lifecycle, p provider.Provider, r provider.Reprovider) provider.System { - sys := provider.NewSystem(p, r) - - if isOnline { - lc.Append(fx.Hook{ - OnStart: func(ctx context.Context) error { - sys.Run() - return nil - }, - OnStop: func(ctx context.Context) error { - return sys.Close() - }, - }) - } - - return sys - } -} - -// BatchedProviderSys creates new provider system -func BatchedProviderSys(isOnline bool, reprovideInterval time.Duration) interface{} { - return func(lc fx.Lifecycle, cr irouting.ProvideManyRouter, q *q.Queue, keyProvider simple.KeyChanFunc, repo repo.Repo) (provider.System, error) { - sys, err := batched.New(cr, q, - batched.ReproviderInterval(reprovideInterval), - batched.Datastore(repo.Datastore()), - batched.KeyProvider(keyProvider)) +func ProviderSys(reprovideInterval time.Duration) fx.Option { + const magicThroughputReportCount = 128 + return fx.Provide(func(lc fx.Lifecycle, cr irouting.ProvideManyRouter, keyProvider provider.KeyChanFunc, repo repo.Repo, bs blockstore.Blockstore) (provider.System, error) { + sys, err := provider.New(repo.Datastore(), + provider.Online(cr), + provider.ReproviderInterval(reprovideInterval), + provider.KeyProvider(keyProvider), + provider.ThroughputReport(func(reprovide bool, complete bool, keysProvided uint, duration time.Duration) bool { + avgProvideSpeed := duration / time.Duration(keysProvided) + count := uint64(keysProvided) + + if !reprovide || !complete { + // We don't know how many CIDs we have to provide, try to fetch it from the blockstore. + // But don't try for too long as this might be very expensive if you have a huge datastore. + ctx, cancel := context.WithTimeout(context.Background(), time.Minute*5) + defer cancel() + + // FIXME: I want a running counter of blocks so size of blockstore can be an O(1) lookup. + ch, err := bs.AllKeysChan(ctx) + if err != nil { + logger.Errorf("fetching AllKeysChain in provider ThroughputReport: %v", err) + return false + } + count = 0 + countLoop: + for { + select { + case _, ok := <-ch: + if !ok { + break countLoop + } + count++ + case <-ctx.Done(): + // really big blockstore mode + + // how many blocks would be in a 10TiB blockstore with 128KiB blocks. + const probableBigBlockstore = (10 * 1024 * 1024 * 1024 * 1024) / (128 * 1024) + // How long per block that lasts us. + expectedProvideSpeed := reprovideInterval / probableBigBlockstore + if avgProvideSpeed > expectedProvideSpeed { + logger.Errorf(` +๐Ÿ””๐Ÿ””๐Ÿ”” YOU MAY BE FALLING BEHIND DHT REPROVIDES! ๐Ÿ””๐Ÿ””๐Ÿ”” + +โš ๏ธ Your system might be struggling to keep up with DHT reprovides! +Meaning your content being partially or completely inacessible on the network. +We observed that you recently provided %d keys at an average rate of %v per key. + +๐Ÿ•‘ An attempt to estimate your blockstore size timed out after 5 minutes, +implying your blockstore might be exceedingly large. Assuming a considerable +size of 10TiB, it would take %v to provide the complete set. + +โฐ The total provide time needs to stay under 24 hours to prevent falling behind! + +๐Ÿ’ก Consider enabling the Accelerated DHT to enhance your system performance. See: +https://github.com/ipfs/kubo/blob/master/docs/config.md#routingaccelerateddhtclient`, + keysProvided, avgProvideSpeed, avgProvideSpeed*probableBigBlockstore) + return false + } + } + } + } + + // How long per block that lasts us. + expectedProvideSpeed := reprovideInterval / time.Duration(count) + if avgProvideSpeed > expectedProvideSpeed { + // FIXME(@Jorropo): add link to the accelerated DHT client docs once this isn't experimental anymore. + logger.Errorf(` +๐Ÿ””๐Ÿ””๐Ÿ”” YOU ARE FALLING BEHIND DHT REPROVIDES! ๐Ÿ””๐Ÿ””๐Ÿ”” + +โš ๏ธ Your system is struggling to keep up with DHT reprovides! +Meaning your content being partially or completely inacessible on the network. +We observed that you recently provided %d keys at an average rate of %v per key. + +๐Ÿ’พ Your total CID count is ~%d which would total at %v reprovide process. + +โฐ The total provide time needs to stay under 24 hours to prevent falling behind! + +๐Ÿ’ก Consider enabling the Accelerated DHT to enhance your reprovide throughput. See: +https://github.com/ipfs/kubo/blob/master/docs/config.md#routingaccelerateddhtclient`, + keysProvided, avgProvideSpeed, count, avgProvideSpeed*time.Duration(count)) + } + return false + }, magicThroughputReportCount), + ) if err != nil { return nil, err } - if isOnline { - lc.Append(fx.Hook{ - OnStart: func(ctx context.Context) error { - sys.Run() - return nil - }, - OnStop: func(ctx context.Context) error { - return sys.Close() - }, - }) - } + lc.Append(fx.Hook{ + OnStop: func(ctx context.Context) error { + return sys.Close() + }, + }) return sys, nil - } + }) } // ONLINE/OFFLINE // OnlineProviders groups units managing provider routing records online -func OnlineProviders(useStrategicProviding bool, useBatchedProviding bool, reprovideStrategy string, reprovideInterval time.Duration) fx.Option { - if useStrategicProviding { - return fx.Provide(provider.NewOfflineProvider) - } - - return fx.Options( - SimpleProviders(reprovideStrategy, reprovideInterval), - maybeProvide(SimpleProviderSys(true), !useBatchedProviding), - maybeProvide(BatchedProviderSys(true, reprovideInterval), useBatchedProviding), - ) -} - -// OfflineProviders groups units managing provider routing records offline -func OfflineProviders(useStrategicProviding bool, useBatchedProviding bool, reprovideStrategy string, reprovideInterval time.Duration) fx.Option { +func OnlineProviders(useStrategicProviding bool, reprovideStrategy string, reprovideInterval time.Duration) fx.Option { if useStrategicProviding { - return fx.Provide(provider.NewOfflineProvider) + return OfflineProviders() } - return fx.Options( - SimpleProviders(reprovideStrategy, reprovideInterval), - maybeProvide(SimpleProviderSys(false), true), - //maybeProvide(BatchedProviderSys(false, reprovideInterval), useBatchedProviding), - ) -} - -// SimpleProviders creates the simple provider/reprovider dependencies -func SimpleProviders(reprovideStrategy string, reproviderInterval time.Duration) fx.Option { var keyProvider fx.Option switch reprovideStrategy { - case "all": - fallthrough - case "": - keyProvider = fx.Provide(simple.NewBlockstoreProvider) + case "all", "": + keyProvider = fx.Provide(provider.NewBlockstoreProvider) case "roots": keyProvider = fx.Provide(pinnedProviderStrategy(true)) case "pinned": keyProvider = fx.Provide(pinnedProviderStrategy(false)) default: - return fx.Error(fmt.Errorf("unknown reprovider strategy '%s'", reprovideStrategy)) + return fx.Error(fmt.Errorf("unknown reprovider strategy %q", reprovideStrategy)) } return fx.Options( - fx.Provide(ProviderQueue), - fx.Provide(SimpleProvider), keyProvider, - fx.Provide(SimpleReprovider(reproviderInterval)), + ProviderSys(reprovideInterval), ) } +// OfflineProviders groups units managing provider routing records offline +func OfflineProviders() fx.Option { + return fx.Provide(provider.NewNoopProvider) +} + func pinnedProviderStrategy(onlyRoots bool) interface{} { type input struct { fx.In Pinner pin.Pinner IPLDFetcher fetcher.Factory `name:"ipldFetcher"` } - return func(in input) simple.KeyChanFunc { - return simple.NewPinnedProvider(onlyRoots, in.Pinner, in.IPLDFetcher) + return func(in input) provider.KeyChanFunc { + return provider.NewPinnedProvider(onlyRoots, in.Pinner, in.IPLDFetcher) } } diff --git a/docs/changelogs/v0.21.md b/docs/changelogs/v0.21.md index 57eb7002bad4..3ece77959dae 100644 --- a/docs/changelogs/v0.21.md +++ b/docs/changelogs/v0.21.md @@ -6,6 +6,7 @@ - [Overview](#overview) - [๐Ÿ”ฆ Highlights](#-highlights) + - [Accelerated DHT Client is no longer experimental](#--empty-repo-is-now-the-default) - [๐Ÿ“ Changelog](#-changelog) - [๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ Contributors](#-contributors) @@ -13,6 +14,16 @@ ### ๐Ÿ”ฆ Highlights +#### Accelerated DHT Client is no longer experimental + +The accelerated DHT client is now the main recomended solution for peoples who +are hosting lots of data. + +We moved the `Experimental.AcceleratedDHTClient` flag in `Routing.AcceleratedDHTClient`. + +A new tracker will try to estimate the providing speed and warn users that they +should be using AcceleratedDHTClient because they are falling behind. + ### ๐Ÿ“ Changelog ### ๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ Contributors diff --git a/docs/config.md b/docs/config.md index 91866a42323c..9e17e0f97da3 100644 --- a/docs/config.md +++ b/docs/config.md @@ -108,6 +108,7 @@ config file at runtime. - [`Reprovider.Strategy`](#reproviderstrategy) - [`Routing`](#routing) - [`Routing.Type`](#routingtype) + - [`Routing.AcceleratedDHTClient`](#routingaccelerateddhtclient) - [`Routing.Routers`](#routingrouters) - [`Routing.Routers: Type`](#routingrouters-type) - [`Routing.Routers: Parameters`](#routingrouters-parameters) @@ -971,7 +972,7 @@ Type: `optionalInteger` (byte count, `null` means default which is 1MB) ### `Internal.Bitswap.ProviderSearchDelay` This parameter determines how long to wait before looking for providers outside of bitswap. -Other routing systems like the DHT are able to provide results in less than a second, so lowering +Other routing systems like the DHT are able to provide results in less than a second, so lowering this number will allow faster peers lookups in some cases. Type: `optionalDuration` (`null` means default which is 1s) @@ -1397,6 +1398,43 @@ Default: `auto` (DHT + IPNI) Type: `optionalString` (`null`/missing means the default) + +### `Routing.AcceleratedDHTClient` + +Utilizes an alternative DHT client that searches for and maintains more information +about the network in exchange for being more performant. + +This is not compatible with `Routing.Type` `custom`. If you are using composable routers +you can configure this individualy on each router. + +When it is enabled: +- DHT operations should complete much faster than with it disabled +- A batching reprovider system will be enabled which takes advantage of some + properties of the experimental client to very efficiently put provider records into the network +- The standard DHT client (and server if enabled) are run alongside the accelerated DHT client +- The operations `ipfs stats dht` will default to showing information about the accelerated DHT client + +**Caveats:** +1. Running the experimental client likely will result in more resource consumption (connections, RAM, CPU, bandwidth) + - Users that are limited in the number of parallel connections their machines/networks can perform will likely suffer + - The resource usage is not smooth as the client crawls the network in rounds and reproviding is similarly done in rounds + - Users who previously had a lot of content but were unable to advertise it on the network will see an increase in + egress bandwidth as their nodes start to advertise all of their CIDs into the network. If you have lots of data + entering your node that you don't want to advertise consider using [Reprovider Strategies](#reproviderstrategy) + to reduce the number of CIDs that you are reproviding. Similarly, if you are running a node that deals mostly with + short-lived temporary data (e.g. you use a separate node for ingesting data then for storing and serving it) then + you may benefit from using [Strategic Providing](experimental-features#strategic-providing) to prevent advertising + of data that you ultimately will not have. +2. Currently, the DHT is not usable for queries for the first 5-10 minutes of operation as the routing table is being +prepared. This means operations like searching the DHT for particular peers or content will not work + - You can see if the DHT has been initially populated by running `ipfs stats dht` +3. Currently, the accelerated DHT client is not compatible with LAN-based DHTs and will not perform operations against +them + +Default: `false` + +Type: `bool` (missing means `false`) + ### `Routing.Routers` **EXPERIMENTAL: `Routing.Routers` configuration may change in future release** @@ -1548,7 +1586,7 @@ another node, even if this other node is on a different network. This may trigger netscan alerts on some hosting providers or cause strain in some setups. The `server` configuration profile fills up this list with sensible defaults, -preventing dials to all non-routable IP addresses (e.g., `/ip4/192.168.0.0/ipcidr/16`, +preventing dials to all non-routable IP addresses (e.g., `/ip4/192.168.0.0/ipcidr/16`, which is the multiaddress representation of `192.168.0.0/16`) but you should always check settings against your own network and/or hosting provider. diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index 52e3c064699c..619619f78d76 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -7,7 +7,7 @@ go 1.18 replace github.com/ipfs/kubo => ./../../.. require ( - github.com/ipfs/boxo v0.8.2-0.20230510114019-33e3f0cd052b + github.com/ipfs/boxo v0.8.2-0.20230519132256-d6bf836e8808 github.com/ipfs/kubo v0.0.0-00010101000000-000000000000 github.com/libp2p/go-libp2p v0.27.3 github.com/multiformats/go-multiaddr v0.9.0 @@ -21,7 +21,6 @@ require ( github.com/benbjohnson/clock v1.3.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect - github.com/cenkalti/backoff v2.2.1+incompatible // indirect github.com/cenkalti/backoff/v4 v4.2.0 // indirect github.com/ceramicnetwork/go-dag-jose v0.1.0 // indirect github.com/cespare/xxhash v1.1.0 // indirect diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index 3111d683f532..c307c036806c 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -74,8 +74,6 @@ github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= -github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= -github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/cenkalti/backoff/v4 v4.2.0 h1:HN5dHm3WBOgndBH6E8V0q2jIYIR3s9yglV8k/+MN3u4= github.com/cenkalti/backoff/v4 v4.2.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -321,8 +319,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.8.2-0.20230510114019-33e3f0cd052b h1:6EVpfwbBgwhfZOA19i55jOGokKOy+OaQAm1dg4RbXmc= -github.com/ipfs/boxo v0.8.2-0.20230510114019-33e3f0cd052b/go.mod h1:Ej2r08Z4VIaFKqY08UXMNhwcLf6VekHhK8c+KqA1B9Y= +github.com/ipfs/boxo v0.8.2-0.20230519132256-d6bf836e8808 h1:8yjrpVR7hgXsHKoEM4rFQKcpDATkr5uv1qeBEYZDp7E= +github.com/ipfs/boxo v0.8.2-0.20230519132256-d6bf836e8808/go.mod h1:LbIQUU7IEKw2AMm8sCLgJ/hljd/4yyiMtPzc68nveOc= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= diff --git a/docs/experimental-features.md b/docs/experimental-features.md index 83a5fdf7b863..07f7f30f5e83 100644 --- a/docs/experimental-features.md +++ b/docs/experimental-features.md @@ -513,7 +513,7 @@ ipfs config --json Experimental.StrategicProviding true - [ ] provide roots - [ ] provide all - [ ] provide strategic - + ## GraphSync ### State @@ -546,59 +546,6 @@ Stable, enabled by default [Noise](https://github.com/libp2p/specs/tree/master/noise) libp2p transport based on the [Noise Protocol Framework](https://noiseprotocol.org/noise.html). While TLS remains the default transport in Kubo, Noise is easier to implement and is thus the "interop" transport between IPFS and libp2p implementations. -## Accelerated DHT Client - -### In Version - -0.9.0 - -### State - -Experimental, default-disabled. - -Utilizes an alternative DHT client that searches for and maintains more information about the network -in exchange for being more performant. - -When it is enabled: -- DHT operations should complete much faster than with it disabled -- A batching reprovider system will be enabled which takes advantage of some properties of the experimental client to - very efficiently put provider records into the network -- The standard DHT client (and server if enabled) are run alongside the alternative client -- The operations `ipfs stats dht` and `ipfs stats provide` will have different outputs - - `ipfs stats provide` only works when the accelerated DHT client is enabled and shows various statistics regarding - the provider/reprovider system - - `ipfs stats dht` will default to showing information about the new client - -**Caveats:** -1. Running the experimental client likely will result in more resource consumption (connections, RAM, CPU, bandwidth) - - Users that are limited in the number of parallel connections their machines/networks can perform will likely suffer - - Currently, the resource usage is not smooth as the client crawls the network in rounds and reproviding is similarly - done in rounds - - Users who previously had a lot of content but were unable to advertise it on the network will see an increase in - egress bandwidth as their nodes start to advertise all of their CIDs into the network. If you have lots of data - entering your node that you don't want to advertise consider using [Reprovider Strategies](config.md#reproviderstrategy) - to reduce the number of CIDs that you are reproviding. Similarly, if you are running a node that deals mostly with - short-lived temporary data (e.g. you use a separate node for ingesting data then for storing and serving it) then - you may benefit from using [Strategic Providing](#strategic-providing) to prevent advertising of data that you - ultimately will not have. -2. Currently, the DHT is not usable for queries for the first 5-10 minutes of operation as the routing table is being -prepared. This means operations like searching the DHT for particular peers or content will not work - - You can see if the DHT has been initially populated by running `ipfs stats dht` -3. Currently, the accelerated DHT client is not compatible with LAN-based DHTs and will not perform operations against -them - -### How to enable - -``` -ipfs config --json Experimental.AcceleratedDHTClient true -``` - -### Road to being a real feature - -- [ ] Needs more people to use and report on how well it works -- [ ] Should be usable for queries (even if slower/less efficient) shortly after startup -- [ ] Should be usable with non-WAN DHTs - ## Optimistic Provide ### In Version @@ -640,7 +587,7 @@ than the classic client. size estimation available the client will transparently fall back to the classic approach. 2. The chosen peers to store the provider records might not be the actual closest ones. Measurements showed that this is not a problem. -3. The optimistic provide process returns already after 15 out of the 20 provider records were stored with peers. The +3. The optimistic provide process returns already after 15 out of the 20 provider records were stored with peers. The reasoning here is that one out of the remaining 5 peers are very likely to time out and delay the whole process. To limit the number of in-flight async requests there is the second `OptimisticProvideJobsPoolSize` setting. Currently, this is set to 60. This means that at most 60 parallel background requests are allowed to be in-flight. If this diff --git a/go.mod b/go.mod index a756f0f32068..93c198201a44 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/gogo/protobuf v1.3.2 github.com/google/uuid v1.3.0 github.com/hashicorp/go-multierror v1.1.1 - github.com/ipfs/boxo v0.8.2-0.20230510114019-33e3f0cd052b + github.com/ipfs/boxo v0.8.2-0.20230519132256-d6bf836e8808 github.com/ipfs/go-block-format v0.1.2 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-cidutil v0.1.0 @@ -81,6 +81,7 @@ require ( go.uber.org/fx v1.19.2 go.uber.org/zap v1.24.0 golang.org/x/crypto v0.9.0 + golang.org/x/exp v0.0.0-20230321023759-10a507213a29 golang.org/x/mod v0.10.0 golang.org/x/sync v0.1.0 golang.org/x/sys v0.8.0 @@ -92,7 +93,6 @@ require ( github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/cenkalti/backoff v2.2.1+incompatible // indirect github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/containerd/cgroups v1.1.0 // indirect @@ -215,7 +215,6 @@ require ( go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.11.0 // indirect go4.org v0.0.0-20230225012048-214862532bf5 // indirect - golang.org/x/exp v0.0.0-20230321023759-10a507213a29 // indirect golang.org/x/net v0.10.0 // indirect golang.org/x/oauth2 v0.5.0 // indirect golang.org/x/term v0.8.0 // indirect diff --git a/go.sum b/go.sum index af7ec2fc01ae..93a8b107ccab 100644 --- a/go.sum +++ b/go.sum @@ -84,8 +84,6 @@ github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= -github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= -github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/cenkalti/backoff/v4 v4.2.0 h1:HN5dHm3WBOgndBH6E8V0q2jIYIR3s9yglV8k/+MN3u4= github.com/cenkalti/backoff/v4 v4.2.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -356,8 +354,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.8.2-0.20230510114019-33e3f0cd052b h1:6EVpfwbBgwhfZOA19i55jOGokKOy+OaQAm1dg4RbXmc= -github.com/ipfs/boxo v0.8.2-0.20230510114019-33e3f0cd052b/go.mod h1:Ej2r08Z4VIaFKqY08UXMNhwcLf6VekHhK8c+KqA1B9Y= +github.com/ipfs/boxo v0.8.2-0.20230519132256-d6bf836e8808 h1:8yjrpVR7hgXsHKoEM4rFQKcpDATkr5uv1qeBEYZDp7E= +github.com/ipfs/boxo v0.8.2-0.20230519132256-d6bf836e8808/go.mod h1:LbIQUU7IEKw2AMm8sCLgJ/hljd/4yyiMtPzc68nveOc= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= diff --git a/repo/fsrepo/fsrepo.go b/repo/fsrepo/fsrepo.go index cb8de08c1deb..3c4a709c1955 100644 --- a/repo/fsrepo/fsrepo.go +++ b/repo/fsrepo/fsrepo.go @@ -37,7 +37,7 @@ const LockFile = "repo.lock" var log = logging.Logger("fsrepo") // RepoVersion is the version number that we are currently expecting to see -var RepoVersion = 13 +var RepoVersion = 14 var migrationInstructions = `See https://github.com/ipfs/fs-repo-migrations/blob/master/run.md Sorry for the inconvenience. In the future, these will run automatically.` diff --git a/repo/fsrepo/migrations/fetcher.go b/repo/fsrepo/migrations/fetcher.go index c174b5e77751..1dc4d0345c78 100644 --- a/repo/fsrepo/migrations/fetcher.go +++ b/repo/fsrepo/migrations/fetcher.go @@ -11,7 +11,7 @@ import ( const ( // Current distribution to fetch migrations from - CurrentIpfsDist = "/ipfs/Qmf4yftD4LuMo8JMNPqqw3BtUwYd2VkXMiAThuPE6usrbQ" // fs-repo-12-to-13 v1.0.0 + CurrentIpfsDist = "/ipfs/QmYerugGRCZWA8yQMKDsd9daEVXUR3C5nuw3VXuX1mggHa" // fs-repo-13-to-14 v1.0.0 // Latest distribution path. Default for fetchers. LatestIpfsDist = "/ipns/dist.ipfs.tech"