feat(routing/http): support IPIP-484 and streaming (#10534)

config/routing.go

@@ -3,14 +3,31 @@ package config
 import (
 	"encoding/json"
 	"fmt"
+	"os"
 	"runtime"
+	"strings"
 )
 
-var (
+const (
 	DefaultAcceleratedDHTClient      = false
 	DefaultLoopbackAddressesOnLanDHT = false
 )
 
+var (
+	// Default HTTP routers used in parallel to DHT when Routing.Type = "auto"
+	DefaultHTTPRouters = getEnvOrDefault("IPFS_HTTP_ROUTERS", []string{
+		"https://cid.contact", // https://github.com/ipfs/kubo/issues/9422#issuecomment-1338142084
+	})
+
+	// Default filter-protocols to pass along with delegated routing requests (as defined in IPIP-484)
+	// and also filter out locally
+	DefaultHTTPRoutersFilterProtocols = getEnvOrDefault("IPFS_HTTP_ROUTERS_FILTER_PROTOCOLS", []string{
+		"unknown", // allow results without protocol list, we can do libp2p identify to test them
+		"transport-bitswap",
+		// TODO: add 'transport-ipfs-gateway-http' once https://github.com/ipfs/rainbow/issues/125 is addressed
+	})
+)
+
 // Routing defines configuration options for libp2p routing.
 type Routing struct {
 	// Type sets default daemon routing mode.
@@ -180,3 +197,13 @@ type ConfigRouter struct {
 type Method struct {
 	RouterName string
 }
+
+// getEnvOrDefault reads space or comma separated strings from env if present,
+// and uses provided defaultValue as a fallback
+func getEnvOrDefault(key string, defaultValue []string) []string {
+	if value, exists := os.LookupEnv(key); exists {
+		splitFunc := func(r rune) bool { return r == ',' || r == ' ' }
+		return strings.FieldsFunc(value, splitFunc)
+	}
+	return defaultValue
+}

core/node/libp2p/routingopt.go

@@ -2,8 +2,6 @@ package libp2p
 
 import (
 	"context"
-	"os"
-	"strings"
 	"time"
 
 	"github.com/ipfs/go-datastore"
@@ -31,23 +29,10 @@ type RoutingOptionArgs struct {
 
 type RoutingOption func(args RoutingOptionArgs) (routing.Routing, error)
 
-// Default HTTP routers used in parallel to DHT when Routing.Type = "auto"
-var defaultHTTPRouters = []string{
-	"https://cid.contact", // https://github.com/ipfs/kubo/issues/9422#issuecomment-1338142084
-	// TODO: add an independent router from Cloudflare
-}
-
-func init() {
-	// Override HTTP routers if custom ones were passed via env
-	if routers := os.Getenv("IPFS_HTTP_ROUTERS"); routers != "" {
-		defaultHTTPRouters = strings.Split(routers, " ")
-	}
-}
-
 func constructDefaultHTTPRouters(cfg *config.Config) ([]*routinghelpers.ParallelRouter, error) {
 	var routers []*routinghelpers.ParallelRouter
 	// Append HTTP routers for additional speed
-	for _, endpoint := range defaultHTTPRouters {
+	for _, endpoint := range config.DefaultHTTPRouters {
 		httpRouter, err := irouting.ConstructHTTPRouter(endpoint, cfg.Identity.PeerID, httpAddrsFromConfig(cfg.Addresses), cfg.Identity.PrivKey)
 		if err != nil {
 			return nil, err

docs/environment-variables.md

@@ -131,6 +131,18 @@ The above will replace implicit HTTP routers with single one, allowing for
 inspection/debug of HTTP requests sent by Kubo via `while true ; do nc -l 7423; done`
 or more advanced tools like [mitmproxy](https://docs.mitmproxy.org/stable/#mitmproxy).
 
+Default: `config.DefaultHTTPRouters`
+
+## `IPFS_HTTP_ROUTERS_FILTER_PROTOCOLS`
+
+Overrides values passed with `filter-protocols` parameter defined in IPIP-484.
+Value is space-separated.
+
+```console
+$ IPFS_HTTP_ROUTERS_FILTER_PROTOCOLS="unknown transport-bitswap transport-foo" ipfs daemon
+```
+
+Default: `config.DefaultHTTPRoutersFilterProtocols`
 
 ## `IPFS_CONTENT_BLOCKING_DISABLE`
 

routing/delegated.go

@@ -206,6 +206,9 @@ func httpRoutingFromConfig(conf config.Router, extraHTTP *ExtraHTTPParams) (rout
 		drclient.WithIdentity(key),
 		drclient.WithProviderInfo(addrInfo.ID, addrInfo.Addrs),
 		drclient.WithUserAgent(version.GetUserAgentVersion()),
+		drclient.WithProtocolFilter(config.DefaultHTTPRoutersFilterProtocols),
+		drclient.WithStreamResultsRequired(),       // https://specs.ipfs.tech/routing/http-routing-v1/#streaming
+		drclient.WithDisabledLocalFiltering(false), // force local filtering in case remote server does not support IPIP-484
 	)
 	if err != nil {
 		return nil, err