feat: Gateway.DeserializedResponses config flag (#9789)

Co-authored-by: Marcin Rataj <lidel@lidel.org>

config/gateway.go

@@ -1,6 +1,9 @@
 package config
 
-const DefaultInlineDNSLink = false
+const (
+	DefaultInlineDNSLink         = false
+	DefaultDeserializedResponses = true
+)
 
 type GatewaySpec struct {
 	// Paths is explicit list of path prefixes that should be handled by
@@ -25,6 +28,11 @@ type GatewaySpec struct {
 	// (FQDN) into a single DNS label in order to interop with wildcard TLS certs
 	// and Origin per CID isolation provided by rules like https://publicsuffix.org
 	InlineDNSLink Flag
+
+	// DeserializedResponses configures this gateway to respond to deserialized
+	// responses. Disabling this option enables a Trustless Gateway, as per:
+	// https://specs.ipfs.tech/http-gateways/trustless-gateway/.
+	DeserializedResponses Flag
 }
 
 // Gateway contains options for the HTTP gateway server.
@@ -56,6 +64,12 @@ type Gateway struct {
 	// This flag can be overridden per FQDN in PublicGateways.
 	NoDNSLink bool
 
+	// DeserializedResponses configures this gateway to respond to deserialized
+	// requests. Disabling this option enables a Trustless only gateway, as per:
+	// https://specs.ipfs.tech/http-gateways/trustless-gateway/. This can
+	// be overridden per FQDN in PublicGateways.
+	DeserializedResponses Flag
+
 	// PublicGateways configures behavior of known public gateways.
 	// Each key is a fully qualified domain name (FQDN).
 	PublicGateways map[string]*GatewaySpec

core/corehttp/gateway.go

@@ -28,22 +28,11 @@ import (
 
 func GatewayOption(paths ...string) ServeOption {
 	return func(n *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) {
-		cfg, err := n.Repo.Config()
+		gwConfig, err := getGatewayConfig(n)
 		if err != nil {
 			return nil, err
 		}
 
-		headers := make(map[string][]string, len(cfg.Gateway.HTTPHeaders))
-		for h, v := range cfg.Gateway.HTTPHeaders {
-			headers[http.CanonicalHeaderKey(h)] = v
-		}
-
-		gateway.AddAccessControlHeaders(headers)
-
-		gwConfig := gateway.Config{
-			Headers: headers,
-		}
-
 		gwAPI, err := newGatewayBackend(n)
 		if err != nil {
 			return nil, err
@@ -65,7 +54,7 @@ func GatewayOption(paths ...string) ServeOption {
 
 func HostnameOption() ServeOption {
 	return func(n *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) {
-		cfg, err := n.Repo.Config()
+		gwConfig, err := getGatewayConfig(n)
 		if err != nil {
 			return nil, err
 		}
@@ -75,9 +64,8 @@ func HostnameOption() ServeOption {
 			return nil, err
 		}
 
-		publicGateways := convertPublicGateways(cfg.Gateway.PublicGateways)
 		childMux := http.NewServeMux()
-		mux.HandleFunc("/", gateway.WithHostname(childMux, gwAPI, publicGateways, cfg.Gateway.NoDNSLink).ServeHTTP)
+		mux.HandleFunc("/", gateway.WithHostname(gwConfig, gwAPI, childMux).ServeHTTP)
 		return childMux, nil
 	}
 }
@@ -212,30 +200,49 @@ var defaultKnownGateways = map[string]*gateway.Specification{
 	"localhost": subdomainGatewaySpec,
 }
 
-func convertPublicGateways(publicGateways map[string]*config.GatewaySpec) map[string]*gateway.Specification {
-	gws := map[string]*gateway.Specification{}
+func getGatewayConfig(n *core.IpfsNode) (gateway.Config, error) {
+	cfg, err := n.Repo.Config()
+	if err != nil {
+		return gateway.Config{}, err
+	}
+
+	// Parse configuration headers and add the default Access Control Headers.
+	headers := make(map[string][]string, len(cfg.Gateway.HTTPHeaders))
+	for h, v := range cfg.Gateway.HTTPHeaders {
+		headers[http.CanonicalHeaderKey(h)] = v
+	}
+	gateway.AddAccessControlHeaders(headers)
+
+	// Initialize gateway configuration, with empty PublicGateways, handled after.
+	gwCfg := gateway.Config{
+		Headers:               headers,
+		DeserializedResponses: cfg.Gateway.DeserializedResponses.WithDefault(config.DefaultDeserializedResponses),
+		NoDNSLink:             cfg.Gateway.NoDNSLink,
+		PublicGateways:        map[string]*gateway.Specification{},
+	}
 
-	// First, implicit defaults such as subdomain gateway on localhost
+	// Add default implicit known gateways, such as subdomain gateway on localhost.
 	for hostname, gw := range defaultKnownGateways {
-		gws[hostname] = gw
+		gwCfg.PublicGateways[hostname] = gw
 	}
 
-	// Then apply values from Gateway.PublicGateways, if present in the config
-	for hostname, gw := range publicGateways {
+	// Apply values from cfg.Gateway.PublicGateways if they exist.
+	for hostname, gw := range cfg.Gateway.PublicGateways {
 		if gw == nil {
 			// Remove any implicit defaults, if present. This is useful when one
-			// wants to disable subdomain gateway on localhost etc.
-			delete(gws, hostname)
+			// wants to disable subdomain gateway on localhost, etc.
+			delete(gwCfg.PublicGateways, hostname)
 			continue
 		}
 
-		gws[hostname] = &gateway.Specification{
-			Paths:         gw.Paths,
-			NoDNSLink:     gw.NoDNSLink,
-			UseSubdomains: gw.UseSubdomains,
-			InlineDNSLink: gw.InlineDNSLink.WithDefault(config.DefaultInlineDNSLink),
+		gwCfg.PublicGateways[hostname] = &gateway.Specification{
+			Paths:                 gw.Paths,
+			NoDNSLink:             gw.NoDNSLink,
+			UseSubdomains:         gw.UseSubdomains,
+			InlineDNSLink:         gw.InlineDNSLink.WithDefault(config.DefaultInlineDNSLink),
+			DeserializedResponses: gw.DeserializedResponses.WithDefault(gwCfg.DeserializedResponses),
 		}
 	}
 
-	return gws
+	return gwCfg, nil
 }

core/corehttp/gateway_test.go

@@ -14,6 +14,7 @@ import (
 	core "github.com/ipfs/kubo/core"
 	"github.com/ipfs/kubo/core/coreapi"
 	repo "github.com/ipfs/kubo/repo"
+	"github.com/stretchr/testify/assert"
 
 	iface "github.com/ipfs/boxo/coreiface"
 	nsopts "github.com/ipfs/boxo/coreiface/options/namesys"
@@ -173,3 +174,42 @@ func TestVersion(t *testing.T) {
 		t.Fatalf("response doesn't contain protocol version:\n%s", s)
 	}
 }
+
+func TestDeserializedResponsesInheritance(t *testing.T) {
+	for _, testCase := range []struct {
+		globalSetting          config.Flag
+		gatewaySetting         config.Flag
+		expectedGatewaySetting bool
+	}{
+		{config.True, config.Default, true},
+		{config.False, config.Default, false},
+		{config.False, config.True, true},
+		{config.True, config.False, false},
+	} {
+		c := config.Config{
+			Identity: config.Identity{
+				PeerID: "QmTFauExutTsy4XP6JbMFcw2Wa9645HJt2bTqL6qYDCKfe", // required by offline node
+			},
+			Gateway: config.Gateway{
+				DeserializedResponses: testCase.globalSetting,
+				PublicGateways: map[string]*config.GatewaySpec{
+					"example.com": {
+						DeserializedResponses: testCase.gatewaySetting,
+					},
+				},
+			},
+		}
+		r := &repo.Mock{
+			C: c,
+			D: syncds.MutexWrap(datastore.NewMapDatastore()),
+		}
+		n, err := core.NewNode(context.Background(), &core.BuildCfg{Repo: r})
+		assert.NoError(t, err)
+
+		gwCfg, err := getGatewayConfig(n)
+		assert.NoError(t, err)
+
+		assert.Contains(t, gwCfg.PublicGateways, "example.com")
+		assert.Equal(t, testCase.expectedGatewaySetting, gwCfg.PublicGateways["example.com"].DeserializedResponses)
+	}
+}

docs/changelogs/v0.21.md

@@ -7,6 +7,7 @@
 - [Overview](#overview)
 - [🔦 Highlights](#-highlights)
   - [Saving previously seen nodes for later bootstrapping](#saving-previously-seen-nodes-for-later-bootstrapping)
+  - [`Gateway.DeserializedResponses` config flag](#gatewaydeserializedresponses-config-flag)
 - [📝 Changelog](#-changelog)
 - [👨‍👩‍👧‍👦 Contributors](#-contributors)
 
@@ -29,6 +30,32 @@ enabled.
 With this update, the same level of robustness is applied to peers that lack
 mDNS peers and solely rely on the public DHT.
 
+
+#### `Gateway.DeserializedResponses` config flag
+
+This release introduces the
+[`Gateway.DeserializedResponses`](https://github.com/ipfs/kubo/blob/master/docs/config.md#gatewaydeserializedresponses)
+configuration flag.
+
+With this flag, one can explicitly configure whether the gateway responds to
+deserialized requests or not. By default, this flag is enabled.
+
+Disabling deserialized responses allows the
+gateway to operate
+as a [Trustless Gateway](https://specs.ipfs.tech/http-gateways/trustless-gateway/)
+limited to three [verifiable](https://docs.ipfs.tech/reference/http/gateway/#trustless-verifiable-retrieval)
+response types:
+[application/vnd.ipld.raw](https://www.iana.org/assignments/media-types/application/vnd.ipld.raw),
+[application/vnd.ipld.car](https://www.iana.org/assignments/media-types/application/vnd.ipld.car),
+and [application/vnd.ipfs.ipns-record](https://www.iana.org/assignments/media-types/application/vnd.ipfs.ipns-record).
+
+With deserialized responses disabled, the Kubo gateway can serve as a block
+backend for other software (like
+[bifrost-gateway](https://github.com/ipfs/bifrost-gateway#readme),
+[IPFS in Chromium](https://github.com/little-bear-labs/ipfs-chromium/blob/main/README.md)
+etc) without the usual risks associated with hosting deserialized data behind
+third-party CIDs.
+
 ### 📝 Changelog
 
 ### 👨‍👩‍👧‍👦 Contributors

docs/config.md

@@ -50,6 +50,7 @@ config file at runtime.
   - [`Gateway`](#gateway)
     - [`Gateway.NoFetch`](#gatewaynofetch)
     - [`Gateway.NoDNSLink`](#gatewaynodnslink)
+    - [`Gateway.DeserializedResponses`](#gatewaydeserializedresponses)
     - [`Gateway.HTTPHeaders`](#gatewayhttpheaders)
     - [`Gateway.RootRedirect`](#gatewayrootredirect)
     - [`Gateway.FastDirIndexThreshold`](#gatewayfastdirindexthreshold)
@@ -60,6 +61,7 @@ config file at runtime.
       - [`Gateway.PublicGateways: UseSubdomains`](#gatewaypublicgateways-usesubdomains)
       - [`Gateway.PublicGateways: NoDNSLink`](#gatewaypublicgateways-nodnslink)
       - [`Gateway.PublicGateways: InlineDNSLink`](#gatewaypublicgateways-inlinednslink)
+      - [`Gateway.PublicGateways: DeserializedResponses`](#gatewaypublicgateways-deserializedresponses)
       - [Implicit defaults of `Gateway.PublicGateways`](#implicit-defaults-of-gatewaypublicgateways)
     - [`Gateway` recipes](#gateway-recipes)
   - [`Identity`](#identity)
@@ -236,7 +238,7 @@ documented in `ipfs config profile --help`.
     smaller than several gigabytes. If you run IPFS with `--enable-gc`, you plan on storing very little data in
     your IPFS node, and disk usage is more critical than performance, consider using
     `flatfs`.
-  - This datastore uses up to several gigabytes of memory.  
+  - This datastore uses up to several gigabytes of memory.
   - Good for medium-size datastores, but may run into performance issues if your dataset is bigger than a terabyte.
   - The current implementation is based on old badger 1.x which is no longer supported by the upstream team.
 
@@ -646,6 +648,16 @@ Default: `false`
 
 Type: `bool`
 
+#### `Gateway.DeserializedResponses`
+
+An optional flag to explicitly configure whether this gateway responds to deserialized
+requests, or not. By default, it is enabled. When disabling this option, the gateway
+operates as a Trustless Gateway only: https://specs.ipfs.tech/http-gateways/trustless-gateway/.
+
+Default: `true`
+
+Type: `flag`
+
 ### `Gateway.HTTPHeaders`
 
 Headers to set on gateway responses.
@@ -790,6 +802,16 @@ Default: `false`
 
 Type: `flag`
 
+#### `Gateway.PublicGateways: DeserializedResponses`
+
+An optional flag to explicitly configure whether this gateway responds to deserialized
+requests, or not. By default, it is enabled. When disabling this option, the gateway
+operates as a Trustless Gateway only: https://specs.ipfs.tech/http-gateways/trustless-gateway/.
+
+Default: same as global `Gateway.DeserializedResponses`
+
+Type: `flag`
+
 #### Implicit defaults of `Gateway.PublicGateways`
 
 Default entries for `localhost` hostname and loopback IPs are always present.
@@ -895,7 +917,7 @@ Type: `string` (base64 encoded)
 
 ## `Internal`
 
-This section includes internal knobs for various subsystems to allow advanced users with big or private infrastructures to fine-tune some behaviors without the need to recompile Kubo.  
+This section includes internal knobs for various subsystems to allow advanced users with big or private infrastructures to fine-tune some behaviors without the need to recompile Kubo.
 
 **Be aware that making informed change here requires in-depth knowledge and most users should leave these untouched. All knobs listed here are subject to breaking changes between versions.**
 
@@ -971,7 +993,7 @@ Type: `optionalInteger` (byte count, `null` means default which is 1MB)
 ### `Internal.Bitswap.ProviderSearchDelay`
 
 This parameter determines how long to wait before looking for providers outside of bitswap.
-Other routing systems like the DHT are able to provide results in less than a second, so lowering 
+Other routing systems like the DHT are able to provide results in less than a second, so lowering
 this number will allow faster peers lookups in some cases.
 
 Type: `optionalDuration` (`null` means default which is 1s)
@@ -1552,7 +1574,7 @@ another node, even if this other node is on a different network. This may
 trigger netscan alerts on some hosting providers or cause strain in some setups.
 
 The `server` configuration profile fills up this list with sensible defaults,
-preventing dials to all non-routable IP addresses (e.g., `/ip4/192.168.0.0/ipcidr/16`, 
+preventing dials to all non-routable IP addresses (e.g., `/ip4/192.168.0.0/ipcidr/16`,
 which is the multiaddress representation of `192.168.0.0/16`) but you should always
 check settings against your own network and/or hosting provider.
 

docs/examples/kubo-as-a-library/go.mod

@@ -7,7 +7,7 @@ go 1.18
 replace github.com/ipfs/kubo => ./../../..
 
 require (
-	github.com/ipfs/boxo v0.8.2-0.20230525115135-a8533c998f49
+	github.com/ipfs/boxo v0.8.2-0.20230529214945-86cdb2485dad
 	github.com/ipfs/kubo v0.0.0-00010101000000-000000000000
 	github.com/libp2p/go-libp2p v0.27.3
 	github.com/multiformats/go-multiaddr v0.9.0

docs/examples/kubo-as-a-library/go.sum

@@ -321,8 +321,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
 github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
-github.com/ipfs/boxo v0.8.2-0.20230525115135-a8533c998f49 h1:hi2x0dCINl9fHIV6YM+IH+Bah45pRAFekjM5MMKWJO4=
-github.com/ipfs/boxo v0.8.2-0.20230525115135-a8533c998f49/go.mod h1:Ej2r08Z4VIaFKqY08UXMNhwcLf6VekHhK8c+KqA1B9Y=
+github.com/ipfs/boxo v0.8.2-0.20230529214945-86cdb2485dad h1:2vkMvvVa5f9fWzts7OcJL6ZS0QaKCcEeOV6I+doPMo0=
+github.com/ipfs/boxo v0.8.2-0.20230529214945-86cdb2485dad/go.mod h1:Ej2r08Z4VIaFKqY08UXMNhwcLf6VekHhK8c+KqA1B9Y=
 github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
 github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
 github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY=

go.mod

@@ -16,7 +16,7 @@ require (
 	github.com/gogo/protobuf v1.3.2
 	github.com/google/uuid v1.3.0
 	github.com/hashicorp/go-multierror v1.1.1
-	github.com/ipfs/boxo v0.8.2-0.20230525115135-a8533c998f49
+	github.com/ipfs/boxo v0.8.2-0.20230529214945-86cdb2485dad
 	github.com/ipfs/go-block-format v0.1.2
 	github.com/ipfs/go-cid v0.4.1
 	github.com/ipfs/go-cidutil v0.1.0

go.sum

@@ -356,8 +356,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
 github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
-github.com/ipfs/boxo v0.8.2-0.20230525115135-a8533c998f49 h1:hi2x0dCINl9fHIV6YM+IH+Bah45pRAFekjM5MMKWJO4=
-github.com/ipfs/boxo v0.8.2-0.20230525115135-a8533c998f49/go.mod h1:Ej2r08Z4VIaFKqY08UXMNhwcLf6VekHhK8c+KqA1B9Y=
+github.com/ipfs/boxo v0.8.2-0.20230529214945-86cdb2485dad h1:2vkMvvVa5f9fWzts7OcJL6ZS0QaKCcEeOV6I+doPMo0=
+github.com/ipfs/boxo v0.8.2-0.20230529214945-86cdb2485dad/go.mod h1:Ej2r08Z4VIaFKqY08UXMNhwcLf6VekHhK8c+KqA1B9Y=
 github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
 github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
 github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY=