diff --git a/CHANGELOG.md b/CHANGELOG.md index ce711da..76bec69 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ The following emojis are used to highlight certain changes: ### Added - Tracing per request with auth header (see `RAINBOW_TRACING_AUTH`) or a fraction of requests (see `RAINBOW_SAMPLING_FRACTION`) +- Debugging with [`Rainbow-No-Blockcache`](./docs/headers.md#rainbow-no-blockcache) that is gated by the `Authorization` header and does not use the local block cache for the request ### Changed diff --git a/docs/headers.md b/docs/headers.md index 42fe4de..b9381a7 100644 --- a/docs/headers.md +++ b/docs/headers.md @@ -1,9 +1,21 @@ ## `Authorization` -Optional request header that guards per-request tracing features. +Optional request header that guards per-request tracing and debugging features. See [`RAINBOW_TRACING_AUTH`](./environment-variables.md#rainbow_tracing_auth) ## `Traceparent` See [`RAINBOW_TRACING_AUTH`](./environment-variables.md#rainbow_tracing_auth) + +## `Tracestate` + +See [`RAINBOW_TRACING_AUTH`](./environment-variables.md#rainbow_tracing_auth) + +## Rainbow-No-Blockcache + +If the value is `true` the associated request will skip the local block cache and leverage a separate in-memory block cache for the request. + +This header is not respected unless the request has a valid `Authorization` header + +See [`RAINBOW_TRACING_AUTH`](./environment-variables.md#rainbow_tracing_auth) diff --git a/go.mod b/go.mod index f91aa38..421ee89 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/ipfs/go-datastore v0.6.0 github.com/ipfs/go-ds-badger4 v0.1.5 github.com/ipfs/go-ds-flatfs v0.5.1 + github.com/ipfs/go-ds-leveldb v0.5.0 github.com/ipfs/go-ipfs-delay v0.0.1 github.com/ipfs/go-log/v2 v2.5.1 github.com/ipfs/go-metrics-interface v0.0.1 @@ -176,6 +177,7 @@ require ( github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/samber/lo v1.39.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect + github.com/syndtr/goleveldb v1.0.0 // indirect github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb // indirect github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc // indirect github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 // indirect diff --git a/go.sum b/go.sum index 4858555..0f63c66 100644 --- a/go.sum +++ b/go.sum @@ -263,6 +263,8 @@ github.com/ipfs/go-ds-badger4 v0.1.5/go.mod h1:LUU2FbhNdmhAbJmMeoahVRbe4GsduAODS github.com/ipfs/go-ds-flatfs v0.5.1 h1:ZCIO/kQOS/PSh3vcF1H6a8fkRGS7pOfwfPdx4n/KJH4= github.com/ipfs/go-ds-flatfs v0.5.1/go.mod h1:RWTV7oZD/yZYBKdbVIFXTX2fdY2Tbvl94NsWqmoyAX4= github.com/ipfs/go-ds-leveldb v0.1.0/go.mod h1:hqAW8y4bwX5LWcCtku2rFNX3vjDZCy5LZCg+cSZvYb8= +github.com/ipfs/go-ds-leveldb v0.5.0 h1:s++MEBbD3ZKc9/8/njrn4flZLnCuY9I79v94gBUNumo= +github.com/ipfs/go-ds-leveldb v0.5.0/go.mod h1:d3XG9RUDzQ6V4SHi8+Xgj9j1XuEk1z82lquxrVbml/Q= github.com/ipfs/go-ipfs-blockstore v1.3.1 h1:cEI9ci7V0sRNivqaOr0elDsamxXFxJMMMy7PTTDQNsQ= github.com/ipfs/go-ipfs-blockstore v1.3.1/go.mod h1:KgtZyc9fq+P2xJUiCAzbRdhhqJHvsw8u2Dlqy2MyRTE= github.com/ipfs/go-ipfs-blocksutil v0.0.1 h1:Eh/H4pc1hsvhzsQoMEP3Bke/aW5P5rVM1IWFJMcGIPQ= @@ -491,8 +493,12 @@ github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/n github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= +github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY= +github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/ginkgo/v2 v2.19.0 h1:9Cnnf7UHo57Hy3k6/m5k3dRfGTMXGvxhHFvkDTCTpvA= github.com/onsi/ginkgo/v2 v2.19.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= @@ -669,6 +675,7 @@ github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk= @@ -1016,6 +1023,7 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/src-d/go-cli.v0 v0.0.0-20181105080154-d492247bbc0d/go.mod h1:z+K8VcOYVYcSwSjGebuDL6176A1XskgbtNl64NSg+n8= gopkg.in/src-d/go-log.v1 v1.0.1/go.mod h1:GN34hKP0g305ysm2/hctJ0Y8nWP3zxXXJ8GFabTyABE= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/handler_test.go b/handler_test.go index c246166..48c041c 100644 --- a/handler_test.go +++ b/handler_test.go @@ -1,8 +1,18 @@ package main import ( + "context" + "crypto/rand" + "io" "net/http" "testing" + "time" + + bsnet "github.com/ipfs/boxo/bitswap/network" + bsserver "github.com/ipfs/boxo/bitswap/server" + "github.com/ipfs/go-metrics-interface" + "github.com/libp2p/go-libp2p" + "github.com/libp2p/go-libp2p/core/peer" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -14,6 +24,7 @@ func TestTrustless(t *testing.T) { ts, gnd := mustTestServer(t, Config{ Bitswap: true, TrustlessGatewayDomains: []string{"trustless.com"}, + disableMetrics: true, }) content := "hello world" @@ -51,3 +62,137 @@ func TestTrustless(t *testing.T) { assert.Equal(t, http.StatusOK, res.StatusCode) }) } + +func TestNoBlockcacheHeader(t *testing.T) { + const authToken = "authorized" + const authHeader = "Authorization" + ts, gnd := mustTestServer(t, Config{ + Bitswap: true, + TracingAuthToken: authToken, + disableMetrics: true, + }) + + content := make([]byte, 1024) + _, err := rand.Read(content) + require.NoError(t, err) + cid := mustAddFile(t, gnd, content) + url := ts.URL + "/ipfs/" + cid.String() + + t.Run("Successful download of data with standard already cached in the node", func(t *testing.T) { + req, err := http.NewRequest(http.MethodGet, url, nil) + require.NoError(t, err) + + res, err := http.DefaultClient.Do(req) + assert.NoError(t, err) + assert.Equal(t, http.StatusOK, res.StatusCode) + responseBody, err := io.ReadAll(res.Body) + assert.NoError(t, err) + assert.Equal(t, content, responseBody) + }) + + t.Run("When caching is explicitly skipped the data is not found", func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + require.NoError(t, err) + + // Both headers present, expect NoBlockcacheHeader to work + req.Header.Set(NoBlockcacheHeader, "true") + req.Header.Set(authHeader, authToken) + + _, err = http.DefaultClient.Do(req) + assert.ErrorIs(t, err, context.DeadlineExceeded) + }) + + t.Run("When caching is explicitly skipped the data is found if a peer has it", func(t *testing.T) { + newHost, err := libp2p.New() + require.NoError(t, err) + + ctx := context.Background() + // pacify metrics reporting code + ctx = metrics.CtxScope(ctx, "test.bsserver.host") + n := bsnet.NewFromIpfsHost(newHost, nil) + bs := bsserver.New(ctx, n, gnd.blockstore) + n.Start(bs) + defer bs.Close() + + require.NoError(t, newHost.Connect(context.Background(), peer.AddrInfo{ + ID: gnd.host.ID(), + Addrs: gnd.host.Addrs(), + })) + + ctx, cancel := context.WithTimeout(ctx, time.Second*1) + defer cancel() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + require.NoError(t, err) + + // Both headers present, expect NoBlockcacheHeader to work + req.Header.Set(NoBlockcacheHeader, "true") + req.Header.Set(authHeader, authToken) + + res, err := http.DefaultClient.Do(req) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, res.StatusCode) + responseBody, err := io.ReadAll(res.Body) + assert.NoError(t, err) + assert.Equal(t, content, responseBody) + }) + + t.Run("Skipping the cache only works when 'true' is passed", func(t *testing.T) { + req, err := http.NewRequest(http.MethodGet, url, nil) + require.NoError(t, err) + + // Both headers present, but NoBlockcacheHeader is not 'true' + req.Header.Set(NoBlockcacheHeader, "1") + req.Header.Set(authHeader, authToken) + + res, err := http.DefaultClient.Do(req) + assert.NoError(t, err) + assert.Equal(t, http.StatusOK, res.StatusCode) + responseBody, err := io.ReadAll(res.Body) + assert.NoError(t, err) + assert.Equal(t, content, responseBody) + }) + + t.Run("Skipping the cache only works when the Authorization header matches", func(t *testing.T) { + req, err := http.NewRequest(http.MethodGet, url, nil) + require.NoError(t, err) + + // Authorization missing, expect NoBlockcacheHeader to be ignored + req.Header.Set(NoBlockcacheHeader, "true") + + res, err := http.DefaultClient.Do(req) + assert.NoError(t, err) + assert.Equal(t, http.StatusOK, res.StatusCode) + responseBody, err := io.ReadAll(res.Body) + assert.NoError(t, err) + assert.Equal(t, content, responseBody) + }) + + t.Run("Skipping the cache only works when RAINBOW_TRACING_AUTH is set", func(t *testing.T) { + // Set up separate server without authToken set + ts2, gnd := mustTestServer(t, Config{ + Bitswap: true, + TracingAuthToken: "", // simulate RAINBOW_TRACING_AUTH being not set + disableMetrics: true, + }) + content := make([]byte, 1024) + _, err := rand.Read(content) + require.NoError(t, err) + cid2 := mustAddFile(t, gnd, content) + url := ts2.URL + "/ipfs/" + cid2.String() + + req, err := http.NewRequest(http.MethodGet, url, nil) + require.NoError(t, err) + + // Authorization missing, expect NoBlockcacheHeader to be ignored + req.Header.Set(NoBlockcacheHeader, "true") + + res, err := http.DefaultClient.Do(req) + assert.NoError(t, err) + assert.Equal(t, http.StatusOK, res.StatusCode) + responseBody, err := io.ReadAll(res.Body) + assert.NoError(t, err) + assert.Equal(t, content, responseBody) + }) +} diff --git a/handlers.go b/handlers.go index ebddfb8..05ba9c8 100644 --- a/handlers.go +++ b/handlers.go @@ -1,6 +1,7 @@ package main import ( + "context" "encoding/json" "fmt" "net/http" @@ -8,6 +9,9 @@ import ( "runtime" "strconv" + "github.com/ipfs/boxo/blockstore" + leveldb "github.com/ipfs/go-ds-leveldb" + _ "embed" _ "net/http/pprof" @@ -84,7 +88,7 @@ func withRequestLogger(next http.Handler) http.Handler { }) } -func setupGatewayHandler(cfg Config, nd *Node, tracingAuth string) (http.Handler, error) { +func setupGatewayHandler(cfg Config, nd *Node) (http.Handler, error) { var ( backend gateway.IPFSBackend err error @@ -175,8 +179,9 @@ func setupGatewayHandler(cfg Config, nd *Node, tracingAuth string) (http.Handler NoDNSLink: noDNSLink, } gwHandler := gateway.NewHandler(gwConf, backend) - ipfsHandler := withHTTPMetrics(gwHandler, "ipfs") - ipnsHandler := withHTTPMetrics(gwHandler, "ipns") + + ipfsHandler := withHTTPMetrics(gwHandler, "ipfs", cfg.disableMetrics) + ipnsHandler := withHTTPMetrics(gwHandler, "ipns", cfg.disableMetrics) topMux := http.NewServeMux() topMux.Handle("/ipfs/", ipfsHandler) @@ -206,25 +211,49 @@ func setupGatewayHandler(cfg Config, nd *Node, tracingAuth string) (http.Handler handler = withRequestLogger(handler) // Add tracing. - handler = otelhttp.NewHandler(handler, "Gateway") + handler = withTracingAndDebug(handler, cfg.TracingAuthToken) + + return handler, nil +} + +func withTracingAndDebug(next http.Handler, authToken string) http.Handler { + next = otelhttp.NewHandler(next, "Gateway") - // Remove tracing headers if not authorized - prevHandler := handler - handler = http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { - if request.Header.Get("Authorization") != tracingAuth { + // Remove tracing and cache skipping headers if not authorized + return http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + // Disable tracing/debug headers if auth token missing or invalid + if authToken == "" || request.Header.Get("Authorization") != authToken { if request.Header.Get("Traceparent") != "" { request.Header.Del("Traceparent") } if request.Header.Get("Tracestate") != "" { request.Header.Del("Tracestate") } + if request.Header.Get(NoBlockcacheHeader) != "" { + request.Header.Del(NoBlockcacheHeader) + } } - prevHandler.ServeHTTP(writer, request) - }) - return handler, nil + // Process cache skipping header + if noBlockCache := request.Header.Get(NoBlockcacheHeader); noBlockCache == "true" { + ds, err := leveldb.NewDatastore("", nil) + if err != nil { + writer.WriteHeader(http.StatusInternalServerError) + _, _ = writer.Write([]byte(err.Error())) + return + } + newCtx := context.WithValue(request.Context(), NoBlockcache{}, blockstore.NewBlockstore(ds)) + request = request.WithContext(newCtx) + } + + next.ServeHTTP(writer, request) + }) } +const NoBlockcacheHeader = "Rainbow-No-Blockcache" + +type NoBlockcache struct{} + // MutexFractionOption allows to set runtime.SetMutexProfileFraction via HTTP // using POST request with parameter 'fraction'. func MutexFractionOption(path string, mux *http.ServeMux) *http.ServeMux { diff --git a/main.go b/main.go index 84f3d2e..dbd71bc 100644 --- a/main.go +++ b/main.go @@ -453,6 +453,7 @@ share the same seed as long as the indexes are different. GCInterval: cctx.Duration("gc-interval"), GCThreshold: cctx.Float64("gc-threshold"), ListenAddrs: cctx.StringSlice("libp2p-listen-addrs"), + TracingAuthToken: cctx.String("tracing-auth"), } var gnd *Node @@ -471,8 +472,7 @@ share the same seed as long as the indexes are different. gatewayListen := cctx.String("gateway-listen-address") ctlListen := cctx.String("ctl-listen-address") - tracingAuth := cctx.String("tracing-auth") - handler, err := setupGatewayHandler(cfg, gnd, tracingAuth) + handler, err := setupGatewayHandler(cfg, gnd) if err != nil { return err } diff --git a/main_test.go b/main_test.go index a4eb39b..33c0354 100644 --- a/main_test.go +++ b/main_test.go @@ -71,7 +71,7 @@ func mustTestNodeWithKey(t *testing.T, cfg Config, sk ic.PrivKey) *Node { func mustTestServer(t *testing.T, cfg Config) (*httptest.Server, *Node) { nd := mustTestNode(t, cfg) - handler, err := setupGatewayHandler(cfg, nd, "") + handler, err := setupGatewayHandler(cfg, nd) if err != nil { require.NoError(t, err) } diff --git a/metrics.go b/metrics.go index 45f47ab..f0c3fe8 100644 --- a/metrics.go +++ b/metrics.go @@ -24,7 +24,10 @@ var defaultDurationHistogramBuckets = []float64{0.05, 0.1, 0.25, 0.5, 1, 2, 5, 1 // withHTTPMetrics collects metrics around HTTP request/response count, duration, and size // per specific handler. Allows us to track them separately for /ipns and /ipfs. -func withHTTPMetrics(handler http.Handler, handlerName string) http.Handler { +func withHTTPMetrics(handler http.Handler, handlerName string, disableMetrics bool) http.Handler { + if disableMetrics { + return handler + } opts := prometheus.HistogramOpts{ Namespace: "ipfs", diff --git a/setup.go b/setup.go index f86c679..06ce132 100644 --- a/setup.go +++ b/setup.go @@ -22,6 +22,8 @@ import ( "github.com/ipfs/boxo/namesys" "github.com/ipfs/boxo/path/resolver" "github.com/ipfs/boxo/peering" + blocks "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" badger4 "github.com/ipfs/go-ds-badger4" flatfs "github.com/ipfs/go-ds-flatfs" @@ -121,6 +123,10 @@ type Config struct { GCInterval time.Duration GCThreshold float64 + + TracingAuthToken string + + disableMetrics bool // only meant to be used during testing } func SetupNoLibp2p(ctx context.Context, cfg Config, dnsCache *cachedDNS) (*Node, error) { @@ -277,6 +283,10 @@ func SetupWithLibp2p(ctx context.Context, cfg Config, key crypto.PrivKey, dnsCac // See also comment in blockservice. blockstore.WriteThrough(), ) + blkst = &switchingBlockstore{ + baseBlockstore: blkst, + contextSwitchingKey: NoBlockcache{}, + } blkst = blockstore.NewIdStore(blkst) n.blockstore = blkst @@ -494,3 +504,50 @@ func setupNamesys(cfg Config, vs routing.ValueStore, blocker *nopfs.Blocker) (na ns = nopfsipfs.WrapNameSystem(ns, blocker) return ns, nil } + +type switchingBlockstore struct { + baseBlockstore blockstore.Blockstore + contextSwitchingKey any +} + +func (s *switchingBlockstore) getBlockstore(ctx context.Context) blockstore.Blockstore { + alternativeBlockstore, ok := ctx.Value(s.contextSwitchingKey).(blockstore.Blockstore) + if ok { + return alternativeBlockstore + } + return s.baseBlockstore +} + +func (s *switchingBlockstore) DeleteBlock(ctx context.Context, c cid.Cid) error { + return s.getBlockstore(ctx).DeleteBlock(ctx, c) +} + +func (s *switchingBlockstore) Has(ctx context.Context, c cid.Cid) (bool, error) { + return s.getBlockstore(ctx).Has(ctx, c) +} + +func (s *switchingBlockstore) Get(ctx context.Context, c cid.Cid) (blocks.Block, error) { + return s.getBlockstore(ctx).Get(ctx, c) +} + +func (s *switchingBlockstore) GetSize(ctx context.Context, c cid.Cid) (int, error) { + return s.getBlockstore(ctx).GetSize(ctx, c) +} + +func (s *switchingBlockstore) Put(ctx context.Context, block blocks.Block) error { + return s.getBlockstore(ctx).Put(ctx, block) +} + +func (s *switchingBlockstore) PutMany(ctx context.Context, blocks []blocks.Block) error { + return s.getBlockstore(ctx).PutMany(ctx, blocks) +} + +func (s *switchingBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) { + return s.getBlockstore(ctx).AllKeysChan(ctx) +} + +func (s *switchingBlockstore) HashOnRead(enabled bool) { + s.baseBlockstore.HashOnRead(enabled) +} + +var _ blockstore.Blockstore = (*switchingBlockstore)(nil)