feat: --compression-level for gzip responses

README.md

@@ -32,9 +32,10 @@ Full argument list:
 * `--announce` - announce the given roots to IPNI on startup. Can be `roots` or `none`. Defaults to `none`.
 * `--listen` - hostname and port to listen on. Defaults to `:3747`.
 * `--public-addr` - multiaddr or URL of this server as seen by the indexer and other peers if it is different to the listen address. Defaults address of the server once started (typically the value of `--listen`).
-* `--log-file` - path to file to append HTTP request and error logs to. Defaults to `stdout`.
+* `--log-file` - path to file to append HTTP request and error logs to. See [Log format](#log-format) for details of the log format. Defaults to `stdout`.
 * `--max-response-duration` - maximum duration to spend responding to a request. Defaults to `5m`.
 * `--max-response-bytes` - maximum size of a response from IPNI. Defaults to `100MiB`.
+* `--compression-level` - compression level to use for HTTP response data where the client accepts it. `0`-`9`, `0` is no compression, `9` is maximum compression. Defaults to `0` (none).
 * `--verbose` - enable verbose logging. Defaults to `false`. Same as using `GOLOG_LOG_LEVEL=debug` as an environment variable. `GOLOG_LOG_LEVEL` can be used for more fine-grained control of log output.
 * `--help` - show help.
 
@@ -54,6 +55,27 @@ See https://pkg.go.dev/github.com/ipld/frisbii for full documentation.
 
 `NewFrisbiiServer()` can be used to create a new server given a `LinkSystem` as a source of IPLD data.
 
+## Log format
+
+Frisbii logs HTTP requests and errors to a log file that is roughly equivalent to a standard nginx or Apache log format; that is, a space-separated list of elements, where the elements that may contain spaces are quoted. The format of each line can be specified as:
+
+```
+%s %s %s "%s" %d %d %d %s "%s" "%s"
+```
+
+Where the elements are:
+
+1. RFC 3339 timestamp
+2. Remote address
+3. Method
+4. Path
+5. Response status code
+6. Response duration (in milliseconds)
+7. Response size
+8. Compression ratio (or `-` if no compression)
+9. User agent
+10. Error (or `""` if no error)
+
 ## Further Development
 
 The goal of Frisbii is to be maximally minimal according to user need. It is not intended to be a full IPFS node, but rather a simple server that can be used to serve IPLD data. However, the limitations of HTTP as the only transport, the restrictions within the current minimal Trustless Gateway implementation, and reliance on IPNI for content announcement present some challenges for data provision to a wide audience.

cmd/frisbii/flags.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"compress/gzip"
 	"errors"
 	"net/url"
 	"path/filepath"
@@ -55,6 +56,11 @@ var Flags = []cli.Flag{
 		Usage: "maximum number of bytes to send in a response (use 0 for no limit)",
 		Value: "100MiB",
 	},
+	&cli.IntFlag{
+		Name:  "compression-level",
+		Usage: "compression level to use for responses, 0-9, 0 is no compression, 9 is maximum compression",
+		Value: gzip.NoCompression,
+	},
 	&cli.BoolFlag{
 		Name:  "verbose",
 		Usage: "enable verbose debug logging to stderr, same as setting GOLOG_LOG_LEVEL=DEBUG",
@@ -78,6 +84,7 @@ type Config struct {
 	LogFile             string
 	MaxResponseDuration time.Duration
 	MaxResponseBytes    int64
+	CompressionLevel    int
 	Verbose             bool
 }
 
@@ -124,6 +131,8 @@ func ToConfig(c *cli.Context) (Config, error) {
 		}
 	}
 
+	compressionLevel := c.Int("compression-level")
+
 	return Config{
 		Cars:                carPaths,
 		Listen:              listen,
@@ -134,6 +143,7 @@ func ToConfig(c *cli.Context) (Config, error) {
 		LogFile:             logFile,
 		MaxResponseDuration: maxResponseDuration,
 		MaxResponseBytes:    int64(maxResponseBytes),
+		CompressionLevel:    compressionLevel,
 		Verbose:             verbose,
 	}, nil
 }

cmd/frisbii/main.go

@@ -7,6 +7,8 @@ import (
 	"os"
 	"os/signal"
 	"strings"
+	"sync"
+	"sync/atomic"
 	"syscall"
 
 	"github.com/ipfs/go-log/v2"
@@ -19,6 +21,7 @@ import (
 	"github.com/libp2p/go-libp2p/core/peer"
 	"github.com/multiformats/go-multiaddr"
 	"github.com/urfave/cli/v2"
+	"go.uber.org/multierr"
 	"golang.org/x/term"
 )
 
@@ -102,9 +105,22 @@ func action(c *cli.Context) error {
 	}
 
 	multicar := frisbii.NewMultiReadableStorage()
+	var wg sync.WaitGroup
+	loader.SetStatus(fmt.Sprintf("Loading CARs (%d / %d) ...", 0, len(config.Cars)))
+	var loaded int64
+	err = nil
 	for ii, carPath := range config.Cars {
-		loader.SetStatus(fmt.Sprintf("Loading CARs (%d / %d) ...", ii+1, len(config.Cars)))
-		util.LoadCar(multicar, carPath)
+		wg.Add(1)
+		go func(ii int, carPath string) {
+			err = multierr.Append(err, util.LoadCar(multicar, carPath))
+			wg.Done()
+			l := atomic.AddInt64(&loaded, 1)
+			loader.SetStatus(fmt.Sprintf("Loading CARs (%d / %d) ...", l, len(config.Cars)))
+		}(ii, carPath)
+	}
+	wg.Wait()
+	if err != nil {
+		return err
 	}
 
 	loader.SetStatus("Loaded CARs, starting server ...")
@@ -128,6 +144,7 @@ func action(c *cli.Context) error {
 		config.Listen,
 		frisbii.WithMaxResponseDuration(config.MaxResponseDuration),
 		frisbii.WithMaxResponseBytes(config.MaxResponseBytes),
+		frisbii.WithCompressionLevel(config.CompressionLevel),
 	)
 	if err != nil {
 		return err

frisbii.go

@@ -50,6 +50,9 @@ func NewFrisbiiServer(
 	if err != nil {
 		return nil, err
 	}
+	if logWriter == nil {
+		logWriter = io.Discard
+	}
 	return &FrisbiiServer{
 		ctx:         ctx,
 		logWriter:   logWriter,
@@ -65,14 +68,13 @@ func (fs *FrisbiiServer) Addr() net.Addr {
 
 func (fs *FrisbiiServer) Serve() error {
 	fs.mux = http.NewServeMux()
-
 	fs.mux.Handle("/ipfs/", NewHttpIpfs(fs.ctx, fs.lsys, fs.httpOptions...))
+	fs.mux.Handle("/", http.NotFoundHandler())
 	server := &http.Server{
 		Addr:        fs.Addr().String(),
 		BaseContext: func(listener net.Listener) context.Context { return fs.ctx },
 		Handler:     NewLogMiddleware(fs.mux, fs.logWriter),
 	}
-	fs.mux.Handle("/", http.NotFoundHandler())
 	logger.Debugf("Serve() server on %s", fs.Addr().String())
 	return server.Serve(fs.listener)
 }

frisbii_test.go

@@ -0,0 +1,170 @@
+package frisbii_test
+
+import (
+	"compress/gzip"
+	"context"
+	"io"
+	"math/rand"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/ipfs/go-cid"
+	unixfstestutil "github.com/ipfs/go-unixfsnode/testutil"
+	"github.com/ipld/frisbii"
+	"github.com/ipld/go-car/v2"
+	unixfsgen "github.com/ipld/go-fixtureplate/generator"
+	cidlink "github.com/ipld/go-ipld-prime/linking/cid"
+	"github.com/ipld/go-ipld-prime/storage/memstore"
+	"github.com/ipld/go-trustless-utils/testutil"
+	"github.com/stretchr/testify/require"
+)
+
+func TestFrisbiiServer(t *testing.T) {
+	testCases := []struct {
+		name                   string
+		acceptGzip             bool
+		noClientCompression    bool
+		serverCompressionLevel int
+		expectGzip             bool
+	}{
+		{
+			name: "default",
+		},
+		{
+			name:                   "no client compression (no server gzip)",
+			noClientCompression:    true,
+			serverCompressionLevel: gzip.NoCompression,
+			expectGzip:             false,
+		},
+		{
+			name:                   "no client compression (with server gzip)",
+			noClientCompression:    true,
+			serverCompressionLevel: gzip.DefaultCompression,
+			expectGzip:             false,
+		},
+		{
+			name:                   "gzip (with server 1)",
+			acceptGzip:             true,
+			serverCompressionLevel: gzip.BestSpeed,
+			expectGzip:             true,
+		},
+		{
+			name:                   "gzip (with server 9)",
+			acceptGzip:             true,
+			serverCompressionLevel: gzip.BestCompression,
+			expectGzip:             true,
+		},
+		{
+			name:                   "gzip (no server gzip)",
+			acceptGzip:             true,
+			serverCompressionLevel: gzip.NoCompression,
+			expectGzip:             false,
+		},
+		{
+			name:                   "gzip transparent (no server gzip)",
+			serverCompressionLevel: gzip.NoCompression,
+			expectGzip:             false,
+		},
+		{
+			name:                   "gzip transparent (with server gzip)",
+			serverCompressionLevel: gzip.DefaultCompression,
+			expectGzip:             true,
+		},
+	}
+
+	rndSeed := time.Now().UTC().UnixNano()
+	t.Logf("random seed: %d", rndSeed)
+	var rndReader io.Reader = rand.New(rand.NewSource(rndSeed))
+
+	store := &testutil.CorrectedMemStore{ParentStore: &memstore.Store{Bag: make(map[string][]byte)}}
+	lsys := cidlink.DefaultLinkSystem()
+	lsys.SetReadStorage(store)
+	lsys.SetWriteStorage(store)
+	lsys.TrustedStorage = true
+
+	entity, err := unixfsgen.Parse("file:1MiB")
+	require.NoError(t, err)
+	t.Logf("Generating: %s", entity.Describe(""))
+	rootEnt, err := entity.Generate(lsys, rndReader)
+	require.NoError(t, err)
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			req := require.New(t)
+			ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+			defer cancel()
+
+			opts := []frisbii.HttpOption{}
+			if tc.serverCompressionLevel != gzip.NoCompression {
+				opts = append(opts, frisbii.WithCompressionLevel(tc.serverCompressionLevel))
+			}
+			server, err := frisbii.NewFrisbiiServer(ctx, nil, lsys, "localhost:0", opts...)
+			req.NoError(err)
+			go func() {
+				req.NoError(server.Serve())
+			}()
+			addr := server.Addr()
+
+			request, err := http.NewRequest("GET", "http://"+addr.String()+"/ipfs/"+rootEnt.Root.String(), nil)
+			request.Header.Set("Accept", "application/vnd.ipld.car")
+			if tc.acceptGzip {
+				request.Header.Set("Accept-Encoding", "gzip")
+			}
+			req.NoError(err)
+			request = request.WithContext(ctx)
+			client := &http.Client{Transport: &http.Transport{DisableCompression: tc.noClientCompression}}
+			response, err := client.Do(request)
+			req.NoError(err)
+			if response.StatusCode != http.StatusOK {
+				body, _ := io.ReadAll(response.Body)
+				req.Failf("wrong response code not received", "expected %d, got %d; body: [%s]", http.StatusOK, response.StatusCode, string(body))
+			}
+
+			req.Equal("application/vnd.ipld.car;version=1;order=dfs;dups=y", response.Header.Get("Content-Type"))
+			req.Equal("Accept, Accept-Encoding", response.Header.Get("Vary"))
+
+			rdr := response.Body
+			if tc.expectGzip {
+				if tc.noClientCompression || tc.acceptGzip { // in either of these cases we expect to handle it ourselves
+					req.Equal("gzip", response.Header.Get("Content-Encoding"))
+					rdr, err = gzip.NewReader(response.Body)
+					req.NoError(err)
+				} // else should be handled by the go client
+				req.Regexp(`\.car\.\w{12,13}\.gz"$`, response.Header.Get("Etag"))
+			} else {
+				req.Regexp(`\.car\.\w{12,13}"$`, response.Header.Get("Etag"))
+			}
+			cr, err := car.NewBlockReader(rdr)
+			req.NoError(err)
+			req.Equal(cr.Version, uint64(1))
+			req.Equal(cr.Roots, []cid.Cid{rootEnt.Root})
+
+			wantCids := toCids(rootEnt)
+			gotCids := make([]cid.Cid, 0)
+			for {
+				blk, err := cr.Next()
+				if err != nil {
+					req.ErrorIs(err, io.EOF)
+					break
+				}
+				req.NoError(err)
+				gotCids = append(gotCids, blk.Cid())
+			}
+			req.ElementsMatch(wantCids, gotCids)
+		})
+	}
+}
+
+func toCids(e unixfstestutil.DirEntry) []cid.Cid {
+	cids := make([]cid.Cid, 0)
+	var r func(e unixfstestutil.DirEntry)
+	r = func(e unixfstestutil.DirEntry) {
+		cids = append(cids, e.SelfCids...)
+		for _, c := range e.Children {
+			r(c)
+		}
+	}
+	r(e)
+	return cids
+}

go.mod

@@ -3,6 +3,7 @@ module github.com/ipld/frisbii
 go 1.20
 
 require (
+	github.com/NYTimes/gziphandler v1.1.1
 	github.com/dustin/go-humanize v1.0.1
 	github.com/ipfs/go-block-format v0.2.0
 	github.com/ipfs/go-cid v0.4.1
@@ -12,9 +13,10 @@ require (
 	github.com/ipfs/go-unixfsnode v1.9.0
 	github.com/ipld/go-car/v2 v2.13.1
 	github.com/ipld/go-codec-dagpb v1.6.0
+	github.com/ipld/go-fixtureplate v0.0.2
 	github.com/ipld/go-ipld-prime v0.21.0
 	github.com/ipld/go-trustless-utils v0.3.1
-	github.com/ipld/ipld/specs v0.0.0-20230907004443-0e4ff95ff474
+	github.com/ipld/ipld/specs v0.0.0-20230927010225-ef4dbd703269
 	github.com/ipni/go-libipni v0.5.2
 	github.com/ipni/index-provider v0.14.2
 	github.com/ipni/storetheindex v0.8.1
@@ -23,6 +25,7 @@ require (
 	github.com/multiformats/go-multihash v0.2.3
 	github.com/stretchr/testify v1.8.4
 	github.com/urfave/cli/v2 v2.25.7
+	go.uber.org/multierr v1.11.0
 	golang.org/x/term v0.12.0
 )
 
@@ -145,7 +148,6 @@ require (
 	go.uber.org/atomic v1.11.0 // indirect
 	go.uber.org/dig v1.17.0 // indirect
 	go.uber.org/fx v1.20.0 // indirect
-	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.25.0 // indirect
 	go4.org v0.0.0-20230225012048-214862532bf5 // indirect
 	golang.org/x/crypto v0.12.0 // indirect