feat!: NewHttpIpfs constructor takes options; add tests

cmd/frisbii/main.go

@@ -101,7 +101,14 @@ func action(c *cli.Context) error {
 	unixfsnode.AddUnixFSReificationToLinkSystem(&lsys)
 	lsys.SetReadStorage(multicar)
 
-	server, err := frisbii.NewFrisbiiServer(ctx, logWriter, lsys, config.MaxResponseDuration, config.MaxResponseBytes, config.Listen)
+	server, err := frisbii.NewFrisbiiServer(
+		ctx,
+		logWriter,
+		lsys,
+		config.Listen,
+		frisbii.WithMaxResponseDuration(config.MaxResponseDuration),
+		frisbii.WithMaxResponseBytes(config.MaxResponseBytes),
+	)
 	if err != nil {
 		return err
 	}

frisbii.go

@@ -6,7 +6,6 @@ import (
 	"io"
 	"net"
 	"net/http"
-	"time"
 
 	"github.com/ipfs/go-cid"
 	"github.com/ipfs/go-log/v2"
@@ -25,11 +24,10 @@ var advMetadata = metadata.Default.New(metadata.IpfsGatewayHttp{})
 // HTTP server to serve data according to the Trustless Gateway spec and it
 // also provides a mechanism to announce the server to the indexer service.
 type FrisbiiServer struct {
-	ctx                 context.Context
-	lsys                linking.LinkSystem
-	logWriter           io.Writer
-	maxResponseDuration time.Duration
-	maxResponseBytes    int64
+	ctx         context.Context
+	lsys        linking.LinkSystem
+	logWriter   io.Writer
+	httpOptions []HttpOption
 
 	listener        net.Listener
 	mux             *http.ServeMux
@@ -45,22 +43,19 @@ func NewFrisbiiServer(
 	ctx context.Context,
 	logWriter io.Writer,
 	lsys linking.LinkSystem,
-	maxResponseDuration time.Duration,
-	maxResponseBytes int64,
 	address string,
+	httpOptions ...HttpOption,
 ) (*FrisbiiServer, error) {
 	listener, err := net.Listen("tcp", address)
 	if err != nil {
 		return nil, err
 	}
 	return &FrisbiiServer{
-		ctx:                 ctx,
-		logWriter:           logWriter,
-		lsys:                lsys,
-		maxResponseDuration: maxResponseDuration,
-		maxResponseBytes:    maxResponseBytes,
-
-		listener: listener,
+		ctx:         ctx,
+		logWriter:   logWriter,
+		lsys:        lsys,
+		httpOptions: httpOptions,
+		listener:    listener,
 	}, nil
 }
 
@@ -71,7 +66,7 @@ func (fs *FrisbiiServer) Addr() net.Addr {
 func (fs *FrisbiiServer) Serve() error {
 	fs.mux = http.NewServeMux()
 
-	fs.mux.Handle("/ipfs/", NewHttpIpfs(fs.ctx, fs.logWriter, fs.lsys, fs.maxResponseDuration, fs.maxResponseBytes))
+	fs.mux.Handle("/ipfs/", NewHttpIpfs(fs.ctx, fs.lsys, fs.httpOptions...))
 	server := &http.Server{
 		Addr:        fs.Addr().String(),
 		BaseContext: func(listener net.Listener) context.Context { return fs.ctx },

go.mod

@@ -10,9 +10,10 @@ require (
 	github.com/ipfs/go-ipld-format v0.6.0
 	github.com/ipfs/go-log/v2 v2.5.1
 	github.com/ipfs/go-unixfsnode v1.8.0
-	github.com/ipld/go-car/v2 v2.11.0
+	github.com/ipld/go-car/v2 v2.12.0
 	github.com/ipld/go-ipld-prime v0.21.0
 	github.com/ipld/go-trustless-utils v0.0.0
+	github.com/ipld/ipld/specs v0.0.0-20230826120441-91918996e8eb
 	github.com/ipni/go-libipni v0.3.4
 	github.com/ipni/index-provider v0.13.5
 	github.com/libp2p/go-libp2p v0.30.0
@@ -70,7 +71,7 @@ require (
 	github.com/ipfs/go-ipfs-chunker v0.0.5 // indirect
 	github.com/ipfs/go-ipfs-pq v0.0.3 // indirect
 	github.com/ipfs/go-ipfs-util v0.0.3 // indirect
-	github.com/ipfs/go-ipld-cbor v0.0.6 // indirect
+	github.com/ipfs/go-ipld-cbor v0.1.0 // indirect
 	github.com/ipfs/go-log v1.0.5 // indirect
 	github.com/ipfs/go-peertaskqueue v0.8.1 // indirect
 	github.com/ipld/go-codec-dagpb v1.6.0 // indirect
@@ -130,8 +131,9 @@ require (
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
 	github.com/twmb/murmur3 v1.1.6 // indirect
+	github.com/warpfork/go-testmark v0.12.1 // indirect
 	github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 // indirect
-	github.com/whyrusleeping/cbor-gen v0.0.0-20230418232409-daab9ece03a0 // indirect
+	github.com/whyrusleeping/cbor-gen v0.0.0-20230818171029-f91ae536ca25 // indirect
 	github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f // indirect
 	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
 	go.opentelemetry.io/otel v1.16.0 // indirect

go.sum

@@ -246,8 +246,8 @@ github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyB
 github.com/ipfs/go-ipfs-util v0.0.3 h1:2RFdGez6bu2ZlZdI+rWfIdbQb1KudQp3VGwPtdNCmE0=
 github.com/ipfs/go-ipfs-util v0.0.3/go.mod h1:LHzG1a0Ig4G+iZ26UUOMjHd+lfM84LZCrn17xAKWBvs=
 github.com/ipfs/go-ipld-cbor v0.0.3/go.mod h1:wTBtrQZA3SoFKMVkp6cn6HMRteIB1VsmHA0AQFOn7Nc=
-github.com/ipfs/go-ipld-cbor v0.0.6 h1:pYuWHyvSpIsOOLw4Jy7NbBkCyzLDcl64Bf/LZW7eBQ0=
-github.com/ipfs/go-ipld-cbor v0.0.6/go.mod h1:ssdxxaLJPXH7OjF5V4NSjBbcfh+evoR4ukuru0oPXMA=
+github.com/ipfs/go-ipld-cbor v0.1.0 h1:dx0nS0kILVivGhfWuB6dUpMa/LAwElHPw1yOGYopoYs=
+github.com/ipfs/go-ipld-cbor v0.1.0/go.mod h1:U2aYlmVrJr2wsUBU67K4KgepApSZddGRDWBYR0H4sCk=
 github.com/ipfs/go-ipld-format v0.0.1/go.mod h1:kyJtbkDALmFHv3QR6et67i35QzO3S0dCDnkOJhcZkms=
 github.com/ipfs/go-ipld-format v0.0.2/go.mod h1:4B6+FM2u9OJ9zCV+kSbgFAZlOrv1Hqbf0INGQgiKf9k=
 github.com/ipfs/go-ipld-format v0.6.0 h1:VEJlA2kQ3LqFSIm5Vu6eIlSxD/Ze90xtc4Meten1F5U=
@@ -271,8 +271,8 @@ github.com/ipfs/go-unixfs v0.4.4 h1:D/dLBOJgny5ZLIur2vIXVQVW0EyDHdOMBDEhgHrt6rY=
 github.com/ipfs/go-unixfsnode v1.8.0 h1:yCkakzuE365glu+YkgzZt6p38CSVEBPgngL9ZkfnyQU=
 github.com/ipfs/go-unixfsnode v1.8.0/go.mod h1:HxRu9HYHOjK6HUqFBAi++7DVoWAHn0o4v/nZ/VA+0g8=
 github.com/ipfs/go-verifcid v0.0.2 h1:XPnUv0XmdH+ZIhLGKg6U2vaPaRDXb9urMyNVCE7uvTs=
-github.com/ipld/go-car/v2 v2.11.0 h1:lkAPwbbTFqbdfawgm+bfmFc8PjGC7D12VcaLXPCLNfM=
-github.com/ipld/go-car/v2 v2.11.0/go.mod h1:aDszqev0zjtU8l96g4lwXHaU9bzArj56Y7eEN0q/xqA=
+github.com/ipld/go-car/v2 v2.12.0 h1:4wpZwCEK2Th7lrVhkAio7fnxZb6COrSHxSz9xCR6FOo=
+github.com/ipld/go-car/v2 v2.12.0/go.mod h1:QkdjjFNGit2GIkpQ953KBwowuoukoM75nP/JI1iDJdo=
 github.com/ipld/go-codec-dagpb v1.6.0 h1:9nYazfyu9B1p3NAgfVdpRco3Fs2nFC72DqVsMj6rOcc=
 github.com/ipld/go-codec-dagpb v1.6.0/go.mod h1:ANzFhfP2uMJxRBr8CE+WQWs5UsNa0pYtmKZ+agnUw9s=
 github.com/ipld/go-ipld-adl-hamt v0.0.0-20220616142416-9004dbd839e0 h1:QAI/Ridj0+foHD6epbxmB4ugxz9B4vmNdYSmQLGa05E=
@@ -283,6 +283,7 @@ github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236
 github.com/ipld/go-trustless-utils v0.0.0 h1:X+hY7lWZSd6kdBGfmGtEIjCUeqETaeahMKjont7+OGo=
 github.com/ipld/go-trustless-utils v0.0.0/go.mod h1:rqvDGdgk2acsKiGyV5mzCZZRK8JtnEZqvUBxLuYxR6A=
 github.com/ipld/ipld/specs v0.0.0-20230826120441-91918996e8eb h1:5ARxkQ9NqZq33RM7i/Eq3bvBj2RBxx2xE63hqfa+9KY=
+github.com/ipld/ipld/specs v0.0.0-20230826120441-91918996e8eb/go.mod h1:AfGlAr20WOjV5PyCowEnGY3pAm5x5i+o0R8IUeir6cs=
 github.com/ipni/go-libipni v0.3.4 h1:ZYgCE2TOZt/QJJcBZb+R63FaBLlA2suZGP2IH1fKv4A=
 github.com/ipni/go-libipni v0.3.4/go.mod h1:6EIUhN83pd1i6q7SCSCIuuUC3XgR7D/gjKkEnVyIQWE=
 github.com/ipni/index-provider v0.13.5 h1:pNOO795k4mR0bwm9npkapSWJld7fYP/8//DMJZi1w/M=
@@ -410,7 +411,6 @@ github.com/multiformats/go-multicodec v0.9.0/go.mod h1:L3QTQvMIaVBkXOXXtVmYE+LI1
 github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U=
 github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew=
 github.com/multiformats/go-multihash v0.0.9/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew=
-github.com/multiformats/go-multihash v0.0.10/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew=
 github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc=
 github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U=
 github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM=
@@ -534,7 +534,9 @@ github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs=
 github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ=
 github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU=
 github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM=
+github.com/warpfork/go-fsx v0.3.0/go.mod h1:oTACCMj+Zle+vgVa5SAhGAh7WksYpLgGUCKEAVc+xPg=
 github.com/warpfork/go-testmark v0.12.1 h1:rMgCpJfwy1sJ50x0M0NgyphxYYPMOODIJHhsXyEHU0s=
+github.com/warpfork/go-testmark v0.12.1/go.mod h1:kHwy7wfvGSPh1rQJYKayD4AbtNaeyZdcGi9tNJTaa5Y=
 github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw=
 github.com/warpfork/go-wish v0.0.0-20190328234359-8b3e70f8e830/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw=
 github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0 h1:GDDkbFiaK8jsSDJfjId/PEGEShv6ugrt4kYsC5UIDaQ=
@@ -545,8 +547,8 @@ github.com/whyrusleeping/cbor-gen v0.0.0-20191216205031-b047b6acb3c0/go.mod h1:x
 github.com/whyrusleeping/cbor-gen v0.0.0-20200123233031-1cdf64d27158/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI=
 github.com/whyrusleeping/cbor-gen v0.0.0-20200710004633-5379fc63235d/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ=
 github.com/whyrusleeping/cbor-gen v0.0.0-20200826160007-0b9f6c5fb163/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ=
-github.com/whyrusleeping/cbor-gen v0.0.0-20230418232409-daab9ece03a0 h1:XYEgH2nJgsrcrj32p+SAbx6T3s/6QknOXezXtz7kzbg=
-github.com/whyrusleeping/cbor-gen v0.0.0-20230418232409-daab9ece03a0/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ=
+github.com/whyrusleeping/cbor-gen v0.0.0-20230818171029-f91ae536ca25 h1:yVYDLoN2gmB3OdBXFW8e1UwgVbmCvNlnAKhvHPaNARI=
+github.com/whyrusleeping/cbor-gen v0.0.0-20230818171029-f91ae536ca25/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ=
 github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E=
 github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f/go.mod h1:p9UJB6dDgdPgMJZs7UjUOdulKyRr9fqkS+6JKAInPy8=
 github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM=

httpipfs.go

@@ -25,49 +25,77 @@ type ErrorLogger interface {
 // HttpIpfs is an http.Handler that serves IPLD data via HTTP according to the
 // Trustless Gateway specification.
 type HttpIpfs struct {
-	ctx                 context.Context
-	logWriter           io.Writer
-	lsys                linking.LinkSystem
-	maxResponseDuration time.Duration
-	maxResponseBytes    int64
+	ctx  context.Context
+	lsys linking.LinkSystem
+	cfg  *httpOptions
+}
+
+type httpOptions struct {
+	MaxResponseDuration time.Duration
+	MaxResponseBytes    int64
+}
+
+type HttpOption func(*httpOptions)
+
+// WithMaxResponseDuration sets the maximum duration for a response to be
+// streamed before the connection is closed. This allows a server to limit the
+// amount of time a client can hold a connection open; and also restricts the
+// ability to serve very large DAGs.
+//
+// A value of 0 will disable the limitation. This is the default.
+func WithMaxResponseDuration(d time.Duration) HttpOption {
+	return func(o *httpOptions) {
+		o.MaxResponseDuration = d
+	}
+}
+
+// WithMaxResponseBytes sets the maximum number of bytes that will be streamed
+// before the connection is closed. This allows a server to limit the amount of
+// data a client can request; and also restricts the ability to serve very large
+// DAGs.
+//
+// A value of 0 will disable the limitation. This is the default.
+func WithMaxResponseBytes(b int64) HttpOption {
+	return func(o *httpOptions) {
+		o.MaxResponseBytes = b
+	}
 }
 
 func NewHttpIpfs(
 	ctx context.Context,
-	logWriter io.Writer,
 	lsys linking.LinkSystem,
-	maxResponseDuration time.Duration,
-	maxResponseBytes int64,
+	opts ...HttpOption,
 ) *HttpIpfs {
+	cfg := &httpOptions{}
+	for _, opt := range opts {
+		opt(cfg)
+	}
 
 	return &HttpIpfs{
-		ctx:                 ctx,
-		logWriter:           logWriter,
-		lsys:                lsys,
-		maxResponseDuration: maxResponseDuration,
-		maxResponseBytes:    maxResponseBytes,
+		ctx:  ctx,
+		lsys: lsys,
+		cfg:  cfg,
 	}
 }
 
 func (hi *HttpIpfs) ServeHTTP(res http.ResponseWriter, req *http.Request) {
 	ctx := hi.ctx
-	if hi.maxResponseDuration > 0 {
+	if hi.cfg.MaxResponseDuration > 0 {
 		var cancel context.CancelFunc
-		ctx, cancel = context.WithTimeout(ctx, hi.maxResponseDuration)
+		ctx, cancel = context.WithTimeout(ctx, hi.cfg.MaxResponseDuration)
 		defer cancel()
 	}
 
 	logError := func(status int, err error) {
+		res.WriteHeader(status)
+		res.Write([]byte(err.Error()))
 		if lrw, ok := res.(ErrorLogger); ok {
 			lrw.LogError(status, err)
 		} else {
-			logger.Debug("Error handling request from [%s] for [%s] status=%d, msg=%s", req.RemoteAddr, req.URL, status, err.Error())
+			logger.Debugf("Error handling request from [%s] for [%s] status=%d, msg=%s", req.RemoteAddr, req.URL, status, err.Error())
 		}
 	}
 
-	path := datamodel.ParsePath(req.URL.Path)
-	_, path = path.Shift() // remove /ipfs
-
 	// filter out everything but GET requests
 	switch req.Method {
 	case http.MethodGet:
@@ -78,6 +106,9 @@ func (hi *HttpIpfs) ServeHTTP(res http.ResponseWriter, req *http.Request) {
 		return
 	}
 
+	path := datamodel.ParsePath(req.URL.Path)
+	_, path = path.Shift() // remove /ipfs
+
 	// check if CID path param is missing
 	if path.Len() == 0 {
 		// not a valid path to hit
@@ -102,7 +133,7 @@ func (hi *HttpIpfs) ServeHTTP(res http.ResponseWriter, req *http.Request) {
 	cidSeg, path = path.Shift()
 	rootCid, err := cid.Parse(cidSeg.String())
 	if err != nil {
-		logError(http.StatusInternalServerError, errors.New("failed to parse CID path parameter"))
+		logError(http.StatusBadRequest, errors.New("failed to parse CID path parameter"))
 		return
 	}
 
@@ -131,7 +162,7 @@ func (hi *HttpIpfs) ServeHTTP(res http.ResponseWriter, req *http.Request) {
 	}
 
 	bytesWrittenCh := make(chan struct{})
-	writer := newIpfsResponseWriter(res, hi.maxResponseBytes, func() {
+	writer := newIpfsResponseWriter(res, hi.cfg.MaxResponseBytes, func() {
 		// called once we start writing blocks into the CAR (on the first Put())
 		res.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%q", fileName))
 		res.Header().Set("Cache-Control", trustlesshttp.ResponseCacheControlHeader)