From e8103a2020afe1ca51976c2680782d66b594d765 Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Thu, 5 Oct 2023 13:30:19 +1100 Subject: [PATCH] fix: unfork frisbii, use dependency --- cmd/booster-http/frisbii/carstream.go | 66 ----- cmd/booster-http/frisbii/carstream_test.go | 227 --------------- cmd/booster-http/frisbii/doc.go | 4 - cmd/booster-http/frisbii/httpipfs.go | 257 ----------------- cmd/booster-http/frisbii/httpipfs_test.go | 315 --------------------- cmd/booster-http/frisbii/logger.go | 7 - cmd/booster-http/server.go | 2 +- go.mod | 9 +- go.sum | 14 +- 9 files changed, 14 insertions(+), 887 deletions(-) delete mode 100644 cmd/booster-http/frisbii/carstream.go delete mode 100644 cmd/booster-http/frisbii/carstream_test.go delete mode 100644 cmd/booster-http/frisbii/doc.go delete mode 100644 cmd/booster-http/frisbii/httpipfs.go delete mode 100644 cmd/booster-http/frisbii/httpipfs_test.go delete mode 100644 cmd/booster-http/frisbii/logger.go diff --git a/cmd/booster-http/frisbii/carstream.go b/cmd/booster-http/frisbii/carstream.go deleted file mode 100644 index c132cec0c..000000000 --- a/cmd/booster-http/frisbii/carstream.go +++ /dev/null @@ -1,66 +0,0 @@ -package frisbii - -import ( - "bytes" - "context" - "io" - - // codecs we care about - - _ "github.com/ipld/go-ipld-prime/codec/cbor" - _ "github.com/ipld/go-ipld-prime/codec/dagcbor" - _ "github.com/ipld/go-ipld-prime/codec/dagjson" - _ "github.com/ipld/go-ipld-prime/codec/json" - _ "github.com/ipld/go-ipld-prime/codec/raw" - "github.com/ipld/go-trustless-utils/traversal" - - "github.com/ipfs/go-cid" - "github.com/ipld/go-car/v2" - "github.com/ipld/go-car/v2/storage/deferred" - "github.com/ipld/go-ipld-prime/datamodel" - "github.com/ipld/go-ipld-prime/linking" - cidlink "github.com/ipld/go-ipld-prime/linking/cid" - trustlessutils "github.com/ipld/go-trustless-utils" -) - -// StreamCar streams a DAG in CARv1 format to the given writer, using the given -// selector. -func StreamCar( - ctx context.Context, - requestLsys linking.LinkSystem, - out io.Writer, - request trustlessutils.Request, -) error { - carWriter := deferred.NewDeferredCarWriterForStream(out, []cid.Cid{request.Root}, car.AllowDuplicatePuts(request.Duplicates)) - requestLsys.StorageReadOpener = carPipe(requestLsys.StorageReadOpener, carWriter) - - cfg := traversal.Config{Root: request.Root, Selector: request.Selector()} - lastPath, err := cfg.Traverse(ctx, requestLsys, nil) - if err != nil { - return err - } - - if err := traversal.CheckPath(datamodel.ParsePath(request.Path), lastPath); err != nil { - logger.Warn(err) - } - - return nil -} - -func carPipe(orig linking.BlockReadOpener, car *deferred.DeferredCarWriter) linking.BlockReadOpener { - return func(lc linking.LinkContext, lnk datamodel.Link) (io.Reader, error) { - r, err := orig(lc, lnk) - if err != nil { - return nil, err - } - byts, err := io.ReadAll(r) - if err != nil { - return nil, err - } - err = car.Put(lc.Ctx, lnk.(cidlink.Link).Cid.KeyString(), byts) - if err != nil { - return nil, err - } - return bytes.NewReader(byts), nil - } -} diff --git a/cmd/booster-http/frisbii/carstream_test.go b/cmd/booster-http/frisbii/carstream_test.go deleted file mode 100644 index 14deb4fd0..000000000 --- a/cmd/booster-http/frisbii/carstream_test.go +++ /dev/null @@ -1,227 +0,0 @@ -package frisbii_test - -import ( - "bytes" - "context" - "crypto/rand" - "io" - "testing" - - "github.com/filecoin-project/boost/cmd/booster-http/frisbii" - blocks "github.com/ipfs/go-block-format" - "github.com/ipfs/go-cid" - gstestutil "github.com/ipfs/go-graphsync/testutil" - "github.com/ipfs/go-unixfsnode" - unixfs "github.com/ipfs/go-unixfsnode/testutil" - "github.com/ipld/go-car/v2" - "github.com/ipld/go-ipld-prime/datamodel" - "github.com/ipld/go-ipld-prime/linking" - cidlink "github.com/ipld/go-ipld-prime/linking/cid" - "github.com/ipld/go-ipld-prime/storage/memstore" - trustlessutils "github.com/ipld/go-trustless-utils" - trustlesstestutil "github.com/ipld/go-trustless-utils/testutil" - "github.com/stretchr/testify/require" -) - -func TestStreamCar(t *testing.T) { - ctx := context.Background() - - chainLsys := makeLsys() - tbc := gstestutil.SetupBlockChain(ctx, t, chainLsys, 1000, 100) - allChainBlocks := tbc.AllBlocks() - - fileLsys := makeLsys() - fileEnt := unixfs.GenerateFile(t, &fileLsys, rand.Reader, 1<<20) - - dirLsys := makeLsys() - var dirEnt unixfs.DirEntry - for { - dirEnt = GenerateNoDupes(func() unixfs.DirEntry { return unixfs.GenerateDirectory(t, &dirLsys, rand.Reader, 4<<20, false) }) - if len(dirEnt.Children) > 2 { // we want at least 3 children to test the path subset selector - break - } - } - - shardedDirLsys := makeLsys() - var shardedDirEnt unixfs.DirEntry - for { - shardedDirEnt = GenerateNoDupes(func() unixfs.DirEntry { return unixfs.GenerateDirectory(t, &shardedDirLsys, rand.Reader, 4<<20, true) }) - if len(dirEnt.Children) > 2 { // we want at least 3 children to test the path subset selector - break - } - } - - testCases := []struct { - name string - path datamodel.Path - scope trustlessutils.DagScope - root cid.Cid - lsys linking.LinkSystem - validate func(t *testing.T, r io.Reader) - expectedErr string - }{ - { - name: "chain: all blocks", - scope: trustlessutils.DagScopeAll, - root: tbc.TipLink.(cidlink.Link).Cid, - lsys: chainLsys, - validate: func(t *testing.T, r io.Reader) { - root, blks := carToBlocks(t, r) - require.Equal(t, tbc.TipLink.(cidlink.Link).Cid, root) - require.Equal(t, allChainBlocks, blks) - }, - }, - { - name: "chain: just root", - scope: trustlessutils.DagScopeBlock, - root: tbc.TipLink.(cidlink.Link).Cid, - lsys: chainLsys, - validate: func(t *testing.T, r io.Reader) { - root, blks := carToBlocks(t, r) - require.Equal(t, tbc.TipLink.(cidlink.Link).Cid, root) - require.Equal(t, []blocks.Block{allChainBlocks[0]}, blks) - }, - }, - { - name: "unixfs file", - scope: trustlessutils.DagScopeAll, - root: fileEnt.Root, - lsys: fileLsys, - validate: func(t *testing.T, r io.Reader) { - root, blks := carToBlocks(t, r) - require.Equal(t, fileEnt.Root, root) - require.ElementsMatch(t, fileEnt.SelfCids, blkCids(blks)) - }, - }, - { - name: "unixfs directory", - scope: trustlessutils.DagScopeAll, - root: dirEnt.Root, - lsys: dirLsys, - validate: func(t *testing.T, r io.Reader) { - root, blks := carToBlocks(t, r) - require.Equal(t, dirEnt.Root, root) - require.ElementsMatch(t, entCids(dirEnt), blkCids(blks)) - }, - }, - { - name: "unixfs sharded directory", - scope: trustlessutils.DagScopeAll, - root: shardedDirEnt.Root, - lsys: shardedDirLsys, - validate: func(t *testing.T, r io.Reader) { - root, blks := carToBlocks(t, r) - require.Equal(t, shardedDirEnt.Root, root) - require.ElementsMatch(t, entCids(shardedDirEnt), blkCids(blks)) - }, - }, - { - // path that (probably) doesn't exist, shouldn't error but shouldn't - // return much (this ought to be tested better with a fixture) - name: "unixfs sharded directory, no such path", - scope: trustlessutils.DagScopeAll, - path: datamodel.ParsePath(shardedDirEnt.Children[0].Path + "/nope"), - root: shardedDirEnt.Root, - lsys: shardedDirLsys, - validate: func(t *testing.T, r io.Reader) { - root, blks := carToBlocks(t, r) - require.Equal(t, shardedDirEnt.Root, root) - cids := blkCids(blks) - require.Contains(t, cids, shardedDirEnt.Root) - require.NotEqual(t, len(entCids(shardedDirEnt)), cids) // shouldn't contain the full thing! - }, - }, - } - - for _, tc := range testCases { - tc := tc - t.Run(tc.name, func(t *testing.T) { - req := require.New(t) - var buf bytes.Buffer - err := frisbii.StreamCar(ctx, tc.lsys, &buf, trustlessutils.Request{Root: tc.root, Path: tc.path.String(), Scope: tc.scope, Duplicates: false}) - if tc.expectedErr != "" { - req.EqualError(err, tc.expectedErr) - } else { - req.NoError(err) - tc.validate(t, &buf) - } - }) - } -} - -func carToBlocks(t *testing.T, r io.Reader) (cid.Cid, []blocks.Block) { - rdr, err := car.NewBlockReader(r) - require.NoError(t, err) - require.Len(t, rdr.Roots, 1) - blks := make([]blocks.Block, 0) - for { - blk, err := rdr.Next() - if err == io.EOF { - break - } - require.NoError(t, err) - blks = append(blks, blk) - } - return rdr.Roots[0], blks -} - -func blkCids(blks []blocks.Block) []cid.Cid { - cids := make([]cid.Cid, len(blks)) - for i, blk := range blks { - cids[i] = blk.Cid() - } - return cids -} - -func entCids(ent unixfs.DirEntry) []cid.Cid { - cids := make([]cid.Cid, 0) - var _entCids func(ent unixfs.DirEntry) - _entCids = func(ent unixfs.DirEntry) { - cids = append(cids, ent.SelfCids...) - for _, c := range ent.Children { - _entCids(c) - } - } - _entCids(ent) - return cids -} - -func makeLsys() linking.LinkSystem { - store := &trustlesstestutil.CorrectedMemStore{ParentStore: &memstore.Store{ - Bag: make(map[string][]byte), - }} - lsys := cidlink.DefaultLinkSystem() - lsys.SetReadStorage(store) - lsys.SetWriteStorage(store) - unixfsnode.AddUnixFSReificationToLinkSystem(&lsys) - return lsys -} - -// TODO: this should probably be an option in unixfsnode/testutil, for -// generators to strictly not return a DAG with duplicates - -func GenerateNoDupes(gen func() unixfs.DirEntry) unixfs.DirEntry { - var check func(unixfs.DirEntry) bool - var seen map[cid.Cid]struct{} - check = func(e unixfs.DirEntry) bool { - for _, c := range e.SelfCids { - if _, ok := seen[c]; ok { - return false - } - seen[c] = struct{}{} - } - for _, c := range e.Children { - if !check(c) { - return false - } - } - return true - } - for { - seen = make(map[cid.Cid]struct{}) - gend := gen() - if check(gend) { - return gend - } - } -} diff --git a/cmd/booster-http/frisbii/doc.go b/cmd/booster-http/frisbii/doc.go deleted file mode 100644 index 434186903..000000000 --- a/cmd/booster-http/frisbii/doc.go +++ /dev/null @@ -1,4 +0,0 @@ -// Package frisbii is a temporary fork of components from -// https://github.com/ipld/frisbii to deal with unresolvable libp2p version -// discrepancies while we wait for a Lotus libp2p upgrade. -package frisbii diff --git a/cmd/booster-http/frisbii/httpipfs.go b/cmd/booster-http/frisbii/httpipfs.go deleted file mode 100644 index 976928736..000000000 --- a/cmd/booster-http/frisbii/httpipfs.go +++ /dev/null @@ -1,257 +0,0 @@ -package frisbii - -import ( - "context" - "errors" - "fmt" - "io" - "net/http" - "strings" - "sync" - "time" - - "github.com/ipfs/go-cid" - "github.com/ipld/go-ipld-prime/datamodel" - "github.com/ipld/go-ipld-prime/linking" - trustlessutils "github.com/ipld/go-trustless-utils" - trustlesshttp "github.com/ipld/go-trustless-utils/http" -) - -var _ http.Handler = (*HttpIpfs)(nil) - -type ErrorLogger interface { - LogError(status int, err error) -} - -// HttpIpfs is an http.Handler that serves IPLD data via HTTP according to the -// Trustless Gateway specification. -type HttpIpfs struct { - ctx context.Context - lsys linking.LinkSystem - cfg *httpOptions -} - -type httpOptions struct { - MaxResponseDuration time.Duration - MaxResponseBytes int64 -} - -type HttpOption func(*httpOptions) - -// WithMaxResponseDuration sets the maximum duration for a response to be -// streamed before the connection is closed. This allows a server to limit the -// amount of time a client can hold a connection open; and also restricts the -// ability to serve very large DAGs. -// -// A value of 0 will disable the limitation. This is the default. -func WithMaxResponseDuration(d time.Duration) HttpOption { - return func(o *httpOptions) { - o.MaxResponseDuration = d - } -} - -// WithMaxResponseBytes sets the maximum number of bytes that will be streamed -// before the connection is closed. This allows a server to limit the amount of -// data a client can request; and also restricts the ability to serve very large -// DAGs. -// -// A value of 0 will disable the limitation. This is the default. -func WithMaxResponseBytes(b int64) HttpOption { - return func(o *httpOptions) { - o.MaxResponseBytes = b - } -} - -func NewHttpIpfs( - ctx context.Context, - lsys linking.LinkSystem, - opts ...HttpOption, -) *HttpIpfs { - cfg := &httpOptions{} - for _, opt := range opts { - opt(cfg) - } - - return &HttpIpfs{ - ctx: ctx, - lsys: lsys, - cfg: cfg, - } -} - -func (hi *HttpIpfs) ServeHTTP(res http.ResponseWriter, req *http.Request) { - ctx := hi.ctx - if hi.cfg.MaxResponseDuration > 0 { - var cancel context.CancelFunc - ctx, cancel = context.WithTimeout(ctx, hi.cfg.MaxResponseDuration) - defer cancel() - } - - var rootCid cid.Cid - bytesWrittenCh := make(chan struct{}) - - logError := func(status int, err error) { - select { - case <-bytesWrittenCh: - cs := "unknown" - if rootCid.Defined() { - cs = rootCid.String() - } - logger.Debugw("forcing unclean close", "cid", cs, "status", status, "err", err) - if err := closeWithUnterminatedChunk(res); err != nil { - log := logger.Infow - if strings.Contains(err.Error(), "use of closed network connection") { - log = logger.Debugw // it's just not as interesting in this case - } - log("unable to send early termination", "err", err) - } - return - default: - res.WriteHeader(status) - if _, werr := res.Write([]byte(err.Error())); werr != nil { - logger.Debugw("unable to write error to response", "err", werr) - } - } - - if lrw, ok := res.(ErrorLogger); ok { - lrw.LogError(status, err) - } else { - logger.Debugf("error handling request from [%s] for [%s] status=%d, msg=%s", req.RemoteAddr, req.URL, status, err.Error()) - } - } - - // filter out everything but GET requests - switch req.Method { - case http.MethodGet: - break - default: - res.Header().Add("Allow", http.MethodGet) - logError(http.StatusMethodNotAllowed, errors.New("method not allowed")) - return - } - - path := datamodel.ParsePath(req.URL.Path) - _, path = path.Shift() // remove /ipfs - - // check if CID path param is missing - if path.Len() == 0 { - // not a valid path to hit - logError(http.StatusNotFound, errors.New("not found")) - return - } - - // get the preferred `Accept` header if one exists; we should be able to - // handle whatever comes back from here, primarily we're looking for - // the `dups` parameter - accept, err := trustlesshttp.CheckFormat(req) - if err != nil { - logError(http.StatusBadRequest, err) - return - } - - fileName, err := trustlesshttp.ParseFilename(req) - if err != nil { - logError(http.StatusBadRequest, err) - return - } - - // validate CID path parameter - var cidSeg datamodel.PathSegment - cidSeg, path = path.Shift() - if rootCid, err = cid.Parse(cidSeg.String()); err != nil { - logError(http.StatusBadRequest, errors.New("failed to parse CID path parameter")) - return - } - - dagScope, err := trustlesshttp.ParseScope(req) - if err != nil { - logError(http.StatusBadRequest, err) - return - } - - byteRange, err := trustlesshttp.ParseByteRange(req) - if err != nil { - logError(http.StatusBadRequest, err) - return - } - - request := trustlessutils.Request{ - Root: rootCid, - Path: path.String(), - Scope: dagScope, - Bytes: byteRange, - Duplicates: accept.Duplicates, - } - - if fileName == "" { - fileName = fmt.Sprintf("%s%s", rootCid.String(), trustlesshttp.FilenameExtCar) - } - - writer := newIpfsResponseWriter(res, hi.cfg.MaxResponseBytes, func() { - // called once we start writing blocks into the CAR (on the first Put()) - res.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%q", fileName)) - res.Header().Set("Cache-Control", trustlesshttp.ResponseCacheControlHeader) - res.Header().Set("Content-Type", accept.WithMimeType(trustlesshttp.MimeTypeCar).WithQuality(1).String()) - res.Header().Set("Etag", request.Etag()) - res.Header().Set("X-Content-Type-Options", "nosniff") - res.Header().Set("X-Ipfs-Path", "/"+datamodel.ParsePath(req.URL.Path).String()) - close(bytesWrittenCh) - }) - - if err := StreamCar(ctx, hi.lsys, writer, request); err != nil { - logger.Debugw("error streaming CAR", "cid", rootCid, "err", err) - logError(http.StatusInternalServerError, err) - } -} - -var _ io.Writer = (*ipfsResponseWriter)(nil) - -type ipfsResponseWriter struct { - w io.Writer - fn func() - byteCount int - once sync.Once - maxBytes int64 -} - -func newIpfsResponseWriter(w io.Writer, maxBytes int64, fn func()) *ipfsResponseWriter { - return &ipfsResponseWriter{ - w: w, - maxBytes: maxBytes, - fn: fn, - } -} - -func (w *ipfsResponseWriter) Write(p []byte) (int, error) { - w.once.Do(w.fn) - w.byteCount += len(p) - if w.maxBytes > 0 && int64(w.byteCount) > w.maxBytes { - return 0, fmt.Errorf("response too large: %d bytes", w.byteCount) - } - return w.w.Write(p) -} - -// closeWithUnterminatedChunk attempts to take control of the the http conn and terminate the stream early -// -// (copied from github.com/filecoin-project/lassie/pkg/server/http/ipfs.go) -func closeWithUnterminatedChunk(res http.ResponseWriter) error { - hijacker, ok := res.(http.Hijacker) - if !ok { - return errors.New("unable to access hijack interface") - } - conn, buf, err := hijacker.Hijack() - if err != nil { - return fmt.Errorf("unable to access conn through hijack interface: %w", err) - } - if _, err := buf.Write(trustlesshttp.ResponseChunkDelimeter); err != nil { - return fmt.Errorf("writing response chunk delimiter: %w", err) - } - if err := buf.Flush(); err != nil { - return fmt.Errorf("flushing buff: %w", err) - } - // attempt to close just the write side - if err := conn.Close(); err != nil { - return fmt.Errorf("closing write conn: %w", err) - } - return nil -} diff --git a/cmd/booster-http/frisbii/httpipfs_test.go b/cmd/booster-http/frisbii/httpipfs_test.go deleted file mode 100644 index e0e187b02..000000000 --- a/cmd/booster-http/frisbii/httpipfs_test.go +++ /dev/null @@ -1,315 +0,0 @@ -package frisbii_test - -import ( - "context" - "fmt" - "io" - "net/http" - "net/http/httptest" - "strings" - "testing" - - "github.com/filecoin-project/boost/cmd/booster-http/frisbii" - "github.com/ipfs/go-cid" - "github.com/ipfs/go-unixfsnode" - "github.com/ipld/go-car/v2" - dagpb "github.com/ipld/go-codec-dagpb" - "github.com/ipld/go-ipld-prime/datamodel" - "github.com/ipld/go-ipld-prime/fluent/qp" - "github.com/ipld/go-ipld-prime/linking" - cidlink "github.com/ipld/go-ipld-prime/linking/cid" - "github.com/ipld/go-ipld-prime/node/basicnode" - "github.com/ipld/go-ipld-prime/storage/memstore" - trustlesshttp "github.com/ipld/go-trustless-utils/http" - trustlesstestutil "github.com/ipld/go-trustless-utils/testutil" - trustlesspathing "github.com/ipld/ipld/specs/pkg-go/trustless-pathing" - "github.com/multiformats/go-multihash" - "github.com/stretchr/testify/require" -) - -func TestHttpIpfsHandler(t *testing.T) { - lsys := cidlink.DefaultLinkSystem() - lsys.SetReadStorage(&trustlesstestutil.CorrectedMemStore{ParentStore: &memstore.Store{}}) - handler := frisbii.NewHttpIpfs(context.Background(), lsys) - testServer := httptest.NewServer(handler) - defer testServer.Close() - - for _, testCase := range []struct { - name string - path string - accept string - expectedStatusCode int - expectedBody string - }{ - { - name: "404", - path: "/not here", - expectedStatusCode: http.StatusNotFound, - expectedBody: "not found", - }, - { - name: "bad cid", - path: "/ipfs/foobarbaz", - accept: trustlesshttp.DefaultContentType().String(), - expectedStatusCode: http.StatusBadRequest, - expectedBody: "failed to parse CID path parameter", - }, - { - name: "bad dag-scope", - path: "/ipfs/bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi?dag-scope=bork", - accept: trustlesshttp.DefaultContentType().String(), - expectedStatusCode: http.StatusBadRequest, - expectedBody: "invalid dag-scope parameter", - }, - { - name: "bad entity-bytes", - path: "/ipfs/bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi?entity-bytes=bork", - accept: trustlesshttp.DefaultContentType().String(), - expectedStatusCode: http.StatusBadRequest, - expectedBody: "invalid entity-bytes parameter", - }, - { - name: "bad Accept", - path: "/ipfs/bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi", - accept: "applicaiton/json", - expectedStatusCode: http.StatusBadRequest, - expectedBody: "invalid Accept header; unsupported: \"applicaiton/json\"", - }, - { - // special case where we get to start the request because everything - // is valid, but the block isn't in our blockstore; passing this - // depends on deferring writing the CAR output until after we've - // at least loaded the first block. - name: "block not found", - path: "/ipfs/bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi", - accept: trustlesshttp.DefaultContentType().String(), - expectedStatusCode: http.StatusInternalServerError, - expectedBody: "failed to load root node: failed to load root CID: ipld: could not find bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi", - }, - } { - t.Run(testCase.name, func(t *testing.T) { - req := require.New(t) - request, err := http.NewRequest(http.MethodGet, testServer.URL+testCase.path, nil) - req.NoError(err) - if testCase.accept != "" { - request.Header.Set("Accept", testCase.accept) - } - res, err := http.DefaultClient.Do(request) - req.NoError(err) - req.Equal(testCase.expectedStatusCode, res.StatusCode) - body, err := io.ReadAll(res.Body) - req.NoError(err) - req.Equal(testCase.expectedBody, string(body)) - }) - } -} - -func TestHttpIpfsIntegration_Unixfs20mVariety(t *testing.T) { - req := require.New(t) - - testCases, _, err := trustlesspathing.Unixfs20mVarietyCases() - req.NoError(err) - storage, closer, err := trustlesspathing.Unixfs20mVarietyReadableStorage() - req.NoError(err) - defer closer.Close() - - lsys := cidlink.DefaultLinkSystem() - unixfsnode.AddUnixFSReificationToLinkSystem(&lsys) - lsys.TrustedStorage = true - lsys.SetReadStorage(storage) - - handler := frisbii.NewHttpIpfs(context.Background(), lsys) - testServer := httptest.NewServer(handler) - defer testServer.Close() - - for _, tc := range testCases { - t.Run(tc.Name, func(t *testing.T) { - req := require.New(t) - - t.Logf("query=%s, blocks=%d", tc.AsQuery(), len(tc.ExpectedCids)) - - request, err := http.NewRequest(http.MethodGet, testServer.URL+tc.AsQuery(), nil) - req.NoError(err) - request.Header.Set("Accept", trustlesshttp.DefaultContentType().WithDuplicates(false).String()) - res, err := http.DefaultClient.Do(request) - req.NoError(err) - req.Equal(http.StatusOK, res.StatusCode) - req.Equal(trustlesshttp.DefaultContentType().WithDuplicates(false).String(), res.Header.Get("Content-Type")) - - carReader, err := car.NewBlockReader(res.Body) - req.NoError(err) - req.Equal(uint64(1), carReader.Version) - req.Equal([]cid.Cid{tc.Root}, carReader.Roots) - - for ii, expectedCid := range tc.ExpectedCids { - blk, err := carReader.Next() - if err != nil { - req.Equal(io.EOF, err) - req.Len(tc.ExpectedCids, ii+1) - break - } - req.Equal(expectedCid, blk.Cid()) - } - }) - } -} - -func TestHttpIpfsDuplicates(t *testing.T) { - store := &trustlesstestutil.CorrectedMemStore{ParentStore: &memstore.Store{}} - lsys := cidlink.DefaultLinkSystem() - lsys.SetReadStorage(store) - lsys.SetWriteStorage(store) - dupyLinks, dupyLinksDeduped := mkDupy(lsys) - - handler := frisbii.NewHttpIpfs(context.Background(), lsys) - testServer := httptest.NewServer(handler) - defer testServer.Close() - - for _, tc := range []struct { - name string - accepts []string - expectedContentType string - expectedCids []cid.Cid - }{ - { - name: "default", - accepts: []string{trustlesshttp.DefaultContentType().String()}, - expectedContentType: trustlesshttp.DefaultContentType().String(), - expectedCids: dupyLinks, - }, - { - // note that we're pretty permissive, as long as you send an Accept - // that vaguely signals you're willing to accept what we give - name: "*/*", - accepts: []string{"*/*"}, - expectedContentType: trustlesshttp.DefaultContentType().String(), - expectedCids: dupyLinks, - }, - { - name: "dups", - accepts: []string{trustlesshttp.DefaultContentType().WithDuplicates(true).String()}, - expectedContentType: trustlesshttp.DefaultContentType().WithDuplicates(true).String(), - expectedCids: dupyLinks, - }, - { - name: "no dups", - accepts: []string{trustlesshttp.DefaultContentType().WithDuplicates(false).String()}, - expectedContentType: trustlesshttp.DefaultContentType().WithDuplicates(false).String(), - expectedCids: dupyLinksDeduped, - }, - { - name: "ranked w/ dups", - accepts: []string{ - "text/html", - trustlesshttp.DefaultContentType().WithDuplicates(false).WithQuality(0.7).String(), - trustlesshttp.DefaultContentType().WithDuplicates(true).WithQuality(0.8).String(), - "*/*;q=0.2", - }, - expectedContentType: trustlesshttp.DefaultContentType().WithDuplicates(true).String(), - expectedCids: dupyLinks, - }, - { - name: "ranked w/ no dups", - accepts: []string{ - "text/html", - trustlesshttp.DefaultContentType().WithDuplicates(false).WithQuality(0.8).String(), - trustlesshttp.DefaultContentType().WithDuplicates(true).WithQuality(0.7).String(), - "*/*;q=0.2", - }, - expectedContentType: trustlesshttp.DefaultContentType().WithDuplicates(false).String(), - expectedCids: dupyLinksDeduped, - }, - } { - t.Run(tc.name, func(t *testing.T) { - req := require.New(t) - - request, err := http.NewRequest(http.MethodGet, testServer.URL+"/ipfs/"+dupyLinks[0].String(), nil) - req.NoError(err) - var accept strings.Builder - for _, a := range tc.accepts { - if accept.Len() > 0 { - accept.WriteString(", ") - } - accept.WriteString(a) - } - request.Header.Set("Accept", accept.String()) - res, err := http.DefaultClient.Do(request) - req.NoError(err) - req.Equal(http.StatusOK, res.StatusCode) - req.Equal(tc.expectedContentType, res.Header.Get("Content-Type")) - - carReader, err := car.NewBlockReader(res.Body) - req.NoError(err) - req.Equal(uint64(1), carReader.Version) - req.Equal([]cid.Cid{dupyLinks[0]}, carReader.Roots) - - for ii, expectedCid := range tc.expectedCids { - blk, err := carReader.Next() - if err != nil { - req.Equal(io.EOF, err) - req.Len(tc.expectedCids, ii+1) - break - } - req.Equal(expectedCid, blk.Cid()) - } - }) - } -} - -var pblp = cidlink.LinkPrototype{ - Prefix: cid.Prefix{ - Version: 1, - Codec: cid.DagProtobuf, - MhType: multihash.SHA2_256, - MhLength: 32, - }, -} - -var rawlp = cidlink.LinkPrototype{ - Prefix: cid.Prefix{ - Version: 1, - Codec: cid.Raw, - MhType: multihash.SHA2_256, - MhLength: 32, - }, -} - -func mkBlockWithBytes(lsys linking.LinkSystem, bytes []byte) cid.Cid { - l, err := lsys.Store(linking.LinkContext{}, rawlp, basicnode.NewBytes(bytes)) - if err != nil { - panic(err) - } - return l.(cidlink.Link).Cid -} - -func mkDupy(lsys linking.LinkSystem) ([]cid.Cid, []cid.Cid) { - dupy := mkBlockWithBytes(lsys, []byte("duplicate data")) - - n, err := qp.BuildMap(dagpb.Type.PBNode, 1, func(ma datamodel.MapAssembler) { - qp.MapEntry(ma, "Links", qp.List(100, func(la datamodel.ListAssembler) { - for i := 0; i < 100; i++ { - qp.ListEntry(la, qp.Map(2, func(ma datamodel.MapAssembler) { - qp.MapEntry(ma, "Name", qp.String(fmt.Sprintf("%03d", i))) - qp.MapEntry(ma, "Hash", qp.Link(cidlink.Link{Cid: dupy})) - })) - } - })) - }) - if err != nil { - panic(err) - } - l, err := lsys.Store(linking.LinkContext{}, pblp, n) - if err != nil { - panic(err) - } - - // dupyLinks contains the duplicates - dupyLinks := []cid.Cid{l.(cidlink.Link).Cid} - for i := 0; i < 100; i++ { - dupyLinks = append(dupyLinks, dupy) - } - // dupyLinksDeduped contains just the unique links - dupyLinksDeduped := []cid.Cid{l.(cidlink.Link).Cid, dupy} - - return dupyLinks, dupyLinksDeduped -} diff --git a/cmd/booster-http/frisbii/logger.go b/cmd/booster-http/frisbii/logger.go deleted file mode 100644 index a919a8553..000000000 --- a/cmd/booster-http/frisbii/logger.go +++ /dev/null @@ -1,7 +0,0 @@ -package frisbii - -import ( - "github.com/ipfs/go-log/v2" -) - -var logger = log.Logger("booster/frisbii") diff --git a/cmd/booster-http/server.go b/cmd/booster-http/server.go index 446841533..4570d1f20 100644 --- a/cmd/booster-http/server.go +++ b/cmd/booster-http/server.go @@ -15,7 +15,6 @@ import ( "github.com/fatih/color" "github.com/filecoin-project/boost-gfm/retrievalmarket" "github.com/filecoin-project/boost-graphsync/storeutil" - "github.com/filecoin-project/boost/cmd/booster-http/frisbii" "github.com/filecoin-project/boost/metrics" "github.com/filecoin-project/boostd-data/model" "github.com/filecoin-project/boostd-data/shared/tracing" @@ -26,6 +25,7 @@ import ( "github.com/ipfs/boxo/blockstore" "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" + "github.com/ipld/frisbii" "github.com/rs/cors" "go.opencensus.io/stats" ) diff --git a/go.mod b/go.mod index 55ddd02b2..df4b73208 100644 --- a/go.mod +++ b/go.mod @@ -311,8 +311,8 @@ require ( go4.org v0.0.0-20230225012048-214862532bf5 // indirect golang.org/x/mod v0.12.0 // indirect golang.org/x/net v0.14.0 // indirect - golang.org/x/sys v0.11.0 // indirect - golang.org/x/term v0.11.0 + golang.org/x/sys v0.12.0 // indirect + golang.org/x/term v0.12.0 golang.org/x/time v0.3.0 // indirect google.golang.org/grpc v1.55.0 // indirect google.golang.org/protobuf v1.31.0 // indirect @@ -336,9 +336,9 @@ require ( github.com/filecoin-project/lotus v1.23.4-rc1 github.com/ipfs/boxo v0.12.0 github.com/ipfs/kubo v0.22.0 - github.com/ipld/go-trustless-utils v0.3.1 + github.com/ipld/frisbii v0.2.1 github.com/ipld/ipld/specs v0.0.0-20230927010225-ef4dbd703269 - github.com/ipni/go-libipni v0.5.1 + github.com/ipni/go-libipni v0.5.2 github.com/ipni/ipni-cli v0.1.1 github.com/schollz/progressbar/v3 v3.13.1 ) @@ -361,6 +361,7 @@ require ( github.com/hashicorp/golang-lru/arc/v2 v2.0.5 // indirect github.com/hashicorp/golang-lru/v2 v2.0.5 // indirect github.com/ipfs/go-ipfs-blockstore v1.3.0 // indirect + github.com/ipld/go-trustless-utils v0.3.1 // indirect github.com/jackc/chunkreader/v2 v2.0.1 // indirect github.com/jackc/pgconn v1.11.0 // indirect github.com/jackc/pgio v1.0.0 // indirect diff --git a/go.sum b/go.sum index e1f34e7b3..93f6ad8ce 100644 --- a/go.sum +++ b/go.sum @@ -898,6 +898,8 @@ github.com/ipfs/go-verifcid v0.0.2 h1:XPnUv0XmdH+ZIhLGKg6U2vaPaRDXb9urMyNVCE7uvT github.com/ipfs/go-verifcid v0.0.2/go.mod h1:40cD9x1y4OWnFXbLNJYRe7MpNvWlMn3LZAG5Wb4xnPU= github.com/ipfs/kubo v0.22.0 h1:HxYkvtFqlF+qQMTxHW+xBhrIWykWm+WbEuQpw1d67mM= github.com/ipfs/kubo v0.22.0/go.mod h1:Sn3hp55POjH9Ni0lEd/+smXpkZ0J1gKlm0Fx+E1LE60= +github.com/ipld/frisbii v0.2.1 h1:DHdDrYFkIoFIjkXHfj3OMJFN17zh9DBMCHjRqumnEOQ= +github.com/ipld/frisbii v0.2.1/go.mod h1:BFeqnuQ+8n2JEyNVmFl7Y5BqsiEDqmS752ZHDdfKq4o= github.com/ipld/go-car v0.1.0/go.mod h1:RCWzaUh2i4mOEkB3W45Vc+9jnS/M6Qay5ooytiBHl3g= github.com/ipld/go-car v0.6.1 h1:blWbEHf1j62JMWFIqWE//YR0m7k5ZMw0AuUOU5hjrH8= github.com/ipld/go-car v0.6.1/go.mod h1:oEGXdwp6bmxJCZ+rARSkDliTeYnVzv3++eXajZ+Bmr8= @@ -931,8 +933,8 @@ github.com/ipld/go-trustless-utils v0.3.1 h1:i7lPoo7HbThj93wJO1aow3UKdL6AV/jeTLX github.com/ipld/go-trustless-utils v0.3.1/go.mod h1:SQR5abLVb2YcZiy9QEsBhNyIPj6ubWISJnrpwNBdmpA= github.com/ipld/ipld/specs v0.0.0-20230927010225-ef4dbd703269 h1:MXBxKsw8geRJitw8f1dr3EfwbEV0WXVbEH7e/o3p+NI= github.com/ipld/ipld/specs v0.0.0-20230927010225-ef4dbd703269/go.mod h1:WcT0DfRe+e2QFY0kcbsOnuT6jL5Q0JNZ83I5DHIdStg= -github.com/ipni/go-libipni v0.5.1 h1:HumuJtKmV8RoDpBakLgxCSl5QPiD2ljTZl/NOyXO6nM= -github.com/ipni/go-libipni v0.5.1/go.mod h1:UnrhEqjVI2Z2HXlaieOBONJmtW557nZkYpB4IIsMD+s= +github.com/ipni/go-libipni v0.5.2 h1:9vaYOnR4dskd8p88NOboqI6yVqBwYPNCQ/zOaRSr59I= +github.com/ipni/go-libipni v0.5.2/go.mod h1:UnrhEqjVI2Z2HXlaieOBONJmtW557nZkYpB4IIsMD+s= github.com/ipni/index-provider v0.14.2 h1:daA3IFnI2n2x/mL0K91SQHNLq6Vvfp5q4uFX9G4glvE= github.com/ipni/index-provider v0.14.2/go.mod h1:mArx7Ou3Y62fIDSj9a1Neh5G14xQcwXGbfEbf47vyuM= github.com/ipni/ipni-cli v0.1.1 h1:TjYAf5CrVx/loQtWQnwEnIYjW7hvRJDRyIibT7WbHjE= @@ -2252,8 +2254,8 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= -golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -2261,8 +2263,8 @@ golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuX golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= -golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0= -golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= +golang.org/x/term v0.12.0 h1:/ZfYdc3zq+q02Rv9vGqTeSItdzZTSNDmfTi0mBAuidU= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=