diff --git a/Makefile b/Makefile index 92508770a..606e28345 100644 --- a/Makefile +++ b/Makefile @@ -143,7 +143,7 @@ update-react: validate-node-version npm run --prefix react build .PHONY: react -build-go: boost boostd boostx boostd-data booster-http booster-bitswap devnet migrate-lid +build-go: boost devnet .PHONY: build-go build: react build-go @@ -217,7 +217,7 @@ docsgen-openrpc-boost: docsgen-openrpc-bin ## DOCKER IMAGES docker_user?=filecoin -lotus_version?=v1.23.2 +lotus_version?=v1.21.0-rc2 ffi_from_source?=0 build_lotus?=0 ifeq ($(build_lotus),1) diff --git a/api/api.go b/api/api.go index 7086848af..5877f52be 100644 --- a/api/api.go +++ b/api/api.go @@ -33,21 +33,21 @@ type Boost interface { Net // MethodGroup: Boost - BoostIndexerAnnounceAllDeals(ctx context.Context) error //perm:admin - BoostIndexerListMultihashes(ctx context.Context, proposalCid cid.Cid) ([]multihash.Multihash, error) //perm:admin - BoostOfflineDealWithData(ctx context.Context, dealUuid uuid.UUID, filePath string) (*ProviderDealRejectionInfo, error) //perm:admin - BoostDeal(ctx context.Context, dealUuid uuid.UUID) (*smtypes.ProviderDealState, error) //perm:admin - BoostDealBySignedProposalCid(ctx context.Context, proposalCid cid.Cid) (*smtypes.ProviderDealState, error) //perm:admin - BoostDummyDeal(context.Context, smtypes.DealParams) (*ProviderDealRejectionInfo, error) //perm:admin - BoostDagstoreRegisterShard(ctx context.Context, key string) error //perm:admin - BoostDagstoreDestroyShard(ctx context.Context, key string) error //perm:admin - BoostDagstoreInitializeShard(ctx context.Context, key string) error //perm:admin - BoostDagstoreInitializeAll(ctx context.Context, params DagstoreInitializeAllParams) (<-chan DagstoreInitializeAllEvent, error) //perm:admin - BoostDagstoreRecoverShard(ctx context.Context, key string) error //perm:admin - BoostDagstoreGC(ctx context.Context) ([]DagstoreShardResult, error) //perm:admin - BoostDagstorePiecesContainingMultihash(ctx context.Context, mh multihash.Multihash) ([]cid.Cid, error) //perm:read - BoostDagstoreListShards(ctx context.Context) ([]DagstoreShardInfo, error) //perm:admin - BoostMakeDeal(context.Context, smtypes.DealParams) (*ProviderDealRejectionInfo, error) //perm:write + BoostIndexerAnnounceAllDeals(ctx context.Context) error //perm:admin + BoostIndexerListMultihashes(ctx context.Context, proposalCid cid.Cid) ([]multihash.Multihash, error) //perm:admin + BoostOfflineDealWithData(ctx context.Context, dealUuid uuid.UUID, filePath string, delAfterImport bool) (*ProviderDealRejectionInfo, error) //perm:admin + BoostDeal(ctx context.Context, dealUuid uuid.UUID) (*smtypes.ProviderDealState, error) //perm:admin + BoostDealBySignedProposalCid(ctx context.Context, proposalCid cid.Cid) (*smtypes.ProviderDealState, error) //perm:admin + BoostDummyDeal(context.Context, smtypes.DealParams) (*ProviderDealRejectionInfo, error) //perm:admin + BoostDagstoreRegisterShard(ctx context.Context, key string) error //perm:admin + BoostDagstoreDestroyShard(ctx context.Context, key string) error //perm:admin + BoostDagstoreInitializeShard(ctx context.Context, key string) error //perm:admin + BoostDagstoreInitializeAll(ctx context.Context, params DagstoreInitializeAllParams) (<-chan DagstoreInitializeAllEvent, error) //perm:admin + BoostDagstoreRecoverShard(ctx context.Context, key string) error //perm:admin + BoostDagstoreGC(ctx context.Context) ([]DagstoreShardResult, error) //perm:admin + BoostDagstorePiecesContainingMultihash(ctx context.Context, mh multihash.Multihash) ([]cid.Cid, error) //perm:read + BoostDagstoreListShards(ctx context.Context) ([]DagstoreShardInfo, error) //perm:admin + BoostMakeDeal(context.Context, smtypes.DealParams) (*ProviderDealRejectionInfo, error) //perm:write // MethodGroup: Blockstore BlockstoreGet(ctx context.Context, c cid.Cid) ([]byte, error) //perm:read diff --git a/build/openrpc/boost.json.gz b/build/openrpc/boost.json.gz index b76f31dca..be25749c3 100644 Binary files a/build/openrpc/boost.json.gz and b/build/openrpc/boost.json.gz differ diff --git a/build/version.go b/build/version.go index ac0ade390..f4436455f 100644 --- a/build/version.go +++ b/build/version.go @@ -2,7 +2,7 @@ package build var CurrentCommit string -const BuildVersion = "1.6.2-rc1" +const BuildVersion = "1.7.0-rc1" func UserVersion() string { return BuildVersion + CurrentCommit diff --git a/cmd/boostd/import_data.go b/cmd/boostd/import_data.go index bed8bcb33..66a7d8c89 100644 --- a/cmd/boostd/import_data.go +++ b/cmd/boostd/import_data.go @@ -67,6 +67,9 @@ var importDataCmd = &cli.Command{ // If the user has supplied a signed proposal cid deleteAfterImport := cctx.Bool("delete-after-import") if proposalCid != nil { + if deleteAfterImport { + return fmt.Errorf("legacy deal data cannot be automatically deleted after import (only new deals)") + } // Look up the deal in the boost database deal, err := napi.BoostDealBySignedProposalCid(cctx.Context, *proposalCid) diff --git a/cmd/booster-bitswap/run.go b/cmd/booster-bitswap/run.go index 3fc52b833..db2444490 100644 --- a/cmd/booster-bitswap/run.go +++ b/cmd/booster-bitswap/run.go @@ -5,9 +5,9 @@ import ( "net/http" _ "net/http/pprof" - "github.com/filecoin-project/boost/cmd/booster-bitswap/filters" - "github.com/filecoin-project/boost/cmd/booster-bitswap/remoteblockstore" "github.com/filecoin-project/boost/cmd/lib" + "github.com/filecoin-project/boost/cmd/lib/filters" + "github.com/filecoin-project/boost/cmd/lib/remoteblockstore" "github.com/filecoin-project/boost/metrics" "github.com/filecoin-project/boost/piecedirectory" "github.com/filecoin-project/boostd-data/shared/tracing" @@ -165,6 +165,20 @@ var runCmd = &cli.Command{ } // Create the bitswap host + bitswapBlockMetrics := remoteblockstore.BlockMetrics{ + GetRequestCount: metrics.BitswapRblsGetRequestCount, + GetFailResponseCount: metrics.BitswapRblsGetFailResponseCount, + GetSuccessResponseCount: metrics.BitswapRblsGetSuccessResponseCount, + BytesSentCount: metrics.BitswapRblsBytesSentCount, + HasRequestCount: metrics.BitswapRblsHasRequestCount, + HasFailResponseCount: metrics.BitswapRblsHasFailResponseCount, + HasSuccessResponseCount: metrics.BitswapRblsHasSuccessResponseCount, + GetSizeRequestCount: metrics.BitswapRblsGetSizeRequestCount, + GetSizeFailResponseCount: metrics.BitswapRblsGetSizeFailResponseCount, + GetSizeSuccessResponseCount: metrics.BitswapRblsGetSizeSuccessResponseCount, + } + + // Create the server API port := cctx.Int("port") repoDir, err := homedir.Expand(cctx.String(FlagRepo.Name)) if err != nil { @@ -183,7 +197,7 @@ var runCmd = &cli.Command{ } pr := &piecedirectory.SectorAccessorAsPieceReader{SectorAccessor: sa} piecedirectory := piecedirectory.NewPieceDirectory(pdClient, pr, cctx.Int("add-index-throttle")) - remoteStore := remoteblockstore.NewRemoteBlockstore(piecedirectory) + remoteStore := remoteblockstore.NewRemoteBlockstore(piecedirectory, &bitswapBlockMetrics) server := NewBitswapServer(remoteStore, host, multiFilter) var proxyAddrInfo *peer.AddrInfo diff --git a/cmd/booster-http/blocks.go b/cmd/booster-http/blocks.go new file mode 100644 index 000000000..0275ddf77 --- /dev/null +++ b/cmd/booster-http/blocks.go @@ -0,0 +1,237 @@ +package main + +// Copied from +// https://github.com/ipfs/go-libipfs/blob/c76138366f1a7c416c481a89b3e71ed29e81a641/examples/gateway/common/blocks.go + +import ( + "context" + "errors" + "fmt" + gopath "path" + + "github.com/ipfs/go-blockservice" + "github.com/ipfs/go-cid" + bsfetcher "github.com/ipfs/go-fetcher/impl/blockservice" + blockstore "github.com/ipfs/go-ipfs-blockstore" + format "github.com/ipfs/go-ipld-format" + "github.com/ipfs/go-libipfs/blocks" + "github.com/ipfs/go-libipfs/files" + "github.com/ipfs/go-merkledag" + "github.com/ipfs/go-namesys" + "github.com/ipfs/go-namesys/resolve" + ipfspath "github.com/ipfs/go-path" + "github.com/ipfs/go-path/resolver" + "github.com/ipfs/go-unixfs" + ufile "github.com/ipfs/go-unixfs/file" + uio "github.com/ipfs/go-unixfs/io" + "github.com/ipfs/go-unixfsnode" + iface "github.com/ipfs/interface-go-ipfs-core" + nsopts "github.com/ipfs/interface-go-ipfs-core/options/namesys" + ifacepath "github.com/ipfs/interface-go-ipfs-core/path" + dagpb "github.com/ipld/go-codec-dagpb" + "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/node/basicnode" + "github.com/ipld/go-ipld-prime/schema" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/libp2p/go-libp2p/core/routing" + mc "github.com/multiformats/go-multicodec" +) + +type BlocksGateway struct { + blockStore blockstore.Blockstore + blockService blockservice.BlockService + dagService format.DAGService + resolver resolver.Resolver + + // Optional routing system to handle /ipns addresses. + namesys namesys.NameSystem + routing routing.ValueStore +} + +func NewBlocksGateway(blockService blockservice.BlockService, routing routing.ValueStore) (*BlocksGateway, error) { + // Setup the DAG services, which use the CAR block store. + dagService := merkledag.NewDAGService(blockService) + + // Setup the UnixFS resolver. + fetcherConfig := bsfetcher.NewFetcherConfig(blockService) + fetcherConfig.PrototypeChooser = dagpb.AddSupportToChooser(func(lnk ipld.Link, lnkCtx ipld.LinkContext) (ipld.NodePrototype, error) { + if tlnkNd, ok := lnkCtx.LinkNode.(schema.TypedLinkNode); ok { + return tlnkNd.LinkTargetNodePrototype(), nil + } + return basicnode.Prototype.Any, nil + }) + fetcher := fetcherConfig.WithReifier(unixfsnode.Reify) + resolver := resolver.NewBasicResolver(fetcher) + + // Setup a name system so that we are able to resolve /ipns links. + var ( + ns namesys.NameSystem + err error + ) + if routing != nil { + ns, err = namesys.NewNameSystem(routing) + if err != nil { + return nil, err + } + } + + return &BlocksGateway{ + blockStore: blockService.Blockstore(), + blockService: blockService, + dagService: dagService, + resolver: resolver, + routing: routing, + namesys: ns, + }, nil +} + +func (api *BlocksGateway) GetUnixFsNode(ctx context.Context, p ifacepath.Resolved) (files.Node, error) { + nd, err := api.resolveNode(ctx, p) + if err != nil { + return nil, err + } + + return ufile.NewUnixfsFile(ctx, api.dagService, nd) +} + +func (api *BlocksGateway) LsUnixFsDir(ctx context.Context, p ifacepath.Resolved) (<-chan iface.DirEntry, error) { + node, err := api.resolveNode(ctx, p) + if err != nil { + return nil, err + } + + dir, err := uio.NewDirectoryFromNode(api.dagService, node) + if err != nil { + return nil, err + } + + out := make(chan iface.DirEntry, uio.DefaultShardWidth) + + go func() { + defer close(out) + for l := range dir.EnumLinksAsync(ctx) { + select { + case out <- api.processLink(ctx, l): + case <-ctx.Done(): + return + } + } + }() + + return out, nil +} + +func (api *BlocksGateway) GetBlock(ctx context.Context, c cid.Cid) (blocks.Block, error) { + return api.blockService.GetBlock(ctx, c) +} + +func (api *BlocksGateway) GetIPNSRecord(ctx context.Context, c cid.Cid) ([]byte, error) { + if api.routing == nil { + return nil, routing.ErrNotSupported + } + + // Fails fast if the CID is not an encoded Libp2p Key, avoids wasteful + // round trips to the remote routing provider. + if mc.Code(c.Type()) != mc.Libp2pKey { + return nil, errors.New("provided cid is not an encoded libp2p key") + } + + // The value store expects the key itself to be encoded as a multihash. + id, err := peer.FromCid(c) + if err != nil { + return nil, err + } + + return api.routing.GetValue(ctx, "/ipns/"+string(id)) +} + +func (api *BlocksGateway) GetDNSLinkRecord(ctx context.Context, hostname string) (ifacepath.Path, error) { + if api.namesys != nil { + p, err := api.namesys.Resolve(ctx, "/ipns/"+hostname, nsopts.Depth(1)) + if err == namesys.ErrResolveRecursion { + err = nil + } + return ifacepath.New(p.String()), err + } + + return nil, errors.New("not implemented") +} + +func (api *BlocksGateway) IsCached(ctx context.Context, p ifacepath.Path) bool { + rp, err := api.ResolvePath(ctx, p) + if err != nil { + return false + } + + has, _ := api.blockStore.Has(ctx, rp.Cid()) + return has +} + +func (api *BlocksGateway) ResolvePath(ctx context.Context, p ifacepath.Path) (ifacepath.Resolved, error) { + if _, ok := p.(ifacepath.Resolved); ok { + return p.(ifacepath.Resolved), nil + } + + err := p.IsValid() + if err != nil { + return nil, err + } + + ipath := ipfspath.Path(p.String()) + if ipath.Segments()[0] == "ipns" { + ipath, err = resolve.ResolveIPNS(ctx, api.namesys, ipath) + if err != nil { + return nil, err + } + } + + if ipath.Segments()[0] != "ipfs" { + return nil, fmt.Errorf("unsupported path namespace: %s", p.Namespace()) + } + + node, rest, err := api.resolver.ResolveToLastNode(ctx, ipath) + if err != nil { + return nil, err + } + + root, err := cid.Parse(ipath.Segments()[1]) + if err != nil { + return nil, err + } + + return ifacepath.NewResolvedPath(ipath, node, root, gopath.Join(rest...)), nil +} + +func (api *BlocksGateway) resolveNode(ctx context.Context, p ifacepath.Path) (format.Node, error) { + rp, err := api.ResolvePath(ctx, p) + if err != nil { + return nil, err + } + + node, err := api.dagService.Get(ctx, rp.Cid()) + if err != nil { + return nil, fmt.Errorf("get node: %w", err) + } + return node, nil +} + +func (api *BlocksGateway) processLink(ctx context.Context, result unixfs.LinkResult) iface.DirEntry { + if result.Err != nil { + return iface.DirEntry{Err: result.Err} + } + + link := iface.DirEntry{ + Name: result.Link.Name, + Cid: result.Link.Cid, + } + + switch link.Cid.Type() { + case cid.Raw: + link.Type = iface.TFile + link.Size = result.Link.Size + case cid.DagProtobuf: + link.Size = result.Link.Size + } + + return link +} diff --git a/cmd/booster-http/gateway_handler.go b/cmd/booster-http/gateway_handler.go index f15663e5f..a7be0a8c4 100644 --- a/cmd/booster-http/gateway_handler.go +++ b/cmd/booster-http/gateway_handler.go @@ -2,11 +2,10 @@ package main import ( "fmt" + "github.com/ipfs/go-libipfs/gateway" "mime" "net/http" "strings" - - "github.com/ipfs/boxo/gateway" ) type gatewayHandler struct { @@ -14,7 +13,7 @@ type gatewayHandler struct { supportedFormats map[string]struct{} } -func newGatewayHandler(gw *gateway.BlocksBackend, supportedFormats []string) http.Handler { +func newGatewayHandler(gw *BlocksGateway, supportedFormats []string) http.Handler { headers := map[string][]string{} gateway.AddAccessControlHeaders(headers) @@ -24,10 +23,7 @@ func newGatewayHandler(gw *gateway.BlocksBackend, supportedFormats []string) htt } return &gatewayHandler{ - gwh: gateway.NewHandler(gateway.Config{ - Headers: headers, - DeserializedResponses: true, - }, gw), + gwh: gateway.NewHandler(gateway.Config{Headers: headers}, gw), supportedFormats: fmtsMap, } } diff --git a/cmd/booster-http/http_test.go b/cmd/booster-http/http_test.go index bc132f747..20bf4ecda 100644 --- a/cmd/booster-http/http_test.go +++ b/cmd/booster-http/http_test.go @@ -22,8 +22,10 @@ const testFile = "test/test_file" func TestNewHttpServer(t *testing.T) { // Create a new mock Http server ctrl := gomock.NewController(t) - httpServer := NewHttpServer("", 7777, mocks_booster_http.NewMockHttpServerApi(ctrl)) - httpServer.Start(context.Background()) + httpServer := NewHttpServer("", 7777, mocks_booster_http.NewMockHttpServerApi(ctrl), nil) + err := httpServer.Start(context.Background()) + require.NoError(t, err) + waitServerUp(t, 7777) // Check that server is responding with 200 status code resp, err := http.Get("http://localhost:7777/") @@ -39,8 +41,10 @@ func TestHttpGzipResponse(t *testing.T) { // Create a new mock Http server with custom functions ctrl := gomock.NewController(t) mockHttpServer := mocks_booster_http.NewMockHttpServerApi(ctrl) - httpServer := NewHttpServer("", 7778, mockHttpServer) - httpServer.Start(context.Background()) + httpServer := NewHttpServer("", 7777, mockHttpServer, nil) + err := httpServer.Start(context.Background()) + require.NoError(t, err) + waitServerUp(t, 7777) // Create mock unsealed file for piece/car f, _ := os.Open(testFile) @@ -65,7 +69,7 @@ func TestHttpGzipResponse(t *testing.T) { //Create a client and make request with Encoding header client := new(http.Client) - request, err := http.NewRequest("GET", "http://localhost:7778/piece/bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi", nil) + request, err := http.NewRequest("GET", "http://localhost:7777/piece/bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi", nil) require.NoError(t, err) request.Header.Add("Accept-Encoding", "gzip") @@ -95,10 +99,12 @@ func TestHttpInfo(t *testing.T) { // Create a new mock Http server ctrl := gomock.NewController(t) - httpServer := NewHttpServer("", 7780, mocks_booster_http.NewMockHttpServerApi(ctrl)) - httpServer.Start(context.Background()) + httpServer := NewHttpServer("", 7777, mocks_booster_http.NewMockHttpServerApi(ctrl), nil) + err := httpServer.Start(context.Background()) + require.NoError(t, err) + waitServerUp(t, 7777) - response, err := http.Get("http://localhost:7780/info") + response, err := http.Get("http://localhost:7777/info") require.NoError(t, err) defer response.Body.Close() @@ -109,3 +115,10 @@ func TestHttpInfo(t *testing.T) { err = httpServer.Stop() require.NoError(t, err) } + +func waitServerUp(t *testing.T, port int) { + require.Eventually(t, func() bool { + _, err := http.Get(fmt.Sprintf("http://localhost:%d", port)) + return err == nil + }, time.Second, 100*time.Millisecond) +} diff --git a/cmd/booster-http/mocks/mock_booster_http.go b/cmd/booster-http/mocks/mock_booster_http.go index 9a143b56d..77cb30a93 100644 --- a/cmd/booster-http/mocks/mock_booster_http.go +++ b/cmd/booster-http/mocks/mock_booster_http.go @@ -1,8 +1,8 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: server.go +// Source: cmd/booster-http/server.go -// Package mocks_booster_http is a generated GoMock package. -package mocks_booster_http +// Package mock_main is a generated GoMock package. +package mock_main import ( context "context" @@ -38,21 +38,6 @@ func (m *MockHttpServerApi) EXPECT() *MockHttpServerApiMockRecorder { return m.recorder } -// GetBlockByCid mocks base method. -func (m *MockHttpServerApi) GetBlockByCid(ctx context.Context, blockCid cid.Cid) ([]byte, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetBlockByCid", ctx, blockCid) - ret0, _ := ret[0].([]byte) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetBlockByCid indicates an expected call of GetBlockByCid. -func (mr *MockHttpServerApiMockRecorder) GetBlockByCid(ctx, blockCid interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetBlockByCid", reflect.TypeOf((*MockHttpServerApi)(nil).GetBlockByCid), ctx, blockCid) -} - // GetPieceDeals mocks base method. func (m *MockHttpServerApi) GetPieceDeals(ctx context.Context, pieceCID cid.Cid) ([]model.DealInfo, error) { m.ctrl.T.Helper() diff --git a/cmd/booster-http/run.go b/cmd/booster-http/run.go index fea91d922..b624e2856 100644 --- a/cmd/booster-http/run.go +++ b/cmd/booster-http/run.go @@ -6,8 +6,13 @@ import ( "fmt" "net/http" _ "net/http/pprof" + "os" + "strings" "github.com/filecoin-project/boost/cmd/lib" + "github.com/filecoin-project/boost/cmd/lib/filters" + "github.com/filecoin-project/boost/cmd/lib/remoteblockstore" + "github.com/filecoin-project/boost/metrics" "github.com/filecoin-project/boost/piecedirectory" "github.com/filecoin-project/boostd-data/model" "github.com/filecoin-project/boostd-data/shared/tracing" @@ -170,6 +175,40 @@ var runCmd = &cli.Command{ // Create the server API pr := &piecedirectory.SectorAccessorAsPieceReader{SectorAccessor: sa} piecedirectory := piecedirectory.NewPieceDirectory(pdClient, pr, cctx.Int("add-index-throttle")) + + opts := &HttpServerOptions{ + ServePieces: servePieces, + SupportedResponseFormats: responseFormats, + } + if enableIpfsGateway { + repoDir, err := createRepoDir(cctx.String(FlagRepo.Name)) + if err != nil { + return err + } + + // Set up badbits filter + multiFilter := filters.NewMultiFilter(repoDir, cctx.String("api-filter-endpoint"), cctx.String("api-filter-auth"), cctx.StringSlice("badbits-denylists")) + err = multiFilter.Start(ctx) + if err != nil { + return fmt.Errorf("starting block filter: %w", err) + } + + httpBlockMetrics := remoteblockstore.BlockMetrics{ + GetRequestCount: metrics.HttpRblsGetRequestCount, + GetFailResponseCount: metrics.HttpRblsGetFailResponseCount, + GetSuccessResponseCount: metrics.HttpRblsGetSuccessResponseCount, + BytesSentCount: metrics.HttpRblsBytesSentCount, + HasRequestCount: metrics.HttpRblsHasRequestCount, + HasFailResponseCount: metrics.HttpRblsHasFailResponseCount, + HasSuccessResponseCount: metrics.HttpRblsHasSuccessResponseCount, + GetSizeRequestCount: metrics.HttpRblsGetSizeRequestCount, + GetSizeFailResponseCount: metrics.HttpRblsGetSizeFailResponseCount, + GetSizeSuccessResponseCount: metrics.HttpRblsGetSizeSuccessResponseCount, + } + rbs := remoteblockstore.NewRemoteBlockstore(piecedirectory, &httpBlockMetrics) + filtered := filters.NewFilteredBlockstore(rbs, multiFilter) + opts.Blockstore = filtered + } sapi := serverApi{ctx: ctx, piecedirectory: piecedirectory, sa: sa} server := NewHttpServer( cctx.String("base-path"), @@ -185,7 +224,17 @@ var runCmd = &cli.Command{ // Start the server log.Infof("Starting booster-http node on port %d with base path '%s'", cctx.Int("port"), cctx.String("base-path")) - server.Start(ctx) + err = server.Start(ctx) + if err != nil { + return fmt.Errorf("starting http server: %w", err) + } + + log.Infof(ipfsGatewayMsg(cctx, server.ipfsBasePath())) + if servePieces { + log.Infof("serving raw pieces at " + server.pieceBasePath()) + } else { + log.Infof("serving raw pieces is disabled") + } // Monitor for shutdown. <-ctx.Done() diff --git a/cmd/booster-http/server.go b/cmd/booster-http/server.go index 32c98baf4..03c38f6b7 100644 --- a/cmd/booster-http/server.go +++ b/cmd/booster-http/server.go @@ -20,12 +20,11 @@ import ( "github.com/filecoin-project/dagstore/mount" "github.com/filecoin-project/go-state-types/abi" "github.com/hashicorp/go-multierror" - "github.com/ipfs/boxo/blockservice" - blockstore "github.com/ipfs/boxo/blockstore" - offline "github.com/ipfs/boxo/exchange/offline" - "github.com/ipfs/boxo/gateway" + "github.com/ipfs/go-blockservice" "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" + blockstore "github.com/ipfs/go-ipfs-blockstore" + offline "github.com/ipfs/go-ipfs-exchange-offline" "go.opencensus.io/stats" ) @@ -43,9 +42,11 @@ type apiVersion struct { } type HttpServer struct { - path string - port int - api HttpServerApi + path string + port int + api HttpServerApi + opts HttpServerOptions + idxPage string ctx context.Context cancel context.CancelFunc @@ -58,8 +59,17 @@ type HttpServerApi interface { UnsealSectorAt(ctx context.Context, sectorID abi.SectorNumber, pieceOffset abi.UnpaddedPieceSize, length abi.UnpaddedPieceSize) (mount.Reader, error) } -func NewHttpServer(path string, port int, api HttpServerApi) *HttpServer { - return &HttpServer{path: path, port: port, api: api} +type HttpServerOptions struct { + Blockstore blockstore.Blockstore + ServePieces bool + SupportedResponseFormats []string +} + +func NewHttpServer(path string, port int, api HttpServerApi, opts *HttpServerOptions) *HttpServer { + if opts == nil { + opts = &HttpServerOptions{ServePieces: true} + } + return &HttpServer{path: path, port: port, api: api, opts: *opts, idxPage: parseTemplate(*opts)} } func (s *HttpServer) pieceBasePath() string { @@ -80,7 +90,7 @@ func (s *HttpServer) Start(ctx context.Context) error { if s.opts.Blockstore != nil { blockService := blockservice.New(s.opts.Blockstore, offline.Exchange(s.opts.Blockstore)) - gw, err := gateway.NewBlocksBackend(blockService) + gw, err := NewBlocksGateway(blockService, nil) if err != nil { return fmt.Errorf("creating blocks gateway: %w", err) } @@ -92,7 +102,7 @@ func (s *HttpServer) Start(ctx context.Context) error { handler.HandleFunc("/info", s.handleInfo) handler.Handle("/metrics", metrics.Exporter("booster_http")) // metrics s.server = &http.Server{ - Addr: fmt.Sprintf("%s:%d", s.listenAddr, s.port), + Addr: fmt.Sprintf(":%d", s.port), Handler: handler, // This context will be the parent of the context associated with all // incoming requests @@ -115,27 +125,6 @@ func (s *HttpServer) Stop() error { return s.server.Close() } -const idxPage = ` - - -

Booster HTTP Server

- Endpoints: - - - - - - - -
- Download a raw piece by its piece CID - - /piece/ -
- - -` - func (s *HttpServer) handleIndex(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) w.Write([]byte(s.idxPage)) //nolint:errcheck @@ -306,7 +295,7 @@ func (s *HttpServer) getPieceContent(ctx context.Context, pieceCid cid.Cid) (io. } func (s *HttpServer) unsealedDeal(ctx context.Context, pieceCid cid.Cid, pieceDeals []model.DealInfo) (*model.DealInfo, error) { - // There should always been deals in the PieceInfo, but check just in case + // There should always be deals in the PieceInfo, but check just in case if len(pieceDeals) == 0 { return nil, fmt.Errorf("there are no deals containing piece %s: %w", pieceCid, ErrNotFound) } diff --git a/cmd/boostx/stats_cmd.go b/cmd/boostx/stats_cmd.go index 1b8775955..42bcc1a96 100644 --- a/cmd/boostx/stats_cmd.go +++ b/cmd/boostx/stats_cmd.go @@ -2,6 +2,8 @@ package main import ( "fmt" + "github.com/filecoin-project/boost/retrievalmarket/lp2pimpl" + transports_types "github.com/filecoin-project/boost/retrievalmarket/types" "regexp" "sort" "strings" diff --git a/cmd/lib/filters/filteredblockstore.go b/cmd/lib/filters/filteredblockstore.go index 5a01a1f7a..bff2d7efe 100644 --- a/cmd/lib/filters/filteredblockstore.go +++ b/cmd/lib/filters/filteredblockstore.go @@ -3,10 +3,10 @@ package filters import ( "context" "fmt" - blockstore "github.com/ipfs/boxo/blockstore" - blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" + blockstore "github.com/ipfs/go-ipfs-blockstore" ipld "github.com/ipfs/go-ipld-format" + "github.com/ipfs/go-libipfs/blocks" ) type FilteredBlockstore struct { diff --git a/cmd/lib/filters/filteredblockstore_test.go b/cmd/lib/filters/filteredblockstore_test.go index b6342e2c0..154a355c1 100644 --- a/cmd/lib/filters/filteredblockstore_test.go +++ b/cmd/lib/filters/filteredblockstore_test.go @@ -43,7 +43,7 @@ func TestFilteredBlockstore(t *testing.T) { filter.EXPECT().FulfillRequest(gomock.Any(), blk.Cid()).Return(true, nil) gotBlk, err := fbs.Get(ctx, blk.Cid()) require.NoError(t, err) - require.Equal(t, blk.RawData(), gotBlk.RawData()) + require.Equal(t, blk, gotBlk) // Filter rejects cid filter.EXPECT().FulfillRequest(gomock.Any(), blk.Cid()).Return(false, nil) diff --git a/db/migrations_tests/deals_announce_to_ipni_test.go b/db/migrations_tests/deals_announce_to_ipni_test.go index 696dc2e63..2a766e828 100644 --- a/db/migrations_tests/deals_announce_to_ipni_test.go +++ b/db/migrations_tests/deals_announce_to_ipni_test.go @@ -29,15 +29,13 @@ func TestDealAnnounceToIPNI(t *testing.T) { // Insert the deal into the DB deal := deals[0] - deal.AnnounceToIPNI = false - _, err = sqldb.Exec(`INSERT INTO Deals ("ID", "CreatedAt", "DealProposalSignature", "PieceCID", "PieceSize", "VerifiedDeal", "IsOffline", "ClientAddress", "ProviderAddress","Label", "StartEpoch", "EndEpoch", "StoragePricePerEpoch", "ProviderCollateral", "ClientCollateral", "ClientPeerID", "DealDataRoot", "InboundFilePath", "TransferType", "TransferParams", "TransferSize", "ChainDealID", "PublishCID", "SectorID", "Offset", "Length", "Checkpoint", "CheckpointAt", "Error", "Retry", "SignedProposalCID", - "FastRetrieval","AnnounceToIPNI") - VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`, + "FastRetrieval") + VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`, deal.DealUuid, deal.CreatedAt, []byte("test"), deal.ClientDealProposal.Proposal.PieceCID.String(), deal.ClientDealProposal.Proposal.PieceSize, deal.ClientDealProposal.Proposal.VerifiedDeal, deal.IsOffline, deal.ClientDealProposal.Proposal.Client.String(), deal.ClientDealProposal.Proposal.Provider.String(), "test", @@ -45,7 +43,7 @@ func TestDealAnnounceToIPNI(t *testing.T) { deal.ClientDealProposal.Proposal.ProviderCollateral.Int64(), deal.ClientDealProposal.Proposal.ClientCollateral.Uint64(), deal.ClientPeerID.String(), deal.DealDataRoot.String(), deal.InboundFilePath, deal.Transfer.Type, deal.Transfer.Params, deal.Transfer.Size, deal.ChainDealID, deal.PublishCID.String(), deal.SectorID, deal.Offset, deal.Length, deal.Checkpoint.String(), deal.CheckpointAt, deal.Err, deal.Retry, []byte("test"), - deal.FastRetrieval, deal.AnnounceToIPNI) + deal.FastRetrieval) req.NoError(err) diff --git a/docker/monitoring/grafana/dashboards/provider-retrievals.json b/docker/monitoring/grafana/dashboards/provider-retrievals.json index d6bdab783..e8acde859 100644 --- a/docker/monitoring/grafana/dashboards/provider-retrievals.json +++ b/docker/monitoring/grafana/dashboards/provider-retrievals.json @@ -24,7 +24,7 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 2, + "id": 3, "links": [], "liveNow": false, "panels": [ @@ -1614,6 +1614,424 @@ "title": "booster-http: ipfs gateway", "type": "row" }, + { + "datasource": { + "type": "prometheus", + "uid": "prom1234" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 66 + }, + "id": 74, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prom1234" + }, + "editorMode": "code", + "exemplar": true, + "expr": "irate(booster_http_http_rbls_bytes_sent_count{}[$__rate_interval]) * 15", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Http Gateway Bytes Served", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prom1234" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 73 + }, + "id": 76, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prom1234" + }, + "editorMode": "code", + "expr": "irate(booster_http_http_rbls_get_request_count{}[$__rate_interval]) * 15", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Http Gateway Block Get Requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prom1234" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 73 + }, + "id": 78, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prom1234" + }, + "editorMode": "code", + "expr": "irate(booster_http_http_rbls_get_success_response_count{}[$__rate_interval]) * 15", + "legendFormat": "__auto", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prom1234" + }, + "editorMode": "code", + "expr": "irate(booster_http_http_rbls_get_fail_response_count{}[$__rate_interval]) * 15", + "hide": false, + "legendFormat": "__auto", + "range": true, + "refId": "B" + } + ], + "title": "Http Gateway Block Get Response Counts", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prom1234" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 73 + }, + "id": 80, + "options": { + "legend": { + "calcs": [ + "last", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prom1234" + }, + "editorMode": "code", + "expr": "(irate(booster_http_http_rbls_get_success_response_count{}[$__rate_interval]) * 15) / (irate(booster_http_http_rbls_get_request_count{}[$__rate_interval]) * 15)", + "legendFormat": "Success", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prom1234" + }, + "editorMode": "code", + "expr": "(irate(booster_http_http_rbls_get_fail_response_count{}[$__rate_interval]) * 15) / (irate(booster_http_http_rbls_get_request_count{}[$__rate_interval]) * 15)", + "hide": false, + "legendFormat": "Fail", + "range": true, + "refId": "B" + } + ], + "title": "Http Block Get Success / Failure Rate", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 81 + }, + "id": 30, + "panels": [], + "title": "booster-http: ipfs gateway", + "type": "row" + }, { "datasource": { "type": "prometheus", @@ -1677,7 +2095,7 @@ "h": 7, "w": 24, "x": 0, - "y": 59 + "y": 82 }, "id": 74, "options": { @@ -1767,6 +2185,101 @@ } ] } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 82 + }, + "id": 3, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0-77684pre", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prom1234" + }, + "editorMode": "code", + "expr": "irate(booster_http_http_piece_by_cid_request_count{host=~\"$host\"}[$__rate_interval]) * 15", + "legendFormat": "Requests", + "range": true, + "refId": "A" + } + ], + "title": "Piece by CID Requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prom1234" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" }, "overrides": [] }, @@ -1774,7 +2287,7 @@ "h": 8, "w": 8, "x": 0, - "y": 66 + "y": 89 }, "id": 76, "options": { @@ -2193,10 +2706,10 @@ "overrides": [] }, "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 83 + "h": 8, + "w": 12, + "x": 12, + "y": 89 }, "id": 37, "options": { @@ -2351,9 +2864,9 @@ }, "gridPos": { "h": 7, - "w": 8, - "x": 8, - "y": 83 + "w": 12, + "x": 0, + "y": 97 }, "id": 3, "options": { @@ -2444,9 +2957,9 @@ }, "gridPos": { "h": 7, - "w": 8, - "x": 16, - "y": 83 + "w": 12, + "x": 12, + "y": 97 }, "id": 6, "options": { @@ -2502,13 +3015,109 @@ "title": "Piece by CID Request Durations", "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "prom1234" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 104 + }, + "id": 9, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prom1234" + }, + "editorMode": "code", + "expr": "go_goroutines{job=~\"booster-http\", host=~\"$host\"}", + "legendFormat": "Goroutines", + "range": true, + "refId": "A" + } + ], + "title": "Goroutines", + "type": "timeseries" + }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 90 + "y": 112 }, "id": 82, "panels": [], @@ -2578,7 +3187,7 @@ "h": 8, "w": 12, "x": 0, - "y": 91 + "y": 113 }, "id": 49, "options": { @@ -2676,7 +3285,7 @@ "h": 8, "w": 12, "x": 12, - "y": 91 + "y": 113 }, "id": 50, "options": { @@ -2773,7 +3382,7 @@ "h": 8, "w": 12, "x": 0, - "y": 99 + "y": 121 }, "id": 51, "options": { @@ -2870,7 +3479,7 @@ "h": 8, "w": 12, "x": 12, - "y": 99 + "y": 121 }, "id": 52, "options": { @@ -2966,7 +3575,7 @@ "h": 8, "w": 12, "x": 0, - "y": 107 + "y": 129 }, "id": 54, "options": { @@ -3062,7 +3671,7 @@ "h": 8, "w": 12, "x": 12, - "y": 107 + "y": 129 }, "id": 57, "options": { @@ -3159,7 +3768,7 @@ "h": 8, "w": 12, "x": 0, - "y": 115 + "y": 137 }, "id": 56, "options": { @@ -3256,7 +3865,7 @@ "h": 8, "w": 12, "x": 12, - "y": 115 + "y": 137 }, "id": 55, "options": { @@ -3333,6 +3942,6 @@ "timezone": "", "title": "Provider Retrievals", "uid": "VG2IXa4Vk", - "version": 5, + "version": 16, "weekStart": "" } \ No newline at end of file diff --git a/go.mod b/go.mod index f8d33b279..edbed72f3 100644 --- a/go.mod +++ b/go.mod @@ -45,7 +45,7 @@ require ( github.com/filecoin-project/go-paramfetch v0.0.4 github.com/filecoin-project/go-state-types v0.11.1 github.com/filecoin-project/go-statestore v0.2.0 - github.com/filecoin-project/lotus v1.21.0-rc1 + github.com/filecoin-project/lotus v1.21.0-rc2 github.com/filecoin-project/specs-actors v0.9.15 github.com/gbrlsnchs/jwt/v3 v3.0.1 github.com/golang/mock v1.6.0 @@ -55,9 +55,9 @@ require ( github.com/graph-gophers/graphql-transport-ws v0.0.2 github.com/hashicorp/go-multierror v1.1.1 github.com/hnlq715/golang-lru v0.3.0 - github.com/ipfs/go-block-format v0.1.2 - github.com/ipfs/go-blockservice v0.5.1 // indirect - github.com/ipfs/go-cid v0.4.1 + github.com/ipfs/go-block-format v0.1.1 // indirect + github.com/ipfs/go-blockservice v0.5.0 + github.com/ipfs/go-cid v0.4.0 github.com/ipfs/go-cidutil v0.1.0 github.com/ipfs/go-datastore v0.6.0 github.com/ipfs/go-graphsync v0.14.3 @@ -71,7 +71,7 @@ require ( github.com/ipfs/go-ipfs-routing v0.3.0 github.com/ipfs/go-ipld-format v0.4.0 github.com/ipfs/go-ipld-legacy v0.1.1 - github.com/ipfs/go-libipfs v0.5.0 + github.com/ipfs/go-libipfs v0.7.0 github.com/ipfs/go-log/v2 v2.5.1 github.com/ipfs/go-merkledag v0.9.0 github.com/ipfs/go-metrics-interface v0.0.1 @@ -122,7 +122,7 @@ require ( golang.org/x/text v0.7.0 golang.org/x/tools v0.3.0 golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 - google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect + google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6 // indirect gopkg.in/cheggaaa/pb.v1 v1.0.28 ) @@ -234,10 +234,11 @@ require ( github.com/ipfs/go-ipfs-util v0.0.3 // indirect github.com/ipfs/go-ipld-cbor v0.0.6 github.com/ipfs/go-log v1.0.5 // indirect + github.com/ipfs/go-path v0.3.1 github.com/ipfs/go-peertaskqueue v0.8.1 // indirect github.com/ipfs/go-unixfsnode v1.5.2 github.com/ipfs/go-verifcid v0.0.2 // indirect - github.com/ipfs/interface-go-ipfs-core v0.11.1 // indirect + github.com/ipfs/interface-go-ipfs-core v0.11.1 github.com/ipld/go-codec-dagpb v1.5.0 github.com/ipld/go-ipld-adl-hamt v0.0.0-20220616142416-9004dbd839e0 // indirect github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 // indirect @@ -273,7 +274,7 @@ require ( github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.17 // indirect - github.com/mattn/go-runewidth v0.0.13 // indirect + github.com/mattn/go-runewidth v0.0.14 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/miekg/dns v1.1.50 // indirect github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect @@ -326,10 +327,10 @@ require ( go4.org v0.0.0-20200411211856-f5505b9728dd // indirect golang.org/x/mod v0.7.0 // indirect golang.org/x/net v0.7.0 // indirect - golang.org/x/sys v0.5.0 // indirect - golang.org/x/term v0.5.0 + golang.org/x/sys v0.6.0 // indirect + golang.org/x/term v0.6.0 golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9 // indirect - google.golang.org/grpc v1.47.0 // indirect + google.golang.org/grpc v1.52.3 // indirect google.golang.org/protobuf v1.28.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect @@ -342,8 +343,17 @@ require ( github.com/filecoin-project/boostd-data v0.0.0-00010101000000-000000000000 github.com/gabriel-vasile/mimetype v1.4.1 // indirect github.com/ipfs/go-ds-flatfs v0.5.1 + github.com/ipfs/go-fetcher v1.6.1 + github.com/ipfs/go-ipfs-redirects-file v0.1.1 // indirect + github.com/ipfs/go-namesys v0.7.0 +) + +require ( + github.com/filecoin-project/boost-gfm v1.26.5 + github.com/filecoin-project/boost-graphsync v0.13.6 + github.com/filecoin-project/go-data-transfer/v2 v2.0.0-rc4 github.com/ipfs/kubo v0.18.1 - github.com/schollz/progressbar/v3 v3.11.0 + github.com/schollz/progressbar/v3 v3.13.1 ) require ( @@ -354,9 +364,6 @@ require ( github.com/docker/distribution v2.8.1+incompatible // indirect github.com/docker/docker v20.10.21+incompatible // indirect github.com/docker/go-connections v0.4.0 // indirect - github.com/filecoin-project/boost-gfm v1.26.5 - github.com/filecoin-project/boost-graphsync v0.13.6 - github.com/filecoin-project/go-data-transfer/v2 v2.0.0-rc4 github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect @@ -364,6 +371,7 @@ require ( require ( github.com/google/pprof v0.0.0-20221203041831-ce31453925ec // indirect + github.com/hashicorp/go-uuid v1.0.2 // indirect github.com/hashicorp/golang-lru/v2 v2.0.2 // indirect github.com/onsi/ginkgo/v2 v2.5.1 // indirect github.com/quic-go/qpack v0.4.0 // indirect @@ -371,6 +379,8 @@ require ( github.com/quic-go/qtls-go1-20 v0.1.1 // indirect github.com/quic-go/quic-go v0.33.0 // indirect github.com/quic-go/webtransport-go v0.5.2 // indirect + github.com/tidwall/gjson v1.14.0 // indirect + github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb // indirect github.com/zyedidia/generic v1.2.1 // indirect nhooyr.io/websocket v1.8.7 // indirect ) diff --git a/go.sum b/go.sum index ae4721ee1..49e7b4d6d 100644 --- a/go.sum +++ b/go.sum @@ -204,11 +204,6 @@ github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= -github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA= @@ -338,8 +333,6 @@ github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4s github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/etclabscore/go-jsonschema-walk v0.0.6 h1:DrNzoKWKd8f8XB5nFGBY00IcjakRE22OTI12k+2LkyY= github.com/etclabscore/go-jsonschema-walk v0.0.6/go.mod h1:VdfDY72AFAiUhy0ZXEaWSpveGjMT5JcDIm903NGqFwQ= @@ -426,8 +419,8 @@ github.com/filecoin-project/go-statestore v0.2.0 h1:cRRO0aPLrxKQCZ2UOQbzFGn4WDNd github.com/filecoin-project/go-statestore v0.2.0/go.mod h1:8sjBYbS35HwPzct7iT4lIXjLlYyPor80aU7t7a/Kspo= github.com/filecoin-project/go-storedcounter v0.1.0 h1:Mui6wSUBC+cQGHbDUBcO7rfh5zQkWJM/CpAZa/uOuus= github.com/filecoin-project/go-storedcounter v0.1.0/go.mod h1:4ceukaXi4vFURIoxYMfKzaRF5Xv/Pinh2oTnoxpv+z8= -github.com/filecoin-project/lotus v1.21.0-rc1 h1:LcU1T4tyGbBYDHA6Hl/lVBcR80wUsLN9xiIB42XuUqM= -github.com/filecoin-project/lotus v1.21.0-rc1/go.mod h1:EQk2GAEwWeLCZNRU1MaF8Kj6cGVMK1F6Tt30OMXnL/M= +github.com/filecoin-project/lotus v1.21.0-rc2 h1:EAADt5PpVei/sANsa//oQK8wm3bxxeFn/lbtiEtT4sI= +github.com/filecoin-project/lotus v1.21.0-rc2/go.mod h1:a4qjLilt0mDENhD3RmV5N8hj8d0VxEcRn53266V0KNY= github.com/filecoin-project/pubsub v1.0.0 h1:ZTmT27U07e54qV1mMiQo4HDr0buo8I1LDHBYLXlsNXM= github.com/filecoin-project/pubsub v1.0.0/go.mod h1:GkpB33CcUtUNrLPhJgfdy4FDx4OMNR9k+46DHx/Lqrg= github.com/filecoin-project/specs-actors v0.9.13/go.mod h1:TS1AW/7LbG+615j4NsjMK1qlpAwaFsG9w0V2tg2gSao= @@ -463,6 +456,8 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/gabriel-vasile/mimetype v1.4.1 h1:TRWk7se+TOjCYgRth7+1/OYLNiRNIotknkFtf/dnN7Q= +github.com/gabriel-vasile/mimetype v1.4.1/go.mod h1:05Vi0w3Y9c/lNvJOdmIwvrrAhX3rYhfQQCaf9VJcv7M= github.com/gbrlsnchs/jwt/v3 v3.0.1 h1:lbUmgAKpxnClrKloyIwpxm4OuWeDl5wLk52G91ODPw4= github.com/gbrlsnchs/jwt/v3 v3.0.1/go.mod h1:AncDcjXz18xetI3A6STfXq2w+LuTx8pQ8bGEwRN8zVM= github.com/gdamore/encoding v1.0.0 h1:+7OoQ1Bc6eTm5niUzBa0Ctsh6JbMW6Ra+YNuAtDBdko= @@ -675,7 +670,6 @@ github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpg github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= -github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8= @@ -788,8 +782,8 @@ github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqg github.com/ipfs/go-cid v0.1.0/go.mod h1:rH5/Xv83Rfy8Rw6xG+id3DYAMUVmem1MowoKwdXmN2o= github.com/ipfs/go-cid v0.2.0/go.mod h1:P+HXFDF4CVhaVayiEb4wkAy7zBHxBwsJyt0Y5U6MLro= github.com/ipfs/go-cid v0.3.2/go.mod h1:gQ8pKqT/sUxGY+tIwy1RPpAojYu7jAyCp5Tz1svoupw= -github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= -github.com/ipfs/go-cid v0.4.1/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk= +github.com/ipfs/go-cid v0.4.0 h1:a4pdZq0sx6ZSxbCizebnKiMCx/xI/aBBFlB73IgH4rA= +github.com/ipfs/go-cid v0.4.0/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk= github.com/ipfs/go-cidutil v0.1.0 h1:RW5hO7Vcf16dplUU60Hs0AKDkQAVPVplr7lk97CFL+Q= github.com/ipfs/go-cidutil v0.1.0/go.mod h1:e7OEVBMIv9JaOxt9zaGEmAoSlXW9jdFZ5lP/0PwcfpA= github.com/ipfs/go-datastore v0.0.1/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= @@ -824,6 +818,10 @@ github.com/ipfs/go-ds-leveldb v0.5.0 h1:s++MEBbD3ZKc9/8/njrn4flZLnCuY9I79v94gBUN github.com/ipfs/go-ds-leveldb v0.5.0/go.mod h1:d3XG9RUDzQ6V4SHi8+Xgj9j1XuEk1z82lquxrVbml/Q= github.com/ipfs/go-ds-measure v0.2.0 h1:sG4goQe0KDTccHMyT45CY1XyUbxe5VwTKpg2LjApYyQ= github.com/ipfs/go-ds-measure v0.2.0/go.mod h1:SEUD/rE2PwRa4IQEC5FuNAmjJCyYObZr9UvVh8V3JxE= +github.com/ipfs/go-fetcher v1.6.1 h1:UFuRVYX5AIllTiRhi5uK/iZkfhSpBCGX7L70nSZEmK8= +github.com/ipfs/go-fetcher v1.6.1/go.mod h1:27d/xMV8bodjVs9pugh/RCjjK2OZ68UgAMspMdingNo= +github.com/ipfs/go-filestore v1.2.0 h1:O2wg7wdibwxkEDcl7xkuQsPvJFRBVgVSsOJ/GP6z3yU= +github.com/ipfs/go-filestore v1.2.0/go.mod h1:HLJrCxRXquTeEEpde4lTLMaE/MYJZD7WHLkp9z6+FF8= github.com/ipfs/go-fs-lock v0.0.6/go.mod h1:OTR+Rj9sHiRubJh3dRhD15Juhd/+w6VPOY28L7zESmM= github.com/ipfs/go-fs-lock v0.0.7 h1:6BR3dajORFrFTkb5EpCUFIAypsoxpGpDSVUdFwzgL9U= github.com/ipfs/go-fs-lock v0.0.7/go.mod h1:Js8ka+FNYmgQRLrRXzU3CB/+Csr1BwrRilEcvYrHhhc= @@ -900,8 +898,8 @@ github.com/ipfs/go-ipld-legacy v0.1.1 h1:BvD8PEuqwBHLTKqlGFTHSwrwFOMkVESEvwIYwR2 github.com/ipfs/go-ipld-legacy v0.1.1/go.mod h1:8AyKFCjgRPsQFf15ZQgDB8Din4DML/fOmKZkkFkrIEg= github.com/ipfs/go-ipns v0.3.0 h1:ai791nTgVo+zTuq2bLvEGmWP1M0A6kGTXUsgv/Yq67A= github.com/ipfs/go-ipns v0.3.0/go.mod h1:3cLT2rbvgPZGkHJoPO1YMJeh6LtkxopCkKFcio/wE24= -github.com/ipfs/go-libipfs v0.5.0 h1:gtvzeoTdUHPUN4B5izzyBS3Cxtpvi+l7hd2mmjN+teM= -github.com/ipfs/go-libipfs v0.5.0/go.mod h1:iZ9QyhzNr3AkxRXrbQYb//rv7iLyvZJX0GNuc3lJDiQ= +github.com/ipfs/go-libipfs v0.7.0 h1:Mi54WJTODaOL2/ZSm5loi3SwI3jI2OuFWUrQIkJ5cpM= +github.com/ipfs/go-libipfs v0.7.0/go.mod h1:KsIf/03CqhICzyRGyGo68tooiBE2iFbI/rXW7FhAYr0= github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM= github.com/ipfs/go-log v1.0.0/go.mod h1:JO7RzlMK6rA+CIxFMLOuB6Wf5b81GDiKElL7UPSIKjA= github.com/ipfs/go-log v1.0.1/go.mod h1:HuWlQttfN6FWNHRhlY5yMk/lW7evQC0HHGOxEwMRR8I= @@ -932,6 +930,10 @@ github.com/ipfs/go-merkledag v0.9.0 h1:DFC8qZ96Dz1hMT7dtIpcY524eFFDiEWAF8hNJHWW2 github.com/ipfs/go-merkledag v0.9.0/go.mod h1:bPHqkHt5OZ0p1n3iqPeDiw2jIBkjAytRjS3WSBwjq90= github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg= github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY= +github.com/ipfs/go-namesys v0.7.0 h1:xqosk71GIVRkFDtF2UNRcXn4LdNeo7tzuy8feHD6NbU= +github.com/ipfs/go-namesys v0.7.0/go.mod h1:KYSZBVZG3VJC34EfqqJPG7T48aWgxseoMPAPA5gLyyQ= +github.com/ipfs/go-path v0.3.1 h1:wkeaCWE/NTuuPGlEkLTsED5UkzfKYZpxaFFPgk8ZVLE= +github.com/ipfs/go-path v0.3.1/go.mod h1:eNLsxJEEMxn/CDzUJ6wuNl+6No6tEUhOZcPKsZsYX0E= github.com/ipfs/go-peertaskqueue v0.1.0/go.mod h1:Jmk3IyCcfl1W3jTW3YpghSwSEC6IJ3Vzz/jUmWw8Z0U= github.com/ipfs/go-peertaskqueue v0.7.0/go.mod h1:M/akTIE/z1jGNXMU7kFB4TeSEFvj68ow0Rrb04donIU= github.com/ipfs/go-peertaskqueue v0.8.0/go.mod h1:cz8hEnnARq4Du5TGqiWKgMr/BOSQ5XOgMOh1K5YYKKM= @@ -941,7 +943,6 @@ github.com/ipfs/go-unixfs v0.2.2-0.20190827150610-868af2e9e5cb/go.mod h1:IwAAgul github.com/ipfs/go-unixfs v0.3.1/go.mod h1:h4qfQYzghiIc8ZNFKiLMFWOTzrWIAtzYQ59W/pCFf1o= github.com/ipfs/go-unixfs v0.4.3 h1:EdDc1sNZNFDUlo4UrVAvvAofVI5EwTnKu8Nv8mgXkWQ= github.com/ipfs/go-unixfs v0.4.3/go.mod h1:TSG7G1UuT+l4pNj91raXAPkX0BhJi3jST1FDTfQ5QyM= -github.com/ipfs/go-unixfsnode v1.1.2/go.mod h1:5dcE2x03pyjHk4JjamXmunTMzz+VUtqvPwZjIEkfV6s= github.com/ipfs/go-unixfsnode v1.4.0/go.mod h1:qc7YFFZ8tABc58p62HnIYbUMwj9chhUuFWmxSokfePo= github.com/ipfs/go-unixfsnode v1.5.2 h1:CvsiTt58W2uR5dD8bqQv+aAY0c1qolmXmSyNbPHYiew= github.com/ipfs/go-unixfsnode v1.5.2/go.mod h1:NlOebRwYx8lMCNMdhAhEspYPBD3obp7TE0LvBqHY+ks= @@ -1756,8 +1757,8 @@ github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= -github.com/schollz/progressbar/v3 v3.11.0 h1:3nIBUF1Zw/pGUaRHP7PZWmARP7ZQbWQ6vL6hwoQiIvU= -github.com/schollz/progressbar/v3 v3.11.0/go.mod h1:R2djRgv58sn00AGysc4fN0ip4piOGd3z88K+zVBjczs= +github.com/schollz/progressbar/v3 v3.13.1 h1:o8rySDYiQ59Mwzy2FELeHY5ZARXZTVJC7iHD6PEFUiE= +github.com/schollz/progressbar/v3 v3.13.1/go.mod h1:xvrbki8kfT1fzWzBT/UZd9L6GA+jdL7HAgq2RFnO6fQ= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= github.com/sercand/kuberesolver v2.4.0+incompatible h1:WE2OlRf6wjLxHwNkkFLQGaZcVLEXjMjBPjjEU5vksH8= @@ -1859,13 +1860,15 @@ github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpP github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= -github.com/tidwall/gjson v1.6.0 h1:9VEQWz6LLMUsUl6PueE49ir4Ka6CzLymOAZDxpFsTDc= github.com/tidwall/gjson v1.6.0/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls= -github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM= +github.com/tidwall/gjson v1.14.0 h1:6aeJ0bzojgWLa82gDQHcx3S0Lr/O51I9bJ5nv6JFx5w= +github.com/tidwall/gjson v1.14.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk= github.com/tklauser/go-sysconf v0.3.5 h1:uu3Xl4nkLzQfXNsWn15rPc/HQCJKObbt1dKJeWp3vU4= github.com/tklauser/go-sysconf v0.3.5/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI= @@ -1879,6 +1882,8 @@ github.com/twmb/murmur3 v1.1.6 h1:mqrRot1BRxm+Yct+vavLMou2/iJt0tNVTTC0QoIjaZg= github.com/twmb/murmur3 v1.1.6/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/uber/jaeger-client-go v2.30.0+incompatible h1:D6wyKGCecFaSRUpo8lCVbaOOb6ThwMmTEbhRwtKR97o= github.com/uber/jaeger-lib v2.4.1+incompatible h1:td4jdvLcExb4cBISKIpHuGoVXh+dVKhn2Um6rjCsSsg= +github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb h1:Ywfo8sUltxogBpFuMOFRrrSifO788kAFxmvVw31PtQQ= +github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb/go.mod h1:ikPs9bRWicNw3S7XpJ8sK/smGwU9WcSVU3dy9qahYBM= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go v1.2.6/go.mod h1:anCg0y61KIhDlPZmnH+so+RQbysYVyDko0IMgJv0Nn0= @@ -2016,8 +2021,8 @@ go.opentelemetry.io/otel/sdk v1.13.0 h1:BHib5g8MvdqS65yo2vV1s6Le42Hm6rrw08qU6yz5 go.opentelemetry.io/otel/sdk v1.13.0/go.mod h1:YLKPx5+6Vx/o1TCUYYs+bpymtkmazOMT6zoRrC7AQ7I= go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= go.opentelemetry.io/otel/trace v1.7.0/go.mod h1:fzLSB9nqR2eXzxPXb2JW9IKE+ScyXA48yyE4TNvoHqU= -go.opentelemetry.io/otel/trace v1.16.0 h1:8JRpaObFoW0pxuVPapkgH8UhHQj+bJW8jJsCZEu5MQs= -go.opentelemetry.io/otel/trace v1.16.0/go.mod h1:Yt9vYq1SdNz3xdjZZK7wcXv1qv2pwLkqr2QVwea0ef0= +go.opentelemetry.io/otel/trace v1.13.0 h1:CBgRZ6ntv+Amuj1jDsMhZtlAPT6gbyIRdaIzFhfBSdY= +go.opentelemetry.io/otel/trace v1.13.0/go.mod h1:muCvmmO9KKpvuXSf3KKAXXB2ygNYHQ+ZfI5X08d3tds= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= @@ -2360,22 +2365,19 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220829200755-d48e67d00261/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw= +golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= @@ -2525,8 +2527,8 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 h1:hrbNEivu7Zn1pxvHk6MBrq9iE22woVILTHqexqBxe6I= -google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6 h1:a2S6M0+660BgMNl++4JPlcAO/CjkqYItDEZwkoDQK7c= +google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= @@ -2549,10 +2551,8 @@ google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc v1.47.0 h1:9n77onPX5F3qfFCqjy9dhn8PbNQsIKeVU04J9G7umt8= -google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.52.3 h1:pf7sOysg4LdgBqduXveGKrcEwbStiK2rtfghdzlUYDQ= +google.golang.org/grpc v1.52.3/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5vorUY= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -2566,7 +2566,7 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= diff --git a/gql/serve_download.go b/gql/serve_download.go index af03446a0..eb6191936 100644 --- a/gql/serve_download.go +++ b/gql/serve_download.go @@ -3,8 +3,8 @@ package gql import ( "context" "fmt" - blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" + "github.com/ipfs/go-libipfs/blocks" "net/http" ) diff --git a/gql/server.go b/gql/server.go index b0d2360e4..1404beb53 100644 --- a/gql/server.go +++ b/gql/server.go @@ -23,22 +23,14 @@ import ( var log = logging.Logger("gql") type Server struct { - resolver *resolver - bstore BlockGetter - cfgHandler http.Handler - srv *http.Server - wg sync.WaitGroup + resolver *resolver + bstore BlockGetter + srv *http.Server + wg sync.WaitGroup } -func NewServer(cfg *config.Boost, resolver *resolver, bstore BlockGetter) *Server { - webCfg := &corsHandler{sub: &webConfigServer{ - cfg: webConfig{ - Ipni: webConfigIpni{ - IndexerHost: cfg.IndexProvider.WebHost, - }, - }, - }} - return &Server{resolver: resolver, bstore: bstore, cfgHandler: webCfg} +func NewServer(resolver *resolver, bstore BlockGetter) *Server { + return &Server{resolver: resolver, bstore: bstore} } //go:embed schema.graphql diff --git a/metrics/metrics.go b/metrics/metrics.go index 15228872e..553d04e1d 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -29,12 +29,12 @@ var ( APIRequestDuration = stats.Float64("api/request_duration_ms", "Duration of API requests", stats.UnitMilliseconds) // http - HttpPayloadByCidRequestCount = stats.Int64("http/payload_by_cid_request_count", "Counter of /payload/ requests", stats.UnitDimensionless) + HttpPayloadByCidRequestCount = stats.Int64("http/payload_by_cid_request_count", "Counter of /ipfs/ requests", stats.UnitDimensionless) HttpPayloadByCidRequestDuration = stats.Float64("http/payload_by_cid_request_duration_ms", "Time spent retrieving a payload by cid", stats.UnitMilliseconds) - HttpPayloadByCid200ResponseCount = stats.Int64("http/payload_by_cid_200_response_count", "Counter of /payload/ 200 responses", stats.UnitDimensionless) - HttpPayloadByCid400ResponseCount = stats.Int64("http/payload_by_cid_400_response_count", "Counter of /payload/ 400 responses", stats.UnitDimensionless) - HttpPayloadByCid404ResponseCount = stats.Int64("http/payload_by_cid_404_response_count", "Counter of /payload/ 404 responses", stats.UnitDimensionless) - HttpPayloadByCid500ResponseCount = stats.Int64("http/payload_by_cid_500_response_count", "Counter of /payload/ 500 responses", stats.UnitDimensionless) + HttpPayloadByCid200ResponseCount = stats.Int64("http/payload_by_cid_200_response_count", "Counter of /ipfs/ 200 responses", stats.UnitDimensionless) + HttpPayloadByCid400ResponseCount = stats.Int64("http/payload_by_cid_400_response_count", "Counter of /ipfs/ 400 responses", stats.UnitDimensionless) + HttpPayloadByCid404ResponseCount = stats.Int64("http/payload_by_cid_404_response_count", "Counter of /ipfs/ 404 responses", stats.UnitDimensionless) + HttpPayloadByCid500ResponseCount = stats.Int64("http/payload_by_cid_500_response_count", "Counter of /ipfs/ 500 responses", stats.UnitDimensionless) HttpPieceByCidRequestCount = stats.Int64("http/piece_by_cid_request_count", "Counter of /piece/ requests", stats.UnitDimensionless) HttpPieceByCidRequestDuration = stats.Float64("http/piece_by_cid_request_duration_ms", "Time spent retrieving a piece by cid", stats.UnitMilliseconds) HttpPieceByCid200ResponseCount = stats.Int64("http/piece_by_cid_200_response_count", "Counter of /piece/ 200 responses", stats.UnitDimensionless) @@ -48,6 +48,18 @@ var ( HttpBlockByCid404ResponseCount = stats.Int64("http/block_by_cid_404_response_count", "Counter of /block/ 404 responses", stats.UnitDimensionless) HttpBlockByCid500ResponseCount = stats.Int64("http/block_by_cid_500_response_count", "Counter of /block/ 500 responses", stats.UnitDimensionless) + // http remote blockstore + HttpRblsGetRequestCount = stats.Int64("http/rbls_get_request_count", "Counter of RemoteBlockstore Get requests", stats.UnitDimensionless) + HttpRblsGetSuccessResponseCount = stats.Int64("http/rbls_get_success_response_count", "Counter of successful RemoteBlockstore Get responses", stats.UnitDimensionless) + HttpRblsGetFailResponseCount = stats.Int64("http/rbls_get_fail_response_count", "Counter of failed RemoteBlockstore Get responses", stats.UnitDimensionless) + HttpRblsGetSizeRequestCount = stats.Int64("http/rbls_getsize_request_count", "Counter of RemoteBlockstore GetSize requests", stats.UnitDimensionless) + HttpRblsGetSizeSuccessResponseCount = stats.Int64("http/rbls_getsize_success_response_count", "Counter of successful RemoteBlockstore GetSize responses", stats.UnitDimensionless) + HttpRblsGetSizeFailResponseCount = stats.Int64("http/rbls_getsize_fail_response_count", "Counter of failed RemoteBlockstore GetSize responses", stats.UnitDimensionless) + HttpRblsHasRequestCount = stats.Int64("http/rbls_has_request_count", "Counter of RemoteBlockstore Has requests", stats.UnitDimensionless) + HttpRblsHasSuccessResponseCount = stats.Int64("http/rbls_has_success_response_count", "Counter of successful RemoteBlockstore Has responses", stats.UnitDimensionless) + HttpRblsHasFailResponseCount = stats.Int64("http/rbls_has_fail_response_count", "Counter of failed RemoteBlockstore Has responses", stats.UnitDimensionless) + HttpRblsBytesSentCount = stats.Int64("http/rbls_bytes_sent_count", "Counter of the number of bytes sent by bitswap since startup", stats.UnitBytes) + // bitswap BitswapRblsGetRequestCount = stats.Int64("bitswap/rbls_get_request_count", "Counter of RemoteBlockstore Get requests", stats.UnitDimensionless) BitswapRblsGetSuccessResponseCount = stats.Int64("bitswap/rbls_get_success_response_count", "Counter of successful RemoteBlockstore Get responses", stats.UnitDimensionless) diff --git a/node/builder.go b/node/builder.go index 34db6318a..7b8bb7369 100644 --- a/node/builder.go +++ b/node/builder.go @@ -44,10 +44,6 @@ import ( "github.com/filecoin-project/boostd-data/shared/tracing" "github.com/filecoin-project/dagstore" "github.com/filecoin-project/go-address" - //"github.com/filecoin-project/go-fil-markets/retrievalmarket" - //rmnet "github.com/filecoin-project/go-fil-markets/retrievalmarket/network" - //lotus_storagemarket "github.com/filecoin-project/go-fil-markets/storagemarket" - //"github.com/filecoin-project/go-fil-markets/storagemarket/impl/storedask" lotus_gfm_storagemarket "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/go-state-types/abi" lotus_api "github.com/filecoin-project/lotus/api" @@ -525,7 +521,7 @@ func ConfigBoost(cfg *config.Boost) Option { Override(new(*mpoolmonitor.MpoolMonitor), modules.NewMpoolMonitor(cfg)), // GraphQL server - Override(new(gql.BlockGetter), modules.NewBlockGetter), + Override(new(gql.BlockGetter), From(new(dtypes.IndexBackedBlockstore))), Override(new(*gql.Server), modules.NewGraphqlServer(cfg)), // Tracing diff --git a/node/impl/boost.go b/node/impl/boost.go index d9ad41870..5b2886f31 100644 --- a/node/impl/boost.go +++ b/node/impl/boost.go @@ -172,8 +172,8 @@ func (sm *BoostAPI) BoostIndexerListMultihashes(ctx context.Context, proposalCid } } -func (sm *BoostAPI) BoostOfflineDealWithData(ctx context.Context, dealUuid uuid.UUID, filePath string) (*api.ProviderDealRejectionInfo, error) { - res, err := sm.StorageProvider.ImportOfflineDealData(ctx, dealUuid, filePath) +func (sm *BoostAPI) BoostOfflineDealWithData(ctx context.Context, dealUuid uuid.UUID, filePath string, delAfterImport bool) (*api.ProviderDealRejectionInfo, error) { + res, err := sm.StorageProvider.ImportOfflineDealData(ctx, dealUuid, filePath, delAfterImport) return res, err } diff --git a/node/modules/graphsync.go b/node/modules/graphsync.go index 5f442fad2..e2c3dcf2b 100644 --- a/node/modules/graphsync.go +++ b/node/modules/graphsync.go @@ -3,12 +3,15 @@ package modules import ( "context" + "sync" + "time" + "github.com/filecoin-project/boost-gfm/retrievalmarket" retrievalimpl "github.com/filecoin-project/boost-gfm/retrievalmarket/impl" graphsync "github.com/filecoin-project/boost-graphsync/impl" gsnet "github.com/filecoin-project/boost-graphsync/network" "github.com/filecoin-project/boost-graphsync/storeutil" - "github.com/filecoin-project/boost/cmd/booster-bitswap/remoteblockstore" + "github.com/filecoin-project/boost/cmd/lib/remoteblockstore" "github.com/filecoin-project/boost/node/config" "github.com/filecoin-project/boost/node/modules/dtypes" "github.com/filecoin-project/boost/piecedirectory" @@ -20,7 +23,6 @@ import ( "github.com/libp2p/go-libp2p/core/host" "go.opencensus.io/stats" "go.uber.org/fx" - "time" ) var _ server.AskGetter = (*ProxyAskGetter)(nil) @@ -55,7 +57,8 @@ func SetAskGetter(proxy *ProxyAskGetter, rp retrievalmarket.RetrievalProvider) { // RetrievalGraphsync creates a graphsync instance used to serve retrievals. func RetrievalGraphsync(parallelTransfersForStorage uint64, parallelTransfersForStoragePerPeer uint64, parallelTransfersForRetrieval uint64) func(mctx lotus_helpers.MetricsCtx, lc fx.Lifecycle, pid *piecedirectory.PieceDirectory, h host.Host, net dtypes.ProviderTransferNetwork, dealDecider dtypes.RetrievalDealFilter, pstore dtypes.ProviderPieceStore, sa retrievalmarket.SectorAccessor, askGetter server.AskGetter) (*server.GraphsyncUnpaidRetrieval, error) { return func(mctx lotus_helpers.MetricsCtx, lc fx.Lifecycle, pid *piecedirectory.PieceDirectory, h host.Host, net dtypes.ProviderTransferNetwork, dealDecider dtypes.RetrievalDealFilter, pstore dtypes.ProviderPieceStore, sa retrievalmarket.SectorAccessor, askGetter server.AskGetter) (*server.GraphsyncUnpaidRetrieval, error) { - rb := remoteblockstore.NewRemoteBlockstore(pid) + // Graphsync tracks metrics separately, pass nothing to the remote blockstore + rb := remoteblockstore.NewRemoteBlockstore(pid, nil) // Create a Graphsync instance mkgs := Graphsync(parallelTransfersForStorage, parallelTransfersForStoragePerPeer, parallelTransfersForRetrieval) @@ -108,6 +111,7 @@ func Graphsync(parallelTransfersForStorage uint64, parallelTransfersForStoragePe } func graphsyncStats(mctx helpers.MetricsCtx, lc fx.Lifecycle, gs dtypes.Graphsync) { + var closeOnce sync.Once stopStats := make(chan struct{}) lc.Append(fx.Hook{ OnStart: func(context.Context) error { @@ -140,7 +144,7 @@ func graphsyncStats(mctx helpers.MetricsCtx, lc fx.Lifecycle, gs dtypes.Graphsyn return nil }, OnStop: func(ctx context.Context) error { - close(stopStats) + closeOnce.Do(func() { close(stopStats) }) return nil }, }) diff --git a/node/modules/legacy_markets.go b/node/modules/legacy_markets.go index 382e7ddca..09c3d99c7 100644 --- a/node/modules/legacy_markets.go +++ b/node/modules/legacy_markets.go @@ -2,6 +2,8 @@ package modules import ( "context" + "fmt" + "os" "path/filepath" piecefilestore "github.com/filecoin-project/boost-gfm/filestore" diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 8e9e209fe..aa6ea06ee 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -629,15 +629,15 @@ func NewStorageMarketProvider(provAddr address.Address, cfg *config.Boost) func( } } -func NewGraphqlServer(cfg *config.Boost) func(lc fx.Lifecycle, r repo.LockedRepo, h host.Host, prov *storagemarket.Provider, dealsDB *db.DealsDB, logsDB *db.LogsDB, retDB *rtvllog.RetrievalLogDB, plDB *db.ProposalLogsDB, fundsDB *db.FundsDB, fundMgr *fundmanager.FundManager, storageMgr *storagemanager.StorageManager, publisher *storageadapter.DealPublisher, spApi sealingpipeline.API, legacyProv gfm_storagemarket.StorageProvider, legacyDT dtypes.ProviderDataTransfer, ps dtypes.ProviderPieceStore, sa retrievalmarket.SectorAccessor, piecedirectory *piecedirectory.PieceDirectory, fullNode v1api.FullNode) *gql.Server { +func NewGraphqlServer(cfg *config.Boost) func(lc fx.Lifecycle, r repo.LockedRepo, h host.Host, prov *storagemarket.Provider, dealsDB *db.DealsDB, logsDB *db.LogsDB, retDB *rtvllog.RetrievalLogDB, plDB *db.ProposalLogsDB, fundsDB *db.FundsDB, fundMgr *fundmanager.FundManager, storageMgr *storagemanager.StorageManager, publisher *storageadapter.DealPublisher, spApi sealingpipeline.API, legacyProv gfm_storagemarket.StorageProvider, legacyDT dtypes.ProviderDataTransfer, ps dtypes.ProviderPieceStore, sa retrievalmarket.SectorAccessor, piecedirectory *piecedirectory.PieceDirectory, fullNode v1api.FullNode, bg gql.BlockGetter) *gql.Server { return func(lc fx.Lifecycle, r repo.LockedRepo, h host.Host, prov *storagemarket.Provider, dealsDB *db.DealsDB, logsDB *db.LogsDB, retDB *rtvllog.RetrievalLogDB, plDB *db.ProposalLogsDB, fundsDB *db.FundsDB, fundMgr *fundmanager.FundManager, storageMgr *storagemanager.StorageManager, publisher *storageadapter.DealPublisher, spApi sealingpipeline.API, legacyProv gfm_storagemarket.StorageProvider, legacyDT dtypes.ProviderDataTransfer, - ps dtypes.ProviderPieceStore, sa retrievalmarket.SectorAccessor, piecedirectory *piecedirectory.PieceDirectory, fullNode v1api.FullNode) *gql.Server { + ps dtypes.ProviderPieceStore, sa retrievalmarket.SectorAccessor, piecedirectory *piecedirectory.PieceDirectory, fullNode v1api.FullNode, bg gql.BlockGetter) *gql.Server { resolverCtx, cancel := context.WithCancel(context.Background()) resolver := gql.NewResolver(resolverCtx, cfg, r, h, dealsDB, logsDB, retDB, plDB, fundsDB, fundMgr, storageMgr, spApi, prov, legacyProv, legacyDT, ps, sa, piecedirectory, publisher, fullNode) - server := gql.NewServer(resolver) + server := gql.NewServer(resolver, bg) lc.Append(fx.Hook{ OnStart: server.Start, diff --git a/react/src/Inspect.css b/react/src/Inspect.css index 6bf974f48..ad98e519a 100644 --- a/react/src/Inspect.css +++ b/react/src/Inspect.css @@ -104,3 +104,11 @@ display: inline-block; margin: 1em 0; } + +.inspect a.download { + background-image: url("./bootstrap-icons/icons/download.svg"); + background-position: left; + background-repeat: no-repeat; + padding-left: 1.25em; + margin-left: 1em; +} diff --git a/react/src/LID.js b/react/src/LID.js index 037326668..24150f5e1 100644 --- a/react/src/LID.js +++ b/react/src/LID.js @@ -615,16 +615,26 @@ function PieceStatus({pieceCid, pieceStatus, searchQuery}) { {searchIsAnyCid ? ( - Searched CID + Searched CID (non-root) {searchQuery} -   Download block ) : null} + {rootCid ? ( + + Data Root CID + + { searchIsRootCid ? {rootCid} : rootCid } + + Download block + + + + ) : null} Piece CID {searchIsPieceCid ? ( diff --git a/storagemarket/provider_test.go b/storagemarket/provider_test.go index f0a870e80..2d39f9eb3 100644 --- a/storagemarket/provider_test.go +++ b/storagemarket/provider_test.go @@ -2429,17 +2429,3 @@ func copyFile(source string, dest string) error { return nil } - -func mockTipset(minerAddr address.Address, height abi.ChainEpoch) (*chaintypes.TipSet, error) { - dummyCid, _ := cid.Parse("bafkqaaa") - return chaintypes.NewTipSet([]*chaintypes.BlockHeader{{ - Miner: minerAddr, - Height: height, - ParentStateRoot: dummyCid, - Messages: dummyCid, - ParentMessageReceipts: dummyCid, - BlockSig: &crypto.Signature{Type: crypto.SigTypeBLS}, - BLSAggregate: &crypto.Signature{Type: crypto.SigTypeBLS}, - Timestamp: 1, - }}) -}