diff --git a/.circleci/config.yml b/.circleci/config.yml index 751ece814..123823f43 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -83,6 +83,19 @@ jobs: paths: - linux + lid-docker-compose: + description: 'Run LID integration tests' + machine: + image: ubuntu-2004:202104-01 + resource_class: xlarge + steps: + - checkout + - run: + name: local index directory docker compose tests + command: | + set -x + make test-lid + test: description: | Run go tests @@ -322,7 +335,5 @@ workflows: suite: all target: "`go list ./... | grep -v boost/itests`" - - test: - name: local index directory - suite: all - cwd: "./extern/boostd-data" + - lid-docker-compose + diff --git a/Makefile b/Makefile index 8c32e638c..81ed6a2bc 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,7 @@ unexport GOFLAGS GOCC?=go +ARCH?=$(shell arch) GOVERSION:=$(shell $(GOCC) version | tr ' ' '\n' | grep go1 | sed 's/^go//' | awk -F. '{printf "%d%03d%03d", $$1, $$2, $$3}') ifeq ($(shell expr $(GOVERSION) \< 1016000), 1) $(warning Your Golang version is go$(shell expr $(GOVERSION) / 1000000).$(shell expr $(GOVERSION) % 1000000 / 1000).$(shell expr $(GOVERSION) % 1000)) @@ -259,6 +260,9 @@ docker/all: $(lotus_build_cmd) docker/boost docker/booster-http docker/booster-b docker/lotus docker/lotus-miner .PHONY: docker/all +test-lid: + cd ./extern/boostd-data && ARCH=$(ARCH) docker-compose up --build --exit-code-from go-tests + devnet/up: rm -rf ./docker/devnet/data && docker compose -f ./docker/devnet/docker-compose.yaml up -d diff --git a/api/api.go b/api/api.go index 5877f52be..56e71b603 100644 --- a/api/api.go +++ b/api/api.go @@ -55,8 +55,7 @@ type Boost interface { BlockstoreGetSize(ctx context.Context, c cid.Cid) (int, error) //perm:read // MethodGroup: PieceDirectory - PdBuildIndexForPieceCid(ctx context.Context, piececid cid.Cid) error //perm:admin - PdMarkIndexErrored(ctx context.Context, piececid cid.Cid, err string) error //perm:admin + PdBuildIndexForPieceCid(ctx context.Context, piececid cid.Cid) error //perm:admin // RuntimeSubsystems returns the subsystems that are enabled // in this instance. diff --git a/api/proxy_gen.go b/api/proxy_gen.go index 1a82a57ec..83f0ce039 100644 --- a/api/proxy_gen.go +++ b/api/proxy_gen.go @@ -129,8 +129,6 @@ type BoostStruct struct { PdBuildIndexForPieceCid func(p0 context.Context, p1 cid.Cid) error `perm:"admin"` - PdMarkIndexErrored func(p0 context.Context, p1 cid.Cid, p2 string) error `perm:"admin"` - RuntimeSubsystems func(p0 context.Context) (lapi.MinerSubsystems, error) `perm:"read"` SectorsRefs func(p0 context.Context) (map[string][]lapi.SealedRef, error) `perm:"read"` @@ -777,17 +775,6 @@ func (s *BoostStub) PdBuildIndexForPieceCid(p0 context.Context, p1 cid.Cid) erro return ErrNotSupported } -func (s *BoostStruct) PdMarkIndexErrored(p0 context.Context, p1 cid.Cid, p2 string) error { - if s.Internal.PdMarkIndexErrored == nil { - return ErrNotSupported - } - return s.Internal.PdMarkIndexErrored(p0, p1, p2) -} - -func (s *BoostStub) PdMarkIndexErrored(p0 context.Context, p1 cid.Cid, p2 string) error { - return ErrNotSupported -} - func (s *BoostStruct) RuntimeSubsystems(p0 context.Context) (lapi.MinerSubsystems, error) { if s.Internal.RuntimeSubsystems == nil { return *new(lapi.MinerSubsystems), ErrNotSupported diff --git a/build/openrpc/boost.json.gz b/build/openrpc/boost.json.gz index 2c5e19aee..966bf3abb 100644 Binary files a/build/openrpc/boost.json.gz and b/build/openrpc/boost.json.gz differ diff --git a/cmd/boostd/piecedir.go b/cmd/boostd/piecedir.go index da8f04628..54436b0aa 100644 --- a/cmd/boostd/piecedir.go +++ b/cmd/boostd/piecedir.go @@ -16,7 +16,6 @@ var pieceDirCmd = &cli.Command{ Usage: "Manage Local Index Directory", Subcommands: []*cli.Command{ pdIndexGenerate, - pdIndexMarkErroredCmd, }, } @@ -56,45 +55,3 @@ var pdIndexGenerate = &cli.Command{ return nil }, } - -var pdIndexMarkErroredCmd = &cli.Command{ - Name: "mark-index", - Usage: "Mark an index errored for a given piece in the local index directory", - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: "piece-cid", - Usage: "piece-cid of the index that will be marked as errored", - Required: true, - }, - &cli.StringFlag{ - Name: "error", - Usage: "error message", - Required: true, - }, - }, - Action: func(cctx *cli.Context) error { - ctx := lcli.ReqContext(cctx) - - // parse piececid - piececid, err := cid.Decode(cctx.String("piece-cid")) - if err != nil { - return err - } - - boostApi, ncloser, err := bcli.GetBoostAPI(cctx) - if err != nil { - return fmt.Errorf("getting boost api: %w", err) - } - defer ncloser() - - errMsg := cctx.String("error") - err = boostApi.PdMarkIndexErrored(ctx, piececid, errMsg) - if err != nil { - return err - } - - fmt.Printf("Marked %s as errored with \"%s\"\n", piececid, errMsg) - - return nil - }, -} diff --git a/documentation/en/api-v1-methods.md b/documentation/en/api-v1-methods.md index 26fdba8d5..8c9d0c610 100644 --- a/documentation/en/api-v1-methods.md +++ b/documentation/en/api-v1-methods.md @@ -85,7 +85,6 @@ * [OnlineBackup](#onlinebackup) * [Pd](#pd) * [PdBuildIndexForPieceCid](#pdbuildindexforpiececid) - * [PdMarkIndexErrored](#pdmarkindexerrored) * [Runtime](#runtime) * [RuntimeSubsystems](#runtimesubsystems) * [Sectors](#sectors) @@ -1765,23 +1764,6 @@ Inputs: Response: `{}` -### PdMarkIndexErrored - - -Perms: admin - -Inputs: -```json -[ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "string value" -] -``` - -Response: `{}` - ## Runtime diff --git a/extern/boostd-data/Dockerfile.test b/extern/boostd-data/Dockerfile.test new file mode 100644 index 000000000..24972f7b8 --- /dev/null +++ b/extern/boostd-data/Dockerfile.test @@ -0,0 +1,8 @@ +FROM golang:1.18-alpine + +WORKDIR /go/src/ + +ENV CGO_ENABLED=0 + +ENTRYPOINT ["go", "test"] +CMD ["-v", "./..."] diff --git a/extern/boostd-data/client/client.go b/extern/boostd-data/client/client.go index e41a7952d..98fcb8609 100644 --- a/extern/boostd-data/client/client.go +++ b/extern/boostd-data/client/client.go @@ -26,8 +26,6 @@ type Store struct { ListPieces func(ctx context.Context) ([]cid.Cid, error) GetPieceMetadata func(ctx context.Context, pieceCid cid.Cid) (model.Metadata, error) GetPieceDeals func(context.Context, cid.Cid) ([]model.DealInfo, error) - SetCarSize func(ctx context.Context, pieceCid cid.Cid, size uint64) error - MarkIndexErrored func(context.Context, cid.Cid, string) error IndexedAt func(context.Context, cid.Cid) (time.Time, error) PiecesContainingMultihash func(context.Context, mh.Multihash) ([]cid.Cid, error) RemoveDealForPiece func(context.Context, cid.Cid, string) error @@ -107,18 +105,10 @@ func (s *Store) PiecesContainingMultihash(ctx context.Context, m mh.Multihash) ( return s.client.PiecesContainingMultihash(ctx, m) } -func (s *Store) MarkIndexErrored(ctx context.Context, pieceCid cid.Cid, err string) error { - return s.client.MarkIndexErrored(ctx, pieceCid, err) -} - func (s *Store) AddDealForPiece(ctx context.Context, pieceCid cid.Cid, dealInfo model.DealInfo) error { return s.client.AddDealForPiece(ctx, pieceCid, dealInfo) } -func (s *Store) SetCarSize(ctx context.Context, pieceCid cid.Cid, size uint64) error { - return s.client.SetCarSize(ctx, pieceCid, size) -} - func (s *Store) AddIndex(ctx context.Context, pieceCid cid.Cid, records []model.Record, isCompleteIndex bool) error { log.Debugw("add-index", "piece-cid", pieceCid, "records", len(records)) diff --git a/extern/boostd-data/cmd/run.go b/extern/boostd-data/cmd/run.go index 109d804d3..a41bb35cd 100644 --- a/extern/boostd-data/cmd/run.go +++ b/extern/boostd-data/cmd/run.go @@ -9,6 +9,7 @@ import ( "github.com/filecoin-project/boostd-data/shared/cliutil" "github.com/filecoin-project/boostd-data/shared/tracing" "github.com/filecoin-project/boostd-data/svc" + "github.com/filecoin-project/boostd-data/yugabyte" "github.com/mitchellh/go-homedir" "github.com/urfave/cli/v2" ) @@ -18,6 +19,7 @@ var runCmd = &cli.Command{ Subcommands: []*cli.Command{ leveldbCmd, couchbaseCmd, + yugabyteCmd, }, } @@ -106,6 +108,35 @@ var couchbaseCmd = &cli.Command{ }, } +var yugabyteCmd = &cli.Command{ + Name: "yugabyte", + Usage: "Run boostd-data with a yugabyte database", + Before: before, + Flags: append([]cli.Flag{ + &cli.StringSliceFlag{ + Name: "hosts", + Usage: "yugabyte hosts to connect to over cassandra interface eg '127.0.0.1'", + Required: true, + }, + &cli.StringFlag{ + Name: "connect-string", + Usage: "postgres connect string eg 'postgresql://postgres:postgres@localhost'", + Required: true, + }}, + runFlags..., + ), + Action: func(cctx *cli.Context) error { + // Create a yugabyte data service + settings := yugabyte.DBSettings{ + Hosts: cctx.StringSlice("hosts"), + ConnectString: cctx.String("connect-string"), + } + + bdsvc := svc.NewYugabyte(settings) + return runAction(cctx, "yugabyte", bdsvc) + }, +} + func runAction(cctx *cli.Context, dbType string, store *svc.Service) error { ctx := cliutil.ReqContext(cctx) diff --git a/extern/boostd-data/couchbase/db.go b/extern/boostd-data/couchbase/db.go index c624b5df7..bc80f118a 100644 --- a/extern/boostd-data/couchbase/db.go +++ b/extern/boostd-data/couchbase/db.go @@ -337,42 +337,6 @@ func (db *DB) setPieceCidsForMultihash(ctx context.Context, mh multihash.Multiha return nil } -func (db *DB) SetCarSize(ctx context.Context, pieceCid cid.Cid, size uint64) error { - ctx, span := tracing.Tracer.Start(ctx, "db.set_car_size") - defer span.End() - - return db.mutatePieceMetadata(ctx, pieceCid, "set-car-size", func(metadata CouchbaseMetadata) *CouchbaseMetadata { - // Set the car size on each deal (should be the same for all deals) - var deals []model.DealInfo - for _, dl := range metadata.Deals { - dl.CarLength = size - - deals = append(deals, dl) - } - metadata.Deals = deals - return &metadata - }) -} - -func (db *DB) MarkIndexErrored(ctx context.Context, pieceCid cid.Cid, idxErr error) error { - ctx, span := tracing.Tracer.Start(ctx, "db.mark_piece_index_errored") - defer span.End() - - return db.mutatePieceMetadata(ctx, pieceCid, "mark-index-errored", func(metadata CouchbaseMetadata) *CouchbaseMetadata { - // If the error was already set, don't overwrite it - if metadata.Error != "" { - // If the error state has already been set, don't over-write the existing error - return nil - } - - // Set the error state - metadata.Error = idxErr.Error() - metadata.ErrorType = fmt.Sprintf("%T", idxErr) - - return &metadata - }) -} - func (db *DB) MarkIndexingComplete(ctx context.Context, pieceCid cid.Cid, blockCount int, isCompleteIndex bool) error { ctx, span := tracing.Tracer.Start(ctx, "db.mark_indexing_complete") defer span.End() @@ -382,8 +346,6 @@ func (db *DB) MarkIndexingComplete(ctx context.Context, pieceCid cid.Cid, blockC metadata.IndexedAt = time.Now() metadata.CompleteIndex = isCompleteIndex metadata.BlockCount = blockCount - metadata.Error = "" - metadata.ErrorType = "" if metadata.Deals == nil { metadata.Deals = []model.DealInfo{} } diff --git a/extern/boostd-data/couchbase/service.go b/extern/boostd-data/couchbase/service.go index 35259b04a..38c9d637b 100644 --- a/extern/boostd-data/couchbase/service.go +++ b/extern/boostd-data/couchbase/service.go @@ -2,7 +2,6 @@ package couchbase import ( "context" - "errors" "fmt" "time" @@ -56,38 +55,6 @@ func (s *Store) AddDealForPiece(ctx context.Context, pieceCid cid.Cid, dealInfo return s.db.AddDealForPiece(ctx, pieceCid, dealInfo) } -func (s *Store) SetCarSize(ctx context.Context, pieceCid cid.Cid, size uint64) error { - log.Debugw("handle.set-car-size", "piece-cid", pieceCid, "size", size) - - ctx, span := tracing.Tracer.Start(ctx, "store.set-car-size") - defer span.End() - - defer func(now time.Time) { - log.Debugw("handled.set-car-size", "took", time.Since(now).String()) - }(time.Now()) - - err := s.db.SetCarSize(ctx, pieceCid, size) - return normalizePieceCidError(pieceCid, err) -} - -func (s *Store) MarkIndexErrored(ctx context.Context, pieceCid cid.Cid, idxErr string) error { - log.Debugw("handle.mark-piece-index-errored", "piece-cid", pieceCid, "err", idxErr) - - ctx, span := tracing.Tracer.Start(ctx, "store.mark-piece-index-errored") - defer span.End() - - defer func(now time.Time) { - log.Debugw("handled.mark-piece-index-errored", "took", time.Since(now).String()) - }(time.Now()) - - err := s.db.MarkIndexErrored(ctx, pieceCid, errors.New(idxErr)) - if err != nil { - return normalizePieceCidError(pieceCid, err) - } - - return s.FlagPiece(ctx, pieceCid) -} - func (s *Store) GetOffsetSize(ctx context.Context, pieceCid cid.Cid, hash mh.Multihash) (*model.OffsetSize, error) { log.Debugw("handle.get-offset-size", "piece-cid", pieceCid) diff --git a/extern/boostd-data/docker-compose.yml b/extern/boostd-data/docker-compose.yml new file mode 100644 index 000000000..5ac7b7368 --- /dev/null +++ b/extern/boostd-data/docker-compose.yml @@ -0,0 +1,17 @@ +version: '3.7' + +services: + + yugabyte: + image: public.ecr.aws/n6b0k8i7/yugabyte-test:${ARCH}-2.17.2.0 + restart: on-failure + + go-tests: + build: + context: . + dockerfile: ./Dockerfile.test + environment: + YUGABYTE_HOST: yugabyte + volumes: + - ./:/go/src/ + command: -tags=test_lid -v -count=1 -p=1 ./... diff --git a/extern/boostd-data/go.mod b/extern/boostd-data/go.mod index 9d1c890d0..32dff3ec5 100644 --- a/extern/boostd-data/go.mod +++ b/extern/boostd-data/go.mod @@ -18,12 +18,15 @@ require ( github.com/ipfs/go-ds-leveldb v0.5.0 github.com/ipfs/go-log/v2 v2.5.1 github.com/ipld/go-car/v2 v2.4.2-0.20220707083113-89de8134e58e + github.com/jackc/pgtype v1.10.0 github.com/mitchellh/go-homedir v1.1.0 github.com/multiformats/go-multicodec v0.6.0 github.com/multiformats/go-multihash v0.2.1 github.com/stretchr/testify v1.8.1 github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 github.com/urfave/cli/v2 v2.24.4 + github.com/yugabyte/gocql v0.0.0-20221110041640-6fc475c5aeb0 + github.com/yugabyte/pgx/v4 v4.14.5 go.opentelemetry.io/otel v1.13.0 go.opentelemetry.io/otel/exporters/jaeger v1.13.0 go.opentelemetry.io/otel/sdk v1.13.0 @@ -51,6 +54,7 @@ require ( github.com/golang/protobuf v1.5.2 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/gorilla/websocket v1.5.0 // indirect + github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect github.com/ipfs/bbloom v0.0.4 // indirect github.com/ipfs/go-block-format v0.0.3 // indirect @@ -75,6 +79,13 @@ require ( github.com/ipfs/go-verifcid v0.0.1 // indirect github.com/ipld/go-codec-dagpb v1.3.2 // indirect github.com/ipld/go-ipld-prime v0.18.0 // indirect + github.com/jackc/chunkreader/v2 v2.0.1 // indirect + github.com/jackc/pgconn v1.11.0 // indirect + github.com/jackc/pgio v1.0.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgproto3/v2 v2.2.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect + github.com/jackc/puddle v1.2.1 // indirect github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c // indirect github.com/jbenet/goprocess v0.1.4 // indirect github.com/klauspost/cpuid/v2 v2.1.1 // indirect @@ -117,10 +128,12 @@ require ( golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e // indirect golang.org/x/exp v0.0.0-20220916125017-b168a2c6b86b // indirect golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8 // indirect + golang.org/x/text v0.3.7 // indirect golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f // indirect google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect google.golang.org/grpc v1.47.0 // indirect google.golang.org/protobuf v1.28.1 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect gotest.tools/v3 v3.4.0 // indirect lukechampine.com/blake3 v1.1.7 // indirect diff --git a/extern/boostd-data/go.sum b/extern/boostd-data/go.sum index cf22ef94e..cc22c597f 100644 --- a/extern/boostd-data/go.sum +++ b/extern/boostd-data/go.sum @@ -67,6 +67,7 @@ github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETF github.com/Kubuxu/imtui v0.0.0-20210401140320-41663d68d0fa/go.mod h1:WUmMvh9wMtqj1Xhf1hf3kp9RvL+y6odtdYxpyZjb90U= github.com/Masterminds/glide v0.13.2/go.mod h1:STyF5vcenH/rUqTEv+/hBXlSTo7KYwg2oc2f4tzPWic= github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= +github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA= github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA= @@ -128,9 +129,13 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bep/debounce v1.2.0/go.mod h1:H8yggRPQKLUhUoqrJC1bO2xNya7vanpDl7xR3ISbCJ0= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 h1:mXoPYz/Ul5HYEDvkta6I8/rnYM5gSdSV2tJ6XbZuEtY= +github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k= github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= github.com/briandowns/spinner v1.11.1/go.mod h1:QOuQk7x+EaDASo80FEXwlwiA+j/PPIcX3FScO+3/ZPQ= github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8= @@ -195,6 +200,8 @@ github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= +github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA= @@ -214,6 +221,7 @@ github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= @@ -481,6 +489,8 @@ github.com/godbus/dbus v0.0.0-20190402143921-271e53dc4968/go.mod h1:/YcGZj5zSblf github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= +github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/googleapis v1.4.0/go.mod h1:5YRNX2z1oM5gXdAkurHa942MDgEJyk02w4OecKY87+c= @@ -617,6 +627,8 @@ github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645/go github.com/gxed/go-shellwords v1.0.3/go.mod h1:N7paucT91ByIjmVJHhvoarjoQnmsi3Jd3vH7VqgtMxQ= github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= +github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8= +github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= github.com/hako/durafmt v0.0.0-20200710122514-c0fb7b4da026/go.mod h1:5Scbynm8dF1XAPwIwkGPqzkM/shndPm79Jd1003hTjE= github.com/hannahhoward/cbor-gen-for v0.0.0-20200817222906-ea96cece81f1/go.mod h1:jvfsLIxk0fY/2BKSQ1xf2406AKA5dwMmKKv0ADcOfN8= github.com/hannahhoward/go-pubsub v0.0.0-20200423002714-8d62886cc36e/go.mod h1:I8h3MITA53gN9OnWGCgaMa0JWVRdXthWw4M3CPM54OY= @@ -939,6 +951,53 @@ github.com/ipld/go-ipld-selector-text-lite v0.0.1/go.mod h1:U2CQmFb+uWzfIEF3I1ar github.com/ipld/go-storethehash v0.1.7/go.mod h1:O2CgbSwJfXCrYsjA1g1M7zJmVzzg71BM00ds6pyMLAQ= github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 h1:QG4CGBqCeuBo6aZlGAamSkxWdgWfZGeE49eUOWJPA4c= github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52/go.mod h1:fdg+/X9Gg4AsAIzWpEHwnqd+QY3b7lajxyjE1m4hkq4= +github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= +github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= +github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA= +github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE= +github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s= +github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= +github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY= +github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= +github.com/jackc/pgconn v1.11.0 h1:HiHArx4yFbwl91X3qqIHtUFoiIfLNJXCQRsnzkiwwaQ= +github.com/jackc/pgconn v1.11.0/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= +github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= +github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= +github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= +github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c= +github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65 h1:DadwsjnMwFjfWc9y5Wi/+Zz7xoE5ALHsRQlOctkOiHc= +github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= +github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.2.0 h1:r7JypeP2D3onoQTCxWdTpCtJ4D+qpKr0TxvoyMhZ5ns= +github.com/jackc/pgproto3/v2 v2.2.0/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= +github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= +github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= +github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= +github.com/jackc/pgtype v1.10.0 h1:ILnBWrRMSXGczYvmkYD6PsYyVFUNLTnIUJHHDLmqk38= +github.com/jackc/pgtype v1.10.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= +github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= +github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= +github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= +github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c h1:Dznn52SgVIVst9UyOT9brctYUgxs+CvVfPaC3jKrA50= +github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= +github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.2.1 h1:gI8os0wpRXFd4FiAY2dWiqRK037tjj3t7rKFeO4X5iw= +github.com/jackc/puddle v1.2.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= @@ -1010,6 +1069,7 @@ github.com/klauspost/cpuid/v2 v2.1.1 h1:t0wUqjowdm8ezddV5k0tLWVklVuvLJpoHeb4WBdy github.com/klauspost/cpuid/v2 v2.1.1/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= github.com/koalacxr/quantile v0.0.1/go.mod h1:bGN/mCZLZ4lrSDHRQ6Lglj9chowGux8sGUIND+DQeD0= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/koron/go-ssdp v0.0.0-20180514024734-4a0ed625a78b/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= github.com/koron/go-ssdp v0.0.0-20191105050749-2e1c40ed0b5d/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= @@ -1023,11 +1083,16 @@ github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lib/pq v0.0.0-20180327071824-d34b9ff171c2/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.10.4 h1:SO9z7FRPzA03QhHKJrH5BXA6HU1rS4V2nIVrrNC1iYk= github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/libp2p/go-addr-util v0.0.1/go.mod h1:4ac6O7n9rIAKB1dnd+s8IbbMXkt+oBpzX4/+RACcnlQ= github.com/libp2p/go-addr-util v0.0.2/go.mod h1:Ecd6Fb3yIuLzq4bD7VcywcVSBtefcAwnUISBM3WG15E= @@ -1469,6 +1534,7 @@ github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= @@ -1805,6 +1871,8 @@ github.com/rogpeppe/go-internal v1.8.1 h1:geMPLpDpQOgVyCg5z5GoRwLHepNdb71NXb67XF github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= +github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= github.com/rs/zerolog v1.21.0/go.mod h1:ZPhntP/xmq1nnND05hhpAh2QMhSsA4UN3MGZ6O2J3hM= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -1813,12 +1881,16 @@ github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= github.com/sercand/kuberesolver v2.1.0+incompatible/go.mod h1:lWF3GL0xptCB/vCiJPl/ZshwPsX/n4Y7u0CW9E7aQIQ= github.com/sercand/kuberesolver v2.4.0+incompatible/go.mod h1:lWF3GL0xptCB/vCiJPl/ZshwPsX/n4Y7u0CW9E7aQIQ= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shirou/gopsutil v2.18.12+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= +github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= +github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY= github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM= github.com/shurcooL/github_flavored_markdown v0.0.0-20181002035957-2122de532470/go.mod h1:2dOwnU2uBioM+SGy2aZoq1f/Sd1l9OkAeAUvjSyvgU0= @@ -1844,6 +1916,7 @@ github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYED github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw= github.com/siebenmann/go-kstat v0.0.0-20160321171754-d34789b79745/go.mod h1:G81aIFAMS9ECrwBYR9YxhlPjWgrItd+Kje78O6+uqm8= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= @@ -1888,6 +1961,7 @@ github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5J github.com/streadway/handy v0.0.0-20200128134331-0f66f006fb2e/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= @@ -1998,12 +2072,17 @@ github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1: github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= github.com/ybbus/jsonrpc/v2 v2.1.6/go.mod h1:rIuG1+ORoiqocf9xs/v+ecaAVeo3zcZHQgInyKFMeg0= +github.com/yugabyte/gocql v0.0.0-20221110041640-6fc475c5aeb0 h1:68nrJsrWe0A6JiKnsjWChAaWhj20v+AwYJObtp86D1k= +github.com/yugabyte/gocql v0.0.0-20221110041640-6fc475c5aeb0/go.mod h1:LAokR6+vevDCrTxk52U7p6ki+4qELu4XU7JUGYa2O2M= +github.com/yugabyte/pgx/v4 v4.14.5 h1:XLzEEiO3d/kWzpyctO8l4kwMLhzPQ9n2er7ATH7CJVA= +github.com/yugabyte/pgx/v4 v4.14.5/go.mod h1:nFSvjsVq4CuA61TWGriwWG74ZVxeuJCcNu42Mkn+rgw= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= github.com/zondax/hid v0.9.0/go.mod h1:l5wttcP0jwtdLjqjMMWFVEE7d1zO0jvSPA9OPZxWpEM= github.com/zondax/ledger-go v0.12.1/go.mod h1:KatxXrVDzgWwbssUWsF5+cOJHXPvzQ09YSlzGNuhOEo= @@ -2093,6 +2172,7 @@ go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95a go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8= go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.14.1/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= @@ -2119,6 +2199,7 @@ golang.org/x/crypto v0.0.0-20190225124518-7f87c0fbb88b/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190228161510-8dd112bcdc25/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -2127,6 +2208,7 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20190927123631-a832865fa7ad/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -2143,6 +2225,7 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= @@ -2325,6 +2408,7 @@ golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190302025703-b6889370fb10/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190316082340-a2f829d7f35f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190411185658-b44545bcd369/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -2341,6 +2425,7 @@ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190902133755-9109b7679e13/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -2436,6 +2521,7 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= @@ -2462,6 +2548,7 @@ golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= @@ -2469,6 +2556,7 @@ golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190927191325-030b2cf1153e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -2520,6 +2608,8 @@ golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/tools v0.1.11/go.mod h1:SgwaegtQh8clINPpECJMqnxLv9I09HLqnW3RMqW0CA4= golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU= +golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -2659,6 +2749,8 @@ gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qS gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= +gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= diff --git a/extern/boostd-data/ldb/db.go b/extern/boostd-data/ldb/db.go index 5489f5bce..2ca08ad41 100644 --- a/extern/boostd-data/ldb/db.go +++ b/extern/boostd-data/ldb/db.go @@ -290,43 +290,6 @@ func (db *DB) GetPieceCidToMetadata(ctx context.Context, pieceCid cid.Cid) (Leve return metadata, nil } -func (db *DB) SetCarSize(ctx context.Context, pieceCid cid.Cid, size uint64) error { - ctx, span := tracing.Tracer.Start(ctx, "db.set_car_size") - defer span.End() - - md, err := db.GetPieceCidToMetadata(ctx, pieceCid) - if err != nil { - return fmt.Errorf("getting piece metadata for piece %s: %w", pieceCid, err) - } - - // Set the car size on each deal (should be the same for all deals) - for _, dl := range md.Deals { - dl.CarLength = size - } - - return db.SetPieceCidToMetadata(ctx, pieceCid, md) -} - -func (db *DB) MarkIndexErrored(ctx context.Context, pieceCid cid.Cid, sourceErr error) error { - ctx, span := tracing.Tracer.Start(ctx, "db.mark_piece_index_errored") - defer span.End() - - md, err := db.GetPieceCidToMetadata(ctx, pieceCid) - if err != nil { - return fmt.Errorf("getting piece metadata for piece %s: %w", pieceCid, err) - } - - if md.Error != "" { - // If the error state has already been set, don't over-write the existing error - return nil - } - - md.Error = sourceErr.Error() - md.ErrorType = fmt.Sprintf("%T", sourceErr) - - return db.SetPieceCidToMetadata(ctx, pieceCid, md) -} - // AllRecords func (db *DB) AllRecords(ctx context.Context, cursor uint64) ([]model.Record, error) { ctx, span := tracing.Tracer.Start(ctx, "db.all_records") diff --git a/extern/boostd-data/ldb/service.go b/extern/boostd-data/ldb/service.go index e3915857d..f41abe3a4 100644 --- a/extern/boostd-data/ldb/service.go +++ b/extern/boostd-data/ldb/service.go @@ -121,44 +121,6 @@ func (s *Store) AddDealForPiece(ctx context.Context, pieceCid cid.Cid, dealInfo return nil } -func (s *Store) SetCarSize(ctx context.Context, pieceCid cid.Cid, size uint64) error { - log.Debugw("handle.set-car-size", "piece-cid", pieceCid, "size", size) - - ctx, span := tracing.Tracer.Start(ctx, "store.set-car-size") - defer span.End() - - defer func(now time.Time) { - log.Debugw("handled.set-car-size", "took", time.Since(now).String()) - }(time.Now()) - - s.Lock() - defer s.Unlock() - - err := s.db.SetCarSize(ctx, pieceCid, size) - return normalizePieceCidError(pieceCid, err) -} - -func (s *Store) MarkIndexErrored(ctx context.Context, pieceCid cid.Cid, idxErr string) error { - log.Debugw("handle.mark-piece-index-errored", "piece-cid", pieceCid, "err", idxErr) - - ctx, span := tracing.Tracer.Start(ctx, "store.mark-piece-index-errored") - defer span.End() - - defer func(now time.Time) { - log.Debugw("handled.mark-piece-index-errored", "took", time.Since(now).String()) - }(time.Now()) - - s.Lock() - defer s.Unlock() - - err := s.db.MarkIndexErrored(ctx, pieceCid, errors.New(idxErr)) - if err != nil { - return normalizePieceCidError(pieceCid, err) - } - - return s.FlagPiece(ctx, pieceCid) -} - func (s *Store) GetOffsetSize(ctx context.Context, pieceCid cid.Cid, hash mh.Multihash) (*model.OffsetSize, error) { log.Debugw("handle.get-offset-size", "piece-cid", pieceCid) @@ -354,8 +316,6 @@ func (s *Store) AddIndex(ctx context.Context, pieceCid cid.Cid, records []model. md.Cursor = cursor md.IndexedAt = time.Now() md.CompleteIndex = isCompleteIndex - md.Error = "" - md.ErrorType = "" err = s.db.SetPieceCidToMetadata(ctx, pieceCid, md) if err != nil { diff --git a/extern/boostd-data/model/model.go b/extern/boostd-data/model/model.go index a994ef7c2..7ab0004a2 100644 --- a/extern/boostd-data/model/model.go +++ b/extern/boostd-data/model/model.go @@ -12,8 +12,8 @@ import ( ) // DealInfo is information about a single deal for a given piece -// PieceOffset -// v +// . PieceOffset +// . v // Sector [..........................] // Piece ......[ ]...... // CAR ......[ ]............ @@ -39,11 +39,10 @@ type Metadata struct { IndexedAt time.Time `json:"i"` // CompleteIndex indicates whether the index has all information or is // missing block size information. Note that indexes imported from the - // dagstore do not have block size information. + // dagstore do not have block size information (they only have block + // offsets). CompleteIndex bool `json:"c"` Deals []DealInfo `json:"d"` - Error string `json:"e"` - ErrorType string `json:"t"` } // Record is the information stored in the index for each block in a piece diff --git a/extern/boostd-data/svc/setup_yugabyte_test_util.go b/extern/boostd-data/svc/setup_yugabyte_test_util.go new file mode 100644 index 000000000..ff7f5e61f --- /dev/null +++ b/extern/boostd-data/svc/setup_yugabyte_test_util.go @@ -0,0 +1,58 @@ +package svc + +import ( + "testing" + "time" + + "github.com/filecoin-project/boostd-data/yugabyte" + "github.com/stretchr/testify/require" + "github.com/yugabyte/gocql" + "github.com/yugabyte/pgx/v4/pgxpool" + "golang.org/x/net/context" +) + +var TestYugabyteSettings = yugabyte.DBSettings{ + Hosts: []string{"yugabyte"}, + ConnectString: "postgresql://postgres:postgres@yugabyte:5433", +} + +func SetupYugabyte(t *testing.T) { + ctx := context.Background() + + tlog.Info("wait for yugabyte start...") + awaitYugabyteUp(t, time.Minute) + tlog.Info("yugabyte started") + + store := yugabyte.NewStore(TestYugabyteSettings) + err := store.Start(ctx) + require.NoError(t, err) + + RecreateTables(ctx, t, store) +} + +func RecreateTables(ctx context.Context, t *testing.T, store *yugabyte.Store) { + err := store.Drop(ctx) + require.NoError(t, err) + err = store.Create(ctx) + require.NoError(t, err) +} + +func awaitYugabyteUp(t *testing.T, duration time.Duration) { + start := time.Now() + cluster := gocql.NewCluster(TestYugabyteSettings.Hosts[0]) + for { + _, err := cluster.CreateSession() + if err == nil { + _, err = pgxpool.Connect(context.Background(), TestYugabyteSettings.ConnectString) + if err == nil { + return + } + } + + tlog.Debugf("waiting for yugabyte: %s", err) + if time.Since(start) > duration { + t.Fatalf("failed to start yugabyte within %s", duration) + } + time.Sleep(time.Second) + } +} diff --git a/extern/boostd-data/svc/svc.go b/extern/boostd-data/svc/svc.go index 56ba36871..bf20b009e 100644 --- a/extern/boostd-data/svc/svc.go +++ b/extern/boostd-data/svc/svc.go @@ -12,6 +12,7 @@ import ( "github.com/filecoin-project/boostd-data/couchbase" "github.com/filecoin-project/boostd-data/ldb" "github.com/filecoin-project/boostd-data/svc/types" + "github.com/filecoin-project/boostd-data/yugabyte" "github.com/filecoin-project/go-jsonrpc" "github.com/gorilla/mux" logging "github.com/ipfs/go-log/v2" @@ -22,11 +23,15 @@ var ( ) type Service struct { - impl types.ServiceImpl + Impl types.ServiceImpl +} + +func NewYugabyte(settings yugabyte.DBSettings) *Service { + return &Service{Impl: yugabyte.NewStore(settings)} } func NewCouchbase(settings couchbase.DBSettings) *Service { - return &Service{impl: couchbase.NewStore(settings)} + return &Service{Impl: couchbase.NewStore(settings)} } func NewLevelDB(repoPath string) (*Service, error) { @@ -38,7 +43,7 @@ func NewLevelDB(repoPath string) (*Service, error) { } } - return &Service{impl: ldb.NewStore(repoPath)}, nil + return &Service{Impl: ldb.NewStore(repoPath)}, nil } func MakeLevelDBDir(repoPath string) (string, error) { @@ -55,13 +60,13 @@ func (s *Service) Start(ctx context.Context, addr string) error { return fmt.Errorf("setting up listener for local index directory service: %w", err) } - err = s.impl.Start(ctx) + err = s.Impl.Start(ctx) if err != nil { return fmt.Errorf("starting local index directory service: %w", err) } server := jsonrpc.NewServer() - server.Register("boostddata", s.impl) + server.Register("boostddata", s.Impl) router := mux.NewRouter() router.Handle("/", server) diff --git a/extern/boostd-data/svc/svc_test.go b/extern/boostd-data/svc/svc_test.go index 498829caa..2ec2d83f7 100644 --- a/extern/boostd-data/svc/svc_test.go +++ b/extern/boostd-data/svc/svc_test.go @@ -1,3 +1,6 @@ +//go:build test_lid +// +build test_lid + package svc import ( @@ -11,8 +14,6 @@ import ( "testing" "time" - "golang.org/x/sync/errgroup" - "github.com/filecoin-project/boost/testutil" "github.com/filecoin-project/boostd-data/client" "github.com/filecoin-project/boostd-data/couchbase" @@ -26,6 +27,7 @@ import ( "github.com/multiformats/go-multicodec" "github.com/multiformats/go-multihash" "github.com/stretchr/testify/require" + "golang.org/x/sync/errgroup" ) var testCouchSettings = couchbase.DBSettings{ @@ -54,8 +56,10 @@ func TestService(t *testing.T) { defer cancel() bdsvc, err := NewLevelDB("") require.NoError(t, err) + testService(ctx, t, bdsvc, "localhost:8042") }) + t.Run("couchbase", func(t *testing.T) { // TODO: Unskip this test once the couchbase instance can be created // from a docker container in CI as part of the test @@ -66,7 +70,23 @@ func TestService(t *testing.T) { defer cancel() SetupCouchbase(t, testCouchSettings) bdsvc := NewCouchbase(testCouchSettings) - testService(ctx, t, bdsvc, "localhost:8043") + + addr := "localhost:8043" + testService(ctx, t, bdsvc, addr) + }) + + t.Run("yugabyte", func(t *testing.T) { + // Running yugabyte tests may require download the docker container + // so set a high timeout + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + defer cancel() + + SetupYugabyte(t) + + bdsvc := NewYugabyte(TestYugabyteSettings) + + addr := "localhost:8044" + testService(ctx, t, bdsvc, addr) }) } @@ -118,6 +138,24 @@ func testService(ctx context.Context, t *testing.T, bdsvc *Service, addr string) require.Len(t, dis, 1) require.Equal(t, di, dis[0]) + // Add a second deal + di2 := model.DealInfo{ + DealUuid: uuid.NewString(), + SectorID: abi.SectorNumber(11), + PieceOffset: 11, + PieceLength: 12, + CarLength: 13, + } + err = cl.AddDealForPiece(ctx, pieceCid, di2) + require.NoError(t, err) + + // There should now be two deals + dis, err = cl.GetPieceDeals(ctx, pieceCid) + require.NoError(t, err) + require.Len(t, dis, 2) + require.Contains(t, dis, di) + require.Contains(t, dis, di2) + b, err := hex.DecodeString("1220ff63d7689e2d9567d1a90a7a68425f430137142e1fbc28fe4780b9ee8a5ef842") require.NoError(t, err) @@ -164,24 +202,39 @@ func TestServiceFuzz(t *testing.T) { t.Run("level db", func(t *testing.T) { bdsvc, err := NewLevelDB("") require.NoError(t, err) - testServiceFuzz(ctx, t, bdsvc, "localhost:8042") + addr := "localhost:8042" + err = bdsvc.Start(ctx, addr) + require.NoError(t, err) + testServiceFuzz(ctx, t, addr) }) + t.Run("couchbase", func(t *testing.T) { // TODO: Unskip this test once the couchbase instance can be created // from a docker container in CI as part of the test t.Skip() SetupCouchbase(t, testCouchSettings) bdsvc := NewCouchbase(testCouchSettings) - testServiceFuzz(ctx, t, bdsvc, "localhost:8043") + addr := "localhost:8043" + err := bdsvc.Start(ctx, addr) + require.NoError(t, err) + testServiceFuzz(ctx, t, addr) }) -} -func testServiceFuzz(ctx context.Context, t *testing.T, bdsvc *Service, addr string) { - err := bdsvc.Start(ctx, addr) - require.NoError(t, err) + t.Run("yugabyte", func(t *testing.T) { + SetupYugabyte(t) + bdsvc := NewYugabyte(TestYugabyteSettings) + + addr := "localhost:8044" + err := bdsvc.Start(ctx, addr) + require.NoError(t, err) + + testServiceFuzz(ctx, t, addr) + }) +} +func testServiceFuzz(ctx context.Context, t *testing.T, addr string) { cl := client.NewStore() - err = cl.Dial(context.Background(), "http://localhost:8042") + err := cl.Dial(context.Background(), "http://"+addr) require.NoError(t, err) defer cl.Close(ctx) @@ -386,6 +439,7 @@ func TestCleanup(t *testing.T) { require.NoError(t, err) testCleanup(ctx, t, bdsvc, "localhost:8042") }) + t.Run("couchbase", func(t *testing.T) { // TODO: Unskip this test once the couchbase instance can be created // from a docker container in CI as part of the test @@ -394,6 +448,18 @@ func TestCleanup(t *testing.T) { bdsvc := NewCouchbase(testCouchSettings) testCleanup(ctx, t, bdsvc, "localhost:8043") }) + + t.Run("yugabyte", func(t *testing.T) { + // Running yugabyte tests may require download the docker container + // so set a high timeout + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + defer cancel() + + SetupYugabyte(t) + + bdsvc := NewYugabyte(TestYugabyteSettings) + testCleanup(ctx, t, bdsvc, "localhost:8044") + }) } func testCleanup(ctx context.Context, t *testing.T, bdsvc *Service, addr string) { @@ -416,29 +482,41 @@ func testCleanup(ctx context.Context, t *testing.T, bdsvc *Service, addr string) records, err := getRecords(subject) require.NoError(t, err) - randomuuid, err := uuid.Parse("4d8f5ce6-dbfd-40dc-8b03-29308e97357b") - require.NoError(t, err) - err = cl.AddIndex(ctx, pieceCid, records, true) require.NoError(t, err) di := model.DealInfo{ - DealUuid: randomuuid.String(), + DealUuid: uuid.NewString(), SectorID: abi.SectorNumber(1), PieceOffset: 1, PieceLength: 2, CarLength: 3, } + di2 := model.DealInfo{ + DealUuid: uuid.NewString(), + SectorID: abi.SectorNumber(10), + PieceOffset: 11, + PieceLength: 12, + CarLength: 13, + } - // Add a deal for the piece + // Add two deals for the piece err = cl.AddDealForPiece(ctx, pieceCid, di) require.NoError(t, err) + err = cl.AddDealForPiece(ctx, pieceCid, di2) + require.NoError(t, err) - // There should only be one deal dis, err := cl.GetPieceDeals(ctx, pieceCid) require.NoError(t, err) + require.Len(t, dis, 2) + + // Remove one deal for the piece + err = cl.RemoveDealForPiece(ctx, pieceCid, di.DealUuid) + require.NoError(t, err) + + dis, err = cl.GetPieceDeals(ctx, pieceCid) + require.NoError(t, err) require.Len(t, dis, 1) - require.Equal(t, di, dis[0]) b, err := hex.DecodeString("1220ff63d7689e2d9567d1a90a7a68425f430137142e1fbc28fe4780b9ee8a5ef842") require.NoError(t, err) @@ -449,7 +527,6 @@ func testCleanup(ctx context.Context, t *testing.T, bdsvc *Service, addr string) offset, err := cl.GetOffsetSize(ctx, pieceCid, mhash) require.NoError(t, err) require.EqualValues(t, 3039040395, offset.Offset) - require.EqualValues(t, 0, offset.Size) pcids, err := cl.PiecesContainingMultihash(ctx, mhash) require.NoError(t, err) @@ -460,18 +537,10 @@ func testCleanup(ctx context.Context, t *testing.T, bdsvc *Service, addr string) require.NoError(t, err) require.True(t, indexed) - recs, err := cl.GetRecords(ctx, pieceCid) - require.NoError(t, err) - require.Equal(t, len(records), len(recs)) - - loadedSubject, err := cl.GetIndex(ctx, pieceCid) - require.NoError(t, err) - - ok, err := compareIndices(subject, loadedSubject) - require.NoError(t, err) - require.True(t, ok) - - err = cl.RemoveDealForPiece(ctx, pieceCid, di.DealUuid) + // Remove the other deal for the piece. + // After this call there are no deals left, so it should also cause the + // piece metadata and indexes to be removed. + err = cl.RemoveDealForPiece(ctx, pieceCid, di2.DealUuid) require.NoError(t, err) _, err = cl.GetPieceDeals(ctx, pieceCid) diff --git a/extern/boostd-data/svc/types/types.go b/extern/boostd-data/svc/types/types.go index 631cf27d7..76106df32 100644 --- a/extern/boostd-data/svc/types/types.go +++ b/extern/boostd-data/svc/types/types.go @@ -18,6 +18,9 @@ var ErrNotFound = errors.New("not found") // We have to do string matching so that it can be used on errors that // cross the RPC boundary (we can't use errors.Is) func IsNotFound(err error) bool { + if err == nil { + return false + } return strings.Contains(err.Error(), ErrNotFound.Error()) } @@ -31,7 +34,6 @@ type Service interface { ListPieces(ctx context.Context) ([]cid.Cid, error) GetPieceMetadata(ctx context.Context, pieceCid cid.Cid) (model.Metadata, error) GetPieceDeals(context.Context, cid.Cid) ([]model.DealInfo, error) - SetCarSize(ctx context.Context, pieceCid cid.Cid, size uint64) error IndexedAt(context.Context, cid.Cid) (time.Time, error) PiecesContainingMultihash(context.Context, mh.Multihash) ([]cid.Cid, error) RemoveDealForPiece(context.Context, cid.Cid, string) error diff --git a/extern/boostd-data/yugabyte/create.cql b/extern/boostd-data/yugabyte/create.cql new file mode 100644 index 000000000..0e7c01e48 --- /dev/null +++ b/extern/boostd-data/yugabyte/create.cql @@ -0,0 +1,44 @@ +create keyspace if not exists idx with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 }; + +CREATE TABLE IF NOT EXISTS idx.PayloadToPieces ( + PayloadMultihash BLOB, + PieceCid BLOB, + -- PayloadMultihash is the partition key: the key that determines which + -- node the row is sent to. + -- PieceCid is the clustering key: the data is sorted by this key. + -- We want both of these to be in the primary key because + -- the piece cids should be a unique set. + PRIMARY KEY (PayloadMultihash, PieceCid) +); + +CREATE TABLE IF NOT EXISTS idx.PieceBlockOffsetSize ( + PieceCid BLOB, + PayloadMultihash BLOB, + BlockOffset BIGINT, + BlockSize BIGINT, + PRIMARY KEY (PieceCid, PayloadMultihash) +); + +CREATE TABLE IF NOT EXISTS idx.PieceMetadata ( + PieceCid TEXT PRIMARY KEY, + Version TEXT, + CreatedAt TIMESTAMP, + IndexedAt TIMESTAMP, + CompleteIndex BOOLEAN, + Error TEXT, + ErrorType Text +) WITH transactions = { 'enabled' : true }; + +CREATE INDEX IF NOT EXISTS PieceMetadataCreatedAt ON idx.PieceMetadata (CreatedAt); + +CREATE TABLE IF NOT EXISTS idx.PieceDeal ( + DealUuid TEXT PRIMARY KEY, + PieceCid BLOB, + IsLegacy BOOLEAN, + ChainDealID BIGINT, + MinerAddr TEXT, + SectorID BIGINT, + PieceOffset BIGINT, + PieceLength BIGINT, + CarLength BIGINT +); diff --git a/extern/boostd-data/yugabyte/create.sql b/extern/boostd-data/yugabyte/create.sql new file mode 100644 index 000000000..d296143af --- /dev/null +++ b/extern/boostd-data/yugabyte/create.sql @@ -0,0 +1,17 @@ +CREATE TABLE IF NOT EXISTS PieceTracker ( + PieceCid TEXT PRIMARY KEY, + CreatedAt TIMESTAMP, + UpdatedAt TIMESTAMP +); + +CREATE INDEX IF NOT EXISTS PieceTrackerCreatedAt ON PieceTracker (CreatedAt); +CREATE INDEX IF NOT EXISTS PieceTrackerUpdatedAt ON PieceTracker (UpdatedAt); + +CREATE TABLE IF NOT EXISTS PieceFlagged ( + PieceCid TEXT PRIMARY KEY, + CreatedAt TIMESTAMP, + UpdatedAt TIMESTAMP +); + +CREATE INDEX IF NOT EXISTS PieceFlaggedCreatedAt ON PieceFlagged (CreatedAt); +CREATE INDEX IF NOT EXISTS PieceFlaggedUpdatedAt ON PieceFlagged (UpdatedAt); diff --git a/extern/boostd-data/yugabyte/drop.cql b/extern/boostd-data/yugabyte/drop.cql new file mode 100644 index 000000000..e50b35d44 --- /dev/null +++ b/extern/boostd-data/yugabyte/drop.cql @@ -0,0 +1,10 @@ +DROP TABLE IF EXISTS idx.PayloadToPieces; +DROP TABLE IF EXISTS idx.PieceBlockOffsetSize; +DROP TABLE IF EXISTS idx.PieceMetadata; +DROP INDEX IF EXISTS idx.PieceMetadataIndexedAt; +DROP TABLE IF EXISTS idx.PieceDeal; +DROP TABLE IF EXISTS idx.PieceTracker; +DROP INDEX IF EXISTS idx.PieceTrackerUpdatedAt; +DROP TABLE IF EXISTS idx.PieceFlagged; +DROP INDEX IF EXISTS idx.PieceFlaggedCreatedAt; +DROP INDEX IF EXISTS idx.PieceFlaggedUpdatedAt; diff --git a/extern/boostd-data/yugabyte/drop.sql b/extern/boostd-data/yugabyte/drop.sql new file mode 100644 index 000000000..92e483c0f --- /dev/null +++ b/extern/boostd-data/yugabyte/drop.sql @@ -0,0 +1,6 @@ +DROP TABLE IF EXISTS PieceTracker; +DROP INDEX IF EXISTS PieceTrackerCreatedAt; +DROP INDEX IF EXISTS PieceTrackerUpdatedAt; +DROP TABLE IF EXISTS PieceFlagged; +DROP INDEX IF EXISTS PieceFlaggedCreatedAt; +DROP INDEX IF EXISTS PieceFlaggedUpdatedAt; diff --git a/extern/boostd-data/yugabyte/encode.go b/extern/boostd-data/yugabyte/encode.go new file mode 100644 index 000000000..33cee6084 --- /dev/null +++ b/extern/boostd-data/yugabyte/encode.go @@ -0,0 +1,18 @@ +package yugabyte + +import ( + "github.com/multiformats/go-multihash" +) + +// Probability of a collision in two 24 byte hashes (birthday problem): +// 2^(24*8/2) = 8 x 10^28 +const multihashLimitBytes = 24 + +// trimMultihash trims the multihash to the last multihashLimitBytes bytes +func trimMultihash(mh multihash.Multihash) []byte { + var idx int + if len(mh) > multihashLimitBytes { + idx = len(mh) - multihashLimitBytes + } + return mh[idx:] +} diff --git a/extern/boostd-data/yugabyte/err.go b/extern/boostd-data/yugabyte/err.go new file mode 100644 index 000000000..176679a4e --- /dev/null +++ b/extern/boostd-data/yugabyte/err.go @@ -0,0 +1,45 @@ +package yugabyte + +import ( + "errors" + "fmt" + "github.com/filecoin-project/boostd-data/svc/types" + "github.com/ipfs/go-cid" + mh "github.com/multiformats/go-multihash" + "github.com/yugabyte/gocql" + "strings" +) + +func normalizePieceCidError(pieceCid cid.Cid, err error) error { + if err == nil { + return nil + } + if isNotFoundErr(err) { + return fmt.Errorf("piece %s: %w", pieceCid, types.ErrNotFound) + } + return err +} + +func normalizeMultihashError(m mh.Multihash, err error) error { + if err == nil { + return nil + } + if isNotFoundErr(err) { + return fmt.Errorf("multihash %s: %w", m, types.ErrNotFound) + } + return err +} + +func isNotFoundErr(err error) bool { + if err == nil { + return false + } + + if errors.Is(err, gocql.ErrNotFound) { + return true + } + + // Unfortunately it seems like the Cassandra driver doesn't always return + // a specific not found error type, so we need to rely on string parsing + return strings.Contains(strings.ToLower(err.Error()), "not found") +} diff --git a/extern/boostd-data/yugabyte/piecedoctor.go b/extern/boostd-data/yugabyte/piecedoctor.go new file mode 100644 index 000000000..93a6a9699 --- /dev/null +++ b/extern/boostd-data/yugabyte/piecedoctor.go @@ -0,0 +1,287 @@ +package yugabyte + +import ( + "context" + "fmt" + "github.com/filecoin-project/boostd-data/model" + "github.com/filecoin-project/boostd-data/shared/tracing" + "github.com/ipfs/go-cid" + "github.com/jackc/pgtype" + "go.opentelemetry.io/otel/attribute" + "golang.org/x/sync/errgroup" + "time" +) + +var TrackerCheckBatchSize = 1024 + +const insertTrackerParallelism = 16 + +type pieceCreated struct { + PieceCid cid.Cid + CreatedAt time.Time +} + +// NextPiecesToCheck is periodically called by the piece doctor. +// It returns a selection of piece cids so the piece doctor can check the +// status of each piece. +// For each piece it saves the time at which the piece was checked, so that +// the piece won't be checked again for a while. +// The implementation uses a PieceTracker table to keep track of when each piece +// was last checked. +func (s *Store) NextPiecesToCheck(ctx context.Context) ([]cid.Cid, error) { + ctx, span := tracing.Tracer.Start(ctx, "store.next_pieces_to_check") + defer span.End() + + // Get the time at which pieces were last copied from the piece metadata + // to the piece tracker table + var lastCopiedRes pgtype.Timestamptz + err := s.db.QueryRow(ctx, `SELECT MAX(CreatedAt) FROM PieceTracker`).Scan(&lastCopiedRes) + if err != nil { + return nil, fmt.Errorf("getting time piece tracker was last updated: %w", err) + } + + lastCopied := lastCopiedRes.Time + if lastCopiedRes.Status&pgtype.Present == 0 { + // If there are no results, set last updated to the zero value of time, + // so that we copy across all rows from piece metadata to piece tracker + lastCopied = time.UnixMilli(0) + } + log.Debugw("got tracker last updated", "updated at", lastCopied.String()) + + // Get the list of pieces that have been added since tracking information + // was last updated + qry := `SELECT PieceCid, CreatedAt from idx.PieceMetadata WHERE CreatedAt >= ?` + iter := s.session.Query(qry, lastCopied).WithContext(ctx).Iter() + var newPieces []pieceCreated + var createdAt time.Time + var pcidstr string + for iter.Scan(&pcidstr, &createdAt) { + c, err := cid.Parse(pcidstr) + if err != nil { + return nil, fmt.Errorf("getting new pieces: parsing piece cid %s: %w", pcidstr, err) + } + newPieces = append(newPieces, pieceCreated{PieceCid: c, CreatedAt: createdAt}) + } + if err := iter.Close(); err != nil { + return nil, fmt.Errorf("getting new pieces: %w", err) + } + + // Add any new pieces into the piece status tracking table + log.Debugw("inserting new pieces into tracker", "count", len(newPieces)) + err = s.execWithConcurrency(ctx, newPieces, insertTrackerParallelism, func(pc pieceCreated) error { + qry := `INSERT INTO PieceTracker (PieceCid, CreatedAt, UpdatedAt) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING` + _, err := s.db.Exec(ctx, qry, pc.PieceCid, pc.CreatedAt, time.UnixMilli(0)) + if err != nil { + return fmt.Errorf("inserting row into piece tracker: %w", err) + } + return nil + }) + if err != nil { + return nil, err + } + + // Work out how frequently to check each piece, based on how many pieces + // there are. + // Any pieces that have not been checked in the last pieceCheckPeriod + // will be checked now (eg check all pieces that haven't been checked + // for 10s) + pieceCheckPeriod, err := s.getPieceCheckPeriod(ctx) + if err != nil { + return nil, fmt.Errorf("getting piece check period: %w", err) + } + + // Get all pieces from the piece tracker table that have not been updated + // since the last piece check period. + // At the same time set the UpdatedAt field so that these pieces are marked + // as checked (and will not be returned until the next piece check period + // elapses again). + // Note that we limit the number of rows to fetch so as not to overload the + // system. Any rows beyond the limit will be fetched the next time + // NextPiecesToCheck is called. + now := time.Now() + qry = `WITH cte AS (SELECT PieceCid FROM PieceTracker WHERE UpdatedAt < $1 LIMIT $2)` + + `UPDATE PieceTracker pt SET UpdatedAt = $3 ` + + `FROM cte WHERE pt.PieceCid = cte.PieceCid RETURNING pt.PieceCid` + rows, err := s.db.Query(ctx, qry, now.Add(-pieceCheckPeriod), TrackerCheckBatchSize, now) + if err != nil { + return nil, fmt.Errorf("getting pieces from piece tracker: %w", err) + } + defer rows.Close() + + pcids := make([]cid.Cid, 0, TrackerCheckBatchSize) + var pcid string + for rows.Next() { + err := rows.Scan(&pcid) + if err != nil { + return nil, fmt.Errorf("scanning piece tracker row: %w", err) + } + + c, err := cid.Parse(pcid) + if err != nil { + return nil, fmt.Errorf("parsing tracker piece cid %s as cid: %w", pcid, err) + } + pcids = append(pcids, c) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("getting pieces to check: %w", err) + } + + log.Debugw("got tracker pieces", "count", len(pcids), + "last updated", lastCopied.String(), "now", now.String(), "piece-check-period", pieceCheckPeriod.String()) + return pcids, nil +} + +func (s *Store) execWithConcurrency(ctx context.Context, pcids []pieceCreated, concurrency int, exec func(created pieceCreated) error) error { + queue := make(chan pieceCreated, len(pcids)) + for _, pc := range pcids { + queue <- pc + } + close(queue) + + var eg errgroup.Group + for i := 0; i < concurrency; i++ { + eg.Go(func() error { + for ctx.Err() == nil { + select { + case <-ctx.Done(): + return ctx.Err() + case pc, ok := <-queue: + if !ok { + // Finished adding all the queued items, exit the thread + return nil + } + + err := exec(pc) + if err != nil { + return err + } + } + } + + return ctx.Err() + }) + } + return eg.Wait() +} + +// The minimum frequency with which to check pieces for errors (eg bad index) +var MinPieceCheckPeriod = 30 * time.Second + +// Work out how frequently to check each piece, based on how many pieces +// there are: if there are many pieces, each piece will be checked +// less frequently +func (s *Store) getPieceCheckPeriod(ctx context.Context) (time.Duration, error) { + var count int + err := s.db.QueryRow(ctx, `SELECT Count(*) FROM PieceTracker`).Scan(&count) + if err != nil { + return 0, fmt.Errorf("getting count of rows in piece tracker: %w", err) + } + + // Check period: + // - 1k pieces; every 10s + // - 100k pieces; every 15m + // - 1m pieces; every 2 hours + period := time.Duration(count*10) * time.Millisecond + if period < MinPieceCheckPeriod { + period = MinPieceCheckPeriod + } + + return period, nil +} + +func (s *Store) FlagPiece(ctx context.Context, pieceCid cid.Cid) error { + ctx, span := tracing.Tracer.Start(ctx, "store.flag_piece") + span.SetAttributes(attribute.String("pieceCid", pieceCid.String())) + defer span.End() + + now := time.Now() + qry := `INSERT INTO PieceFlagged (PieceCid, CreatedAt, UpdatedAt) ` + + `VALUES ($1, $2, $3) ` + + `ON CONFLICT (PieceCid) DO UPDATE SET UpdatedAt = excluded.UpdatedAt` + _, err := s.db.Exec(ctx, qry, pieceCid.String(), now, now) + if err != nil { + return fmt.Errorf("flagging piece %s: %w", pieceCid, err) + } + return nil +} + +func (s *Store) UnflagPiece(ctx context.Context, pieceCid cid.Cid) error { + ctx, span := tracing.Tracer.Start(ctx, "store.unflag_piece") + span.SetAttributes(attribute.String("pieceCid", pieceCid.String())) + defer span.End() + + qry := `DELETE FROM PieceFlagged WHERE PieceCid = $1` + _, err := s.db.Exec(ctx, qry, pieceCid.String()) + if err != nil { + return fmt.Errorf("unflagging piece %s: %w", pieceCid, err) + } + + return nil +} + +func (s *Store) FlaggedPiecesList(ctx context.Context, cursor *time.Time, offset int, limit int) ([]model.FlaggedPiece, error) { + ctx, span := tracing.Tracer.Start(ctx, "store.flagged_pieces") + var spanCursor int + if cursor != nil { + spanCursor = int(cursor.UnixMilli()) + } + span.SetAttributes(attribute.Int("cursor", spanCursor)) + span.SetAttributes(attribute.Int("offset", offset)) + span.SetAttributes(attribute.Int("limit", limit)) + defer span.End() + + var args []interface{} + idx := 0 + qry := `SELECT PieceCid, CreatedAt from PieceFlagged ` + if cursor != nil { + qry += `WHERE CreatedAt < $1 ` + args = append(args, cursor) + idx++ + } + qry += `ORDER BY CreatedAt desc ` + + qry += fmt.Sprintf(`LIMIT $%d OFFSET $%d`, idx+1, idx+2) + args = append(args, limit, offset) + + rows, err := s.db.Query(ctx, qry, args...) + if err != nil { + return nil, fmt.Errorf("getting flagged pieces: %w", err) + } + defer rows.Close() + + var pieces []model.FlaggedPiece + var pcid string + var createdAt time.Time + for rows.Next() { + err := rows.Scan(&pcid, &createdAt) + if err != nil { + return nil, fmt.Errorf("scanning flagged piece: %w", err) + } + + c, err := cid.Parse(pcid) + if err != nil { + return nil, fmt.Errorf("parsing flagged piece cid %s: %w", pcid, err) + } + pieces = append(pieces, model.FlaggedPiece{PieceCid: c, CreatedAt: createdAt}) + } + + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("getting new pieces: %w", err) + } + + return pieces, nil +} + +func (s *Store) FlaggedPiecesCount(ctx context.Context) (int, error) { + ctx, span := tracing.Tracer.Start(ctx, "store.flagged_pieces_count") + defer span.End() + + var count int + qry := `SELECT COUNT(*) FROM PieceFlagged` + err := s.db.QueryRow(ctx, qry).Scan(&count) + if err != nil { + return 0, fmt.Errorf("getting flagged pieces count: %w", err) + } + + return count, nil +} diff --git a/extern/boostd-data/yugabyte/service.go b/extern/boostd-data/yugabyte/service.go new file mode 100644 index 000000000..b529cc7e6 --- /dev/null +++ b/extern/boostd-data/yugabyte/service.go @@ -0,0 +1,534 @@ +package yugabyte + +import ( + "context" + _ "embed" + "fmt" + "github.com/filecoin-project/boostd-data/model" + "github.com/filecoin-project/boostd-data/shared/tracing" + "github.com/filecoin-project/boostd-data/svc/types" + "github.com/filecoin-project/go-address" + "github.com/ipfs/go-cid" + logging "github.com/ipfs/go-log/v2" + "github.com/multiformats/go-multihash" + mh "github.com/multiformats/go-multihash" + "github.com/yugabyte/gocql" + "github.com/yugabyte/pgx/v4/pgxpool" + "golang.org/x/sync/errgroup" + "sync" + "time" +) + +var log = logging.Logger("boostd-data-yb") + +// The current piece metadata version. This version will be used when doing +// data migrations (migrations are not yet implemented in version 1). +const pieceMetadataVersion = "1" + +type DBSettings struct { + // The cassandra hosts to connect to + Hosts []string + // The postgres connect string + ConnectString string + // The number of threads to use when inserting into the PayloadToPieces index + PayloadPiecesParallelism int +} + +type Store struct { + settings DBSettings + cluster *gocql.ClusterConfig + session *gocql.Session + db *pgxpool.Pool + startOnce sync.Once +} + +var _ types.ServiceImpl = (*Store)(nil) + +func NewStore(settings DBSettings) *Store { + if settings.PayloadPiecesParallelism == 0 { + settings.PayloadPiecesParallelism = 16 + } + + cluster := gocql.NewCluster(settings.Hosts...) + return &Store{ + settings: settings, + cluster: cluster, + } +} + +func (s *Store) Start(ctx context.Context) error { + var startErr error + s.startOnce.Do(func() { + session, err := s.cluster.CreateSession() + if err != nil { + startErr = fmt.Errorf("creating yugabyte cluster: %w", err) + return + } + s.session = session + + db, err := pgxpool.Connect(context.Background(), s.settings.ConnectString) + if err != nil { + startErr = fmt.Errorf("connecting to database: %w", err) + return + } + s.db = db + + startErr = s.Create(ctx) + }) + + return startErr +} + +func (s *Store) AddDealForPiece(ctx context.Context, pieceCid cid.Cid, dealInfo model.DealInfo) error { + ctx, span := tracing.Tracer.Start(ctx, "store.add_deal_for_piece") + defer span.End() + + err := s.createPieceMetadata(ctx, pieceCid) + if err != nil { + return err + } + + qry := `INSERT INTO idx.PieceDeal ` + + `(DealUuid, PieceCid, IsLegacy, ChainDealID, MinerAddr, SectorID, PieceOffset, PieceLength, CarLength) ` + + `VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) ` + + `IF NOT EXISTS` + err = s.session.Query(qry, + dealInfo.DealUuid, pieceCid.Bytes(), dealInfo.IsLegacy, dealInfo.ChainDealID, dealInfo.MinerAddr.String(), + dealInfo.SectorID, dealInfo.PieceOffset, dealInfo.PieceLength, dealInfo.CarLength). + WithContext(ctx).Exec() + if err != nil { + return fmt.Errorf("inserting deal %s for piece %s", dealInfo.DealUuid, pieceCid) + } + + return nil +} + +func (s *Store) createPieceMetadata(ctx context.Context, pieceCid cid.Cid) error { + qry := `INSERT INTO idx.PieceMetadata (PieceCid, Version, CreatedAt) VALUES (?, ?, ?) IF NOT EXISTS` + err := s.session.Query(qry, pieceCid.String(), pieceMetadataVersion, time.Now()).WithContext(ctx).Exec() + if err != nil { + return fmt.Errorf("inserting piece metadata for piece %s: %w", pieceCid, err) + } + return nil +} + +func (s *Store) GetOffsetSize(ctx context.Context, pieceCid cid.Cid, hash mh.Multihash) (*model.OffsetSize, error) { + ctx, span := tracing.Tracer.Start(ctx, "store.get_offset_size") + defer span.End() + + var offset, size uint64 + qry := `SELECT BlockOffset, BlockSize FROM idx.PieceBlockOffsetSize WHERE PieceCid = ? AND PayloadMultihash = ?` + err := s.session.Query(qry, pieceCid.Bytes(), hash).WithContext(ctx).Scan(&offset, &size) + if err != nil { + err = normalizePieceCidError(pieceCid, err) + return nil, fmt.Errorf("getting offset / size: %w", err) + } + + return &model.OffsetSize{Offset: offset, Size: size}, nil +} + +// Get piece metadata with deals +func (s *Store) GetPieceMetadata(ctx context.Context, pieceCid cid.Cid) (model.Metadata, error) { + md, err := s.getPieceMetadata(ctx, pieceCid) + if err != nil { + return md, err + } + + deals, err := s.GetPieceDeals(ctx, pieceCid) + if err != nil { + return md, fmt.Errorf("getting deals for piece %s: %w", pieceCid, err) + } + md.Deals = deals + + return md, nil +} + +// Get the piece metadata without deal information +func (s *Store) getPieceMetadata(ctx context.Context, pieceCid cid.Cid) (model.Metadata, error) { + ctx, span := tracing.Tracer.Start(ctx, "store.get_piece_metadata") + defer span.End() + + // Get piece metadata + var md model.Metadata + qry := `SELECT Version, IndexedAt, CompleteIndex FROM idx.PieceMetadata WHERE PieceCid = ?` + err := s.session.Query(qry, pieceCid.String()).WithContext(ctx). + Scan(&md.Version, &md.IndexedAt, &md.CompleteIndex) + if err != nil { + err = normalizePieceCidError(pieceCid, err) + return md, fmt.Errorf("getting piece metadata: %w", err) + } + + return md, nil +} + +func (s *Store) GetPieceDeals(ctx context.Context, pieceCid cid.Cid) ([]model.DealInfo, error) { + ctx, span := tracing.Tracer.Start(ctx, "store.get_piece_deals") + defer span.End() + + // Get deals for piece + qry := `SELECT DealUuid, IsLegacy, ChainDealID, MinerAddr, ` + + `SectorID, PieceOffset, PieceLength, CarLength ` + + `FROM idx.PieceDeal WHERE PieceCid = ?` + iter := s.session.Query(qry, pieceCid.Bytes()).WithContext(ctx).Iter() + + var deals []model.DealInfo + var deal model.DealInfo + var minerAddr string + for iter.Scan(&deal.DealUuid, &deal.IsLegacy, &deal.ChainDealID, &minerAddr, + &deal.SectorID, &deal.PieceOffset, &deal.PieceLength, &deal.CarLength) { + + ma, err := address.NewFromString(minerAddr) + if err != nil { + return nil, fmt.Errorf("parsing miner address for piece %s / deal %s: %w", pieceCid, deal.DealUuid, err) + } + + deal.MinerAddr = ma + deals = append(deals, deal) + } + if err := iter.Close(); err != nil { + return nil, fmt.Errorf("getting piece deals: %w", err) + } + + // For correctness, we should always return a not found error if there is + // no piece with the piece cid + if len(deals) == 0 { + _, err := s.getPieceMetadata(ctx, pieceCid) + if err != nil { + return nil, err + } + } + + return deals, nil +} + +// Get all pieces that contain a multihash (used when retrieving by payload CID) +func (s *Store) PiecesContainingMultihash(ctx context.Context, m mh.Multihash) ([]cid.Cid, error) { + ctx, span := tracing.Tracer.Start(ctx, "store.pieces_containing_multihash") + defer span.End() + + // Get all piece cids referred to by the multihash + pcids := make([]cid.Cid, 0, 1) + var bz []byte + qry := `SELECT PieceCid FROM idx.PayloadToPieces WHERE PayloadMultihash = ?` + iter := s.session.Query(qry, trimMultihash(m)).WithContext(ctx).Iter() + for iter.Scan(&bz) { + pcid, err := cid.Parse(bz) + if err != nil { + return nil, fmt.Errorf("parsing piece cid: %w", err) + } + pcids = append(pcids, pcid) + } + if err := iter.Close(); err != nil { + return nil, fmt.Errorf("getting pieces containing multihash %s: %w", m, err) + } + + // No pieces found for multihash, return a "not found" error + if len(pcids) == 0 { + return nil, normalizeMultihashError(m, types.ErrNotFound) + } + + return pcids, nil +} + +func (s *Store) GetIndex(ctx context.Context, pieceCid cid.Cid) ([]model.Record, error) { + ctx, span := tracing.Tracer.Start(ctx, "store.get_index") + defer span.End() + + qry := `SELECT PayloadMultihash, BlockOffset, BlockSize FROM idx.PieceBlockOffsetSize WHERE PieceCid = ?` + iter := s.session.Query(qry, pieceCid.Bytes()).WithContext(ctx).Iter() + + var records []model.Record + var payloadMHBz []byte + var offset, size uint64 + for iter.Scan(&payloadMHBz, &offset, &size) { + _, pmh, err := multihash.MHFromBytes(payloadMHBz) + if err != nil { + return nil, fmt.Errorf("scanning mulithash: %w", err) + } + + records = append(records, model.Record{ + Cid: cid.NewCidV1(cid.Raw, pmh), + OffsetSize: model.OffsetSize{ + Offset: offset, + Size: size, + }, + }) + } + if err := iter.Close(); err != nil { + return nil, fmt.Errorf("getting piece index for piece %s: %w", pieceCid, err) + } + + // For correctness, we should always return a not found error if there is + // no piece with the piece cid + if len(records) == 0 { + _, err := s.getPieceMetadata(ctx, pieceCid) + if err != nil { + return nil, err + } + } + + return records, nil +} + +func (s *Store) AddIndex(ctx context.Context, pieceCid cid.Cid, recs []model.Record, isCompleteIndex bool) error { + ctx, span := tracing.Tracer.Start(ctx, "store.add_index") + defer span.End() + + if len(recs) == 0 { + return nil + } + + // Add a mapping from multihash -> piece cid so that clients can look up + // which pieces contain a multihash + err := s.addMultihashesToPieces(ctx, pieceCid, recs) + if err != nil { + return err + } + + // Add a mapping from piece cid -> offset / size of each block so that + // clients can get the block info for all blocks in a piece + err = s.addPieceInfos(ctx, pieceCid, recs) + if err != nil { + return err + } + + // Ensure the piece metadata exists + err = s.createPieceMetadata(ctx, pieceCid) + if err != nil { + return err + } + + // Mark indexing as complete for the piece + qry := `UPDATE idx.PieceMetadata ` + + `SET IndexedAt = ?, CompleteIndex = ? ` + + `WHERE PieceCid = ?` + err = s.session.Query(qry, time.Now(), isCompleteIndex, pieceCid.String()).WithContext(ctx).Exec() + if err != nil { + return fmt.Errorf("marking indexing as complete for piece %s", pieceCid) + } + + return nil +} + +func (s *Store) addMultihashesToPieces(ctx context.Context, pieceCid cid.Cid, recs []model.Record) error { + ctx, span := tracing.Tracer.Start(ctx, "store.add_index.payloadpiece") + defer span.End() + + return s.execParallel(ctx, recs, s.settings.PayloadPiecesParallelism, func(rec model.Record) error { + multihashBytes := rec.Cid.Hash() + q := `INSERT INTO idx.PayloadToPieces (PayloadMultihash, PieceCid) VALUES (?, ?)` + err := s.session.Query(q, trimMultihash(multihashBytes), pieceCid.Bytes()).Exec() + if err != nil { + return fmt.Errorf("inserting into PayloadToPieces: %w", err) + } + return nil + }) +} + +func (s *Store) addPieceInfos(ctx context.Context, pieceCid cid.Cid, recs []model.Record) error { + ctx, span := tracing.Tracer.Start(ctx, "store.add_index.pieceinfo") + defer span.End() + + batchEntries := make([]gocql.BatchEntry, 0, len(recs)) + insertPieceOffsetsQry := `INSERT INTO idx.PieceBlockOffsetSize (PieceCid, PayloadMultihash, BlockOffset, BlockSize) VALUES (?, ?, ?, ?)` + for _, rec := range recs { + batchEntries = append(batchEntries, gocql.BatchEntry{ + Stmt: insertPieceOffsetsQry, + Args: []interface{}{pieceCid.Bytes(), rec.Cid.Hash(), rec.Offset, rec.Size}, + Idempotent: true, + }) + } + + // The Cassandra driver has a 50k limit on batch statements. Keeping + // batch size small makes sure we're under the limit. + const batchSize = 49000 + var batch *gocql.Batch + for allIdx, entry := range batchEntries { + if batch == nil { + batch = s.session.NewBatch(gocql.UnloggedBatch).WithContext(ctx) + } + + batch.Entries = append(batch.Entries, entry) + + if allIdx == len(batchEntries)-1 || len(batch.Entries) == batchSize { + err := s.session.ExecuteBatch(batch) + if err != nil { + return fmt.Errorf("executing offset / size batch insert for piece %s: %w", pieceCid, err) + } + batch = nil + continue + } + } + + return nil +} + +func (s *Store) IsCompleteIndex(ctx context.Context, pieceCid cid.Cid) (bool, error) { + ctx, span := tracing.Tracer.Start(ctx, "store.is_incomplete_index") + defer span.End() + + md, err := s.getPieceMetadata(ctx, pieceCid) + if err != nil { + return false, err + } + + return md.CompleteIndex, nil +} + +func (s *Store) IsIndexed(ctx context.Context, pieceCid cid.Cid) (bool, error) { + t, err := s.IndexedAt(ctx, pieceCid) + if err != nil { + if isNotFoundErr(err) { + return false, nil + } + return false, err + } + return !t.IsZero(), nil +} + +func (s *Store) IndexedAt(ctx context.Context, pieceCid cid.Cid) (time.Time, error) { + ctx, span := tracing.Tracer.Start(ctx, "store.indexed_at") + defer span.End() + + md, err := s.getPieceMetadata(ctx, pieceCid) + if err != nil { + if isNotFoundErr(err) { + return time.Time{}, nil + } + return time.Time{}, err + } + + return md.IndexedAt, nil +} + +func (s *Store) ListPieces(ctx context.Context) ([]cid.Cid, error) { + ctx, span := tracing.Tracer.Start(ctx, "store.list_pieces") + defer span.End() + + iter := s.session.Query("SELECT PieceCid FROM idx.PieceMetadata").WithContext(ctx).Iter() + var pcids []cid.Cid + var cstr string + for iter.Scan(&cstr) { + c, err := cid.Parse(cstr) + if err != nil { + return nil, fmt.Errorf("parsing piece cid: %w", err) + } + + pcids = append(pcids, c) + } + if err := iter.Close(); err != nil { + return nil, fmt.Errorf("getting piece cids: %w", err) + } + + return pcids, nil +} + +// RemoveDealForPiece removes Single deal for pieceCID. +func (s *Store) RemoveDealForPiece(ctx context.Context, pieceCid cid.Cid, dealId string) error { + ctx, span := tracing.Tracer.Start(ctx, "store.remove_deal_for_piece") + defer span.End() + + qry := `DELETE FROM idx.PieceDeal WHERE DealUuid = ?` + err := s.session.Query(qry, dealId).WithContext(ctx).Exec() + if err != nil { + return fmt.Errorf("removing deal %s for piece %s: %w", dealId, pieceCid, err) + } + + dls, err := s.GetPieceDeals(ctx, pieceCid) + if err != nil { + return fmt.Errorf("getting remaining deals in remove deal for piece %s: %w", pieceCid, err) + } + + if len(dls) > 0 { + return nil + } + + err = s.RemoveIndexes(ctx, pieceCid) + if err != nil { + return fmt.Errorf("removing deal: %w", err) + } + + err = s.RemovePieceMetadata(ctx, pieceCid) + if err != nil { + return fmt.Errorf("removing deal: %w", err) + } + + return nil +} + +// RemovePieceMetadata removes all Metadata for pieceCID. +func (s *Store) RemovePieceMetadata(ctx context.Context, pieceCid cid.Cid) error { + ctx, span := tracing.Tracer.Start(ctx, "store.remove_piece_metadata") + defer span.End() + + qry := `DELETE FROM idx.PieceMetadata WHERE PieceCid = ?` + err := s.session.Query(qry, pieceCid.String()).WithContext(ctx).Exec() + if err != nil { + return fmt.Errorf("removing piece metadata for piece %s: %w", pieceCid, err) + } + + return nil +} + +// RemoveIndexes removes all multihash -> piece cid mappings, and all +// offset / size information for the piece. +func (s *Store) RemoveIndexes(ctx context.Context, pieceCid cid.Cid) error { + ctx, span := tracing.Tracer.Start(ctx, "store.remove_indexes") + defer span.End() + + recs, err := s.GetIndex(ctx, pieceCid) + if err != nil { + return fmt.Errorf("removing indexes for piece %s: getting recs: %w", pieceCid, err) + } + + err = s.execParallel(ctx, recs, s.settings.PayloadPiecesParallelism, func(rec model.Record) error { + multihashBytes := rec.Cid.Hash() + q := `DELETE FROM idx.PayloadToPieces WHERE PayloadMultihash = ? AND PieceCid = ?` + err := s.session.Query(q, trimMultihash(multihashBytes), pieceCid.Bytes()).Exec() + if err != nil { + return fmt.Errorf("inserting into PayloadToPieces: %w", err) + } + return nil + }) + + qry := `DELETE FROM idx.PieceBlockOffsetSize WHERE PieceCid = ?` + err = s.session.Query(qry, pieceCid.Bytes()).WithContext(ctx).Exec() + if err != nil { + return fmt.Errorf("removing indexes for piece %s: deleting offset / size info: %w", pieceCid, err) + } + + return nil +} + +func (s *Store) execParallel(ctx context.Context, recs []model.Record, parallelism int, f func(record model.Record) error) error { + queue := make(chan model.Record, len(recs)) + for _, rec := range recs { + queue <- rec + } + close(queue) + + var eg errgroup.Group + for i := 0; i < parallelism; i++ { + eg.Go(func() error { + for ctx.Err() == nil { + select { + case <-ctx.Done(): + return ctx.Err() + case rec, ok := <-queue: + if !ok { + // Finished adding all the queued items, exit the thread + return nil + } + + err := f(rec) + if err != nil { + return err + } + } + } + + return ctx.Err() + }) + } + return eg.Wait() +} diff --git a/extern/boostd-data/yugabyte/setup.go b/extern/boostd-data/yugabyte/setup.go new file mode 100644 index 000000000..849db882b --- /dev/null +++ b/extern/boostd-data/yugabyte/setup.go @@ -0,0 +1,65 @@ +package yugabyte + +import ( + "context" + _ "embed" + "fmt" + "strings" +) + +//go:embed create.cql +var createCQL string + +//go:embed create.sql +var createSQL string + +func (s *Store) Create(ctx context.Context) error { + log.Infow("creating cassandra tables") + err := s.execScript(ctx, createCQL, s.execCQL) + if err != nil { + return err + } + + log.Infow("creating postgres tables") + return s.execScript(ctx, createSQL, s.execSQL) +} + +//go:embed drop.cql +var dropCQL string + +//go:embed drop.sql +var dropSQL string + +func (s *Store) Drop(ctx context.Context) error { + err := s.execScript(ctx, dropCQL, s.execCQL) + if err != nil { + return err + } + return s.execScript(ctx, dropSQL, s.execSQL) +} + +func (s *Store) execScript(ctx context.Context, cqlstr string, exec func(context.Context, string) error) error { + lines := strings.Split(cqlstr, ";") + for _, line := range lines { + line = strings.Trim(line, "\n \t") + if line == "" { + continue + } + log.Debug(line) + err := exec(ctx, line) + if err != nil { + return fmt.Errorf("executing\n%s\n%w", line, err) + } + } + + return nil +} + +func (s *Store) execCQL(ctx context.Context, query string) error { + return s.session.Query(query).WithContext(ctx).Exec() +} + +func (s *Store) execSQL(ctx context.Context, query string) error { + _, err := s.db.Exec(ctx, query) + return err +} diff --git a/go.mod b/go.mod index 574fa987d..669fd8128 100644 --- a/go.mod +++ b/go.mod @@ -369,8 +369,17 @@ require ( require ( github.com/google/pprof v0.0.0-20221203041831-ce31453925ec // indirect + github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect github.com/hashicorp/go-uuid v1.0.2 // indirect github.com/hashicorp/golang-lru/v2 v2.0.2 // indirect + github.com/jackc/chunkreader/v2 v2.0.1 // indirect + github.com/jackc/pgconn v1.11.0 // indirect + github.com/jackc/pgio v1.0.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgproto3/v2 v2.2.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect + github.com/jackc/pgtype v1.10.0 // indirect + github.com/jackc/puddle v1.2.1 // indirect github.com/onsi/ginkgo/v2 v2.5.1 // indirect github.com/quic-go/qpack v0.4.0 // indirect github.com/quic-go/qtls-go1-19 v0.2.1 // indirect @@ -379,6 +388,9 @@ require ( github.com/quic-go/webtransport-go v0.5.2 // indirect github.com/tidwall/gjson v1.14.0 // indirect github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb // indirect + github.com/yugabyte/gocql v0.0.0-20221110041640-6fc475c5aeb0 // indirect + github.com/yugabyte/pgx/v4 v4.14.5 // indirect github.com/zyedidia/generic v1.2.1 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect nhooyr.io/websocket v1.8.7 // indirect ) diff --git a/go.sum b/go.sum index c7d3cfb1c..c17988896 100644 --- a/go.sum +++ b/go.sum @@ -74,6 +74,7 @@ github.com/Kubuxu/imtui v0.0.0-20210401140320-41663d68d0fa h1:1PPxEyGdIGVkX/kqMv github.com/Kubuxu/imtui v0.0.0-20210401140320-41663d68d0fa/go.mod h1:WUmMvh9wMtqj1Xhf1hf3kp9RvL+y6odtdYxpyZjb90U= github.com/Masterminds/glide v0.13.2/go.mod h1:STyF5vcenH/rUqTEv+/hBXlSTo7KYwg2oc2f4tzPWic= github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= +github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA= github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA= @@ -134,8 +135,12 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/bep/debounce v1.2.1 h1:v67fRdBA9UQu2NhLFXrSg0Brw7CexQekrBwDMM8bzeY= github.com/bep/debounce v1.2.1/go.mod h1:H8yggRPQKLUhUoqrJC1bO2xNya7vanpDl7xR3ISbCJ0= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 h1:mXoPYz/Ul5HYEDvkta6I8/rnYM5gSdSV2tJ6XbZuEtY= +github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k= github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= @@ -197,6 +202,8 @@ github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= +github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA= @@ -214,6 +221,7 @@ github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= @@ -530,6 +538,8 @@ github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5x github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= +github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/googleapis v1.4.1 h1:1Yx4Myt7BxzvUr5ldGSbwYiZG6t9wGBZ+8/fX3Wvtq0= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -664,6 +674,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= +github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8= +github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= github.com/hako/durafmt v0.0.0-20200710122514-c0fb7b4da026 h1:BpJ2o0OR5FV7vrkDYfXYVJQeMNWa8RhklZOpW2ITAIQ= github.com/hako/durafmt v0.0.0-20200710122514-c0fb7b4da026/go.mod h1:5Scbynm8dF1XAPwIwkGPqzkM/shndPm79Jd1003hTjE= github.com/hannahhoward/cbor-gen-for v0.0.0-20230214144701-5d17c9d5243c h1:iiD+p+U0M6n/FsO6XIZuOgobnNa48FxtyYFfWwLttUQ= @@ -974,6 +986,53 @@ github.com/ipni/storetheindex v0.5.10 h1:r97jIZsXPuwQvePJQuStu2a/kn+Zn8X4MAdA0rU github.com/ipni/storetheindex v0.5.10/go.mod h1:SJKFCnSx4X/4ekQuZvq8pVU/7tmxkEv632Qmgu3m2bQ= github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 h1:QG4CGBqCeuBo6aZlGAamSkxWdgWfZGeE49eUOWJPA4c= github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52/go.mod h1:fdg+/X9Gg4AsAIzWpEHwnqd+QY3b7lajxyjE1m4hkq4= +github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= +github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= +github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA= +github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE= +github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s= +github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= +github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY= +github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= +github.com/jackc/pgconn v1.11.0 h1:HiHArx4yFbwl91X3qqIHtUFoiIfLNJXCQRsnzkiwwaQ= +github.com/jackc/pgconn v1.11.0/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= +github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= +github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= +github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= +github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c= +github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65 h1:DadwsjnMwFjfWc9y5Wi/+Zz7xoE5ALHsRQlOctkOiHc= +github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= +github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.2.0 h1:r7JypeP2D3onoQTCxWdTpCtJ4D+qpKr0TxvoyMhZ5ns= +github.com/jackc/pgproto3/v2 v2.2.0/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= +github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= +github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= +github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= +github.com/jackc/pgtype v1.10.0 h1:ILnBWrRMSXGczYvmkYD6PsYyVFUNLTnIUJHHDLmqk38= +github.com/jackc/pgtype v1.10.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= +github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= +github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= +github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= +github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c h1:Dznn52SgVIVst9UyOT9brctYUgxs+CvVfPaC3jKrA50= +github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= +github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.2.1 h1:gI8os0wpRXFd4FiAY2dWiqRK037tjj3t7rKFeO4X5iw= +github.com/jackc/puddle v1.2.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= @@ -1049,6 +1108,7 @@ github.com/klauspost/cpuid/v2 v2.1.0/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8t github.com/klauspost/cpuid/v2 v2.2.3 h1:sxCkb+qR91z4vsqw4vGGZlDgPz3G7gjaLyK3V8y70BU= github.com/klauspost/cpuid/v2 v2.2.3/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/koron/go-ssdp v0.0.0-20180514024734-4a0ed625a78b/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= github.com/koron/go-ssdp v0.0.0-20191105050749-2e1c40ed0b5d/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= @@ -1062,6 +1122,7 @@ github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NB github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= @@ -1069,6 +1130,10 @@ github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/lib/pq v0.0.0-20180327071824-d34b9ff171c2/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.10.4 h1:SO9z7FRPzA03QhHKJrH5BXA6HU1rS4V2nIVrrNC1iYk= github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/libp2p/go-addr-util v0.0.1/go.mod h1:4ac6O7n9rIAKB1dnd+s8IbbMXkt+oBpzX4/+RACcnlQ= github.com/libp2p/go-addr-util v0.0.2/go.mod h1:Ecd6Fb3yIuLzq4bD7VcywcVSBtefcAwnUISBM3WG15E= @@ -1324,6 +1389,7 @@ github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaO github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= @@ -1331,6 +1397,7 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= @@ -1664,6 +1731,8 @@ github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZV github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= +github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= github.com/rs/zerolog v1.21.0/go.mod h1:ZPhntP/xmq1nnND05hhpAh2QMhSsA4UN3MGZ6O2J3hM= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -1672,6 +1741,7 @@ github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/schollz/progressbar/v3 v3.13.1 h1:o8rySDYiQ59Mwzy2FELeHY5ZARXZTVJC7iHD6PEFUiE= github.com/schollz/progressbar/v3 v3.13.1/go.mod h1:xvrbki8kfT1fzWzBT/UZd9L6GA+jdL7HAgq2RFnO6fQ= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= @@ -1681,6 +1751,9 @@ github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAm github.com/shirou/gopsutil v2.18.12+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible h1:Bn1aCHHRnjv4Bl16T8rcaFjYSrGrIZvpiGO6P3Q4GpU= github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= +github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= +github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY= github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM= github.com/shurcooL/github_flavored_markdown v0.0.0-20181002035957-2122de532470/go.mod h1:2dOwnU2uBioM+SGy2aZoq1f/Sd1l9OkAeAUvjSyvgU0= @@ -1705,6 +1778,7 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeV github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYEDaXHZDBsXlPCDqdhQuJkuw4NOtaxYe3xii4= github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= @@ -1748,6 +1822,7 @@ github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3 github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= @@ -1872,6 +1947,10 @@ github.com/xorcare/golden v0.6.1-0.20191112154924-b87f686d7542 h1:oWgZJmC1DorFZD github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= +github.com/yugabyte/gocql v0.0.0-20221110041640-6fc475c5aeb0 h1:68nrJsrWe0A6JiKnsjWChAaWhj20v+AwYJObtp86D1k= +github.com/yugabyte/gocql v0.0.0-20221110041640-6fc475c5aeb0/go.mod h1:LAokR6+vevDCrTxk52U7p6ki+4qELu4XU7JUGYa2O2M= +github.com/yugabyte/pgx/v4 v4.14.5 h1:XLzEEiO3d/kWzpyctO8l4kwMLhzPQ9n2er7ATH7CJVA= +github.com/yugabyte/pgx/v4 v4.14.5/go.mod h1:nFSvjsVq4CuA61TWGriwWG74ZVxeuJCcNu42Mkn+rgw= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -1879,6 +1958,7 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= github.com/zondax/hid v0.9.0/go.mod h1:l5wttcP0jwtdLjqjMMWFVEE7d1zO0jvSPA9OPZxWpEM= github.com/zondax/hid v0.9.1 h1:gQe66rtmyZ8VeGFcOpbuH3r7erYtNEAezCAYu8LdkJo= @@ -1953,6 +2033,7 @@ go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95a go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.14.1/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= @@ -1976,6 +2057,7 @@ golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190225124518-7f87c0fbb88b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -1984,6 +2066,7 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190927123631-a832865fa7ad/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -1997,10 +2080,12 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= +golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220210151621-f4118a5b28e2/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= @@ -2167,6 +2252,7 @@ golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190302025703-b6889370fb10/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190316082340-a2f829d7f35f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -2180,6 +2266,7 @@ golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -2277,6 +2364,7 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= @@ -2301,6 +2389,7 @@ golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= @@ -2308,6 +2397,7 @@ golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190927191325-030b2cf1153e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -2355,6 +2445,8 @@ golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.3.0 h1:SrNbZl6ECOS1qFzgTdQfWXZM9XBkiA6tkFrH9YSTPHM= golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= +golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -2477,6 +2569,8 @@ gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qS gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= +gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/src-d/go-cli.v0 v0.0.0-20181105080154-d492247bbc0d/go.mod h1:z+K8VcOYVYcSwSjGebuDL6176A1XskgbtNl64NSg+n8= diff --git a/gql/resolver_piece.go b/gql/resolver_piece.go index 6842bcf89..65fc7a569 100644 --- a/gql/resolver_piece.go +++ b/gql/resolver_piece.go @@ -362,9 +362,6 @@ func (r *resolver) getIndexStatus(ctx context.Context, pieceCid cid.Cid, md pdty idxerr = mdErr.Error() case md.Indexing: idxst = IndexStatusIndexing - case md.Error != "": - idxst = IndexStatusFailed - idxerr = md.Error case md.IndexedAt.IsZero(): idxst = IndexStatusRegistered default: diff --git a/node/config/def.go b/node/config/def.go index 7c29ceba4..74daa704d 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -76,6 +76,9 @@ func DefaultBoost() *Boost { }, LocalIndexDirectory: LocalIndexDirectoryConfig{ + Yugabyte: LocalIndexDirectoryYugabyteConfig{ + Enabled: false, + }, Couchbase: LocalIndexDirectoryCouchbaseConfig{ ConnectString: "", Username: "", diff --git a/node/config/doc_gen.go b/node/config/doc_gen.go index b4f8e2acd..c0da49c90 100644 --- a/node/config/doc_gen.go +++ b/node/config/doc_gen.go @@ -443,6 +443,12 @@ for any other deal.`, }, }, "LocalIndexDirectoryConfig": []DocField{ + { + Name: "Yugabyte", + Type: "LocalIndexDirectoryYugabyteConfig", + + Comment: ``, + }, { Name: "Couchbase", Type: "LocalIndexDirectoryCouchbaseConfig", @@ -521,6 +527,26 @@ If empty, a leveldb database is used instead.`, Comment: ``, }, }, + "LocalIndexDirectoryYugabyteConfig": []DocField{ + { + Name: "Enabled", + Type: "bool", + + Comment: ``, + }, + { + Name: "ConnectString", + Type: "string", + + Comment: `The yugabyte postgres connect string eg "postgresql://postgres:postgres@localhost"`, + }, + { + Name: "Hosts", + Type: "[]string", + + Comment: `The yugabyte cassandra hosts eg ["127.0.0.1"]`, + }, + }, "LotusDealmakingConfig": []DocField{ { Name: "PieceCidBlocklist", diff --git a/node/config/types.go b/node/config/types.go index 34a52c2cd..2b6db413e 100644 --- a/node/config/types.go +++ b/node/config/types.go @@ -323,7 +323,16 @@ type LocalIndexDirectoryCouchbaseConfig struct { PieceOffsetsBucket LocalIndexDirectoryCouchbaseBucketConfig } +type LocalIndexDirectoryYugabyteConfig struct { + Enabled bool + // The yugabyte postgres connect string eg "postgresql://postgres:postgres@localhost" + ConnectString string + // The yugabyte cassandra hosts eg ["127.0.0.1"] + Hosts []string +} + type LocalIndexDirectoryConfig struct { + Yugabyte LocalIndexDirectoryYugabyteConfig Couchbase LocalIndexDirectoryCouchbaseConfig // The maximum number of add index operations allowed to execute in parallel. // The add index operation is executed when a new deal is created - it fetches diff --git a/node/impl/boost.go b/node/impl/boost.go index 5b2886f31..fb2d24d57 100644 --- a/node/impl/boost.go +++ b/node/impl/boost.go @@ -543,10 +543,6 @@ func (sm *BoostAPI) PdBuildIndexForPieceCid(ctx context.Context, piececid cid.Ci return sm.Pd.BuildIndexForPiece(ctx, piececid) } -func (sm *BoostAPI) PdMarkIndexErrored(ctx context.Context, piececid cid.Cid, err string) error { - return sm.Pd.MarkIndexErrored(ctx, piececid, err) -} - func (sm *BoostAPI) OnlineBackup(ctx context.Context, dstDir string) error { return sm.Bkp.Backup(ctx, dstDir) } diff --git a/node/modules/piecedirectory.go b/node/modules/piecedirectory.go index 994542b12..e1a73a760 100644 --- a/node/modules/piecedirectory.go +++ b/node/modules/piecedirectory.go @@ -16,6 +16,7 @@ import ( "github.com/filecoin-project/boostd-data/couchbase" "github.com/filecoin-project/boostd-data/model" "github.com/filecoin-project/boostd-data/svc" + "github.com/filecoin-project/boostd-data/yugabyte" "github.com/filecoin-project/dagstore" "github.com/filecoin-project/dagstore/shard" "github.com/filecoin-project/go-address" @@ -52,9 +53,20 @@ func NewPieceDirectoryStore(cfg *config.Boost) func(lc fx.Lifecycle, r lotus_rep } svcCtx, cancel = context.WithCancel(ctx) - var bdsvc *svc.Service - if cfg.LocalIndexDirectory.Couchbase.ConnectString != "" { + switch { + case cfg.LocalIndexDirectory.Yugabyte.Enabled: + log.Infow("local index directory: connecting to yugabyte server", + "connect-string", cfg.LocalIndexDirectory.Yugabyte.ConnectString, + "hosts", cfg.LocalIndexDirectory.Yugabyte.Hosts) + + // Set up a local index directory service that connects to the yugabyte db + bdsvc = svc.NewYugabyte(yugabyte.DBSettings{ + Hosts: cfg.LocalIndexDirectory.Yugabyte.Hosts, + ConnectString: cfg.LocalIndexDirectory.Yugabyte.ConnectString, + }) + + case cfg.LocalIndexDirectory.Couchbase.ConnectString != "": log.Infow("local index directory: connecting to couchbase server", "connect-string", cfg.LocalIndexDirectory.Couchbase.ConnectString) @@ -76,7 +88,8 @@ func NewPieceDirectoryStore(cfg *config.Boost) func(lc fx.Lifecycle, r lotus_rep RAMQuotaMB: cfg.LocalIndexDirectory.Couchbase.PieceOffsetsBucket.RAMQuotaMB, }, }) - } else { + + default: log.Infow("local index directory: connecting to leveldb instance") // Setup a local index directory service that connects to the leveldb diff --git a/piecedirectory/doctor.go b/piecedirectory/doctor.go index 805b5359f..ec6cfe77e 100644 --- a/piecedirectory/doctor.go +++ b/piecedirectory/doctor.go @@ -86,16 +86,6 @@ func (d *Doctor) checkPiece(ctx context.Context, pieceCid cid.Cid) error { return fmt.Errorf("failed to get piece %s from local index directory: %w", pieceCid, err) } - // Check if the piece is in an error state - if md.Error != "" { - err = d.store.FlagPiece(ctx, pieceCid) - if err != nil { - return fmt.Errorf("failed to flag piece in error state %s: %w", pieceCid, err) - } - doclog.Debugw("piece is in error state", "err", md.Error) - return nil - } - // Check if piece has been indexed isIndexed, err := d.store.IsIndexed(ctx, pieceCid) if err != nil { diff --git a/piecedirectory/doctor_test.go b/piecedirectory/doctor_test.go index 8e6ec2110..eb8338d0f 100644 --- a/piecedirectory/doctor_test.go +++ b/piecedirectory/doctor_test.go @@ -1,3 +1,6 @@ +//go:build test_lid +// +build test_lid + package piecedirectory import ( @@ -7,11 +10,13 @@ import ( "testing" "time" + "github.com/filecoin-project/boost/testutil" "github.com/filecoin-project/boostd-data/client" "github.com/filecoin-project/boostd-data/couchbase" "github.com/filecoin-project/boostd-data/ldb" "github.com/filecoin-project/boostd-data/model" "github.com/filecoin-project/boostd-data/svc" + "github.com/filecoin-project/boostd-data/yugabyte" "github.com/filecoin-project/go-state-types/abi" "github.com/google/uuid" "github.com/ipfs/go-cid" @@ -28,9 +33,6 @@ func TestPieceDoctor(t *testing.T) { prev := ldb.MinPieceCheckPeriod ldb.MinPieceCheckPeriod = 1 * time.Second - prevp := ldb.PiecesToTrackerBatchSize - ldb.PiecesToTrackerBatchSize = 4 - bdsvc, err := svc.NewLevelDB("") require.NoError(t, err) @@ -44,7 +46,11 @@ func TestPieceDoctor(t *testing.T) { defer cl.Close(ctx) t.Run("next pieces pagination", func(t *testing.T) { - testNextPiecesPagination(ctx, t, cl, ldb.MinPieceCheckPeriod) + prevp := ldb.PiecesToTrackerBatchSize + testNextPiecesPagination(ctx, t, cl, func(pageSize int) { + ldb.PiecesToTrackerBatchSize = pageSize + }) + ldb.PiecesToTrackerBatchSize = prevp }) t.Run("check pieces", func(t *testing.T) { @@ -52,8 +58,8 @@ func TestPieceDoctor(t *testing.T) { }) ldb.MinPieceCheckPeriod = prev - ldb.PiecesToTrackerBatchSize = prevp }) + t.Run("couchbase", func(t *testing.T) { // TODO: Unskip this test once the couchbase instance can be created // from a docker container in CI @@ -84,6 +90,47 @@ func TestPieceDoctor(t *testing.T) { couchbase.MinPieceCheckPeriod = prev }) + + t.Run("yugabyte", func(t *testing.T) { + prev := yugabyte.MinPieceCheckPeriod + yugabyte.MinPieceCheckPeriod = 1 * time.Second + + svc.SetupYugabyte(t) + + bdsvc := svc.NewYugabyte(svc.TestYugabyteSettings) + + addr := "localhost:8044" + err := bdsvc.Start(ctx, addr) + require.NoError(t, err) + + cl := client.NewStore() + err = cl.Dial(ctx, fmt.Sprintf("http://%s", addr)) + require.NoError(t, err) + defer cl.Close(ctx) + + ybstore := bdsvc.Impl.(*yugabyte.Store) + + t.Run("next pieces", func(t *testing.T) { + svc.RecreateTables(ctx, t, ybstore) + testNextPieces(ctx, t, cl, yugabyte.MinPieceCheckPeriod) + }) + + t.Run("next pieces pagination", func(t *testing.T) { + svc.RecreateTables(ctx, t, ybstore) + prevp := yugabyte.TrackerCheckBatchSize + testNextPiecesPagination(ctx, t, cl, func(pageSize int) { + yugabyte.TrackerCheckBatchSize = pageSize + }) + yugabyte.TrackerCheckBatchSize = prevp + }) + + t.Run("check pieces", func(t *testing.T) { + svc.RecreateTables(ctx, t, ybstore) + testCheckPieces(ctx, t, cl) + }) + + yugabyte.MinPieceCheckPeriod = prev + }) } // Verify that after a new piece is added @@ -92,7 +139,6 @@ func TestPieceDoctor(t *testing.T) { func testNextPieces(ctx context.Context, t *testing.T, cl *client.Store, pieceCheckPeriod time.Duration) { // Add a new piece pieceCid := blocks.NewBlock([]byte(fmt.Sprintf("%d", time.Now().UnixMilli()))).Cid() - fmt.Println(pieceCid) di := model.DealInfo{ DealUuid: uuid.New().String(), ChainDealID: 1, @@ -127,14 +173,14 @@ func testNextPieces(ctx context.Context, t *testing.T, cl *client.Store, pieceCh require.Contains(t, pcids, pieceCid) } -func testNextPiecesPagination(ctx context.Context, t *testing.T, cl *client.Store, pieceCheckPeriod time.Duration) { - // Add 8 pieces - allPcids := make(map[cid.Cid]struct{}) +func testNextPiecesPagination(ctx context.Context, t *testing.T, cl *client.Store, setPageSize func(int)) { + setPageSize(4) + + // Add 9 pieces seen := make(map[cid.Cid]int) for i := 1; i <= 9; i++ { - pieceCid := blocks.NewBlock([]byte(fmt.Sprintf("%d%d", time.Now().UnixMilli(), i))).Cid() - fmt.Println(pieceCid) + pieceCid := testutil.GenerateCid() di := model.DealInfo{ DealUuid: uuid.New().String(), ChainDealID: abi.DealID(i), @@ -144,8 +190,6 @@ func testNextPiecesPagination(ctx context.Context, t *testing.T, cl *client.Stor } err := cl.AddDealForPiece(ctx, pieceCid, di) require.NoError(t, err) - - allPcids[pieceCid] = struct{}{} } // expect to get 4 pieces @@ -191,19 +235,16 @@ func testNextPiecesPagination(ctx context.Context, t *testing.T, cl *client.Stor // Add 1 more piece for i := 1; i <= 1; i++ { - pieceCid := blocks.NewBlock([]byte(fmt.Sprintf("%d%d", time.Now().UnixMilli(), i))).Cid() - fmt.Println(pieceCid) + pieceCid := testutil.GenerateCid() di := model.DealInfo{ DealUuid: uuid.New().String(), - ChainDealID: abi.DealID(i), - SectorID: abi.SectorNumber(i), + ChainDealID: abi.DealID(100), + SectorID: abi.SectorNumber(100), PieceOffset: 0, PieceLength: 2048, } err := cl.AddDealForPiece(ctx, pieceCid, di) require.NoError(t, err) - - allPcids[pieceCid] = struct{}{} } // wait to reset the interval and start from scratch diff --git a/piecedirectory/piecedirectory.go b/piecedirectory/piecedirectory.go index 5b3aecef6..1e41cab14 100644 --- a/piecedirectory/piecedirectory.go +++ b/piecedirectory/piecedirectory.go @@ -3,7 +3,6 @@ package piecedirectory import ( "bufio" "context" - "encoding/binary" "errors" "fmt" "io" @@ -111,84 +110,6 @@ func (ps *PieceDirectory) GetOffsetSize(ctx context.Context, pieceCid cid.Cid, h return ps.store.GetOffsetSize(ctx, pieceCid, hash) } -func (ps *PieceDirectory) GetCarSize(ctx context.Context, pieceCid cid.Cid) (uint64, error) { - // Get the deals for the piece - dls, err := ps.GetPieceDeals(ctx, pieceCid) - if err != nil { - return 0, fmt.Errorf("getting piece deals for piece %s: %w", pieceCid, err) - } - - if len(dls) == 0 { - return 0, fmt.Errorf("no deals for piece %s in index: piece not found", pieceCid) - } - - // The size of the CAR should be the same for any deal, so just return the - // first non-zero CAR size - for _, dl := range dls { - if dl.CarLength > 0 { - return dl.CarLength, nil - } - } - - // There are no deals with a non-zero CAR size. - // The CAR size is zero if it's been imported from the dagstore (the - // dagstore doesn't store CAR size information). So instead work out the - // size of the CAR by getting the offset of the last section in the CAR - // file, then reading the section information. - - // Get the offset of the last section in the CAR file from the index. - var lastSectionOffset uint64 - idx, err := ps.GetIterableIndex(ctx, pieceCid) - if err != nil { - return 0, fmt.Errorf("getting index for piece %s: %w", pieceCid, err) - } - err = idx.ForEach(func(_ mh.Multihash, offset uint64) error { - if offset > lastSectionOffset { - lastSectionOffset = offset - } - return nil - }) - if err != nil { - return 0, fmt.Errorf("iterating index for piece %s: %w", pieceCid, err) - } - - // Get a reader over the piece - pieceReader, err := ps.GetPieceReader(ctx, pieceCid) - if err != nil { - return 0, fmt.Errorf("getting piece reader for piece %s: %w", pieceCid, err) - } - - // Seek to the last section - _, err = pieceReader.Seek(int64(lastSectionOffset), io.SeekStart) - if err != nil { - return 0, fmt.Errorf("seeking to offset %d in piece data: %w", lastSectionOffset, err) - } - - // A section consists of - // - - // Get - cr := &countReader{r: bufio.NewReader(pieceReader)} - dataLength, err := binary.ReadUvarint(cr) - if err != nil { - return 0, fmt.Errorf("reading CAR section length: %w", err) - } - - // The number of bytes in the uvarint that records - dataLengthUvarSize := cr.count - - // Get the size of the (unpadded) CAR file - unpaddedCarSize := lastSectionOffset + dataLengthUvarSize + dataLength - - // Write the CAR size back to the store so that it's cached for next time - err = ps.store.SetCarSize(ctx, pieceCid, unpaddedCarSize) - if err != nil { - log.Errorw("writing CAR size to local index directory store", "pieceCid", pieceCid, "err", err) - } - - return unpaddedCarSize, nil -} - func (ps *PieceDirectory) AddDealForPiece(ctx context.Context, pieceCid cid.Cid, dealInfo model.DealInfo) error { ctx, span := tracing.Tracer.Start(ctx, "pm.add_deal_for_piece") defer span.End() @@ -348,10 +269,6 @@ func (ps *PieceDirectory) RemoveDealForPiece(ctx context.Context, pieceCid cid.C return nil } -func (ps *PieceDirectory) MarkIndexErrored(ctx context.Context, pieceCid cid.Cid, err string) error { - return ps.store.MarkIndexErrored(ctx, pieceCid, err) -} - //func (ps *piecedirectory) deleteIndexForPiece(pieceCid cid.Cid) interface{} { // TODO: Maybe mark for GC instead of deleting immediately @@ -586,20 +503,6 @@ func (ps *PieceDirectory) GetBlockstore(ctx context.Context, pieceCid cid.Cid) ( return bs, nil } -// countReader just counts the number of bytes read -type countReader struct { - r *bufio.Reader - count uint64 -} - -func (c *countReader) ReadByte() (byte, error) { - b, err := c.r.ReadByte() - if err == nil { - c.count++ - } - return b, err -} - type SectorAccessorAsPieceReader struct { dagstore.SectorAccessor } diff --git a/piecedirectory/piecedirectory_test.go b/piecedirectory/piecedirectory_test.go index ea4017d7a..567f4c8bc 100644 --- a/piecedirectory/piecedirectory_test.go +++ b/piecedirectory/piecedirectory_test.go @@ -1,3 +1,6 @@ +//go:build test_lid +// +build test_lid + package piecedirectory import ( @@ -22,6 +25,24 @@ import ( "github.com/stretchr/testify/require" ) +var testCouchSettings = couchbase.DBSettings{ + ConnectString: "couchbase://localhost", + Auth: couchbase.DBSettingsAuth{ + Username: "Administrator", + Password: "boostdemo", + }, + PieceMetadataBucket: couchbase.DBSettingsBucket{ + RAMQuotaMB: 128, + }, + MultihashToPiecesBucket: couchbase.DBSettingsBucket{ + RAMQuotaMB: 128, + }, + PieceOffsetsBucket: couchbase.DBSettingsBucket{ + RAMQuotaMB: 128, + }, + TestMode: true, +} + func TestPieceDirectory(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 300*time.Second) defer cancel() @@ -29,20 +50,31 @@ func TestPieceDirectory(t *testing.T) { t.Run("leveldb", func(t *testing.T) { bdsvc, err := svc.NewLevelDB("") require.NoError(t, err) - testPieceDirectory(ctx, t, bdsvc) + addr := "localhost:8042" + + testPieceDirectory(ctx, t, bdsvc, addr) }) + t.Run("couchbase", func(t *testing.T) { // TODO: Unskip this test once the couchbase instance can be created // from a docker container in CI as part of the test t.Skip() svc.SetupCouchbase(t, testCouchSettings) bdsvc := svc.NewCouchbase(testCouchSettings) - testPieceDirectory(ctx, t, bdsvc) + addr := "localhost:8043" + testPieceDirectory(ctx, t, bdsvc, addr) + }) + + t.Run("yugabyte", func(t *testing.T) { + svc.SetupYugabyte(t) + + bdsvc := svc.NewYugabyte(svc.TestYugabyteSettings) + addr := "localhost:8044" + testPieceDirectory(ctx, t, bdsvc, addr) }) } -func testPieceDirectory(ctx context.Context, t *testing.T, bdsvc *svc.Service) { - addr := "localhost:8044" +func testPieceDirectory(ctx context.Context, t *testing.T, bdsvc *svc.Service, addr string) { err := bdsvc.Start(ctx, addr) require.NoError(t, err) @@ -63,10 +95,6 @@ func testPieceDirectory(ctx context.Context, t *testing.T, bdsvc *svc.Service) { testImportedIndex(ctx, t, cl) }) - t.Run("car file size", func(t *testing.T) { - testCarFileSize(ctx, t, cl) - }) - t.Run("flagging pieces", func(t *testing.T) { testFlaggingPieces(ctx, t, cl) }) @@ -252,55 +280,6 @@ func testImportedIndex(ctx context.Context, t *testing.T, cl *client.Store) { require.Equal(t, len(blk.RawData()), sz) } -// Verify that if the deal info has been imported from the DAG store, meaning -// it does not have CAR size information, GetCarSize will correctly calculate -// the CAR size from the index + piece data -func testCarFileSize(ctx context.Context, t *testing.T, cl *client.Store) { - // Create a random CAR file - carFilePath := CreateCarFile(t) - carFile, err := os.Open(carFilePath) - require.NoError(t, err) - defer carFile.Close() - - carReader, err := car.OpenReader(carFilePath) - require.NoError(t, err) - defer carReader.Close() - carv1Reader, err := carReader.DataReader() - require.NoError(t, err) - - // Read the CAR bytes - carBytes, err := io.ReadAll(carv1Reader) - require.NoError(t, err) - - // Any calls to get a reader over data should return a reader over the random CAR file - pr := CreateMockPieceReader(t, carv1Reader) - - recs := GetRecords(t, carv1Reader) - commpCalc := CalculateCommp(t, carv1Reader) - err = cl.AddIndex(ctx, commpCalc.PieceCID, recs, false) - require.NoError(t, err) - - // Add deal info for the piece without a CAR file - di := model.DealInfo{ - DealUuid: uuid.New().String(), - ChainDealID: 1, - SectorID: 1, - PieceOffset: 0, - PieceLength: commpCalc.PieceSize, - } - err = cl.AddDealForPiece(ctx, commpCalc.PieceCID, di) - require.NoError(t, err) - - // Verify that getting the size of the CAR file works correctly: - // There is no CAR size information in the deal info, so the piece - // directory should work it out from the index and piece data. - pm := NewPieceDirectory(cl, pr, 1) - pm.Start(ctx) - size, err := pm.GetCarSize(ctx, commpCalc.PieceCID) - require.NoError(t, err) - require.Equal(t, len(carBytes), int(size)) -} - func testFlaggingPieces(ctx context.Context, t *testing.T, cl *client.Store) { // Create a random CAR file carFilePath := CreateCarFile(t) diff --git a/piecedirectory/test_util.go b/piecedirectory/test_util.go index f3aac7840..cc4bef77c 100644 --- a/piecedirectory/test_util.go +++ b/piecedirectory/test_util.go @@ -9,7 +9,6 @@ import ( "github.com/filecoin-project/boost/piecedirectory/types" mock_piecedirectory "github.com/filecoin-project/boost/piecedirectory/types/mocks" "github.com/filecoin-project/boost/testutil" - "github.com/filecoin-project/boostd-data/couchbase" "github.com/filecoin-project/boostd-data/model" "github.com/filecoin-project/go-commp-utils/writer" "github.com/filecoin-project/go-state-types/abi" @@ -18,24 +17,6 @@ import ( "github.com/stretchr/testify/require" ) -var testCouchSettings = couchbase.DBSettings{ - ConnectString: "couchbase://localhost", - Auth: couchbase.DBSettingsAuth{ - Username: "Administrator", - Password: "boostdemo", - }, - PieceMetadataBucket: couchbase.DBSettingsBucket{ - RAMQuotaMB: 128, - }, - MultihashToPiecesBucket: couchbase.DBSettingsBucket{ - RAMQuotaMB: 128, - }, - PieceOffsetsBucket: couchbase.DBSettingsBucket{ - RAMQuotaMB: 128, - }, - TestMode: true, -} - // Get the index records from the CAR file func GetRecords(t *testing.T, reader car.SectionReader) []model.Record { _, err := reader.Seek(0, io.SeekStart) diff --git a/piecedirectory/types/mocks/piecedirectory.go b/piecedirectory/types/mocks/piecedirectory.go index 70b61ff8e..d44e2031d 100644 --- a/piecedirectory/types/mocks/piecedirectory.go +++ b/piecedirectory/types/mocks/piecedirectory.go @@ -338,20 +338,6 @@ func (mr *MockStoreMockRecorder) ListPieces(arg0 interface{}) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListPieces", reflect.TypeOf((*MockStore)(nil).ListPieces), arg0) } -// MarkIndexErrored mocks base method. -func (m *MockStore) MarkIndexErrored(arg0 context.Context, arg1 cid.Cid, arg2 string) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "MarkIndexErrored", arg0, arg1, arg2) - ret0, _ := ret[0].(error) - return ret0 -} - -// MarkIndexErrored indicates an expected call of MarkIndexErrored. -func (mr *MockStoreMockRecorder) MarkIndexErrored(arg0, arg1, arg2 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MarkIndexErrored", reflect.TypeOf((*MockStore)(nil).MarkIndexErrored), arg0, arg1, arg2) -} - // NextPiecesToCheck mocks base method. func (m *MockStore) NextPiecesToCheck(arg0 context.Context) ([]cid.Cid, error) { m.ctrl.T.Helper() @@ -424,20 +410,6 @@ func (mr *MockStoreMockRecorder) RemovePieceMetadata(arg0, arg1 interface{}) *go return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemovePieceMetadata", reflect.TypeOf((*MockStore)(nil).RemovePieceMetadata), arg0, arg1) } -// SetCarSize mocks base method. -func (m *MockStore) SetCarSize(arg0 context.Context, arg1 cid.Cid, arg2 uint64) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SetCarSize", arg0, arg1, arg2) - ret0, _ := ret[0].(error) - return ret0 -} - -// SetCarSize indicates an expected call of SetCarSize. -func (mr *MockStoreMockRecorder) SetCarSize(arg0, arg1, arg2 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetCarSize", reflect.TypeOf((*MockStore)(nil).SetCarSize), arg0, arg1, arg2) -} - // UnflagPiece mocks base method. func (m *MockStore) UnflagPiece(arg0 context.Context, arg1 cid.Cid) error { m.ctrl.T.Helper() diff --git a/piecedirectory/types/types.go b/piecedirectory/types/types.go index 8221dd5d3..8ad5dae8a 100644 --- a/piecedirectory/types/types.go +++ b/piecedirectory/types/types.go @@ -39,9 +39,7 @@ type Store interface { GetPieceMetadata(ctx context.Context, pieceCid cid.Cid) (model.Metadata, error) ListPieces(ctx context.Context) ([]cid.Cid, error) GetPieceDeals(ctx context.Context, pieceCid cid.Cid) ([]model.DealInfo, error) - SetCarSize(ctx context.Context, pieceCid cid.Cid, size uint64) error PiecesContainingMultihash(ctx context.Context, m multihash.Multihash) ([]cid.Cid, error) - MarkIndexErrored(ctx context.Context, pieceCid cid.Cid, err string) error RemoveDealForPiece(context.Context, cid.Cid, string) error RemovePieceMetadata(context.Context, cid.Cid) error RemoveIndexes(context.Context, cid.Cid) error