LID yugabyte db impl (#1391)

* feat: yugabyte db impl * feat: run yugabyte tests against a dockerized yugabyte * fix: use out own yugabyte docker image * fix: use yugabyte 2.17.2.0 docker image * feat: piece doctor yugabyte impl * fix: go mod tidy * refactor: remove SetCarSize as its not longer being used * refactor: remove functionality to mark index as errored (not being used) * feat: implement delete commands * refactor: consolidate test params * feat: add lid yugabyte config * fix: port map yugabyte postgres to standard port * Fix yugabyte CI (#1433) * fix: yugabyte tests in CI * docker-compose.yml ; Dockerfile.test ; connect to `yugabyte` and not localhost * add tag * test lid * make gen * fixup * move couchbase settings under build tag --------- Co-authored-by: Anton Evangelatov <anton.evangelatov@gmail.com> --------- Co-authored-by: Anton Evangelatov <anton.evangelatov@gmail.com>
filecoin-project · Jul 20, 2023 · 97ea9ba · 97ea9ba
1 parent e01a047
commit 97ea9ba
Show file tree

Hide file tree

Showing 37 changed files with 447 additions and 740 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -340,7 +340,5 @@ workflows:
           suite: all
           target: "`go list ./... | grep -v boost/itests`"
 
-      - test:
-          name: local index directory
-          suite: all
-          cwd: "./extern/boostd-data"
+      - lid-docker-compose
+
diff --git a/api/api.go b/api/api.go
@@ -55,8 +55,7 @@ type Boost interface {
 	BlockstoreGetSize(ctx context.Context, c cid.Cid) (int, error) //perm:read
 
 	// MethodGroup: PieceDirectory
-	PdBuildIndexForPieceCid(ctx context.Context, piececid cid.Cid) error        //perm:admin
-	PdMarkIndexErrored(ctx context.Context, piececid cid.Cid, err string) error //perm:admin
+	PdBuildIndexForPieceCid(ctx context.Context, piececid cid.Cid) error //perm:admin
 
 	// RuntimeSubsystems returns the subsystems that are enabled
 	// in this instance.

diff --git a/api/proxy_gen.go b/api/proxy_gen.go
diff --git a/build/openrpc/boost.json.gz b/build/openrpc/boost.json.gz
diff --git a/cmd/boostd/piecedir.go b/cmd/boostd/piecedir.go
@@ -16,7 +16,6 @@ var pieceDirCmd = &cli.Command{
 	Usage: "Manage Local Index Directory",
 	Subcommands: []*cli.Command{
 		pdIndexGenerate,
-		pdIndexMarkErroredCmd,
 	},
 }
 
@@ -56,45 +55,3 @@ var pdIndexGenerate = &cli.Command{
 		return nil
 	},
 }
-
-var pdIndexMarkErroredCmd = &cli.Command{
-	Name:  "mark-index",
-	Usage: "Mark an index errored for a given piece in the local index directory",
-	Flags: []cli.Flag{
-		&cli.StringFlag{
-			Name:     "piece-cid",
-			Usage:    "piece-cid of the index that will be marked as errored",
-			Required: true,
-		},
-		&cli.StringFlag{
-			Name:     "error",
-			Usage:    "error message",
-			Required: true,
-		},
-	},
-	Action: func(cctx *cli.Context) error {
-		ctx := lcli.ReqContext(cctx)
-
-		// parse piececid
-		piececid, err := cid.Decode(cctx.String("piece-cid"))
-		if err != nil {
-			return err
-		}
-
-		boostApi, ncloser, err := bcli.GetBoostAPI(cctx)
-		if err != nil {
-			return fmt.Errorf("getting boost api: %w", err)
-		}
-		defer ncloser()
-
-		errMsg := cctx.String("error")
-		err = boostApi.PdMarkIndexErrored(ctx, piececid, errMsg)
-		if err != nil {
-			return err
-		}
-
-		fmt.Printf("Marked %s as errored with \"%s\"\n", piececid, errMsg)
-
-		return nil
-	},
-}
diff --git a/documentation/en/api-v1-methods.md b/documentation/en/api-v1-methods.md
@@ -87,7 +87,6 @@
   * [OnlineBackup](#onlinebackup)
 * [Pd](#pd)
   * [PdBuildIndexForPieceCid](#pdbuildindexforpiececid)
-  * [PdMarkIndexErrored](#pdmarkindexerrored)
 * [Runtime](#runtime)
   * [RuntimeSubsystems](#runtimesubsystems)
 * [Sectors](#sectors)
@@ -1791,23 +1790,6 @@ Inputs:
 
 Response: `{}`
 
-### PdMarkIndexErrored
-
-
-Perms: admin
-
-Inputs:
-```json
-[
-  {
-    "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4"
-  },
-  "string value"
-]
-```
-
-Response: `{}`
-
 ## Runtime
 
 

diff --git a/extern/boostd-data/client/client.go b/extern/boostd-data/client/client.go
@@ -26,8 +26,6 @@ type Store struct {
 		ListPieces                func(ctx context.Context) ([]cid.Cid, error)
 		GetPieceMetadata          func(ctx context.Context, pieceCid cid.Cid) (model.Metadata, error)
 		GetPieceDeals             func(context.Context, cid.Cid) ([]model.DealInfo, error)
-		SetCarSize                func(ctx context.Context, pieceCid cid.Cid, size uint64) error
-		MarkIndexErrored          func(context.Context, cid.Cid, string) error
 		IndexedAt                 func(context.Context, cid.Cid) (time.Time, error)
 		PiecesContainingMultihash func(context.Context, mh.Multihash) ([]cid.Cid, error)
 		RemoveDealForPiece        func(context.Context, cid.Cid, string) error
@@ -110,18 +108,10 @@ func (s *Store) PiecesContainingMultihash(ctx context.Context, m mh.Multihash) (
 	return s.client.PiecesContainingMultihash(ctx, m)
 }
 
-func (s *Store) MarkIndexErrored(ctx context.Context, pieceCid cid.Cid, err string) error {
-	return s.client.MarkIndexErrored(ctx, pieceCid, err)
-}
-
 func (s *Store) AddDealForPiece(ctx context.Context, pieceCid cid.Cid, dealInfo model.DealInfo) error {
 	return s.client.AddDealForPiece(ctx, pieceCid, dealInfo)
 }
 
-func (s *Store) SetCarSize(ctx context.Context, pieceCid cid.Cid, size uint64) error {
-	return s.client.SetCarSize(ctx, pieceCid, size)
-}
-
 func (s *Store) AddIndex(ctx context.Context, pieceCid cid.Cid, records []model.Record, isCompleteIndex bool) error {
 	log.Debugw("add-index", "piece-cid", pieceCid, "records", len(records))
 

diff --git a/extern/boostd-data/cmd/run.go b/extern/boostd-data/cmd/run.go
@@ -9,6 +9,7 @@ import (
 	"github.com/filecoin-project/boostd-data/shared/cliutil"
 	"github.com/filecoin-project/boostd-data/shared/tracing"
 	"github.com/filecoin-project/boostd-data/svc"
+	"github.com/filecoin-project/boostd-data/yugabyte"
 	"github.com/mitchellh/go-homedir"
 	"github.com/urfave/cli/v2"
 )
@@ -18,6 +19,7 @@ var runCmd = &cli.Command{
 	Subcommands: []*cli.Command{
 		leveldbCmd,
 		couchbaseCmd,
+		yugabyteCmd,
 	},
 }
 
@@ -106,6 +108,35 @@ var couchbaseCmd = &cli.Command{
 	},
 }
 
+var yugabyteCmd = &cli.Command{
+	Name:   "yugabyte",
+	Usage:  "Run boostd-data with a yugabyte database",
+	Before: before,
+	Flags: append([]cli.Flag{
+		&cli.StringSliceFlag{
+			Name:     "hosts",
+			Usage:    "yugabyte hosts to connect to over cassandra interface eg '127.0.0.1'",
+			Required: true,
+		},
+		&cli.StringFlag{
+			Name:     "connect-string",
+			Usage:    "postgres connect string eg 'postgresql://postgres:postgres@localhost'",
+			Required: true,
+		}},
+		runFlags...,
+	),
+	Action: func(cctx *cli.Context) error {
+		// Create a yugabyte data service
+		settings := yugabyte.DBSettings{
+			Hosts:         cctx.StringSlice("hosts"),
+			ConnectString: cctx.String("connect-string"),
+		}
+
+		bdsvc := svc.NewYugabyte(settings)
+		return runAction(cctx, "yugabyte", bdsvc)
+	},
+}
+
 func runAction(cctx *cli.Context, dbType string, store *svc.Service) error {
 	ctx := cliutil.ReqContext(cctx)
 

diff --git a/extern/boostd-data/couchbase/db.go b/extern/boostd-data/couchbase/db.go
@@ -337,42 +337,6 @@ func (db *DB) setPieceCidsForMultihash(ctx context.Context, mh multihash.Multiha
 	return nil
 }
 
-func (db *DB) SetCarSize(ctx context.Context, pieceCid cid.Cid, size uint64) error {
-	ctx, span := tracing.Tracer.Start(ctx, "db.set_car_size")
-	defer span.End()
-
-	return db.mutatePieceMetadata(ctx, pieceCid, "set-car-size", func(metadata CouchbaseMetadata) *CouchbaseMetadata {
-		// Set the car size on each deal (should be the same for all deals)
-		var deals []model.DealInfo
-		for _, dl := range metadata.Deals {
-			dl.CarLength = size
-
-			deals = append(deals, dl)
-		}
-		metadata.Deals = deals
-		return &metadata
-	})
-}
-
-func (db *DB) MarkIndexErrored(ctx context.Context, pieceCid cid.Cid, idxErr error) error {
-	ctx, span := tracing.Tracer.Start(ctx, "db.mark_piece_index_errored")
-	defer span.End()
-
-	return db.mutatePieceMetadata(ctx, pieceCid, "mark-index-errored", func(metadata CouchbaseMetadata) *CouchbaseMetadata {
-		// If the error was already set, don't overwrite it
-		if metadata.Error != "" {
-			// If the error state has already been set, don't over-write the existing error
-			return nil
-		}
-
-		// Set the error state
-		metadata.Error = idxErr.Error()
-		metadata.ErrorType = fmt.Sprintf("%T", idxErr)
-
-		return &metadata
-	})
-}
-
 func (db *DB) MarkIndexingComplete(ctx context.Context, pieceCid cid.Cid, blockCount int, isCompleteIndex bool) error {
 	ctx, span := tracing.Tracer.Start(ctx, "db.mark_indexing_complete")
 	defer span.End()
@@ -382,8 +346,6 @@ func (db *DB) MarkIndexingComplete(ctx context.Context, pieceCid cid.Cid, blockC
 		metadata.IndexedAt = time.Now()
 		metadata.CompleteIndex = isCompleteIndex
 		metadata.BlockCount = blockCount
-		metadata.Error = ""
-		metadata.ErrorType = ""
 		if metadata.Deals == nil {
 			metadata.Deals = []model.DealInfo{}
 		}

diff --git a/extern/boostd-data/go.mod b/extern/boostd-data/go.mod
@@ -18,6 +18,7 @@ require (
 	github.com/ipfs/go-ds-leveldb v0.5.0
 	github.com/ipfs/go-log/v2 v2.5.1
 	github.com/ipld/go-car/v2 v2.4.2-0.20220707083113-89de8134e58e
+	github.com/jackc/pgtype v1.10.0
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/multiformats/go-multicodec v0.6.0
 	github.com/multiformats/go-multihash v0.2.1
@@ -53,6 +54,7 @@ require (
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/gorilla/websocket v1.5.0 // indirect
+	github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect
 	github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect
 	github.com/ipfs/bbloom v0.0.4 // indirect
 	github.com/ipfs/go-block-format v0.0.3 // indirect
@@ -77,6 +79,13 @@ require (
 	github.com/ipfs/go-verifcid v0.0.1 // indirect
 	github.com/ipld/go-codec-dagpb v1.3.2 // indirect
 	github.com/ipld/go-ipld-prime v0.18.0 // indirect
+	github.com/jackc/chunkreader/v2 v2.0.1 // indirect
+	github.com/jackc/pgconn v1.11.0 // indirect
+	github.com/jackc/pgio v1.0.0 // indirect
+	github.com/jackc/pgpassfile v1.0.0 // indirect
+	github.com/jackc/pgproto3/v2 v2.2.0 // indirect
+	github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect
+	github.com/jackc/puddle v1.2.1 // indirect
 	github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c // indirect
 	github.com/jbenet/goprocess v0.1.4 // indirect
 	github.com/klauspost/cpuid/v2 v2.1.1 // indirect
@@ -117,10 +126,12 @@ require (
 	golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e // indirect
 	golang.org/x/exp v0.0.0-20220916125017-b168a2c6b86b // indirect
 	golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8 // indirect
+	golang.org/x/text v0.3.7 // indirect
 	golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f // indirect
 	google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect
 	google.golang.org/grpc v1.47.0 // indirect
 	google.golang.org/protobuf v1.28.1 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	gotest.tools/v3 v3.4.0 // indirect
 	lukechampine.com/blake3 v1.1.7 // indirect