LID yugabyte db impl (#1391)

* feat: yugabyte db impl * feat: run yugabyte tests against a dockerized yugabyte * fix: use out own yugabyte docker image * fix: use yugabyte 2.17.2.0 docker image * feat: piece doctor yugabyte impl * fix: go mod tidy * refactor: remove SetCarSize as its not longer being used * refactor: remove functionality to mark index as errored (not being used) * feat: implement delete commands * refactor: consolidate test params * feat: add lid yugabyte config * fix: port map yugabyte postgres to standard port * Fix yugabyte CI (#1433) * fix: yugabyte tests in CI * docker-compose.yml ; Dockerfile.test ; connect to `yugabyte` and not localhost * add tag * test lid * make gen * fixup * move couchbase settings under build tag --------- Co-authored-by: Anton Evangelatov <anton.evangelatov@gmail.com> --------- Co-authored-by: Anton Evangelatov <anton.evangelatov@gmail.com>
filecoin-project · May 10, 2023 · f5de340 · f5de340
1 parent dac04ee
commit f5de340
Show file tree

Hide file tree

Showing 46 changed files with 1,637 additions and 521 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -83,6 +83,19 @@ jobs:
           paths:
             - linux
 
+  lid-docker-compose:
+    description: 'Run LID integration tests'
+    machine:
+      image: ubuntu-2004:202104-01
+      resource_class: xlarge
+    steps:
+      - checkout
+      - run:
+          name: local index directory docker compose tests
+          command: |
+            set -x
+            make test-lid
+
   test:
     description: |
       Run go tests
@@ -322,7 +335,5 @@ workflows:
           suite: all
           target: "`go list ./... | grep -v boost/itests`"
 
-      - test:
-          name: local index directory
-          suite: all
-          cwd: "./extern/boostd-data"
+      - lid-docker-compose
+
diff --git a/Makefile b/Makefile
@@ -7,6 +7,7 @@ unexport GOFLAGS
 
 GOCC?=go
 
+ARCH?=$(shell arch)
 GOVERSION:=$(shell $(GOCC) version | tr ' ' '\n' | grep go1 | sed 's/^go//' | awk -F. '{printf "%d%03d%03d", $$1, $$2, $$3}')
 ifeq ($(shell expr $(GOVERSION) \< 1016000), 1)
 $(warning Your Golang version is go$(shell expr $(GOVERSION) / 1000000).$(shell expr $(GOVERSION) % 1000000 / 1000).$(shell expr $(GOVERSION) % 1000))
@@ -259,6 +260,9 @@ docker/all: $(lotus_build_cmd) docker/boost docker/booster-http docker/booster-b
 	docker/lotus docker/lotus-miner
 .PHONY: docker/all
 
+test-lid:
+	cd ./extern/boostd-data && ARCH=$(ARCH) docker-compose up --build --exit-code-from go-tests
+
 devnet/up:
 	rm -rf ./docker/devnet/data && docker compose -f ./docker/devnet/docker-compose.yaml up -d
 

diff --git a/api/api.go b/api/api.go
@@ -55,8 +55,7 @@ type Boost interface {
 	BlockstoreGetSize(ctx context.Context, c cid.Cid) (int, error) //perm:read
 
 	// MethodGroup: PieceDirectory
-	PdBuildIndexForPieceCid(ctx context.Context, piececid cid.Cid) error        //perm:admin
-	PdMarkIndexErrored(ctx context.Context, piececid cid.Cid, err string) error //perm:admin
+	PdBuildIndexForPieceCid(ctx context.Context, piececid cid.Cid) error //perm:admin
 
 	// RuntimeSubsystems returns the subsystems that are enabled
 	// in this instance.

diff --git a/api/proxy_gen.go b/api/proxy_gen.go
diff --git a/build/openrpc/boost.json.gz b/build/openrpc/boost.json.gz
diff --git a/cmd/boostd/piecedir.go b/cmd/boostd/piecedir.go
@@ -16,7 +16,6 @@ var pieceDirCmd = &cli.Command{
 	Usage: "Manage Local Index Directory",
 	Subcommands: []*cli.Command{
 		pdIndexGenerate,
-		pdIndexMarkErroredCmd,
 	},
 }
 
@@ -56,45 +55,3 @@ var pdIndexGenerate = &cli.Command{
 		return nil
 	},
 }
-
-var pdIndexMarkErroredCmd = &cli.Command{
-	Name:  "mark-index",
-	Usage: "Mark an index errored for a given piece in the local index directory",
-	Flags: []cli.Flag{
-		&cli.StringFlag{
-			Name:     "piece-cid",
-			Usage:    "piece-cid of the index that will be marked as errored",
-			Required: true,
-		},
-		&cli.StringFlag{
-			Name:     "error",
-			Usage:    "error message",
-			Required: true,
-		},
-	},
-	Action: func(cctx *cli.Context) error {
-		ctx := lcli.ReqContext(cctx)
-
-		// parse piececid
-		piececid, err := cid.Decode(cctx.String("piece-cid"))
-		if err != nil {
-			return err
-		}
-
-		boostApi, ncloser, err := bcli.GetBoostAPI(cctx)
-		if err != nil {
-			return fmt.Errorf("getting boost api: %w", err)
-		}
-		defer ncloser()
-
-		errMsg := cctx.String("error")
-		err = boostApi.PdMarkIndexErrored(ctx, piececid, errMsg)
-		if err != nil {
-			return err
-		}
-
-		fmt.Printf("Marked %s as errored with \"%s\"\n", piececid, errMsg)
-
-		return nil
-	},
-}
diff --git a/documentation/en/api-v1-methods.md b/documentation/en/api-v1-methods.md
@@ -85,7 +85,6 @@
   * [OnlineBackup](#onlinebackup)
 * [Pd](#pd)
   * [PdBuildIndexForPieceCid](#pdbuildindexforpiececid)
-  * [PdMarkIndexErrored](#pdmarkindexerrored)
 * [Runtime](#runtime)
   * [RuntimeSubsystems](#runtimesubsystems)
 * [Sectors](#sectors)
@@ -1765,23 +1764,6 @@ Inputs:
 
 Response: `{}`
 
-### PdMarkIndexErrored
-
-
-Perms: admin
-
-Inputs:
-```json
-[
-  {
-    "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4"
-  },
-  "string value"
-]
-```
-
-Response: `{}`
-
 ## Runtime
 
 

diff --git a/extern/boostd-data/Dockerfile.test b/extern/boostd-data/Dockerfile.test
@@ -0,0 +1,8 @@
+FROM golang:1.18-alpine
+
+WORKDIR /go/src/
+
+ENV CGO_ENABLED=0
+
+ENTRYPOINT ["go", "test"]
+CMD ["-v", "./..."]
diff --git a/extern/boostd-data/client/client.go b/extern/boostd-data/client/client.go
@@ -26,8 +26,6 @@ type Store struct {
 		ListPieces                func(ctx context.Context) ([]cid.Cid, error)
 		GetPieceMetadata          func(ctx context.Context, pieceCid cid.Cid) (model.Metadata, error)
 		GetPieceDeals             func(context.Context, cid.Cid) ([]model.DealInfo, error)
-		SetCarSize                func(ctx context.Context, pieceCid cid.Cid, size uint64) error
-		MarkIndexErrored          func(context.Context, cid.Cid, string) error
 		IndexedAt                 func(context.Context, cid.Cid) (time.Time, error)
 		PiecesContainingMultihash func(context.Context, mh.Multihash) ([]cid.Cid, error)
 		RemoveDealForPiece        func(context.Context, cid.Cid, string) error
@@ -107,18 +105,10 @@ func (s *Store) PiecesContainingMultihash(ctx context.Context, m mh.Multihash) (
 	return s.client.PiecesContainingMultihash(ctx, m)
 }
 
-func (s *Store) MarkIndexErrored(ctx context.Context, pieceCid cid.Cid, err string) error {
-	return s.client.MarkIndexErrored(ctx, pieceCid, err)
-}
-
 func (s *Store) AddDealForPiece(ctx context.Context, pieceCid cid.Cid, dealInfo model.DealInfo) error {
 	return s.client.AddDealForPiece(ctx, pieceCid, dealInfo)
 }
 
-func (s *Store) SetCarSize(ctx context.Context, pieceCid cid.Cid, size uint64) error {
-	return s.client.SetCarSize(ctx, pieceCid, size)
-}
-
 func (s *Store) AddIndex(ctx context.Context, pieceCid cid.Cid, records []model.Record, isCompleteIndex bool) error {
 	log.Debugw("add-index", "piece-cid", pieceCid, "records", len(records))
 

diff --git a/extern/boostd-data/cmd/run.go b/extern/boostd-data/cmd/run.go
@@ -9,6 +9,7 @@ import (
 	"github.com/filecoin-project/boostd-data/shared/cliutil"
 	"github.com/filecoin-project/boostd-data/shared/tracing"
 	"github.com/filecoin-project/boostd-data/svc"
+	"github.com/filecoin-project/boostd-data/yugabyte"
 	"github.com/mitchellh/go-homedir"
 	"github.com/urfave/cli/v2"
 )
@@ -18,6 +19,7 @@ var runCmd = &cli.Command{
 	Subcommands: []*cli.Command{
 		leveldbCmd,
 		couchbaseCmd,
+		yugabyteCmd,
 	},
 }
 
@@ -106,6 +108,35 @@ var couchbaseCmd = &cli.Command{
 	},
 }
 
+var yugabyteCmd = &cli.Command{
+	Name:   "yugabyte",
+	Usage:  "Run boostd-data with a yugabyte database",
+	Before: before,
+	Flags: append([]cli.Flag{
+		&cli.StringSliceFlag{
+			Name:     "hosts",
+			Usage:    "yugabyte hosts to connect to over cassandra interface eg '127.0.0.1'",
+			Required: true,
+		},
+		&cli.StringFlag{
+			Name:     "connect-string",
+			Usage:    "postgres connect string eg 'postgresql://postgres:postgres@localhost'",
+			Required: true,
+		}},
+		runFlags...,
+	),
+	Action: func(cctx *cli.Context) error {
+		// Create a yugabyte data service
+		settings := yugabyte.DBSettings{
+			Hosts:         cctx.StringSlice("hosts"),
+			ConnectString: cctx.String("connect-string"),
+		}
+
+		bdsvc := svc.NewYugabyte(settings)
+		return runAction(cctx, "yugabyte", bdsvc)
+	},
+}
+
 func runAction(cctx *cli.Context, dbType string, store *svc.Service) error {
 	ctx := cliutil.ReqContext(cctx)
 

diff --git a/extern/boostd-data/couchbase/db.go b/extern/boostd-data/couchbase/db.go
@@ -337,42 +337,6 @@ func (db *DB) setPieceCidsForMultihash(ctx context.Context, mh multihash.Multiha
 	return nil
 }
 
-func (db *DB) SetCarSize(ctx context.Context, pieceCid cid.Cid, size uint64) error {
-	ctx, span := tracing.Tracer.Start(ctx, "db.set_car_size")
-	defer span.End()
-
-	return db.mutatePieceMetadata(ctx, pieceCid, "set-car-size", func(metadata CouchbaseMetadata) *CouchbaseMetadata {
-		// Set the car size on each deal (should be the same for all deals)
-		var deals []model.DealInfo
-		for _, dl := range metadata.Deals {
-			dl.CarLength = size
-
-			deals = append(deals, dl)
-		}
-		metadata.Deals = deals
-		return &metadata
-	})
-}
-
-func (db *DB) MarkIndexErrored(ctx context.Context, pieceCid cid.Cid, idxErr error) error {
-	ctx, span := tracing.Tracer.Start(ctx, "db.mark_piece_index_errored")
-	defer span.End()
-
-	return db.mutatePieceMetadata(ctx, pieceCid, "mark-index-errored", func(metadata CouchbaseMetadata) *CouchbaseMetadata {
-		// If the error was already set, don't overwrite it
-		if metadata.Error != "" {
-			// If the error state has already been set, don't over-write the existing error
-			return nil
-		}
-
-		// Set the error state
-		metadata.Error = idxErr.Error()
-		metadata.ErrorType = fmt.Sprintf("%T", idxErr)
-
-		return &metadata
-	})
-}
-
 func (db *DB) MarkIndexingComplete(ctx context.Context, pieceCid cid.Cid, blockCount int, isCompleteIndex bool) error {
 	ctx, span := tracing.Tracer.Start(ctx, "db.mark_indexing_complete")
 	defer span.End()
@@ -382,8 +346,6 @@ func (db *DB) MarkIndexingComplete(ctx context.Context, pieceCid cid.Cid, blockC
 		metadata.IndexedAt = time.Now()
 		metadata.CompleteIndex = isCompleteIndex
 		metadata.BlockCount = blockCount
-		metadata.Error = ""
-		metadata.ErrorType = ""
 		if metadata.Deals == nil {
 			metadata.Deals = []model.DealInfo{}
 		}

diff --git a/extern/boostd-data/couchbase/service.go b/extern/boostd-data/couchbase/service.go
@@ -2,7 +2,6 @@ package couchbase
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"time"
 
@@ -56,38 +55,6 @@ func (s *Store) AddDealForPiece(ctx context.Context, pieceCid cid.Cid, dealInfo
 	return s.db.AddDealForPiece(ctx, pieceCid, dealInfo)
 }
 
-func (s *Store) SetCarSize(ctx context.Context, pieceCid cid.Cid, size uint64) error {
-	log.Debugw("handle.set-car-size", "piece-cid", pieceCid, "size", size)
-
-	ctx, span := tracing.Tracer.Start(ctx, "store.set-car-size")
-	defer span.End()
-
-	defer func(now time.Time) {
-		log.Debugw("handled.set-car-size", "took", time.Since(now).String())
-	}(time.Now())
-
-	err := s.db.SetCarSize(ctx, pieceCid, size)
-	return normalizePieceCidError(pieceCid, err)
-}
-
-func (s *Store) MarkIndexErrored(ctx context.Context, pieceCid cid.Cid, idxErr string) error {
-	log.Debugw("handle.mark-piece-index-errored", "piece-cid", pieceCid, "err", idxErr)
-
-	ctx, span := tracing.Tracer.Start(ctx, "store.mark-piece-index-errored")
-	defer span.End()
-
-	defer func(now time.Time) {
-		log.Debugw("handled.mark-piece-index-errored", "took", time.Since(now).String())
-	}(time.Now())
-
-	err := s.db.MarkIndexErrored(ctx, pieceCid, errors.New(idxErr))
-	if err != nil {
-		return normalizePieceCidError(pieceCid, err)
-	}
-
-	return s.FlagPiece(ctx, pieceCid)
-}
-
 func (s *Store) GetOffsetSize(ctx context.Context, pieceCid cid.Cid, hash mh.Multihash) (*model.OffsetSize, error) {
 	log.Debugw("handle.get-offset-size", "piece-cid", pieceCid)
 

diff --git a/extern/boostd-data/docker-compose.yml b/extern/boostd-data/docker-compose.yml
@@ -0,0 +1,17 @@
+version: '3.7'
+
+services:
+
+  yugabyte:
+    image: public.ecr.aws/n6b0k8i7/yugabyte-test:${ARCH}-2.17.2.0
+    restart: on-failure
+
+  go-tests:
+    build:
+      context: .
+      dockerfile: ./Dockerfile.test
+    environment:
+      YUGABYTE_HOST: yugabyte
+    volumes:
+      - ./:/go/src/
+    command: -tags=test_lid -v -count=1 -p=1 ./...