Fix yugabyte CI (#1433)

* fix: yugabyte tests in CI

* docker-compose.yml ; Dockerfile.test ; connect to `yugabyte` and not localhost

* add tag

* test lid

* make gen

* fixup

* move couchbase settings under build tag

---------

Co-authored-by: Anton Evangelatov <anton.evangelatov@gmail.com>

.circleci/config.yml

@@ -83,6 +83,19 @@ jobs:
           paths:
             - linux
 
+  lid-docker-compose:
+    description: 'Run LID integration tests'
+    machine:
+      image: ubuntu-2004:202104-01
+      resource_class: xlarge
+    steps:
+      - checkout
+      - run:
+          name: local index directory docker compose tests
+          command: |
+            set -x
+            make test-lid
+
   test:
     description: |
       Run go tests
@@ -322,7 +335,5 @@ workflows:
           suite: all
           target: "`go list ./... | grep -v boost/itests`"
 
-      - test:
-          name: local index directory
-          suite: all
-          cwd: "./extern/boostd-data"
+      - lid-docker-compose
+

Makefile

@@ -7,6 +7,7 @@ unexport GOFLAGS
 
 GOCC?=go
 
+ARCH?=$(shell arch)
 GOVERSION:=$(shell $(GOCC) version | tr ' ' '\n' | grep go1 | sed 's/^go//' | awk -F. '{printf "%d%03d%03d", $$1, $$2, $$3}')
 ifeq ($(shell expr $(GOVERSION) \< 1016000), 1)
 $(warning Your Golang version is go$(shell expr $(GOVERSION) / 1000000).$(shell expr $(GOVERSION) % 1000000 / 1000).$(shell expr $(GOVERSION) % 1000))
@@ -259,6 +260,9 @@ docker/all: $(lotus_build_cmd) docker/boost docker/booster-http docker/booster-b
 	docker/lotus docker/lotus-miner
 .PHONY: docker/all
 
+test-lid:
+	cd ./extern/boostd-data && ARCH=$(ARCH) docker-compose up --build --exit-code-from go-tests
+
 devnet/up:
 	rm -rf ./docker/devnet/data && docker compose -f ./docker/devnet/docker-compose.yaml up -d
 

extern/boostd-data/Dockerfile.test

@@ -0,0 +1,8 @@
+FROM golang:1.18-alpine
+
+WORKDIR /go/src/
+
+ENV CGO_ENABLED=0
+
+ENTRYPOINT ["go", "test"]
+CMD ["-v", "./..."]

extern/boostd-data/docker-compose.yml

@@ -0,0 +1,17 @@
+version: '3.7'
+
+services:
+
+  yugabyte:
+    image: public.ecr.aws/n6b0k8i7/yugabyte-test:${ARCH}-2.17.2.0
+    restart: on-failure
+
+  go-tests:
+    build:
+      context: .
+      dockerfile: ./Dockerfile.test
+    environment:
+      YUGABYTE_HOST: yugabyte
+    volumes:
+      - ./:/go/src/
+    command: -tags=test_lid -v -count=1 -p=1 ./... 

extern/boostd-data/svc/setup_yugabyte_test_util.go

@@ -1,83 +1,30 @@
 package svc
 
 import (
-	"github.com/davecgh/go-spew/spew"
-	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/api/types/container"
-	dockercl "github.com/docker/docker/client"
-	"github.com/docker/go-connections/nat"
+	"testing"
+	"time"
+
 	"github.com/filecoin-project/boostd-data/yugabyte"
 	"github.com/stretchr/testify/require"
 	"github.com/yugabyte/gocql"
 	"github.com/yugabyte/pgx/v4/pgxpool"
 	"golang.org/x/net/context"
-	"io"
-	"os"
-	"testing"
-	"time"
 )
 
 var TestYugabyteSettings = yugabyte.DBSettings{
-	Hosts:         []string{"127.0.0.1"},
-	ConnectString: "postgresql://postgres:postgres@localhost",
+	Hosts:         []string{"yugabyte"},
+	ConnectString: "postgresql://postgres:postgres@yugabyte:5433",
 }
 
 func SetupYugabyte(t *testing.T) {
 	ctx := context.Background()
-	cli, err := dockercl.NewClientWithOpts(dockercl.FromEnv)
-	require.NoError(t, err)
-
-	imageName := "public.ecr.aws/n6b0k8i7/yugabyte-test:aarch64-2.17.2.0"
-	out, err := cli.ImagePull(ctx, imageName, types.ImagePullOptions{})
-	require.NoError(t, err)
-
-	_, err = io.Copy(os.Stdout, out)
-	require.NoError(t, err)
-
-	tlog.Info("yugabyte docker container create...")
-	resp, err := cli.ContainerCreate(ctx, &container.Config{
-		Image: imageName,
-		ExposedPorts: nat.PortSet{
-			"7000": struct{}{},
-			"9000": struct{}{},
-			"5433": struct{}{},
-			"9042": struct{}{},
-		},
-	}, &container.HostConfig{
-		PortBindings: map[nat.Port][]nat.PortBinding{
-			"7000": {{HostIP: "127.0.0.1", HostPort: "7001"}},
-			"9000": {{HostIP: "127.0.0.1", HostPort: "9000"}},
-			// Yugabyte's postgres interface in docker runs on 5433
-			// whereas the standard postgres port is 5432
-			"5433": {{HostIP: "127.0.0.1", HostPort: "5432"}},
-			"9042": {{HostIP: "127.0.0.1", HostPort: "9042"}},
-		},
-	}, nil, nil, "")
-	require.NoError(t, err)
-	tlog.Info("yugabyte docker container created")
-
-	tlog.Info("yugabyte docker container start...")
-	err = cli.ContainerStart(ctx, resp.ID, types.ContainerStartOptions{})
-	require.NoError(t, err)
-	tlog.Info("yugabyte docker container started")
-
-	inspect, err := cli.ContainerInspect(ctx, resp.ID)
-	require.NoError(t, err)
-	spew.Dump(inspect)
-
-	t.Cleanup(func() {
-		tlog.Info("yugabyte docker container remove...")
-		err := cli.ContainerRemove(ctx, resp.ID, types.ContainerRemoveOptions{Force: true})
-		require.NoError(t, err)
-		tlog.Info("yugabyte docker container removed")
-	})
 
 	tlog.Info("wait for yugabyte start...")
 	awaitYugabyteUp(t, time.Minute)
 	tlog.Info("yugabyte started")
 
 	store := yugabyte.NewStore(TestYugabyteSettings)
-	err = store.Start(ctx)
+	err := store.Start(ctx)
 	require.NoError(t, err)
 
 	RecreateTables(ctx, t, store)

extern/boostd-data/svc/svc_test.go

@@ -1,3 +1,6 @@
+//go:build test_lid
+// +build test_lid
+
 package svc
 
 import (

piecedirectory/doctor_test.go

@@ -1,3 +1,6 @@
+//go:build test_lid
+// +build test_lid
+
 package piecedirectory
 
 import (

piecedirectory/piecedirectory_test.go

@@ -1,3 +1,6 @@
+//go:build test_lid
+// +build test_lid
+
 package piecedirectory
 
 import (
@@ -22,6 +25,24 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+var testCouchSettings = couchbase.DBSettings{
+	ConnectString: "couchbase://localhost",
+	Auth: couchbase.DBSettingsAuth{
+		Username: "Administrator",
+		Password: "boostdemo",
+	},
+	PieceMetadataBucket: couchbase.DBSettingsBucket{
+		RAMQuotaMB: 128,
+	},
+	MultihashToPiecesBucket: couchbase.DBSettingsBucket{
+		RAMQuotaMB: 128,
+	},
+	PieceOffsetsBucket: couchbase.DBSettingsBucket{
+		RAMQuotaMB: 128,
+	},
+	TestMode: true,
+}
+
 func TestPieceDirectory(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), 300*time.Second)
 	defer cancel()

piecedirectory/test_util.go

@@ -9,7 +9,6 @@ import (
 	"github.com/filecoin-project/boost/piecedirectory/types"
 	mock_piecedirectory "github.com/filecoin-project/boost/piecedirectory/types/mocks"
 	"github.com/filecoin-project/boost/testutil"
-	"github.com/filecoin-project/boostd-data/couchbase"
 	"github.com/filecoin-project/boostd-data/model"
 	"github.com/filecoin-project/go-commp-utils/writer"
 	"github.com/filecoin-project/go-state-types/abi"
@@ -18,24 +17,6 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-var testCouchSettings = couchbase.DBSettings{
-	ConnectString: "couchbase://localhost",
-	Auth: couchbase.DBSettingsAuth{
-		Username: "Administrator",
-		Password: "boostdemo",
-	},
-	PieceMetadataBucket: couchbase.DBSettingsBucket{
-		RAMQuotaMB: 128,
-	},
-	MultihashToPiecesBucket: couchbase.DBSettingsBucket{
-		RAMQuotaMB: 128,
-	},
-	PieceOffsetsBucket: couchbase.DBSettingsBucket{
-		RAMQuotaMB: 128,
-	},
-	TestMode: true,
-}
-
 // Get the index records from the CAR file
 func GetRecords(t *testing.T, reader car.SectionReader) []model.Record {
 	_, err := reader.Seek(0, io.SeekStart)