ci: Implement integration tests (#46)

* ci: Implement integration tests

Signed-off-by: Xuanwo <github@xuanwo.io>

* Fix build

Signed-off-by: Xuanwo <github@xuanwo.io>

* Fix test

Signed-off-by: Xuanwo <github@xuanwo.io>

* Fix go.mod

Signed-off-by: Xuanwo <github@xuanwo.io>

* There is no need for generate

Signed-off-by: Xuanwo <github@xuanwo.io>

.dockerignore

@@ -6,3 +6,4 @@
 .github/
 bin/
 docs/
+Makefile.env

.github/workflows/integration-test.yml

@@ -0,0 +1,39 @@
+name: "Integration Test"
+
+on: [push,pull_request]
+
+jobs:
+  integration_test:
+    name: Integration Test
+    runs-on: ubuntu-latest
+
+    services:
+      minio:
+        image: wktk/minio-server
+        ports:
+          - 9000:9000
+        env:
+          MINIO_ACCESS_KEY: "minioadmin"
+          MINIO_SECRET_KEY: "minioadmin"
+
+    steps:
+      - name: Set up Go 1.x
+        uses: actions/setup-go@v2
+        with:
+          go-version: "1.16"
+
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Setup test bucket
+        env:
+          AWS_ACCESS_KEY_ID: "minioadmin"
+          AWS_SECRET_ACCESS_KEY: "minioadmin"
+          AWS_EC2_METADATA_DISABLED: "true"
+        run: aws --endpoint-url http://127.0.0.1:9000/ s3 mb s3://test
+
+      - name: Test
+        env:
+          BEYOND_CTL_INTEGRATION_TEST: "on"
+          BEYOND_CTL_TEST_SERVICE: "s3://test/%s?credential=hmac:minioadmin:minioadmin&endpoint=http:127.0.0.1:9000&location=test&force_path_style=true"
+        run: make integration_test

.gitignore

@@ -19,4 +19,6 @@ bin/
 release/
 coverage/
 coverage.*
-tests/*.yaml
+tests/*.yaml
+
+Makefile.env

Makefile

@@ -2,6 +2,8 @@ SHELL := /bin/bash
 
 .PHONY: all check format vet build test generate tidy release
 
+-include Makefile.env
+
 VERSION := v0.0.1
 
 GOOS ?= $(shell go env GOOS)
@@ -20,10 +22,7 @@ format:
 vet:
 	go vet ./...
 
-generate:
-	go generate ./...
-
-build: tidy generate format vet
+build: tidy format vet
 	${GO_BUILD} -o bin/beyondctl ./cmd/beyondctl
 
 release:
@@ -47,9 +46,12 @@ release-windows-amd64: release
 release-all: release-linux-amd64 release-darwin-amd64 release-windows-amd64
 
 test:
-	go test -race -coverprofile=coverage.txt -covermode=atomic -v ./...
+	BEYOND_CTL_INTEGRATION_TEST=off go test -race -coverprofile=coverage.txt -covermode=atomic -v ./...
 	go tool cover -html="coverage.txt" -o "coverage.html"
 
+integration_test:
+	go test -race -coverprofile=coverage.txt -covermode=atomic -v ./cmd/beyondctl
+
 tidy:
 	go mod tidy
 	go mod verify

Makefile.env.example

@@ -0,0 +1,2 @@
+export BEYOND_CTL_INTEGRATION_TEST=on
+export BEYOND_CTL_TEST_SERVICE=s3://test/%s?credential=hmac:minioadmin:minioadmin&endpoint=http:127.0.0.1:9000&location=test&force_path_style=true

cmd/beyondctl/ls_test.go

@@ -0,0 +1,118 @@
+package main
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"math/rand"
+	"os"
+	"testing"
+
+	"github.com/google/uuid"
+
+	"github.com/beyondstorage/go-storage/v4/pkg/randbytes"
+	"github.com/beyondstorage/go-storage/v4/services"
+	"github.com/beyondstorage/go-storage/v4/types"
+)
+
+func getTestService(s string) string {
+	if s != "" {
+		s += "/"
+	}
+	return fmt.Sprintf(os.Getenv("BEYOND_CTL_TEST_SERVICE"), s)
+}
+
+type object struct {
+	path string
+	size int64
+}
+
+func setupLs(t *testing.T) (base string, obs []object) {
+	store, err := services.NewStoragerFromString(getTestService(""))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	base = uuid.NewString()
+
+	obs = make([]object, 0)
+
+	for i := 0; i < rand.Intn(1024); i++ {
+		path := fmt.Sprintf("%s/%s", base, uuid.NewString())
+
+		// Limit the content under 1MB.
+		size := rand.Intn(1024 * 1024)
+		bs := make([]byte, size)
+		_, err := io.ReadFull(randbytes.NewRand(), bs)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		_, err = store.Write(path, bytes.NewReader(bs), int64(size))
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		obs = append(obs, object{
+			path: path,
+			size: int64(size),
+		})
+	}
+
+	err = os.Setenv(
+		fmt.Sprintf("BEYOND_CTL_PROFILE_%s", base),
+		getTestService(base),
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return base, obs
+}
+
+func tearDownLs(t *testing.T, base string) {
+	store, err := services.NewStoragerFromString(os.Getenv("BEYOND_CTL_TEST_SERVICE"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	it, err := store.List(base)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for {
+		o, err := it.Next()
+		if err != nil && errors.Is(err, types.IterateDone) {
+			break
+		}
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		err = store.Delete(o.Path)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	err = os.Unsetenv(fmt.Sprintf("BEYOND_CTL_PROFILE_%s", base))
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestLs(t *testing.T) {
+	if os.Getenv("BEYOND_CTL_INTEGRATION_TEST") != "on" {
+		t.Skipf("BEYOND_CTL_INTEGRATION_TEST is not 'on', skipped")
+	}
+
+	base, _ := setupLs(t)
+	defer tearDownLs(t, base)
+
+	err := app.Run([]string{"beyondctl", "ls", "-l",
+		fmt.Sprintf("%s:", base)})
+	if err != nil {
+		t.Error(err)
+	}
+}

cmd/beyondctl/main.go

@@ -13,7 +13,7 @@ var (
 	Version string
 )
 
-var App = cli.App{
+var app = cli.App{
 	Name:        "beyondctl",
 	Description: "the command-line tool for all storage services",
 	Version:     Version,
@@ -27,7 +27,7 @@ var App = cli.App{
 }
 
 func main() {
-	err := App.Run(os.Args)
+	err := app.Run(os.Args)
 	if err != nil {
 		// FIXME: we need to respect platform style later.
 		os.Exit(1)

go.mod

@@ -21,6 +21,7 @@ require (
 	github.com/beyondstorage/go-service-uss/v2 v2.2.0
 	github.com/beyondstorage/go-storage/v4 v4.6.0
 	github.com/docker/go-units v0.4.0
+	github.com/google/uuid v1.3.0
 	github.com/panjf2000/ants/v2 v2.4.6
 	github.com/stretchr/testify v1.7.0
 	github.com/urfave/cli/v2 v2.3.0