chore: add a docker resource for minio (#169)

testhelper/docker/resource/minio.go

@@ -0,0 +1,119 @@
+package resource
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"net/http"
+
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+	"github.com/ory/dockertest/v3"
+
+	"github.com/rudderlabs/rudder-go-kit/httputil"
+	minioconfig "github.com/rudderlabs/rudder-go-kit/testhelper/docker/resource/minio"
+)
+
+type MinioResource struct {
+	BucketName      string
+	AccessKeyID     string
+	AccessKeySecret string
+	Endpoint        string
+	Region          string
+	Client          *minio.Client
+}
+
+func (mr *MinioResource) ToFileManagerConfig(prefix string) map[string]any {
+	return map[string]any{
+		"bucketName":       mr.BucketName,
+		"accessKeyID":      mr.AccessKeyID,
+		"secretAccessKey":  mr.AccessKeySecret,
+		"accessKey":        mr.AccessKeySecret,
+		"enableSSE":        false,
+		"prefix":           prefix,
+		"endPoint":         mr.Endpoint,
+		"s3ForcePathStyle": true,
+		"disableSSL":       true,
+		"useSSL":           false,
+		"region":           mr.Region,
+	}
+}
+
+func SetupMinio(pool *dockertest.Pool, d cleaner, opts ...func(*minioconfig.Config)) (*MinioResource, error) {
+	const (
+		bucket          = "rudder-saas"
+		region          = "us-east-1"
+		accessKeyId     = "MYACCESSKEY"
+		secretAccessKey = "MYSECRETKEY"
+		prefix          = "some-prefix"
+	)
+
+	c := &minioconfig.Config{
+		Tag:     "latest",
+		Options: []string{},
+	}
+	for _, opt := range opts {
+		opt(c)
+	}
+
+	minioContainer, err := pool.RunWithOptions(&dockertest.RunOptions{
+		Repository: "minio/minio",
+		Tag:        c.Tag,
+		Cmd:        []string{"server", "/data"},
+		Env: append([]string{
+			fmt.Sprintf("MINIO_ACCESS_KEY=%s", accessKeyId),
+			fmt.Sprintf("MINIO_SECRET_KEY=%s", secretAccessKey),
+			fmt.Sprintf("MINIO_SITE_REGION=%s", region),
+			"MINIO_API_SELECT_PARQUET=on",
+		}, c.Options...),
+	})
+	if err != nil {
+		return nil, fmt.Errorf("could not start resource: %s", err)
+	}
+	d.Cleanup(func() {
+		if err := pool.Purge(minioContainer); err != nil {
+			log.Printf("Could not purge minio resource: %s \n", err)
+		}
+	})
+
+	endpoint := fmt.Sprintf("localhost:%s", minioContainer.GetPort("9000/tcp"))
+
+	// check if minio server is up & running.
+	if err := pool.Retry(func() error {
+		url := fmt.Sprintf("http://%s/minio/health/live", endpoint)
+		resp, err := http.Get(url)
+		if err != nil {
+			return err
+		}
+		defer func() { httputil.CloseResponse(resp) }()
+
+		if resp.StatusCode != http.StatusOK {
+			return fmt.Errorf("status code not OK")
+		}
+		return nil
+	}); err != nil {
+		log.Fatalf("Could not connect to docker: %s", err)
+	}
+
+	client, err := minio.New(endpoint, &minio.Options{
+		Creds:  credentials.NewStaticV4(accessKeyId, secretAccessKey, ""),
+		Secure: false,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("could not create minio client: %w", err)
+	}
+
+	// creating bucket inside minio where testing will happen.
+	if err := client.MakeBucket(context.Background(), bucket, minio.MakeBucketOptions{Region: region}); err != nil {
+		return nil, fmt.Errorf("could not create bucket %q: %w", bucket, err)
+	}
+
+	return &MinioResource{
+		BucketName:      bucket,
+		AccessKeyID:     accessKeyId,
+		AccessKeySecret: secretAccessKey,
+		Endpoint:        endpoint,
+		Region:          region,
+		Client:          client,
+	}, nil
+}

testhelper/docker/resource/minio/config.go

@@ -0,0 +1,20 @@
+package minio
+
+type Opt func(*Config)
+
+func WithTag(tag string) Opt {
+	return func(c *Config) {
+		c.Tag = tag
+	}
+}
+
+func WithOptions(options ...string) Opt {
+	return func(c *Config) {
+		c.Options = options
+	}
+}
+
+type Config struct {
+	Tag     string
+	Options []string
+}

testhelper/docker/resource/minio_test.go

@@ -0,0 +1,51 @@
+package resource_test
+
+import (
+	"context"
+	"testing"
+
+	"github.com/minio/minio-go/v7"
+	"github.com/ory/dockertest/v3"
+	"github.com/stretchr/testify/require"
+
+	"github.com/rudderlabs/rudder-go-kit/filemanager"
+	"github.com/rudderlabs/rudder-go-kit/testhelper/docker/resource"
+)
+
+func TestMinioResource(t *testing.T) {
+	const prefix = "some-prefix"
+	const objectName = "minio.object"
+	pool, err := dockertest.NewPool("")
+	require.NoError(t, err)
+	minioResource, err := resource.SetupMinio(pool, t)
+	require.NoError(t, err)
+	_, err = minioResource.Client.FPutObject(context.Background(), minioResource.BucketName, prefix+"/"+objectName, "testdata/minio.object", minio.PutObjectOptions{})
+	require.NoError(t, err)
+	c := minioResource.ToFileManagerConfig("some-prefix")
+
+	t.Run("can use a minio filemanager", func(t *testing.T) {
+		fm, err := filemanager.New(&filemanager.Settings{
+			Provider: "MINIO",
+			Config:   c,
+		})
+		require.NoError(t, err)
+
+		it := fm.ListFilesWithPrefix(context.Background(), "", "some-prefix", 1)
+		items, err := it.Next()
+		require.NoError(t, err)
+		require.Len(t, items, 1)
+	})
+
+	t.Run("can use a s3 filemanager", func(t *testing.T) {
+		fm, err := filemanager.New(&filemanager.Settings{
+			Provider: "S3",
+			Config:   c,
+		})
+		require.NoError(t, err)
+
+		it := fm.ListFilesWithPrefix(context.Background(), "", "some-prefix", 1)
+		items, err := it.Next()
+		require.NoError(t, err)
+		require.Len(t, items, 1)
+	})
+}

testhelper/docker/resource/testdata/minio.object

@@ -0,0 +1 @@
+minio object