Add bucket package

artefactual-labs · Apr 24, 2024 · 1803eeb · 1803eeb
1 parent e47c40f
commit 1803eeb
Show file tree

Hide file tree

Showing 4 changed files with 458 additions and 1,440 deletions.
diff --git a/bucket/bucket.go b/bucket/bucket.go
@@ -0,0 +1,106 @@
+// Package bucket provides functions to open gocloud buckets.
+package bucket
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/url"
+	"strings"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"gocloud.dev/blob"
+	"gocloud.dev/blob/s3blob"
+)
+
+type Config struct {
+	// URL specifies the connection string for a bucket as described in
+	// https://gocloud.dev/howto/blob/. If provided, this URL will be used to
+	// open the bucket directly.
+	URL string
+
+	// These fields are used for direct access to S3-compatible services when
+	// the URL field is not specified. They provide a more granular
+	// configuration using specific credentials and connection details.
+	Endpoint  string
+	Bucket    string
+	AccessKey string
+	SecretKey string
+	Token     string
+	Profile   string
+	Region    string
+	PathStyle bool
+}
+
+// WithConfig opens a bucket based on the provided configuration. It defaults to
+// using AWS SDK v2 via s3blob.OpenBucketV2 unless the URL field is specified,
+// in which case it uses blob.OpenBucket.
+func WithConfig(ctx context.Context, c *Config) (*blob.Bucket, error) {
+	if c == nil {
+		return nil, errors.New("config is undefined")
+	}
+
+	var (
+		b   *blob.Bucket
+		err error
+	)
+
+	if c.URL != "" {
+		b, err = openWithURL(ctx, c.URL)
+	} else {
+		b, err = openWithConfig(ctx, c)
+	}
+
+	return b, err
+}
+
+func openWithURL(ctx context.Context, url string) (*blob.Bucket, error) {
+	b, err := blob.OpenBucket(ctx, url)
+	if err != nil {
+		return nil, fmt.Errorf("open bucket from URL %q: %v", url, err)
+	}
+	return b, nil
+}
+
+func openWithConfig(ctx context.Context, c *Config) (*blob.Bucket, error) {
+	addr := c.Endpoint
+	if u, err := url.Parse(c.Endpoint); err == nil {
+		if !strings.HasPrefix(u.Scheme, "http") {
+			addr = "http://" + addr
+		}
+	}
+
+	awscfg, err := config.LoadDefaultConfig(
+		ctx,
+		config.WithSharedConfigProfile(c.Profile),
+		config.WithRegion(c.Region),
+		config.WithCredentialsProvider(
+			credentials.NewStaticCredentialsProvider(
+				c.AccessKey, c.SecretKey, c.Token,
+			),
+		),
+		config.WithEndpointResolverWithOptions(
+			aws.EndpointResolverWithOptionsFunc(
+				func(service, region string, options ...interface{}) (aws.Endpoint, error) {
+					return aws.Endpoint{URL: addr}, nil
+				},
+			),
+		),
+	)
+	if err != nil {
+		return nil, fmt.Errorf("load AWS default config: %v", err)
+	}
+
+	client := s3.NewFromConfig(awscfg, func(opts *s3.Options) {
+		opts.UsePathStyle = c.PathStyle
+	})
+	b, err := s3blob.OpenBucketV2(ctx, client, c.Bucket, nil)
+	if err != nil {
+		return nil, fmt.Errorf("open bucket: %v", err)
+	}
+
+	return b, nil
+}
diff --git a/bucket/bucket_test.go b/bucket/bucket_test.go
@@ -0,0 +1,116 @@
+package bucket_test
+
+import (
+	"context"
+	"testing"
+
+	s3v2 "github.com/aws/aws-sdk-go-v2/service/s3"
+	"gocloud.dev/blob"
+	_ "gocloud.dev/blob/fileblob"
+	_ "gocloud.dev/blob/memblob"
+	"gotest.tools/v3/assert"
+
+	bucket "go.artefactual.dev/tools/bucket"
+)
+
+func TestWithConfig(t *testing.T) {
+	t.Parallel()
+
+	type test struct {
+		config  *bucket.Config
+		errMsg  string
+		require func(*blob.Bucket)
+	}
+	tests := map[string]test{
+		"Opens URL-based config": {
+			config: &bucket.Config{
+				URL: "mem://",
+			},
+		},
+		"Opens attr-based config": {
+			config: &bucket.Config{
+				Endpoint:  "http://foobar:12345",
+				Bucket:    "name",
+				Region:    "region",
+				AccessKey: "access",
+				SecretKey: "secret",
+				PathStyle: true,
+			},
+			require: func(b *blob.Bucket) {
+				var client *s3v2.Client
+				assert.Equal(t, b.As(&client), true)
+
+				opts := client.Options()
+				assert.Equal(t, opts.Region, "region")
+				assert.Equal(t, opts.UsePathStyle, true)
+
+				_, err := client.ListBuckets(context.Background(), &s3v2.ListBucketsInput{})
+				assert.ErrorContains(t, err, "http://foobar:12345/?x-id=ListBuckets")
+			},
+		},
+		"Appends http if scheme is undefined": {
+			config: &bucket.Config{
+				Endpoint:  "foobar:12345",
+				Bucket:    "name",
+				Region:    "region",
+				AccessKey: "access",
+				SecretKey: "secret",
+			},
+			require: func(b *blob.Bucket) {
+				var client *s3v2.Client
+				assert.Equal(t, b.As(&client), true)
+
+				_, err := client.ListBuckets(context.Background(), &s3v2.ListBucketsInput{})
+				assert.ErrorContains(t, err, "http://foobar:12345/?x-id=ListBuckets")
+			},
+		},
+		"Rejects nil config": {
+			config: nil,
+			errMsg: "config is undefined",
+		},
+		"Rejects non-existent shared config profile": {
+			config: &bucket.Config{
+				Profile: "profile",
+			},
+			errMsg: "load AWS default config: failed to get shared config profile, profile",
+		},
+		"Rejects URL-based config with unknown scheme": {
+			config: &bucket.Config{
+				URL: "unknown://",
+			},
+			errMsg: `open bucket from URL "unknown://": open blob.Bucket: no driver registered for "unknown" for URL "unknown:"; available schemes: file, mem, s3`,
+		},
+		"Rejects bucket with empty name": {
+			config: &bucket.Config{
+				Endpoint:  "foobar:12345",
+				Bucket:    "",
+				Region:    "region",
+				AccessKey: "access",
+				SecretKey: "secret",
+			},
+			errMsg: "open bucket: s3blob.OpenBucket: bucketName is required",
+		},
+	}
+
+	for name, tc := range tests {
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+
+			b, err := bucket.WithConfig(context.Background(), tc.config)
+			if b != nil {
+				defer b.Close()
+			}
+
+			if tc.errMsg != "" {
+				assert.Assert(t, b == nil)
+				assert.Error(t, err, tc.errMsg)
+				return
+			}
+			assert.NilError(t, err)
+
+			if tc.require != nil {
+				tc.require(b)
+			}
+		})
+	}
+}
diff --git a/go.mod b/go.mod
@@ -1,42 +1,69 @@
 module go.artefactual.dev/tools
 
-go 1.21.1
+go 1.21
 
 require (
-	github.com/go-logr/logr v1.2.4
-	github.com/go-logr/zapr v1.2.4
-	github.com/google/go-cmp v0.5.9
-	go.temporal.io/api v1.21.0
-	go.temporal.io/sdk v1.24.0
-	go.uber.org/mock v0.3.0
-	go.uber.org/zap v1.24.0
-	golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df
-	gotest.tools/v3 v3.4.0
+	github.com/aws/aws-sdk-go-v2 v1.26.1
+	github.com/aws/aws-sdk-go-v2/config v1.27.11
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.11
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1
+	github.com/go-logr/logr v1.4.1
+	github.com/go-logr/zapr v1.3.0
+	github.com/google/go-cmp v0.6.0
+	go.temporal.io/api v1.29.2
+	go.temporal.io/sdk v1.26.0
+	go.uber.org/mock v0.4.0
+	go.uber.org/zap v1.27.0
+	gocloud.dev v0.37.0
+	golang.org/x/exp v0.0.0-20231219180239-dc181d75b848
+	gotest.tools/v3 v3.5.1
 )
 
 require (
-	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/aws/aws-sdk-go v1.50.36 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.9 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.20.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 // indirect
+	github.com/aws/smithy-go v1.20.2 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect
-	github.com/gogo/googleapis v1.4.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/gogo/status v1.1.1 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/mock v1.6.0 // indirect
-	github.com/golang/protobuf v1.5.3 // indirect
-	github.com/google/uuid v1.3.0 // indirect
-	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/google/wire v0.6.0 // indirect
+	github.com/googleapis/gax-go/v2 v2.12.2 // indirect
+	github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/pborman/uuid v1.2.1 // indirect
-	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/robfig/cron v1.2.0 // indirect
-	github.com/stretchr/objx v0.5.0 // indirect
-	github.com/stretchr/testify v1.8.3 // indirect
-	go.uber.org/atomic v1.9.0 // indirect
-	go.uber.org/multierr v1.6.0 // indirect
-	golang.org/x/net v0.10.0 // indirect
-	golang.org/x/sys v0.8.0 // indirect
-	golang.org/x/text v0.9.0 // indirect
-	golang.org/x/time v0.3.0 // indirect
-	google.golang.org/genproto v0.0.0-20230525154841-bd750badd5c6 // indirect
-	google.golang.org/grpc v1.55.0 // indirect
-	google.golang.org/protobuf v1.30.0 // indirect
+	github.com/stretchr/objx v0.5.2 // indirect
+	github.com/stretchr/testify v1.9.0 // indirect
+	go.opencensus.io v0.24.0 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	golang.org/x/net v0.24.0 // indirect
+	golang.org/x/sys v0.19.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/time v0.5.0 // indirect
+	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
+	google.golang.org/api v0.169.0 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20240311173647-c811ad7063a7 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240311173647-c811ad7063a7 // indirect
+	google.golang.org/grpc v1.62.1 // indirect
+	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )