Skip to content

Commit

Permalink
feat(QAB-164): Update s3 steps to use aws golang sdk v2 (#104)
Browse files Browse the repository at this point in the history
  • Loading branch information
jordipuigbou authored Jul 1, 2022
1 parent c23b831 commit 74fc76e
Show file tree
Hide file tree
Showing 13 changed files with 272 additions and 140 deletions.
8 changes: 6 additions & 2 deletions .github/workflows/sonarcloud.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,10 @@ permissions:
contents: read
# Optional: allow read access to pull request. Use with `only-new-issues` option.
# pull-requests: read

env:
GOLANGCI_LINT_VERSION: v1.46.2

jobs:
analysis:
name: lint
Expand All @@ -30,12 +34,12 @@ jobs:
- name: Golangci increment
if: "!contains(github.ref, 'master')"
run: |
curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.44.2
curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin $GOLANGCI_LINT_VERSION
golangci-lint run -c .golangci.yml --timeout 5m --new-from-rev origin/master
- name: Golangci main
if: contains(github.ref, 'master')
run: |
curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.44.2
curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin $GOLANGCI_LINT_VERSION
golangci-lint run -c .golangci.yml --timeout 5m
- name: SonarCloud Scan
if: always()
Expand Down
28 changes: 28 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
UNAME := $(shell uname)
GO_PATH:=$(shell go env GOPATH)

LINTER_ARGS = run -c .golangci.yml --timeout 5m
GODOG_FORMAT = pretty
GOLANGCI_LINT_VERSION = v1.46.2

.PHONY: help
help: ## Show a list of available commands
@fgrep -h "##" $(MAKEFILE_LIST) | fgrep -v fgrep | sed -e 's/\\$$//' | sed -e 's/##//'

.PHONY: test-acceptance
test-acceptance: ## Run acceptance tests
go test ./test/acceptance -v --godog.format=$(GODOG_FORMAT)

.PHONY: test-run-tag
test-run-tag: ## Run feature from tag using variable TAG='<@tag_name>'
go test ./test/acceptance -v --godog.tags=$(TAG) --godog.format=$(GODOG_FORMAT)

.PHONY: download-tools
download-tools: ## Download all required tools to validate and generate documentation, code analysis...
@echo "Installing tools on $(GO_PATH)/bin"
go install github.com/golangci/golangci-lint/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION)
@echo "Tools installed"

.PHONY: lint
lint: ## Run static linting of source files. See .golangci.yml for options
golangci-lint $(LINTER_ARGS)
11 changes: 9 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ The library has a default configuration. However, these settings can be changed
## Example

The library includes a complete example with some scenarios for HTTP and DNS protocols in the directory [test/acceptance](test/acceptance).

### Environment

Using `docker-compose` to build test environment with all dependent services for each example: redis, rabbit, elasticsearch and minio:
Expand All @@ -39,10 +40,16 @@ docker-compose build

### Tests

Run tests:
Run acceptance tests:

```bash
docker-compose run --rm golium sh -c "make test-acceptance"
```

Run tests filtering by tag:

```bash
docker-compose run --rm golium sh -c "cd test/acceptance && go test -v ./..."
docker-compose run --rm golium sh -c "make test-run-tag TAG=@rabbit"
```

The examples contains the following directories:
Expand Down
19 changes: 18 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,11 @@ go 1.18

require (
github.com/AdguardTeam/dnsproxy v0.41.2
github.com/aws/aws-sdk-go v1.38.71
github.com/aws/aws-sdk-go-v2 v1.16.6
github.com/aws/aws-sdk-go-v2/config v1.15.12
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.17
github.com/aws/aws-sdk-go-v2/service/s3 v1.26.12
github.com/aws/smithy-go v1.12.0
github.com/cucumber/godog v0.12.0
github.com/cucumber/messages-go/v16 v16.0.1
github.com/elastic/go-elasticsearch/v7 v7.12.0
Expand All @@ -30,6 +34,19 @@ require (
github.com/aead/poly1305 v0.0.0-20180717145839-3fee0db0b635 // indirect
github.com/ameshkov/dnscrypt/v2 v2.2.3 // indirect
github.com/ameshkov/dnsstamps v1.0.3 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.3 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.12.7 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.7 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.13 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.7 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.14 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.4 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.3 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.8 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.7 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.7 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.11.10 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.16.8 // indirect
github.com/cespare/xxhash/v2 v2.1.1 // indirect
github.com/cheekybits/genny v1.0.0 // indirect
github.com/cucumber/gherkin-go/v19 v19.0.3 // indirect
Expand Down
42 changes: 38 additions & 4 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,42 @@ github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYU
github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
github.com/aws/aws-sdk-go v1.38.71 h1:aWhtgoOiDhBCfaAj9XbxzcyvjEAKovbtv7d5mCVBZXw=
github.com/aws/aws-sdk-go v1.38.71/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
github.com/aws/aws-sdk-go-v2 v1.16.6 h1:kzafGZYwkwVgLZ2zEX7P+vTwLli6uIMXF8aGjunN6UI=
github.com/aws/aws-sdk-go-v2 v1.16.6/go.mod h1:6CpKuLXg2w7If3ABZCl/qZ6rEgwtjZTn4eAf4RcEyuw=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.3 h1:S/ZBwevQkr7gv5YxONYpGQxlMFFYSRfz3RMcjsC9Qhk=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.3/go.mod h1:gNsR5CaXKmQSSzrmGxmwmct/r+ZBfbxorAuXYsj/M5Y=
github.com/aws/aws-sdk-go-v2/config v1.15.12 h1:D4mdf0cOSmZRgJe0DDOd1Qm6tkwHJ7r5i1lz0asa+AA=
github.com/aws/aws-sdk-go-v2/config v1.15.12/go.mod h1:oxRNnH11J580bxDEXyfTqfB3Auo2fxzhV052LD4HnyA=
github.com/aws/aws-sdk-go-v2/credentials v1.12.7 h1:e2DcCR0gP+T2zVj5eQPMQoRdxo+vd2p9BkpJ72BdyzA=
github.com/aws/aws-sdk-go-v2/credentials v1.12.7/go.mod h1:8b1nSHdDaKLho9VEK+K8WivifA/2K5pPm4sfI21NlQ8=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.7 h1:8yi2ORCwXpXEPnj0vP3DjYhejwDQD/5klgBoxXcKOxY=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.7/go.mod h1:81k6q0UUZj6AdQZ1E/VQ27cLrTUpJGraZR6/hVHRxjE=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.17 h1:9Y+OvoIvC8KocGNqbbBNDvMu0zsIgzKg3r+ZllSuH5Y=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.17/go.mod h1:z/7g6Z78jPG0l3HeShseUWzA+aBJDK4Mu5DkKkYdIW0=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.13 h1:WuQ1yGs3TMJgxpGVLspcsU/5q1omSA0SG6Cu0yZ4jkM=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.13/go.mod h1:wLLesU+LdMZDM3U0PP9vZXJW39zmD/7L4nY2pSrYZ/g=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.7 h1:mCeDDYeDXp3loo/xKi7nkx34eeh7q3n1mUBtzptsj8c=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.7/go.mod h1:93Uot80ddyVzSl//xEJreNKMhxntr71WtR3v/A1cRYk=
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.14 h1:bJv4Y9QOiW0GZPStgLgpGrpdfRDSR3XM4V4M3YCQRZo=
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.14/go.mod h1:R1HF8ZDdcRFfAGF+13En4LSHi2IrrNuPQCaxgWCeGyY=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.4 h1:wusoY1MJ9JNrPoX3n4kxY4MTIUivCiXvTYQbYh59yxs=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.4/go.mod h1:cHTMyJVEXRUZ25f8V+pq6CAwoYARarJRFGf3XH4eIxE=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.3 h1:4n4KCtv5SUoT5Er5XV41huuzrCqepxlW3SDI9qHQebc=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.3/go.mod h1:gkb2qADY+OHaGLKNTYxMaQNacfeyQpZ4csDTQMeFmcw=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.8 h1:BzBekDihMMeBexBhdK7xS3AIh2Jg/mECyLWO5RRwwHY=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.8/go.mod h1:a1BSeQI9IVr1j5Dwn73cdAKi4MdizTaV9YovUaHefGI=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.7 h1:M7/BzQNsu0XXiJRe3gUn8UA8tExF6kLMAfvo5PT/KJY=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.7/go.mod h1:HvVdEh/x4jsPBsjNvDy+MH3CDCPy4gTZEzFe2r4uJY8=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.7 h1:imb0NhTQZaTDSAQvgFyiZbKTwl0F+AkZL1ZNoEHtuQc=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.7/go.mod h1:V952z/yIT247sKya+CB+Ls3sxpB9jeBj5TkLraCGKGU=
github.com/aws/aws-sdk-go-v2/service/s3 v1.26.12 h1:/JTTdNObz+GygQqnbdBzummuxFIcuB6hbra1mqS+Wic=
github.com/aws/aws-sdk-go-v2/service/s3 v1.26.12/go.mod h1:eas8WnpTDJtCvEjRXAINFuox9TmEGeevxiUKEKv2tQ8=
github.com/aws/aws-sdk-go-v2/service/sso v1.11.10 h1:icon5WWg9Yg5nkB0pJF6bfKw6M0xozukeGKSNKtnqzw=
github.com/aws/aws-sdk-go-v2/service/sso v1.11.10/go.mod h1:UHxA35uPrCykRySBV5iSPZhZRlYnWSS2c/aaZVsoU94=
github.com/aws/aws-sdk-go-v2/service/sts v1.16.8 h1:GLGfpqX+1bmjNvUJkwB1ZaDpNFXQwJ3z9RkQDA58OBY=
github.com/aws/aws-sdk-go-v2/service/sts v1.16.8/go.mod h1:50YdFq1WIuxA0AGrygvYGucnNYrG24WYzu5fNp7lMgY=
github.com/aws/smithy-go v1.12.0 h1:gXpeZel/jPoWQ7OEmLIgCUnhkFftqNfwWUwAHSlp1v0=
github.com/aws/smithy-go v1.12.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
Expand Down Expand Up @@ -135,8 +169,9 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
Expand Down Expand Up @@ -462,7 +497,6 @@ golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/
golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
Expand Down
99 changes: 57 additions & 42 deletions steps/s3/client.go → steps/aws/s3/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,46 +15,54 @@
package s3steps

import (
"context"
"io"
"strings"

"github.com/aws/aws-sdk-go/aws"
aws_s "github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
"github.com/aws/aws-sdk-go-v2/aws"
s3manager "github.com/aws/aws-sdk-go-v2/feature/s3/manager"
"github.com/aws/aws-sdk-go-v2/service/s3"
)

type ClientFunctions interface {
NewSession(cfgs *aws.Config) (*aws_s.Session, error)
NewUploader(client *aws_s.Session) *s3manager.Uploader
New(cfgs aws.Config) *s3.Client
NewUploader(s3Client *s3.Client) *s3manager.Uploader
Upload(
ctx context.Context,
s3Client *s3.Client,
uploader *s3manager.Uploader,
bucket, key, message string,
) (*s3manager.UploadOutput, error)
New(client *aws_s.Session) *s3.S3
CreateBucket(
s3Client *s3.S3,
CreateBucket(ctx context.Context,
s3Client *s3.Client,
input *s3.CreateBucketInput,
) (*s3.CreateBucketOutput, error)
DeleteBucket(
s3Client *s3.S3,
ctx context.Context,
s3Client *s3.Client,
input *s3.DeleteBucketInput,
) (*s3.DeleteBucketOutput, error)
GetBucketLocation(
s3Client *s3.S3,
ctx context.Context,
s3Client *s3.Client,
input *s3.GetBucketLocationInput,
) (*s3.GetBucketLocationOutput, error)
HeadObject(
s3Client *s3.S3,
ctx context.Context,
s3Client *s3.Client,
input *s3.HeadObjectInput,
) (*s3.HeadObjectOutput, error)
NewDownloader(client *aws_s.Session) *s3manager.Downloader
NewDownloader(s3Client *s3.Client) *s3manager.Downloader
Download(
ctx context.Context,
s3Client *s3.Client,
downloader *s3manager.Downloader,
w io.WriterAt,
input *s3.GetObjectInput,
) (int64, error)
DeleteObject(s3Client *s3.S3,
DeleteObject(
ctx context.Context,
s3Client *s3.Client,
input *s3.DeleteObjectInput,
) (*s3.DeleteObjectOutput, error)
}
Expand All @@ -65,70 +73,77 @@ func NewS3ClientService() *ClientService {
return &ClientService{}
}

func (c ClientService) NewSession(cfgs *aws.Config) (*aws_s.Session, error) {
return aws_s.NewSession(cfgs)
func (c ClientService) New(cfgs aws.Config) *s3.Client {
return s3.NewFromConfig(cfgs)
}

func (c ClientService) NewUploader(client *aws_s.Session) *s3manager.Uploader {
return s3manager.NewUploader(client)
func (c ClientService) NewUploader(s3Client *s3.Client) *s3manager.Uploader {
return s3manager.NewUploader(s3Client)
}

func (c ClientService) Upload(
ctx context.Context,
s3Client *s3.Client,
uploader *s3manager.Uploader,
bucket, key, message string,
) (*s3manager.UploadOutput, error) {
return uploader.Upload(&s3manager.UploadInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
Body: strings.NewReader(message),
})
}

func (c ClientService) New(client *aws_s.Session) *s3.S3 {
return s3.New(client)
return uploader.Upload(
ctx,
&s3.PutObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
Body: strings.NewReader(message),
})
}

func (c ClientService) CreateBucket(
s3Client *s3.S3,
ctx context.Context,
s3Client *s3.Client,
input *s3.CreateBucketInput,
) (*s3.CreateBucketOutput, error) {
return s3Client.CreateBucket(input)
return s3Client.CreateBucket(ctx, input)
}

func (c ClientService) DeleteBucket(
s3Client *s3.S3,
ctx context.Context,
s3Client *s3.Client,
input *s3.DeleteBucketInput,
) (*s3.DeleteBucketOutput, error) {
return s3Client.DeleteBucket(input)
return s3Client.DeleteBucket(ctx, input)
}

func (c ClientService) GetBucketLocation(
s3Client *s3.S3,
ctx context.Context,
s3Client *s3.Client,
input *s3.GetBucketLocationInput,
) (*s3.GetBucketLocationOutput, error) {
return s3Client.GetBucketLocation(input)
return s3Client.GetBucketLocation(ctx, input)
}

func (c ClientService) HeadObject(
s3Client *s3.S3,
ctx context.Context,
s3Client *s3.Client,
input *s3.HeadObjectInput,
) (*s3.HeadObjectOutput, error) {
return s3Client.HeadObject(input)
return s3Client.HeadObject(ctx, input)
}

func (c ClientService) NewDownloader(client *aws_s.Session) *s3manager.Downloader {
return s3manager.NewDownloader(client)
func (c ClientService) NewDownloader(s3Client *s3.Client) *s3manager.Downloader {
return s3manager.NewDownloader(s3Client)
}

func (c ClientService) Download(
ctx context.Context,
s3Client *s3.Client,
downloader *s3manager.Downloader,
w io.WriterAt,
input *s3.GetObjectInput,
) (int64, error) {
return downloader.Download(w, input)
return downloader.Download(ctx, w, input)
}

func (c ClientService) DeleteObject(s3Client *s3.S3,
func (c ClientService) DeleteObject(
ctx context.Context,
s3Client *s3.Client,
input *s3.DeleteObjectInput,
) (*s3.DeleteObjectOutput, error) {
return s3Client.DeleteObject(input)
return s3Client.DeleteObject(ctx, input)
}
Loading

0 comments on commit 74fc76e

Please sign in to comment.