Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(QAB-164): Update s3 steps to use aws golang sdk v2 #104

Merged
merged 8 commits into from
Jul 1, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions .github/workflows/sonarcloud.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,10 @@ permissions:
contents: read
# Optional: allow read access to pull request. Use with `only-new-issues` option.
# pull-requests: read

env:
GOLANGCI_LINT_VERSION: v1.46.2

jobs:
analysis:
name: lint
Expand All @@ -30,12 +34,12 @@ jobs:
- name: Golangci increment
if: "!contains(github.ref, 'master')"
run: |
curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.44.2
curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin $GOLANGCI_LINT_VERSION
golangci-lint run -c .golangci.yml --timeout 5m --new-from-rev origin/master
- name: Golangci main
if: contains(github.ref, 'master')
run: |
curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.44.2
curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin $GOLANGCI_LINT_VERSION
golangci-lint run -c .golangci.yml --timeout 5m
- name: SonarCloud Scan
if: always()
Expand Down
28 changes: 28 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
UNAME := $(shell uname)
GO_PATH:=$(shell go env GOPATH)

LINTER_ARGS = run -c .golangci.yml --timeout 5m
GODOG_FORMAT = pretty
GOLANGCI_LINT_VERSION = v1.46.2

.PHONY: help
help: ## Show a list of available commands
@fgrep -h "##" $(MAKEFILE_LIST) | fgrep -v fgrep | sed -e 's/\\$$//' | sed -e 's/##//'

.PHONY: test-acceptance
test-acceptance: ## Run acceptance tests
go test ./test/acceptance -v --godog.format=$(GODOG_FORMAT)

.PHONY: test-run-tag
test-run-tag: ## Run feature from tag using variable TAG='<@tag_name>'
go test ./test/acceptance -v --godog.tags=$(TAG) --godog.format=$(GODOG_FORMAT)

.PHONY: download-tools
download-tools: ## Download all required tools to validate and generate documentation, code analysis...
@echo "Installing tools on $(GO_PATH)/bin"
go install github.com/golangci/golangci-lint/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION)
@echo "Tools installed"

.PHONY: lint
lint: ## Run static linting of source files. See .golangci.yml for options
golangci-lint $(LINTER_ARGS)
11 changes: 9 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ The library has a default configuration. However, these settings can be changed
## Example

The library includes a complete example with some scenarios for HTTP and DNS protocols in the directory [test/acceptance](test/acceptance).

### Environment

Using `docker-compose` to build test environment with all dependent services for each example: redis, rabbit, elasticsearch and minio:
Expand All @@ -39,10 +40,16 @@ docker-compose build

### Tests

Run tests:
Run acceptance tests:

```bash
docker-compose run --rm golium sh -c "make test-acceptance"
```

Run tests filtering by tag:

```bash
docker-compose run --rm golium sh -c "cd test/acceptance && go test -v ./..."
docker-compose run --rm golium sh -c "make test-run-tag TAG=@rabbit"
```

The examples contains the following directories:
Expand Down
19 changes: 18 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,11 @@ go 1.18

require (
github.com/AdguardTeam/dnsproxy v0.41.2
github.com/aws/aws-sdk-go v1.38.71
github.com/aws/aws-sdk-go-v2 v1.16.6
github.com/aws/aws-sdk-go-v2/config v1.15.12
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.17
github.com/aws/aws-sdk-go-v2/service/s3 v1.26.12
github.com/aws/smithy-go v1.12.0
github.com/cucumber/godog v0.12.0
github.com/cucumber/messages-go/v16 v16.0.1
github.com/elastic/go-elasticsearch/v7 v7.12.0
Expand All @@ -30,6 +34,19 @@ require (
github.com/aead/poly1305 v0.0.0-20180717145839-3fee0db0b635 // indirect
github.com/ameshkov/dnscrypt/v2 v2.2.3 // indirect
github.com/ameshkov/dnsstamps v1.0.3 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.3 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.12.7 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.7 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.13 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.7 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.14 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.4 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.3 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.8 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.7 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.7 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.11.10 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.16.8 // indirect
github.com/cespare/xxhash/v2 v2.1.1 // indirect
github.com/cheekybits/genny v1.0.0 // indirect
github.com/cucumber/gherkin-go/v19 v19.0.3 // indirect
Expand Down
42 changes: 38 additions & 4 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,42 @@ github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYU
github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
github.com/aws/aws-sdk-go v1.38.71 h1:aWhtgoOiDhBCfaAj9XbxzcyvjEAKovbtv7d5mCVBZXw=
github.com/aws/aws-sdk-go v1.38.71/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
github.com/aws/aws-sdk-go-v2 v1.16.6 h1:kzafGZYwkwVgLZ2zEX7P+vTwLli6uIMXF8aGjunN6UI=
github.com/aws/aws-sdk-go-v2 v1.16.6/go.mod h1:6CpKuLXg2w7If3ABZCl/qZ6rEgwtjZTn4eAf4RcEyuw=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.3 h1:S/ZBwevQkr7gv5YxONYpGQxlMFFYSRfz3RMcjsC9Qhk=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.3/go.mod h1:gNsR5CaXKmQSSzrmGxmwmct/r+ZBfbxorAuXYsj/M5Y=
github.com/aws/aws-sdk-go-v2/config v1.15.12 h1:D4mdf0cOSmZRgJe0DDOd1Qm6tkwHJ7r5i1lz0asa+AA=
github.com/aws/aws-sdk-go-v2/config v1.15.12/go.mod h1:oxRNnH11J580bxDEXyfTqfB3Auo2fxzhV052LD4HnyA=
github.com/aws/aws-sdk-go-v2/credentials v1.12.7 h1:e2DcCR0gP+T2zVj5eQPMQoRdxo+vd2p9BkpJ72BdyzA=
github.com/aws/aws-sdk-go-v2/credentials v1.12.7/go.mod h1:8b1nSHdDaKLho9VEK+K8WivifA/2K5pPm4sfI21NlQ8=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.7 h1:8yi2ORCwXpXEPnj0vP3DjYhejwDQD/5klgBoxXcKOxY=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.7/go.mod h1:81k6q0UUZj6AdQZ1E/VQ27cLrTUpJGraZR6/hVHRxjE=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.17 h1:9Y+OvoIvC8KocGNqbbBNDvMu0zsIgzKg3r+ZllSuH5Y=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.17/go.mod h1:z/7g6Z78jPG0l3HeShseUWzA+aBJDK4Mu5DkKkYdIW0=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.13 h1:WuQ1yGs3TMJgxpGVLspcsU/5q1omSA0SG6Cu0yZ4jkM=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.13/go.mod h1:wLLesU+LdMZDM3U0PP9vZXJW39zmD/7L4nY2pSrYZ/g=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.7 h1:mCeDDYeDXp3loo/xKi7nkx34eeh7q3n1mUBtzptsj8c=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.7/go.mod h1:93Uot80ddyVzSl//xEJreNKMhxntr71WtR3v/A1cRYk=
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.14 h1:bJv4Y9QOiW0GZPStgLgpGrpdfRDSR3XM4V4M3YCQRZo=
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.14/go.mod h1:R1HF8ZDdcRFfAGF+13En4LSHi2IrrNuPQCaxgWCeGyY=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.4 h1:wusoY1MJ9JNrPoX3n4kxY4MTIUivCiXvTYQbYh59yxs=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.4/go.mod h1:cHTMyJVEXRUZ25f8V+pq6CAwoYARarJRFGf3XH4eIxE=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.3 h1:4n4KCtv5SUoT5Er5XV41huuzrCqepxlW3SDI9qHQebc=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.3/go.mod h1:gkb2qADY+OHaGLKNTYxMaQNacfeyQpZ4csDTQMeFmcw=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.8 h1:BzBekDihMMeBexBhdK7xS3AIh2Jg/mECyLWO5RRwwHY=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.8/go.mod h1:a1BSeQI9IVr1j5Dwn73cdAKi4MdizTaV9YovUaHefGI=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.7 h1:M7/BzQNsu0XXiJRe3gUn8UA8tExF6kLMAfvo5PT/KJY=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.7/go.mod h1:HvVdEh/x4jsPBsjNvDy+MH3CDCPy4gTZEzFe2r4uJY8=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.7 h1:imb0NhTQZaTDSAQvgFyiZbKTwl0F+AkZL1ZNoEHtuQc=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.7/go.mod h1:V952z/yIT247sKya+CB+Ls3sxpB9jeBj5TkLraCGKGU=
github.com/aws/aws-sdk-go-v2/service/s3 v1.26.12 h1:/JTTdNObz+GygQqnbdBzummuxFIcuB6hbra1mqS+Wic=
github.com/aws/aws-sdk-go-v2/service/s3 v1.26.12/go.mod h1:eas8WnpTDJtCvEjRXAINFuox9TmEGeevxiUKEKv2tQ8=
github.com/aws/aws-sdk-go-v2/service/sso v1.11.10 h1:icon5WWg9Yg5nkB0pJF6bfKw6M0xozukeGKSNKtnqzw=
github.com/aws/aws-sdk-go-v2/service/sso v1.11.10/go.mod h1:UHxA35uPrCykRySBV5iSPZhZRlYnWSS2c/aaZVsoU94=
github.com/aws/aws-sdk-go-v2/service/sts v1.16.8 h1:GLGfpqX+1bmjNvUJkwB1ZaDpNFXQwJ3z9RkQDA58OBY=
github.com/aws/aws-sdk-go-v2/service/sts v1.16.8/go.mod h1:50YdFq1WIuxA0AGrygvYGucnNYrG24WYzu5fNp7lMgY=
github.com/aws/smithy-go v1.12.0 h1:gXpeZel/jPoWQ7OEmLIgCUnhkFftqNfwWUwAHSlp1v0=
github.com/aws/smithy-go v1.12.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
Expand Down Expand Up @@ -135,8 +169,9 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
Expand Down Expand Up @@ -462,7 +497,6 @@ golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/
golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
Expand Down
99 changes: 57 additions & 42 deletions steps/s3/client.go → steps/aws/s3/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,46 +15,54 @@
package s3steps

import (
"context"
"io"
"strings"

"github.com/aws/aws-sdk-go/aws"
aws_s "github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
"github.com/aws/aws-sdk-go-v2/aws"
s3manager "github.com/aws/aws-sdk-go-v2/feature/s3/manager"
"github.com/aws/aws-sdk-go-v2/service/s3"
)

type ClientFunctions interface {
NewSession(cfgs *aws.Config) (*aws_s.Session, error)
NewUploader(client *aws_s.Session) *s3manager.Uploader
New(cfgs aws.Config) *s3.Client
NewUploader(s3Client *s3.Client) *s3manager.Uploader
Upload(
ctx context.Context,
s3Client *s3.Client,
uploader *s3manager.Uploader,
bucket, key, message string,
) (*s3manager.UploadOutput, error)
New(client *aws_s.Session) *s3.S3
CreateBucket(
s3Client *s3.S3,
CreateBucket(ctx context.Context,
s3Client *s3.Client,
input *s3.CreateBucketInput,
) (*s3.CreateBucketOutput, error)
DeleteBucket(
s3Client *s3.S3,
ctx context.Context,
s3Client *s3.Client,
input *s3.DeleteBucketInput,
) (*s3.DeleteBucketOutput, error)
GetBucketLocation(
s3Client *s3.S3,
ctx context.Context,
s3Client *s3.Client,
input *s3.GetBucketLocationInput,
) (*s3.GetBucketLocationOutput, error)
HeadObject(
s3Client *s3.S3,
ctx context.Context,
s3Client *s3.Client,
input *s3.HeadObjectInput,
) (*s3.HeadObjectOutput, error)
NewDownloader(client *aws_s.Session) *s3manager.Downloader
NewDownloader(s3Client *s3.Client) *s3manager.Downloader
Download(
ctx context.Context,
s3Client *s3.Client,
downloader *s3manager.Downloader,
w io.WriterAt,
input *s3.GetObjectInput,
) (int64, error)
DeleteObject(s3Client *s3.S3,
DeleteObject(
ctx context.Context,
s3Client *s3.Client,
input *s3.DeleteObjectInput,
) (*s3.DeleteObjectOutput, error)
}
Expand All @@ -65,70 +73,77 @@ func NewS3ClientService() *ClientService {
return &ClientService{}
}

func (c ClientService) NewSession(cfgs *aws.Config) (*aws_s.Session, error) {
return aws_s.NewSession(cfgs)
func (c ClientService) New(cfgs aws.Config) *s3.Client {
return s3.NewFromConfig(cfgs)
}

func (c ClientService) NewUploader(client *aws_s.Session) *s3manager.Uploader {
return s3manager.NewUploader(client)
func (c ClientService) NewUploader(s3Client *s3.Client) *s3manager.Uploader {
return s3manager.NewUploader(s3Client)
}

func (c ClientService) Upload(
ctx context.Context,
s3Client *s3.Client,
uploader *s3manager.Uploader,
bucket, key, message string,
) (*s3manager.UploadOutput, error) {
return uploader.Upload(&s3manager.UploadInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
Body: strings.NewReader(message),
})
}

func (c ClientService) New(client *aws_s.Session) *s3.S3 {
return s3.New(client)
return uploader.Upload(
ctx,
&s3.PutObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
Body: strings.NewReader(message),
})
}

func (c ClientService) CreateBucket(
s3Client *s3.S3,
ctx context.Context,
s3Client *s3.Client,
input *s3.CreateBucketInput,
) (*s3.CreateBucketOutput, error) {
return s3Client.CreateBucket(input)
return s3Client.CreateBucket(ctx, input)
}

func (c ClientService) DeleteBucket(
s3Client *s3.S3,
ctx context.Context,
s3Client *s3.Client,
input *s3.DeleteBucketInput,
) (*s3.DeleteBucketOutput, error) {
return s3Client.DeleteBucket(input)
return s3Client.DeleteBucket(ctx, input)
}

func (c ClientService) GetBucketLocation(
s3Client *s3.S3,
ctx context.Context,
s3Client *s3.Client,
input *s3.GetBucketLocationInput,
) (*s3.GetBucketLocationOutput, error) {
return s3Client.GetBucketLocation(input)
return s3Client.GetBucketLocation(ctx, input)
}

func (c ClientService) HeadObject(
s3Client *s3.S3,
ctx context.Context,
s3Client *s3.Client,
input *s3.HeadObjectInput,
) (*s3.HeadObjectOutput, error) {
return s3Client.HeadObject(input)
return s3Client.HeadObject(ctx, input)
}

func (c ClientService) NewDownloader(client *aws_s.Session) *s3manager.Downloader {
return s3manager.NewDownloader(client)
func (c ClientService) NewDownloader(s3Client *s3.Client) *s3manager.Downloader {
return s3manager.NewDownloader(s3Client)
}

func (c ClientService) Download(
ctx context.Context,
s3Client *s3.Client,
downloader *s3manager.Downloader,
w io.WriterAt,
input *s3.GetObjectInput,
) (int64, error) {
return downloader.Download(w, input)
return downloader.Download(ctx, w, input)
}

func (c ClientService) DeleteObject(s3Client *s3.S3,
func (c ClientService) DeleteObject(
ctx context.Context,
s3Client *s3.Client,
input *s3.DeleteObjectInput,
) (*s3.DeleteObjectOutput, error) {
return s3Client.DeleteObject(input)
return s3Client.DeleteObject(ctx, input)
}
Loading