Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add express integration test #2402

Merged
merged 1 commit into from
Dec 1, 2023
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions service/internal/integrationtest/go.mod
Original file line number Diff line number Diff line change
@@ -3,6 +3,7 @@ module github.com/aws/aws-sdk-go-v2/service/internal/integrationtest
require (
github.com/aws/aws-sdk-go-v2 v1.23.5
github.com/aws/aws-sdk-go-v2/config v1.25.11
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.4
github.com/aws/aws-sdk-go-v2/service/acm v1.22.2
github.com/aws/aws-sdk-go-v2/service/apigateway v1.21.2
github.com/aws/aws-sdk-go-v2/service/applicationautoscaling v1.25.2
@@ -115,6 +116,8 @@ replace github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream => ../../../aws/pr

replace github.com/aws/aws-sdk-go-v2/config => ../../../config/

replace github.com/aws/aws-sdk-go-v2/feature/s3/manager => ../../../feature/s3/manager/

replace github.com/aws/aws-sdk-go-v2/credentials => ../../../credentials/

replace github.com/aws/aws-sdk-go-v2/feature/ec2/imds => ../../../feature/ec2/imds/
227 changes: 227 additions & 0 deletions service/internal/integrationtest/s3/express_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,227 @@
//go:build integration
// +build integration

package s3

import (
"bytes"
"context"
"fmt"
"io"
"log"
"net/http"
"net/url"
"strings"
"testing"

"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
"github.com/aws/aws-sdk-go-v2/internal/awstesting"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/aws/smithy-go/middleware"
smithyhttp "github.com/aws/smithy-go/transport/http"
)

func TestExpressRoundTripObject(t *testing.T) {
const key = "TestExpressRoundTripObject"
const value = "TestExpressRoundTripObjectValue"

_, err := s3client.PutObject(context.Background(), &s3.PutObjectInput{
Bucket: &setupMetadata.ExpressBucket,
Key: aws.String(key),
Body: strings.NewReader(value),
}, withAssertExpress)
if err != nil {
t.Fatalf("put object: %v", err)
}

resp, err := s3client.GetObject(context.Background(), &s3.GetObjectInput{
Bucket: &setupMetadata.ExpressBucket,
Key: aws.String(key),
}, withAssertExpress)
if err != nil {
t.Fatalf("get object: %v", err)
}

obj, err := io.ReadAll(resp.Body)
if err != nil {
t.Fatalf("read object response body: %v", err)
}

if string(obj) != value {
t.Fatalf("round-trip object didn't match: %q", obj)
}
}

func TestExpressPresignGetObject(t *testing.T) {
const key = "TestExpressPresignGetObject"
const value = "TestExpressPresignGetObjectValue"

_, err := s3client.PutObject(context.Background(), &s3.PutObjectInput{
Bucket: &setupMetadata.ExpressBucket,
Key: aws.String(key),
Body: strings.NewReader(value),
}, withAssertExpress)
if err != nil {
t.Fatalf("put object: %v", err)
}

presigner := s3.NewPresignClient(s3client)
req, err := presigner.PresignGetObject(context.Background(), &s3.GetObjectInput{
Bucket: &setupMetadata.ExpressBucket,
Key: aws.String(key),
})
if err != nil {
log.Fatalf("presign get object: %v", err)
}

u, err := url.Parse(req.URL)
if err != nil {
log.Fatalf("parse url: %v", err)
}

resp, err := http.DefaultClient.Do(&http.Request{
Method: req.Method,
URL: u,
})
if err != nil {
log.Fatalf("call presigned get object: %v", err)
}

obj, err := io.ReadAll(resp.Body)
if err != nil {
t.Fatalf("read response obj: %v", err)
}

if string(obj) != value { // ignore the status code, response body wouldn't match anyway
t.Fatalf("presigned get didn't match: %q", obj)
}
}

func TestExpressPresignPutObject(t *testing.T) {
const key = "TestExpressPresignPutObject"
const value = "TestExpressPresignPutObjectValue"

presigner := s3.NewPresignClient(s3client)
req, err := presigner.PresignPutObject(context.Background(), &s3.PutObjectInput{
Bucket: &setupMetadata.ExpressBucket,
Key: aws.String(key),
})
if err != nil {
log.Fatalf("presign put object: %v", err)
}

u, err := url.Parse(req.URL)
if err != nil {
log.Fatal(err)
}

presp, err := http.DefaultClient.Do(&http.Request{
Method: req.Method,
URL: u,
Body: io.NopCloser(strings.NewReader(value)),
ContentLength: int64(len(value)),
})
if err != nil {
log.Fatal(err)
}
if presp.StatusCode != http.StatusOK {
msg, err := io.ReadAll(presp.Body)
if err != nil {
log.Fatalf("read presigned put object response body: %s", msg)
}

log.Fatalf("call presigned put object: %s", msg)
}

gresp, err := s3client.GetObject(context.Background(), &s3.GetObjectInput{
Bucket: &setupMetadata.ExpressBucket,
Key: aws.String(key),
}, withAssertExpress)
if err != nil {
log.Fatalf("get object: %v", err)
}

obj, err := io.ReadAll(gresp.Body)
if err != nil {
log.Fatalf("read response body: %v", err)
}

if string(obj) != value {
t.Fatalf("presigned put didn't match: %q", obj)
}
}

func TestExpressUploaderDefaultChecksum(t *testing.T) {
const key = "TestExpressUploaderDefaultChecksum"
const valueLen = 12 * 1024 * 1024 // default/min part size is 5MiB, guarantee 2 full + 1 partial

value := make([]byte, valueLen)

uploader := manager.NewUploader(s3client, func(u *manager.Uploader) {
u.ClientOptions = append(u.ClientOptions, withAssertExpress)
})
out, err := uploader.Upload(context.Background(), &s3.PutObjectInput{
Bucket: &setupMetadata.ExpressBucket,
Key: aws.String(key),
Body: bytes.NewBuffer(value),
})
if err != nil {
log.Fatal(err)
}

if out.ChecksumCRC32 == nil {
log.Fatal("upload didn't default to crc32")
}
}

func TestExpressUploaderManualChecksum(t *testing.T) {
const key = "TestExpressUploaderManualChecksum"
const valueLen = 12 * 1024 * 1024

value := make([]byte, valueLen)

uploader := manager.NewUploader(s3client, func(u *manager.Uploader) {
u.ClientOptions = append(u.ClientOptions, withAssertExpress)
})
out, err := uploader.Upload(context.Background(), &s3.PutObjectInput{
Bucket: &setupMetadata.ExpressBucket,
Key: aws.String(key),
Body: bytes.NewBuffer(value),
ChecksumAlgorithm: types.ChecksumAlgorithmCrc32c,
})
if err != nil {
log.Fatal(err)
}

if out.ChecksumCRC32C == nil {
log.Fatal("upload didn't use explicit crc32c")
}
}

var withAssertExpress = s3.WithAPIOptions(func(s *middleware.Stack) error {
return s.Finalize.Add(&assertExpress{}, middleware.After)
})

type assertExpress struct{}

func (*assertExpress) ID() string {
return "assertExpress"
}

func (m *assertExpress) HandleFinalize(ctx context.Context, in middleware.FinalizeInput, next middleware.FinalizeHandler) (
out middleware.FinalizeOutput, metadata middleware.Metadata, err error,
) {
req, ok := in.Request.(*smithyhttp.Request)
if !ok {
return out, metadata, fmt.Errorf("unexpected transport type %T", in.Request)
}

sig := awstesting.ParseSigV4Signature(req.Header)
if sig.SigningName != "s3express" {
return out, metadata, fmt.Errorf("signing name is not s3express: %q", sig.SigningName)
}

return next.HandleFinalize(ctx, in)
}
17 changes: 17 additions & 0 deletions service/internal/integrationtest/s3/setup_test.go
Original file line number Diff line number Diff line change
@@ -39,6 +39,8 @@ var setupMetadata = struct {
}
}

ExpressBucket string

AccessPoints struct {
Source struct {
Name string
@@ -196,11 +198,15 @@ func getAccountID(ctx context.Context) (string, error) {

func setupBuckets(ctx context.Context) (func(), error) {
var cleanups []func()
var expressCleanups []func()

cleanup := func() {
for i := range cleanups {
cleanups[i]()
}
for i := range expressCleanups {
expressCleanups[i]()
}
}

bucketCreates := []struct {
@@ -237,6 +243,17 @@ func setupBuckets(ctx context.Context) (func(), error) {
})
}

setupMetadata.ExpressBucket = s3shared.GenerateExpressBucketName()
if err := s3shared.SetupExpressBucket(ctx, s3client, setupMetadata.ExpressBucket); err != nil {
return cleanup, fmt.Errorf("setup express bucket: %v", err)
}

expressCleanups = append(expressCleanups, func() {
if err := s3shared.CleanupBucket(ctx, s3client, setupMetadata.ExpressBucket); err != nil {
fmt.Fprintln(os.Stderr, err)
}
})

return cleanup, nil

}
54 changes: 53 additions & 1 deletion service/internal/integrationtest/s3shared/integ_test_setup.go
Original file line number Diff line number Diff line change
@@ -6,13 +6,20 @@ package s3shared
import (
"context"
"fmt"
"strings"
"time"

"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/internal/integrationtest"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/aws/aws-sdk-go-v2/service/s3control"
)

const expressAZID = "usw2-az3"

const expressSuffix = "--usw2-az3--x-s3"

// BucketPrefix is the root prefix of integration test buckets.
const BucketPrefix = "aws-sdk-go-v2-integration"

@@ -22,6 +29,16 @@ func GenerateBucketName() string {
BucketPrefix, integrationtest.UniqueID())
}

// GenerateBucketName returns a unique express-formatted bucket name.
func GenerateExpressBucketName() string {
return fmt.Sprintf(
"%s-%s%s",
BucketPrefix,
integrationtest.UniqueID()[0:8], // express suffix adds length, regain that here
expressSuffix,
)
}

// SetupBucket returns a test bucket created for the integration tests.
func SetupBucket(ctx context.Context, svc *s3.Client, bucketName string) (err error) {
fmt.Println("Setup: Creating test bucket,", bucketName)
@@ -67,7 +84,7 @@ func CleanupBucket(ctx context.Context, svc *s3.Client, bucketName string) (err
var errs = make([]error, 0)

fmt.Println("TearDown: Deleting objects from test bucket,", bucketName)
listObjectsResp, err := svc.ListObjects(ctx, &s3.ListObjectsInput{
listObjectsResp, err := svc.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
Bucket: &bucketName,
})
if err != nil {
@@ -146,3 +163,38 @@ func CleanupAccessPoint(ctx context.Context, svc *s3control.Client, accountID, a
}
return nil
}

// SetupExpressBucket returns an express bucket for testing.
func SetupExpressBucket(ctx context.Context, svc *s3.Client, bucketName string) error {
if !strings.HasSuffix(bucketName, expressSuffix) {
return fmt.Errorf("bucket name %s is missing required suffix %s", bucketName, expressSuffix)
}

fmt.Println("Setup: Creating test express bucket,", bucketName)
_, err := svc.CreateBucket(ctx, &s3.CreateBucketInput{
Bucket: &bucketName,
CreateBucketConfiguration: &types.CreateBucketConfiguration{
Location: &types.LocationInfo{
Name: aws.String(expressAZID),
Type: types.LocationTypeAvailabilityZone,
},
Bucket: &types.BucketInfo{
DataRedundancy: types.DataRedundancySingleAvailabilityZone,
Type: types.BucketTypeDirectory,
},
},
})
if err != nil {
return fmt.Errorf("create express bucket %s: %v", bucketName, err)
}

w := s3.NewBucketExistsWaiter(svc)
err = w.Wait(ctx, &s3.HeadBucketInput{
Bucket: &bucketName,
}, 10*time.Second)
if err != nil {
return fmt.Errorf("wait for express bucket %s: %v", bucketName, err)
}

return nil
}

Unchanged files with check annotations Beta

)
go func() {
g.Do("key", func() (i interface{}, e error) {

Check failure on line 107 in internal/sync/singleflight/singleflight_test.go

GitHub Actions / lint (1.20.x, ubuntu-latest)

Error return value of `g.Do` is not checked (errcheck)
close(firstStarted)
<-unblockFirst
close(firstFinished)
}
}()
g.Do("key", fn)

Check failure on line 180 in internal/sync/singleflight/singleflight_test.go

GitHub Actions / lint (1.20.x, ubuntu-latest)

Error return value of `g.Do` is not checked (errcheck)
}()
}
if os.Getenv("TEST_PANIC_DOCHAN") != "" {
defer func() {
recover()

Check failure on line 233 in internal/sync/singleflight/singleflight_test.go

GitHub Actions / lint (1.20.x, ubuntu-latest)

Error return value is not checked (errcheck)
}()
g := new(Group)
g := new(Group)
go func() {
defer func() {
recover()

Check failure on line 280 in internal/sync/singleflight/singleflight_test.go

GitHub Actions / lint (1.20.x, ubuntu-latest)

Error return value is not checked (errcheck)
}()
g.Do("", func() (interface{}, error) {

Check failure on line 282 in internal/sync/singleflight/singleflight_test.go

GitHub Actions / lint (1.20.x, ubuntu-latest)

Error return value of `g.Do` is not checked (errcheck)
close(blocked)
<-unblock
panic("Panicking in Do")
trc := testutils.TestReadCloser{Data: []byte(expectedData)}
reader := New(mctx, &trc)
readData, _ := io.ReadAll(reader)

Check failure on line 21 in aws/middleware/private/metrics/readcloserwithmetrics/read_closer_with_metrics_test.go

GitHub Actions / lint (1.20.x, ubuntu-latest)

Error return value of `io.ReadAll` is not checked (errcheck)
actualData := string(readData)
if actualData != expectedData {
Request: func() *smithyhttp.Request {
req := smithyhttp.NewStackRequest().(*smithyhttp.Request)
req.Method = "POST"
req, _ = req.SetStream(bytes.NewReader([]byte("some=field")))

Check failure on line 21 in aws/protocol/query/middleware_test.go

GitHub Actions / lint (1.20.x, ubuntu-latest)

Error return value of `req.SetStream` is not checked (errcheck)
return req
}(),
req := smithyhttp.NewStackRequest().(*smithyhttp.Request)
req.Method = "GET"
req.URL.RawQuery = "existing=query"
req, _ = req.SetStream(bytes.NewReader([]byte("some=field")))

Check failure on line 55 in aws/protocol/query/middleware_test.go

GitHub Actions / lint (1.20.x, ubuntu-latest)

Error return value of `req.SetStream` is not checked (errcheck)
return req
}(),
req := smithyhttp.NewStackRequest().(*smithyhttp.Request)
req.Method = "POST"
req.URL.RawQuery = "existing=query"
req, _ = req.SetStream(bytes.NewReader([]byte("some=field")))

Check failure on line 73 in aws/protocol/query/middleware_test.go

GitHub Actions / lint (1.20.x, ubuntu-latest)

Error return value of `req.SetStream` is not checked (errcheck)
return req
}(),
// running under.
func UserHomeDir() string {
// Ignore errors since we only care about Windows and *nix.
home, _ := os.UserHomeDir()

Check failure on line 35 in internal/shareddefaults/shared_config.go

GitHub Actions / lint (1.20.x, ubuntu-latest)

Error return value of `os.UserHomeDir` is not checked (errcheck)
if len(home) > 0 {
return home