diff --git a/service/internal/integrationtest/go.mod b/service/internal/integrationtest/go.mod index eb4f8504c6f..8f1f0882cfa 100644 --- a/service/internal/integrationtest/go.mod +++ b/service/internal/integrationtest/go.mod @@ -3,6 +3,7 @@ module github.com/aws/aws-sdk-go-v2/service/internal/integrationtest require ( github.com/aws/aws-sdk-go-v2 v1.23.5 github.com/aws/aws-sdk-go-v2/config v1.25.11 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.4 github.com/aws/aws-sdk-go-v2/service/acm v1.22.2 github.com/aws/aws-sdk-go-v2/service/apigateway v1.21.2 github.com/aws/aws-sdk-go-v2/service/applicationautoscaling v1.25.2 @@ -115,6 +116,8 @@ replace github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream => ../../../aws/pr replace github.com/aws/aws-sdk-go-v2/config => ../../../config/ +replace github.com/aws/aws-sdk-go-v2/feature/s3/manager => ../../../feature/s3/manager/ + replace github.com/aws/aws-sdk-go-v2/credentials => ../../../credentials/ replace github.com/aws/aws-sdk-go-v2/feature/ec2/imds => ../../../feature/ec2/imds/ diff --git a/service/internal/integrationtest/s3/express_test.go b/service/internal/integrationtest/s3/express_test.go new file mode 100644 index 00000000000..e25c44a5e7e --- /dev/null +++ b/service/internal/integrationtest/s3/express_test.go @@ -0,0 +1,227 @@ +//go:build integration +// +build integration + +package s3 + +import ( + "bytes" + "context" + "fmt" + "io" + "log" + "net/http" + "net/url" + "strings" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/feature/s3/manager" + "github.com/aws/aws-sdk-go-v2/internal/awstesting" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/aws/smithy-go/middleware" + smithyhttp "github.com/aws/smithy-go/transport/http" +) + +func TestExpressRoundTripObject(t *testing.T) { + const key = "TestExpressRoundTripObject" + const value = "TestExpressRoundTripObjectValue" + + _, err := s3client.PutObject(context.Background(), &s3.PutObjectInput{ + Bucket: &setupMetadata.ExpressBucket, + Key: aws.String(key), + Body: strings.NewReader(value), + }, withAssertExpress) + if err != nil { + t.Fatalf("put object: %v", err) + } + + resp, err := s3client.GetObject(context.Background(), &s3.GetObjectInput{ + Bucket: &setupMetadata.ExpressBucket, + Key: aws.String(key), + }, withAssertExpress) + if err != nil { + t.Fatalf("get object: %v", err) + } + + obj, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("read object response body: %v", err) + } + + if string(obj) != value { + t.Fatalf("round-trip object didn't match: %q", obj) + } +} + +func TestExpressPresignGetObject(t *testing.T) { + const key = "TestExpressPresignGetObject" + const value = "TestExpressPresignGetObjectValue" + + _, err := s3client.PutObject(context.Background(), &s3.PutObjectInput{ + Bucket: &setupMetadata.ExpressBucket, + Key: aws.String(key), + Body: strings.NewReader(value), + }, withAssertExpress) + if err != nil { + t.Fatalf("put object: %v", err) + } + + presigner := s3.NewPresignClient(s3client) + req, err := presigner.PresignGetObject(context.Background(), &s3.GetObjectInput{ + Bucket: &setupMetadata.ExpressBucket, + Key: aws.String(key), + }) + if err != nil { + log.Fatalf("presign get object: %v", err) + } + + u, err := url.Parse(req.URL) + if err != nil { + log.Fatalf("parse url: %v", err) + } + + resp, err := http.DefaultClient.Do(&http.Request{ + Method: req.Method, + URL: u, + }) + if err != nil { + log.Fatalf("call presigned get object: %v", err) + } + + obj, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("read response obj: %v", err) + } + + if string(obj) != value { // ignore the status code, response body wouldn't match anyway + t.Fatalf("presigned get didn't match: %q", obj) + } +} + +func TestExpressPresignPutObject(t *testing.T) { + const key = "TestExpressPresignPutObject" + const value = "TestExpressPresignPutObjectValue" + + presigner := s3.NewPresignClient(s3client) + req, err := presigner.PresignPutObject(context.Background(), &s3.PutObjectInput{ + Bucket: &setupMetadata.ExpressBucket, + Key: aws.String(key), + }) + if err != nil { + log.Fatalf("presign put object: %v", err) + } + + u, err := url.Parse(req.URL) + if err != nil { + log.Fatal(err) + } + + presp, err := http.DefaultClient.Do(&http.Request{ + Method: req.Method, + URL: u, + Body: io.NopCloser(strings.NewReader(value)), + ContentLength: int64(len(value)), + }) + if err != nil { + log.Fatal(err) + } + if presp.StatusCode != http.StatusOK { + msg, err := io.ReadAll(presp.Body) + if err != nil { + log.Fatalf("read presigned put object response body: %s", msg) + } + + log.Fatalf("call presigned put object: %s", msg) + } + + gresp, err := s3client.GetObject(context.Background(), &s3.GetObjectInput{ + Bucket: &setupMetadata.ExpressBucket, + Key: aws.String(key), + }, withAssertExpress) + if err != nil { + log.Fatalf("get object: %v", err) + } + + obj, err := io.ReadAll(gresp.Body) + if err != nil { + log.Fatalf("read response body: %v", err) + } + + if string(obj) != value { + t.Fatalf("presigned put didn't match: %q", obj) + } +} + +func TestExpressUploaderDefaultChecksum(t *testing.T) { + const key = "TestExpressUploaderDefaultChecksum" + const valueLen = 12 * 1024 * 1024 // default/min part size is 5MiB, guarantee 2 full + 1 partial + + value := make([]byte, valueLen) + + uploader := manager.NewUploader(s3client, func(u *manager.Uploader) { + u.ClientOptions = append(u.ClientOptions, withAssertExpress) + }) + out, err := uploader.Upload(context.Background(), &s3.PutObjectInput{ + Bucket: &setupMetadata.ExpressBucket, + Key: aws.String(key), + Body: bytes.NewBuffer(value), + }) + if err != nil { + log.Fatal(err) + } + + if out.ChecksumCRC32 == nil { + log.Fatal("upload didn't default to crc32") + } +} + +func TestExpressUploaderManualChecksum(t *testing.T) { + const key = "TestExpressUploaderManualChecksum" + const valueLen = 12 * 1024 * 1024 + + value := make([]byte, valueLen) + + uploader := manager.NewUploader(s3client, func(u *manager.Uploader) { + u.ClientOptions = append(u.ClientOptions, withAssertExpress) + }) + out, err := uploader.Upload(context.Background(), &s3.PutObjectInput{ + Bucket: &setupMetadata.ExpressBucket, + Key: aws.String(key), + Body: bytes.NewBuffer(value), + ChecksumAlgorithm: types.ChecksumAlgorithmCrc32c, + }) + if err != nil { + log.Fatal(err) + } + + if out.ChecksumCRC32C == nil { + log.Fatal("upload didn't use explicit crc32c") + } +} + +var withAssertExpress = s3.WithAPIOptions(func(s *middleware.Stack) error { + return s.Finalize.Add(&assertExpress{}, middleware.After) +}) + +type assertExpress struct{} + +func (*assertExpress) ID() string { + return "assertExpress" +} + +func (m *assertExpress) HandleFinalize(ctx context.Context, in middleware.FinalizeInput, next middleware.FinalizeHandler) ( + out middleware.FinalizeOutput, metadata middleware.Metadata, err error, +) { + req, ok := in.Request.(*smithyhttp.Request) + if !ok { + return out, metadata, fmt.Errorf("unexpected transport type %T", in.Request) + } + + sig := awstesting.ParseSigV4Signature(req.Header) + if sig.SigningName != "s3express" { + return out, metadata, fmt.Errorf("signing name is not s3express: %q", sig.SigningName) + } + + return next.HandleFinalize(ctx, in) +} diff --git a/service/internal/integrationtest/s3/setup_test.go b/service/internal/integrationtest/s3/setup_test.go index 4ce1237ba69..ea00037d6e3 100644 --- a/service/internal/integrationtest/s3/setup_test.go +++ b/service/internal/integrationtest/s3/setup_test.go @@ -39,6 +39,8 @@ var setupMetadata = struct { } } + ExpressBucket string + AccessPoints struct { Source struct { Name string @@ -196,11 +198,15 @@ func getAccountID(ctx context.Context) (string, error) { func setupBuckets(ctx context.Context) (func(), error) { var cleanups []func() + var expressCleanups []func() cleanup := func() { for i := range cleanups { cleanups[i]() } + for i := range expressCleanups { + expressCleanups[i]() + } } bucketCreates := []struct { @@ -237,6 +243,17 @@ func setupBuckets(ctx context.Context) (func(), error) { }) } + setupMetadata.ExpressBucket = s3shared.GenerateExpressBucketName() + if err := s3shared.SetupExpressBucket(ctx, s3client, setupMetadata.ExpressBucket); err != nil { + return cleanup, fmt.Errorf("setup express bucket: %v", err) + } + + expressCleanups = append(expressCleanups, func() { + if err := s3shared.CleanupBucket(ctx, s3client, setupMetadata.ExpressBucket); err != nil { + fmt.Fprintln(os.Stderr, err) + } + }) + return cleanup, nil } diff --git a/service/internal/integrationtest/s3shared/integ_test_setup.go b/service/internal/integrationtest/s3shared/integ_test_setup.go index fd839c5dff0..34c9e1fd9cb 100644 --- a/service/internal/integrationtest/s3shared/integ_test_setup.go +++ b/service/internal/integrationtest/s3shared/integ_test_setup.go @@ -6,13 +6,20 @@ package s3shared import ( "context" "fmt" + "strings" + "time" + "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/internal/integrationtest" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/aws/aws-sdk-go-v2/service/s3control" ) +const expressAZID = "usw2-az3" + +const expressSuffix = "--usw2-az3--x-s3" + // BucketPrefix is the root prefix of integration test buckets. const BucketPrefix = "aws-sdk-go-v2-integration" @@ -22,6 +29,16 @@ func GenerateBucketName() string { BucketPrefix, integrationtest.UniqueID()) } +// GenerateBucketName returns a unique express-formatted bucket name. +func GenerateExpressBucketName() string { + return fmt.Sprintf( + "%s-%s%s", + BucketPrefix, + integrationtest.UniqueID()[0:8], // express suffix adds length, regain that here + expressSuffix, + ) +} + // SetupBucket returns a test bucket created for the integration tests. func SetupBucket(ctx context.Context, svc *s3.Client, bucketName string) (err error) { fmt.Println("Setup: Creating test bucket,", bucketName) @@ -67,7 +84,7 @@ func CleanupBucket(ctx context.Context, svc *s3.Client, bucketName string) (err var errs = make([]error, 0) fmt.Println("TearDown: Deleting objects from test bucket,", bucketName) - listObjectsResp, err := svc.ListObjects(ctx, &s3.ListObjectsInput{ + listObjectsResp, err := svc.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ Bucket: &bucketName, }) if err != nil { @@ -146,3 +163,38 @@ func CleanupAccessPoint(ctx context.Context, svc *s3control.Client, accountID, a } return nil } + +// SetupExpressBucket returns an express bucket for testing. +func SetupExpressBucket(ctx context.Context, svc *s3.Client, bucketName string) error { + if !strings.HasSuffix(bucketName, expressSuffix) { + return fmt.Errorf("bucket name %s is missing required suffix %s", bucketName, expressSuffix) + } + + fmt.Println("Setup: Creating test express bucket,", bucketName) + _, err := svc.CreateBucket(ctx, &s3.CreateBucketInput{ + Bucket: &bucketName, + CreateBucketConfiguration: &types.CreateBucketConfiguration{ + Location: &types.LocationInfo{ + Name: aws.String(expressAZID), + Type: types.LocationTypeAvailabilityZone, + }, + Bucket: &types.BucketInfo{ + DataRedundancy: types.DataRedundancySingleAvailabilityZone, + Type: types.BucketTypeDirectory, + }, + }, + }) + if err != nil { + return fmt.Errorf("create express bucket %s: %v", bucketName, err) + } + + w := s3.NewBucketExistsWaiter(svc) + err = w.Wait(ctx, &s3.HeadBucketInput{ + Bucket: &bucketName, + }, 10*time.Second) + if err != nil { + return fmt.Errorf("wait for express bucket %s: %v", bucketName, err) + } + + return nil +}