Merge pull request #848 from aws/jasdel/cust/EndpointResolver

Adds a new member to the `aws.Endpoint` struct, `HostnameImmutable`. This member directs if the SDK is allowed to modify the resolved endpoint to meet the requirements of the API.  If `HostnameImmutable` is true, the SDK will not attempt to modify the hostname via any customizations, or prefix behavior. Flag defaults to false.

- Fixes #827 (SDK requiring region for endpoint resolution)
- Fixes #328
- Fixes #364
- Related to #836 When Amazon S3 Accesspoint support is implemented.
- Related to #479

Depends on aws/smithy-go#222

aws/endpoints.go

@@ -4,12 +4,44 @@ import (
 	"fmt"
 )
 
-// Endpoint represents the endpoint a service client should make requests to.
+// Endpoint represents the endpoint a service client should make API operation
+// calls to.
+//
+// The SDK will automatically resolve these endpoints per API client using an
+// internal endpoint resolvers. If you'd like to provide custom endpoint
+// resolving behavior you can implement the EndpointResolver interface.
 type Endpoint struct {
-	// The URL of the endpoint.
+	// The base URL endpoint the SDK API clients will use to make API calls to.
+	// The SDK will suffix URI path and query elements to this endpoint.
 	URL string
 
-	// The endpoint partition
+	// Specifies if the endpoint's hostname can be modified by the SDK's API
+	// client.
+	//
+	// If the hostname is mutable the SDK API clients may modify any part of
+	// the hostname based on the requirements of the API, (e.g. adding, or
+	// removing content in the hostname). Such as, Amazon S3 API client
+	// prefixing "bucketname" to the hostname, or changing the
+	// hostname service name component from "s3." to "s3-accesspoint.dualstack."
+	// for the dualstack endpoint of an S3 Accesspoint resource.
+	//
+	// Care should be taken when providing a custom endpoint for an API. If the
+	// endpoint hostname is mutable, and the client cannot modify the endpoint
+	// correctly, the operation call will most likely fail, or have undefined
+	// behavior.
+	//
+	// If hostname is immutable, the SDK API clients will not modify the
+	// hostname of the URL. This may cause the API client not to function
+	// correctly if the API requires the operation specific hostname values
+	// to be used by the client.
+	//
+	// This flag does not modify the API client's behavior if this endpoint
+	// will be used instead of Endpoint Discovery, or if the endpoint will be
+	// used to perform Endpoint Discovery. That behavior is configured via the
+	// API Client's Options.
+	HostnameImmutable bool
+
+	// The AWS partition the endpoint belongs to.
 	PartitionID string
 
 	// The service name that should be used for signing the requests to the
@@ -24,9 +56,11 @@ type Endpoint struct {
 	SigningMethod string
 }
 
-// EndpointNotFoundError is a sentinel error to indicate that the EndpointResolver implementation was unable
-// to resolve an endpoint for the given service and region. Resolvers should use this to indicate that
-// a client should fallback and attempt to use it's default resolver to resolve the endpoint.
+// EndpointNotFoundError is a sentinel error to indicate that the
+// EndpointResolver implementation was unable to resolve an endpoint for the
+// given service and region. Resolvers should use this to indicate that an API
+// client should fallback and attempt to use it's internal default resolver to
+// resolve the endpoint.
 type EndpointNotFoundError struct {
 	Err error
 }
@@ -41,18 +75,20 @@ func (e *EndpointNotFoundError) Unwrap() error {
 	return e.Err
 }
 
-// EndpointResolver is an endpoint resolver that can be used to provide or override an endpoint for the given
-// service and region. Clients will attempt to use the EndpointResolver first to resolve an endpoint if available.
-// If the EndpointResolver returns an EndpointNotFoundError error, clients will fallback to attempting to resolve the endpoint
-// using their default endpoint resolver.
+// EndpointResolver is an endpoint resolver that can be used to provide or
+// override an endpoint for the given service and region. API clients will
+// attempt to use the EndpointResolver first to resolve an endpoint if
+// available. If the EndpointResolver returns an EndpointNotFoundError error,
+// API clients will fallback to attempting to resolve the endpoint using its
+// internal default endpoint resolver.
 type EndpointResolver interface {
 	ResolveEndpoint(service, region string) (Endpoint, error)
 }
 
 // EndpointResolverFunc wraps a function to satisfy the EndpointResolver interface.
 type EndpointResolverFunc func(service, region string) (Endpoint, error)
 
-// ResolveEndpoint calls the wrapped function and returns the results
+// ResolveEndpoint calls the wrapped function and returns the results.
 func (e EndpointResolverFunc) ResolveEndpoint(service, region string) (Endpoint, error) {
 	return e(service, region)
 }

aws/errors.go

@@ -1,15 +1,9 @@
 package aws
 
-// MissingRegionError is an error that is returned if region configuration is not found.
+// MissingRegionError is an error that is returned if region configuration
+// value was not found.
 type MissingRegionError struct{}
 
 func (*MissingRegionError) Error() string {
-	return "could not find region configuration"
-}
-
-// MissingEndpointError is an error that is returned if an endpoint cannot be resolved for a service.
-type MissingEndpointError struct{}
-
-func (*MissingEndpointError) Error() string {
-	return "'Endpoint' configuration is required for this service"
+	return "an AWS region is required, but was not found"
 }

codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AwsGoDependency.java

@@ -82,6 +82,6 @@ protected static GoDependency module(
     }
 
     private static final class Versions {
-        private static final String AWS_SDK = "v0.28.0";
+        private static final String AWS_SDK = "v0.28.1-0.20201027184009-8eb8fc303e7c";
     }
 }

codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/EndpointGenerator.java

@@ -48,7 +48,7 @@ final class EndpointGenerator implements Runnable {
     public static final String ADD_MIDDLEWARE_HELPER_NAME = String.format("add%sMiddleware", MIDDLEWARE_NAME);
     public static final String RESOLVER_INTERFACE_NAME = "EndpointResolver";
     public static final String RESOLVER_FUNC_NAME = "EndpointResolverFunc";
-    public static final String RESOLVER_OPTIONS = "ResolverOptions";
+    public static final String RESOLVER_OPTIONS = "EndpointResolverOptions";
     public static final String CLIENT_CONFIG_RESOLVER = "resolveDefaultEndpointConfiguration";
     public static final String RESOLVER_CONSTRUCTOR_NAME = "NewDefaultEndpointResolver";
     public static final String AWS_ENDPOINT_RESOLVER_HELPER = "WithEndpointResolver";
@@ -217,38 +217,47 @@ private void generateMiddlewareResolverBody(GoStackStepMiddlewareGenerator g, Go
         w.addUseImports(SmithyGoDependency.FMT);
         w.addUseImports(SmithyGoDependency.NET_URL);
         w.addUseImports(AwsGoDependency.AWS_MIDDLEWARE);
+        w.addUseImports(SmithyGoDependency.SMITHY_MIDDLEWARE);
         w.addUseImports(SmithyGoDependency.SMITHY_HTTP_TRANSPORT);
 
         w.write("req, ok := in.Request.(*smithyhttp.Request)");
         w.openBlock("if !ok {", "}", () -> {
             w.write("return out, metadata, fmt.Errorf(\"unknown transport type %T\", in.Request)");
         });
         w.write("");
+
         w.openBlock("if m.Resolver == nil {", "}", () -> {
             w.write("return out, metadata, fmt.Errorf(\"expected endpoint resolver to not be nil\")");
         });
         w.write("");
+
         w.write("var endpoint $T", SymbolUtils.createValueSymbolBuilder("Endpoint", AwsGoDependency.AWS_CORE)
                 .build());
         w.write("endpoint, err = m.Resolver.ResolveEndpoint(awsmiddleware.GetRegion(ctx), m.Options)");
         w.openBlock("if err != nil {", "}", () -> {
-            w.write("return out, metadata, fmt.Errorf(\"failed to resolve service endpoint\")");
+            w.write("return out, metadata, fmt.Errorf(\"failed to resolve service endpoint, %w\", err)");
         });
         w.write("");
+
         w.write("req.URL, err = url.Parse(endpoint.URL)");
         w.openBlock("if err != nil {", "}", () -> {
             w.write("return out, metadata, fmt.Errorf(\"failed to parse endpoint URL: %w\", err)");
         });
         w.write("");
+
         w.openBlock("if len(awsmiddleware.GetSigningName(ctx)) == 0 {", "}", () -> {
             w.write("signingName := endpoint.SigningName");
             w.openBlock("if len(signingName) == 0 {", "}", () -> {
                 w.write("signingName = $S", serviceShape.expectTrait(ServiceTrait.class).getArnNamespace());
             });
             w.write("ctx = awsmiddleware.SetSigningName(ctx, signingName)");
         });
+        w.write("");
+
         w.write("ctx = awsmiddleware.SetSigningRegion(ctx, endpoint.SigningRegion)");
+        w.write("ctx = smithyhttp.SetHostnameImmutable(ctx, endpoint.HostnameImmutable)");
         w.write("");
+
         w.write("return next.HandleSerialize(ctx, in)");
     }
 
@@ -387,6 +396,12 @@ private void generateInternalResolverImplementation(GoWriter writer) {
         writer.write("");
         writer.writeDocs("ResolveEndpoint resolves the service endpoint for the given region and options");
         writeInternalResolveEndpointImplementation(writer, resolverImplSymbol, "r", () -> {
+            // Currently all APIs require a region to derive the endpoint for that API. If there are ever a truly
+            // region-less API then this should be gated at codegen.
+            writer.addUseImports(AwsGoDependency.AWS_CORE);
+            writer.write("if len(region) == 0 { return endpoint, &aws.MissingRegionError{} }");
+            writer.write("");
+
             Symbol sharedOptions = SymbolUtils.createPointableSymbolBuilder("Options",
                     AwsGoDependency.AWS_ENDPOINTS).build();
             writer.openBlock("opt := $T{", "}", sharedOptions, () -> {

codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/customization/AwsCustomGoDependency.java

@@ -48,7 +48,7 @@ private AwsCustomGoDependency() {
     }
 
     private static final class Versions {
-        private static final String INTERNAL_S3SHARED = "v0.3.0";
+        private static final String INTERNAL_S3SHARED = "v0.3.1-0.20201027184009-8eb8fc303e7c";
         private static final String INTERNAL_ACCEPTENCODING = "v0.3.0";
         private static final String INTERNAL_PRESIGNURL = "v0.1.0";
     }

codegen/smithy-aws-go-codegen/src/main/resources/software/amazon/smithy/aws/go/codegen/customization/presign_url.go.template

@@ -55,7 +55,7 @@ func TestClient${operation}_presignURLCustomization(t *testing.T) {
 					return smithyhttp.NopClient{}.Do(r)
 				}),
 				EndpointResolver: EndpointResolverFunc(
-					func(region string, options ResolverOptions) (aws.Endpoint, error) {
+					func(region string, options EndpointResolverOptions) (aws.Endpoint, error) {
 						return aws.Endpoint{
 							URL:           "https://service." + region + ".amazonaws.com",
 							SigningRegion: c.ClientRegion,

config/go.mod

@@ -3,11 +3,11 @@ module github.com/aws/aws-sdk-go-v2/config
 go 1.15
 
 require (
-	github.com/aws/aws-sdk-go-v2 v0.28.0
+	github.com/aws/aws-sdk-go-v2 v0.28.1-0.20201027184009-8eb8fc303e7c
 	github.com/aws/aws-sdk-go-v2/credentials v0.1.3
 	github.com/aws/aws-sdk-go-v2/ec2imds v0.1.3
 	github.com/aws/aws-sdk-go-v2/service/sts v0.28.0
-	github.com/awslabs/smithy-go v0.2.1
+	github.com/awslabs/smithy-go v0.2.2-0.20201026231331-345290040c23
 )
 
 replace (

config/go.sum

@@ -1,5 +1,7 @@
 github.com/awslabs/smithy-go v0.2.1 h1:guvR5teu1q6IoZnE9PFyPjxh/H7YEcZGysx9vu5ZwZ0=
 github.com/awslabs/smithy-go v0.2.1/go.mod h1:hPOQwnmBLHsUphH13tVSjQhTAFma0/0XoZGbBcOuABI=
+github.com/awslabs/smithy-go v0.2.2-0.20201026231331-345290040c23 h1:v/tQbfgJxSE2tScYeZY1SrGmUDjcFsjbREOZrZ/pGxE=
+github.com/awslabs/smithy-go v0.2.2-0.20201026231331-345290040c23/go.mod h1:hPOQwnmBLHsUphH13tVSjQhTAFma0/0XoZGbBcOuABI=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/google/go-cmp v0.4.1 h1:/exdXoGamhu5ONeUJH0deniYLWYvQwW66yvlfiiKTu0=
 github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=

credentials/go.mod

@@ -3,10 +3,10 @@ module github.com/aws/aws-sdk-go-v2/credentials
 go 1.15
 
 require (
-	github.com/aws/aws-sdk-go-v2 v0.28.0
+	github.com/aws/aws-sdk-go-v2 v0.28.1-0.20201027184009-8eb8fc303e7c
 	github.com/aws/aws-sdk-go-v2/ec2imds v0.1.3
 	github.com/aws/aws-sdk-go-v2/service/sts v0.28.0
-	github.com/awslabs/smithy-go v0.2.1
+	github.com/awslabs/smithy-go v0.2.2-0.20201026231331-345290040c23
 )
 
 replace (

credentials/go.sum

@@ -1,5 +1,7 @@
 github.com/awslabs/smithy-go v0.2.1 h1:guvR5teu1q6IoZnE9PFyPjxh/H7YEcZGysx9vu5ZwZ0=
 github.com/awslabs/smithy-go v0.2.1/go.mod h1:hPOQwnmBLHsUphH13tVSjQhTAFma0/0XoZGbBcOuABI=
+github.com/awslabs/smithy-go v0.2.2-0.20201026231331-345290040c23 h1:v/tQbfgJxSE2tScYeZY1SrGmUDjcFsjbREOZrZ/pGxE=
+github.com/awslabs/smithy-go v0.2.2-0.20201026231331-345290040c23/go.mod h1:hPOQwnmBLHsUphH13tVSjQhTAFma0/0XoZGbBcOuABI=
 github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/google/go-cmp v0.4.1 h1:/exdXoGamhu5ONeUJH0deniYLWYvQwW66yvlfiiKTu0=

example/service/s3/listObjects/go.mod

@@ -4,7 +4,7 @@ go 1.15
 
 require (
 	github.com/aws/aws-sdk-go-v2/config v0.2.1
-	github.com/aws/aws-sdk-go-v2/service/s3 v0.28.0
+	github.com/aws/aws-sdk-go-v2/service/s3 v0.28.1-0.20201027190912-8f6733991e52
 )
 
 replace github.com/aws/aws-sdk-go-v2/config => ../../../../config/

example/service/s3/listObjects/go.sum

@@ -1,5 +1,7 @@
 github.com/awslabs/smithy-go v0.2.1 h1:guvR5teu1q6IoZnE9PFyPjxh/H7YEcZGysx9vu5ZwZ0=
 github.com/awslabs/smithy-go v0.2.1/go.mod h1:hPOQwnmBLHsUphH13tVSjQhTAFma0/0XoZGbBcOuABI=
+github.com/awslabs/smithy-go v0.2.2-0.20201026231331-345290040c23 h1:v/tQbfgJxSE2tScYeZY1SrGmUDjcFsjbREOZrZ/pGxE=
+github.com/awslabs/smithy-go v0.2.2-0.20201026231331-345290040c23/go.mod h1:hPOQwnmBLHsUphH13tVSjQhTAFma0/0XoZGbBcOuABI=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/google/go-cmp v0.4.1 h1:/exdXoGamhu5ONeUJH0deniYLWYvQwW66yvlfiiKTu0=
 github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=

feature/s3/manager/bucket_region_test.go

@@ -67,7 +67,7 @@ func TestGetBucketRegion_Exists(t *testing.T) {
 		server := testSetupGetBucketRegionServer(c.RespRegion, c.StatusCode, true)
 
 		client := s3.New(s3.Options{
-			EndpointResolver: s3testing.EndpointResolverFunc(func(region string, options s3.ResolverOptions) (aws.Endpoint, error) {
+			EndpointResolver: s3testing.EndpointResolverFunc(func(region string, options s3.EndpointResolverOptions) (aws.Endpoint, error) {
 				return aws.Endpoint{
 					URL: server.URL,
 				}, nil
@@ -95,7 +95,7 @@ func TestGetBucketRegion_NotExists(t *testing.T) {
 	defer server.Close()
 
 	client := s3.New(s3.Options{
-		EndpointResolver: s3testing.EndpointResolverFunc(func(region string, options s3.ResolverOptions) (aws.Endpoint, error) {
+		EndpointResolver: s3testing.EndpointResolverFunc(func(region string, options s3.EndpointResolverOptions) (aws.Endpoint, error) {
 			return aws.Endpoint{
 				URL: server.URL,
 			}, nil

feature/s3/manager/download_test.go

@@ -464,7 +464,9 @@ func TestDownloadPartBodyRetry_FailRetry(t *testing.T) {
 }
 
 func TestDownloadWithContextCanceled(t *testing.T) {
-	d := manager.NewDownloader(s3.New(s3.Options{}))
+	d := manager.NewDownloader(s3.New(s3.Options{
+		Region: "mock-region",
+	}))
 
 	params := s3.GetObjectInput{
 		Bucket: aws.String("bucket"),

feature/s3/manager/go.mod

@@ -3,10 +3,10 @@ module github.com/aws/aws-sdk-go-v2/feature/s3/manager
 go 1.15
 
 require (
-	github.com/aws/aws-sdk-go-v2 v0.28.0
+	github.com/aws/aws-sdk-go-v2 v0.28.1-0.20201027184009-8eb8fc303e7c
 	github.com/aws/aws-sdk-go-v2/config v0.2.1
-	github.com/aws/aws-sdk-go-v2/service/s3 v0.28.0
-	github.com/awslabs/smithy-go v0.2.1
+	github.com/aws/aws-sdk-go-v2/service/s3 v0.28.1-0.20201027190912-8f6733991e52
+	github.com/awslabs/smithy-go v0.2.2-0.20201026231331-345290040c23
 	github.com/google/go-cmp v0.4.1
 )
 

feature/s3/manager/go.sum

@@ -1,5 +1,7 @@
 github.com/awslabs/smithy-go v0.2.1 h1:guvR5teu1q6IoZnE9PFyPjxh/H7YEcZGysx9vu5ZwZ0=
 github.com/awslabs/smithy-go v0.2.1/go.mod h1:hPOQwnmBLHsUphH13tVSjQhTAFma0/0XoZGbBcOuABI=
+github.com/awslabs/smithy-go v0.2.2-0.20201026231331-345290040c23 h1:v/tQbfgJxSE2tScYeZY1SrGmUDjcFsjbREOZrZ/pGxE=
+github.com/awslabs/smithy-go v0.2.2-0.20201026231331-345290040c23/go.mod h1:hPOQwnmBLHsUphH13tVSjQhTAFma0/0XoZGbBcOuABI=
 github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/google/go-cmp v0.4.1 h1:/exdXoGamhu5ONeUJH0deniYLWYvQwW66yvlfiiKTu0=

feature/s3/manager/internal/testing/endpoints.go

@@ -6,9 +6,9 @@ import (
 )
 
 // EndpointResolverFunc is a mock s3 endpoint resolver that wraps the given function
-type EndpointResolverFunc func(region string, options s3.ResolverOptions) (aws.Endpoint, error)
+type EndpointResolverFunc func(region string, options s3.EndpointResolverOptions) (aws.Endpoint, error)
 
 // ResolveEndpoint returns the results from the wrapped function.
-func (m EndpointResolverFunc) ResolveEndpoint(region string, options s3.ResolverOptions) (aws.Endpoint, error) {
+func (m EndpointResolverFunc) ResolveEndpoint(region string, options s3.EndpointResolverOptions) (aws.Endpoint, error) {
 	return m(region, options)
 }

feature/s3/manager/upload_test.go

@@ -741,6 +741,7 @@ func TestSSE(t *testing.T) {
 func TestUploadWithContextCanceled(t *testing.T) {
 	u := manager.NewUploader(s3.New(s3.Options{
 		UsePathStyle: true,
+		Region:       "mock-region",
 	}))
 
 	params := s3.PutObjectInput{
@@ -878,7 +879,7 @@ func TestUploadRetry(t *testing.T) {
 			defer server.Close()
 
 			client := s3.New(s3.Options{
-				EndpointResolver: s3testing.EndpointResolverFunc(func(region string, options s3.ResolverOptions) (aws.Endpoint, error) {
+				EndpointResolver: s3testing.EndpointResolverFunc(func(region string, options s3.EndpointResolverOptions) (aws.Endpoint, error) {
 					return aws.Endpoint{
 						URL: server.URL,
 					}, nil