Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use S3 regional endpoint in us-east-1 for directory buckets #34893

Merged
merged 35 commits into from
Dec 13, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
35 commits
Select commit Hold shift + click to select a range
32c2c5e
Document default value of 's3_us_east_1_regional_endpoint'.
ewbankkit Dec 12, 2023
14c7942
r/aws_s3_directory_bucket: Use regional endpoint is us-east-1 in acce…
ewbankkit Dec 12, 2023
550e3a8
s3: Add 'isDirectoryBucket'.
ewbankkit Dec 12, 2023
31fa5ef
Add 'AWSClient.S3ExpressClient()'.
ewbankkit Dec 12, 2023
270c7f9
Add 'extra' to AWS SDK for Go v1 API client factory.
ewbankkit Dec 12, 2023
2f627d5
Add 'extra' to AWS SDK for Go v2 API client factory.
ewbankkit Dec 12, 2023
ec76778
Extras overwrite per-service defaults.
ewbankkit Dec 12, 2023
b2254e1
Only default service client is cached.
ewbankkit Dec 12, 2023
e573491
Implement 'AWSClient.S3ExpressClient()'.
ewbankkit Dec 12, 2023
bb43b55
s3: Replace 'useRegionalEndpointInUSEast1' with 'S3ExpressClient()'.
ewbankkit Dec 12, 2023
ff8b163
Cache 'AWSClient.s3ExpressClient'.
ewbankkit Dec 12, 2023
3bc2ab6
Add CHANGELOG entry.
ewbankkit Dec 12, 2023
b11170b
d/aws_s3_directory_buckets: Use 'S3ExpressClient'.
ewbankkit Dec 12, 2023
bae67c8
r/aws_s3_bucket_policy: Use 'S3ExpressClient'.
ewbankkit Dec 12, 2023
af5cc8a
Optimize 'AWSClient.S3ExpressClient()'.
ewbankkit Dec 12, 2023
728198d
r/aws_s3_object: Use 'S3ExpressClient'.
ewbankkit Dec 12, 2023
42049a3
d/aws_s3_object: Use 'S3ExpressClient'.
ewbankkit Dec 12, 2023
18570a5
r/aws_s3_object_copy: Use 'S3ExpressClient'.
ewbankkit Dec 12, 2023
c12d9d7
d/aws_s3_objects: Use 'S3ExpressClient'.
ewbankkit Dec 12, 2023
3533f23
Tweak conn for directory bucket tests.
ewbankkit Dec 12, 2023
c00e674
Correct CHANGELOG entry file name.
ewbankkit Dec 12, 2023
b09aa45
Tweak documentation for 's3_us_east_1_regional_endpoint'.
ewbankkit Dec 12, 2023
e34a41b
s3: Use 'S3ExpressClient' in sweeper.
ewbankkit Dec 12, 2023
831611e
Consolidate CHANGELOG entries.
ewbankkit Dec 13, 2023
2bb4795
d/aws_s3_objects: Fix 'Invalid configuration: region from ARN `us-eas…
ewbankkit Dec 13, 2023
ede566d
d/aws_s3_object: Fix 'Invalid configuration: region from ARN `us-east…
ewbankkit Dec 13, 2023
82657c7
r/aws_s3_object: Fix 'Invalid configuration: region from ARN `us-east…
ewbankkit Dec 13, 2023
2e68eda
Add 'TestAccS3ObjectCopy_basicViaAccessPoint'.
ewbankkit Dec 13, 2023
c97b30f
r/aws_s3_object_copy: Fix 'Invalid configuration: region from ARN `us…
ewbankkit Dec 13, 2023
ac7857a
Suppress 'ci.semgrep.migrate.aws-api-context' for 'conn.Options()'.
ewbankkit Dec 13, 2023
470dcdb
testAccCheckBucketAddObjects: Use 'S3ExpressClient' if necessary.
ewbankkit Dec 13, 2023
bf90c78
findBucket: Explain need for 'errs.Contains'.
ewbankkit Dec 13, 2023
acf2604
r/aws_s3_object: Remove FIXMEs from acceptance tests.
ewbankkit Dec 13, 2023
bb0978b
r/aws_s3_object_copy: Remove FIXMEs from acceptance tests.
ewbankkit Dec 13, 2023
e7e4e83
Update 34893.txt
ewbankkit Dec 13, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .changelog/34893.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
```release-note:bug
provider: Always use the S3 regional endpoint in `us-east-1` for S3 directory bucket operations. This fixes `no such host` errors
```
1 change: 1 addition & 0 deletions .ci/semgrep/migrate/context.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ rules:
- pattern-not: tfkafkaconnect.$API()
- pattern-not: conn.Handlers.$X(...)
- pattern-not: conn.Handlers.$X.$Y(...)
- pattern-not: conn.Options()
severity: ERROR
- id: context-todo
languages: [go]
Expand Down
171 changes: 108 additions & 63 deletions internal/conns/awsclient.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,16 @@ import (
"sync"

aws_sdkv2 "github.com/aws/aws-sdk-go-v2/aws"
s3_sdkv2 "github.com/aws/aws-sdk-go-v2/service/s3"
endpoints_sdkv1 "github.com/aws/aws-sdk-go/aws/endpoints"
session_sdkv1 "github.com/aws/aws-sdk-go/aws/session"
apigatewayv2_sdkv1 "github.com/aws/aws-sdk-go/service/apigatewayv2"
mediaconvert_sdkv1 "github.com/aws/aws-sdk-go/service/mediaconvert"
baselogging "github.com/hashicorp/aws-sdk-go-base/v2/logging"
"github.com/hashicorp/terraform-provider-aws/internal/errs"
tftags "github.com/hashicorp/terraform-provider-aws/internal/tags"
"github.com/hashicorp/terraform-provider-aws/names"
"golang.org/x/exp/maps"
)

type AWSClient struct {
Expand All @@ -40,92 +43,115 @@ type AWSClient struct {
httpClient *http.Client
lock sync.Mutex
logger baselogging.Logger
s3ExpressClient *s3_sdkv2.Client
s3UsePathStyle bool // From provider configuration.
s3UsEast1RegionalEndpoint endpoints_sdkv1.S3UsEast1RegionalEndpoint // From provider configuration.
stsRegion string // From provider configuration.
}

// CredentialsProvider returns the AWS SDK for Go v2 credentials provider.
func (client *AWSClient) CredentialsProvider() aws_sdkv2.CredentialsProvider {
if client.awsConfig == nil {
func (c *AWSClient) CredentialsProvider() aws_sdkv2.CredentialsProvider {
if c.awsConfig == nil {
return nil
}
return client.awsConfig.Credentials
return c.awsConfig.Credentials
}

func (client *AWSClient) AwsConfig() aws_sdkv2.Config { // nosemgrep:ci.aws-in-func-name
return client.awsConfig.Copy()
func (c *AWSClient) AwsConfig() aws_sdkv2.Config { // nosemgrep:ci.aws-in-func-name
return c.awsConfig.Copy()
}

// PartitionHostname returns a hostname with the provider domain suffix for the partition
// e.g. PREFIX.amazonaws.com
// The prefix should not contain a trailing period.
func (client *AWSClient) PartitionHostname(prefix string) string {
return fmt.Sprintf("%s.%s", prefix, client.DNSSuffix)
func (c *AWSClient) PartitionHostname(prefix string) string {
return fmt.Sprintf("%s.%s", prefix, c.DNSSuffix)
}

// RegionalHostname returns a hostname with the provider domain suffix for the region and partition
// e.g. PREFIX.us-west-2.amazonaws.com
// The prefix should not contain a trailing period.
func (client *AWSClient) RegionalHostname(prefix string) string {
return fmt.Sprintf("%s.%s.%s", prefix, client.Region, client.DNSSuffix)
func (c *AWSClient) RegionalHostname(prefix string) string {
return fmt.Sprintf("%s.%s.%s", prefix, c.Region, c.DNSSuffix)
}

// S3ExpressClient returns an S3 API client suitable for use with S3 Express (directory buckets).
// This client differs from the standard S3 API client only in us-east-1 if the global S3 endpoint is used.
// In that case the returned client uses the regional S3 endpoint.
func (c *AWSClient) S3ExpressClient(ctx context.Context) *s3_sdkv2.Client {
s3Client := c.S3Client(ctx)

c.lock.Lock() // OK since a non-default client is created.
defer c.lock.Unlock()

if c.s3ExpressClient == nil {
if s3Client.Options().Region == names.GlobalRegionID {
c.s3ExpressClient = errs.Must(client[*s3_sdkv2.Client](ctx, c, names.S3, map[string]any{
"s3_us_east_1_regional_endpoint": endpoints_sdkv1.RegionalS3UsEast1Endpoint,
}))
} else {
c.s3ExpressClient = s3Client
}
}

return c.s3ExpressClient
}

// S3UsePathStyle returns the s3_force_path_style provider configuration value.
func (client *AWSClient) S3UsePathStyle() bool {
return client.s3UsePathStyle
func (c *AWSClient) S3UsePathStyle() bool {
return c.s3UsePathStyle
}

// SetHTTPClient sets the http.Client used for AWS API calls.
// To have effect it must be called before the AWS SDK v1 Session is created.
func (client *AWSClient) SetHTTPClient(httpClient *http.Client) {
if client.Session == nil {
client.httpClient = httpClient
func (c *AWSClient) SetHTTPClient(httpClient *http.Client) {
if c.Session == nil {
c.httpClient = httpClient
}
}

// HTTPClient returns the http.Client used for AWS API calls.
func (client *AWSClient) HTTPClient() *http.Client {
return client.httpClient
func (c *AWSClient) HTTPClient() *http.Client {
return c.httpClient
}

// RegisterLogger places the configured logger into Context so it can be used via `tflog`.
func (client *AWSClient) RegisterLogger(ctx context.Context) context.Context {
return baselogging.RegisterLogger(ctx, client.logger)
func (c *AWSClient) RegisterLogger(ctx context.Context) context.Context {
return baselogging.RegisterLogger(ctx, c.logger)
}

// APIGatewayInvokeURL returns the Amazon API Gateway (REST APIs) invoke URL for the configured AWS Region.
// See https://docs.aws.amazon.com/apigateway/latest/developerguide/how-to-call-api.html.
func (client *AWSClient) APIGatewayInvokeURL(restAPIID, stageName string) string {
return fmt.Sprintf("https://%s/%s", client.RegionalHostname(fmt.Sprintf("%s.execute-api", restAPIID)), stageName)
func (c *AWSClient) APIGatewayInvokeURL(restAPIID, stageName string) string {
return fmt.Sprintf("https://%s/%s", c.RegionalHostname(fmt.Sprintf("%s.execute-api", restAPIID)), stageName)
}

// APIGatewayV2InvokeURL returns the Amazon API Gateway v2 (WebSocket & HTTP APIs) invoke URL for the configured AWS Region.
// See https://docs.aws.amazon.com/apigateway/latest/developerguide/http-api-publish.html and
// https://docs.aws.amazon.com/apigateway/latest/developerguide/apigateway-set-up-websocket-deployment.html.
func (client *AWSClient) APIGatewayV2InvokeURL(protocolType, apiID, stageName string) string {
func (c *AWSClient) APIGatewayV2InvokeURL(protocolType, apiID, stageName string) string {
if protocolType == apigatewayv2_sdkv1.ProtocolTypeWebsocket {
return fmt.Sprintf("wss://%s/%s", client.RegionalHostname(fmt.Sprintf("%s.execute-api", apiID)), stageName)
return fmt.Sprintf("wss://%s/%s", c.RegionalHostname(fmt.Sprintf("%s.execute-api", apiID)), stageName)
}

if stageName == "$default" {
return fmt.Sprintf("https://%s/", client.RegionalHostname(fmt.Sprintf("%s.execute-api", apiID)))
return fmt.Sprintf("https://%s/", c.RegionalHostname(fmt.Sprintf("%s.execute-api", apiID)))
}

return fmt.Sprintf("https://%s/%s", client.RegionalHostname(fmt.Sprintf("%s.execute-api", apiID)), stageName)
return fmt.Sprintf("https://%s/%s", c.RegionalHostname(fmt.Sprintf("%s.execute-api", apiID)), stageName)
}

// CloudFrontDistributionHostedZoneID returns the Route 53 hosted zone ID
// for Amazon CloudFront distributions in the configured AWS partition.
func (client *AWSClient) CloudFrontDistributionHostedZoneID() string {
if client.Partition == endpoints_sdkv1.AwsCnPartitionID {
func (c *AWSClient) CloudFrontDistributionHostedZoneID() string {
if c.Partition == endpoints_sdkv1.AwsCnPartitionID {
return "Z3RFFRIM2A3IF5" // See https://docs.amazonaws.cn/en_us/aws/latest/userguide/route53.html
}
return "Z2FDTNDATAQYW2" // See https://docs.aws.amazon.com/Route53/latest/APIReference/API_AliasTarget.html#Route53-Type-AliasTarget-HostedZoneId
}

// DefaultKMSKeyPolicy returns the default policy for KMS keys in the configured AWS partition.
func (client *AWSClient) DefaultKMSKeyPolicy() string {
func (c *AWSClient) DefaultKMSKeyPolicy() string {
return fmt.Sprintf(`
{
"Id": "default",
Expand All @@ -142,52 +168,57 @@ func (client *AWSClient) DefaultKMSKeyPolicy() string {
}
]
}
`, client.Partition, client.AccountID)
`, c.Partition, c.AccountID)
}

// GlobalAcceleratorHostedZoneID returns the Route 53 hosted zone ID
// for AWS Global Accelerator accelerators in the configured AWS partition.
func (client *AWSClient) GlobalAcceleratorHostedZoneID() string {
func (c *AWSClient) GlobalAcceleratorHostedZoneID() string {
return "Z2BJ6XQ5FK7U4H" // See https://docs.aws.amazon.com/general/latest/gr/global_accelerator.html#global_accelerator_region
}

// apiClientConfig returns the AWS API client configuration parameters for the specified service.
func (client *AWSClient) apiClientConfig(servicePackageName string) map[string]any {
func (c *AWSClient) apiClientConfig(servicePackageName string) map[string]any {
m := map[string]any{
"aws_sdkv2_config": client.awsConfig,
"endpoint": client.endpoints[servicePackageName],
"partition": client.Partition,
"session": client.Session,
"aws_sdkv2_config": c.awsConfig,
"endpoint": c.endpoints[servicePackageName],
"partition": c.Partition,
"session": c.Session,
}
switch servicePackageName {
case names.S3:
m["s3_use_path_style"] = client.s3UsePathStyle
m["s3_use_path_style"] = c.s3UsePathStyle
// AWS SDK for Go v2 does not use the AWS_S3_US_EAST_1_REGIONAL_ENDPOINT environment variable during configuration.
// For compatibility, read it now.
if client.s3UsEast1RegionalEndpoint == endpoints_sdkv1.UnsetS3UsEast1Endpoint {
if c.s3UsEast1RegionalEndpoint == endpoints_sdkv1.UnsetS3UsEast1Endpoint {
if v, err := endpoints_sdkv1.GetS3UsEast1RegionalEndpoint(os.Getenv("AWS_S3_US_EAST_1_REGIONAL_ENDPOINT")); err == nil {
client.s3UsEast1RegionalEndpoint = v
c.s3UsEast1RegionalEndpoint = v
}
}
m["s3_us_east_1_regional_endpoint"] = client.s3UsEast1RegionalEndpoint
m["s3_us_east_1_regional_endpoint"] = c.s3UsEast1RegionalEndpoint
case names.STS:
m["sts_region"] = client.stsRegion
m["sts_region"] = c.stsRegion
}

return m
}

// conn returns the AWS SDK for Go v1 API client for the specified service.
func conn[T any](ctx context.Context, c *AWSClient, servicePackageName string) (T, error) {
c.lock.Lock()
defer c.lock.Unlock()

if raw, ok := c.conns[servicePackageName]; ok {
if conn, ok := raw.(T); ok {
return conn, nil
} else {
var zero T
return zero, fmt.Errorf("AWS SDK v1 API client (%s): %T, want %T", servicePackageName, raw, zero)
// The default service client (`extra` is empty) is cached. In this case the AWSClient lock is held.
func conn[T any](ctx context.Context, c *AWSClient, servicePackageName string, extra map[string]any) (T, error) {
isDefault := len(extra) == 0
// Default service client is cached.
if isDefault {
c.lock.Lock()
defer c.lock.Unlock() // Runs at function exit, NOT block.

if raw, ok := c.conns[servicePackageName]; ok {
if conn, ok := raw.(T); ok {
return conn, nil
} else {
var zero T
return zero, fmt.Errorf("AWS SDK v1 API client (%s): %T, want %T", servicePackageName, raw, zero)
}
}
}

Expand All @@ -205,7 +236,9 @@ func conn[T any](ctx context.Context, c *AWSClient, servicePackageName string) (
return zero, fmt.Errorf("no AWS SDK v1 API client factory: %s", servicePackageName)
}

conn, err := v.NewConn(ctx, c.apiClientConfig(servicePackageName))
config := c.apiClientConfig(servicePackageName)
maps.Copy(config, extra) // Extras overwrite per-service defaults.
conn, err := v.NewConn(ctx, config)
if err != nil {
var zero T
return zero, err
Expand All @@ -221,22 +254,30 @@ func conn[T any](ctx context.Context, c *AWSClient, servicePackageName string) (
}
}

c.conns[servicePackageName] = conn
// Default service client is cached.
if isDefault {
c.conns[servicePackageName] = conn
}

return conn, nil
}

// client returns the AWS SDK for Go v2 API client for the specified service.
func client[T any](ctx context.Context, c *AWSClient, servicePackageName string) (T, error) {
c.lock.Lock()
defer c.lock.Unlock()

if raw, ok := c.clients[servicePackageName]; ok {
if client, ok := raw.(T); ok {
return client, nil
} else {
var zero T
return zero, fmt.Errorf("AWS SDK v2 API client (%s): %T, want %T", servicePackageName, raw, zero)
// The default service client (`extra` is empty) is cached. In this case the AWSClient lock is held.
func client[T any](ctx context.Context, c *AWSClient, servicePackageName string, extra map[string]any) (T, error) {
isDefault := len(extra) == 0
// Default service client is cached.
if isDefault {
c.lock.Lock()
defer c.lock.Unlock() // Runs at function exit, NOT block.

if raw, ok := c.clients[servicePackageName]; ok {
if client, ok := raw.(T); ok {
return client, nil
} else {
var zero T
return zero, fmt.Errorf("AWS SDK v2 API client (%s): %T, want %T", servicePackageName, raw, zero)
}
}
}

Expand All @@ -254,15 +295,19 @@ func client[T any](ctx context.Context, c *AWSClient, servicePackageName string)
return zero, fmt.Errorf("no AWS SDK v2 API client factory: %s", servicePackageName)
}

client, err := v.NewClient(ctx, c.apiClientConfig(servicePackageName))
config := c.apiClientConfig(servicePackageName)
maps.Copy(config, extra) // Extras overwrite per-service defaults.
client, err := v.NewClient(ctx, config)
if err != nil {
var zero T
return zero, err
}

// All customization for AWS SDK for Go v2 API clients must be done during construction.

c.clients[servicePackageName] = client
if isDefault {
c.clients[servicePackageName] = client
}

return client, nil
}
Loading
Loading