Deprecate aws_partition and use endpoint,regions instead (elastic#23539)

CHANGELOG.next.asciidoc

@@ -582,6 +582,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Honor kube event resysncs to handle missed watch events {pull}22668[22668]
 - Add autodiscover provider and metadata processor for Nomad. {pull}14954[14954] {pull}23324[23324]
 - Add `processors.rate_limit.n.dropped` monitoring counter metric for the `rate_limit` processor. {pull}23330[23330]
+- Deprecate aws_partition config parameter for AWS, use endpoint instead. {pull}23539[23539]
 - Update the baseline version of Sarama (Kafka support library) to 1.27.2. {pull}23595[23595]
 - Add kubernetes.volume.fs.used.pct field. {pull}23564[23564]
 

metricbeat/docs/modules/aws.asciidoc

@@ -31,7 +31,7 @@ Please see <<aws-credentials-config,AWS credentials options>> for more details.
 This module also accepts optional configuration `regions` to specify which
 AWS regions to query metrics from. If the `regions` parameter is not set in the
 config file, then by default, the `aws` module will query metrics from all available
-AWS regions.
+AWS regions. If `endpoint` is specified, `regions` becomes a required config parameter.
 
 * *latency*
 
@@ -40,6 +40,27 @@ process larger than Metricbeat collection period. This case, please specify a
 `latency` parameter so collection start time and end time will be shifted by the
 given latency amount.
 
+* *endpoint*
+
+Most AWS services offer a regional endpoint that can be used to make requests.
+The general syntax of a regional endpoint is `protocol://service-code.region-code.endpoint-code`.
+Some services, such as IAM, do not support regions. The endpoints for these
+services do not include a region. In `aws` module, `endpoint` config is to set
+the `endpoint-code` part, such as `amazonaws.com`, `amazonaws.com.cn`, `c2s.ic.gov`,
+`sc2s.sgov.gov`.
+
+If endpoint is specified, `regions` config becomes required. For example:
+
+[source,yaml]
+----
+- module: aws
+  period: 5m
+  endpoint: amazonaws.com.cn
+  regions: cn-north-1
+  metricsets:
+    - ec2
+----
+
 The aws module comes with a predefined dashboard. For example:
 
 image::./images/metricbeat-aws-overview.png[]

x-pack/filebeat/input/awss3/input.go

@@ -113,6 +113,7 @@ func (in *s3Input) createCollector(ctx v2.Context, pipeline beat.Pipeline) (*s3C
 
 	log.Debug("s3 service name = ", s3Servicename)
 	log.Debug("s3 input config max_number_of_messages = ", in.config.MaxNumberOfMessages)
+	log.Debug("s3 input config endpoint = ", in.config.AwsConfig.Endpoint)
 	return &s3Collector{
 		cancellation:      ctxtool.FromCanceller(ctx.Cancelation),
 		logger:            log,

x-pack/libbeat/common/aws/credentials.go

@@ -12,6 +12,7 @@ import (
 	"github.com/aws/aws-sdk-go-v2/service/sts"
 	"github.com/pkg/errors"
 
+	"github.com/elastic/beats/v7/libbeat/common/cfgwarn"
 	"github.com/elastic/beats/v7/libbeat/logp"
 )
 
@@ -24,7 +25,7 @@ type ConfigAWS struct {
 	SharedCredentialFile string `config:"shared_credential_file"`
 	Endpoint             string `config:"endpoint"`
 	RoleArn              string `config:"role_arn"`
-	AWSPartition         string `config:"aws_partition"`
+	AWSPartition         string `config:"aws_partition"` // Deprecated.
 }
 
 // GetAWSCredentials function gets aws credentials from the config.
@@ -104,3 +105,11 @@ func EnrichAWSConfigWithEndpoint(endpoint string, serviceName string, regionName
 	}
 	return awsConfig
 }
+
+// Validate checks for deprecated config option
+func (c ConfigAWS) Validate() error {
+	if c.AWSPartition != "" {
+		cfgwarn.Deprecate("8.0.0", "aws_partition is deprecated. Please use endpoint instead.")
+	}
+	return nil
+}

x-pack/libbeat/docs/aws-credentials-config.asciidoc

@@ -9,9 +9,14 @@ To configure AWS credentials, either put the credentials into the {beatname_uc}
 * *session_token*: required when using temporary security credentials.
 * *credential_profile_name*: profile name in shared credentials file.
 * *shared_credential_file*: directory of the shared credentials file.
-* *endpoint*: URL of the entry point for an AWS web service.
 * *role_arn*: AWS IAM Role to assume.
-* *aws_partition*: AWS region parttion name, value is one of `aws, aws-cn, aws-us-gov`, default is `aws`.
+* *endpoint*: URL of the entry point for an AWS web service.
+Most AWS services offer a regional endpoint that can be used to make requests.
+The general syntax of a regional endpoint is `protocol://service-code.region-code.endpoint-code`.
+Some services, such as IAM, do not support regions. The endpoints for these
+services do not include a region. In `aws` module, `endpoint` config is to set
+the `endpoint-code` part, such as `amazonaws.com`, `amazonaws.com.cn`, `c2s.ic.gov`,
+`sc2s.sgov.gov`.
 
 [float]
 ==== Supported Formats

x-pack/metricbeat/module/aws/_meta/docs.asciidoc

@@ -23,7 +23,7 @@ Please see <<aws-credentials-config,AWS credentials options>> for more details.
 This module also accepts optional configuration `regions` to specify which
 AWS regions to query metrics from. If the `regions` parameter is not set in the
 config file, then by default, the `aws` module will query metrics from all available
-AWS regions.
+AWS regions. If `endpoint` is specified, `regions` becomes a required config parameter.
 
 * *latency*
 
@@ -32,6 +32,27 @@ process larger than Metricbeat collection period. This case, please specify a
 `latency` parameter so collection start time and end time will be shifted by the
 given latency amount.
 
+* *endpoint*
+
+Most AWS services offer a regional endpoint that can be used to make requests.
+The general syntax of a regional endpoint is `protocol://service-code.region-code.endpoint-code`.
+Some services, such as IAM, do not support regions. The endpoints for these
+services do not include a region. In `aws` module, `endpoint` config is to set
+the `endpoint-code` part, such as `amazonaws.com`, `amazonaws.com.cn`, `c2s.ic.gov`,
+`sc2s.sgov.gov`.
+
+If endpoint is specified, `regions` config becomes required. For example:
+
+[source,yaml]
+----
+- module: aws
+  period: 5m
+  endpoint: amazonaws.com.cn
+  regions: cn-north-1
+  metricsets:
+    - ec2
+----
+
 The aws module comes with a predefined dashboard. For example:
 
 image::./images/metricbeat-aws-overview.png[]

x-pack/metricbeat/module/aws/aws.go

@@ -6,6 +6,7 @@ package aws
 
 import (
 	"context"
+	"fmt"
 	"time"
 
 	awssdk "github.com/aws/aws-sdk-go-v2/aws"
@@ -16,7 +17,6 @@ import (
 	"github.com/aws/aws-sdk-go-v2/service/rds"
 	"github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi"
 	"github.com/aws/aws-sdk-go-v2/service/sts"
-	"github.com/pkg/errors"
 
 	"github.com/elastic/beats/v7/libbeat/common"
 	"github.com/elastic/beats/v7/metricbeat/mb"
@@ -78,20 +78,22 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) {
 
 	awsConfig, err := awscommon.GetAWSCredentials(config.AWSConfig)
 	if err != nil {
-		return nil, errors.Wrap(err, "failed to get aws credentials, please check AWS credential in config")
+		return nil, fmt.Errorf("failed to get aws credentials, please check AWS credential in config: %w", err)
 	}
 
 	_, err = awsConfig.Credentials.Retrieve()
 	if err != nil {
-		return nil, errors.Wrap(err, "failed to retrieve aws credentials, please check AWS credential in config")
+		return nil, fmt.Errorf("failed to retrieve aws credentials, please check AWS credential in config: %w", err)
 	}
 
+	base.Logger().Debug("aws config endpoint = ", config.AWSConfig.Endpoint)
 	metricSet := MetricSet{
 		BaseMetricSet: base,
 		Period:        config.Period,
 		Latency:       config.Latency,
 		AwsConfig:     &awsConfig,
 		TagsFilter:    config.TagsFilter,
+		Endpoint:      config.AWSConfig.Endpoint,
 	}
 
 	base.Logger().Debug("Metricset level config for period: ", metricSet.Period)
@@ -100,6 +102,9 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) {
 
 	// Get IAM account name, set region by aws_partition, default is aws global partition
 	// refer https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html
+	if config.AWSConfig.AWSPartition != "" && config.AWSConfig.Endpoint != "" {
+		base.Logger().Warn("aws_partition is deprecated. Please use endpoint and regions instead.")
+	}
 	switch config.AWSConfig.AWSPartition {
 	case "aws-cn":
 		awsConfig.Region = "cn-north-1"
@@ -109,6 +114,11 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) {
 		awsConfig.Region = "us-east-1"
 	}
 
+	// If regions in config is not empty, then overwrite the awsConfig.Region
+	if len(config.Regions) > 0 {
+		awsConfig.Region = config.Regions[0]
+	}
+
 	// Get IAM account id
 	svcSts := sts.New(awscommon.EnrichAWSConfigWithEndpoint(
 		config.AWSConfig.Endpoint, "sts", "", awsConfig))
@@ -151,7 +161,7 @@ func getRegions(svc ec2iface.ClientAPI) (completeRegionsList []string, err error
 	req := svc.DescribeRegionsRequest(input)
 	output, err := req.Send(context.TODO())
 	if err != nil {
-		err = errors.Wrap(err, "Failed DescribeRegions")
+		err = fmt.Errorf("failed DescribeRegions: %w", err)
 		return
 	}
 	for _, region := range output.Regions {