Skip to content

Commit

Permalink
Feature/aws ecs support (hashicorp#197)
Browse files Browse the repository at this point in the history 
* Add service argument to aws provider, where service can be either ec2 or ecs (defaulting to ec2)
* Add optional ecs_cluster and ecs_family arguments
  • Loading branch information
@fdr2 @elprans
fdr2 authored and elprans committed Dec 2, 2022
1 parent 20d4832 commit 701f0c7
Show file tree
Hide file tree
Showing 6 changed files with 537 additions and 18 deletions.
4 changes: 2 additions & 2 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -182,7 +182,7 @@ environment variables.
**Note: This will make real API calls to the account provided by the credentials.**

```
$ AWS_ACCESS_KEY_ID=... AWS_ACCESS_KEY_SECRET=... AWS_REGION=... go test -v ./provider/aws
$ AWS_ACCESS_KEY_ID=... AWS_SECRET_ACCESS_KEY=... AWS_REGION=... go test -v ./provider/aws
```

This requires resources to exist that match those specified in tests
Expand All @@ -195,7 +195,7 @@ environment variables should be applicable and read by Terraform.

```
$ cd test/tf/aws
$ export AWS_ACCESS_KEY_ID=... AWS_ACCESS_KEY_SECRET=... AWS_REGION=...
$ export AWS_ACCESS_KEY_ID=... AWS_SECRET_ACCESS_KEY=... AWS_REGION=...
$ terraform init
...
$ terraform apply
Expand Down
261 changes: 249 additions & 12 deletions provider/aws/aws_discover.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,14 @@
package aws

import (
"encoding/json"
"fmt"
"github.com/aws/aws-sdk-go/aws/arn"
"github.com/aws/aws-sdk-go/service/ecs"
"io/ioutil"
"log"
"net/http"
"os"

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
Expand All @@ -16,6 +21,12 @@ import (

type Provider struct{}

const ECSMetadataURIEnvVar = "ECS_CONTAINER_METADATA_URI_V4"

type ECSTaskMeta struct {
TaskARN string `json:"TaskARN"`
}

func (p *Provider) Help() string {
return `Amazon AWS:
Expand All @@ -26,13 +37,25 @@ func (p *Provider) Help() string {
addr_type: "private_v4", "public_v4" or "public_v6". Defaults to "private_v4".
access_key_id: The AWS access key to use
secret_access_key: The AWS secret access key to use
endpoint: The endpoint URL of EC2 to use. If not set the AWS client will set
this value, which defaults to the ec2 public dns for the specified
region
service: The AWS service to filter. "ec2" or "ecs". Defaults to "ec2".
ecs_cluster: The AWS ECS Cluster Name or Full ARN to limit searching within. Default none, search all.
ecs_family: The AWS ECS Task Definition Family to limit searching within. Default none, search all.
endpoint: The endpoint URL of the AWS Service to use. If not set the AWS
client will set this value, which defaults to the public DNS name
for the service in the specified region.
The only required IAM permission is 'ec2:DescribeInstances'. If the Consul agent is
running on AWS instance it is recommended you use an IAM role, otherwise it is
recommended you make a dedicated IAM user and access key used only for auto-joining.
For EC2 discovery the only required IAM permission is 'ec2:DescribeInstances'.
If the Consul agent is running on AWS instance it is recommended you use an IAM role,
otherwise it is recommended you make a dedicated IAM user and access key used only
for auto-joining.
For ECS discovery the following IAM permissions are required on the AWS ECS Task Role
associated with the Service performing discovery.
"ecs:ListClusters"
"ecs:ListServices"
"ecs:DescribeServices"
"ecs:ListTasks"
"ecs:DescribeTasks"
`
}

Expand All @@ -52,8 +75,19 @@ func (p *Provider) Addrs(args map[string]string, l *log.Logger) ([]string, error
accessKey := args["access_key_id"]
secretKey := args["secret_access_key"]
sessionToken := args["session_token"]
service := args["service"]
ecsCluster := args["ecs_cluster"]
ecsFamily := args["ecs_family"]
endpoint := args["endpoint"]

if service != "ec2" && service != "ecs" {
l.Printf("[INFO] discover-aws: Service type %s is not supported. Valid values are {ec2,ecs}. Falling back to 'ec2'", service)
service = "ec2"
} else if service == "ecs" && addrType != "private_v4" {
l.Printf("[INFO] discover-aws: Address Type %s is not supported for ECS. Valid values are {private_v4}. Falling back to 'private_v4'", addrType)
addrType = "private_v4"
}

if addrType != "private_v4" && addrType != "public_v4" && addrType != "public_v6" {
l.Printf("[INFO] discover-aws: Address type %s is not supported. Valid values are {private_v4,public_v4,public_v6}. Falling back to 'private_v4'", addrType)
addrType = "private_v4"
Expand All @@ -73,13 +107,28 @@ func (p *Provider) Addrs(args map[string]string, l *log.Logger) ([]string, error
}

if region == "" {
l.Printf("[INFO] discover-aws: Region not provided. Looking up region in metadata...")
ec2meta := ec2metadata.New(session.New())
identity, err := ec2meta.GetInstanceIdentityDocument()
if err != nil {
return nil, fmt.Errorf("discover-aws: GetInstanceIdentityDocument failed: %s", err)
_, ecsEnabled := os.LookupEnv("ECS_CONTAINER_METADATA_URI_V4")
if ecsEnabled {
// Get ECS Task Region from metadata, so it works on Fargate and EC2-ECS
l.Printf("[INFO] discover-aws: Region not provided. Looking up region in ecs metadata...")
taskMetadata, err := getECSTaskMetadata()
if err != nil {
return nil, fmt.Errorf("discover-aws: Failed retrieving ECS Task Metadata: %s", err)
}

region, err = getEcsTaskRegion(taskMetadata)
if err != nil {
return nil, fmt.Errorf("discover-aws: Failed retrieving ECS Task Region: %s", err)
}
} else {
l.Printf("[INFO] discover-aws: Region not provided. Looking up region in ec2 metadata...")
ec2meta := ec2metadata.New(session.New())
identity, err := ec2meta.GetInstanceIdentityDocument()
if err != nil {
return nil, fmt.Errorf("discover-aws: GetInstanceIdentityDocument failed: %s", err)
}
region = identity.Region
}
region = identity.Region
}
l.Printf("[INFO] discover-aws: Region is %s", region)

Expand All @@ -105,6 +154,51 @@ func (p *Provider) Addrs(args map[string]string, l *log.Logger) ([]string, error
config.Endpoint = &endpoint
}

// Split here for ec2 vs ecs decision tree
if service == "ecs" {
svc := ecs.New(session.New(), &config)

log.Printf("[INFO] discover-aws: Filter ECS tasks with %s=%s", tagKey, tagValue)
var clusterArns []*string

// If an ECS Cluster Name (ARN) was specified, dont lookup all the cluster arns
if ecsCluster == "" {
arns, err := getEcsClusters(svc)
if err != nil {
return nil, fmt.Errorf("discover-aws: Failed to get ECS clusters: %s", err)
}
clusterArns = arns
} else {
clusterArns = []*string{&ecsCluster}
}

var taskIps []string
for _, clusterArn := range clusterArns {
taskArns, err := getEcsTasks(svc, clusterArn, &ecsFamily)
if err != nil {
return nil, fmt.Errorf("discover-aws: Failed to get ECS Tasks: %s", err)
}
log.Printf("[DEBUG] discover-aws: Found %d ECS Tasks", len(taskArns))

// Once all the possibly paged task arns are collected, collect task descriptions with 100 task maximum
// ref: https://docs.aws.amazon.com/AmazonECS/latest/APIReference/API_DescribeTasks.html#ECS-DescribeTasks-request-tasks
pageLimit := 100
for i := 0; i < len(taskArns); i += pageLimit {
taskGroup := taskArns[i:min(i+pageLimit, len(taskArns))]
ecsTaskIps, err := getEcsTaskIps(svc, clusterArn, taskGroup, &tagKey, &tagValue)
if err != nil {
return nil, fmt.Errorf("discover-aws: Failed to get ECS Task IPs: %s", err)
}
taskIps = append(taskIps, ecsTaskIps...)
log.Printf("[DEBUG] discover-aws: Found %d ECS IPs", len(ecsTaskIps))
}
}
log.Printf("[DEBUG] discover-aws: Discovered ECS Task IPs: %v", taskIps)
return taskIps, nil
}

// When not using ECS continue with the default EC2 search

svc := ec2.New(session.New(), &config)

l.Printf("[INFO] discover-aws: Filter instances with %s=%s", tagKey, tagValue)
Expand Down Expand Up @@ -174,3 +268,146 @@ func (p *Provider) Addrs(args map[string]string, l *log.Logger) ([]string, error
l.Printf("[DEBUG] discover-aws: Found ip addresses: %v", addrs)
return addrs, nil
}

func min(a, b int) int {
if a <= b {
return a
}
return b
}

func getEcsClusters(svc *ecs.ECS) ([]*string, error) {
pageNum := 0
var clusterArns []*string
err := svc.ListClustersPages(&ecs.ListClustersInput{}, func(page *ecs.ListClustersOutput, lastPage bool) bool {
pageNum++
clusterArns = append(clusterArns, page.ClusterArns...)
log.Printf("[DEBUG] discover-aws: Retrieved %d TaskArns from page %d", len(clusterArns), pageNum)
return !lastPage // return false to exit page function
})

if err != nil {
return nil, fmt.Errorf("ListClusters failed: %s", err)
}

return clusterArns, nil
}

func getECSTaskMetadata() (ECSTaskMeta, error) {
var metadataResp ECSTaskMeta

metadataURI := os.Getenv(ECSMetadataURIEnvVar)
if metadataURI == "" {
return metadataResp, fmt.Errorf("%s env var not set", ECSMetadataURIEnvVar)
}
resp, err := http.Get(fmt.Sprintf("%s/task", metadataURI))
if err != nil {
return metadataResp, fmt.Errorf("calling metadata uri: %s", err)
}
respBytes, err := ioutil.ReadAll(resp.Body)
if err != nil {
return metadataResp, fmt.Errorf("reading metadata uri response body: %s", err)
}
if err := json.Unmarshal(respBytes, &metadataResp); err != nil {
return metadataResp, fmt.Errorf("unmarshalling metadata uri response: %s", err)
}
return metadataResp, nil
}

func getEcsTaskRegion(e ECSTaskMeta) (string, error) {
// Task ARN: "arn:aws:ecs:us-east-1:000000000000:task/cluster/00000000000000000000000000000000"
// https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html
// See also: https://github.com/aws/containers-roadmap/issues/337
a, err := arn.Parse(e.TaskARN)
if err != nil {
return "", fmt.Errorf("unable to determine AWS region from ECS Task ARN: %q", e.TaskARN)
}
return a.Region, nil
}

func getEcsTasks(svc *ecs.ECS, clusterArn *string, family *string) ([]*string, error) {
var taskArns []*string
lti := ecs.ListTasksInput{
Cluster: clusterArn,
DesiredStatus: aws.String("RUNNING"),
}
if *family != "" {
lti.Family = family
}

pageNum := 0
err := svc.ListTasksPages(&lti, func(page *ecs.ListTasksOutput, lastPage bool) bool {
pageNum++
taskArns = append(taskArns, page.TaskArns...)
log.Printf("[DEBUG] discover-aws: Retrieved %d TaskArns from page %d", len(taskArns), pageNum)
return !lastPage // return false to exit page function
})

if err != nil {
return nil, fmt.Errorf("ListTasks failed: %s", err)
}

return taskArns, nil
}

func getEcsTaskIps(svc *ecs.ECS, clusterArn *string, taskArns []*string, tagKey *string, tagValue *string) ([]string, error) {
// Describe all the tasks listed for this cluster
taskDescriptions, err := svc.DescribeTasks(&ecs.DescribeTasksInput{
Cluster: clusterArn,
Include: []*string{aws.String(ecs.TaskFieldTags)},
Tasks: taskArns,
})

if err != nil {
return nil, fmt.Errorf("DescribeTasks failed: %s", err)
}

taskRequestFailures := taskDescriptions.Failures
tasks := taskDescriptions.Tasks
log.Printf("[INFO] discover-aws: Retrieved %d Task Descriptions and %d Failures", len(tasks), len(taskRequestFailures))

// Filter tasks by Tag and Connectivity Status
var ipList []string
for _, taskDescription := range tasks {

for _, tag := range taskDescription.Tags {
if *tag.Key == *tagKey && *tag.Value == *tagValue {
log.Printf("[DEBUG] discover-aws: Tag Match: %s : %s, desiredStatus: %s", *tag.Key, *tag.Value, *taskDescription.DesiredStatus)

if *taskDescription.DesiredStatus == "RUNNING" {
log.Printf("[INFO] discover-aws: Found Running Instance: %s", *taskDescription.TaskArn)
ip := getIpFromTaskDescription(taskDescription)

if ip != nil {
log.Printf("[DEBUG] discover-aws: Found Private IP: %s", *ip)
ipList = append(ipList, *ip)
}

}

}

}
}

log.Printf("[INFO] discover-aws: Retrieved %d IPs from %d Tasks", len(ipList), len(taskArns))
return ipList, nil
}

func getIpFromTaskDescription(taskDesc *ecs.Task) *string {
log.Printf("[DEBUG] discover-aws: Searching %d attachments for IPs", len(taskDesc.Attachments))
for _, attachment := range taskDesc.Attachments {

log.Printf("[DEBUG] discover-aws: Searching %d attachment details for IPs", len(attachment.Details))
for _, detail := range attachment.Details {

if *detail.Name == "privateIPv4Address" {
log.Printf("[DEBUG] discover-aws: Parsing Private IPv4: %s", *detail.Value)
return detail.Value
}

}

}
return nil
}
Loading

0 comments on commit 701f0c7

Please sign in to comment.