From e15af51297f2edad5736d014e56a95920ac7ec3a Mon Sep 17 00:00:00 2001 From: Yusuke KUOKA Date: Wed, 22 Nov 2017 17:33:48 +0900 Subject: [PATCH 1/4] Fix the default FleetIamRole Closes #1022 --- core/controlplane/config/templates/cluster.yaml | 2 +- model/spot_fleet.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/controlplane/config/templates/cluster.yaml b/core/controlplane/config/templates/cluster.yaml index 6ffe380de..63d25990c 100644 --- a/core/controlplane/config/templates/cluster.yaml +++ b/core/controlplane/config/templates/cluster.yaml @@ -397,7 +397,7 @@ worker: # # IAM role to grant the Spot fleet permission to bid on, launch, and terminate instances on your behalf # # See http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/spot-fleet-requests.html#spot-fleet-prerequisites # # -# # Defaults to "arn:aws:iam::youraccountid:role/aws-ec2-spot-fleet-role" assuming you've arrived "Spot Requests" in EC2 Dashboard +# # Defaults to "arn:aws:iam::youraccountid:role/aws-ec2-spot-fleet-tagging-role" assuming you've arrived "Spot Requests" in EC2 Dashboard # # hence the role is automatically created for you # iamFleetRoleArn: "arn:aws:iam::youraccountid:role/kube-aws-doesnt-create-this-for-you" # diff --git a/model/spot_fleet.go b/model/spot_fleet.go index e071b43f2..bc825e64e 100644 --- a/model/spot_fleet.go +++ b/model/spot_fleet.go @@ -60,7 +60,7 @@ func (f *SpotFleet) UnmarshalYAML(unmarshal func(interface{}) error) error { func (f SpotFleet) IAMFleetRoleRef() string { if f.IAMFleetRoleARN == "" { - return `{"Fn::Join":["", [ "arn:aws:iam::", {"Ref":"AWS::AccountId"}, ":role/aws-ec2-spot-fleet-role" ]]}` + return `{"Fn::Join":["", [ "arn:aws:iam::", {"Ref":"AWS::AccountId"}, ":role/aws-ec2-spot-fleet-tagging-role" ]]}` } else { return fmt.Sprintf(`"%s"`, f.IAMFleetRoleARN) } From c50d38f79909afb6f8ec12b0a31f2004b599bd41 Mon Sep 17 00:00:00 2001 From: Yusuke KUOKA Date: Wed, 22 Nov 2017 17:33:48 +0900 Subject: [PATCH 2/4] Fix the default FleetIamRole Closes #1022 --- core/controlplane/config/templates/cluster.yaml | 2 +- model/spot_fleet.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/controlplane/config/templates/cluster.yaml b/core/controlplane/config/templates/cluster.yaml index 6ffe380de..63d25990c 100644 --- a/core/controlplane/config/templates/cluster.yaml +++ b/core/controlplane/config/templates/cluster.yaml @@ -397,7 +397,7 @@ worker: # # IAM role to grant the Spot fleet permission to bid on, launch, and terminate instances on your behalf # # See http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/spot-fleet-requests.html#spot-fleet-prerequisites # # -# # Defaults to "arn:aws:iam::youraccountid:role/aws-ec2-spot-fleet-role" assuming you've arrived "Spot Requests" in EC2 Dashboard +# # Defaults to "arn:aws:iam::youraccountid:role/aws-ec2-spot-fleet-tagging-role" assuming you've arrived "Spot Requests" in EC2 Dashboard # # hence the role is automatically created for you # iamFleetRoleArn: "arn:aws:iam::youraccountid:role/kube-aws-doesnt-create-this-for-you" # diff --git a/model/spot_fleet.go b/model/spot_fleet.go index e071b43f2..bc825e64e 100644 --- a/model/spot_fleet.go +++ b/model/spot_fleet.go @@ -60,7 +60,7 @@ func (f *SpotFleet) UnmarshalYAML(unmarshal func(interface{}) error) error { func (f SpotFleet) IAMFleetRoleRef() string { if f.IAMFleetRoleARN == "" { - return `{"Fn::Join":["", [ "arn:aws:iam::", {"Ref":"AWS::AccountId"}, ":role/aws-ec2-spot-fleet-role" ]]}` + return `{"Fn::Join":["", [ "arn:aws:iam::", {"Ref":"AWS::AccountId"}, ":role/aws-ec2-spot-fleet-tagging-role" ]]}` } else { return fmt.Sprintf(`"%s"`, f.IAMFleetRoleARN) } From 12e30ec78efbcb7bdbf871869945eb9da0907b8b Mon Sep 17 00:00:00 2001 From: Yusuke KUOKA Date: Thu, 23 Nov 2017 00:11:37 +0900 Subject: [PATCH 3/4] Support EC2 instance tags per node role This feature will be handy when e.g. your monitoring tools discovers EC2 instances and then groups resource metrics with EC2 instance tags. Changes: - Add support for stackTags to SpotFleet-based node pool - Add a new configuration key `instanceTags` to cluster.yaml per `worker.nodePools[]`, `controller` and `etcd`. I recomment to use `stackTags` for adding cluster-wide metadata to all the resources managed by Cfn, whereas `instanceTags` are by definition used for node-group-wide metadata to EC2 instances(node pool name, node role, and anything configurable per node pool for example). A typical cluster.yaml using both `stackTags` and `instanceTags` would look like: ```yaml worker: nodePools: - name: pool1 instanceTags: myrole: worker type: ondemand # Propagated to (hopefully, it's up to cfn) all the stack resources for the pool1 stack stackTags: env: prod - name: pool2 spotFleet: targetCapacity: 2 # Propagated to EC2 instances managed by this spot fleet (via tag-spot-instance.service) instanceTags: myrole: worker type: spot # Propagated to (hopefully, it's up to cfn) all the stack resources for the pool2 stack stackTags: env: prod controller: instanceTags: myrole: controller type: ondemand # invalid as controller nodes don't have a dedicated cfn stack # stackTags: etcd: instanceTags: myrole: etcd type: ondemand # invalid as etcd nodes don't have a dedicated cfn stack # stackTags: stackTags: env: prod ``` Resolves #1026 --- .../config/templates/cloud-config-worker | 21 ++++++++++++++++++- .../config/templates/cluster.yaml | 12 +++++++++++ .../config/templates/stack-template.json | 14 +++++++++++++ .../config/templates/stack-template.json | 7 +++++++ model/ec2_instance.go | 3 ++- 5 files changed, 55 insertions(+), 2 deletions(-) diff --git a/core/controlplane/config/templates/cloud-config-worker b/core/controlplane/config/templates/cloud-config-worker index d05101e12..925af28ca 100644 --- a/core/controlplane/config/templates/cloud-config-worker +++ b/core/controlplane/config/templates/cloud-config-worker @@ -881,6 +881,25 @@ write_files: instance_id=$(curl http://169.254.169.254/latest/meta-data/instance-id) + TAGS="" + TAGS="${TAGS}Key=\"kubernetes.io/cluster/{{ .ClusterName }}\",Value=\"owned\" " + TAGS="${TAGS}Key=\"kube-aws:node-pool:name\",Value=\"{{.NodePoolName}}\" " + TAGS="${TAGS}Key=\"Name\",Value=\"{{.ClusterName}}-{{.StackName}}-kube-aws-worker\" " + + {{if .Autoscaling.ClusterAutoscaler.Enabled -}} + TAGS="${TAGS}Key=\"{{.Autoscaling.ClusterAutoscaler.AutoDiscoveryTagKey}}\",Value=\"\" " + {{end -}} + + {{range $k, $v := .StackTags -}} + TAGS="${TAGS}Key=\"{{$k}}\",Value=\"{{$v}}\" " + {{end -}} + + {{range $k, $v := .InstanceTags -}} + TAGS="${TAGS}Key=\"{{$k}}\",Value=\"{{$v}}\" " + {{end -}} + + echo Tagging this EC2 instance with: "$TAGS" + rkt run \ --volume=ssl,kind=host,source=/etc/kubernetes/ssl,readOnly=false \ --mount=volume=ssl,target=/etc/kubernetes/ssl \ @@ -896,7 +915,7 @@ write_files: /usr/bin/aws \ --region {{.Region}} ec2 create-tags \ --resource $instance_id \ - --tags '"'"'Key=kubernetes.io/cluster/{{.ClusterName}},Value=""'"'"' '"'"'Key=Name,Value="{{.ClusterName}}-{{.StackName}}-kube-aws-worker"'"'"' '"'"'Key="kube-aws:node-pool:name",Value="{{.NodePoolName}}"'"'"' + --tags '"$TAGS"' echo done.' rkt rm --uuid-file=/var/run/coreos/tag-spot-instance.uuid || : diff --git a/core/controlplane/config/templates/cluster.yaml b/core/controlplane/config/templates/cluster.yaml index 63d25990c..825fb1273 100644 --- a/core/controlplane/config/templates/cluster.yaml +++ b/core/controlplane/config/templates/cluster.yaml @@ -164,6 +164,10 @@ kmsKeyArn: "{{.KMSKeyARN}}" # # CAUTION: Don't use t2.micro or the cluster won't work. See https://github.com/kubernetes/kubernetes/issues/18975 # instanceType: t2.medium # +# # EC2 instance tags for controller nodes +# instanceTags: +# instanceRole: controller +# # rootVolume: # # Disk size (GiB) for controller node # size: 30 @@ -343,6 +347,10 @@ worker: # # CAUTION: Don't use t2.micro or the cluster won't work. See https://github.com/kubernetes/kubernetes/issues/16122 # instanceType: t2.medium # +# # EC2 instance tags for worker nodes +# instanceTags: +# instanceRole: worker +# # rootVolume: # # Disk size (GiB) for worker nodes # size: 30 @@ -577,6 +585,10 @@ worker: # # Instance type for etcd node # instanceType: t2.medium # +# # EC2 instance tags for etcd nodes +# instanceTags: +# instanceRole: etcd +# # rootVolume: # # Root volume size (GiB) for etcd node # size: 30 diff --git a/core/controlplane/config/templates/stack-template.json b/core/controlplane/config/templates/stack-template.json index 1a80f67e0..7cb603041 100644 --- a/core/controlplane/config/templates/stack-template.json +++ b/core/controlplane/config/templates/stack-template.json @@ -26,6 +26,13 @@ ], "MinSize": "{{.MinControllerCount}}", "Tags": [ + {{range $k, $v := $.Controller.InstanceTags -}} + { + "Key": "{{$k}}", + "PropagateAtLaunch": "true", + "Value": "{{$v}}" + }, + {{end -}} { "Key": "kubernetes.io/cluster/{{.ClusterName}}", "PropagateAtLaunch": "true", @@ -647,6 +654,13 @@ ], "MinSize": "1", "Tags": [ + {{range $k, $v := $.Etcd.InstanceTags -}} + { + "Key": "{{$k}}", + "PropagateAtLaunch": "true", + "Value": "{{$v}}" + }, + {{end -}} { "Key": "kubernetes.io/cluster/{{$.ClusterName}}", "PropagateAtLaunch": "true", diff --git a/core/nodepool/config/templates/stack-template.json b/core/nodepool/config/templates/stack-template.json index b89737898..6735deaea 100644 --- a/core/nodepool/config/templates/stack-template.json +++ b/core/nodepool/config/templates/stack-template.json @@ -134,6 +134,13 @@ "Value": "" }, {{end}} + {{range $k, $v := .InstanceTags -}} + { + "Key": "{{$k}}", + "PropagateAtLaunch": "true", + "Value": "{{$v}}" + }, + {{end -}} { "Key": "kubernetes.io/cluster/{{ .ClusterName }}", "PropagateAtLaunch": "true", diff --git a/model/ec2_instance.go b/model/ec2_instance.go index bfe247223..5f820c261 100644 --- a/model/ec2_instance.go +++ b/model/ec2_instance.go @@ -5,5 +5,6 @@ type EC2Instance struct { CreateTimeout string `yaml:"createTimeout,omitempty"` InstanceType string `yaml:"instanceType,omitempty"` RootVolume `yaml:"rootVolume,omitempty"` - Tenancy string `yaml:"tenancy,omitempty"` + Tenancy string `yaml:"tenancy,omitempty"` + InstanceTags map[string]string `yaml:"instanceTags,omitempty"` } From 1d0db14e531b02022e16cc464a58133766f84183 Mon Sep 17 00:00:00 2001 From: David Thomas Date: Tue, 21 Nov 2017 15:35:03 +0000 Subject: [PATCH 4/4] Add rkt container cleanup to journald-cloudwatch-logs service --- core/controlplane/config/templates/cloud-config-controller | 2 ++ core/controlplane/config/templates/cloud-config-etcd | 2 ++ core/controlplane/config/templates/cloud-config-worker | 2 ++ 3 files changed, 6 insertions(+) diff --git a/core/controlplane/config/templates/cloud-config-controller b/core/controlplane/config/templates/cloud-config-controller index b31d87151..8a17f791b 100644 --- a/core/controlplane/config/templates/cloud-config-controller +++ b/core/controlplane/config/templates/cloud-config-controller @@ -106,7 +106,9 @@ coreos: --mount volume=journal,target=/var/log/journal \ --volume machine-id,kind=host,source=/etc/machine-id,readOnly=true \ --mount volume=machine-id,target=/etc/machine-id \ + --uuid-file-save=/var/journald-cloudwatch-logs/journald-cloudwatch-logs.uuid \ {{ .JournaldCloudWatchLogsImage.RktRepo }} -- {{.ClusterName}} + ExecStopPost=/usr/bin/rkt rm --uuid-file=/var/journald-cloudwatch-logs/journald-cloudwatch-logs.uuid Restart=always RestartSec=60s diff --git a/core/controlplane/config/templates/cloud-config-etcd b/core/controlplane/config/templates/cloud-config-etcd index efe832941..cf306f6bc 100644 --- a/core/controlplane/config/templates/cloud-config-etcd +++ b/core/controlplane/config/templates/cloud-config-etcd @@ -129,7 +129,9 @@ coreos: --mount volume=journal,target=/var/log/journal \ --volume machine-id,kind=host,source=/etc/machine-id,readOnly=true \ --mount volume=machine-id,target=/etc/machine-id \ + --uuid-file-save=/var/journald-cloudwatch-logs/journald-cloudwatch-logs.uuid \ {{ .JournaldCloudWatchLogsImage.RktRepo }} -- {{.ClusterName}} + ExecStopPost=/usr/bin/rkt rm --uuid-file=/var/journald-cloudwatch-logs/journald-cloudwatch-logs.uuid Restart=always RestartSec=60s diff --git a/core/controlplane/config/templates/cloud-config-worker b/core/controlplane/config/templates/cloud-config-worker index d05101e12..d501d6cdb 100644 --- a/core/controlplane/config/templates/cloud-config-worker +++ b/core/controlplane/config/templates/cloud-config-worker @@ -141,7 +141,9 @@ coreos: --mount volume=journal,target=/var/log/journal \ --volume machine-id,kind=host,source=/etc/machine-id,readOnly=true \ --mount volume=machine-id,target=/etc/machine-id \ + --uuid-file-save=/var/journald-cloudwatch-logs/journald-cloudwatch-logs.uuid \ {{ .JournaldCloudWatchLogsImage.RktRepo }} -- {{.ClusterName}} + ExecStopPost=/usr/bin/rkt rm --uuid-file=/var/journald-cloudwatch-logs/journald-cloudwatch-logs.uuid Restart=always RestartSec=60s