diff --git a/core/controlplane/cluster/cluster.go b/core/controlplane/cluster/cluster.go index 546e8e74c..318e61ea1 100644 --- a/core/controlplane/cluster/cluster.go +++ b/core/controlplane/cluster/cluster.go @@ -138,7 +138,7 @@ func (c *Cluster) Assets() (cfnstack.Assets, error) { return cfnstack.NewAssetsBuilder(c.StackName(), c.StackConfig.S3URI). Add("userdata-controller", c.UserDataController). - Add("userdata-worker", c.UserDataWorker). + Add("userdata-etcd", c.UserDataEtcd). Add(STACK_TEMPLATE_FILENAME, stackTemplate). Build(), nil } diff --git a/core/controlplane/config/config.go b/core/controlplane/config/config.go index e87d70ad3..8554ba457 100644 --- a/core/controlplane/config/config.go +++ b/core/controlplane/config/config.go @@ -15,6 +15,7 @@ import ( "github.com/coreos/kube-aws/coreos/amiregistry" "github.com/coreos/kube-aws/filereader/userdatatemplate" "github.com/coreos/kube-aws/model" + "github.com/coreos/kube-aws/model/derived" "github.com/coreos/kube-aws/netutil" yaml "gopkg.in/yaml.v2" ) @@ -630,32 +631,10 @@ func (c Cluster) Config() (*Config, error) { config.AMI = c.AmiId } - config.EtcdInstances = make([]model.EtcdInstance, config.EtcdCount) - - for etcdIndex := 0; etcdIndex < config.EtcdCount; etcdIndex++ { - - //Round-robin etcd instances across all available subnets - subnetIndex := etcdIndex % len(config.Etcd.Subnets) - subnet := config.Etcd.Subnets[subnetIndex] - - var instance model.EtcdInstance - - if subnet.ManageNATGateway() { - ngw, err := c.FindNATGatewayForPrivateSubnet(subnet) - - if err != nil { - return nil, fmt.Errorf("failed getting a NAT gateway for the subnet %s in %v: %v", subnet.LogicalName(), c.NATGateways(), err) - } - - instance = model.NewEtcdInstanceDependsOnNewlyCreatedNGW(subnet, *ngw) - } else { - instance = model.NewEtcdInstance(subnet) - } - - config.EtcdInstances[etcdIndex] = instance - - //http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-instance-addressing.html#concepts-private-addresses - + var err error + config.EtcdNodes, err = derived.NewEtcdNodes(c.Etcd.Nodes, c.EtcdCluster()) + if err != nil { + return nil, fmt.Errorf("failed to derived etcd nodes configuration: %v", err) } // Populate top-level subnets to model @@ -670,6 +649,12 @@ func (c Cluster) Config() (*Config, error) { return &config, nil } +func (c *Cluster) EtcdCluster() derived.EtcdCluster { + region := model.RegionForName(c.Region) + etcdNetwork := derived.NewNetwork(c.Etcd.Subnets, c.NATGateways()) + return derived.NewEtcdCluster(c.Etcd.Cluster, region, etcdNetwork, c.EtcdCount) +} + // releaseVersionIsGreaterThan will return true if the supplied version is greater then // or equal to the current CoreOS release indicated by the given release // channel. @@ -731,7 +716,7 @@ func (c Cluster) StackConfig(opts StackTemplateOptions) (*StackConfig, error) { if stackConfig.UserDataController, err = userdatatemplate.GetString(opts.ControllerTmplFile, stackConfig.Config); err != nil { return nil, fmt.Errorf("failed to render controller cloud config: %v", err) } - if stackConfig.userDataEtcd, err = userdatatemplate.GetString(opts.EtcdTmplFile, stackConfig.Config); err != nil { + if stackConfig.UserDataEtcd, err = userdatatemplate.GetString(opts.EtcdTmplFile, stackConfig.Config); err != nil { return nil, fmt.Errorf("failed to render etcd cloud config: %v", err) } @@ -750,7 +735,7 @@ func (c Cluster) StackConfig(opts StackTemplateOptions) (*StackConfig, error) { type Config struct { Cluster - EtcdInstances []model.EtcdInstance + EtcdNodes []derived.EtcdNode // Encoded TLS assets TLSConfig *CompactTLSAssets @@ -764,6 +749,18 @@ func (c Cluster) StackName() string { return "control-plane" } +func (c Cluster) StackNameEnvVarName() string { + return "KUBE_AWS_STACK_NAME" +} + +func (c Cluster) EtcdNodeEnvFileName() string { + return "/var/run/coreos/etcd-node.env" +} + +func (c Cluster) EtcdIndexEnvVarName() string { + return "KUBE_AWS_ETCD_INDEX" +} + func (c Config) VPCLogicalName() string { return vpcLogicalName } diff --git a/core/controlplane/config/stack_config.go b/core/controlplane/config/stack_config.go index a1dd6bf0b..a19d481a0 100644 --- a/core/controlplane/config/stack_config.go +++ b/core/controlplane/config/stack_config.go @@ -4,7 +4,6 @@ import ( "fmt" "github.com/coreos/kube-aws/coreos/userdatavalidation" "github.com/coreos/kube-aws/filereader/jsontemplate" - "github.com/coreos/kube-aws/gzipcompressor" "net/url" ) @@ -13,13 +12,12 @@ type StackConfig struct { StackTemplateOptions UserDataWorker string UserDataController string - userDataEtcd string + UserDataEtcd string ControllerSubnetIndex int } type CompressedStackConfig struct { *StackConfig - UserDataEtcd string } func (c *StackConfig) UserDataControllerS3Path() (string, error) { @@ -30,28 +28,27 @@ func (c *StackConfig) UserDataControllerS3Path() (string, error) { return fmt.Sprintf("%s%s/%s/userdata-controller", s3uri.Host, s3uri.Path, c.StackName()), nil } +func (c *StackConfig) UserDataEtcdS3Path() (string, error) { + s3uri, err := url.Parse(c.S3URI) + if err != nil { + return "", fmt.Errorf("Error in UserDataEtcdS3Path : %v", err) + } + return fmt.Sprintf("%s%s/%s/userdata-etcd", s3uri.Host, s3uri.Path, c.StackName()), nil +} + func (c *StackConfig) ValidateUserData() error { err := userdatavalidation.Execute([]userdatavalidation.Entry{ {Name: "UserDataWorker", Content: c.UserDataWorker}, {Name: "UserDataController", Content: c.UserDataController}, - {Name: "UserDataEtcd", Content: c.userDataEtcd}, + {Name: "UserDataEtcd", Content: c.UserDataEtcd}, }) return err } func (c *StackConfig) Compress() (*CompressedStackConfig, error) { - var err error - var compressedEtcdUserData string - - if compressedEtcdUserData, err = gzipcompressor.CompressString(c.userDataEtcd); err != nil { - return nil, err - } - var stackConfig CompressedStackConfig stackConfig.StackConfig = &(*c) - stackConfig.UserDataEtcd = compressedEtcdUserData - return &stackConfig, nil } diff --git a/core/controlplane/config/templates/cloud-config-etcd b/core/controlplane/config/templates/cloud-config-etcd index 3bcf24bc6..95eaf2307 100644 --- a/core/controlplane/config/templates/cloud-config-etcd +++ b/core/controlplane/config/templates/cloud-config-etcd @@ -9,14 +9,20 @@ coreos: runtime: true content: | [Unit] - Description=Fetches etcd static IP addresses list from CF + Description=Configures EBS volume and R53 record set for this node and derives env vars for etcd bootstrap After=network-online.target + Before=format-etcd2-volume.service [Service] + EnvironmentFile={{.EtcdNodeEnvFileName}} Restart=on-failure RemainAfterExit=true ExecStartPre=/opt/bin/cfn-etcd-environment ExecStart=/usr/bin/mv -f /var/run/coreos/etcd-environment /etc/etcd-environment + TimeoutStartSec=120 + + [Install] + RequiredBy=format-etcd2-volume.service - name: etcd2.service drop-ins: @@ -27,28 +33,11 @@ coreos: After=decrypt-tls-assets.service cfn-etcd-environment.service [Service] - Environment=ETCD_NAME=%H - - Environment=ETCD_PEER_TRUSTED_CA_FILE=/etc/etcd2/ssl/ca.pem - Environment=ETCD_PEER_CERT_FILE=/etc/etcd2/ssl/etcd.pem - Environment=ETCD_PEER_KEY_FILE=/etc/etcd2/ssl/etcd-key.pem - - Environment=ETCD_CLIENT_CERT_AUTH=true - Environment=ETCD_TRUSTED_CA_FILE=/etc/etcd2/ssl/ca.pem - Environment=ETCD_CERT_FILE=/etc/etcd2/ssl/etcd.pem - Environment=ETCD_KEY_FILE=/etc/etcd2/ssl/etcd-key.pem - - Environment=ETCD_INITIAL_CLUSTER_STATE=new EnvironmentFile=-/etc/etcd-environment - Environment=ETCD_DATA_DIR=/var/lib/etcd2 - Environment=ETCD_LISTEN_CLIENT_URLS=https://%H:2379 - Environment=ETCD_ADVERTISE_CLIENT_URLS=https://%H:2379 - Environment=ETCD_LISTEN_PEER_URLS=https://%H:2380 - Environment=ETCD_INITIAL_ADVERTISE_PEER_URLS=https://%H:2380 + PermissionsStartOnly=true ExecStartPre=/usr/bin/systemctl is-active cfn-etcd-environment.service ExecStartPre=/usr/bin/systemctl is-active decrypt-tls-assets.service - ExecStartPre=/usr/bin/sed -i 's/^ETCDCTL_ENDPOINT.*$/ETCDCTL_ENDPOINT=https:\/\/%H:2379/' /etc/environment ExecStartPre=/usr/bin/chown -R etcd:etcd /var/lib/etcd2 enable: true command: start @@ -121,6 +110,21 @@ coreos: [Install] RequiredBy=etcd2.service +{{ if .WaitSignal.Enabled }} + - name: cfn-signal.service + command: start + content: | + [Unit] + Wants=etcd2.service + After=etcd2.service + + [Service] + Type=oneshot + EnvironmentFile={{.EtcdNodeEnvFileName}} + ExecStartPre=/usr/bin/systemctl is-active etcd2.service + ExecStart=/opt/bin/cfn-signal +{{end}} + {{if .SSHAuthorizedKeys}} ssh_authorized_keys: {{range $sshkey := .SSHAuthorizedKeys}} @@ -130,34 +134,312 @@ ssh_authorized_keys: write_files: + - path: /opt/bin/cfn-init-etcd-server + owner: root:root + permissions: 0700 + content: | + #!/bin/bash -vxe + + cfn-init -v -c "etcd-server" --region {{.Region}} --resource {{.Etcd.LogicalName}}${{.EtcdIndexEnvVarName}} --stack ${{.StackNameEnvVarName}} + + - path: /opt/bin/attach-etcd-volume + owner: root:root + permissions: 0700 + content: | + #!/bin/bash -vxe + + # To omit the `--region {{.Region}}` flag for every aws-cli invocation + export AWS_DEFAULT_REGION={{.Region}} + + instance_id=$(curl http://169.254.169.254/latest/meta-data/instance-id) + az=$(curl http://169.254.169.254/latest/meta-data/placement/availability-zone) + + # values shared between cloud-config-etcd and stack-template.json + stack_name=${{.StackNameEnvVarName}} + name_tag_key="{{$.Etcd.NameTagKey}}" + advertised_hostname_tag_key="{{$.Etcd.AdvertisedFQDNTagKey}}" + eip_allocation_id_tag_key="{{$.Etcd.EIPAllocationIDTagKey}}" + network_interface_id_tag_key="{{$.Etcd.NetworkInterfaceIDTagKey}}" + + etcd_index=${{.EtcdIndexEnvVarName}} + + state_prefix=/var/run/coreos/etcd-volume + output_prefix=/var/run/coreos/ + common_volume_filter="Name=tag:aws:cloudformation:stack-name,Values=$stack_name Name=tag:kube-aws:etcd:index,Values=$etcd_index" + + export $(cat /var/run/coreos/etcd-environment | grep -v ^# | xargs) + + export | grep ETCD + + # TODO: Locate the corresponding EBS volume via a tag on the ASG managing this EC2 instance + # See https://github.com/coreos/kube-aws/pull/332#issuecomment-281531769 + + # Skip the `while` block below when the EBS volume is already attached to this EC2 instance + aws ec2 describe-volumes \ + --filters $common_volume_filter Name=attachment.instance-id,Values=$instance_id \ + | jq -r '([] + .Volumes)[0]' \ + > ${state_prefix}.json + + attached_vol_id=$( + cat ${state_prefix}.json \ + | jq -r '"" + .VolumeId' + ) + + # Decide which volume to attach hence hostname to assume + while [ "$attached_vol_id" = "" ]; do + sleep 3 + + aws ec2 describe-volumes \ + --filters $common_volume_filter Name=status,Values=available Name=availability-zone,Values=$az \ + > ${state_prefix}-candidates.json + + cat ${state_prefix}-candidates.json \ + | jq -r '([] + .Volumes)[0]' \ + > ${state_prefix}.json + + candidate_vol_id=$( + cat ${state_prefix}.json \ + | jq -r '"" + .VolumeId' + ) + + if [ "$candidate_vol_id" = "" ]; then + echo "[bug] no etcd volume found" 1>&2 + exit 1 + fi + + # See http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/device_naming.html for device naming + if aws ec2 attach-volume --volume-id $candidate_vol_id --instance-id $instance_id --device "/dev/xvdf"; then + attached_vol_id=$candidate_vol_id + fi + done + + # Wait until the volume attachment completes + until [ "$volume_status" = ok ]; do + sleep 3 + describe_volume_status_result=$(aws ec2 describe-volume-status --volume-id $attached_vol_id) + volume_status=$(echo "$describe_volume_status_result" | jq -r "([] + .VolumeStatuses)[0].VolumeStatus.Status") + done + + cat ${state_prefix}.json \ + | jq -r "([] + .Tags)[] | select(.Key == \"$name_tag_key\").Value" \ + > ${output_prefix}name + + cat ${state_prefix}.json \ + | jq -r "([] + .Tags)[] | select(.Key == \"$advertised_hostname_tag_key\").Value" \ + > ${output_prefix}advertised-hostname + + cat ${state_prefix}.json \ + | jq -r "([] + .Tags)[] | select(.Key == \"$eip_allocation_id_tag_key\").Value" \ + > ${output_prefix}eip-allocation-id + + cat ${state_prefix}.json \ + | jq -r "([] + .Tags)[] | select(.Key == \"$network_interface_id_tag_key\").Value" \ + > ${output_prefix}network-interface-id + + {{if $.Etcd.NodeShouldHaveSecondaryENI -}} + - path: /opt/bin/assume-advertised-hostname-with-eni + owner: root:root + permissions: 0700 + content: | + #!/bin/bash -vxe + + # To omit the `--region {{.Region}}` flag for every aws-cli invocation + export AWS_DEFAULT_REGION={{.Region}} + + instance_id=$(curl http://169.254.169.254/latest/meta-data/instance-id) + network_interface_id=$1 + + # Persist outputs from awscli instead of just capturing them into shell variables and then echoing, + # so that we can make debugging easier while making it won't break when + # a possible huge output from awscli exceeds the bash limit of ARG_MAX + state_prefix=/var/run/coreos/network-interface + state_attached=${state_prefix}-attached.json + state_attachment_id=${state_prefix}-attachment-id + state_attachment=${state_prefix}-attachment.json + state_attachment_status=${state_prefix}-status + state_network_interface=${state_prefix}.json + + aws ec2 describe-network-interfaces \ + --network-interface-id $network_interface_id \ + | jq -r '.NetworkInterfaces[0]' \ + > $state_network_interface + + attached=$( + cat $state_network_interface \ + | jq -r 'select(.Attachment.InstanceId) | "yes"' \ + ) + + if [ "$attached" != yes ]; then + aws ec2 attach-network-interface \ + --network-interface-id $network_interface_id \ + --instance-id $instance_id \ + --device-index {{$.Etcd.NetworkInterfaceDeviceIndex}} \ + > $state_attached + fi + + until [ "$status" = attached ]; do + sleep 3 + + aws ec2 describe-network-interface-attribute \ + --network-interface-id $network_interface_id \ + --attribute attachment \ + > $state_attachment + + cat $state_attachment \ + | jq -r '.Attachment.Status' \ + > $state_attachment_status + + status=$(cat $state_attachment_status) + done + + aws ec2 describe-network-interfaces \ + --network-interface-id $network_interface_id \ + > $state_network_interface + + cat $state_network_interface \ + | jq -r '.NetworkInterfaces[0].PrivateIpAddresses[] | select(.Primary == true).PrivateIpAddress' \ + > /var/run/coreos/listen-private-ip + + - path: /opt/bin/reconfigure-ip-routing + owner: root:root + permissions: 0700 + content: | + #!/bin/bash -vxe + + # Reconfigure ip routes and rules so that etcd can communicate via the newly attached ENI + # Otherwise, an etcd process ends up producing `publish error: etcdserver: request timed out` errors repeatedly and + # the etcd cluster never come up + + primary_ip=$(curl http://169.254.169.254/latest/meta-data/local-ipv4) + secondary_ip=$(cat /var/run/coreos/listen-private-ip) + + # There's some possibility that the network interface kept configuring thus unable to be used at all. + # Anyway, set the device down and then up to see if it alleviates the issue. + # See https://gist.github.com/mumoshu/2e82cab514dd82e165df4ca223f554e2 for how it looked like when happened + device=eth{{.Etcd.NetworkInterfaceDeviceIndex}} + + networkctl status $device + ip link set $device down + ip link set $device up + + configured=1 + while [ $configured -ne 0 ]; do + sleep 3 + networkctl status $device + networkctl status $device | grep State | grep routable + configured=$? + done + + # Dump various ip configs for debugging purpose + ip rule show + ip route show table main + + # TODO: Use subnet CIDR +1 instead? + default_gw_for_subnet=$(ip route show | grep default | sed 's/default\svia \([0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\) .*/\1/' | head -n 1) + + ip route add default via $default_gw_for_subnet dev eth0 tab 1 + ip route add default via $default_gw_for_subnet dev $device tab 2 + + ip rule add from $primary_ip/32 tab 1 priority 500 + ip rule add from $secondary_ip/32 tab 2 priority 600 + + # Clear the rule from eth0 to subnets inside the VPC from the default table to so that packets to other etcd nodes goes through the newly attached ENI + # Without losing internet connectivity provided via eth0(which has a public IP when this EC2 instance is in a public subnet) + ip route show | grep eth0 | grep -v metric | while read -r route; do ip route del ${route}; done + + ip route show + {{- end }} + + {{if $.Etcd.NodeShouldHaveEIP -}} + - path: /opt/bin/assume-advertised-hostname-with-eip + owner: root:root + permissions: 0700 + content: | + #!/bin/bash -vxe + + # To omit the `--region {{.Region}}` flag for every aws-cli invocation + export AWS_DEFAULT_REGION={{.Region}} + + instance_id=$(curl http://169.254.169.254/latest/meta-data/instance-id) + eip_alloc_id=$1 + + aws ec2 associate-address --instance-id $instance_id --allocation-id $eip_alloc_id + + curl http://169.254.169.254/latest/meta-data/public-hostname + + curl http://169.254.169.254/latest/meta-data/local-ipv4 > /var/run/coreos/listen-private-ip + {{- end }} + + - path: /opt/bin/append-etcd-server-env + owner: root:root + permissions: 0700 + content: | + #!/bin/bash -vxe + + private_ip=$(cat /var/run/coreos/listen-private-ip) + name=$(cat /var/run/coreos/name) + advertised_hostname=$(cat /var/run/coreos/advertised-hostname) + + echo "KUBE_AWS_ASSUMED_HOSTNAME=$advertised_hostname + ETCD_NAME=$name + ETCD_PEER_TRUSTED_CA_FILE=/etc/etcd2/ssl/ca.pem + ETCD_PEER_CERT_FILE=/etc/etcd2/ssl/etcd.pem + ETCD_PEER_KEY_FILE=/etc/etcd2/ssl/etcd-key.pem + + ETCD_CLIENT_CERT_AUTH=true + ETCD_TRUSTED_CA_FILE=/etc/etcd2/ssl/ca.pem + ETCD_CERT_FILE=/etc/etcd2/ssl/etcd.pem + ETCD_KEY_FILE=/etc/etcd2/ssl/etcd-key.pem + + ETCD_INITIAL_CLUSTER_STATE=new + ETCD_DATA_DIR=/var/lib/etcd2 + ETCD_LISTEN_CLIENT_URLS=https://$private_ip:2379 + ETCD_ADVERTISE_CLIENT_URLS=https://$advertised_hostname:2379 + ETCD_LISTEN_PEER_URLS=https://$private_ip:2380 + ETCD_INITIAL_ADVERTISE_PEER_URLS=https://$advertised_hostname:2380" >> /var/run/coreos/etcd-environment + - path: /opt/bin/cfn-etcd-environment owner: root:root permissions: 0700 content: | #!/bin/bash -e - rkt run \ - --volume=dns,kind=host,source=/etc/resolv.conf,readOnly=true \ - --mount volume=dns,target=/etc/resolv.conf \ - --volume=awsenv,kind=host,source=/var/run/coreos,readOnly=false \ - --mount volume=awsenv,target=/var/run/coreos \ - --uuid-file-save=/var/run/coreos/cfn-etcd-environment.uuid \ - --net=host \ - --trust-keys-from-https \ - {{.AWSCliImageRepo}}:{{.AWSCliTag}} --exec=/bin/bash -- \ - -ec \ - 'instance_id=$(curl http://169.254.169.254/latest/meta-data/instance-id) - stack_name=$( - aws ec2 describe-tags --region {{.Region}} --filters \ - "Name=resource-id,Values=$instance_id" \ - "Name=key,Values=aws:cloudformation:stack-name" \ - --output json \ - | jq -r ".Tags[].Value" - ) - cfn-init -v -c "etcd-server" --region {{.Region}} --resource {{.Controller.LogicalName}} --stack $stack_name - ' + run() { + rkt run \ + --volume=dns,kind=host,source=/etc/resolv.conf,readOnly=true \ + --mount volume=dns,target=/etc/resolv.conf \ + --volume=awsenv,kind=host,source=/var/run/coreos,readOnly=false \ + --mount volume=awsenv,target=/var/run/coreos \ + --volume=optbin,kind=host,source=/opt/bin,readOnly=false \ + --mount volume=optbin,target=/opt/bin \ + --uuid-file-save=/var/run/coreos/$1.uuid \ + --set-env={{.StackNameEnvVarName}}=${{.StackNameEnvVarName}} \ + --set-env={{.EtcdIndexEnvVarName}}=${{.EtcdIndexEnvVarName}} \ + --net=host \ + --trust-keys-from-https \ + {{.AWSCliImageRepo}}:{{.AWSCliTag}} --exec=/opt/bin/$1 -- $2 + + rkt rm --uuid-file=/var/run/coreos/$1.uuid || : + } + + run cfn-init-etcd-server + run attach-etcd-volume + + eip_allocation_id=$(cat /var/run/coreos/eip-allocation-id) + network_interface_id=$(cat /var/run/coreos/network-interface-id) + if [ "$eip_allocation_id" != "" ]; then + run assume-advertised-hostname-with-eip $eip_allocation_id + elif [ "$network_interface_id" != "" ]; then + run assume-advertised-hostname-with-eni $network_interface_id + /opt/bin/reconfigure-ip-routing + else + echo '[bug] neither eip_allocation_id nor network_interface_id for this node found' + fi + + run append-etcd-server-env + + /usr/bin/sed -i "s/^ETCDCTL_ENDPOINT.*$/ETCDCTL_ENDPOINT=https:\/\/$(cat /var/run/coreos/advertised-hostname):2379/" /etc/environment - rkt rm --uuid-file=/var/run/coreos/cfn-etcd-environment.uuid || : - path: /etc/environment permissions: 0644 @@ -180,6 +462,32 @@ write_files: echo "volume $1 is already formatted" fi +{{ if .WaitSignal.Enabled }} + - path: /opt/bin/cfn-signal + owner: root:root + permissions: 0700 + content: | + #!/bin/bash -e + + rkt run \ + --volume=dns,kind=host,source=/etc/resolv.conf,readOnly=true \ + --mount volume=dns,target=/etc/resolv.conf \ + --volume=awsenv,kind=host,source=/var/run/coreos,readOnly=false \ + --mount volume=awsenv,target=/var/run/coreos \ + --uuid-file-save=/var/run/coreos/cfn-signal.uuid \ + --set-env={{.StackNameEnvVarName}}=${{.StackNameEnvVarName}} \ + --set-env={{.EtcdIndexEnvVarName}}=${{.EtcdIndexEnvVarName}} \ + --net=host \ + --trust-keys-from-https \ + {{.AWSCliImageRepo}}:{{.AWSCliTag}} --exec=/bin/bash -- \ + -vxec \ + ' + cfn-signal -e 0 --region {{.Region}} --resource {{.Etcd.LogicalName}}${{.EtcdIndexEnvVarName}} --stack ${{.StackNameEnvVarName}} + ' + + rkt rm --uuid-file=/var/run/coreos/cfn-signal.uuid || : +{{end}} + {{ if .ManageCertificates }} - path: /etc/etcd2/ssl/ca.pem.enc diff --git a/core/controlplane/config/templates/cluster.yaml b/core/controlplane/config/templates/cluster.yaml index b0fa644b8..12ae81de8 100644 --- a/core/controlplane/config/templates/cluster.yaml +++ b/core/controlplane/config/templates/cluster.yaml @@ -302,6 +302,55 @@ worker: # # References subnets defined under the top-level `subnets` key by their names # - name: ManagedPrivateSubnet1 # - name: ManagedPrivateSubnet2 +# +# # The strategy to provide your etcd nodes, in combination with floating EBS volumes, stable member identities. Defaults to "eip". +# # +# # Available options: eip, eni +# # +# # With every option, etcd nodes communicate to each other via their private IPs. +# # +# # eip: Use EC2 public hostnames stabilized with EIPs and "resolved eventually to private IPs" +# # Requires Amazon DNS (at the second IP of your VPC, e.g. 10.0.0.2) to work. +# # 1st recommendation because less moving parts and relatively easy disaster recovery for a single-AZ etcd cluster. +# # If you run a single-AZ etcd cluster and the AZ failed, EBS volumes created from latest snapshots and EIPs can be reused in an another AZ to reproduce your etcd cluster in the AZ. +# # +# # eni: [EXPERIMENTAL] Use secondary ENIs and Route53 record sets to provide etcd nodes stable hostnames +# # Requires Amazon DNS (at the second IP of your VPC, e.g. 10.0.0.2) or an another DNS which can resolve dns names in the hosted zone managed by kube-aws to work. +# # 2nd recommendation because relatively easy disaster recovery for a single-AZ etcd cluster but more moving parts than "eip". +# # If you run a single-AZ etcd cluster and the AZ failed, EBS volumes created from latest snapshots and record sets can be reused in an another AZ to reproduce your etcd cluster in the AZ. +# # +# memberIdentityProvider: eip +# +# # Domain of the hostname used for etcd peer discovery. +# # Used only when `memberIdentityProvider: eni` for TLS key/cert generation +# # If omitted, defaults to "ec2.internal" for us-east-1 region and ".compute.internal" otherwise +# internalDomainName: +# +# # Set to `false` to disable creation of record sets. +# # Used only when `memberIdentityProvider` is set to `eni` +# # When disabled, it's your responsibility to configure all the etcd nodes so that +# # they can resolve each other's FQDN(specified via the below `etcd.nods[].fqdn` settings) via your DNS(can be the Amazon DNS or your own DNS. Configure it with e.g. coroes-cloudinit) +# manageRecordSets: +# +# # Advanced configuration used only when `memberIdentityProvider: eni` +# hostedZone: +# # The hosted zone where record sets for etcd nodes managed by kube-aws are created +# # If omitted, kube-aws creates a hosted zone for you +# id: +# +# # CAUTION: Advanced configuration. This should be omitted unless you have very deep knowledge of etcd and kube-aws +# nodes: +# - # The name of this etcd node. Specified only when you want to customize the etcd member's name shown in ETCD_INITIAL_CLUSTER and ETCD_NAME +# name: etcd0 +# # The FQDN of this etcd node +# # Usually this should be omitted so that kube-aws can choose a proper value. +# # Specified only when `memberIdentityProvider: eni` and `manageRecordSets: false` i.e. +# # it is your responsibility to properly configure EC2 instances to use a DNS which is able to resolve the FQDN. +# fqdn: etcd0. +# - name: etcd1 +# fqdn: etcd1. +# - name: etcd2 +# fqdn: etcd2. # Instance type for etcd node # etcdInstanceType: t2.medium diff --git a/core/controlplane/config/templates/stack-template.json b/core/controlplane/config/templates/stack-template.json index 73144dc9f..bd946bf8f 100644 --- a/core/controlplane/config/templates/stack-template.json +++ b/core/controlplane/config/templates/stack-template.json @@ -1,13 +1,6 @@ { "AWSTemplateFormatVersion": "2010-09-09", "Description": "kube-aws Kubernetes cluster {{.ClusterName}}", - "Mappings" : { - "EtcdInstanceParams" : { - "UserData" : { - "cloudconfig" : "{{.UserDataEtcd}}" - } - } - }, "Resources": { "{{.Controller.LogicalName}}": { "Type": "AWS::AutoScaling::AutoScalingGroup", @@ -69,7 +62,6 @@ "Metadata" : { "AWS::CloudFormation::Init" : { "configSets" : { - "etcd-server": [ "etcd-server-env" ], "etcd-client": [ "etcd-client-env" ]{{if .Experimental.AwsEnvironment.Enabled}}, "aws-environment": [ "aws-environment-env" ]{{end}} }, @@ -86,32 +78,15 @@ } }, {{ end }} - "etcd-server-env": { - "files" : { - "/var/run/coreos/etcd-environment": { - "content": { "Fn::Join" : [ "", [ - "ETCD_INITIAL_CLUSTER='", - {{range $index, $_ := $.EtcdInstances}} - {{if $index}}",", {{end}} - { "Fn::GetAtt" : [ "InstanceEtcd{{$index}}", "PrivateDnsName" ] }, - "=https://", - { "Fn::GetAtt" : [ "InstanceEtcd{{$index}}", "PrivateDnsName" ] }, - ":2380", - {{end}} - "'\n" - ]]} - } - } - }, "etcd-client-env": { "files" : { "/var/run/coreos/etcd-environment": { "content": { "Fn::Join" : [ "", [ "ETCD_ENDPOINTS='", -{{range $index, $_ := $.EtcdInstances}} + {{range $index, $etcdInstance := $.EtcdNodes}} {{if $index}}",", {{end}} "https://", - { "Fn::GetAtt" : [ "InstanceEtcd{{$index}}", "PrivateDnsName" ] }, ":2379", -{{end}} + {{$etcdInstance.AdvertisedFQDNRef}}, ":2379", + {{end}} "'\n" ]]} } @@ -119,7 +94,7 @@ } } }, - "DependsOn": ["InstanceEtcd0"] + "DependsOn": ["{{$.Etcd.LogicalName}}{{minus $.EtcdCount 1}}"] }, {{ if .CreateRecordSet }} "ExternalDNS": { @@ -305,6 +280,57 @@ "Action": "ec2:DescribeTags", "Effect": "Allow", "Resource": "*" + }, + {{/* Required for cfn-etcd-environment.service to discover the volume */}} + { + "Action": "ec2:DescribeVolumes", + "Effect": "Allow", + "Resource": "*" + }, + {{/* Required for cfn-etcd-environment.service to start attaching the volume */}} + { + "Action": "ec2:AttachVolume", + "Effect": "Allow", + "Resource": "*" + }, + {{/* Required for cfn-etcd-environment.service to wait until the volume is attached */}} + { + "Action": "ec2:DescribeVolumeStatus", + "Effect": "Allow", + "Resource": "*" + }, + {{if $.Etcd.NodeShouldHaveEIP -}} + {{/* Required for cfn-etcd-environment.service to associate an EIP */}} + { + "Action": "ec2:AssociateAddress", + "Effect": "Allow", + "Resource": "*" + }, + {{end -}} + {{if $.Etcd.NodeShouldHaveSecondaryENI -}} + {{/* Required for cfn-etcd-environment.service to associate a network interface */}} + { + "Action": "ec2:AttachNetworkInterface", + "Effect": "Allow", + "Resource": "*" + }, + { + "Action": "ec2:DescribeNetworkInterfaces", + "Effect": "Allow", + "Resource": "*" + }, + { + "Action": "ec2:DescribeNetworkInterfaceAttribute", + "Effect": "Allow", + "Resource": "*" + }, + {{end -}} + { + "Effect": "Allow", + "Action": [ + "s3:GetObject" + ], + "Resource": "arn:aws:s3:::{{$.UserDataEtcdS3Path}}" } ], "Version": "2012-10-17" @@ -316,8 +342,43 @@ "Type": "AWS::IAM::Role" }, - {{range $etcdIndex, $etcdInstance := .EtcdInstances}} - "InstanceEtcd{{$etcdIndex}}eni": { + {{if $.Etcd.HostedZoneManaged}} + "{{$.Etcd.HostedZoneLogicalName}}": { + "Type": "AWS::Route53::HostedZone", + "Properties": { + "HostedZoneConfig": { + "Comment": "My hosted zone for {{$.Etcd.InternalDomainName}}" + }, + "Name": "{{$.Etcd.InternalDomainName}}", + "VPCs": [{ + "VPCId": {{$.VPCRef}}, + "VPCRegion": { "Ref": "AWS::Region" } + }], + "HostedZoneTags" : [{ + "Key": "KubernetesCluster", + "Value": "{{$.ClusterName}}" + }] + } + }, + {{end}} + {{range $etcdIndex, $etcdInstance := .EtcdNodes}} + {{if $etcdInstance.RecordSetManaged}} + "{{$etcdInstance.RecordSetLogicalName}}" : { + "Type" : "AWS::Route53::RecordSet", + "Properties" : { + "HostedZoneId": {{$.Etcd.HostedZoneRef}}, + "Name": {{$etcdInstance.AdvertisedFQDNRef}}, + "Comment" : "A record for the private IP address of Etcd node named {{$etcdInstance.Name}} at index {{$etcdIndex}}", + "Type" : "A", + "TTL" : "300", + "ResourceRecords" : [ + {{$etcdInstance.NetworkInterfacePrivateIPRef}} + ] + } + }, + {{end}} + {{if $etcdInstance.NetworkInterfaceManaged}} + "{{$etcdInstance.NetworkInterfaceLogicalName}}": { "Properties": { "SubnetId": {{$etcdInstance.SubnetRef}}, "GroupSet": [ @@ -328,20 +389,141 @@ }, "Type": "AWS::EC2::NetworkInterface" }, + {{end}} + {{if $etcdInstance.EIPManaged}} + "{{$etcdInstance.EIPLogicalName}}": { + "Properties": { + "Domain": "vpc" + }, + "Type": "AWS::EC2::EIP" + }, + {{end}} {{if not $.EtcdDataVolumeEphemeral}} - "InstanceEtcd{{$etcdIndex}}ebs": { + "{{$etcdInstance.EBSLogicalName}}": { "Properties": { - "AvailabilityZone": "{{ $etcdInstance.SubnetAvailabilityZone}}", + "AvailabilityZone": "{{$etcdInstance.SubnetAvailabilityZone}}", "Size": "{{$.EtcdDataVolumeSize}}", {{if gt $.EtcdDataVolumeIOPS 0}} "Iops": "{{$.EtcdDataVolumeIOPS}}", {{end}} - "VolumeType": "{{$.EtcdDataVolumeType}}" + "VolumeType": "{{$.EtcdDataVolumeType}}", + "Tags": [ + { + "Key": "kube-aws:etcd:index", + "Value": "{{$etcdIndex}}" + }, + {{if $etcdInstance.EIPManaged}}{ + "Key": "{{$.Etcd.EIPAllocationIDTagKey}}", + "Value": {{$etcdInstance.EIPAllocationIDRef}} + },{{end}} + {{if $etcdInstance.NetworkInterfaceManaged}}{ + "Key": "{{$.Etcd.NetworkInterfaceIDTagKey}}", + "Value": {{$etcdInstance.NetworkInterfaceIDRef}} + },{{end}} + { + "Key": "{{$.Etcd.AdvertisedFQDNTagKey}}", + "Value": {{$etcdInstance.AdvertisedFQDNRef}} + }, + { + "Key": "{{$.Etcd.NameTagKey}}", + "Value": "{{$etcdInstance.Name}}" + } + ] }, "Type": "AWS::EC2::Volume" }, {{end}} - "InstanceEtcd{{$etcdIndex}}": { + "{{$etcdInstance.LogicalName}}": { + "Type": "AWS::AutoScaling::AutoScalingGroup", + "Properties": { + "HealthCheckGracePeriod": 600, + "HealthCheckType": "EC2", + "LaunchConfigurationName": { + "Ref": "{{$etcdInstance.LaunchConfigurationLogicalName}}" + }, + "MaxSize": "1", + "MetricsCollection": [ + { + "Granularity": "1Minute" + } + ], + "MinSize": "1", + "Tags": [ + { + "Key": "KubernetesCluster", + "PropagateAtLaunch": "true", + "Value": "{{$.ClusterName}}" + }, + { + "Key": "Name", + "PropagateAtLaunch": "true", + "Value": "{{$.ClusterName}}-{{$.StackName}}-kube-aws-etcd-{{$etcdIndex}}" + } + ], + "VPCZoneIdentifier": [ + {{$etcdInstance.SubnetRef}} + ] + }, + {{if $.WaitSignal.Enabled}} + "CreationPolicy" : { + "ResourceSignal" : { + "Count" : "1", + "Timeout" : "{{$.ControllerCreateTimeout}}" + } + }, + {{end}} + "UpdatePolicy" : { + "AutoScalingRollingUpdate" : { + "MinInstancesInService" : "0", + "MaxBatchSize" : "1", + {{if $.WaitSignal.Enabled}} + "WaitOnResourceSignals" : "true", + "PauseTime": "{{$.ControllerCreateTimeout}}" + {{else}} + "PauseTime": "PT2M" + {{end}} + } + }, + "Metadata" : { + "AWS::CloudFormation::Init" : { + "configSets" : { + "etcd-server": [ "etcd-server-env" ] + }, + "etcd-server-env": { + "files" : { + "/var/run/coreos/etcd-environment": { + "content": { "Fn::Join" : [ "", [ + "ETCD_INITIAL_CLUSTER='", + {{range $etcdIndex, $etcdInstance := $.EtcdNodes}} + {{if $etcdIndex}}",", {{end}} + "{{$etcdInstance.Name}}", + "=https://", + {{$etcdInstance.AdvertisedFQDNRef}}, + ":2380", + {{end}} + "'\n" + ]]} + } + } + } + } + }, + "DependsOn": [ + {{if $etcdInstance.DependencyExists}}{{$etcdInstance.DependencyRef}},{{end}} + {{if $etcdIndex}}"{{$.Etcd.LogicalName}}{{minus $etcdIndex 1}}",{{end}} + {{if $etcdInstance.EIPManaged}} + "{{$etcdInstance.EIPLogicalName}}", + {{end}} + {{if $etcdInstance.NetworkInterfaceManaged}} + "{{$etcdInstance.NetworkInterfaceLogicalName}}", + {{end}} + {{if $etcdInstance.RecordSetManaged}} + "{{$etcdInstance.RecordSetLogicalName}}", + {{end}} + "{{$etcdInstance.EBSLogicalName}}" + ] + }, + "{{$etcdInstance.LaunchConfigurationLogicalName}}": { "Properties": { "BlockDeviceMappings": [ { @@ -368,35 +550,31 @@ "ImageId": "{{$.AMI}}", "InstanceType": "{{$.EtcdInstanceType}}", {{if $.KeyName}}"KeyName": "{{$.KeyName}}",{{end}} - "NetworkInterfaces": [ - { - "NetworkInterfaceId": { "Ref": "InstanceEtcd{{$etcdIndex}}eni" }, - "DeviceIndex": "0" - } - ], - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "{{$.ClusterName}}" - }, + "SecurityGroups": [ { - "Key": "Name", - "Value": "{{$.ClusterName}}-{{$.StackName}}-kube-aws-etcd-{{$etcdIndex}}" + "Ref": "SecurityGroupEtcd" } ], - "Tenancy": "{{$.EtcdTenancy}}", - "UserData": { "Fn::FindInMap" : [ "EtcdInstanceParams", "UserData", "cloudconfig"] } - {{if not $.EtcdDataVolumeEphemeral}} - , - "Volumes": [ - { "VolumeId": { "Ref": "InstanceEtcd{{$etcdIndex}}ebs" }, "Device": "/dev/xvdf" } - ] - {{end}} + "PlacementTenancy": "{{$.EtcdTenancy}}", + "UserData": { "Fn::Base64": { "Fn::Join" : ["\n", [ + "#!/bin/bash", + "# userdata hash: {{ $.UserDataEtcd | sha1 }}", + {"Fn::Join":["",[ "echo '{{$.StackNameEnvVarName}}=", { "Ref": "AWS::StackName" }, "' >> {{$.EtcdNodeEnvFileName}}" ]]}, + "echo '{{$.EtcdIndexEnvVarName}}={{$etcdIndex}}' >> {{$.EtcdNodeEnvFileName}}", + " . /etc/environment", + "export COREOS_PRIVATE_IPV4 COREOS_PRIVATE_IPV6 COREOS_PUBLIC_IPV4 COREOS_PUBLIC_IPV6", + "REGION=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r '.region')", + "USERDATA_FILE=userdata-etcd", + "/usr/bin/rkt run \\", + " --net=host \\", + " --volume=dns,kind=host,source=/etc/resolv.conf,readOnly=true --mount volume=dns,target=/etc/resolv.conf \\", + " --volume=awsenv,kind=host,source=/var/run/coreos,readOnly=false --mount volume=awsenv,target=/var/run/coreos \\", + " --trust-keys-from-https \\", + " {{$.AWSCliImageRepo}}:{{$.AWSCliTag}} -- aws s3 --region $REGION cp {{ $.S3URI }}/{{ $.StackName }}/$USERDATA_FILE /var/run/coreos/", + "exec /usr/bin/coreos-cloudinit --from-file /var/run/coreos/$USERDATA_FILE" + ]]}} }, - {{if $etcdInstance.DependencyExists}} - "DependsOn": [{{$etcdInstance.DependencyRef}}], - {{end}} - "Type": "AWS::EC2::Instance" + "Type": "AWS::AutoScaling::LaunchConfiguration" }, {{end}} "{{.Controller.LogicalName}}LC": { @@ -1098,6 +1276,22 @@ }, {{end}} {{end}} + {{range $index, $etcdInstance := $.EtcdNodes}} + {{if $etcdInstance.EIPManaged}} + "{{$etcdInstance.EIPLogicalName}}": { + "Description": "The EIP for etcd node {{$index}}", + "Value": {{$etcdInstance.EIPRef}}, + "Export": { "Name" : {"Fn::Sub": "${AWS::StackName}-{{$etcdInstance.EIPLogicalName}}" }} + }, + {{end}} + {{if $etcdInstance.NetworkInterfaceManaged}} + "{{$etcdInstance.NetworkInterfacePrivateIPLogicalName}}": { + "Description": "The private IP for etcd node {{$index}}", + "Value": {{$etcdInstance.NetworkInterfacePrivateIPRef}}, + "Export": { "Name" : {"Fn::Sub": "${AWS::StackName}-{{$etcdInstance.NetworkInterfacePrivateIPLogicalName}}" }} + }, + {{end}} + {{end}} "WorkerSecurityGroup" : { "Description" : "The security group assigned to worker nodes", "Value" : { "Ref" : "SecurityGroupWorker" }, @@ -1106,13 +1300,6 @@ "StackName": { "Description": "The name of this stack which is used by node pool stacks to import outputs from this stack", "Value": { "Ref": "AWS::StackName" } - }, - {{range $index, $_ := $.EtcdInstances}} - {{if $index}},{{end}}"InstanceEtcd{{$index}}PrivateDnsName": { - "Description": "The resolvable hostname of etcd node {{$index}}", - "Value": { "Fn::GetAtt" : [ "InstanceEtcd{{$index}}", "PrivateDnsName" ] }, - "Export": { "Name" : "{{$.ClusterName}}-InstanceEtcd{{$index}}PrivateDnsName" } } - {{end}} } } diff --git a/core/controlplane/config/tls_config.go b/core/controlplane/config/tls_config.go index 9424abd51..f23894e89 100644 --- a/core/controlplane/config/tls_config.go +++ b/core/controlplane/config/tls_config.go @@ -130,10 +130,7 @@ func (c *Cluster) NewTLSAssets(caKey *rsa.PrivateKey, caCert *x509.Certificate) etcdConfig := tlsutil.ServerCertConfig{ CommonName: "kube-etcd", - DNSNames: []string{ - fmt.Sprintf("*.%s.compute.internal", c.Region), - "*.ec2.internal", - }, + DNSNames: c.EtcdCluster().DNSNames(), //etcd https client/peer interfaces are not exposed externally //will live the full year with the CA Duration: tlsutil.Duration365d, diff --git a/core/nodepool/config/config.go b/core/nodepool/config/config.go index fe5504fd9..7c731fb1f 100644 --- a/core/nodepool/config/config.go +++ b/core/nodepool/config/config.go @@ -12,6 +12,7 @@ import ( "github.com/coreos/kube-aws/coreos/amiregistry" "github.com/coreos/kube-aws/filereader/userdatatemplate" "github.com/coreos/kube-aws/model" + "github.com/coreos/kube-aws/model/derived" "gopkg.in/yaml.v2" "strconv" ) @@ -43,7 +44,7 @@ type DeploymentSettings struct { } type MainClusterSettings struct { - EtcdInstances []model.EtcdInstance + EtcdNodes []derived.EtcdNode } type StackTemplateOptions struct { @@ -178,7 +179,7 @@ define one or more public subnets in cluster.yaml or explicitly reference privat } } - c.EtcdInstances = main.EtcdInstances + c.EtcdNodes = main.EtcdNodes return nil } diff --git a/core/nodepool/config/templates/stack-template.json b/core/nodepool/config/templates/stack-template.json index 8942bf1a4..5b4a8b857 100644 --- a/core/nodepool/config/templates/stack-template.json +++ b/core/nodepool/config/templates/stack-template.json @@ -43,9 +43,10 @@ "/var/run/coreos/etcd-environment": { "content": { "Fn::Join" : [ "", [ "ETCD_ENDPOINTS='", - {{range $index, $_ := $.EtcdInstances}} + {{range $index, $instance := $.EtcdNodes}} {{if $index}}",", {{end}} "https://", - { "Fn::ImportValue" : "{{$.ClusterName}}-InstanceEtcd{{$index}}PrivateDnsName" }, ":2379", + {{$instance.ImportedAdvertisedFQDNRef}}, + ":2379", {{end}} "'\n" ]]} diff --git a/e2e/README.md b/e2e/README.md index 6ea35660b..0921cddbb 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -78,6 +78,10 @@ The test cluster created by the `e2e/run` script can be customized via various e `KUBE_AWS_USE_CALICO=true`: Enable Calico throughout the cluster. -`KUBE_AWS_CLUSTER_NAME=mycluster`: The name of a kube-aws main cluster i.e. a cloudformation stack for the main cluster. Must be unique in your AWS account. +`ETCD_COUNT=1`: Number of etcd nodes in the cluster. Defaults to 1. Set to an odd number greater than or equal to 3 for testing a H/A etcd cluster + +`ETCD_MEMBER_IDENTITY_PROVIDER=`: "eni" for using secondary NICs=ENIs for etcd node=member identity. "eip" for using EIPs and Amazon DNS. +`ETCD_INTERNAL_DOMAIN_NAME=internal.example.com`: Used only when `ETCD_MEMBER_IDENTITY_PROVIDER` is set to "eni". See comments in `cluster.yaml` for more details. +`KUBE_AWS_CLUSTER_NAME=mycluster`: The name of a kube-aws main cluster i.e. a cloudformation stack for the main cluster. Must be unique in your AWS account. diff --git a/e2e/run b/e2e/run index dcbb5cc87..b8fe10b88 100755 --- a/e2e/run +++ b/e2e/run @@ -7,6 +7,7 @@ TESTINFRA_DIR=${E2E_DIR}/testinfra KUBE_AWS_TEST_INFRA_STACK_NAME=${KUBE_AWS_TEST_INFRA_STACK_NAME:-${KUBE_AWS_CLUSTER_NAME}-testinfra} SRC_DIR=$(cd $(dirname $0); cd ..; pwd) KUBECONFIG=${WORK_DIR}/kubeconfig +ETCD_COUNT=${ETCD_COUNT:-1} export KUBECONFIG @@ -99,7 +100,7 @@ configure() { echo 'useCalico: true' >> cluster.yaml fi - customize_worker + customize_cluster_yaml ${KUBE_AWS_CMD} render @@ -111,7 +112,7 @@ configure() { find . } -customize_worker() { +customize_cluster_yaml() { echo Writing to $(pwd)/cluster.yaml if [ "${KUBE_AWS_DEPLOY_TO_EXISTING_VPC}" != "" ]; then @@ -154,15 +155,15 @@ worker: loadBalancer: enabled: true names: - - $(testinfra_public_lb) + - $(testinfra_public_elb) securityGroupIds: - - $(testinfra_public_lb_backend_sg) + - $(testinfra_public_elb_backend_sg) targetGroup: enabled: true arns: - $(testinfra_target_group) securityGroupIds: - - $(testinfra_public_lb_backend_sg)" >> cluster.yaml + - $(testinfra_public_alb_backend_sg)" >> cluster.yaml fi echo -e " @@ -194,15 +195,15 @@ worker: loadBalancer: enabled: true names: - - $(testinfra_public_lb) + - $(testinfra_public_elb) securityGroupIds: - - $(testinfra_public_lb_backend_sg) + - $(testinfra_public_elb_backend_sg) targetGroup: enabled: true arns: - $(testinfra_target_group) securityGroupIds: - - $(testinfra_public_lb_backend_sg)" >> cluster.yaml + - $(testinfra_public_alb_backend_sg)" >> cluster.yaml fi echo -e " @@ -211,7 +212,20 @@ waitSignal: enabled: true awsNodeLabels: enabled: true +etcdCount: $ETCD_COUNT " >> cluster.yaml + + if [ "${ETCD_MEMBER_IDENTITY_PROVIDER}" != "" ]; then + echo -e " +# etcd configuration +etcd: + memberIdentityProvider: ${ETCD_MEMBER_IDENTITY_PROVIDER} +" >> cluster.yaml + fi + + if [ "${ETCD_INTERNAL_DOMAIN_NAME}" != "" ]; then + echo -e " internalDomainName: ${ETCD_INTERNAL_DOMAIN_NAME}" >> cluster.yaml + fi } clean() { @@ -227,7 +241,7 @@ up() { starttime=$(date +%s) - ${KUBE_AWS_CMD} up --s3-uri ${KUBE_AWS_S3_URI} + ${KUBE_AWS_CMD} up --s3-uri ${KUBE_AWS_S3_URI} --pretty-print set +vx @@ -378,7 +392,7 @@ testinfra_up() { aws cloudformation create-stack \ --template-body file://$(pwd)/stack-template.yaml \ --stack-name ${KUBE_AWS_TEST_INFRA_STACK_NAME} \ - --parameter ParameterKey=AZ1,ParameterValue=${KUBE_AWS_AZ_1} + --parameter ParameterKey=AZ1,ParameterValue=${KUBE_AWS_AZ_1} ParameterKey=Id,ParameterValue=${KUBE_AWS_CLUSTER_NAME}-infra aws cloudformation wait stack-create-complete \ --stack-name ${KUBE_AWS_TEST_INFRA_STACK_NAME} } @@ -412,12 +426,16 @@ testinfra_public_routetable() { testinfra_output PublicRouteTable } -testinfra_public_lb_backend_sg() { - testinfra_output PublicLBBackendSG +testinfra_public_elb_backend_sg() { + testinfra_output PublicELBBackendSG +} + +testinfra_public_alb_backend_sg() { + testinfra_output PublicALBBackendSG } -testinfra_public_lb() { - testinfra_output PublicLB +testinfra_public_elb() { + testinfra_output PublicELB } testinfra_target_group() { diff --git a/e2e/testinfra/stack-template.yaml b/e2e/testinfra/stack-template.yaml index c21e58bba..c234f6b10 100644 --- a/e2e/testinfra/stack-template.yaml +++ b/e2e/testinfra/stack-template.yaml @@ -125,13 +125,13 @@ Resources: DestinationCidrBlock: 0.0.0.0/0 NatGatewayId: !Ref NAT - PublicLB: + PublicELB: Type: AWS::ElasticLoadBalancing::LoadBalancer Properties: Subnets: - !Ref PublicSubnet1 SecurityGroups: - - !Ref PublicLBSG + - !Ref PublicELBSG Listeners: - LoadBalancerPort: "443" InstancePort: "80" @@ -151,13 +151,13 @@ Resources: - Key: Name Value: !Join [ "-", [ "Ref":"Id" , "public" ] ] - PrivateLB: + PrivateELB: Type: AWS::ElasticLoadBalancing::LoadBalancer Properties: Subnets: - !Ref PrivateSubnet1 SecurityGroups: - - !Ref PrivateLBSG + - !Ref PrivateELBSG Listeners: - LoadBalancerPort: "80" InstancePort: "80" @@ -192,7 +192,7 @@ Resources: UnhealthyThresholdCount: "5" VpcId: !Ref VPC - PrivateLBSG: + PrivateELBSG: Type: AWS::EC2::SecurityGroup Properties: GroupDescription: !Join [ "-", [ "Ref":"Id" , "private-lb" ] ] @@ -203,7 +203,7 @@ Resources: CidrIp: 0.0.0.0/0 VpcId: !Ref VPC - PublicLBSG: + PublicELBSG: Type: AWS::EC2::SecurityGroup Properties: GroupDescription: !Join [ "-", [ "Ref":"Id" , "public-lb" ] ] @@ -214,26 +214,37 @@ Resources: CidrIp: 0.0.0.0/0 VpcId: !Ref VPC - PrivateLBBackendSG: + PrivateELBBackendSG: Type: AWS::EC2::SecurityGroup Properties: - GroupDescription: !Join [ "-", [ "Ref":"Id" , "private-lb-backend" ] ] + GroupDescription: !Join [ "-", [ "Ref":"Id" , "private-elb-backend" ] ] SecurityGroupIngress: - IpProtocol: tcp FromPort: '80' ToPort: '80' - SourceSecurityGroupId: !Ref PrivateLBSG + SourceSecurityGroupId: !Ref PrivateELBSG VpcId: !Ref VPC - PublicLBBackendSG: + PublicALBBackendSG: Type: AWS::EC2::SecurityGroup Properties: - GroupDescription: !Join [ "-", [ "Ref":"Id" , "public-lb-backend" ] ] + GroupDescription: !Join [ "-", [ "Ref":"Id" , "public-alb-backend" ] ] SecurityGroupIngress: - IpProtocol: tcp FromPort: '80' ToPort: '80' - SourceSecurityGroupId: !Ref PublicLBSG + SourceSecurityGroupId: !Ref PublicELBSG + VpcId: !Ref VPC + + PublicELBBackendSG: + Type: AWS::EC2::SecurityGroup + Properties: + GroupDescription: !Join [ "-", [ "Ref":"Id" , "public-elb-backend" ] ] + SecurityGroupIngress: + - IpProtocol: tcp + FromPort: '80' + ToPort: '80' + SourceSecurityGroupId: !Ref PublicELBSG VpcId: !Ref VPC GlueSG: @@ -264,19 +275,21 @@ Outputs: Value: !Ref PublicRouteTable PrivateRouteTable: Value: !Ref PrivateRouteTable - PublicLBBackendSG: - Value: !Ref PublicLBBackendSG - PrivateLBBackendSG: - Value: !Ref PrivateLBBackendSG + PublicELBBackendSG: + Value: !Ref PublicELBBackendSG + PublicALBBackendSG: + Value: !Ref PublicALBBackendSG + PrivateELBBackendSG: + Value: !Ref PrivateELBBackendSG GlueSG: Value: !Ref GlueSG - PublicLB: - Value: !Ref PublicLB - PrivateLB: - Value: !Ref PrivateLB + PublicELB: + Value: !Ref PublicELB + PrivateELB: + Value: !Ref PrivateELB TargetGroup: Value: !Ref TargetGroup - PublicLBDNSName: - Value: !GetAtt PublicLB.DNSName - PrivateLBDNSName: - Value: !GetAtt PrivateLB.DNSName + PublicELBDNSName: + Value: !GetAtt PublicELB.DNSName + PrivateELBDNSName: + Value: !GetAtt PrivateELB.DNSName diff --git a/filereader/texttemplate/texttemplate.go b/filereader/texttemplate/texttemplate.go index 57df376d7..9198a8679 100644 --- a/filereader/texttemplate/texttemplate.go +++ b/filereader/texttemplate/texttemplate.go @@ -14,7 +14,8 @@ func GetBytesBuffer(filename string, data interface{}) (*bytes.Buffer, error) { return nil, err } funcMap := template.FuncMap{ - "sha1": func(v string) string { return fmt.Sprintf("%x", sha1.Sum([]byte(v))) }, + "sha1": func(v string) string { return fmt.Sprintf("%x", sha1.Sum([]byte(v))) }, + "minus": func(a, b int) int { return a - b }, } tmpl, err := template.New(filename).Funcs(funcMap).Parse(string(raw)) diff --git a/model/derived/etcd_cluster.go b/model/derived/etcd_cluster.go new file mode 100644 index 000000000..de3cead63 --- /dev/null +++ b/model/derived/etcd_cluster.go @@ -0,0 +1,47 @@ +package derived + +import ( + "fmt" + "github.com/coreos/kube-aws/model" +) + +type EtcdCluster struct { + model.EtcdCluster + Network + region model.Region + nodeCount int +} + +func NewEtcdCluster(config model.EtcdCluster, region model.Region, network Network, nodeCount int) EtcdCluster { + return EtcdCluster{ + EtcdCluster: config, + region: region, + Network: network, + nodeCount: nodeCount, + } +} + +func (c EtcdCluster) Region() model.Region { + return c.region +} + +func (c EtcdCluster) NodeCount() int { + return c.nodeCount +} + +func (c EtcdCluster) DNSNames() []string { + var dnsName string + if c.GetMemberIdentityProvider() == model.MemberIdentityProviderEIP { + // Used when `etcd.memberIdentityProvider` is set to "eip" + dnsName = fmt.Sprintf("*.%s", c.region.PublicDomainName()) + } + if c.GetMemberIdentityProvider() == model.MemberIdentityProviderENI { + if c.InternalDomainName != "" { + // Used when `etcd.memberIdentityProvider` is set to "eni" with non-empty `etcd.internalDomainName` + dnsName = fmt.Sprintf("*.%s", c.InternalDomainName) + } else { + dnsName = fmt.Sprintf("*.%s", c.region.PrivateDomainName()) + } + } + return []string{dnsName} +} diff --git a/model/derived/etcd_cluster_test.go b/model/derived/etcd_cluster_test.go new file mode 100644 index 000000000..957df6a2a --- /dev/null +++ b/model/derived/etcd_cluster_test.go @@ -0,0 +1,82 @@ +package derived + +import ( + "github.com/coreos/kube-aws/model" + "reflect" + "testing" +) + +func TestEtcdClusterDNSNames(t *testing.T) { + usEast1 := model.RegionForName("us-east-1") + usWest1 := model.RegionForName("us-west-1") + etcdNet := NewNetwork([]model.Subnet{}, []model.NATGateway{}) + etcdCount := 1 + + t.Run("WithENI", func(t *testing.T) { + t.Run("WithoutCustomDomain", func(t *testing.T) { + config := model.EtcdCluster{ + MemberIdentityProvider: "eni", + } + t.Run("us-east-1", func(t *testing.T) { + cluster := NewEtcdCluster(config, usEast1, etcdNet, etcdCount) + actual := cluster.DNSNames() + expected := []string{"*.ec2.internal"} + if !reflect.DeepEqual(actual, expected) { + t.Errorf("invalid dns names: expecetd=%v, got=%v", expected, actual) + } + }) + t.Run("us-west-1", func(t *testing.T) { + cluster := NewEtcdCluster(config, usWest1, etcdNet, etcdCount) + actual := cluster.DNSNames() + expected := []string{"*.us-west-1.compute.internal"} + if !reflect.DeepEqual(actual, expected) { + t.Errorf("invalid dns names: expecetd=%v, got=%v", expected, actual) + } + }) + }) + t.Run("WithCustomDomain", func(t *testing.T) { + config := model.EtcdCluster{ + MemberIdentityProvider: "eni", + InternalDomainName: "internal.example.com", + } + t.Run("us-east-1", func(t *testing.T) { + cluster := NewEtcdCluster(config, usEast1, etcdNet, etcdCount) + actual := cluster.DNSNames() + expected := []string{"*.internal.example.com"} + if !reflect.DeepEqual(actual, expected) { + t.Errorf("invalid dns names: expecetd=%v, got=%v", expected, actual) + } + }) + t.Run("us-west-1", func(t *testing.T) { + cluster := NewEtcdCluster(config, usWest1, etcdNet, etcdCount) + actual := cluster.DNSNames() + expected := []string{"*.internal.example.com"} + if !reflect.DeepEqual(actual, expected) { + t.Errorf("invalid dns names: expecetd=%v, got=%v", expected, actual) + } + }) + }) + }) + + t.Run("WithEIP", func(t *testing.T) { + config := model.EtcdCluster{ + MemberIdentityProvider: "eip", + } + t.Run("us-east-1", func(t *testing.T) { + cluster := NewEtcdCluster(config, usEast1, etcdNet, etcdCount) + actual := cluster.DNSNames() + expected := []string{"*.us-east-1.compute.amazonaws.com"} + if !reflect.DeepEqual(actual, expected) { + t.Errorf("invalid dns names: expecetd=%v, got=%v", expected, actual) + } + }) + t.Run("us-west-1", func(t *testing.T) { + cluster := NewEtcdCluster(config, usWest1, etcdNet, etcdCount) + actual := cluster.DNSNames() + expected := []string{"*.us-west-1.compute.amazonaws.com"} + if !reflect.DeepEqual(actual, expected) { + t.Errorf("invalid dns names: expecetd=%v, got=%v", expected, actual) + } + }) + }) +} diff --git a/model/derived/etcd_node.go b/model/derived/etcd_node.go new file mode 100644 index 000000000..64e1be6f1 --- /dev/null +++ b/model/derived/etcd_node.go @@ -0,0 +1,241 @@ +package derived + +import ( + "fmt" + "github.com/coreos/kube-aws/model" +) + +type EtcdNode interface { + AdvertisedFQDNRef() (string, error) + DependencyExists() bool + DependencyRef() (string, error) + EBSLogicalName() string + EBSRef() string + EIPAllocationIDRef() (string, error) + EIPLogicalName() (string, error) + EIPManaged() bool + EIPRef() (string, error) + // The name of the etcd member runs on this etcd node + Name() string + NetworkInterfaceIDRef() string + NetworkInterfaceLogicalName() string + NetworkInterfaceManaged() bool + NetworkInterfacePrivateIPRef() string + NetworkInterfacePrivateIPLogicalName() string + ImportedAdvertisedFQDNRef() (string, error) + LaunchConfigurationLogicalName() string + LogicalName() string + RecordSetManaged() bool + RecordSetLogicalName() string + SubnetRef() string +} + +type etcdNodeImpl struct { + cluster EtcdCluster + index int + config model.EtcdNode + subnet model.Subnet + natGateway model.NATGateway +} + +func NewEtcdNodeDependsOnManagedNGW(cluster EtcdCluster, index int, nodeConfig model.EtcdNode, s model.Subnet, ngw model.NATGateway) EtcdNode { + return etcdNodeImpl{ + cluster: cluster, + index: index, + config: nodeConfig, + subnet: s, + natGateway: ngw, + } +} + +func NewEtcdNode(cluster EtcdCluster, index int, nodeConfig model.EtcdNode, s model.Subnet) EtcdNode { + return etcdNodeImpl{ + cluster: cluster, + index: index, + config: nodeConfig, + subnet: s, + } +} + +func (i etcdNodeImpl) Name() string { + if i.config.Name != "" { + return i.config.Name + } + return fmt.Sprintf("etcd%d", i.index) +} + +func (i etcdNodeImpl) region() model.Region { + return i.cluster.Region() +} + +func (i etcdNodeImpl) customPrivateDNSName() string { + if i.config.FQDN != "" { + return i.config.FQDN + } + return fmt.Sprintf("%s.%s", i.Name(), i.cluster.InternalDomainName) +} + +func (i etcdNodeImpl) privateDNSNameRef() string { + if i.cluster.EC2InternalDomainUsed() { + return i.defaultPrivateDNSNameRefFromIPRef(i.NetworkInterfacePrivateIPRef()) + } + return fmt.Sprintf(`"%s"`, i.customPrivateDNSName()) +} + +func (i etcdNodeImpl) importedPrivateDNSNameRef() string { + if i.cluster.EC2InternalDomainUsed() { + return i.defaultPrivateDNSNameRefFromIPRef(fmt.Sprintf(`{ "Fn::ImportValue": {"Fn::Sub" : "${ControlPlaneStackName}-%s"} }`, i.NetworkInterfacePrivateIPLogicalName())) + } + return fmt.Sprintf(`"%s"`, i.customPrivateDNSName()) +} + +func (i etcdNodeImpl) defaultPrivateDNSNameRefFromIPRef(ipRef string) string { + hostnameRef := fmt.Sprintf(` + { "Fn::Join" : [ "-", + [ + "ip", + { "Fn::Join" : [ "-", + { "Fn::Split" : [ ".", %s ] } + ] } + ] + ]}`, ipRef) + return fmt.Sprintf(`{ "Fn::Join" : [ ".", [ + %s, + "%s" + ]]}`, hostnameRef, i.region().PrivateDomainName()) +} + +func (i etcdNodeImpl) defaultPublicDNSNameRef() (string, error) { + eipRef, err := i.EIPRef() + if err != nil { + return "", fmt.Errorf("failed to determine an ec2 default public dns name: %v", err) + } + return i.defaultPublicDNSNameRefFromIPRef(eipRef), nil +} + +func (i etcdNodeImpl) importedDefaultPublicDNSNameRef() (string, error) { + eipLogicalName, err := i.EIPLogicalName() + if err != nil { + return "", fmt.Errorf("failed to determine an ec2 default public dns name: %v", err) + } + eipRef := fmt.Sprintf(`{ "Fn::ImportValue": {"Fn::Sub" : "${ControlPlaneStackName}-%s"} }`, eipLogicalName) + return i.defaultPublicDNSNameRefFromIPRef(eipRef), nil +} + +func (i etcdNodeImpl) defaultPublicDNSNameRefFromIPRef(ipRef string) string { + return fmt.Sprintf(`{ "Fn::Join" : [ ".", [ + { "Fn::Join" : [ "-", [ + "ec2", + { "Fn::Join" : [ "-", { "Fn::Split" : [ ".", %s ] } ] } + ]]}, + "%s" + ]]}`, ipRef, i.region().PublicDomainName()) +} + +func (i etcdNodeImpl) AdvertisedFQDNRef() (string, error) { + if i.cluster.NodeShouldHaveSecondaryENI() { + return i.privateDNSNameRef(), nil + } + return i.defaultPublicDNSNameRef() +} + +func (i etcdNodeImpl) ImportedAdvertisedFQDNRef() (string, error) { + if i.cluster.NodeShouldHaveSecondaryENI() { + return i.importedPrivateDNSNameRef(), nil + } + return i.importedDefaultPublicDNSNameRef() +} + +func (i etcdNodeImpl) SubnetRef() string { + return i.subnet.Ref() +} + +func (i etcdNodeImpl) SubnetAvailabilityZone() string { + return i.subnet.AvailabilityZone +} + +func (i etcdNodeImpl) DependencyExists() bool { + return i.subnet.Private && i.subnet.ManageRouteToNATGateway() +} + +func (i etcdNodeImpl) DependencyRef() (string, error) { + // We have to wait until the route to the NAT gateway if it doesn't exist yet(hence ManageRoute=true) or the etcd node fails due to inability to connect internet + if i.DependencyExists() { + name := i.subnet.NATGatewayRouteLogicalName() + return fmt.Sprintf(`"%s"`, name), nil + } + return "", nil +} + +func (i etcdNodeImpl) EBSLogicalName() string { + return fmt.Sprintf("Etcd%dEBS", i.index) +} + +func (i etcdNodeImpl) EBSRef() string { + return fmt.Sprintf(`{ "Ref" : "%s" }`, i.EBSLogicalName()) +} + +func (i etcdNodeImpl) EIPAllocationIDRef() (string, error) { + eipLogicalName, err := i.EIPLogicalName() + if err != nil { + return "", fmt.Errorf("failed to derive the ref to the allocation id of an EIP: %v", err) + } + return fmt.Sprintf(`{ "Fn::GetAtt" : [ "%s", "AllocationId" ] }`, eipLogicalName), nil +} + +func (i etcdNodeImpl) EIPLogicalName() (string, error) { + if !i.EIPManaged() { + return "", fmt.Errorf("[bug] EIPLogicalName invoked when EIP is not managed. Etcd node name: %s", i.Name()) + } + return fmt.Sprintf("Etcd%dEIP", i.index), nil +} + +func (i etcdNodeImpl) EIPManaged() bool { + return i.cluster.NodeShouldHaveEIP() +} + +func (i etcdNodeImpl) EIPRef() (string, error) { + eipLogicalName, err := i.EIPLogicalName() + if err != nil { + return "", fmt.Errorf("failed to derive the ref to an EIP: %v", err) + } + return fmt.Sprintf(`{ "Ref" : "%s" }`, eipLogicalName), nil +} + +func (i etcdNodeImpl) NetworkInterfaceIDRef() string { + return fmt.Sprintf(`{ "Ref" : "%s" }`, i.NetworkInterfaceLogicalName()) +} + +func (i etcdNodeImpl) NetworkInterfaceLogicalName() string { + return fmt.Sprintf("Etcd%dENI", i.index) +} + +func (i etcdNodeImpl) NetworkInterfaceManaged() bool { + return i.cluster.NodeShouldHaveSecondaryENI() +} + +func (i etcdNodeImpl) NetworkInterfacePrivateIPRef() string { + return fmt.Sprintf(`{ "Fn::GetAtt" : [ "%s", "PrimaryPrivateIpAddress" ] }`, i.NetworkInterfaceLogicalName()) +} + +// NetworkInterfacePrivateIPLogicalName returns the logical name of the launch configuration specific to this etcd node +func (i etcdNodeImpl) NetworkInterfacePrivateIPLogicalName() string { + return fmt.Sprintf("%sPrivateIP", i.LogicalName()) +} + +// LaunchConfigurationLogicalName returns the logical name of the launch configuration specific to this etcd node +func (i etcdNodeImpl) LaunchConfigurationLogicalName() string { + return fmt.Sprintf("%sLC", i.LogicalName()) +} + +func (i etcdNodeImpl) LogicalName() string { + return fmt.Sprintf("Etcd%d", i.index) +} + +func (i etcdNodeImpl) RecordSetManaged() bool { + return i.cluster.NodeShouldHaveSecondaryENI() && i.cluster.RecordSetsManaged() +} + +func (i etcdNodeImpl) RecordSetLogicalName() string { + return fmt.Sprintf("Etcd%dInternalRecordSet", i.index) +} diff --git a/model/derived/etcd_nodes.go b/model/derived/etcd_nodes.go new file mode 100644 index 000000000..37ddbddb1 --- /dev/null +++ b/model/derived/etcd_nodes.go @@ -0,0 +1,40 @@ +package derived + +import ( + "fmt" + "github.com/coreos/kube-aws/model" +) + +// NewEtcdNodes derives etcd nodes from user-provided etcd node configs +func NewEtcdNodes(nodeConfigs []model.EtcdNode, cluster EtcdCluster) ([]EtcdNode, error) { + count := cluster.NodeCount() + + result := make([]EtcdNode, count) + for etcdIndex := 0; etcdIndex < count; etcdIndex++ { + + //Round-robin etcd instances across all available subnets + subnetIndex := etcdIndex % len(cluster.Subnets()) + subnet := cluster.Subnets()[subnetIndex] + + nodeConfig := model.EtcdNode{} + if len(nodeConfigs) == count { + nodeConfig = nodeConfigs[etcdIndex] + } + + if subnet.ManageNATGateway() { + ngw, err := cluster.NATGatewayForSubnet(subnet) + + if err != nil { + return nil, fmt.Errorf("failed to determine nat gateway for subnet %s: %v", subnet.LogicalName(), err) + } + + result[etcdIndex] = NewEtcdNodeDependsOnManagedNGW(cluster, etcdIndex, nodeConfig, subnet, *ngw) + } else { + result[etcdIndex] = NewEtcdNode(cluster, etcdIndex, nodeConfig, subnet) + } + + //http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-instance-addressing.html#concepts-private-addresses + + } + return result, nil +} diff --git a/model/derived/network.go b/model/derived/network.go new file mode 100644 index 000000000..9805e20a0 --- /dev/null +++ b/model/derived/network.go @@ -0,0 +1,41 @@ +package derived + +import ( + "fmt" + "github.com/coreos/kube-aws/model" +) + +type Network interface { + Subnets() []model.Subnet + NATGateways() []model.NATGateway + NATGatewayForSubnet(model.Subnet) (*model.NATGateway, error) +} + +type networkImpl struct { + subnets []model.Subnet + natGateways []model.NATGateway +} + +func NewNetwork(subnets []model.Subnet, natGateways []model.NATGateway) Network { + return networkImpl{ + subnets: subnets, + natGateways: natGateways, + } +} + +func (n networkImpl) Subnets() []model.Subnet { + return n.subnets +} + +func (n networkImpl) NATGateways() []model.NATGateway { + return n.natGateways +} + +func (n networkImpl) NATGatewayForSubnet(s model.Subnet) (*model.NATGateway, error) { + for _, ngw := range n.NATGateways() { + if ngw.IsConnectedToPrivateSubnet(s) { + return &ngw, nil + } + } + return nil, fmt.Errorf(`subnet "%s" doesn't have a corresponding nat gateway in: %v`, s.LogicalName(), n.natGateways) +} diff --git a/model/etcd.go b/model/etcd.go index 4ebf6b184..ed5855d2b 100644 --- a/model/etcd.go +++ b/model/etcd.go @@ -1,52 +1,70 @@ package model -import "fmt" +import ( + "errors" +) type Etcd struct { - Subnets []Subnet `yaml:"subnets,omitempty"` + Subnets []Subnet `yaml:"subnets,omitempty"` + Nodes []EtcdNode `yaml:"nodes,omitempty"` + Cluster EtcdCluster `yaml:",inline"` } -type EtcdInstance interface { - SubnetRef() string - DependencyExists() bool - DependencyRef() (string, error) +func (i Etcd) LogicalName() string { + return "Etcd" } -type etcdInstanceImpl struct { - subnet Subnet - natGateway NATGateway +// NameTagKey returns the key of the tag used to identify the name of the etcd member of an EBS volume +func (e Etcd) NameTagKey() string { + return "kube-aws:etcd:name" } -func NewEtcdInstanceDependsOnNewlyCreatedNGW(s Subnet, ngw NATGateway) EtcdInstance { - return etcdInstanceImpl{ - subnet: s, - natGateway: ngw, - } +// AdvertisedFQDNTagKey returns the key of the tag used to identify the advertised hostname of the etcd member of an EBS volume +func (e Etcd) AdvertisedFQDNTagKey() string { + return "kube-aws:etcd:advertised-hostname" } -func NewEtcdInstance(s Subnet) EtcdInstance { - return etcdInstanceImpl{ - subnet: s, - } +// EIPAllocationIDTagKey returns the key of the tag used to identify the EIP for the etcd member of an EBS volume +func (e Etcd) EIPAllocationIDTagKey() string { + return "kube-aws:etcd:eip-allocation-id" +} + +// NetworkInterfaceIDTagKey returns the key of the tag used to identify the ENI for the etcd member of an EBS volume +func (e Etcd) NetworkInterfaceIDTagKey() string { + return "kube-aws:etcd:network-interface-id" +} + +// NetworkInterfaceDeviceIndex represents that the network interface at index 1 is reserved by kube-aws for etcd peer communication +// Please submit a feature request if this is inconvenient for you +func (e Etcd) NetworkInterfaceDeviceIndex() int { + return 1 +} + +func (e Etcd) NodeShouldHaveEIP() bool { + return e.Cluster.NodeShouldHaveEIP() +} + +func (e Etcd) NodeShouldHaveSecondaryENI() bool { + return e.Cluster.NodeShouldHaveSecondaryENI() } -func (i etcdInstanceImpl) SubnetRef() string { - return i.subnet.Ref() +func (e Etcd) HostedZoneManaged() bool { + return e.Cluster.hostedZoneManaged() } -func (i etcdInstanceImpl) SubnetAvailabilityZone() string { - return i.subnet.AvailabilityZone +func (e Etcd) HostedZoneRef() (string, error) { + return e.Cluster.HostedZone.RefOrError(func() (string, error) { + return e.HostedZoneLogicalName() + }) } -func (i etcdInstanceImpl) DependencyExists() bool { - return i.subnet.Private && i.subnet.ManageRouteToNATGateway() +func (e Etcd) InternalDomainName() (string, error) { + return e.Cluster.InternalDomainName, nil } -func (i etcdInstanceImpl) DependencyRef() (string, error) { - // We have to wait until the route to the NAT gateway if it doesn't exist yet(hence ManageRoute=true) or the etcd node fails due to inability to connect internet - if i.DependencyExists() { - name := i.subnet.NATGatewayRouteLogicalName() - return fmt.Sprintf(`"%s"`, name), nil +func (e Etcd) HostedZoneLogicalName() (string, error) { + if !e.Cluster.hostedZoneManaged() { + return "", errors.New("[bug] HostedZoneLogicalName called for an etcd cluster without a managed hosted zone") } - return "", nil + return "EtcdHostedZone", nil } diff --git a/model/etcd_cluster.go b/model/etcd_cluster.go new file mode 100644 index 000000000..f38a17ce3 --- /dev/null +++ b/model/etcd_cluster.go @@ -0,0 +1,54 @@ +package model + +import "fmt" + +type EtcdCluster struct { + InternalDomainName string `yaml:"internalDomainName,omitempty"` + MemberIdentityProvider string `yaml:"memberIdentityProvider,omitempty"` + HostedZone Identifier `yaml:"hostedZone,omitempty"` + ManageRecordSets *bool `yaml:"manageRecordSets,omitempty"` +} + +const ( + MemberIdentityProviderEIP = "eip" + MemberIdentityProviderENI = "eni" +) + +func (c EtcdCluster) EC2InternalDomainUsed() bool { + return c.InternalDomainName == "" +} + +func (c EtcdCluster) GetMemberIdentityProvider() string { + p := c.MemberIdentityProvider + + if p == MemberIdentityProviderEIP || p == MemberIdentityProviderENI { + return p + } else if p == "" { + return MemberIdentityProviderEIP + } + + panic(fmt.Errorf("Unsupported memberIdentityProvider: %s", p)) +} + +func (e EtcdCluster) hostedZoneManaged() bool { + return e.GetMemberIdentityProvider() == MemberIdentityProviderENI && + !e.HostedZone.HasIdentifier() && !e.EC2InternalDomainUsed() +} + +// Notes: +// * EC2's default domain like .compute.internal for internalDomainName implies not to manage record sets +// * Managed hosted zone implies managed record sets +func (e EtcdCluster) RecordSetsManaged() bool { + return e.GetMemberIdentityProvider() == MemberIdentityProviderENI && !e.EC2InternalDomainUsed() && + (e.hostedZoneManaged() || (e.ManageRecordSets == nil || *e.ManageRecordSets)) +} + +// NodeShouldHaveSecondaryENI returns true if all the etcd nodes should have secondary ENIs for their identities +func (c EtcdCluster) NodeShouldHaveSecondaryENI() bool { + return c.GetMemberIdentityProvider() == MemberIdentityProviderENI +} + +// NodeShouldHaveEIP returns true if all the etcd nodes should have EIPs for their identities +func (c EtcdCluster) NodeShouldHaveEIP() bool { + return c.GetMemberIdentityProvider() == MemberIdentityProviderEIP +} diff --git a/model/etcd_node.go b/model/etcd_node.go new file mode 100644 index 000000000..956fb25d5 --- /dev/null +++ b/model/etcd_node.go @@ -0,0 +1,6 @@ +package model + +type EtcdNode struct { + Name string `yaml:"name,omitempty"` + FQDN string `yaml:"fqdn,omitempty"` +} diff --git a/model/region.go b/model/region.go new file mode 100644 index 000000000..d4bcde0be --- /dev/null +++ b/model/region.go @@ -0,0 +1,36 @@ +package model + +import ( + "fmt" +) + +type Region interface { + PrivateDomainName() string + PublicDomainName() string + String() string +} + +type regionImpl struct { + name string +} + +func RegionForName(name string) Region { + return regionImpl{ + name: name, + } +} + +func (r regionImpl) PrivateDomainName() string { + if r.name == "us-east-1" { + return "ec2.internal" + } + return fmt.Sprintf("%s.compute.internal", r.name) +} + +func (r regionImpl) PublicDomainName() string { + return fmt.Sprintf("%s.compute.amazonaws.com", r.name) +} + +func (r regionImpl) String() string { + return r.name +} diff --git a/test/integration/aws_test.go b/test/integration/aws_test.go index 10f335a46..abc17d4a5 100644 --- a/test/integration/aws_test.go +++ b/test/integration/aws_test.go @@ -3,6 +3,7 @@ package integration import ( "fmt" "github.com/coreos/kube-aws/core/controlplane/config" + "github.com/coreos/kube-aws/model" "github.com/coreos/kube-aws/test/helper" "os" "testing" @@ -28,13 +29,14 @@ func useRealAWS() bool { } type kubeAwsSettings struct { - clusterName string - externalDNSName string - keyName string - kmsKeyArn string - region string - mainClusterYaml string - encryptService config.EncryptService + clusterName string + etcdNodeDefaultInternalDomain string + externalDNSName string + keyName string + kmsKeyArn string + region string + mainClusterYaml string + encryptService config.EncryptService } func newKubeAwsSettingsFromEnv(t *testing.T) kubeAwsSettings { @@ -65,12 +67,13 @@ region: "%s" region, ) return kubeAwsSettings{ - clusterName: clusterName, - externalDNSName: externalDnsName, - keyName: keyName, - kmsKeyArn: kmsKeyArn, - region: region, - mainClusterYaml: yaml, + clusterName: clusterName, + etcdNodeDefaultInternalDomain: model.RegionForName(region).PrivateDomainName(), + externalDNSName: externalDnsName, + keyName: keyName, + kmsKeyArn: kmsKeyArn, + region: region, + mainClusterYaml: yaml, } } else { return kubeAwsSettings{ @@ -81,7 +84,8 @@ keyName: test-key-name kmsKeyArn: "arn:aws:kms:us-west-1:xxxxxxxxx:key/xxxxxxxxxxxxxxxxxxx" region: us-west-1 `, clusterName), - encryptService: helper.DummyEncryptService{}, + encryptService: helper.DummyEncryptService{}, + etcdNodeDefaultInternalDomain: model.RegionForName("us-west-1").PrivateDomainName(), } } } diff --git a/test/integration/maincluster_test.go b/test/integration/maincluster_test.go index fc449461b..ece3815a4 100644 --- a/test/integration/maincluster_test.go +++ b/test/integration/maincluster_test.go @@ -19,6 +19,8 @@ type ClusterTester func(c root.Cluster, t *testing.T) // Integration testing with real AWS services including S3, KMS, CloudFormation func TestMainClusterConfig(t *testing.T) { + kubeAwsSettings := newKubeAwsSettingsFromEnv(t) + s3URI, s3URIExists := os.LookupEnv("KUBE_AWS_S3_DIR_URI") if !s3URIExists || s3URI == "" { @@ -287,8 +289,6 @@ func TestMainClusterConfig(t *testing.T) { } } - kubeAwsSettings := newKubeAwsSettingsFromEnv(t) - hasDefaultCluster := func(c root.Cluster, t *testing.T) { assets, err := c.Assets() if err != nil { @@ -414,6 +414,439 @@ worker: hasDefaultCluster, }, }, + { + context: "WithEtcdMemberIdentityProviderEIP", + configYaml: minimalValidConfigYaml + ` +etcd: + memberIdentityProvider: eip +`, + assertConfig: []ConfigTester{ + func(c *config.Config, t *testing.T) { + subnet1 := model.NewPublicSubnet("us-west-1c", "10.0.0.0/24") + subnet1.Name = "Subnet0" + expected := controlplane_config.EtcdSettings{ + Etcd: model.Etcd{ + Cluster: model.EtcdCluster{ + MemberIdentityProvider: "eip", + }, + Subnets: []model.Subnet{ + subnet1, + }, + }, + EtcdCount: 1, + EtcdInstanceType: "t2.medium", + EtcdRootVolumeSize: 30, + EtcdRootVolumeType: "gp2", + EtcdRootVolumeIOPS: 0, + EtcdDataVolumeSize: 30, + EtcdDataVolumeType: "gp2", + EtcdDataVolumeIOPS: 0, + EtcdDataVolumeEphemeral: false, + EtcdTenancy: "default", + } + actual := c.EtcdSettings + if !reflect.DeepEqual(expected, actual) { + t.Errorf( + "EtcdSettings didn't match: expected=%v actual=%v", + expected, + actual, + ) + } + + if !actual.NodeShouldHaveEIP() { + t.Errorf( + "NodeShouldHaveEIP returned unexpected value: %v", + actual.NodeShouldHaveEIP(), + ) + } + }, + }, + assertCluster: []ClusterTester{ + hasDefaultCluster, + }, + }, + { + context: "WithEtcdMemberIdentityProviderENI", + configYaml: minimalValidConfigYaml + ` +etcd: + memberIdentityProvider: eni +`, + assertConfig: []ConfigTester{ + func(c *config.Config, t *testing.T) { + subnet1 := model.NewPublicSubnet("us-west-1c", "10.0.0.0/24") + subnet1.Name = "Subnet0" + expected := controlplane_config.EtcdSettings{ + Etcd: model.Etcd{ + Cluster: model.EtcdCluster{ + MemberIdentityProvider: "eni", + }, + Subnets: []model.Subnet{ + subnet1, + }, + }, + EtcdCount: 1, + EtcdInstanceType: "t2.medium", + EtcdRootVolumeSize: 30, + EtcdRootVolumeType: "gp2", + EtcdRootVolumeIOPS: 0, + EtcdDataVolumeSize: 30, + EtcdDataVolumeType: "gp2", + EtcdDataVolumeIOPS: 0, + EtcdDataVolumeEphemeral: false, + EtcdTenancy: "default", + } + actual := c.EtcdSettings + if !reflect.DeepEqual(expected, actual) { + t.Errorf( + "EtcdSettings didn't match: expected=%v actual=%v", + expected, + actual, + ) + } + + if !actual.NodeShouldHaveSecondaryENI() { + t.Errorf( + "NodeShouldHaveSecondaryENI returned unexpected value: %v", + actual.NodeShouldHaveSecondaryENI(), + ) + } + }, + }, + assertCluster: []ClusterTester{ + hasDefaultCluster, + }, + }, + { + context: "WithEtcdMemberIdentityProviderENIWithCustomDomain", + configYaml: minimalValidConfigYaml + ` +etcd: + memberIdentityProvider: eni + internalDomainName: internal.example.com +`, + assertConfig: []ConfigTester{ + func(c *config.Config, t *testing.T) { + subnet1 := model.NewPublicSubnet("us-west-1c", "10.0.0.0/24") + subnet1.Name = "Subnet0" + expected := controlplane_config.EtcdSettings{ + Etcd: model.Etcd{ + Cluster: model.EtcdCluster{ + MemberIdentityProvider: "eni", + InternalDomainName: "internal.example.com", + }, + Subnets: []model.Subnet{ + subnet1, + }, + }, + EtcdCount: 1, + EtcdInstanceType: "t2.medium", + EtcdRootVolumeSize: 30, + EtcdRootVolumeType: "gp2", + EtcdRootVolumeIOPS: 0, + EtcdDataVolumeSize: 30, + EtcdDataVolumeType: "gp2", + EtcdDataVolumeIOPS: 0, + EtcdDataVolumeEphemeral: false, + EtcdTenancy: "default", + } + actual := c.EtcdSettings + if !reflect.DeepEqual(expected, actual) { + t.Errorf( + "EtcdSettings didn't match: expected=%v actual=%v", + expected, + actual, + ) + } + + if !actual.NodeShouldHaveSecondaryENI() { + t.Errorf( + "NodeShouldHaveSecondaryENI returned unexpected value: %v", + actual.NodeShouldHaveSecondaryENI(), + ) + } + }, + }, + assertCluster: []ClusterTester{ + hasDefaultCluster, + }, + }, + { + context: "WithEtcdMemberIdentityProviderENIWithCustomFQDNs", + configYaml: minimalValidConfigYaml + ` +etcd: + memberIdentityProvider: eni + internalDomainName: internal.example.com + nodes: + - fqdn: etcd1a.internal.example.com + - fqdn: etcd1b.internal.example.com + - fqdn: etcd1c.internal.example.com +`, + assertConfig: []ConfigTester{ + func(c *config.Config, t *testing.T) { + subnet1 := model.NewPublicSubnet("us-west-1c", "10.0.0.0/24") + subnet1.Name = "Subnet0" + expected := controlplane_config.EtcdSettings{ + Etcd: model.Etcd{ + Cluster: model.EtcdCluster{ + MemberIdentityProvider: "eni", + InternalDomainName: "internal.example.com", + }, + Nodes: []model.EtcdNode{ + model.EtcdNode{ + FQDN: "etcd1a.internal.example.com", + }, + model.EtcdNode{ + FQDN: "etcd1b.internal.example.com", + }, + model.EtcdNode{ + FQDN: "etcd1c.internal.example.com", + }, + }, + Subnets: []model.Subnet{ + subnet1, + }, + }, + EtcdCount: 1, + EtcdInstanceType: "t2.medium", + EtcdRootVolumeSize: 30, + EtcdRootVolumeType: "gp2", + EtcdRootVolumeIOPS: 0, + EtcdDataVolumeSize: 30, + EtcdDataVolumeType: "gp2", + EtcdDataVolumeIOPS: 0, + EtcdDataVolumeEphemeral: false, + EtcdTenancy: "default", + } + actual := c.EtcdSettings + if !reflect.DeepEqual(expected, actual) { + t.Errorf( + "EtcdSettings didn't match: expected=%v actual=%v", + expected, + actual, + ) + } + + if !actual.NodeShouldHaveSecondaryENI() { + t.Errorf( + "NodeShouldHaveSecondaryENI returned unexpected value: %v", + actual.NodeShouldHaveSecondaryENI(), + ) + } + }, + }, + assertCluster: []ClusterTester{ + hasDefaultCluster, + }, + }, + { + context: "WithEtcdMemberIdentityProviderENIWithCustomNames", + configYaml: minimalValidConfigYaml + ` +etcd: + memberIdentityProvider: eni + internalDomainName: internal.example.com + nodes: + - name: etcd1a + - name: etcd1b + - name: etcd1c +`, + assertConfig: []ConfigTester{ + func(c *config.Config, t *testing.T) { + subnet1 := model.NewPublicSubnet("us-west-1c", "10.0.0.0/24") + subnet1.Name = "Subnet0" + expected := controlplane_config.EtcdSettings{ + Etcd: model.Etcd{ + Cluster: model.EtcdCluster{ + MemberIdentityProvider: "eni", + InternalDomainName: "internal.example.com", + }, + Nodes: []model.EtcdNode{ + model.EtcdNode{ + Name: "etcd1a", + }, + model.EtcdNode{ + Name: "etcd1b", + }, + model.EtcdNode{ + Name: "etcd1c", + }, + }, + Subnets: []model.Subnet{ + subnet1, + }, + }, + EtcdCount: 1, + EtcdInstanceType: "t2.medium", + EtcdRootVolumeSize: 30, + EtcdRootVolumeType: "gp2", + EtcdRootVolumeIOPS: 0, + EtcdDataVolumeSize: 30, + EtcdDataVolumeType: "gp2", + EtcdDataVolumeIOPS: 0, + EtcdDataVolumeEphemeral: false, + EtcdTenancy: "default", + } + actual := c.EtcdSettings + if !reflect.DeepEqual(expected, actual) { + t.Errorf( + "EtcdSettings didn't match: expected=%v actual=%v", + expected, + actual, + ) + } + + if !actual.NodeShouldHaveSecondaryENI() { + t.Errorf( + "NodeShouldHaveSecondaryENI returned unexpected value: %v", + actual.NodeShouldHaveSecondaryENI(), + ) + } + }, + }, + assertCluster: []ClusterTester{ + hasDefaultCluster, + }, + }, + { + context: "WithEtcdMemberIdentityProviderENIWithoutRecordSets", + configYaml: minimalValidConfigYaml + ` +etcd: + memberIdentityProvider: eni + internalDomainName: internal.example.com + manageRecordSets: false + nodes: + - name: etcd1a + - name: etcd1b + - name: etcd1c +`, + assertConfig: []ConfigTester{ + func(c *config.Config, t *testing.T) { + subnet1 := model.NewPublicSubnet("us-west-1c", "10.0.0.0/24") + subnet1.Name = "Subnet0" + manageRecordSets := false + expected := controlplane_config.EtcdSettings{ + Etcd: model.Etcd{ + Cluster: model.EtcdCluster{ + ManageRecordSets: &manageRecordSets, + MemberIdentityProvider: "eni", + InternalDomainName: "internal.example.com", + }, + Nodes: []model.EtcdNode{ + model.EtcdNode{ + Name: "etcd1a", + }, + model.EtcdNode{ + Name: "etcd1b", + }, + model.EtcdNode{ + Name: "etcd1c", + }, + }, + Subnets: []model.Subnet{ + subnet1, + }, + }, + EtcdCount: 1, + EtcdInstanceType: "t2.medium", + EtcdRootVolumeSize: 30, + EtcdRootVolumeType: "gp2", + EtcdRootVolumeIOPS: 0, + EtcdDataVolumeSize: 30, + EtcdDataVolumeType: "gp2", + EtcdDataVolumeIOPS: 0, + EtcdDataVolumeEphemeral: false, + EtcdTenancy: "default", + } + actual := c.EtcdSettings + if !reflect.DeepEqual(expected, actual) { + t.Errorf( + "EtcdSettings didn't match: expected=%v actual=%v", + expected, + actual, + ) + } + + if !actual.NodeShouldHaveSecondaryENI() { + t.Errorf( + "NodeShouldHaveSecondaryENI returned unexpected value: %v", + actual.NodeShouldHaveSecondaryENI(), + ) + } + }, + }, + assertCluster: []ClusterTester{ + hasDefaultCluster, + }, + }, + { + context: "WithEtcdMemberIdentityProviderENIWithHostedZoneID", + configYaml: minimalValidConfigYaml + ` +etcd: + memberIdentityProvider: eni + internalDomainName: internal.example.com + hostedZone: + id: hostedzone-abcdefg + nodes: + - name: etcd1a + - name: etcd1b + - name: etcd1c +`, + assertConfig: []ConfigTester{ + func(c *config.Config, t *testing.T) { + subnet1 := model.NewPublicSubnet("us-west-1c", "10.0.0.0/24") + subnet1.Name = "Subnet0" + expected := controlplane_config.EtcdSettings{ + Etcd: model.Etcd{ + Cluster: model.EtcdCluster{ + HostedZone: model.Identifier{ID: "hostedzone-abcdefg"}, + MemberIdentityProvider: "eni", + InternalDomainName: "internal.example.com", + }, + Nodes: []model.EtcdNode{ + model.EtcdNode{ + Name: "etcd1a", + }, + model.EtcdNode{ + Name: "etcd1b", + }, + model.EtcdNode{ + Name: "etcd1c", + }, + }, + Subnets: []model.Subnet{ + subnet1, + }, + }, + EtcdCount: 1, + EtcdInstanceType: "t2.medium", + EtcdRootVolumeSize: 30, + EtcdRootVolumeType: "gp2", + EtcdRootVolumeIOPS: 0, + EtcdDataVolumeSize: 30, + EtcdDataVolumeType: "gp2", + EtcdDataVolumeIOPS: 0, + EtcdDataVolumeEphemeral: false, + EtcdTenancy: "default", + } + actual := c.EtcdSettings + if !reflect.DeepEqual(expected, actual) { + t.Errorf( + "EtcdSettings didn't match: expected=%v actual=%v", + expected, + actual, + ) + } + + if !actual.NodeShouldHaveSecondaryENI() { + t.Errorf( + "NodeShouldHaveSecondaryENI returned unexpected value: %v", + actual.NodeShouldHaveSecondaryENI(), + ) + } + }, + }, + assertCluster: []ClusterTester{ + hasDefaultCluster, + }, + }, { context: "WithExperimentalFeatures", configYaml: minimalValidConfigYaml + `