Support for multiple k8s API endpoints

Including the ability to either reuse an existing ELB or a creation of a managed one, and whether to create a record set or not, and of course DNS name, per API endpoint.

Support for DNS round-robin(kubernetes-retired#281) is planned but not included in this commit

core/controlplane/cluster/cluster.go

@@ -348,11 +348,6 @@ func (c *Cluster) Update() (string, error) {
 	return updateOutput, err
 }
 
-func (c *ClusterRef) Info() (*Info, error) {
-	describer := NewClusterDescriber(c.ClusterName, c.StackName(), c.session)
-	return describer.Info()
-}
-
 func (c *ClusterRef) Destroy() error {
 	return cfnstack.NewDestroyer(c.StackName(), c.session).Destroy()
 }

core/controlplane/cluster/describer.go

@@ -13,58 +13,64 @@ type ClusterDescriber interface {
 }
 
 type clusterDescriberImpl struct {
-	session     *session.Session
-	clusterName string
-	stackName   string
+	clusterName             string
+	elbResourceLogicalNames []string
+	session                 *session.Session
+	stackName               string
 }
 
-func NewClusterDescriber(clusterName string, stackName string, session *session.Session) ClusterDescriber {
+func NewClusterDescriber(clusterName string, stackName string, elbResourceLogicalNames []string, session *session.Session) ClusterDescriber {
 	return clusterDescriberImpl{
-		clusterName: clusterName,
-		stackName:   stackName,
-		session:     session,
+		clusterName:             clusterName,
+		elbResourceLogicalNames: elbResourceLogicalNames,
+		stackName:               stackName,
+		session:                 session,
 	}
 }
 
 func (c clusterDescriberImpl) Info() (*Info, error) {
-	var elbName string
+	elbNameRefs := []*string{}
+	elbNames := []string{}
 	{
 		cfSvc := cloudformation.New(c.session)
-		resp, err := cfSvc.DescribeStackResource(
-			&cloudformation.DescribeStackResourceInput{
-				LogicalResourceId: aws.String("ElbAPIServer"),
-				StackName:         aws.String(c.stackName),
-			},
-		)
-		if err != nil {
-			errmsg := "unable to get public IP of controller instance:\n" + err.Error()
-			return nil, fmt.Errorf(errmsg)
+		for _, lb := range c.elbResourceLogicalNames {
+			resp, err := cfSvc.DescribeStackResource(
+				&cloudformation.DescribeStackResourceInput{
+					LogicalResourceId: aws.String(lb),
+					StackName:         aws.String(c.stackName),
+				},
+			)
+			if err != nil {
+				errmsg := "unable to get public IP of controller instance:\n" + err.Error()
+				return nil, fmt.Errorf(errmsg)
+			}
+			elbNameRefs = append(elbNameRefs, resp.StackResourceDetail.PhysicalResourceId)
+			elbNames = append(elbNames, *resp.StackResourceDetail.PhysicalResourceId)
 		}
-		elbName = *resp.StackResourceDetail.PhysicalResourceId
 	}
 
 	elbSvc := elb.New(c.session)
 
 	var info Info
 	{
 		resp, err := elbSvc.DescribeLoadBalancers(&elb.DescribeLoadBalancersInput{
-			LoadBalancerNames: []*string{
-				aws.String(elbName),
-			},
-			PageSize: aws.Int64(2),
+			LoadBalancerNames: elbNameRefs,
+			PageSize:          aws.Int64(2),
 		})
 		if err != nil {
-			return nil, fmt.Errorf("error describing load balancer %s: %v", elbName, err)
+			return nil, fmt.Errorf("error describing load balancers %v: %v", elbNames, err)
 		}
 		if len(resp.LoadBalancerDescriptions) == 0 {
-			return nil, fmt.Errorf("could not find a load balancer with name %s", elbName)
+			return nil, fmt.Errorf("could not find load balancers with names %v", elbNames)
 		}
-		if len(resp.LoadBalancerDescriptions) > 1 {
-			return nil, fmt.Errorf("found multiple load balancers with name %s: %v", elbName, resp)
+
+		dnsNames := []string{}
+		for _, d := range resp.LoadBalancerDescriptions {
+			dnsNames = append(dnsNames, *d.DNSName)
 		}
 
 		info.Name = c.clusterName
-		info.ControllerHost = *resp.LoadBalancerDescriptions[0].DNSName
+		info.ControllerHosts = dnsNames
 	}
 	return &info, nil
 }

core/controlplane/cluster/info.go

@@ -3,12 +3,13 @@ package cluster
 import (
 	"bytes"
 	"fmt"
+	"strings"
 	"text/tabwriter"
 )
 
 type Info struct {
-	Name           string
-	ControllerHost string
+	Name            string
+	ControllerHosts []string
 }
 
 func (c *Info) String() string {
@@ -17,7 +18,7 @@ func (c *Info) String() string {
 	w.Init(buf, 0, 8, 0, '\t', 0)
 
 	fmt.Fprintf(w, "Cluster Name:\t%s\n", c.Name)
-	fmt.Fprintf(w, "Controller DNS Name:\t%s\n", c.ControllerHost)
+	fmt.Fprintf(w, "Controller DNS Names:\t%s\n", strings.Join(c.ControllerHosts, ", "))
 
 	w.Flush()
 	return buf.String()

core/controlplane/config/config.go

@@ -227,6 +227,30 @@ func (c *Cluster) Load() error {
 
 	c.SetDefaults()
 
+	if c.ExternalDNSName != "" {
+		// TODO: Deprecate externalDNSName?
+
+		if len(c.APIEndpointConfigs) != 0 {
+			return errors.New("invalid cluster: you can only specify either externalDNSName or apiEndpoints, but not both")
+		}
+
+		subnetRefs := []model.SubnetReference{}
+		for _, s := range c.Controller.LoadBalancer.Subnets {
+			subnetRefs = append(subnetRefs, model.SubnetReference{Name: s.Name})
+		}
+		hostedZoneId := c.HostedZoneID
+		createRecordSet := c.CreateRecordSet
+		private := c.Controller.LoadBalancer.Private
+
+		c.APIEndpointConfigs = model.NewDefaultAPIEndpoints(
+			c.ExternalDNSName,
+			subnetRefs,
+			hostedZoneId,
+			createRecordSet,
+			private,
+		)
+	}
+
 	return nil
 }
 
@@ -326,6 +350,7 @@ func ClusterFromBytesWithEncryptService(data []byte, encryptService EncryptServi
 // Part of configuration which is shared between controller nodes and worker nodes.
 // Its name is prefixed with `Kube` because it doesn't relate to etcd.
 type KubeClusterSettings struct {
+	APIEndpointConfigs []model.APIEndpoint `yaml:"apiEndpoints,omitempty"`
 	// Required by kubelet to locate the kube-apiserver
 	ExternalDNSName string `yaml:"externalDNSName,omitempty"`
 	// Required by kubelet to locate the cluster-internal dns hosted on controller nodes in the base cluster
@@ -689,6 +714,13 @@ func (c Cluster) Config() (*Config, error) {
 		}
 	}
 
+	apiEndpoints, err := derived.NewAPIEndpoints(c.APIEndpointConfigs, c.Subnets)
+	if err != nil {
+		return nil, fmt.Errorf("invalid cluster: %v", err)
+	}
+
+	config.APIEndpoints = apiEndpoints
+
 	return &config, nil
 }
 
@@ -810,6 +842,8 @@ func (c Cluster) StackConfig(opts StackTemplateOptions) (*StackConfig, error) {
 type Config struct {
 	Cluster
 
+	APIEndpoints derived.APIEndpoints
+
 	EtcdNodes []derived.EtcdNode
 
 	AuthTokensConfig *CompactAuthTokens
@@ -860,6 +894,23 @@ func (c Config) InternetGatewayRef() string {
 	}
 }
 
+// ExternalDNSNames returns all the DNS names of Kubernetes API endpoints should be covered in the TLS cert for k8s API
+func (c Cluster) ExternalDNSNames() []string {
+	names := []string{}
+
+	if c.ExternalDNSName != "" {
+		names = append(names, c.ExternalDNSName)
+	}
+
+	for _, e := range c.APIEndpointConfigs {
+		names = append(names, e.DNSName)
+	}
+
+	sort.Strings(names)
+
+	return names
+}
+
 // NestedStackName returns a sanitized name of this control-plane which is usable as a valid cloudformation nested stack name
 func (c Cluster) NestedStackName() string {
 	// Convert stack name into something valid as a cfn resource name or
@@ -988,8 +1039,14 @@ type InfrastructureValidationResult struct {
 }
 
 func (c KubeClusterSettings) Valid() (*InfrastructureValidationResult, error) {
-	if c.ExternalDNSName == "" {
-		return nil, errors.New("externalDNSName must be set")
+	if c.ExternalDNSName == "" && len(c.APIEndpointConfigs) == 0 {
+		return nil, errors.New("Either externalDNSName or apiEndpoints must be set")
+	}
+
+	for i, apiEndpoint := range c.APIEndpointConfigs {
+		if err := apiEndpoint.Validate(); err != nil {
+			return nil, fmt.Errorf("invalid apiEndpoint at index %d: %v", i, err)
+		}
 	}
 
 	dnsServiceIPAddr := net.ParseIP(c.DNSServiceIP)
@@ -1348,6 +1405,11 @@ func (c *Cluster) ValidateExistingVPC(existingVPCCIDR string, existingSubnetCIDR
 	return nil
 }
 
+// ManageELBLogicalNames returns all the logical names of the cfn resources corresponding to ELBs managed by kube-aws for API endpoints
+func (c *Config) ManagedELBLogicalNames() []string {
+	return c.APIEndpoints.ManagedELBLogicalNames()
+}
+
 func WithTrailingDot(s string) string {
 	if s == "" {
 		return s

core/controlplane/config/templates/cloud-config-worker

@@ -233,7 +233,7 @@ coreos:
         --net=host \
         {{.HyperkubeImage.RepoWithTag}} \
           --exec=/kubectl -- \
-          --server=https://{{.ExternalDNSName}}:443 \
+          --server=https://{{.APIEndpoint.DNSName}}:443 \
           --kubeconfig=/etc/kubernetes/worker-kubeconfig.yaml \
           drain $(hostname) \
           --ignore-daemonsets \
@@ -396,7 +396,7 @@ coreos:
           -e LAUNCHCONFIGURATION=${LAUNCHCONFIGURATION} \
           {{.HyperkubeImage.RepoWithTag}} /bin/bash \
             -ec 'echo "placing labels and annotations with additional AWS parameters."; \
-             kctl="/kubectl --server=https://{{.ExternalDNSName}}:443 --kubeconfig=/etc/kubernetes/worker-kubeconfig.yaml"; \
+             kctl="/kubectl --server=https://{{.APIEndpoint.DNSName}}:443 --kubeconfig=/etc/kubernetes/worker-kubeconfig.yaml"; \
              kctl_label="$kctl label --overwrite nodes/$(hostname)"; \
              kctl_annotate="$kctl annotate --overwrite nodes/$(hostname)"; \
              $kctl_label kube-aws.coreos.com/autoscalinggroup=${AUTOSCALINGGROUP}; \
@@ -698,7 +698,7 @@ write_files:
           'echo "tainting this node."
            hostname="'${hostname}'"
            taints=({{range $i, $taint := .Experimental.Taints}}"{{$taint.String}}" {{end}})
-           kubectl="/kubectl --server=https://{{.ExternalDNSName}}:443 --kubeconfig=/etc/kubernetes/worker-kubeconfig.yaml"
+           kubectl="/kubectl --server=https://{{.APIEndpoint.DNSName}}:443 --kubeconfig=/etc/kubernetes/worker-kubeconfig.yaml"
            taint="$kubectl taint node --overwrite"
            for t in ${taints[@]}; do
              $taint "$hostname" "$t"

core/controlplane/config/templates/cluster.yaml

@@ -33,6 +33,60 @@ externalDNSName: {{.ExternalDNSName}}
 # Either specify hostedZoneId or hostedZone, but not both
 #hostedZoneId: ""
 
+# Kubernetes API endpoints with each one has a DNS name and is with/without a managed/unmanaged ELB, Route 53 record set
+# CAUTION: `externalDNSName` must be omitted when there are one or more items under `apiEndpoints`
+#apiEndpoints:
+#  # The unique name of this API endpoint used to identify it inside CloudFormation stacks or
+#  # to be referenced from other parts of cluster.yaml
+#- name: template
+#
+#  # Specifies an existing load-balancer used for load-balancing controller nodes and serving this endpoint
+#  # Setting id requires all the other settings excluding `name` to be omitted because reusing an ELB implies that configuring other resources
+#  # like a Route 53 record set for the endpoint is now your responsibility!
+#  id: existing-elb
+#
+#  # Set to false when you want to disable creation of the record set for this api load balancer
+#  # Must be omitted when `id` is specified
+#  createRecordSet: true
+#
+#  # All the subnets assigned to this load-balancer. Specified only when this load balancer is not reused but managed one
+#  # Must be omitted when `id` is specified
+#  subnets:
+#  - name: managedPublic1
+#
+#  # Set to true so that the managed ELB becomes an `internal` one rather than `internet-facing` one
+#  # When set to true while subnets are omitted, one or more private subnets in the top-level `subnets` must exist
+#  # Must be omitted when `id` is specified
+#  private: true
+#
+#  # The Route 53 hosted zone is where the resulting Alias record is created for this endpoint
+#  # Must be omitted when `id` is specified
+#  hostedZone:
+#    id: hostedzone-abc
+#
+#  #
+#  # Common configuration #1: Unversioned, internet-facing API endpoint
+#  #
+#  - name: unversionedPublic
+#    dnsName: api.example.com
+#    loadBalancer:
+#      id: elb-abcdefg
+#
+#  #
+#  # Common configuration #2: Versioned, internet-facing API endpoint
+#  #
+#  - name: versionedPublic
+#    dnsName: v1api.example.com
+#    loadBalancer:
+#      hostedZone:
+#        id: hostedzone-abcedfg
+#
+#  #
+#  # Common configuration #3: Unmanaged endpoint a.k.a Extra DNS name added to the kube-apiserver TLS cert
+#  #
+#  - name: extra
+#    dnsName: youralias.example.com
+
 # Name of the SSH keypair already loaded into the AWS
 # account being used to deploy this cluster.
 keyName: {{.KeyName}}