Skip to content
This repository has been archived by the owner on Sep 4, 2021. It is now read-only.

Commit

Permalink
kube-aws: Support Multi-AZ workers on AWS
Browse files Browse the repository at this point in the history 
One step forward to achieve high-availability throught the cluster.
This change allows you to specify multiple subnets in cluster.yaml to make workers' ASG spread instances over those subnets. Differentiating each subnet's availability zone results in H/A of workers.
Beware that this change itself does nothing with H/A of masters.

Possibly relates to #147, #100
  • Loading branch information
@mumoshu
mumoshu committed Apr 27, 2016
1 parent e541a59 commit b6f5bba
Show file tree
Hide file tree
Showing 3 changed files with 212 additions and 34 deletions.
57 changes: 38 additions & 19 deletions multi-node/aws/pkg/config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ func newDefaultCluster() *Cluster {
WorkerRootVolumeSize: 30,
CreateRecordSet: false,
RecordSetTTL: 300,
Subnets: []Subnet{},
}
}

Expand Down Expand Up @@ -74,6 +75,15 @@ func ClusterFromBytes(data []byte) (*Cluster, error) {
// as it will with RecordSets
c.HostedZone = WithTrailingDot(c.HostedZone)

if len(c.Subnets) == 0 {
c.Subnets = []Subnet{
{
AvailabilityZone: c.AvailabilityZone,
InstanceCIDR: c.InstanceCIDR,
},
}
}

if err := c.valid(); err != nil {
return nil, fmt.Errorf("invalid cluster: %v", err)
}
Expand Down Expand Up @@ -108,6 +118,12 @@ type Cluster struct {
RecordSetTTL int `yaml:"recordSetTTL"`
HostedZone string `yaml:"hostedZone"`
StackTags map[string]string `yaml:"stackTags"`
Subnets []Subnet `yaml:"subnets"`
}

type Subnet struct {
AvailabilityZone string `yaml:"availabilityZone"`
InstanceCIDR string `yaml:"instanceCIDR"`
}

const (
Expand Down Expand Up @@ -366,9 +382,6 @@ func (cfg Cluster) valid() error {
if cfg.Region == "" {
return errors.New("region must be set")
}
if cfg.AvailabilityZone == "" {
return errors.New("availabilityZone must be set")
}
if cfg.ClusterName == "" {
return errors.New("clusterName must be set")
}
Expand All @@ -385,26 +398,32 @@ func (cfg Cluster) valid() error {
return fmt.Errorf("invalid vpcCIDR: %v", err)
}

_, instancesNet, err := net.ParseCIDR(cfg.InstanceCIDR)
if err != nil {
return fmt.Errorf("invalid instanceCIDR: %v", err)
}
if !vpcNet.Contains(instancesNet.IP) {
return fmt.Errorf("vpcCIDR (%s) does not contain instanceCIDR (%s)",
cfg.VPCCIDR,
cfg.InstanceCIDR,
)
}

controllerIPAddr := net.ParseIP(cfg.ControllerIP)
if controllerIPAddr == nil {
return fmt.Errorf("invalid controllerIP: %s", cfg.ControllerIP)
}
if !instancesNet.Contains(controllerIPAddr) {
return fmt.Errorf("instanceCIDR (%s) does not contain controllerIP (%s)",
cfg.InstanceCIDR,
cfg.ControllerIP,
)

for i, subnet := range cfg.Subnets {
if subnet.AvailabilityZone == "" {
return fmt.Errorf("availabilityZone must be set for subnet #%d", i)
}
_, instancesNet, err := net.ParseCIDR(subnet.InstanceCIDR)
if err != nil {
return fmt.Errorf("invalid instanceCIDR for subnet #%d: %v", i, err)
}
if !vpcNet.Contains(instancesNet.IP) {
return fmt.Errorf("vpcCIDR (%s) does not contain instanceCIDR (%s) for subnet #%d",
cfg.VPCCIDR,
cfg.InstanceCIDR,
i,
)
}
if i == 0 && !instancesNet.Contains(controllerIPAddr) {
return fmt.Errorf("instanceCIDR (%s) does not contain controllerIP (%s)",
subnet.InstanceCIDR,
cfg.ControllerIP,
)
}
}

_, podNet, err := net.ParseCIDR(cfg.PodCIDR)
Expand Down
137 changes: 137 additions & 0 deletions multi-node/aws/pkg/config/config_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package config

import (
"net"
"reflect"
"testing"
)

Expand Down Expand Up @@ -309,3 +310,139 @@ releaseChannel: non-existant #this release channel will never exist
}

}

func TestMultipleSubnets(t *testing.T) {

validConfigs := []struct {
conf string
subnets []Subnet
}{
{
conf: `
# You can specify multiple subnets to be created in order to achieve H/A
vpcCIDR: 10.4.3.0/16
controllerIP: 10.4.3.50
subnets:
- availabilityZone: ap-northeast-1a
instanceCIDR: 10.4.3.0/24
- availabilityZone: ap-northeast-1c
instanceCIDR: 10.4.4.0/24
`,
subnets: []Subnet{
{
InstanceCIDR: "10.4.3.0/24",
AvailabilityZone: "ap-northeast-1a",
},
{
InstanceCIDR: "10.4.4.0/24",
AvailabilityZone: "ap-northeast-1c",
},
},
},
{
conf: `
# Given AZ/CIDR, missing subnets fall-back to the single subnet with the AZ/CIDR given.
vpcCIDR: 10.4.3.0/16
controllerIP: 10.4.3.50
availabilityZone: ap-northeast-1a
instanceCIDR: 10.4.3.0/24
`,
subnets: []Subnet{
{
AvailabilityZone: "ap-northeast-1a",
InstanceCIDR: "10.4.3.0/24",
},
},
},
{
conf: `
# Given AZ/CIDR, empty subnets fall-back to the single subnet with the AZ/CIDR given.
vpcCIDR: 10.4.3.0/16
controllerIP: 10.4.3.50
availabilityZone: ap-northeast-1a
instanceCIDR: 10.4.3.0/24
subnets: []
`,
subnets: []Subnet{
{
AvailabilityZone: "ap-northeast-1a",
InstanceCIDR: "10.4.3.0/24",
},
},
},
{
conf: `
# Given no AZ/CIDR, empty subnets fall-backs to the single subnet with the default az/cidr.
availabilityZone: "ap-northeast-1a"
subnets: []
`,
subnets: []Subnet{
{
AvailabilityZone: "ap-northeast-1a",
InstanceCIDR: "10.0.0.0/24",
},
},
},
{
conf: `
# Missing subnets field fall-backs to the single subnet with the default az/cidr.
availabilityZone: "ap-northeast-1a"
`,
subnets: []Subnet{
{
AvailabilityZone: "ap-northeast-1a",
InstanceCIDR: "10.0.0.0/24",
},
},
},
}

invalidConfigs := []string{
`
availabilityZone: "ap-northeast-1a"
subnets:
# Missing AZ like this
# - availabilityZone: "ap-northeast-1a"
- instanceCIDR: 10.0.0.0/24
`,
`
subnets:
# Missing AZ like this
# - availabilityZone: "ap-northeast-1a"
- instanceCIDR: 10.0.0.0/24
`,
`
subnets:
# Both AZ/instanceCIDR is given. This is O.K. but...
- availabilityZone: "ap-northeast-1a"
# instanceCIDR does not include the default controllerIP
- instanceCIDR: 10.0.5.0/24
`,
}

for _, conf := range validConfigs {
confBody := minimalConfigYaml + conf.conf
c, err := ClusterFromBytes([]byte(confBody))
if err != nil {
t.Errorf("failed to parse config %s: %v", confBody, err)
continue
}
if !reflect.DeepEqual(c.Subnets, conf.subnets) {
t.Errorf(
"parsed subnets %s does not expected subnets %s in config: %s",
c.Subnets,
conf.subnets,
confBody,
)
}
}

for _, conf := range invalidConfigs {
confBody := minimalConfigYaml + conf
_, err := ClusterFromBytes([]byte(confBody))
if err == nil {
t.Errorf("expected error parsing invalid config: %s", confBody)
}
}

}
52 changes: 37 additions & 15 deletions multi-node/aws/pkg/config/templates/stack-template.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,10 @@
"AutoScaleWorker": {
"Properties": {
"AvailabilityZones": [
"{{.AvailabilityZone}}"
{{range $index, $subnet := .Subnets}}
{{if gt $index 0}},{{end}}
"{{$subnet.AvailabilityZone}}"
{{end}}
],
"DesiredCapacity": "{{.WorkerCount}}",
"HealthCheckGracePeriod": 600,
Expand All @@ -63,9 +66,14 @@
}
],
"VPCZoneIdentifier": [
{{range $index, $subnet := .Subnets}}
{{with $subnetLogicalName := printf "Subnet%d" $index}}
{{if gt $index 0}},{{end}}
{
"Ref": "Subnet"
"Ref": "{{$subnetLogicalName}}"
}
{{end}}
{{end}}
]
},
"Type": "AWS::AutoScaling::AutoScalingGroup",
Expand Down Expand Up @@ -226,7 +234,7 @@
},
"InstanceController": {
"Properties": {
"AvailabilityZone": "{{.AvailabilityZone}}",
"AvailabilityZone": "{{(index .Subnets 0).AvailabilityZone}}",
"BlockDeviceMappings": [
{
"DeviceName": "/dev/xvda",
Expand All @@ -253,7 +261,7 @@
],
"PrivateIpAddress": "{{.ControllerIP}}",
"SubnetId": {
"Ref": "Subnet"
"Ref": "Subnet0"
}
}
],
Expand Down Expand Up @@ -502,22 +510,27 @@
"ToPort": 10255
},
"Type": "AWS::EC2::SecurityGroupIngress"
},
"Subnet": {
}
{{range $index, $subnet := .Subnets}}
{{with $subnetLogicalName := printf "Subnet%d" $index}}
,
"{{$subnetLogicalName}}": {
"Properties": {
"AvailabilityZone": "{{.AvailabilityZone}}",
"CidrBlock": "{{.InstanceCIDR}}",
"AvailabilityZone": "{{$subnet.AvailabilityZone}}",
"CidrBlock": "{{$subnet.InstanceCIDR}}",
"MapPublicIpOnLaunch": true,
"Tags": [
{
"Key": "KubernetesCluster",
"Value": "{{.ClusterName}}"
"Value": "{{$.ClusterName}}"
}
],
"VpcId": {{.VPCRef}}
"VpcId": {{$.VPCRef}}
},
"Type": "AWS::EC2::Subnet"
}
{{end}}
{{end}}
{{if not .VPCID}}
,
"{{.VPCLogicalName}}": {
Expand Down Expand Up @@ -580,30 +593,39 @@
"VpcId": {{.VPCRef}}
},
"Type": "AWS::EC2::VPCGatewayAttachment"
},
"SubnetRouteTableAssociation": {
}
{{range $index, $subnet := .Subnets}}
{{with $subnetLogicalName := printf "Subnet%d" $index}}
,
"{{$subnetLogicalName}}RouteTableAssociation": {
"Properties": {
"RouteTableId": { "Ref" : "RouteTable"},
"SubnetId": {
"Ref": "Subnet"
"Ref": "{{$subnetLogicalName}}"
}
},
"Type": "AWS::EC2::SubnetRouteTableAssociation"
}
{{end}}
{{end}}
{{else}}
{{if .RouteTableID}}
{{range $index, $subnet := .Subnets}}
{{with $subnetLogicalName := printf "Subnet%d" $index}}
,
"SubnetRouteTableAssociation": {
"{{$subnetLogicalName}}RouteTableAssociation": {
"Properties": {
"RouteTableId": "{{.RouteTableID}}",
"SubnetId": {
"Ref": "Subnet"
"Ref": "{{$subnetLogicalName}}"
}
},
"Type": "AWS::EC2::SubnetRouteTableAssociation"
}
{{end}}
{{end}}
{{end}}
{{end}}

}
}

0 comments on commit b6f5bba

Please sign in to comment.