Skip to content
This repository has been archived by the owner on Jun 15, 2021. It is now read-only.

Commit

Permalink
Merge pull request kubernetes-retired#342 from jollinshead/feature/ne…
Browse files Browse the repository at this point in the history
…w-ebs-volume

Add additional EBS volumes to worker nodes.
  • Loading branch information
mumoshu authored Feb 28, 2017
2 parents cae024e + 51d3c66 commit 7133773
Show file tree
Hide file tree
Showing 8 changed files with 271 additions and 2 deletions.
25 changes: 25 additions & 0 deletions core/controlplane/config/templates/cloud-config-worker
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,31 @@ coreos:
etcd_keyfile: /etc/kubernetes/ssl/etcd-client-key.pem

units:
{{range $volumeMountSpecIndex, $volumeMountSpec := .VolumeMounts}}
- name: format-{{$volumeMountSpec.SystemdMountName}}.service
command: start
content: |
[Unit]
Description=Formats the EBS persistent volume drive for {{$volumeMountSpec.Device}}
Before=local-fs-pre.target

[Service]
Type=oneshot
ExecStart=-/usr/sbin/mkfs.xfs {{$volumeMountSpec.Device}}

[Install]
WantedBy=local-fs-pre.target

- name: {{$volumeMountSpec.SystemdMountName}}.mount
command: start
content: |
[Unit]
Description=Mount volume to {{$volumeMountSpec.Path}}

[Mount]
What={{$volumeMountSpec.Device}}
Where={{$volumeMountSpec.Path}}
{{end}}
- name: cfn-etcd-environment.service
enable: true
command: start
Expand Down
11 changes: 11 additions & 0 deletions core/controlplane/config/templates/cluster.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -174,8 +174,19 @@ worker:
# # Must be within the range between 100 and 2000
# # Defaults to worker.spotFleet.unitRootVolumeIOPS * weightedCapacity if omitted
# #rootVolumeIOPS:
#
# - weightedCapacity: 2
# instanceType: c4.xlarge
#
# # # Additional EBS volumes mounted on the worker
# # # No additional EBS volume by default. All parameter values do not default - they must be explicitly defined
# # volumeMounts:
# # - type: "gp2"
# # iops: 0
# # sizeGb: 30
# # # Follow the aws convention of '/dev/xvd*' where '*' is a single letter from 'f' to 'z'
# # device: "/dev/xvdf"
# # mountPath: "/ebs"
# # Used to provide `/etc/environment` env vars with values from arbitrary CloudFormation refs
# awsEnvironment:
# enabled: true
Expand Down
22 changes: 22 additions & 0 deletions core/nodepool/config/templates/stack-template.json
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,18 @@
{{end}}
"VolumeType": "{{$spec.RootVolumeType}}"
}
}{{range $volumeMountSpecIndex, $volumeMountSpec := $.VolumeMounts}},
{
"DeviceName": "{{$volumeMountSpec.Device}}",
"Ebs": {
"VolumeSize": "{{$volumeMountSpec.Size}}",
{{if gt $volumeMountSpec.Iops 0}}
"Iops": "{{$volumeMountSpec.Iops}}",
{{end}}
"VolumeType": "{{$volumeMountSpec.Type}}"
}
}
{{- end -}}
],
"SecurityGroups": [
{{range $sgIndex, $sgRef := $.SecurityGroupRefs}}
Expand Down Expand Up @@ -207,7 +218,18 @@
{{end}}
"VolumeType": "{{.RootVolumeType}}"
}
}{{range $volumeMountSpecIndex, $volumeMountSpec := .VolumeMounts}},
{
"DeviceName": "{{$volumeMountSpec.Device}}",
"Ebs": {
"VolumeSize": "{{$volumeMountSpec.Size}}",
{{if gt $volumeMountSpec.Iops 0}}
"Iops": "{{$volumeMountSpec.Iops}}",
{{end}}
"VolumeType": "{{$volumeMountSpec.Type}}"
}
}
{{- end -}}
],
"IamInstanceProfile": {
"Ref": "IAMInstanceProfileWorker"
Expand Down
6 changes: 6 additions & 0 deletions model/node_pool_config.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ type NodePoolConfig struct {
SecurityGroupIds []string `yaml:"securityGroupIds,omitempty"`
Tenancy string `yaml:"tenancy,omitempty"`
CustomSettings map[string]interface{} `yaml:"customSettings,omitempty"`
VolumeMounts []VolumeMount `yaml:"volumeMounts,omitempty"`
}

type ClusterAutoscaler struct {
Expand Down Expand Up @@ -106,6 +107,10 @@ func (c NodePoolConfig) Valid() error {
return err
}

if err := ValidateVolumeMounts(c.VolumeMounts); err != nil {
return err
}

return nil
}

Expand Down Expand Up @@ -137,5 +142,6 @@ func (c LaunchSpecification) Valid() error {
if err := c.RootVolume.Validate(); err != nil {
return err
}

return nil
}
4 changes: 2 additions & 2 deletions model/root_volume.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,11 +31,11 @@ func (v RootVolume) Validate() error {
}
} else {
if v.RootVolumeIOPS != 0 {
return fmt.Errorf(`invalid rootVolumeIOPS %d for volume type "%s" in %+v": rootVolumeIOPS must be 0 when rootVolumeType is "standard" or "gp1"`, v.RootVolumeIOPS, v.RootVolumeType, v)
return fmt.Errorf(`invalid rootVolumeIOPS %d for volume type "%s" in %+v": rootVolumeIOPS must be 0 when rootVolumeType is "standard" or "gp2"`, v.RootVolumeIOPS, v.RootVolumeType, v)
}

if v.RootVolumeType != "standard" && v.RootVolumeType != "gp2" {
return fmt.Errorf(`invalid rootVolumeType "%s" in %+v: rootVolumeType must be one of "standard", "gp1", "io1"`, v.RootVolumeType, v)
return fmt.Errorf(`invalid rootVolumeType "%s" in %+v: rootVolumeType must be one of "standard", "gp2", "io1"`, v.RootVolumeType, v)
}
}
return nil
Expand Down
1 change: 1 addition & 0 deletions model/spot_fleet.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ func (c SpotFleet) Valid() error {
return fmt.Errorf("invalid launchSpecification at index %d: %v", i, err)
}
}

return nil
}

Expand Down
70 changes: 70 additions & 0 deletions model/volume_mount.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
package model

import (
"fmt"
"regexp"
"strings"
)

type VolumeMount struct {
Type string `yaml:"type,omitempty"`
Iops int `yaml:"iops,omitempty"`
Size int `yaml:"size,omitempty"`
Device string `yaml:"device,omitempty"`
Path string `yaml:"path,omitempty"`
}

func (v VolumeMount) SystemdMountName() string {
return strings.Replace(strings.TrimLeft(v.Path, "/"), "/", "-", -1)
}

func (v VolumeMount) Validate() error {
if v.Type == "io1" {
if v.Iops < 100 || v.Iops > 2000 {
return fmt.Errorf(`invalid iops "%d" in %+v: iops must be between "100" and "2000"`, v.Iops, v)
}
} else {
if v.Iops != 0 {
return fmt.Errorf(`invalid iops "%d" for volume type "%s" in %+v: iops must be "0" when type is "standard" or "gp2"`, v.Iops, v.Type, v)
}

if v.Type != "standard" && v.Type != "gp2" {
return fmt.Errorf(`invalid type "%s" in %+v: type must be one of "standard", "gp2", "io1"`, v.Type, v)
}
}

if v.Size <= 0 {
return fmt.Errorf(`invalid size "%d" in %+v: size must be greater than "0"`, v.Size, v)
}

if v.Path == "" {
return fmt.Errorf(`invalid path "%s" in %v: path cannot be empty`, v.Path, v)
} else if regexp.MustCompile("^[a-zA-Z0-9/]*$").MatchString(v.Path) != true || strings.HasSuffix(v.Path, "/") || strings.HasPrefix(v.Path, "/") == false || strings.Contains(v.Path, "//") {
return fmt.Errorf(`invalid path "%s" in %+v`, v.Path, v)
}

if strings.Compare(v.Device, "/dev/xvdf") == -1 || strings.Compare(v.Device, "/dev/xvdz") == 1 {
return fmt.Errorf(`invalid device "%s" in %+v: device must be a value from "/dev/xvdf" to "/dev/xvdz"`, v.Device, v)
}

return nil
}

func ValidateVolumeMounts(volumes []VolumeMount) error {
paths := make(map[string]bool)
devices := make(map[string]bool)
for _, volume := range volumes {
if err := volume.Validate(); err != nil {
return err
}
if paths[volume.Path] == true {
return fmt.Errorf("duplicate volumeMount path detected (%s) - values must be unique", volume.Path)
}
paths[volume.Path] = true
if devices[volume.Device] == true {
return fmt.Errorf("duplicate volumeMount device detected (%s) - values must be unique", volume.Device)
}
devices[volume.Device] = true
}
return nil
}
134 changes: 134 additions & 0 deletions model/volume_mount_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
package model

import (
"testing"
)

func TestVolumeMountSystemdMountName(t *testing.T) {

c1 := VolumeMount{Path: "/ebs"}
if c1.SystemdMountName() != "ebs" {
t.Errorf("systemdMountName has prodcued an unexpected value '%+v' (expected '%+v') when using a 'path' value of '%+v'", c1.SystemdMountName(), "ebs", c1.Path)
}

c2 := VolumeMount{Path: "/ebs/sbe"}
if c2.SystemdMountName() != "ebs-sbe" {
t.Errorf("systemdMountName has prodcued an unexpected value '%+v' (expected '%+v') when using a 'path' value of '%+v'", c2.SystemdMountName(), "ebs-sbe", c2.Path)
}

c3 := VolumeMount{Path: "/az/AZ/09"}
if c3.SystemdMountName() != "az-AZ-09" {
t.Errorf("systemdMountName has prodcued an unexpected value '%+v' (expected '%+v') when using a 'path' value of '%+v'", c3.SystemdMountName(), "az-AZ-09", c3.Path)
}
}

func TestVolumeMountValidate(t *testing.T) {

c1 := VolumeMount{"gp2", 0, 100, "/dev/xvdf", "/ebs"}
if c1.Validate() != nil {
t.Errorf("validate should not return an error (%+v) with a valid configuration %+v", c1.Validate(), c1)
}

c2 := VolumeMount{"standard", 0, 100, "/dev/xvdf", "/ebs"}
if c2.Validate() != nil {
t.Errorf("validate should not return an error (%+v) with a valid configuration %+v", c2.Validate(), c2)
}

c3 := VolumeMount{"io1", 200, 100, "/dev/xvdf", "/ebs"}
if c3.Validate() != nil {
t.Errorf("validate should not return an error (%+v) with a valid configuration %+v", c3.Validate(), c3)
}

c4 := VolumeMount{"", 0, 100, "/dev/xvdf", "/ebs"}
if c4.Validate() == nil {
t.Errorf("validate should return a 'type' error for using an invalid 'type' value (%+v)", c4.Type)
}

c5 := VolumeMount{"gp2", 0, -5, "/dev/xvdf", "/ebs"}
if c5.Validate() == nil {
t.Errorf("validate should return a 'size' error for using an invalid 'size' value (%d)", c5.Size)
}

c6 := VolumeMount{"io1", 0, 100, "/dev/xvdf", "/ebs"}
if c6.Validate() == nil {
t.Errorf("validate should return a 'iops' error for using an invalid 'iops' value (%d)", c6.Iops)
}

c7 := VolumeMount{"io1", 1E9, 100, "/dev/xvdf", "/ebs"}
if c7.Validate() == nil {
t.Errorf("validate should return a 'size' error for using an invalid 'size' value (%d)", c7.Iops)
}

c8 := VolumeMount{"gp2", 0, 100, "/dev/xvda", "/ebs"}
if c8.Validate() == nil {
t.Errorf("validate should return a 'device' error for using an invalid 'device' value (%+v)", c8.Device)
}

c9 := VolumeMount{"gp2", 0, 100, "/dev/xvdF", "/ebs"}
if c9.Validate() == nil {
t.Errorf("validate should return a 'device' error for using an invalid 'device' value (%+v)", c9.Device)
}

c10 := VolumeMount{"gp2", 0, 100, "/dev/xvdf", "/"}
if c10.Validate() == nil {
t.Errorf("validate should return a 'path' error for using an invalid 'path' value (%+v)", c10.Path)
}

c11 := VolumeMount{"gp2", 0, 100, "/dev/xvdf", "ebs"}
if c11.Validate() == nil {
t.Errorf("validate should return a 'path' error for using an invalid 'path' value (%+v)", c11.Path)
}

c12 := VolumeMount{"gp2", 0, 100, "/dev/xvdf", "/ebs/"}
if c12.Validate() == nil {
t.Errorf("validate should return a 'path' error for using an invalid 'path' value (%+v)", c12.Path)
}

c13 := VolumeMount{"gp2", 0, 100, "/dev/xvdf", "/ebs//sbe"}
if c13.Validate() == nil {
t.Errorf("validate should return a 'path' error for using an invalid 'path' value (%+v)", c13.Path)
}

c14 := VolumeMount{"gp2", 0, 100, "/dev/xvdf", ""}
if c14.Validate() == nil {
t.Errorf("validate should return a 'path' error for using an invalid 'path' value (%+v)", c14.Path)
}

c15 := VolumeMount{"gp2", 0, 100, "/dev/xvdf", "/ebs/sbe"}
if c15.Validate() != nil {
t.Errorf("validate should not return an error (%+v) with a valid configuration %+v", c15.Validate(), c15)
}
}

func TestVolumeMountValidateVolumeMounts(t *testing.T) {

c1 := VolumeMount{"gp2", 0, 100, "/dev/xvdf", "/ebs"}
if c1.Validate() != nil {
t.Errorf("validate should not return an error (%+v) with a valid configuration %+v", c1.Validate(), c1)
}

c2 := VolumeMount{"gp2", 0, 100, "/dev/xvdf", "/ebs2"}
if c2.Validate() != nil {
t.Errorf("validate should not return an error (%+v) with a valid configuration %+v", c2.Validate(), c2)
}

c3 := VolumeMount{"gp2", 0, 100, "/dev/xvdg", "/ebs"}
if c3.Validate() != nil {
t.Errorf("validate should not return an error (%+v) with a valid configuration %+v", c3.Validate(), c3)
}

c4 := []VolumeMount{c2, c3}
if ValidateVolumeMounts(c4) != nil {
t.Errorf("validateEBSVolume should not return an error (%+v) with a valid configuration %+v", ValidateVolumeMounts(c4), c4)
}

c5 := []VolumeMount{c1, c2}
if ValidateVolumeMounts(c5) == nil {
t.Errorf("validate should return a 'device' duplication error for using duplicate 'device' values (%+v) (%+v)", c1.Device, c2.Device)
}

c6 := []VolumeMount{c1, c2}
if ValidateVolumeMounts(c6) == nil {
t.Errorf("validate should return a 'path' duplication error for using duplicate 'path' values (%+v) (%+v)", c1.Path, c2.Path)
}
}

0 comments on commit 7133773

Please sign in to comment.