[kubeadm] Pass features gates to components

pkg/minikube/bootstrapper/kubeadm/kubeadm.go

@@ -191,7 +191,14 @@ func NewKubeletConfig(k8s bootstrapper.KubernetesConfig) (string, error) {
 
 	extraFlags := convertToFlags(extraOpts)
 	b := bytes.Buffer{}
-	if err := kubeletSystemdTemplate.Execute(&b, map[string]string{"ExtraOptions": extraFlags}); err != nil {
+	opts := struct {
+		ExtraOptions string
+		FeatureGates string
+	}{
+		ExtraOptions: extraFlags,
+		FeatureGates: k8s.FeatureGates,
+	}
+	if err := kubeletSystemdTemplate.Execute(&b, opts); err != nil {
 		return "", err
 	}
 
@@ -262,17 +269,35 @@ sudo systemctl start kubelet
 	return nil
 }
 
+func parseFeatureGates(featureGates string) (map[string]string, error) {
+	if featureGates == "" {
+		return nil, nil
+	}
+	fgMap := map[string]string{}
+	fg := strings.Split(featureGates, ",")
+	for _, f := range fg {
+		kv := strings.SplitN(f, "=", 2)
+		if len(kv) != 2 {
+			return nil, fmt.Errorf("Invalid feature gate format: %s", f)
+		}
+		fgMap[kv[0]] = kv[1]
+	}
+
+	return fgMap, nil
+}
+
 func generateConfig(k8s bootstrapper.KubernetesConfig) (string, error) {
 	version, err := ParseKubernetesVersion(k8s.KubernetesVersion)
 	if err != nil {
 		return "", errors.Wrap(err, "parsing kubernetes version")
 	}
 
 	// generates a map of component to extra args for apiserver, controller-manager, and scheduler
-	extraComponentConfig, err := NewComponentExtraArgs(k8s.ExtraOptions, version)
+	extraComponentConfig, err := NewComponentExtraArgs(k8s.ExtraOptions, version, k8s.FeatureGates)
 	if err != nil {
 		return "", errors.Wrap(err, "generating extra component config for kubeadm")
 	}
+
 	opts := struct {
 		CertDir           string
 		ServiceCIDR       string

pkg/minikube/bootstrapper/kubeadm/kubeadm_test.go

@@ -17,6 +17,7 @@ limitations under the License.
 package kubeadm
 
 import (
+	"reflect"
 	"testing"
 
 	"k8s.io/minikube/pkg/minikube/bootstrapper"
@@ -88,11 +89,111 @@ etcd:
   dataDir: /data
 nodeName: extra-args-minikube
 apiServerExtraArgs:
-  fail-no-swap: true
+  fail-no-swap: "true"
 controllerManagerExtraArgs:
-  kube-api-burst: 32
+  kube-api-burst: "32"
 schedulerExtraArgs:
-  scheduler-name: mini-scheduler
+  scheduler-name: "mini-scheduler"
+`,
+		},
+		{
+			description: "two extra args for one component",
+			cfg: bootstrapper.KubernetesConfig{
+				NodeIP:            "192.168.1.101",
+				KubernetesVersion: "v1.8.0-alpha.0",
+				NodeName:          "extra-args-minikube",
+				ExtraOptions: util.ExtraOptionSlice{
+					util.ExtraOption{
+						Component: Apiserver,
+						Key:       "fail-no-swap",
+						Value:     "true",
+					},
+					util.ExtraOption{
+						Component: Apiserver,
+						Key:       "kube-api-burst",
+						Value:     "32",
+					},
+				},
+			},
+			expectedCfg: `apiVersion: kubeadm.k8s.io/v1alpha1
+kind: MasterConfiguration
+api:
+  advertiseAddress: 192.168.1.101
+  bindPort: 8443
+kubernetesVersion: v1.8.0-alpha.0
+certificatesDir: /var/lib/localkube/certs/
+networking:
+  serviceSubnet: 10.0.0.0/24
+etcd:
+  dataDir: /data
+nodeName: extra-args-minikube
+apiServerExtraArgs:
+  fail-no-swap: "true"
+  kube-api-burst: "32"
+`,
+		},
+		{
+			description: "enable feature gates",
+			cfg: bootstrapper.KubernetesConfig{
+				NodeIP:            "192.168.1.101",
+				KubernetesVersion: "v1.8.0-alpha.0",
+				NodeName:          "extra-args-minikube",
+				FeatureGates:      "HugePages=true,OtherFeature=false",
+			},
+			expectedCfg: `apiVersion: kubeadm.k8s.io/v1alpha1
+kind: MasterConfiguration
+api:
+  advertiseAddress: 192.168.1.101
+  bindPort: 8443
+kubernetesVersion: v1.8.0-alpha.0
+certificatesDir: /var/lib/localkube/certs/
+networking:
+  serviceSubnet: 10.0.0.0/24
+etcd:
+  dataDir: /data
+nodeName: extra-args-minikube
+apiServerExtraArgs:
+  feature-gates: "HugePages=true,OtherFeature=false"
+controllerManagerExtraArgs:
+  feature-gates: "HugePages=true,OtherFeature=false"
+schedulerExtraArgs:
+  feature-gates: "HugePages=true,OtherFeature=false"
+`,
+		},
+		{
+			description: "enable feature gates and extra config",
+			cfg: bootstrapper.KubernetesConfig{
+				NodeIP:            "192.168.1.101",
+				KubernetesVersion: "v1.8.0-alpha.0",
+				NodeName:          "extra-args-minikube",
+				FeatureGates:      "HugePages=true,OtherFeature=false",
+				ExtraOptions: util.ExtraOptionSlice{
+					util.ExtraOption{
+						Component: Apiserver,
+						Key:       "fail-no-swap",
+						Value:     "true",
+					},
+				},
+			},
+			expectedCfg: `apiVersion: kubeadm.k8s.io/v1alpha1
+kind: MasterConfiguration
+api:
+  advertiseAddress: 192.168.1.101
+  bindPort: 8443
+kubernetesVersion: v1.8.0-alpha.0
+certificatesDir: /var/lib/localkube/certs/
+networking:
+  serviceSubnet: 10.0.0.0/24
+etcd:
+  dataDir: /data
+nodeName: extra-args-minikube
+apiServerExtraArgs:
+  fail-no-swap: "true"
+  feature-gates: "HugePages=true,OtherFeature=false"
+controllerManagerExtraArgs:
+  feature-gates: "HugePages=true,OtherFeature=false"
+schedulerExtraArgs:
+  feature-gates: "HugePages=true,OtherFeature=false"
 `,
 		},
 		{
@@ -132,3 +233,54 @@ schedulerExtraArgs:
 		})
 	}
 }
+
+func TestParseFeatureGates(t *testing.T) {
+	tests := []struct {
+		description string
+		fg          string
+		expected    map[string]string
+		shouldErr   bool
+	}{
+		{
+			description: "no feature gates",
+		},
+		{
+			description: "one feature gate",
+			fg:          "AppArmor=true",
+			expected: map[string]string{
+				"AppArmor": "true",
+			},
+		},
+		{
+			description: "two feature gates",
+			fg:          "AppArmor=true,HugePages=true",
+			expected: map[string]string{
+				"AppArmor":  "true",
+				"HugePages": "true",
+			},
+		},
+		{
+			description: "missing value pair",
+			fg:          "AppArmor=true,HugePages",
+			shouldErr:   true,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.description, func(t *testing.T) {
+			actual, err := parseFeatureGates(test.fg)
+			t.Logf("%+v", actual)
+			if err == nil && test.shouldErr {
+				t.Errorf("Expected error but got none: fg: %v", actual)
+				return
+			}
+			if err != nil && !test.shouldErr {
+				t.Errorf("Unexpected error: %s", err)
+				return
+			}
+			if !reflect.DeepEqual(actual, test.expected) {
+				t.Errorf("Actual not equal expected: Actual: %v Expected: %v", actual, test.expected)
+			}
+		})
+	}
+}

pkg/minikube/bootstrapper/kubeadm/templates.go

@@ -16,9 +16,15 @@ limitations under the License.
 
 package kubeadm
 
-import "html/template"
+import (
+	"fmt"
+	"sort"
+	"text/template"
+)
 
-var kubeadmConfigTemplate = template.Must(template.New("kubeadmConfigTemplate").Parse(`apiVersion: kubeadm.k8s.io/v1alpha1
+var kubeadmConfigTemplate = template.Must(template.New("kubeadmConfigTemplate").Funcs(template.FuncMap{
+	"printMapInOrder": printMapInOrder,
+}).Parse(`apiVersion: kubeadm.k8s.io/v1alpha1
 kind: MasterConfiguration
 api:
   advertiseAddress: {{.AdvertiseAddress}}
@@ -30,9 +36,9 @@ networking:
 etcd:
   dataDir: {{.EtcdDataDir}}
 nodeName: {{.NodeName}}
-{{range .ExtraArgs}}{{.Component}}:{{range $key, $value := .Options}}
-  {{$key}}: {{$value}}
-{{end}}{{end}}`))
+{{range .ExtraArgs}}{{.Component}}:{{range $i, $val := printMapInOrder .Options ": " }}
+  {{$val}}{{end}}
+{{end}}`))
 
 var kubeletSystemdTemplate = template.Must(template.New("kubeletSystemdTemplate").Parse(`
 [Service]
@@ -42,7 +48,7 @@ Environment="KUBELET_DNS_ARGS=--cluster-dns=10.0.0.10 --cluster-domain=cluster.l
 Environment="KUBELET_CADVISOR_ARGS=--cadvisor-port=0"
 Environment="KUBELET_CGROUP_ARGS=--cgroup-driver=cgroupfs"
 ExecStart=
-ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_SYSTEM_PODS_ARGS $KUBELET_DNS_ARGS $KUBELET_CADVISOR_ARGS $KUBELET_CGROUP_ARGS {{.ExtraOptions}}
+ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_SYSTEM_PODS_ARGS $KUBELET_DNS_ARGS $KUBELET_CADVISOR_ARGS $KUBELET_CGROUP_ARGS {{.ExtraOptions}} {{if .FeatureGates}}--feature-gates={{.FeatureGates}}{{end}}
 `))
 
 const kubeletService = `
@@ -68,3 +74,22 @@ sudo /usr/bin/kubeadm alpha phase etcd local --config {{.KubeadmConfigFile}}
 `))
 
 var kubeadmInitTemplate = template.Must(template.New("kubeadmInitTemplate").Parse("sudo /usr/bin/kubeadm init --config {{.KubeadmConfigFile}} --skip-preflight-checks"))
+
+// printMapInOrder sorts the keys and prints the map in order, combining key
+// value pairs with the separator character
+//
+// Note: this is not necessary, but makes testing easy
+func printMapInOrder(m map[string]string, sep string) []string {
+	if m == nil {
+		return nil
+	}
+	keys := []string{}
+	for k := range m {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	for i, k := range keys {
+		keys[i] = fmt.Sprintf("%s%s\"%s\"", k, sep, m[k])
+	}
+	return keys
+}

pkg/minikube/bootstrapper/kubeadm/templates_test.go

@@ -0,0 +1,62 @@
+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kubeadm
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestPrintMapInOrder(t *testing.T) {
+	tests := []struct {
+		description string
+		m           map[string]string
+		sep         string
+		expected    []string
+	}{
+		{
+			description: "single kv",
+			sep:         ": ",
+			m: map[string]string{
+				"a": "1",
+			},
+			expected: []string{`a: "1"`},
+		},
+		{
+			description: "two kv",
+			sep:         "=",
+			m: map[string]string{
+				"b": "2",
+				"a": "1",
+			},
+			expected: []string{`a="1"`, `b="2"`},
+		},
+		{
+			description: "no kv",
+			sep:         ",",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.description, func(t *testing.T) {
+			actual := printMapInOrder(test.m, test.sep)
+			if !reflect.DeepEqual(actual, test.expected) {
+				t.Errorf("Actual: %v, Expected: %v", actual, test.expected)
+			}
+		})
+	}
+}