Add convenience container-runtime flag for kubeadm

hack/jenkins/minikube_set_pending.sh

@@ -27,7 +27,7 @@
 set -e
 set +x
 
-for job in "Linux-KVM-Kubeadm" "OSX-Virtualbox-Kubeadm" "OSX-Virtualbox" "OSX-XHyve" "OSX-Hyperkit" "Linux-Virtualbox" "Linux-KVM" "Linux-KVM-Alt" "Linux-None" "Windows-Virtualbox" "Linux-Container"; do
+for job in "Linux-KVM-Kubeadm" "OSX-Virtualbox-Kubeadm" "OSX-Virtualbox" "OSX-XHyve" "OSX-Hyperkit" "Linux-Virtualbox" "Linux-KVM" "Linux-KVM-Alt" "Linux-None" "Windows-Virtualbox" "Windows-Kubeadm-CRI-O" "Linux-Container"; do
   target_url="https://storage.googleapis.com/minikube-builds/logs/${ghprbPullId}/${job}.txt"
   curl "https://api.github.com/repos/kubernetes/minikube/statuses/${ghprbActualCommit}?access_token=$access_token" \
     -H "Content-Type: application/json" \

hack/jenkins/windows_integration_test_virtualbox_kubeadm_crio.ps1

@@ -0,0 +1,35 @@
+# Copyright 2016 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+mkdir -p out
+gsutil.cmd cp gs://minikube-builds/$env:MINIKUBE_LOCATION/minikube-windows-amd64.exe out/
+gsutil.cmd cp gs://minikube-builds/$env:MINIKUBE_LOCATION/e2e-windows-amd64.exe out/
+gsutil.cmd cp -r gs://minikube-builds/$env:MINIKUBE_LOCATION/testdata .
+
+
+./out/minikube-windows-amd64.exe delete
+Remove-Item -Recurse -Force C:\Users\jenkins\.minikube
+
+out/e2e-windows-amd64.exe --% -minikube-start-args="--vm-driver=virtualbox --container-runtime=cri-o" -minikube-args="--v=10 --logtostderr --bootstrapper=kubeadm" -test.v -test.timeout=30m -binary=out/minikube-windows-amd64.exe
+
+$env:result=$lastexitcode
+# If the last exit code was 0->success, x>0->error
+If($env:result -eq 0){$env:status="success"}
+Else {$env:status="failure"}
+
+$env:target_url="https://storage.googleapis.com/minikube-builds/logs/$env:MINIKUBE_LOCATION/Windows-Kubeadm-CRI-O.txt"
+$json = "{`"state`": `"$env:status`", `"description`": `"Jenkins`", `"target_url`": `"$env:target_url`", `"context`": `"Windows-Kubeadm-CRI-O`"}"
+Invoke-WebRequest -Uri "https://api.github.com/repos/kubernetes/minikube/statuses/$env:COMMIT`?access_token=$env:access_token" -Body $json -ContentType "application/json" -Method Post -usebasicparsing
+
+Exit $env:result

pkg/minikube/assets/vm_assets.go

@@ -20,7 +20,7 @@ import (
 	"bytes"
 	"io"
 	"os"
-	"path/filepath"
+	"path"
 
 	"github.com/pkg/errors"
 )
@@ -65,7 +65,7 @@ type FileAsset struct {
 }
 
 func NewMemoryAssetTarget(d []byte, targetPath, permissions string) *MemoryAsset {
-	return NewMemoryAsset(d, filepath.Dir(targetPath), filepath.Base(targetPath), permissions)
+	return NewMemoryAsset(d, path.Dir(targetPath), path.Base(targetPath), permissions)
 }
 
 func NewFileAsset(assetName, targetDir, targetName, permissions string) (*FileAsset, error) {

pkg/minikube/bootstrapper/kubeadm/kubeadm.go

@@ -21,12 +21,13 @@ import (
 	"crypto"
 	"fmt"
 	"os"
-	"path/filepath"
+	"path"
 	"strings"
 	"time"
 
 	"github.com/docker/machine/libmachine"
 	"github.com/docker/machine/libmachine/state"
+	"github.com/golang/glog"
 	download "github.com/jimmidyson/go-download"
 	"github.com/pkg/errors"
 	"golang.org/x/sync/errgroup"
@@ -178,6 +179,35 @@ func (k *KubeadmBootstrapper) SetupCerts(k8s bootstrapper.KubernetesConfig) erro
 	return bootstrapper.SetupCerts(k.c, k8s)
 }
 
+// SetContainerRuntime possibly sets the container runtime, if it hasn't already
+// been specified by the extra-config option.  It has a set of defaults known to
+// work for a particular runtime.
+func SetContainerRuntime(cfg map[string]string, runtime string) map[string]string {
+	if _, ok := cfg["container-runtime"]; ok {
+		glog.Infoln("Container runtime already set through extra options, ignoring --container-runtime flag.")
+		return cfg
+	}
+
+	if runtime == "" {
+		glog.Infoln("Container runtime flag provided with no value, using defaults.")
+		return cfg
+	}
+
+	switch runtime {
+	case "crio", "cri-o":
+		cfg["container-runtime"] = "remote"
+		cfg["container-runtime-endpoint"] = "/var/run/crio.sock"
+		cfg["image-service-endpoint"] = "/var/run/crio.sock"
+		cfg["runtime-request-timeout"] = "15m"
+	default:
+		cfg["container-runtime"] = runtime
+	}
+
+	return cfg
+}
+
+// NewKubeletConfig generates a new systemd unit containing a configured kubelet
+// based on the options present in the KubernetesConfig.
 func NewKubeletConfig(k8s bootstrapper.KubernetesConfig) (string, error) {
 	version, err := ParseKubernetesVersion(k8s.KubernetesVersion)
 	if err != nil {
@@ -189,9 +219,19 @@ func NewKubeletConfig(k8s bootstrapper.KubernetesConfig) (string, error) {
 		return "", errors.Wrap(err, "generating extra configuration for kubelet")
 	}
 
+	extraOpts = SetContainerRuntime(extraOpts, k8s.ContainerRuntime)
 	extraFlags := convertToFlags(extraOpts)
 	b := bytes.Buffer{}
-	if err := kubeletSystemdTemplate.Execute(&b, map[string]string{"ExtraOptions": extraFlags}); err != nil {
+	opts := struct {
+		ExtraOptions     string
+		FeatureGates     string
+		ContainerRuntime string
+	}{
+		ExtraOptions:     extraFlags,
+		FeatureGates:     k8s.FeatureGates,
+		ContainerRuntime: k8s.ContainerRuntime,
+	}
+	if err := kubeletSystemdTemplate.Execute(&b, opts); err != nil {
 		return "", err
 	}
 
@@ -269,10 +309,11 @@ func generateConfig(k8s bootstrapper.KubernetesConfig) (string, error) {
 	}
 
 	// generates a map of component to extra args for apiserver, controller-manager, and scheduler
-	extraComponentConfig, err := NewComponentExtraArgs(k8s.ExtraOptions, version)
+	extraComponentConfig, err := NewComponentExtraArgs(k8s.ExtraOptions, version, k8s.FeatureGates)
 	if err != nil {
 		return "", errors.Wrap(err, "generating extra component config for kubeadm")
 	}
+
 	opts := struct {
 		CertDir           string
 		ServiceCIDR       string
@@ -303,7 +344,7 @@ func generateConfig(k8s bootstrapper.KubernetesConfig) (string, error) {
 
 func maybeDownloadAndCache(binary, version string) (string, error) {
 	targetDir := constants.MakeMiniPath("cache", version)
-	targetFilepath := filepath.Join(targetDir, binary)
+	targetFilepath := path.Join(targetDir, binary)
 
 	_, err := os.Stat(targetFilepath)
 	// If it exists, do no verification and continue

pkg/minikube/bootstrapper/kubeadm/kubeadm_test.go

@@ -88,11 +88,111 @@ etcd:
   dataDir: /data
 nodeName: extra-args-minikube
 apiServerExtraArgs:
-  fail-no-swap: true
+  fail-no-swap: "true"
 controllerManagerExtraArgs:
-  kube-api-burst: 32
+  kube-api-burst: "32"
 schedulerExtraArgs:
-  scheduler-name: mini-scheduler
+  scheduler-name: "mini-scheduler"
+`,
+		},
+		{
+			description: "two extra args for one component",
+			cfg: bootstrapper.KubernetesConfig{
+				NodeIP:            "192.168.1.101",
+				KubernetesVersion: "v1.8.0-alpha.0",
+				NodeName:          "extra-args-minikube",
+				ExtraOptions: util.ExtraOptionSlice{
+					util.ExtraOption{
+						Component: Apiserver,
+						Key:       "fail-no-swap",
+						Value:     "true",
+					},
+					util.ExtraOption{
+						Component: Apiserver,
+						Key:       "kube-api-burst",
+						Value:     "32",
+					},
+				},
+			},
+			expectedCfg: `apiVersion: kubeadm.k8s.io/v1alpha1
+kind: MasterConfiguration
+api:
+  advertiseAddress: 192.168.1.101
+  bindPort: 8443
+kubernetesVersion: v1.8.0-alpha.0
+certificatesDir: /var/lib/localkube/certs/
+networking:
+  serviceSubnet: 10.0.0.0/24
+etcd:
+  dataDir: /data
+nodeName: extra-args-minikube
+apiServerExtraArgs:
+  fail-no-swap: "true"
+  kube-api-burst: "32"
+`,
+		},
+		{
+			description: "enable feature gates",
+			cfg: bootstrapper.KubernetesConfig{
+				NodeIP:            "192.168.1.101",
+				KubernetesVersion: "v1.8.0-alpha.0",
+				NodeName:          "extra-args-minikube",
+				FeatureGates:      "HugePages=true,OtherFeature=false",
+			},
+			expectedCfg: `apiVersion: kubeadm.k8s.io/v1alpha1
+kind: MasterConfiguration
+api:
+  advertiseAddress: 192.168.1.101
+  bindPort: 8443
+kubernetesVersion: v1.8.0-alpha.0
+certificatesDir: /var/lib/localkube/certs/
+networking:
+  serviceSubnet: 10.0.0.0/24
+etcd:
+  dataDir: /data
+nodeName: extra-args-minikube
+apiServerExtraArgs:
+  feature-gates: "HugePages=true,OtherFeature=false"
+controllerManagerExtraArgs:
+  feature-gates: "HugePages=true,OtherFeature=false"
+schedulerExtraArgs:
+  feature-gates: "HugePages=true,OtherFeature=false"
+`,
+		},
+		{
+			description: "enable feature gates and extra config",
+			cfg: bootstrapper.KubernetesConfig{
+				NodeIP:            "192.168.1.101",
+				KubernetesVersion: "v1.8.0-alpha.0",
+				NodeName:          "extra-args-minikube",
+				FeatureGates:      "HugePages=true,OtherFeature=false",
+				ExtraOptions: util.ExtraOptionSlice{
+					util.ExtraOption{
+						Component: Apiserver,
+						Key:       "fail-no-swap",
+						Value:     "true",
+					},
+				},
+			},
+			expectedCfg: `apiVersion: kubeadm.k8s.io/v1alpha1
+kind: MasterConfiguration
+api:
+  advertiseAddress: 192.168.1.101
+  bindPort: 8443
+kubernetesVersion: v1.8.0-alpha.0
+certificatesDir: /var/lib/localkube/certs/
+networking:
+  serviceSubnet: 10.0.0.0/24
+etcd:
+  dataDir: /data
+nodeName: extra-args-minikube
+apiServerExtraArgs:
+  fail-no-swap: "true"
+  feature-gates: "HugePages=true,OtherFeature=false"
+controllerManagerExtraArgs:
+  feature-gates: "HugePages=true,OtherFeature=false"
+schedulerExtraArgs:
+  feature-gates: "HugePages=true,OtherFeature=false"
 `,
 		},
 		{

pkg/minikube/bootstrapper/kubeadm/templates.go

@@ -16,9 +16,15 @@ limitations under the License.
 
 package kubeadm
 
-import "html/template"
+import (
+	"fmt"
+	"sort"
+	"text/template"
+)
 
-var kubeadmConfigTemplate = template.Must(template.New("kubeadmConfigTemplate").Parse(`apiVersion: kubeadm.k8s.io/v1alpha1
+var kubeadmConfigTemplate = template.Must(template.New("kubeadmConfigTemplate").Funcs(template.FuncMap{
+	"printMapInOrder": printMapInOrder,
+}).Parse(`apiVersion: kubeadm.k8s.io/v1alpha1
 kind: MasterConfiguration
 api:
   advertiseAddress: {{.AdvertiseAddress}}
@@ -30,19 +36,17 @@ networking:
 etcd:
   dataDir: {{.EtcdDataDir}}
 nodeName: {{.NodeName}}
-{{range .ExtraArgs}}{{.Component}}:{{range $key, $value := .Options}}
-  {{$key}}: {{$value}}
-{{end}}{{end}}`))
+{{range .ExtraArgs}}{{.Component}}:{{range $i, $val := printMapInOrder .Options ": " }}
+  {{$val}}{{end}}
+{{end}}`))
 
 var kubeletSystemdTemplate = template.Must(template.New("kubeletSystemdTemplate").Parse(`
 [Service]
-Environment="KUBELET_KUBECONFIG_ARGS=--kubeconfig=/etc/kubernetes/kubelet.conf --require-kubeconfig=true"
-Environment="KUBELET_SYSTEM_PODS_ARGS=--pod-manifest-path=/etc/kubernetes/manifests --allow-privileged=true"
-Environment="KUBELET_DNS_ARGS=--cluster-dns=10.0.0.10 --cluster-domain=cluster.local"
-Environment="KUBELET_CADVISOR_ARGS=--cadvisor-port=0"
-Environment="KUBELET_CGROUP_ARGS=--cgroup-driver=cgroupfs"
 ExecStart=
-ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_SYSTEM_PODS_ARGS $KUBELET_DNS_ARGS $KUBELET_CADVISOR_ARGS $KUBELET_CGROUP_ARGS {{.ExtraOptions}}
+ExecStart=/usr/bin/kubelet {{.ExtraOptions}} {{if .FeatureGates}}--feature-gates={{.FeatureGates}}{{end}}
+
+[Install]
+{{if or (eq .ContainerRuntime "cri-o") (eq .ContainerRuntime "cri")}}Wants=crio.service{{else}}Wants=docker.socket{{end}}
 `))
 
 const kubeletService = `
@@ -68,3 +72,22 @@ sudo /usr/bin/kubeadm alpha phase etcd local --config {{.KubeadmConfigFile}}
 `))
 
 var kubeadmInitTemplate = template.Must(template.New("kubeadmInitTemplate").Parse("sudo /usr/bin/kubeadm init --config {{.KubeadmConfigFile}} --skip-preflight-checks"))
+
+// printMapInOrder sorts the keys and prints the map in order, combining key
+// value pairs with the separator character
+//
+// Note: this is not necessary, but makes testing easy
+func printMapInOrder(m map[string]string, sep string) []string {
+	if m == nil {
+		return nil
+	}
+	keys := []string{}
+	for k := range m {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	for i, k := range keys {
+		keys[i] = fmt.Sprintf("%s%s\"%s\"", k, sep, m[k])
+	}
+	return keys
+}