From f8561770909a55f1ab9b58770cbdc83ca212f23e Mon Sep 17 00:00:00 2001 From: Ian Lewis Date: Fri, 29 Mar 2019 11:16:02 +0000 Subject: [PATCH] Update gvisor runsc version - Updates the gvisor addon to use containerd shim v2 - Updates the version of runsc - Auto-installs a gvisor RuntimeClass Issue #4482 --- deploy/addons/gvisor/README.md | 30 +++--- deploy/addons/gvisor/gvisor-config.toml | 11 +- .../addons/gvisor/gvisor-containerd-shim.toml | 3 - deploy/addons/gvisor/gvisor-pod.yaml.tmpl | 6 +- deploy/addons/gvisor/gvisor-runtimeclass.yaml | 22 ++++ pkg/gvisor/enable.go | 14 +-- pkg/minikube/assets/addons.go | 12 +-- pkg/minikube/constants/constants.go | 8 +- test/integration/containerd_test.go | 100 +++++++++++++++++- test/integration/testdata/nginx-gvisor.yaml | 12 +++ .../integration/testdata/nginx-untrusted.yaml | 1 + 11 files changed, 167 insertions(+), 52 deletions(-) delete mode 100644 deploy/addons/gvisor/gvisor-containerd-shim.toml create mode 100644 deploy/addons/gvisor/gvisor-runtimeclass.yaml create mode 100644 test/integration/testdata/nginx-gvisor.yaml diff --git a/deploy/addons/gvisor/README.md b/deploy/addons/gvisor/README.md index 6433189f615a..d6414715b6f4 100644 --- a/deploy/addons/gvisor/README.md +++ b/deploy/addons/gvisor/README.md @@ -1,5 +1,5 @@ ## gVisor Addon -[gVisor](https://github.com/google/gvisor/blob/master/README.md), a sandboxed container runtime, allows users to securely run pods with untrusted workloads within Minikube. +[gVisor](https://gvisor.dev/), a sandboxed container runtime, allows users to securely run pods with untrusted workloads within Minikube. ### Starting Minikube gVisor depends on the containerd runtime to run in Minikube. @@ -17,21 +17,27 @@ To enable this addon, simply run: $ minikube addons enable gvisor ``` -Within one minute, the addon manager should pick up the change and you should see the `gvisor` pod: +Within one minute, the addon manager should pick up the change and you should +see the `gvisor` pod and `gvisor` [Runtime Class](https://kubernetes.io/docs/concepts/containers/runtime-class/): ``` -$ kubectl get pod gvisor -n kube-system -NAME READY STATUS RESTARTS AGE -gvisor 1/1 Running 0 3m +$ kubectl get pod,runtimeclass gvisor -n kube-system +NAME READY STATUS RESTARTS AGE +pod/gvisor 1/1 Running 0 2m52s + +NAME CREATED AT +runtimeclass.node.k8s.io/gvisor 2019-06-15T04:35:09Z ``` -Once the pod has status `Running`, gVisor is enabled in Minikube. +Once the pod has status `Running`, gVisor is enabled in Minikube. ### Running pods in gVisor -To run a pod in gVisor, add this annotation to the Kubernetes yaml: + +To run a pod in gVisor, add the `gvisor` runtime class to the Pod spec in your +Kubernetes yaml: ``` -io.kubernetes.cri.untrusted-workload: "true" +runtimeClassName: gvisor ``` An example Pod is shown below: @@ -41,17 +47,15 @@ apiVersion: v1 kind: Pod metadata: name: nginx-untrusted - annotations: - io.kubernetes.cri.untrusted-workload: "true" spec: + runtimeClassName: gvisor containers: - name: nginx image: nginx ``` -_Note: this annotation will not be necessary once the RuntimeClass Kubernetes feature is available broadly._ - ### Disabling gVisor + To disable gVisor, run: ``` @@ -67,4 +71,4 @@ NAME READY STATUS RESTARTS AGE gvisor 1/1 Terminating 0 5m ``` -_Note: Once gVisor is disabled, any pod with the `io.kubernetes.cri.untrusted-workload` annotation will fail with a FailedCreatePodSandBox error._ +_Note: Once gVisor is disabled, any pod with the `gvisor` Runtime Class or `io.kubernetes.cri.untrusted-workload` annotation will fail with a FailedCreatePodSandBox error._ diff --git a/deploy/addons/gvisor/gvisor-config.toml b/deploy/addons/gvisor/gvisor-config.toml index 034be8a4e57c..073d43912bd2 100644 --- a/deploy/addons/gvisor/gvisor-config.toml +++ b/deploy/addons/gvisor/gvisor-config.toml @@ -41,10 +41,10 @@ oom_score = 0 runtime_type = "io.containerd.runtime.v1.linux" runtime_engine = "" runtime_root = "" - [plugins.cri.containerd.untrusted_workload_runtime] - runtime_type = "io.containerd.runtime.v1.linux" - runtime_engine = "/usr/local/bin/runsc" - runtime_root = "/run/containerd/runsc" + [plugins.cri.containerd.runtimes.untrusted] + runtime_type = "io.containerd.runsc.v1" + [plugins.cri.containerd.runtimes.runsc] + runtime_type = "io.containerd.runsc.v1" [plugins.cri.cni] bin_dir = "/opt/cni/bin" conf_dir = "/etc/cni/net.d" @@ -56,7 +56,6 @@ oom_score = 0 [plugins.diff-service] default = ["walking"] [plugins.linux] - shim = "gvisor-containerd-shim" runtime = "runc" runtime_root = "" no_shim = false @@ -66,4 +65,4 @@ oom_score = 0 deletion_threshold = 0 mutation_threshold = 100 schedule_delay = "0s" - startup_delay = "100ms" \ No newline at end of file + startup_delay = "100ms" diff --git a/deploy/addons/gvisor/gvisor-containerd-shim.toml b/deploy/addons/gvisor/gvisor-containerd-shim.toml deleted file mode 100644 index b4afb9c5150c..000000000000 --- a/deploy/addons/gvisor/gvisor-containerd-shim.toml +++ /dev/null @@ -1,3 +0,0 @@ -runc_shim = "/bin/containerd-shim" -[runsc_config] - user-log="/tmp/runsc/user-log-%ID%.log" \ No newline at end of file diff --git a/deploy/addons/gvisor/gvisor-pod.yaml.tmpl b/deploy/addons/gvisor/gvisor-pod.yaml.tmpl index 0fb277b72677..652dbb681087 100644 --- a/deploy/addons/gvisor/gvisor-pod.yaml.tmpl +++ b/deploy/addons/gvisor/gvisor-pod.yaml.tmpl @@ -14,15 +14,15 @@ apiVersion: v1 kind: Pod -metadata: +metadata: name: gvisor namespace: kube-system labels: addonmanager.kubernetes.io/mode: Reconcile kubernetes.io/minikube-addons: gvisor -spec: +spec: hostPID: true - containers: + containers: - name: gvisor image: {{default "gcr.io/k8s-minikube" .ImageRepository}}/gvisor-addon:latest securityContext: diff --git a/deploy/addons/gvisor/gvisor-runtimeclass.yaml b/deploy/addons/gvisor/gvisor-runtimeclass.yaml new file mode 100644 index 000000000000..f37fa4a9ba64 --- /dev/null +++ b/deploy/addons/gvisor/gvisor-runtimeclass.yaml @@ -0,0 +1,22 @@ +# Copyright 2018 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: node.k8s.io/v1beta1 +kind: RuntimeClass +metadata: + name: gvisor + labels: + kubernetes.io/minikube-addons: gvisor + addonmanager.kubernetes.io/mode: Reconcile +handler: runsc diff --git a/pkg/gvisor/enable.go b/pkg/gvisor/enable.go index 7eacf10f9ae1..403ab0d4d8e8 100644 --- a/pkg/gvisor/enable.go +++ b/pkg/gvisor/enable.go @@ -80,12 +80,6 @@ func makeGvisorDirs() error { return errors.Wrap(err, "creating runsc dir") } - // Make /usr/local/bin to store the runsc binary - fp = filepath.Join(nodeDir, "usr/local/bin") - if err := os.MkdirAll(fp, 0755); err != nil { - return errors.Wrap(err, "creating usr/local/bin dir") - } - // Make /tmp/runsc to also hold logs fp = filepath.Join(nodeDir, "tmp/runsc") if err := os.MkdirAll(fp, 0755); err != nil { @@ -107,13 +101,13 @@ func downloadBinaries() error { // downloads the gvisor-containerd-shim func gvisorContainerdShim() error { - dest := filepath.Join(nodeDir, "usr/bin/gvisor-containerd-shim") + dest := filepath.Join(nodeDir, "usr/bin/containerd-shim-runsc-v1") return downloadFileToDest(constants.GvisorContainerdShimURL, dest) } // downloads the runsc binary and returns a path to the binary func runsc() error { - dest := filepath.Join(nodeDir, "usr/local/bin/runsc") + dest := filepath.Join(nodeDir, "usr/bin/runsc") return downloadFileToDest(constants.GvisorURL, dest) } @@ -159,10 +153,6 @@ func copyConfigFiles() error { if err := mcnutils.CopyFile(filepath.Join(nodeDir, constants.ContainerdConfigTomlPath), filepath.Join(nodeDir, constants.StoredContainerdConfigTomlPath)); err != nil { return errors.Wrap(err, "copying default config.toml") } - log.Print("Copying gvisor-containerd-shim.toml...") - if err := copyAssetToDest(constants.GvisorContainerdShimTargetName, filepath.Join(nodeDir, constants.GvisorContainerdShimTomlPath)); err != nil { - return errors.Wrap(err, "copying gvisor-containerd-shim.toml") - } log.Print("Copying containerd config.toml with gvisor...") if err := copyAssetToDest(constants.GvisorConfigTomlTargetName, filepath.Join(nodeDir, constants.ContainerdConfigTomlPath)); err != nil { return errors.Wrap(err, "copying gvisor version of config.toml") diff --git a/pkg/minikube/assets/addons.go b/pkg/minikube/assets/addons.go index ec72f813e3b8..47cd7eb5122b 100644 --- a/pkg/minikube/assets/addons.go +++ b/pkg/minikube/assets/addons.go @@ -314,18 +314,18 @@ var Addons = map[string]*Addon{ "gvisor-pod.yaml", "0640", true), + MustBinAsset( + "deploy/addons/gvisor/gvisor-runtimeclass.yaml", + constants.GuestAddonsDir, + "gvisor-runtimeclass.yaml", + "0640", + false), MustBinAsset( "deploy/addons/gvisor/gvisor-config.toml", constants.GvisorFilesPath, constants.GvisorConfigTomlTargetName, "0640", true), - MustBinAsset( - "deploy/addons/gvisor/gvisor-containerd-shim.toml", - constants.GvisorFilesPath, - constants.GvisorContainerdShimTargetName, - "0640", - false), }, false, "gvisor"), } diff --git a/pkg/minikube/constants/constants.go b/pkg/minikube/constants/constants.go index ca3a47d8a856..aa54dbb522ce 100644 --- a/pkg/minikube/constants/constants.go +++ b/pkg/minikube/constants/constants.go @@ -384,20 +384,16 @@ const ( GvisorFilesPath = "/tmp/gvisor" // ContainerdConfigTomlPath is the path to the containerd config.toml ContainerdConfigTomlPath = "/etc/containerd/config.toml" - // GvisorContainerdShimTomlPath is the path to gvisor-containerd-shim.toml - GvisorContainerdShimTomlPath = "/etc/containerd/gvisor-containerd-shim.toml" // StoredContainerdConfigTomlPath is the path where the default config.toml will be stored StoredContainerdConfigTomlPath = "/tmp/config.toml" // GvisorConfigTomlTargetName is the go-bindata target name for the gvisor config.toml GvisorConfigTomlTargetName = "gvisor-config.toml" - // GvisorContainerdShimTargetName is the go-bindata target name for gvisor-containerd-shim - GvisorContainerdShimTargetName = "gvisor-containerd-shim.toml" // GvisorContainerdShimURL is the url to download gvisor-containerd-shim - GvisorContainerdShimURL = "https://github.com/google/gvisor-containerd-shim/releases/download/v0.0.1-rc.0/gvisor-containerd-shim-v0.0.1-rc.0.linux-amd64" + GvisorContainerdShimURL = "https://github.com/google/gvisor-containerd-shim/releases/download/v0.0.3/containerd-shim-runsc-v1.linux-amd64" // GvisorURL is the url to download gvisor - GvisorURL = "https://storage.googleapis.com/gvisor/releases/nightly/2018-12-07/runsc" + GvisorURL = "https://storage.googleapis.com/gvisor/releases/nightly/2019-01-14/runsc" ) const ( diff --git a/test/integration/containerd_test.go b/test/integration/containerd_test.go index 8ee9863db132..945a53778223 100644 --- a/test/integration/containerd_test.go +++ b/test/integration/containerd_test.go @@ -37,9 +37,55 @@ func TestContainerd(t *testing.T) { t.Parallel() } + t.Run("GvisorUntrustedWorkload", testGvisorUntrustedWorkload) + t.Run("GvisorRuntimeClass", testGvisorRuntimeClass) t.Run("GvisorRestart", testGvisorRestart) } +func testGvisorUntrustedWorkload(t *testing.T) { + p := profileName(t) + if shouldRunInParallel(t) { + t.Parallel() + } + mk := NewMinikubeRunner(t, p, "--wait=false") + defer mk.TearDown(t) + + mk.RunCommand("addons enable gvisor", true) + + t.Log("waiting for gvisor controller to come up") + if err := waitForGvisorControllerRunning(p); err != nil { + t.Fatalf("waiting for gvisor controller to be up: %v", err) + } + + createUntrustedWorkload(t, p) + t.Log("making sure untrusted workload is Running") + if err := waitForUntrustedNginxRunning(p); err != nil { + t.Fatalf("waiting for nginx to be up: %v", err) + } +} + +func testGvisorRuntimeClass(t *testing.T) { + p := profileName(t) + if shouldRunInParallel(t) { + t.Parallel() + } + mk := NewMinikubeRunner(t, p, "--wait=false") + defer mk.TearDown(t) + + mk.RunCommand("addons enable gvisor", true) + + t.Log("waiting for gvisor controller to come up") + if err := waitForGvisorControllerRunning(p); err != nil { + t.Fatalf("waiting for gvisor controller to be up: %v", err) + } + + createGvisorWorkload(t, p) + t.Log("making sure gvisor workload is Running") + if err := waitForGvisorNginxRunning(p); err != nil { + t.Fatalf("waiting for nginx to be up: %v", err) + } +} + func testGvisorRestart(t *testing.T) { p := profileName(t) if shouldRunInParallel(t) { @@ -98,7 +144,24 @@ func deleteUntrustedWorkload(t *testing.T, profile string) { } } -// waitForGvisorControllerRunning waits for the gvisor controller pod to be running +func createGvisorWorkload(t *testing.T, profile string) { + kr := util.NewKubectlRunner(t, profile) + gvisorPath := filepath.Join(*testdataDir, "nginx-gvisor.yaml") + t.Log("creating pod with gvisor workload annotation") + if _, err := kr.RunCommand([]string{"replace", "-f", gvisorPath, "--force"}); err != nil { + t.Fatalf("creating gvisor nginx resource: %v", err) + } +} + +func deleteGvisorWorkload(t *testing.T, profile string) { + kr := util.NewKubectlRunner(t, profile) + gvisorPath := filepath.Join(*testdataDir, "nginx-gvisor.yaml") + if _, err := kr.RunCommand([]string{"delete", "-f", gvisorPath}); err != nil { + t.Logf("error deleting gvisor nginx resource: %v", err) + } +} + +// waitForGvisorControllerRunning waits for the gvisor controller pod to be running. func waitForGvisorControllerRunning(p string) error { client, err := kapi.Client(p) if err != nil { @@ -112,14 +175,45 @@ func waitForGvisorControllerRunning(p string) error { return nil } -// waitForUntrustedNginxRunning waits for the untrusted nginx pod to start running +// waitForGvisorControllerRunning waits for the gvisor controller pod to +// terminate. +func waitForGvisorControllerDeleted(p string) error { + client, err := kapi.Client(p) + if err != nil { + return errors.Wrap(err, "getting kubernetes client") + } + + selector := labels.SelectorFromSet(labels.Set(map[string]string{"kubernetes.io/minikube-addons": "gvisor"})) + if err := kapi.WaitForPodsWithLabelRunning(client, "kube-system", selector); err != nil { + return errors.Wrap(err, "waiting for gvisor controller pod to stabilize") + } + return nil +} + +// waitForUntrustedNginxRunning waits for the untrusted nginx pod to start +// running. func waitForUntrustedNginxRunning(miniProfile string) error { client, err := kapi.Client(miniProfile) if err != nil { return errors.Wrap(err, "getting kubernetes client") } - selector := labels.SelectorFromSet(labels.Set(map[string]string{"run": "nginx"})) + selector := labels.SelectorFromSet(labels.Set(map[string]string{"run": "nginx", "untrusted": "true"})) + if err := kapi.WaitForPodsWithLabelRunning(client, "default", selector); err != nil { + return errors.Wrap(err, "waiting for nginx pods") + } + return nil +} + +// waitForGvisorNginxRunning waits for the nginx pod with gvisor runtime class +// to start running. +func waitForGvisorNginxRunning(miniProfile string) error { + client, err := kapi.Client(miniProfile) + if err != nil { + return errors.Wrap(err, "getting kubernetes client") + } + + selector := labels.SelectorFromSet(labels.Set(map[string]string{"run": "nginx", "runtime": "gvisor"})) if err := kapi.WaitForPodsWithLabelRunning(client, "default", selector); err != nil { return errors.Wrap(err, "waiting for nginx pods") } diff --git a/test/integration/testdata/nginx-gvisor.yaml b/test/integration/testdata/nginx-gvisor.yaml new file mode 100644 index 000000000000..bc7550703d52 --- /dev/null +++ b/test/integration/testdata/nginx-gvisor.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Pod +metadata: + name: nginx-gvisor + labels: + run: nginx + runtime: gvisor +spec: + runtimeClassName: gvisor + containers: + - name: nginx + image: nginx diff --git a/test/integration/testdata/nginx-untrusted.yaml b/test/integration/testdata/nginx-untrusted.yaml index 7ff9ce9fb704..5559bf476048 100644 --- a/test/integration/testdata/nginx-untrusted.yaml +++ b/test/integration/testdata/nginx-untrusted.yaml @@ -4,6 +4,7 @@ metadata: name: nginx-untrusted labels: run: nginx + untrusted: "true" annotations: io.kubernetes.cri.untrusted-workload: "true" spec: