feat(cni): taint controller (#4650)

Signed-off-by: slonka <slonka@users.noreply.github.com> * feat(cni): implement cni taint controller to guard against possible race condition
kumahq · Aug 4, 2022 · db95c0e · db95c0e
1 parent f9e25ee
commit db95c0e
Show file tree

Hide file tree

Showing 26 changed files with 737 additions and 65 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -334,7 +334,7 @@ jobs:
               export K3D=true
             fi
             if [[ "<< parameters.arch >>" == "arm64" ]]; then
-              export GINKGO_E2E_TEST_FLAGS="--label-filter=\"!arm-not-supported\""
+              export GINKGO_E2E_TEST_FLAGS="$GINKGO_E2E_TEST_FLAGS --label-filter=\"!arm-not-supported\""
               export MAKE_PARAMETERS="-j1"
             else
               export MAKE_PARAMETERS="-j2"

diff --git a/app/cni/pkg/install/main.go b/app/cni/pkg/install/main.go
@@ -7,6 +7,7 @@ import (
 	"os/signal"
 	"path"
 	"path/filepath"
+	"strings"
 	"syscall"
 	"time"
 
@@ -23,6 +24,7 @@ const (
 	primaryBinDir      = "/host/opt/cni/bin"
 	secondaryBinDir    = "/host/secondary-bin-dir"
 	serviceAccountPath = "/var/run/secrets/kubernetes.io/serviceaccount"
+	readyFilePath      = "/tmp/ready"
 )
 
 var (
@@ -50,6 +52,11 @@ func cleanup(ic *InstallerConfig) {
 	} else {
 		log.V(1).Info("removed kubeconfig")
 	}
+	if err := os.Remove(readyFilePath); err != nil {
+		log.Error(err, "couldn't remove ready file")
+	} else {
+		log.V(1).Info("removed ready file")
+	}
 	log.Info("finished cleanup")
 }
 
@@ -213,12 +220,19 @@ func Run() {
 		log.Error(err, "error occurred during config loading")
 		os.Exit(1)
 	}
+
 	err = install(installerConfig)
 	if err != nil {
 		log.Error(err, "error occurred during cni installation")
 		os.Exit(1)
 	}
 
+	err = atomic.WriteFile(readyFilePath, strings.NewReader(""))
+	if err != nil {
+		log.Error(err, "unable to mark as ready")
+		os.Exit(1)
+	}
+
 	if err := runLoop(installerConfig); err != nil {
 		log.Error(err, "checking installation failed - exiting")
 		os.Exit(1)

diff --git a/app/kumactl/cmd/install/context/install_control_plane_context.go b/app/kumactl/cmd/install/context/install_control_plane_context.go
@@ -41,8 +41,8 @@ type InstallControlPlaneArgs struct {
 	Cni_image_registry                           string            `helm:"cni.image.registry,omitempty"`
 	Cni_image_repository                         string            `helm:"cni.image.repository"`
 	Cni_image_tag                                string            `helm:"cni.image.tag"`
-	Cni_imageExperimental_repository             string            `helm:"cni.imageExperimental.repository"`
-	Cni_imageExperimental_tag                    string            `helm:"cni.imageExperimental.tag"`
+	Cni_experimental_image_repository            string            `helm:"cni.experimental.image.repository"`
+	Cni_experimental_image_tag                   string            `helm:"cni.experimental.image.tag"`
 	Cni_nodeSelector                             map[string]string `helm:"cni.nodeSelector"`
 	ControlPlane_mode                            string            `helm:"controlPlane.mode"`
 	ControlPlane_zone                            string            `helm:"controlPlane.zone"`
@@ -106,8 +106,8 @@ func DefaultInstallCpContext() InstallCpContext {
 			Cni_image_registry:                      "",
 			Cni_image_repository:                    "install-cni",
 			Cni_image_tag:                           "0.0.10",
-			Cni_imageExperimental_repository:        "kuma-cni",
-			Cni_imageExperimental_tag:               kuma_version.Build.Version,
+			Cni_experimental_image_repository:       "kuma-cni",
+			Cni_experimental_image_tag:              kuma_version.Build.Version,
 			ControlPlane_mode:                       core.Standalone,
 			ControlPlane_zone:                       "",
 			ControlPlane_globalZoneSyncService_type: "LoadBalancer",

diff --git a/app/kumactl/cmd/install/install_control_plane.go b/app/kumactl/cmd/install/install_control_plane.go
@@ -35,7 +35,7 @@ func (cv *componentVersion) Set(v string) error {
 	cv.args.ControlPlane_image_tag = v
 	cv.args.DataPlane_image_tag = v
 	cv.args.DataPlane_initImage_tag = v
-	cv.args.Cni_imageExperimental_tag = v
+	cv.args.Cni_experimental_image_tag = v
 	return nil
 }
 

diff --git a/app/kumactl/cmd/install/testdata/install-control-plane.cni-enabled.golden.yaml b/app/kumactl/cmd/install/testdata/install-control-plane.cni-enabled.golden.yaml
@@ -1978,7 +1978,8 @@ spec:
         - name: install-cni
           image: "docker.io/kumahq/install-cni:0.0.10"
           imagePullPolicy: Always
-          command: ["/install-cni.sh"]
+          command: [ "/bin/sh", "-c", "--" ]
+          args: [ "sleep 0 && exec /install-cni.sh" ]
           env:
             # Name of the CNI config file to create.
             - name: CNI_CONF_NAME

diff --git a/app/kumactl/cmd/install/testdata/install-control-plane.cni-experimental-enabled.golden.yaml b/app/kumactl/cmd/install/testdata/install-control-plane.cni-experimental-enabled.golden.yaml
@@ -1787,6 +1787,18 @@ rules:
       - get
       - list
       - watch
+  - apiGroups:
+      - ""
+    resources:
+      - nodes
+    verbs:
+      - update
+  - apiGroups:
+      - "pods"
+    resources:
+      - pods
+    verbs:
+      - list
   # validate k8s token before issuing mTLS cert
   - apiGroups:
       - authentication.k8s.io
@@ -1978,7 +1990,14 @@ spec:
         - name: install-cni
           image: "docker.io/kumahq/kuma-cni:0.0.1"
           imagePullPolicy: IfNotPresent
-          command: ["/install-cni"]
+          readinessProbe:
+            initialDelaySeconds: 0
+            exec:
+              command:
+                - cat
+                - /tmp/ready
+          command: [ "/bin/sh", "-c", "--" ]
+          args: [ "sleep 0 && exec /install-cni" ]
           env:
             # Name of the CNI config file to create.
             - name: CNI_CONF_NAME
@@ -2075,12 +2094,16 @@ spec:
               value: "false"
             - name: KUMA_API_SERVER_READ_ONLY
               value: "true"
+            - name: KUMA_CNI_APP
+              value: "kuma-cni"
             - name: KUMA_DEFAULTS_SKIP_MESH_CREATION
               value: "false"
             - name: KUMA_DP_SERVER_HDS_ENABLED
               value: "false"
             - name: KUMA_ENVIRONMENT
               value: "kubernetes"
+            - name: KUMA_EXPERIMENTAL_CNI
+              value: "true"
             - name: KUMA_GENERAL_TLS_CERT_FILE
               value: "/var/run/secrets/kuma.io/tls-cert/tls.crt"
             - name: KUMA_GENERAL_TLS_KEY_FILE

diff --git a/app/kumactl/cmd/install/testdata/install-control-plane.dump-values.yaml b/app/kumactl/cmd/install/testdata/install-control-plane.dump-values.yaml
@@ -320,12 +320,16 @@ cni:
     # -- CNI image tag
     tag: "0.0.10"
 
+  # -- it's only useful in tests to trigger a possible race condition
+  delayStartupSeconds: 0
+
   # -- use new CNI image (experimental)
-  imageExperimental:
-    # -- CNI experimental image repository
-    repository: "kuma-cni"
-    # -- CNI experimental image tag - defaults to .Chart.AppVersion
-    tag:
+  experimental:
+    image:
+      # -- CNI experimental image repository
+      repository: "kuma-cni"
+      # -- CNI experimental image tag - defaults to .Chart.AppVersion
+      tag:
 
   # -- Security context at the pod level for cni
   podSecurityContext: {}

diff --git a/deployments/charts/kuma/README.md b/deployments/charts/kuma/README.md
@@ -88,9 +88,10 @@ A Helm chart for the Kuma Control Plane
 | cni.image.registry | string | `"docker.io/kumahq"` | CNI image registry |
 | cni.image.repository | string | `"install-cni"` | CNI image repository |
 | cni.image.tag | string | `"0.0.10"` | CNI image tag |
-| cni.imageExperimental | object | `{"repository":"kuma-cni","tag":null}` | use new CNI image (experimental) |
-| cni.imageExperimental.repository | string | `"kuma-cni"` | CNI experimental image repository |
-| cni.imageExperimental.tag | string | `nil` | CNI experimental image tag - defaults to .Chart.AppVersion |
+| cni.delayStartupSeconds | int | `0` | it's only useful in tests to trigger a possible race condition |
+| cni.experimental | object | `{"image":{"repository":"kuma-cni","tag":null}}` | use new CNI image (experimental) |
+| cni.experimental.image.repository | string | `"kuma-cni"` | CNI experimental image repository |
+| cni.experimental.image.tag | string | `nil` | CNI experimental image tag - defaults to .Chart.AppVersion |
 | cni.podSecurityContext | object | `{}` | Security context at the pod level for cni |
 | cni.containerSecurityContext | object | `{}` | Security context at the container level for cni |
 | dataPlane.image.repository | string | `"kuma-dp"` | The Kuma DP image repository |

diff --git a/deployments/charts/kuma/templates/_helpers.tpl b/deployments/charts/kuma/templates/_helpers.tpl
@@ -273,6 +273,12 @@ env:
 - name: KUMA_EXPERIMENTAL_GATEWAY_API
   value: "true"
 {{- end }}
+{{- if .Values.experimental.cni }}
+- name: KUMA_EXPERIMENTAL_CNI
+  value: "true"
+- name: KUMA_CNI_APP
+  value: "kuma-cni"
+{{- end }}
 {{- if .Values.experimental.ebpf.enabled }}
 - name: KUMA_RUNTIME_KUBERNETES_INJECTOR_EBPF_ENABLED
   value: "true"

diff --git a/deployments/charts/kuma/templates/cni-daemonset.yaml b/deployments/charts/kuma/templates/cni-daemonset.yaml
@@ -51,13 +51,21 @@ spec:
       containers:
         - name: install-cni
           {{- if .Values.experimental.cni }}
-          image: {{ include "kuma.formatImage" (dict "image" .Values.cni.imageExperimental "root" $) | quote }}
+          image: {{ include "kuma.formatImage" (dict "image" .Values.cni.experimental.image "root" $) | quote }}
           imagePullPolicy: IfNotPresent
-          command: ["/install-cni"]
+          readinessProbe:
+            initialDelaySeconds: {{ .Values.cni.delayStartupSeconds }}
+            exec:
+              command:
+                - cat
+                - /tmp/ready
+          command: [ "/bin/sh", "-c", "--" ]
+          args: [ "sleep {{.Values.cni.delayStartupSeconds}} && exec /install-cni" ]
           {{- else }}
           image: {{ include "kuma.formatImage" (dict "image" .Values.cni.image "root" $) | quote }}
           imagePullPolicy: Always
-          command: ["/install-cni.sh"]
+          command: [ "/bin/sh", "-c", "--" ]
+          args: [ "sleep {{.Values.cni.delayStartupSeconds}} && exec /install-cni.sh" ]
           {{- end }}
           {{- if .Values.cni.containerSecurityContext }}
           securityContext:

diff --git a/deployments/charts/kuma/templates/cp-rbac.yaml b/deployments/charts/kuma/templates/cp-rbac.yaml
@@ -182,6 +182,20 @@ rules:
       - get
       - list
       - watch
+  {{- if .Values.experimental.cni }}
+  - apiGroups:
+      - ""
+    resources:
+      - nodes
+    verbs:
+      - update
+  - apiGroups:
+      - "pods"
+    resources:
+      - pods
+    verbs:
+      - list
+  {{- end }}
   {{- end }}
   # validate k8s token before issuing mTLS cert
   - apiGroups:

diff --git a/deployments/charts/kuma/values.yaml b/deployments/charts/kuma/values.yaml
@@ -320,12 +320,16 @@ cni:
     # -- CNI image tag
     tag: "0.0.10"
 
+  # -- it's only useful in tests to trigger a possible race condition
+  delayStartupSeconds: 0
+
   # -- use new CNI image (experimental)
-  imageExperimental:
-    # -- CNI experimental image repository
-    repository: "kuma-cni"
-    # -- CNI experimental image tag - defaults to .Chart.AppVersion
-    tag:
+  experimental:
+    image:
+      # -- CNI experimental image repository
+      repository: "kuma-cni"
+      # -- CNI experimental image tag - defaults to .Chart.AppVersion
+      tag:
 
   # -- Security context at the pod level for cni
   podSecurityContext: {}

diff --git a/go.mod b/go.mod
@@ -51,6 +51,7 @@ require (
 	github.com/testcontainers/testcontainers-go v0.13.0
 	go.uber.org/multierr v1.8.0
 	go.uber.org/zap v1.21.0
+	golang.org/x/exp v0.0.0-20220407100705-7b9b53b0aca4
 	golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
 	golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a
@@ -180,7 +181,6 @@ require (
 	go.opencensus.io v0.23.0 // indirect
 	go.uber.org/atomic v1.9.0 // indirect
 	golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 // indirect
-	golang.org/x/exp v0.0.0-20220407100705-7b9b53b0aca4 // indirect
 	golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect
 	golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 // indirect
 	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect

diff --git a/mk/e2e.new.mk b/mk/e2e.new.mk
@@ -38,6 +38,7 @@ K8S_CLUSTER_TOOL=k3d
 E2E_ENV_VARS += KUMA_K8S_TYPE=k3d
 else
 K8S_CLUSTER_TOOL=kind
+GINKGO_E2E_TEST_FLAGS += --label-filter="!kind-not-supported"
 endif
 
 ifdef IPV6

diff --git a/pkg/config/app/kuma-cp/config.go b/pkg/config/app/kuma-cp/config.go
@@ -322,6 +322,11 @@ type ExperimentalConfig struct {
 	// If true, instead of embedding kubernetes outbounds into Dataplane object, they are persisted next to VIPs in ConfigMap
 	// This can improve performance, but it should be enabled only after all instances are migrated to version that supports this config
 	KubeOutboundsAsVIPs bool `yaml:"kubeOutboundsAsVIPs" envconfig:"KUMA_EXPERIMENTAL_KUBE_OUTBOUNDS_AS_VIPS"`
+
+	// If true, new experimental CNI taint controller is enabled
+	Cni bool `yaml:"cni" envconfig:"KUMA_EXPERIMENTAL_CNI"`
+	// Name of the CNI pod
+	CniApp string `yaml:"cniApp" envconfig:"KUMA_CNI_APP"`
 }
 
 func (e ExperimentalConfig) Validate() error {

diff --git a/pkg/config/loader_test.go b/pkg/config/loader_test.go
@@ -260,6 +260,8 @@ var _ = Describe("Config loader", func() {
 			Expect(cfg.Access.Static.ViewClusters.Groups).To(Equal([]string{"zt-group1", "zt-group2"}))
 
 			Expect(cfg.Experimental.GatewayAPI).To(BeTrue())
+			Expect(cfg.Experimental.Cni).To(BeTrue())
+			Expect(cfg.Experimental.CniApp).To(Equal("kuma-cni"))
 			Expect(cfg.Experimental.KubeOutboundsAsVIPs).To(BeTrue())
 		},
 		Entry("from config file", testCase{
@@ -490,6 +492,8 @@ access:
 experimental:
   gatewayAPI: true
   kubeOutboundsAsVIPs: true
+  cni: true
+  cniApp: "kuma-cni"
 `,
 		}),
 		Entry("from env variables", testCase{
@@ -648,6 +652,8 @@ experimental:
 				"KUMA_ACCESS_STATIC_VIEW_CLUSTERS_USERS":                                                   "zt-admin1,zt-admin2",
 				"KUMA_ACCESS_STATIC_VIEW_CLUSTERS_GROUPS":                                                  "zt-group1,zt-group2",
 				"KUMA_EXPERIMENTAL_GATEWAY_API":                                                            "true",
+				"KUMA_EXPERIMENTAL_CNI":                                                                    "true",
+				"KUMA_CNI_APP":                                                                             "kuma-cni",
 				"KUMA_EXPERIMENTAL_KUBE_OUTBOUNDS_AS_VIPS":                                                 "true",
 			},
 			yamlFileConfig: "",