Merge pull request #2098 from CecileRobertMichon/rm-master-taints

Add control-plane role taints and tolerations
kubernetes-sigs · Feb 18, 2022 · 795082c · 795082c
2 parents e5063d1 + 5d309d0
commit 795082c
Show file tree

Hide file tree

Showing 27 changed files with 117 additions and 21 deletions.
diff --git a/Makefile b/Makefile
@@ -248,7 +248,7 @@ create-management-cluster: $(KUSTOMIZE) $(ENVSUBST) ## Create a management clust
 	./hack/create-identity-secret.sh
 
 	# Deploy CAPI
-	curl --retry $(CURL_RETRIES) -sSL https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.1.0/cluster-api-components.yaml | $(ENVSUBST) | kubectl apply -f -
+	curl --retry $(CURL_RETRIES) -sSL https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.1.2/cluster-api-components.yaml | $(ENVSUBST) | kubectl apply -f -
 
 	# Deploy CAPZ
 	kind load docker-image $(CONTROLLER_IMG)-$(ARCH):$(TAG) --name=capz
@@ -284,7 +284,7 @@ create-workload-cluster: $(ENVSUBST) ## Create a workload cluster.
 	timeout --foreground 300 bash -c "while ! kubectl get secrets | grep $(CLUSTER_NAME)-kubeconfig; do sleep 1; done"
 	# Get kubeconfig and store it locally.
 	kubectl get secrets $(CLUSTER_NAME)-kubeconfig -o json | jq -r .data.value | base64 --decode > ./kubeconfig
-	timeout --foreground 600 bash -c "while ! kubectl --kubeconfig=./kubeconfig get nodes | grep master; do sleep 1; done"
+	timeout --foreground 600 bash -c "while ! kubectl --kubeconfig=./kubeconfig get nodes | grep control-plane; do sleep 1; done"
 
 	@echo 'run "kubectl --kubeconfig=./kubeconfig ..." to work with the new target cluster'
 
@@ -297,7 +297,7 @@ create-aks-cluster: $(KUSTOMIZE) $(ENVSUBST) ## Create a aks cluster.
 	timeout --foreground 300 bash -c "while ! kubectl get secrets | grep $(CLUSTER_NAME)-kubeconfig; do sleep 1; done"
 	# Get kubeconfig and store it locally.
 	kubectl get secrets $(CLUSTER_NAME)-kubeconfig -o json | jq -r .data.value | base64 --decode > ./kubeconfig
-	timeout --foreground 600 bash -c "while ! kubectl --kubeconfig=./kubeconfig get nodes | grep master; do sleep 1; done"
+	timeout --foreground 600 bash -c "while ! kubectl --kubeconfig=./kubeconfig get nodes | grep control-plane; do sleep 1; done"
 
 	@echo 'run "kubectl --kubeconfig=./kubeconfig ..." to work with the new target cluster'
 

diff --git a/Tiltfile b/Tiltfile
@@ -17,7 +17,7 @@ settings = {
     "deploy_cert_manager": True,
     "preload_images_for_kind": True,
     "kind_cluster_name": "capz",
-    "capi_version": "v1.1.0",
+    "capi_version": "v1.1.2",
     "cert_manager_version": "v1.1.0",
     "kubernetes_version": "v1.22.6",
     "aks_kubernetes_version": "v1.22.4",

diff --git a/go.mod b/go.mod
@@ -43,8 +43,8 @@ require (
 	k8s.io/kubectl v0.23.0
 	k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b
 	sigs.k8s.io/cluster-api v1.1.1
-	sigs.k8s.io/cluster-api/test v1.1.0
-	sigs.k8s.io/controller-runtime v0.11.0
+	sigs.k8s.io/cluster-api/test v1.1.2
+	sigs.k8s.io/controller-runtime v0.11.1
 	sigs.k8s.io/kind v0.11.1
 	sigs.k8s.io/yaml v1.3.0
 )

diff --git a/go.sum b/go.sum
@@ -1674,10 +1674,11 @@ sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyz
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.25/go.mod h1:Mlj9PNLmG9bZ6BHFwFKDo5afkpWyUISkb9Me0GnK66I=
 sigs.k8s.io/cluster-api v1.1.1 h1:wU1J+Yyu9xDC5M2uACH4Ii8vrIk37oZiCKeMEfyL0pk=
 sigs.k8s.io/cluster-api v1.1.1/go.mod h1:aSxmmNwDjmg9rDaL0RhRELpGX9uSTTwLF02AMtk/XGY=
-sigs.k8s.io/cluster-api/test v1.1.0 h1:eBVFCWxu9/fb/By02uFjOreG9GAd9eax9uZKsCXP58s=
-sigs.k8s.io/cluster-api/test v1.1.0/go.mod h1:ct7zQrWXsYIAhL/lMLFPfCAqGLDBTEzx+AtwIXpI45o=
-sigs.k8s.io/controller-runtime v0.11.0 h1:DqO+c8mywcZLFJWILq4iktoECTyn30Bkj0CwgqMpZWQ=
+sigs.k8s.io/cluster-api/test v1.1.2 h1:7kGGYqQc1Vn0p/geYXBDOypXJOwLQOcRz9WrFrTHmBY=
+sigs.k8s.io/cluster-api/test v1.1.2/go.mod h1:dk1BBIkLLcvOPuwgKWJ4zfJryGbfCFAZJtWRYo9QrZw=
 sigs.k8s.io/controller-runtime v0.11.0/go.mod h1:KKwLiTooNGu+JmLZGn9Sl3Gjmfj66eMbCQznLP5zcqA=
+sigs.k8s.io/controller-runtime v0.11.1 h1:7YIHT2QnHJArj/dk9aUkYhfqfK5cIxPOX5gPECfdZLU=
+sigs.k8s.io/controller-runtime v0.11.1/go.mod h1:KKwLiTooNGu+JmLZGn9Sl3Gjmfj66eMbCQznLP5zcqA=
 sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 h1:fD1pz4yfdADVNfFmcP2aBEtudwUQ1AlLnRBALr33v3s=
 sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs=
 sigs.k8s.io/kind v0.11.1 h1:pVzOkhUwMBrCB0Q/WllQDO3v14Y+o2V0tFgjTqIUjwA=

diff --git a/hack/log/log-dump-daemonset.yaml b/hack/log/log-dump-daemonset.yaml
@@ -29,6 +29,10 @@ spec:
         key: node-role.kubernetes.io/master
         operator: Equal
         value: "true"
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/control-plane
+        operator: Equal
+        value: "true"
       - effect: NoExecute
         operator: Exists
       - effect: NoSchedule

diff --git a/templates/addons/calico-ipv6.yaml b/templates/addons/calico-ipv6.yaml
@@ -3702,6 +3702,8 @@ spec:
         operator: Exists
       - effect: NoSchedule
         key: node-role.kubernetes.io/master
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/control-plane
 ---
 apiVersion: apps/v1
 kind: Deployment

diff --git a/templates/addons/calico-ipv6/kustomization.yaml b/templates/addons/calico-ipv6/kustomization.yaml
@@ -12,3 +12,8 @@ patches:
     name: calico-node
     namespace: kube-system
   path: patches/calico-node.yaml
+- path: patches/control-plane-tolerations.yaml
+  target:
+    kind: Deployment
+    name: calico-kube-controllers
+    namespace: kube-system
diff --git a/templates/addons/calico-ipv6/patches/control-plane-tolerations.yaml b/templates/addons/calico-ipv6/patches/control-plane-tolerations.yaml
@@ -0,0 +1,15 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: calico-kube-controllers
+  namespace: kube-system
+spec:
+  template:
+    spec:
+      tolerations:
+      - key: CriticalAddonsOnly
+        operator: Exists
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/master
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/control-plane
diff --git a/templates/addons/calico.yaml b/templates/addons/calico.yaml
@@ -3716,6 +3716,8 @@ spec:
         operator: Exists
       - effect: NoSchedule
         key: node-role.kubernetes.io/master
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/control-plane
 ---
 apiVersion: policy/v1beta1
 kind: PodDisruptionBudget

diff --git a/templates/addons/calico/kustomization.yaml b/templates/addons/calico/kustomization.yaml
@@ -13,4 +13,9 @@ patches:
     kind: DaemonSet
     name: calico-node
     namespace: kube-system
-  path: patches/remove-aws-reference.yaml
+  path: patches/remove-aws-reference.yaml
+- path: patches/control-plane-tolerations.yaml
+  target:
+    kind: Deployment
+    name: calico-kube-controllers
+    namespace: kube-system
diff --git a/templates/addons/calico/patches/control-plane-tolerations.yaml b/templates/addons/calico/patches/control-plane-tolerations.yaml
@@ -0,0 +1,15 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: calico-kube-controllers
+  namespace: kube-system
+spec:
+  template:
+    spec:
+      tolerations:
+      - key: CriticalAddonsOnly
+        operator: Exists
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/master
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/control-plane
diff --git a/templates/addons/metrics-server/metrics-server.yaml b/templates/addons/metrics-server/metrics-server.yaml
@@ -176,6 +176,9 @@ spec:
       - effect: NoSchedule
         key: node-role.kubernetes.io/master
         operator: Exists
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/control-plane
+        operator: Exists
       volumes:
       - emptyDir: {}
         name: tmp-dir

diff --git a/templates/addons/metrics-server/patches/control-plane-toleration.yaml b/templates/addons/metrics-server/patches/control-plane-toleration.yaml
@@ -12,3 +12,6 @@ spec:
       - key: "node-role.kubernetes.io/master"
         operator: "Exists"
         effect: "NoSchedule"
+      - key: "node-role.kubernetes.io/control-plane"
+        operator: "Exists"
+        effect: "NoSchedule"
diff --git a/templates/test/ci/cluster-template-prow-ci-version-windows-containerd-2022.yaml b/templates/test/ci/cluster-template-prow-ci-version-windows-containerd-2022.yaml
@@ -4309,6 +4309,8 @@ data:
             operator: Exists
           - effect: NoSchedule
             key: node-role.kubernetes.io/master
+          - effect: NoSchedule
+            key: node-role.kubernetes.io/control-plane
     ---
     apiVersion: policy/v1beta1
     kind: PodDisruptionBudget
@@ -4916,6 +4918,9 @@ data:
           - effect: NoSchedule
             key: node-role.kubernetes.io/master
             operator: Exists
+          - effect: NoSchedule
+            key: node-role.kubernetes.io/control-plane
+            operator: Exists
           volumes:
           - emptyDir: {}
             name: tmp-dir

diff --git a/templates/test/ci/cluster-template-prow-ci-version.yaml b/templates/test/ci/cluster-template-prow-ci-version.yaml
@@ -4309,6 +4309,8 @@ data:
             operator: Exists
           - effect: NoSchedule
             key: node-role.kubernetes.io/master
+          - effect: NoSchedule
+            key: node-role.kubernetes.io/control-plane
     ---
     apiVersion: policy/v1beta1
     kind: PodDisruptionBudget
@@ -4916,6 +4918,9 @@ data:
           - effect: NoSchedule
             key: node-role.kubernetes.io/master
             operator: Exists
+          - effect: NoSchedule
+            key: node-role.kubernetes.io/control-plane
+            operator: Exists
           volumes:
           - emptyDir: {}
             name: tmp-dir

diff --git a/templates/test/ci/cluster-template-prow-custom-vnet.yaml b/templates/test/ci/cluster-template-prow-custom-vnet.yaml
@@ -3989,6 +3989,8 @@ data:
             operator: Exists
           - effect: NoSchedule
             key: node-role.kubernetes.io/master
+          - effect: NoSchedule
+            key: node-role.kubernetes.io/control-plane
     ---
     apiVersion: policy/v1beta1
     kind: PodDisruptionBudget

diff --git a/templates/test/ci/cluster-template-prow-external-cloud-provider.yaml b/templates/test/ci/cluster-template-prow-external-cloud-provider.yaml
@@ -4303,6 +4303,8 @@ data:
             operator: Exists
           - effect: NoSchedule
             key: node-role.kubernetes.io/master
+          - effect: NoSchedule
+            key: node-role.kubernetes.io/control-plane
     ---
     apiVersion: policy/v1beta1
     kind: PodDisruptionBudget

diff --git a/templates/test/ci/cluster-template-prow-ipv6.yaml b/templates/test/ci/cluster-template-prow-ipv6.yaml
@@ -4005,6 +4005,8 @@ data:
             operator: Exists
           - effect: NoSchedule
             key: node-role.kubernetes.io/master
+          - effect: NoSchedule
+            key: node-role.kubernetes.io/control-plane
     ---
     apiVersion: apps/v1
     kind: Deployment

diff --git a/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml b/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml
@@ -4257,6 +4257,8 @@ data:
             operator: Exists
           - effect: NoSchedule
             key: node-role.kubernetes.io/master
+          - effect: NoSchedule
+            key: node-role.kubernetes.io/control-plane
     ---
     apiVersion: policy/v1beta1
     kind: PodDisruptionBudget

diff --git a/templates/test/ci/cluster-template-prow-machine-pool.yaml b/templates/test/ci/cluster-template-prow-machine-pool.yaml
@@ -4070,6 +4070,8 @@ data:
             operator: Exists
           - effect: NoSchedule
             key: node-role.kubernetes.io/master
+          - effect: NoSchedule
+            key: node-role.kubernetes.io/control-plane
     ---
     apiVersion: policy/v1beta1
     kind: PodDisruptionBudget

diff --git a/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml b/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml
@@ -7898,6 +7898,8 @@ data:
             operator: Exists
           - effect: NoSchedule
             key: node-role.kubernetes.io/master
+          - effect: NoSchedule
+            key: node-role.kubernetes.io/control-plane
     ---
     apiVersion: policy/v1beta1
     kind: PodDisruptionBudget

diff --git a/templates/test/ci/cluster-template-prow-private.yaml b/templates/test/ci/cluster-template-prow-private.yaml
@@ -3998,6 +3998,8 @@ data:
             operator: Exists
           - effect: NoSchedule
             key: node-role.kubernetes.io/master
+          - effect: NoSchedule
+            key: node-role.kubernetes.io/control-plane
     ---
     apiVersion: policy/v1beta1
     kind: PodDisruptionBudget

diff --git a/templates/test/ci/cluster-template-prow.yaml b/templates/test/ci/cluster-template-prow.yaml
@@ -4109,6 +4109,8 @@ data:
             operator: Exists
           - effect: NoSchedule
             key: node-role.kubernetes.io/master
+          - effect: NoSchedule
+            key: node-role.kubernetes.io/control-plane
     ---
     apiVersion: policy/v1beta1
     kind: PodDisruptionBudget

diff --git a/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml b/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml
@@ -4160,6 +4160,8 @@ data:
             operator: Exists
           - effect: NoSchedule
             key: node-role.kubernetes.io/master
+          - effect: NoSchedule
+            key: node-role.kubernetes.io/control-plane
     ---
     apiVersion: policy/v1beta1
     kind: PodDisruptionBudget

diff --git a/templates/test/dev/cluster-template-custom-builds.yaml b/templates/test/dev/cluster-template-custom-builds.yaml
@@ -4198,6 +4198,8 @@ data:
             operator: Exists
           - effect: NoSchedule
             key: node-role.kubernetes.io/master
+          - effect: NoSchedule
+            key: node-role.kubernetes.io/control-plane
     ---
     apiVersion: policy/v1beta1
     kind: PodDisruptionBudget

diff --git a/test/e2e/config/azure-dev.yaml b/test/e2e/config/azure-dev.yaml
@@ -3,11 +3,11 @@ managementClusterName: capz-e2e
 images:
   - name: ${MANAGER_IMAGE}
     loadBehavior: mustLoad
-  - name: k8s.gcr.io/cluster-api/cluster-api-controller:v1.1.0
+  - name: k8s.gcr.io/cluster-api/cluster-api-controller:v1.1.2
     loadBehavior: tryLoad
-  - name: k8s.gcr.io/cluster-api/kubeadm-bootstrap-controller:v1.1.0
+  - name: k8s.gcr.io/cluster-api/kubeadm-bootstrap-controller:v1.1.2
     loadBehavior: tryLoad
-  - name: k8s.gcr.io/cluster-api/kubeadm-control-plane-controller:v1.1.0
+  - name: k8s.gcr.io/cluster-api/kubeadm-control-plane-controller:v1.1.2
     loadBehavior: tryLoad
 
 providers:
@@ -32,8 +32,8 @@ providers:
           new: --metrics-addr=:8080
       files:
         - sourcePath: "../data/shared/v1alpha4/metadata.yaml"
-    - name: v1.1.0
-      value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.1.0/core-components.yaml
+    - name: v1.1.2
+      value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.1.2/core-components.yaml
       type: url
       contract: v1beta1
       files:
@@ -64,8 +64,8 @@ providers:
           new: --metrics-addr=:8080
       files:
         - sourcePath: "../data/shared/v1alpha4/metadata.yaml"
-    - name: v1.1.0
-      value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.1.0/bootstrap-components.yaml
+    - name: v1.1.2
+      value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.1.2/bootstrap-components.yaml
       type: url
       contract: v1beta1
       files:
@@ -95,8 +95,8 @@ providers:
           new: --metrics-addr=:8080
       files:
         - sourcePath: "../data/shared/v1alpha4/metadata.yaml"
-    - name: v1.1.0
-      value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.1.0/control-plane-components.yaml
+    - name: v1.1.2
+      value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.1.2/control-plane-components.yaml
       type: url
       contract: v1beta1
       files:

diff --git a/test/e2e/conformance_test.go b/test/e2e/conformance_test.go
@@ -1,3 +1,4 @@
+//go:build e2e
 // +build e2e
 
 /*
@@ -184,12 +185,22 @@ var _ = Describe("Conformance Tests", func() {
 			}
 
 			noScheduleTaint := &corev1.Taint{
-				Key:    "node-role.kubernetes.io/master",
+				Key:    "node-role.kubernetes.io/control-plane",
 				Value:  "",
 				Effect: "NoSchedule",
 			}
 
-			err := node.TaintNode(workloadProxy.GetClientSet(), options, noScheduleTaint)
+			if v, err := semver.ParseTolerant(kubernetesVersion); err == nil {
+				if v.LT(semver.MustParse("1.24.0-alpha.0.0")) {
+					noScheduleTaint = &corev1.Taint{
+						Key:    "node-role.kubernetes.io/master",
+						Value:  "",
+						Effect: "NoSchedule",
+					}
+				}
+			}
+
+			err = node.TaintNode(workloadProxy.GetClientSet(), options, noScheduleTaint)
 			Expect(err).NotTo(HaveOccurred())
 
 			// Windows requires a repo-list because some images are not in k8s gcr