Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add control-plane role taints and tolerations #2098

Merged
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions Makefile
Original file line number Diff line number Diff line change
@@ -248,7 +248,7 @@ create-management-cluster: $(KUSTOMIZE) $(ENVSUBST) ## Create a management clust
./hack/create-identity-secret.sh

# Deploy CAPI
curl --retry $(CURL_RETRIES) -sSL https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.1.0/cluster-api-components.yaml | $(ENVSUBST) | kubectl apply -f -
curl --retry $(CURL_RETRIES) -sSL https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.1.2/cluster-api-components.yaml | $(ENVSUBST) | kubectl apply -f -

# Deploy CAPZ
kind load docker-image $(CONTROLLER_IMG)-$(ARCH):$(TAG) --name=capz
@@ -284,7 +284,7 @@ create-workload-cluster: $(ENVSUBST) ## Create a workload cluster.
timeout --foreground 300 bash -c "while ! kubectl get secrets | grep $(CLUSTER_NAME)-kubeconfig; do sleep 1; done"
# Get kubeconfig and store it locally.
kubectl get secrets $(CLUSTER_NAME)-kubeconfig -o json | jq -r .data.value | base64 --decode > ./kubeconfig
timeout --foreground 600 bash -c "while ! kubectl --kubeconfig=./kubeconfig get nodes | grep master; do sleep 1; done"
timeout --foreground 600 bash -c "while ! kubectl --kubeconfig=./kubeconfig get nodes | grep control-plane; do sleep 1; done"

@echo 'run "kubectl --kubeconfig=./kubeconfig ..." to work with the new target cluster'

@@ -297,7 +297,7 @@ create-aks-cluster: $(KUSTOMIZE) $(ENVSUBST) ## Create a aks cluster.
timeout --foreground 300 bash -c "while ! kubectl get secrets | grep $(CLUSTER_NAME)-kubeconfig; do sleep 1; done"
# Get kubeconfig and store it locally.
kubectl get secrets $(CLUSTER_NAME)-kubeconfig -o json | jq -r .data.value | base64 --decode > ./kubeconfig
timeout --foreground 600 bash -c "while ! kubectl --kubeconfig=./kubeconfig get nodes | grep master; do sleep 1; done"
timeout --foreground 600 bash -c "while ! kubectl --kubeconfig=./kubeconfig get nodes | grep control-plane; do sleep 1; done"

@echo 'run "kubectl --kubeconfig=./kubeconfig ..." to work with the new target cluster'

2 changes: 1 addition & 1 deletion Tiltfile
Original file line number Diff line number Diff line change
@@ -17,7 +17,7 @@ settings = {
"deploy_cert_manager": True,
"preload_images_for_kind": True,
"kind_cluster_name": "capz",
"capi_version": "v1.1.0",
"capi_version": "v1.1.2",
"cert_manager_version": "v1.1.0",
"kubernetes_version": "v1.22.6",
"aks_kubernetes_version": "v1.22.4",
4 changes: 2 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
@@ -43,8 +43,8 @@ require (
k8s.io/kubectl v0.23.0
k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b
sigs.k8s.io/cluster-api v1.1.1
sigs.k8s.io/cluster-api/test v1.1.0
sigs.k8s.io/controller-runtime v0.11.0
sigs.k8s.io/cluster-api/test v1.1.2
sigs.k8s.io/controller-runtime v0.11.1
sigs.k8s.io/kind v0.11.1
sigs.k8s.io/yaml v1.3.0
)
7 changes: 4 additions & 3 deletions go.sum
Original file line number Diff line number Diff line change
@@ -1674,10 +1674,11 @@ sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyz
sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.25/go.mod h1:Mlj9PNLmG9bZ6BHFwFKDo5afkpWyUISkb9Me0GnK66I=
sigs.k8s.io/cluster-api v1.1.1 h1:wU1J+Yyu9xDC5M2uACH4Ii8vrIk37oZiCKeMEfyL0pk=
sigs.k8s.io/cluster-api v1.1.1/go.mod h1:aSxmmNwDjmg9rDaL0RhRELpGX9uSTTwLF02AMtk/XGY=
sigs.k8s.io/cluster-api/test v1.1.0 h1:eBVFCWxu9/fb/By02uFjOreG9GAd9eax9uZKsCXP58s=
sigs.k8s.io/cluster-api/test v1.1.0/go.mod h1:ct7zQrWXsYIAhL/lMLFPfCAqGLDBTEzx+AtwIXpI45o=
sigs.k8s.io/controller-runtime v0.11.0 h1:DqO+c8mywcZLFJWILq4iktoECTyn30Bkj0CwgqMpZWQ=
sigs.k8s.io/cluster-api/test v1.1.2 h1:7kGGYqQc1Vn0p/geYXBDOypXJOwLQOcRz9WrFrTHmBY=
sigs.k8s.io/cluster-api/test v1.1.2/go.mod h1:dk1BBIkLLcvOPuwgKWJ4zfJryGbfCFAZJtWRYo9QrZw=
sigs.k8s.io/controller-runtime v0.11.0/go.mod h1:KKwLiTooNGu+JmLZGn9Sl3Gjmfj66eMbCQznLP5zcqA=
sigs.k8s.io/controller-runtime v0.11.1 h1:7YIHT2QnHJArj/dk9aUkYhfqfK5cIxPOX5gPECfdZLU=
sigs.k8s.io/controller-runtime v0.11.1/go.mod h1:KKwLiTooNGu+JmLZGn9Sl3Gjmfj66eMbCQznLP5zcqA=
sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 h1:fD1pz4yfdADVNfFmcP2aBEtudwUQ1AlLnRBALr33v3s=
sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs=
sigs.k8s.io/kind v0.11.1 h1:pVzOkhUwMBrCB0Q/WllQDO3v14Y+o2V0tFgjTqIUjwA=
4 changes: 4 additions & 0 deletions hack/log/log-dump-daemonset.yaml
Original file line number Diff line number Diff line change
@@ -29,6 +29,10 @@ spec:
key: node-role.kubernetes.io/master
operator: Equal
value: "true"
- effect: NoSchedule
key: node-role.kubernetes.io/control-plane
operator: Equal
value: "true"
- effect: NoExecute
operator: Exists
- effect: NoSchedule
2 changes: 2 additions & 0 deletions templates/addons/calico-ipv6.yaml
Original file line number Diff line number Diff line change
@@ -3702,6 +3702,8 @@ spec:
operator: Exists
- effect: NoSchedule
key: node-role.kubernetes.io/master
- effect: NoSchedule
key: node-role.kubernetes.io/control-plane
---
apiVersion: apps/v1
kind: Deployment
5 changes: 5 additions & 0 deletions templates/addons/calico-ipv6/kustomization.yaml
Original file line number Diff line number Diff line change
@@ -12,3 +12,8 @@ patches:
name: calico-node
namespace: kube-system
path: patches/calico-node.yaml
- path: patches/control-plane-tolerations.yaml
target:
kind: Deployment
name: calico-kube-controllers
namespace: kube-system
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: calico-kube-controllers
namespace: kube-system
spec:
template:
spec:
tolerations:
- key: CriticalAddonsOnly
operator: Exists
- effect: NoSchedule
key: node-role.kubernetes.io/master
- effect: NoSchedule
key: node-role.kubernetes.io/control-plane
2 changes: 2 additions & 0 deletions templates/addons/calico.yaml
Original file line number Diff line number Diff line change
@@ -3716,6 +3716,8 @@ spec:
operator: Exists
- effect: NoSchedule
key: node-role.kubernetes.io/master
- effect: NoSchedule
key: node-role.kubernetes.io/control-plane
---
apiVersion: policy/v1beta1
kind: PodDisruptionBudget
7 changes: 6 additions & 1 deletion templates/addons/calico/kustomization.yaml
Original file line number Diff line number Diff line change
@@ -13,4 +13,9 @@ patches:
kind: DaemonSet
name: calico-node
namespace: kube-system
path: patches/remove-aws-reference.yaml
path: patches/remove-aws-reference.yaml
- path: patches/control-plane-tolerations.yaml
target:
kind: Deployment
name: calico-kube-controllers
namespace: kube-system
15 changes: 15 additions & 0 deletions templates/addons/calico/patches/control-plane-tolerations.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: calico-kube-controllers
namespace: kube-system
spec:
template:
spec:
tolerations:
- key: CriticalAddonsOnly
operator: Exists
- effect: NoSchedule
key: node-role.kubernetes.io/master
- effect: NoSchedule
key: node-role.kubernetes.io/control-plane
3 changes: 3 additions & 0 deletions templates/addons/metrics-server/metrics-server.yaml
Original file line number Diff line number Diff line change
@@ -176,6 +176,9 @@ spec:
- effect: NoSchedule
key: node-role.kubernetes.io/master
operator: Exists
- effect: NoSchedule
key: node-role.kubernetes.io/control-plane
operator: Exists
volumes:
- emptyDir: {}
name: tmp-dir
Original file line number Diff line number Diff line change
@@ -12,3 +12,6 @@ spec:
- key: "node-role.kubernetes.io/master"
operator: "Exists"
effect: "NoSchedule"
- key: "node-role.kubernetes.io/control-plane"
operator: "Exists"
effect: "NoSchedule"
Original file line number Diff line number Diff line change
@@ -4309,6 +4309,8 @@ data:
operator: Exists
- effect: NoSchedule
key: node-role.kubernetes.io/master
- effect: NoSchedule
key: node-role.kubernetes.io/control-plane
---
apiVersion: policy/v1beta1
kind: PodDisruptionBudget
@@ -4916,6 +4918,9 @@ data:
- effect: NoSchedule
key: node-role.kubernetes.io/master
operator: Exists
- effect: NoSchedule
key: node-role.kubernetes.io/control-plane
operator: Exists
volumes:
- emptyDir: {}
name: tmp-dir
5 changes: 5 additions & 0 deletions templates/test/ci/cluster-template-prow-ci-version.yaml
Original file line number Diff line number Diff line change
@@ -4309,6 +4309,8 @@ data:
operator: Exists
- effect: NoSchedule
key: node-role.kubernetes.io/master
- effect: NoSchedule
key: node-role.kubernetes.io/control-plane
---
apiVersion: policy/v1beta1
kind: PodDisruptionBudget
@@ -4916,6 +4918,9 @@ data:
- effect: NoSchedule
key: node-role.kubernetes.io/master
operator: Exists
- effect: NoSchedule
key: node-role.kubernetes.io/control-plane
operator: Exists
volumes:
- emptyDir: {}
name: tmp-dir
2 changes: 2 additions & 0 deletions templates/test/ci/cluster-template-prow-custom-vnet.yaml
Original file line number Diff line number Diff line change
@@ -3989,6 +3989,8 @@ data:
operator: Exists
- effect: NoSchedule
key: node-role.kubernetes.io/master
- effect: NoSchedule
key: node-role.kubernetes.io/control-plane
---
apiVersion: policy/v1beta1
kind: PodDisruptionBudget
Original file line number Diff line number Diff line change
@@ -4303,6 +4303,8 @@ data:
operator: Exists
- effect: NoSchedule
key: node-role.kubernetes.io/master
- effect: NoSchedule
key: node-role.kubernetes.io/control-plane
---
apiVersion: policy/v1beta1
kind: PodDisruptionBudget
2 changes: 2 additions & 0 deletions templates/test/ci/cluster-template-prow-ipv6.yaml
Original file line number Diff line number Diff line change
@@ -4005,6 +4005,8 @@ data:
operator: Exists
- effect: NoSchedule
key: node-role.kubernetes.io/master
- effect: NoSchedule
key: node-role.kubernetes.io/control-plane
---
apiVersion: apps/v1
kind: Deployment
Original file line number Diff line number Diff line change
@@ -4257,6 +4257,8 @@ data:
operator: Exists
- effect: NoSchedule
key: node-role.kubernetes.io/master
- effect: NoSchedule
key: node-role.kubernetes.io/control-plane
---
apiVersion: policy/v1beta1
kind: PodDisruptionBudget
2 changes: 2 additions & 0 deletions templates/test/ci/cluster-template-prow-machine-pool.yaml
Original file line number Diff line number Diff line change
@@ -4070,6 +4070,8 @@ data:
operator: Exists
- effect: NoSchedule
key: node-role.kubernetes.io/master
- effect: NoSchedule
key: node-role.kubernetes.io/control-plane
---
apiVersion: policy/v1beta1
kind: PodDisruptionBudget
2 changes: 2 additions & 0 deletions templates/test/ci/cluster-template-prow-nvidia-gpu.yaml
Original file line number Diff line number Diff line change
@@ -7898,6 +7898,8 @@ data:
operator: Exists
- effect: NoSchedule
key: node-role.kubernetes.io/master
- effect: NoSchedule
key: node-role.kubernetes.io/control-plane
---
apiVersion: policy/v1beta1
kind: PodDisruptionBudget
2 changes: 2 additions & 0 deletions templates/test/ci/cluster-template-prow-private.yaml
Original file line number Diff line number Diff line change
@@ -3998,6 +3998,8 @@ data:
operator: Exists
- effect: NoSchedule
key: node-role.kubernetes.io/master
- effect: NoSchedule
key: node-role.kubernetes.io/control-plane
---
apiVersion: policy/v1beta1
kind: PodDisruptionBudget
2 changes: 2 additions & 0 deletions templates/test/ci/cluster-template-prow.yaml
Original file line number Diff line number Diff line change
@@ -4109,6 +4109,8 @@ data:
operator: Exists
- effect: NoSchedule
key: node-role.kubernetes.io/master
- effect: NoSchedule
key: node-role.kubernetes.io/control-plane
---
apiVersion: policy/v1beta1
kind: PodDisruptionBudget
Original file line number Diff line number Diff line change
@@ -4160,6 +4160,8 @@ data:
operator: Exists
- effect: NoSchedule
key: node-role.kubernetes.io/master
- effect: NoSchedule
key: node-role.kubernetes.io/control-plane
---
apiVersion: policy/v1beta1
kind: PodDisruptionBudget
2 changes: 2 additions & 0 deletions templates/test/dev/cluster-template-custom-builds.yaml
Original file line number Diff line number Diff line change
@@ -4198,6 +4198,8 @@ data:
operator: Exists
- effect: NoSchedule
key: node-role.kubernetes.io/master
- effect: NoSchedule
key: node-role.kubernetes.io/control-plane
---
apiVersion: policy/v1beta1
kind: PodDisruptionBudget
18 changes: 9 additions & 9 deletions test/e2e/config/azure-dev.yaml
Original file line number Diff line number Diff line change
@@ -3,11 +3,11 @@ managementClusterName: capz-e2e
images:
- name: ${MANAGER_IMAGE}
loadBehavior: mustLoad
- name: k8s.gcr.io/cluster-api/cluster-api-controller:v1.1.0
- name: k8s.gcr.io/cluster-api/cluster-api-controller:v1.1.2
loadBehavior: tryLoad
- name: k8s.gcr.io/cluster-api/kubeadm-bootstrap-controller:v1.1.0
- name: k8s.gcr.io/cluster-api/kubeadm-bootstrap-controller:v1.1.2
loadBehavior: tryLoad
- name: k8s.gcr.io/cluster-api/kubeadm-control-plane-controller:v1.1.0
- name: k8s.gcr.io/cluster-api/kubeadm-control-plane-controller:v1.1.2
loadBehavior: tryLoad

providers:
@@ -32,8 +32,8 @@ providers:
new: --metrics-addr=:8080
files:
- sourcePath: "../data/shared/v1alpha4/metadata.yaml"
- name: v1.1.0
value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.1.0/core-components.yaml
- name: v1.1.2
value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.1.2/core-components.yaml
type: url
contract: v1beta1
files:
@@ -64,8 +64,8 @@ providers:
new: --metrics-addr=:8080
files:
- sourcePath: "../data/shared/v1alpha4/metadata.yaml"
- name: v1.1.0
value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.1.0/bootstrap-components.yaml
- name: v1.1.2
value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.1.2/bootstrap-components.yaml
type: url
contract: v1beta1
files:
@@ -95,8 +95,8 @@ providers:
new: --metrics-addr=:8080
files:
- sourcePath: "../data/shared/v1alpha4/metadata.yaml"
- name: v1.1.0
value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.1.0/control-plane-components.yaml
- name: v1.1.2
value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.1.2/control-plane-components.yaml
type: url
contract: v1beta1
files:
15 changes: 13 additions & 2 deletions test/e2e/conformance_test.go
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
//go:build e2e
// +build e2e

/*
@@ -184,12 +185,22 @@ var _ = Describe("Conformance Tests", func() {
}

noScheduleTaint := &corev1.Taint{
CecileRobertMichon marked this conversation as resolved.
Show resolved Hide resolved
Key: "node-role.kubernetes.io/master",
Key: "node-role.kubernetes.io/control-plane",
Value: "",
Effect: "NoSchedule",
}

err := node.TaintNode(workloadProxy.GetClientSet(), options, noScheduleTaint)
if v, err := semver.ParseTolerant(kubernetesVersion); err == nil {
if v.LT(semver.MustParse("1.24.0-alpha.0.0")) {
noScheduleTaint = &corev1.Taint{
Key: "node-role.kubernetes.io/master",
Value: "",
Effect: "NoSchedule",
}
}
}

err = node.TaintNode(workloadProxy.GetClientSet(), options, noScheduleTaint)
Expect(err).NotTo(HaveOccurred())

// Windows requires a repo-list because some images are not in k8s gcr