Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add control-plane role taints and tolerations #2098

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -248,7 +248,7 @@ create-management-cluster: $(KUSTOMIZE) $(ENVSUBST) ## Create a management clust
./hack/create-identity-secret.sh

# Deploy CAPI
curl --retry $(CURL_RETRIES) -sSL https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.1.0/cluster-api-components.yaml | $(ENVSUBST) | kubectl apply -f -
curl --retry $(CURL_RETRIES) -sSL https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.1.2/cluster-api-components.yaml | $(ENVSUBST) | kubectl apply -f -

# Deploy CAPZ
kind load docker-image $(CONTROLLER_IMG)-$(ARCH):$(TAG) --name=capz
Expand Down Expand Up @@ -284,7 +284,7 @@ create-workload-cluster: $(ENVSUBST) ## Create a workload cluster.
timeout --foreground 300 bash -c "while ! kubectl get secrets | grep $(CLUSTER_NAME)-kubeconfig; do sleep 1; done"
# Get kubeconfig and store it locally.
kubectl get secrets $(CLUSTER_NAME)-kubeconfig -o json | jq -r .data.value | base64 --decode > ./kubeconfig
timeout --foreground 600 bash -c "while ! kubectl --kubeconfig=./kubeconfig get nodes | grep master; do sleep 1; done"
timeout --foreground 600 bash -c "while ! kubectl --kubeconfig=./kubeconfig get nodes | grep control-plane; do sleep 1; done"

@echo 'run "kubectl --kubeconfig=./kubeconfig ..." to work with the new target cluster'

Expand All @@ -297,7 +297,7 @@ create-aks-cluster: $(KUSTOMIZE) $(ENVSUBST) ## Create a aks cluster.
timeout --foreground 300 bash -c "while ! kubectl get secrets | grep $(CLUSTER_NAME)-kubeconfig; do sleep 1; done"
# Get kubeconfig and store it locally.
kubectl get secrets $(CLUSTER_NAME)-kubeconfig -o json | jq -r .data.value | base64 --decode > ./kubeconfig
timeout --foreground 600 bash -c "while ! kubectl --kubeconfig=./kubeconfig get nodes | grep master; do sleep 1; done"
timeout --foreground 600 bash -c "while ! kubectl --kubeconfig=./kubeconfig get nodes | grep control-plane; do sleep 1; done"

@echo 'run "kubectl --kubeconfig=./kubeconfig ..." to work with the new target cluster'

Expand Down
2 changes: 1 addition & 1 deletion Tiltfile
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ settings = {
"deploy_cert_manager": True,
"preload_images_for_kind": True,
"kind_cluster_name": "capz",
"capi_version": "v1.1.0",
"capi_version": "v1.1.2",
"cert_manager_version": "v1.1.0",
"kubernetes_version": "v1.22.6",
"aks_kubernetes_version": "v1.22.4",
Expand Down
4 changes: 2 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,8 @@ require (
k8s.io/kubectl v0.23.0
k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b
sigs.k8s.io/cluster-api v1.1.1
sigs.k8s.io/cluster-api/test v1.1.0
sigs.k8s.io/controller-runtime v0.11.0
sigs.k8s.io/cluster-api/test v1.1.2
sigs.k8s.io/controller-runtime v0.11.1
sigs.k8s.io/kind v0.11.1
sigs.k8s.io/yaml v1.3.0
)
Expand Down
7 changes: 4 additions & 3 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -1674,10 +1674,11 @@ sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyz
sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.25/go.mod h1:Mlj9PNLmG9bZ6BHFwFKDo5afkpWyUISkb9Me0GnK66I=
sigs.k8s.io/cluster-api v1.1.1 h1:wU1J+Yyu9xDC5M2uACH4Ii8vrIk37oZiCKeMEfyL0pk=
sigs.k8s.io/cluster-api v1.1.1/go.mod h1:aSxmmNwDjmg9rDaL0RhRELpGX9uSTTwLF02AMtk/XGY=
sigs.k8s.io/cluster-api/test v1.1.0 h1:eBVFCWxu9/fb/By02uFjOreG9GAd9eax9uZKsCXP58s=
sigs.k8s.io/cluster-api/test v1.1.0/go.mod h1:ct7zQrWXsYIAhL/lMLFPfCAqGLDBTEzx+AtwIXpI45o=
sigs.k8s.io/controller-runtime v0.11.0 h1:DqO+c8mywcZLFJWILq4iktoECTyn30Bkj0CwgqMpZWQ=
sigs.k8s.io/cluster-api/test v1.1.2 h1:7kGGYqQc1Vn0p/geYXBDOypXJOwLQOcRz9WrFrTHmBY=
sigs.k8s.io/cluster-api/test v1.1.2/go.mod h1:dk1BBIkLLcvOPuwgKWJ4zfJryGbfCFAZJtWRYo9QrZw=
sigs.k8s.io/controller-runtime v0.11.0/go.mod h1:KKwLiTooNGu+JmLZGn9Sl3Gjmfj66eMbCQznLP5zcqA=
sigs.k8s.io/controller-runtime v0.11.1 h1:7YIHT2QnHJArj/dk9aUkYhfqfK5cIxPOX5gPECfdZLU=
sigs.k8s.io/controller-runtime v0.11.1/go.mod h1:KKwLiTooNGu+JmLZGn9Sl3Gjmfj66eMbCQznLP5zcqA=
sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 h1:fD1pz4yfdADVNfFmcP2aBEtudwUQ1AlLnRBALr33v3s=
sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs=
sigs.k8s.io/kind v0.11.1 h1:pVzOkhUwMBrCB0Q/WllQDO3v14Y+o2V0tFgjTqIUjwA=
Expand Down
4 changes: 4 additions & 0 deletions hack/log/log-dump-daemonset.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,10 @@ spec:
key: node-role.kubernetes.io/master
operator: Equal
value: "true"
- effect: NoSchedule
key: node-role.kubernetes.io/control-plane
operator: Equal
value: "true"
- effect: NoExecute
operator: Exists
- effect: NoSchedule
Expand Down
2 changes: 2 additions & 0 deletions templates/addons/calico-ipv6.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -3702,6 +3702,8 @@ spec:
operator: Exists
- effect: NoSchedule
key: node-role.kubernetes.io/master
- effect: NoSchedule
key: node-role.kubernetes.io/control-plane
---
apiVersion: apps/v1
kind: Deployment
Expand Down
5 changes: 5 additions & 0 deletions templates/addons/calico-ipv6/kustomization.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -12,3 +12,8 @@ patches:
name: calico-node
namespace: kube-system
path: patches/calico-node.yaml
- path: patches/control-plane-tolerations.yaml
target:
kind: Deployment
name: calico-kube-controllers
namespace: kube-system
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: calico-kube-controllers
namespace: kube-system
spec:
template:
spec:
tolerations:
- key: CriticalAddonsOnly
operator: Exists
- effect: NoSchedule
key: node-role.kubernetes.io/master
- effect: NoSchedule
key: node-role.kubernetes.io/control-plane
2 changes: 2 additions & 0 deletions templates/addons/calico.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -3716,6 +3716,8 @@ spec:
operator: Exists
- effect: NoSchedule
key: node-role.kubernetes.io/master
- effect: NoSchedule
key: node-role.kubernetes.io/control-plane
---
apiVersion: policy/v1beta1
kind: PodDisruptionBudget
Expand Down
7 changes: 6 additions & 1 deletion templates/addons/calico/kustomization.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -13,4 +13,9 @@ patches:
kind: DaemonSet
name: calico-node
namespace: kube-system
path: patches/remove-aws-reference.yaml
path: patches/remove-aws-reference.yaml
- path: patches/control-plane-tolerations.yaml
target:
kind: Deployment
name: calico-kube-controllers
namespace: kube-system
15 changes: 15 additions & 0 deletions templates/addons/calico/patches/control-plane-tolerations.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: calico-kube-controllers
namespace: kube-system
spec:
template:
spec:
tolerations:
- key: CriticalAddonsOnly
operator: Exists
- effect: NoSchedule
key: node-role.kubernetes.io/master
- effect: NoSchedule
key: node-role.kubernetes.io/control-plane
3 changes: 3 additions & 0 deletions templates/addons/metrics-server/metrics-server.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,9 @@ spec:
- effect: NoSchedule
key: node-role.kubernetes.io/master
operator: Exists
- effect: NoSchedule
key: node-role.kubernetes.io/control-plane
operator: Exists
volumes:
- emptyDir: {}
name: tmp-dir
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,3 +12,6 @@ spec:
- key: "node-role.kubernetes.io/master"
operator: "Exists"
effect: "NoSchedule"
- key: "node-role.kubernetes.io/control-plane"
operator: "Exists"
effect: "NoSchedule"
Original file line number Diff line number Diff line change
Expand Up @@ -4309,6 +4309,8 @@ data:
operator: Exists
- effect: NoSchedule
key: node-role.kubernetes.io/master
- effect: NoSchedule
key: node-role.kubernetes.io/control-plane
---
apiVersion: policy/v1beta1
kind: PodDisruptionBudget
Expand Down Expand Up @@ -4916,6 +4918,9 @@ data:
- effect: NoSchedule
key: node-role.kubernetes.io/master
operator: Exists
- effect: NoSchedule
key: node-role.kubernetes.io/control-plane
operator: Exists
volumes:
- emptyDir: {}
name: tmp-dir
Expand Down
5 changes: 5 additions & 0 deletions templates/test/ci/cluster-template-prow-ci-version.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4309,6 +4309,8 @@ data:
operator: Exists
- effect: NoSchedule
key: node-role.kubernetes.io/master
- effect: NoSchedule
key: node-role.kubernetes.io/control-plane
---
apiVersion: policy/v1beta1
kind: PodDisruptionBudget
Expand Down Expand Up @@ -4916,6 +4918,9 @@ data:
- effect: NoSchedule
key: node-role.kubernetes.io/master
operator: Exists
- effect: NoSchedule
key: node-role.kubernetes.io/control-plane
operator: Exists
volumes:
- emptyDir: {}
name: tmp-dir
Expand Down
2 changes: 2 additions & 0 deletions templates/test/ci/cluster-template-prow-custom-vnet.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -3989,6 +3989,8 @@ data:
operator: Exists
- effect: NoSchedule
key: node-role.kubernetes.io/master
- effect: NoSchedule
key: node-role.kubernetes.io/control-plane
---
apiVersion: policy/v1beta1
kind: PodDisruptionBudget
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4303,6 +4303,8 @@ data:
operator: Exists
- effect: NoSchedule
key: node-role.kubernetes.io/master
- effect: NoSchedule
key: node-role.kubernetes.io/control-plane
---
apiVersion: policy/v1beta1
kind: PodDisruptionBudget
Expand Down
2 changes: 2 additions & 0 deletions templates/test/ci/cluster-template-prow-ipv6.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4005,6 +4005,8 @@ data:
operator: Exists
- effect: NoSchedule
key: node-role.kubernetes.io/master
- effect: NoSchedule
key: node-role.kubernetes.io/control-plane
---
apiVersion: apps/v1
kind: Deployment
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4257,6 +4257,8 @@ data:
operator: Exists
- effect: NoSchedule
key: node-role.kubernetes.io/master
- effect: NoSchedule
key: node-role.kubernetes.io/control-plane
---
apiVersion: policy/v1beta1
kind: PodDisruptionBudget
Expand Down
2 changes: 2 additions & 0 deletions templates/test/ci/cluster-template-prow-machine-pool.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4070,6 +4070,8 @@ data:
operator: Exists
- effect: NoSchedule
key: node-role.kubernetes.io/master
- effect: NoSchedule
key: node-role.kubernetes.io/control-plane
---
apiVersion: policy/v1beta1
kind: PodDisruptionBudget
Expand Down
2 changes: 2 additions & 0 deletions templates/test/ci/cluster-template-prow-nvidia-gpu.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -7898,6 +7898,8 @@ data:
operator: Exists
- effect: NoSchedule
key: node-role.kubernetes.io/master
- effect: NoSchedule
key: node-role.kubernetes.io/control-plane
---
apiVersion: policy/v1beta1
kind: PodDisruptionBudget
Expand Down
2 changes: 2 additions & 0 deletions templates/test/ci/cluster-template-prow-private.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -3998,6 +3998,8 @@ data:
operator: Exists
- effect: NoSchedule
key: node-role.kubernetes.io/master
- effect: NoSchedule
key: node-role.kubernetes.io/control-plane
---
apiVersion: policy/v1beta1
kind: PodDisruptionBudget
Expand Down
2 changes: 2 additions & 0 deletions templates/test/ci/cluster-template-prow.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4109,6 +4109,8 @@ data:
operator: Exists
- effect: NoSchedule
key: node-role.kubernetes.io/master
- effect: NoSchedule
key: node-role.kubernetes.io/control-plane
---
apiVersion: policy/v1beta1
kind: PodDisruptionBudget
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4160,6 +4160,8 @@ data:
operator: Exists
- effect: NoSchedule
key: node-role.kubernetes.io/master
- effect: NoSchedule
key: node-role.kubernetes.io/control-plane
---
apiVersion: policy/v1beta1
kind: PodDisruptionBudget
Expand Down
2 changes: 2 additions & 0 deletions templates/test/dev/cluster-template-custom-builds.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4198,6 +4198,8 @@ data:
operator: Exists
- effect: NoSchedule
key: node-role.kubernetes.io/master
- effect: NoSchedule
key: node-role.kubernetes.io/control-plane
---
apiVersion: policy/v1beta1
kind: PodDisruptionBudget
Expand Down
18 changes: 9 additions & 9 deletions test/e2e/config/azure-dev.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,11 @@ managementClusterName: capz-e2e
images:
- name: ${MANAGER_IMAGE}
loadBehavior: mustLoad
- name: k8s.gcr.io/cluster-api/cluster-api-controller:v1.1.0
- name: k8s.gcr.io/cluster-api/cluster-api-controller:v1.1.2
loadBehavior: tryLoad
- name: k8s.gcr.io/cluster-api/kubeadm-bootstrap-controller:v1.1.0
- name: k8s.gcr.io/cluster-api/kubeadm-bootstrap-controller:v1.1.2
loadBehavior: tryLoad
- name: k8s.gcr.io/cluster-api/kubeadm-control-plane-controller:v1.1.0
- name: k8s.gcr.io/cluster-api/kubeadm-control-plane-controller:v1.1.2
loadBehavior: tryLoad

providers:
Expand All @@ -32,8 +32,8 @@ providers:
new: --metrics-addr=:8080
files:
- sourcePath: "../data/shared/v1alpha4/metadata.yaml"
- name: v1.1.0
value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.1.0/core-components.yaml
- name: v1.1.2
value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.1.2/core-components.yaml
type: url
contract: v1beta1
files:
Expand Down Expand Up @@ -64,8 +64,8 @@ providers:
new: --metrics-addr=:8080
files:
- sourcePath: "../data/shared/v1alpha4/metadata.yaml"
- name: v1.1.0
value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.1.0/bootstrap-components.yaml
- name: v1.1.2
value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.1.2/bootstrap-components.yaml
type: url
contract: v1beta1
files:
Expand Down Expand Up @@ -95,8 +95,8 @@ providers:
new: --metrics-addr=:8080
files:
- sourcePath: "../data/shared/v1alpha4/metadata.yaml"
- name: v1.1.0
value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.1.0/control-plane-components.yaml
- name: v1.1.2
value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.1.2/control-plane-components.yaml
type: url
contract: v1beta1
files:
Expand Down
15 changes: 13 additions & 2 deletions test/e2e/conformance_test.go
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
//go:build e2e
// +build e2e

/*
Expand Down Expand Up @@ -184,12 +185,22 @@ var _ = Describe("Conformance Tests", func() {
}

noScheduleTaint := &corev1.Taint{
CecileRobertMichon marked this conversation as resolved.
Show resolved Hide resolved
Key: "node-role.kubernetes.io/master",
Key: "node-role.kubernetes.io/control-plane",
Value: "",
Effect: "NoSchedule",
}

err := node.TaintNode(workloadProxy.GetClientSet(), options, noScheduleTaint)
if v, err := semver.ParseTolerant(kubernetesVersion); err == nil {
if v.LT(semver.MustParse("1.24.0-alpha.0.0")) {
noScheduleTaint = &corev1.Taint{
Key: "node-role.kubernetes.io/master",
Value: "",
Effect: "NoSchedule",
}
}
}

err = node.TaintNode(workloadProxy.GetClientSet(), options, noScheduleTaint)
Expect(err).NotTo(HaveOccurred())

// Windows requires a repo-list because some images are not in k8s gcr
Expand Down