Change default kube-system DaemonSet tolerations

conditional.tf

@@ -8,9 +8,10 @@ locals {
     "manifests-networking/${name}" => templatefile(
       "${path.module}/resources/flannel/${name}",
       {
-        flannel_image     = var.container_images["flannel"]
-        flannel_cni_image = var.container_images["flannel_cni"]
-        pod_cidr          = var.pod_cidr
+        flannel_image         = var.container_images["flannel"]
+        flannel_cni_image     = var.container_images["flannel_cni"]
+        pod_cidr              = var.pod_cidr
+        daemonset_tolerations = var.daemonset_tolerations
       }
     )
     if var.networking == "flannel"
@@ -33,6 +34,7 @@ locals {
         network_ip_autodetection_method = var.network_ip_autodetection_method
         pod_cidr                        = var.pod_cidr
         enable_reporting                = var.enable_reporting
+        daemonset_tolerations           = var.daemonset_tolerations
       }
     )
     if var.networking == "calico"
@@ -45,9 +47,10 @@ locals {
     "manifests-networking/${name}" => templatefile(
       "${path.module}/resources/kube-router/${name}",
       {
-        kube_router_image = var.container_images["kube_router"]
-        flannel_cni_image = var.container_images["flannel_cni"]
-        network_mtu       = var.network_mtu
+        kube_router_image     = var.container_images["kube_router"]
+        flannel_cni_image     = var.container_images["flannel_cni"]
+        network_mtu           = var.network_mtu
+        daemonset_tolerations = var.daemonset_tolerations
       }
     )
     if var.networking == "kube-router"
@@ -77,3 +80,4 @@ resource "local_file" "kube-router-manifests" {
   filename = "${var.asset_dir}/${each.key}"
   content  = each.value
 }
+

manifests.tf

@@ -35,6 +35,7 @@ locals {
         cluster_dns_service_ip = cidrhost(var.service_cidr, 10)
         trusted_certs_dir      = var.trusted_certs_dir
         server                 = format("https://%s:%s", var.api_servers[0], var.external_apiserver_port)
+        daemonset_tolerations  = var.daemonset_tolerations
       }
     )
   }

resources/calico/daemonset.yaml

@@ -24,10 +24,14 @@ spec:
       priorityClassName: system-node-critical
       serviceAccountName: calico-node
       tolerations:
-        - effect: NoSchedule
-          operator: Exists
-        - effect: NoExecute
-          operator: Exists
+      - key: node-role.kubernetes.io/master
+        operator: Exists
+      - key: node.kubernetes.io/not-ready
+        operator: Exists
+      %{~ for key in daemonset_tolerations ~}
+      - key: ${key}
+        operator: Exists
+      %{~ endfor ~}
       initContainers:
         # Install Calico CNI binaries and CNI network config file on nodes
         - name: install-cni

resources/flannel/daemonset.yaml

@@ -24,10 +24,14 @@ spec:
       priorityClassName: system-node-critical
       serviceAccountName: flannel
       tolerations:
-        - effect: NoSchedule
-          operator: Exists
-        - effect: NoExecute
-          operator: Exists
+      - key: node-role.kubernetes.io/master
+        operator: Exists
+      - key: node.kubernetes.io/not-ready
+        operator: Exists
+      %{~ for key in daemonset_tolerations ~}
+      - key: ${key}
+        operator: Exists
+      %{~ endfor ~}
       containers:
         - name: flannel
           image: ${flannel_image}

resources/kube-router/daemonset.yaml

@@ -24,10 +24,14 @@ spec:
       priorityClassName: system-node-critical
       serviceAccountName: kube-router
       tolerations:
-        - effect: NoSchedule
-          operator: Exists
-        - effect: NoExecute
-          operator: Exists
+      - key: node-role.kubernetes.io/master
+        operator: Exists
+      - key: node.kubernetes.io/not-ready
+        operator: Exists
+      %{~ for key in daemonset_tolerations ~}
+      - key: ${key}
+        operator: Exists
+      %{~ endfor ~}
       containers:
         - name: kube-router
           image: ${kube_router_image}

resources/manifests/kube-proxy.yaml

@@ -27,10 +27,14 @@ spec:
       priorityClassName: system-node-critical
       serviceAccountName: kube-proxy
       tolerations:
-      - effect: NoSchedule
+      - key: node-role.kubernetes.io/master
         operator: Exists
-      - effect: NoExecute
+      - key: node.kubernetes.io/not-ready
         operator: Exists
+      %{~ for key in daemonset_tolerations ~}
+      - key: ${key}
+        operator: Exists
+      %{~ endfor ~}
       containers:
       - name: kube-proxy
         image: ${kube_proxy_image}

variables.tf

@@ -117,3 +117,9 @@ variable "cluster_domain_suffix" {
   default     = "cluster.local"
 }
 
+variable "daemonset_tolerations" {
+  type        = list(string)
+  description = "List of additional taint keys kube-system DaemonSets should tolerate (e.g. ['custom-role', 'gpu-role'])"
+  default     = []
+}
+