From 6bd1bc103955fa4f4af8aca53e2afddd68a43a1c Mon Sep 17 00:00:00 2001 From: Ilia Lazebnik Date: Fri, 19 Jul 2024 11:34:48 -0400 Subject: [PATCH] feat: bump kubelet config to ga + minor doc fixes (#1994) Co-authored-by: Andrew Peabody --- README.md | 7 +++++ autogen/main/README.md | 6 ++-- autogen/main/cluster.tf.tmpl | 16 +++++------ cluster.tf | 26 +++++++++++++++++ .../README.md | 2 +- .../cluster.tf | 28 +++++++++---------- modules/beta-private-cluster/README.md | 2 +- modules/beta-private-cluster/cluster.tf | 28 +++++++++---------- .../README.md | 2 +- .../cluster.tf | 28 +++++++++---------- modules/beta-public-cluster/README.md | 2 +- modules/beta-public-cluster/cluster.tf | 28 +++++++++---------- .../private-cluster-update-variant/README.md | 6 ++++ .../private-cluster-update-variant/cluster.tf | 26 +++++++++++++++++ modules/private-cluster/README.md | 6 ++++ modules/private-cluster/cluster.tf | 26 +++++++++++++++++ 16 files changed, 167 insertions(+), 72 deletions(-) diff --git a/README.md b/README.md index 9ec5ef5031..aba792c6d4 100644 --- a/README.md +++ b/README.md @@ -298,6 +298,10 @@ The node_pools variable takes the following parameters: | autoscaling | Configuration required by cluster autoscaler to adjust the size of the node pool to the current cluster usage | true | Optional | | auto_upgrade | Whether the nodes will be automatically upgraded | true (if cluster is regional) | Optional | | boot_disk_kms_key | The Customer Managed Encryption Key used to encrypt the boot disk attached to each node in the node pool. This should be of the form projects/[KEY_PROJECT_ID]/locations/[LOCATION]/keyRings/[RING_NAME]/cryptoKeys/[KEY_NAME]. | " " | Optional | +| cpu_manager_policy | The CPU manager policy on the node. One of "none" or "static". | "static" | Optional | +| cpu_cfs_quota | Enforces the Pod's CPU limit. Setting this value to false means that the CPU limits for Pods are ignored | null | Optional | +| cpu_cfs_quota_period | The CPU CFS quota period value, which specifies the period of how often a cgroup's access to CPU resources should be reallocated | null | Optional | +| pod_pids_limit | Controls the maximum number of processes allowed to run in a pod. The value must be greater than or equal to 1024 and less than 4194304. | null | Optional | | disk_size_gb | Size of the disk attached to each node, specified in GB. The smallest allowed disk size is 10GB | 100 | Optional | | disk_type | Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd') | pd-standard | Optional | | effect | Effect for the taint | | Required | @@ -330,6 +334,9 @@ The node_pools variable takes the following parameters: | min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count. Should be used when autoscaling is true. Cannot be used with total limits. | 1 | Optional | | total_min_count | Total minimum number of nodes in the NodePool. Must be >=0 and <= max_count. Should be used when autoscaling is true. Cannot be used with per zone limits. | null | Optional | | name | The name of the node pool | | Required | +| placement_policy | Placement type to set for nodes in a node pool. Can be set as [COMPACT](https://cloud.google.com/kubernetes-engine/docs/how-to/compact-placement#overview) if desired | | Optional | +| pod_range | The name of the secondary range for pod IPs. | | Optional | +| enable_private_nodes | Whether nodes have internal IP addresses only. | | Optional | | node_count | The number of nodes in the nodepool when autoscaling is false. Otherwise defaults to 1. Only valid for non-autoscaling clusters | | Required | | node_locations | The list of zones in which the cluster's nodes are located. Nodes must be in the region of their regional cluster or in the same region as their cluster's zone for zonal clusters. Defaults to cluster level node locations if nothing is specified | " " | Optional | | node_metadata | Options to expose the node metadata to the workload running on the node | | Optional | diff --git a/autogen/main/README.md b/autogen/main/README.md index 4206111f2f..7a3960f1a1 100644 --- a/autogen/main/README.md +++ b/autogen/main/README.md @@ -197,11 +197,11 @@ The node_pools variable takes the following parameters: | autoscaling | Configuration required by cluster autoscaler to adjust the size of the node pool to the current cluster usage | true | Optional | | auto_upgrade | Whether the nodes will be automatically upgraded | true (if cluster is regional) | Optional | | boot_disk_kms_key | The Customer Managed Encryption Key used to encrypt the boot disk attached to each node in the node pool. This should be of the form projects/[KEY_PROJECT_ID]/locations/[LOCATION]/keyRings/[RING_NAME]/cryptoKeys/[KEY_NAME]. | " " | Optional | -{% if beta_cluster %} | cpu_manager_policy | The CPU manager policy on the node. One of "none" or "static". | "static" | Optional | | cpu_cfs_quota | Enforces the Pod's CPU limit. Setting this value to false means that the CPU limits for Pods are ignored | null | Optional | | cpu_cfs_quota_period | The CPU CFS quota period value, which specifies the period of how often a cgroup's access to CPU resources should be reallocated | null | Optional | | pod_pids_limit | Controls the maximum number of processes allowed to run in a pod. The value must be greater than or equal to 1024 and less than 4194304. | null | Optional | +{% if beta_cluster %} | enable\_confidential\_nodes | An optional flag to enable confidential node config. | `bool` | `false` | no | {% endif %} | disk_size_gb | Size of the disk attached to each node, specified in GB. The smallest allowed disk size is 10GB | 100 | Optional | @@ -239,13 +239,11 @@ The node_pools variable takes the following parameters: | min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count. Should be used when autoscaling is true. Cannot be used with total limits. | 1 | Optional | | total_min_count | Total minimum number of nodes in the NodePool. Must be >=0 and <= max_count. Should be used when autoscaling is true. Cannot be used with per zone limits. | null | Optional | | name | The name of the node pool | | Required | -{% if beta_cluster %} -| placement_policy | Placement type to set for nodes in a node pool. Can be set as [COMPACT](https://cloud.google.com/kubernetes-engine/docs/how-to/compact-placement#overview) if desired | Optional | +| placement_policy | Placement type to set for nodes in a node pool. Can be set as [COMPACT](https://cloud.google.com/kubernetes-engine/docs/how-to/compact-placement#overview) if desired | | Optional | | pod_range | The name of the secondary range for pod IPs. | | Optional | {% if not private_cluster %} | enable_private_nodes | Whether nodes have internal IP addresses only. | | Optional | {% endif %} -{% endif %} | node_count | The number of nodes in the nodepool when autoscaling is false. Otherwise defaults to 1. Only valid for non-autoscaling clusters | | Required | | node_locations | The list of zones in which the cluster's nodes are located. Nodes must be in the region of their regional cluster or in the same region as their cluster's zone for zonal clusters. Defaults to cluster level node locations if nothing is specified | " " | Optional | | node_metadata | Options to expose the node metadata to the workload running on the node | | Optional | diff --git a/autogen/main/cluster.tf.tmpl b/autogen/main/cluster.tf.tmpl index 927f5012dc..88780e48c3 100644 --- a/autogen/main/cluster.tf.tmpl +++ b/autogen/main/cluster.tf.tmpl @@ -1000,14 +1000,6 @@ resource "google_container_node_pool" "windows_pools" { } } - {% if beta_cluster %} - dynamic "sandbox_config" { - for_each = tobool((lookup(each.value, "sandbox_enabled", var.sandbox_enabled))) ? ["gvisor"] : [] - content { - sandbox_type = sandbox_config.value - } - } - dynamic "kubelet_config" { for_each = length(setintersection( keys(each.value), @@ -1021,6 +1013,14 @@ resource "google_container_node_pool" "windows_pools" { pod_pids_limit = lookup(each.value, "pod_pids_limit", null) } } + {% if beta_cluster %} + + dynamic "sandbox_config" { + for_each = tobool((lookup(each.value, "sandbox_enabled", var.sandbox_enabled))) ? ["gvisor"] : [] + content { + sandbox_type = sandbox_config.value + } + } {% endif %} {% if i == 0 %} diff --git a/cluster.tf b/cluster.tf index 07d93f26fd..ec7f020cc4 100644 --- a/cluster.tf +++ b/cluster.tf @@ -709,6 +709,19 @@ resource "google_container_node_pool" "pools" { } } + dynamic "kubelet_config" { + for_each = length(setintersection( + keys(each.value), + ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "pod_pids_limit"] + )) != 0 ? [1] : [] + + content { + cpu_manager_policy = lookup(each.value, "cpu_manager_policy", "static") + cpu_cfs_quota = lookup(each.value, "cpu_cfs_quota", null) + cpu_cfs_quota_period = lookup(each.value, "cpu_cfs_quota_period", null) + pod_pids_limit = lookup(each.value, "pod_pids_limit", null) + } + } dynamic "linux_node_config" { for_each = length(merge( @@ -968,6 +981,19 @@ resource "google_container_node_pool" "windows_pools" { } } + dynamic "kubelet_config" { + for_each = length(setintersection( + keys(each.value), + ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "pod_pids_limit"] + )) != 0 ? [1] : [] + + content { + cpu_manager_policy = lookup(each.value, "cpu_manager_policy", "static") + cpu_cfs_quota = lookup(each.value, "cpu_cfs_quota", null) + cpu_cfs_quota_period = lookup(each.value, "cpu_cfs_quota_period", null) + pod_pids_limit = lookup(each.value, "pod_pids_limit", null) + } + } boot_disk_kms_key = lookup(each.value, "boot_disk_kms_key", "") diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md index 5c823a11f8..55578df95b 100644 --- a/modules/beta-private-cluster-update-variant/README.md +++ b/modules/beta-private-cluster-update-variant/README.md @@ -392,7 +392,7 @@ The node_pools variable takes the following parameters: | min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count. Should be used when autoscaling is true. Cannot be used with total limits. | 1 | Optional | | total_min_count | Total minimum number of nodes in the NodePool. Must be >=0 and <= max_count. Should be used when autoscaling is true. Cannot be used with per zone limits. | null | Optional | | name | The name of the node pool | | Required | -| placement_policy | Placement type to set for nodes in a node pool. Can be set as [COMPACT](https://cloud.google.com/kubernetes-engine/docs/how-to/compact-placement#overview) if desired | Optional | +| placement_policy | Placement type to set for nodes in a node pool. Can be set as [COMPACT](https://cloud.google.com/kubernetes-engine/docs/how-to/compact-placement#overview) if desired | | Optional | | pod_range | The name of the secondary range for pod IPs. | | Optional | | node_count | The number of nodes in the nodepool when autoscaling is false. Otherwise defaults to 1. Only valid for non-autoscaling clusters | | Required | | node_locations | The list of zones in which the cluster's nodes are located. Nodes must be in the region of their regional cluster or in the same region as their cluster's zone for zonal clusters. Defaults to cluster level node locations if nothing is specified | " " | Optional | diff --git a/modules/beta-private-cluster-update-variant/cluster.tf b/modules/beta-private-cluster-update-variant/cluster.tf index 2e79946dd3..fd3606e19f 100644 --- a/modules/beta-private-cluster-update-variant/cluster.tf +++ b/modules/beta-private-cluster-update-variant/cluster.tf @@ -876,13 +876,6 @@ resource "google_container_node_pool" "pools" { } } - dynamic "sandbox_config" { - for_each = tobool((lookup(each.value, "sandbox_enabled", var.sandbox_enabled))) ? ["gvisor"] : [] - content { - sandbox_type = sandbox_config.value - } - } - dynamic "kubelet_config" { for_each = length(setintersection( keys(each.value), @@ -897,6 +890,13 @@ resource "google_container_node_pool" "pools" { } } + dynamic "sandbox_config" { + for_each = tobool((lookup(each.value, "sandbox_enabled", var.sandbox_enabled))) ? ["gvisor"] : [] + content { + sandbox_type = sandbox_config.value + } + } + dynamic "linux_node_config" { for_each = length(merge( local.node_pools_linux_node_configs_sysctls["all"], @@ -1162,13 +1162,6 @@ resource "google_container_node_pool" "windows_pools" { } } - dynamic "sandbox_config" { - for_each = tobool((lookup(each.value, "sandbox_enabled", var.sandbox_enabled))) ? ["gvisor"] : [] - content { - sandbox_type = sandbox_config.value - } - } - dynamic "kubelet_config" { for_each = length(setintersection( keys(each.value), @@ -1183,6 +1176,13 @@ resource "google_container_node_pool" "windows_pools" { } } + dynamic "sandbox_config" { + for_each = tobool((lookup(each.value, "sandbox_enabled", var.sandbox_enabled))) ? ["gvisor"] : [] + content { + sandbox_type = sandbox_config.value + } + } + boot_disk_kms_key = lookup(each.value, "boot_disk_kms_key", "") diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md index b56788608b..509e399a97 100644 --- a/modules/beta-private-cluster/README.md +++ b/modules/beta-private-cluster/README.md @@ -370,7 +370,7 @@ The node_pools variable takes the following parameters: | min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count. Should be used when autoscaling is true. Cannot be used with total limits. | 1 | Optional | | total_min_count | Total minimum number of nodes in the NodePool. Must be >=0 and <= max_count. Should be used when autoscaling is true. Cannot be used with per zone limits. | null | Optional | | name | The name of the node pool | | Required | -| placement_policy | Placement type to set for nodes in a node pool. Can be set as [COMPACT](https://cloud.google.com/kubernetes-engine/docs/how-to/compact-placement#overview) if desired | Optional | +| placement_policy | Placement type to set for nodes in a node pool. Can be set as [COMPACT](https://cloud.google.com/kubernetes-engine/docs/how-to/compact-placement#overview) if desired | | Optional | | pod_range | The name of the secondary range for pod IPs. | | Optional | | node_count | The number of nodes in the nodepool when autoscaling is false. Otherwise defaults to 1. Only valid for non-autoscaling clusters | | Required | | node_locations | The list of zones in which the cluster's nodes are located. Nodes must be in the region of their regional cluster or in the same region as their cluster's zone for zonal clusters. Defaults to cluster level node locations if nothing is specified | " " | Optional | diff --git a/modules/beta-private-cluster/cluster.tf b/modules/beta-private-cluster/cluster.tf index 9e25b40547..d93c6e7b5c 100644 --- a/modules/beta-private-cluster/cluster.tf +++ b/modules/beta-private-cluster/cluster.tf @@ -797,13 +797,6 @@ resource "google_container_node_pool" "pools" { } } - dynamic "sandbox_config" { - for_each = tobool((lookup(each.value, "sandbox_enabled", var.sandbox_enabled))) ? ["gvisor"] : [] - content { - sandbox_type = sandbox_config.value - } - } - dynamic "kubelet_config" { for_each = length(setintersection( keys(each.value), @@ -818,6 +811,13 @@ resource "google_container_node_pool" "pools" { } } + dynamic "sandbox_config" { + for_each = tobool((lookup(each.value, "sandbox_enabled", var.sandbox_enabled))) ? ["gvisor"] : [] + content { + sandbox_type = sandbox_config.value + } + } + dynamic "linux_node_config" { for_each = length(merge( local.node_pools_linux_node_configs_sysctls["all"], @@ -1082,13 +1082,6 @@ resource "google_container_node_pool" "windows_pools" { } } - dynamic "sandbox_config" { - for_each = tobool((lookup(each.value, "sandbox_enabled", var.sandbox_enabled))) ? ["gvisor"] : [] - content { - sandbox_type = sandbox_config.value - } - } - dynamic "kubelet_config" { for_each = length(setintersection( keys(each.value), @@ -1103,6 +1096,13 @@ resource "google_container_node_pool" "windows_pools" { } } + dynamic "sandbox_config" { + for_each = tobool((lookup(each.value, "sandbox_enabled", var.sandbox_enabled))) ? ["gvisor"] : [] + content { + sandbox_type = sandbox_config.value + } + } + boot_disk_kms_key = lookup(each.value, "boot_disk_kms_key", "") diff --git a/modules/beta-public-cluster-update-variant/README.md b/modules/beta-public-cluster-update-variant/README.md index 666df46159..e3c7edb87e 100644 --- a/modules/beta-public-cluster-update-variant/README.md +++ b/modules/beta-public-cluster-update-variant/README.md @@ -379,7 +379,7 @@ The node_pools variable takes the following parameters: | min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count. Should be used when autoscaling is true. Cannot be used with total limits. | 1 | Optional | | total_min_count | Total minimum number of nodes in the NodePool. Must be >=0 and <= max_count. Should be used when autoscaling is true. Cannot be used with per zone limits. | null | Optional | | name | The name of the node pool | | Required | -| placement_policy | Placement type to set for nodes in a node pool. Can be set as [COMPACT](https://cloud.google.com/kubernetes-engine/docs/how-to/compact-placement#overview) if desired | Optional | +| placement_policy | Placement type to set for nodes in a node pool. Can be set as [COMPACT](https://cloud.google.com/kubernetes-engine/docs/how-to/compact-placement#overview) if desired | | Optional | | pod_range | The name of the secondary range for pod IPs. | | Optional | | enable_private_nodes | Whether nodes have internal IP addresses only. | | Optional | | node_count | The number of nodes in the nodepool when autoscaling is false. Otherwise defaults to 1. Only valid for non-autoscaling clusters | | Required | diff --git a/modules/beta-public-cluster-update-variant/cluster.tf b/modules/beta-public-cluster-update-variant/cluster.tf index c0db8debc1..9d48543804 100644 --- a/modules/beta-public-cluster-update-variant/cluster.tf +++ b/modules/beta-public-cluster-update-variant/cluster.tf @@ -857,13 +857,6 @@ resource "google_container_node_pool" "pools" { } } - dynamic "sandbox_config" { - for_each = tobool((lookup(each.value, "sandbox_enabled", var.sandbox_enabled))) ? ["gvisor"] : [] - content { - sandbox_type = sandbox_config.value - } - } - dynamic "kubelet_config" { for_each = length(setintersection( keys(each.value), @@ -878,6 +871,13 @@ resource "google_container_node_pool" "pools" { } } + dynamic "sandbox_config" { + for_each = tobool((lookup(each.value, "sandbox_enabled", var.sandbox_enabled))) ? ["gvisor"] : [] + content { + sandbox_type = sandbox_config.value + } + } + dynamic "linux_node_config" { for_each = length(merge( local.node_pools_linux_node_configs_sysctls["all"], @@ -1143,13 +1143,6 @@ resource "google_container_node_pool" "windows_pools" { } } - dynamic "sandbox_config" { - for_each = tobool((lookup(each.value, "sandbox_enabled", var.sandbox_enabled))) ? ["gvisor"] : [] - content { - sandbox_type = sandbox_config.value - } - } - dynamic "kubelet_config" { for_each = length(setintersection( keys(each.value), @@ -1164,6 +1157,13 @@ resource "google_container_node_pool" "windows_pools" { } } + dynamic "sandbox_config" { + for_each = tobool((lookup(each.value, "sandbox_enabled", var.sandbox_enabled))) ? ["gvisor"] : [] + content { + sandbox_type = sandbox_config.value + } + } + boot_disk_kms_key = lookup(each.value, "boot_disk_kms_key", "") diff --git a/modules/beta-public-cluster/README.md b/modules/beta-public-cluster/README.md index 7c23ef23de..be951c213d 100644 --- a/modules/beta-public-cluster/README.md +++ b/modules/beta-public-cluster/README.md @@ -357,7 +357,7 @@ The node_pools variable takes the following parameters: | min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count. Should be used when autoscaling is true. Cannot be used with total limits. | 1 | Optional | | total_min_count | Total minimum number of nodes in the NodePool. Must be >=0 and <= max_count. Should be used when autoscaling is true. Cannot be used with per zone limits. | null | Optional | | name | The name of the node pool | | Required | -| placement_policy | Placement type to set for nodes in a node pool. Can be set as [COMPACT](https://cloud.google.com/kubernetes-engine/docs/how-to/compact-placement#overview) if desired | Optional | +| placement_policy | Placement type to set for nodes in a node pool. Can be set as [COMPACT](https://cloud.google.com/kubernetes-engine/docs/how-to/compact-placement#overview) if desired | | Optional | | pod_range | The name of the secondary range for pod IPs. | | Optional | | enable_private_nodes | Whether nodes have internal IP addresses only. | | Optional | | node_count | The number of nodes in the nodepool when autoscaling is false. Otherwise defaults to 1. Only valid for non-autoscaling clusters | | Required | diff --git a/modules/beta-public-cluster/cluster.tf b/modules/beta-public-cluster/cluster.tf index 13d35fac17..d57ef15329 100644 --- a/modules/beta-public-cluster/cluster.tf +++ b/modules/beta-public-cluster/cluster.tf @@ -778,13 +778,6 @@ resource "google_container_node_pool" "pools" { } } - dynamic "sandbox_config" { - for_each = tobool((lookup(each.value, "sandbox_enabled", var.sandbox_enabled))) ? ["gvisor"] : [] - content { - sandbox_type = sandbox_config.value - } - } - dynamic "kubelet_config" { for_each = length(setintersection( keys(each.value), @@ -799,6 +792,13 @@ resource "google_container_node_pool" "pools" { } } + dynamic "sandbox_config" { + for_each = tobool((lookup(each.value, "sandbox_enabled", var.sandbox_enabled))) ? ["gvisor"] : [] + content { + sandbox_type = sandbox_config.value + } + } + dynamic "linux_node_config" { for_each = length(merge( local.node_pools_linux_node_configs_sysctls["all"], @@ -1063,13 +1063,6 @@ resource "google_container_node_pool" "windows_pools" { } } - dynamic "sandbox_config" { - for_each = tobool((lookup(each.value, "sandbox_enabled", var.sandbox_enabled))) ? ["gvisor"] : [] - content { - sandbox_type = sandbox_config.value - } - } - dynamic "kubelet_config" { for_each = length(setintersection( keys(each.value), @@ -1084,6 +1077,13 @@ resource "google_container_node_pool" "windows_pools" { } } + dynamic "sandbox_config" { + for_each = tobool((lookup(each.value, "sandbox_enabled", var.sandbox_enabled))) ? ["gvisor"] : [] + content { + sandbox_type = sandbox_config.value + } + } + boot_disk_kms_key = lookup(each.value, "boot_disk_kms_key", "") diff --git a/modules/private-cluster-update-variant/README.md b/modules/private-cluster-update-variant/README.md index cb64b72928..030e13b3c0 100644 --- a/modules/private-cluster-update-variant/README.md +++ b/modules/private-cluster-update-variant/README.md @@ -333,6 +333,10 @@ The node_pools variable takes the following parameters: | autoscaling | Configuration required by cluster autoscaler to adjust the size of the node pool to the current cluster usage | true | Optional | | auto_upgrade | Whether the nodes will be automatically upgraded | true (if cluster is regional) | Optional | | boot_disk_kms_key | The Customer Managed Encryption Key used to encrypt the boot disk attached to each node in the node pool. This should be of the form projects/[KEY_PROJECT_ID]/locations/[LOCATION]/keyRings/[RING_NAME]/cryptoKeys/[KEY_NAME]. | " " | Optional | +| cpu_manager_policy | The CPU manager policy on the node. One of "none" or "static". | "static" | Optional | +| cpu_cfs_quota | Enforces the Pod's CPU limit. Setting this value to false means that the CPU limits for Pods are ignored | null | Optional | +| cpu_cfs_quota_period | The CPU CFS quota period value, which specifies the period of how often a cgroup's access to CPU resources should be reallocated | null | Optional | +| pod_pids_limit | Controls the maximum number of processes allowed to run in a pod. The value must be greater than or equal to 1024 and less than 4194304. | null | Optional | | disk_size_gb | Size of the disk attached to each node, specified in GB. The smallest allowed disk size is 10GB | 100 | Optional | | disk_type | Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd') | pd-standard | Optional | | effect | Effect for the taint | | Required | @@ -365,6 +369,8 @@ The node_pools variable takes the following parameters: | min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count. Should be used when autoscaling is true. Cannot be used with total limits. | 1 | Optional | | total_min_count | Total minimum number of nodes in the NodePool. Must be >=0 and <= max_count. Should be used when autoscaling is true. Cannot be used with per zone limits. | null | Optional | | name | The name of the node pool | | Required | +| placement_policy | Placement type to set for nodes in a node pool. Can be set as [COMPACT](https://cloud.google.com/kubernetes-engine/docs/how-to/compact-placement#overview) if desired | | Optional | +| pod_range | The name of the secondary range for pod IPs. | | Optional | | node_count | The number of nodes in the nodepool when autoscaling is false. Otherwise defaults to 1. Only valid for non-autoscaling clusters | | Required | | node_locations | The list of zones in which the cluster's nodes are located. Nodes must be in the region of their regional cluster or in the same region as their cluster's zone for zonal clusters. Defaults to cluster level node locations if nothing is specified | " " | Optional | | node_metadata | Options to expose the node metadata to the workload running on the node | | Optional | diff --git a/modules/private-cluster-update-variant/cluster.tf b/modules/private-cluster-update-variant/cluster.tf index 4d0622639b..f980d096e5 100644 --- a/modules/private-cluster-update-variant/cluster.tf +++ b/modules/private-cluster-update-variant/cluster.tf @@ -807,6 +807,19 @@ resource "google_container_node_pool" "pools" { } } + dynamic "kubelet_config" { + for_each = length(setintersection( + keys(each.value), + ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "pod_pids_limit"] + )) != 0 ? [1] : [] + + content { + cpu_manager_policy = lookup(each.value, "cpu_manager_policy", "static") + cpu_cfs_quota = lookup(each.value, "cpu_cfs_quota", null) + cpu_cfs_quota_period = lookup(each.value, "cpu_cfs_quota_period", null) + pod_pids_limit = lookup(each.value, "pod_pids_limit", null) + } + } dynamic "linux_node_config" { for_each = length(merge( @@ -1067,6 +1080,19 @@ resource "google_container_node_pool" "windows_pools" { } } + dynamic "kubelet_config" { + for_each = length(setintersection( + keys(each.value), + ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "pod_pids_limit"] + )) != 0 ? [1] : [] + + content { + cpu_manager_policy = lookup(each.value, "cpu_manager_policy", "static") + cpu_cfs_quota = lookup(each.value, "cpu_cfs_quota", null) + cpu_cfs_quota_period = lookup(each.value, "cpu_cfs_quota_period", null) + pod_pids_limit = lookup(each.value, "pod_pids_limit", null) + } + } boot_disk_kms_key = lookup(each.value, "boot_disk_kms_key", "") diff --git a/modules/private-cluster/README.md b/modules/private-cluster/README.md index 419120b95f..c6ed02ec12 100644 --- a/modules/private-cluster/README.md +++ b/modules/private-cluster/README.md @@ -311,6 +311,10 @@ The node_pools variable takes the following parameters: | autoscaling | Configuration required by cluster autoscaler to adjust the size of the node pool to the current cluster usage | true | Optional | | auto_upgrade | Whether the nodes will be automatically upgraded | true (if cluster is regional) | Optional | | boot_disk_kms_key | The Customer Managed Encryption Key used to encrypt the boot disk attached to each node in the node pool. This should be of the form projects/[KEY_PROJECT_ID]/locations/[LOCATION]/keyRings/[RING_NAME]/cryptoKeys/[KEY_NAME]. | " " | Optional | +| cpu_manager_policy | The CPU manager policy on the node. One of "none" or "static". | "static" | Optional | +| cpu_cfs_quota | Enforces the Pod's CPU limit. Setting this value to false means that the CPU limits for Pods are ignored | null | Optional | +| cpu_cfs_quota_period | The CPU CFS quota period value, which specifies the period of how often a cgroup's access to CPU resources should be reallocated | null | Optional | +| pod_pids_limit | Controls the maximum number of processes allowed to run in a pod. The value must be greater than or equal to 1024 and less than 4194304. | null | Optional | | disk_size_gb | Size of the disk attached to each node, specified in GB. The smallest allowed disk size is 10GB | 100 | Optional | | disk_type | Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd') | pd-standard | Optional | | effect | Effect for the taint | | Required | @@ -343,6 +347,8 @@ The node_pools variable takes the following parameters: | min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count. Should be used when autoscaling is true. Cannot be used with total limits. | 1 | Optional | | total_min_count | Total minimum number of nodes in the NodePool. Must be >=0 and <= max_count. Should be used when autoscaling is true. Cannot be used with per zone limits. | null | Optional | | name | The name of the node pool | | Required | +| placement_policy | Placement type to set for nodes in a node pool. Can be set as [COMPACT](https://cloud.google.com/kubernetes-engine/docs/how-to/compact-placement#overview) if desired | | Optional | +| pod_range | The name of the secondary range for pod IPs. | | Optional | | node_count | The number of nodes in the nodepool when autoscaling is false. Otherwise defaults to 1. Only valid for non-autoscaling clusters | | Required | | node_locations | The list of zones in which the cluster's nodes are located. Nodes must be in the region of their regional cluster or in the same region as their cluster's zone for zonal clusters. Defaults to cluster level node locations if nothing is specified | " " | Optional | | node_metadata | Options to expose the node metadata to the workload running on the node | | Optional | diff --git a/modules/private-cluster/cluster.tf b/modules/private-cluster/cluster.tf index 4f3296f4db..f15e1b02ba 100644 --- a/modules/private-cluster/cluster.tf +++ b/modules/private-cluster/cluster.tf @@ -728,6 +728,19 @@ resource "google_container_node_pool" "pools" { } } + dynamic "kubelet_config" { + for_each = length(setintersection( + keys(each.value), + ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "pod_pids_limit"] + )) != 0 ? [1] : [] + + content { + cpu_manager_policy = lookup(each.value, "cpu_manager_policy", "static") + cpu_cfs_quota = lookup(each.value, "cpu_cfs_quota", null) + cpu_cfs_quota_period = lookup(each.value, "cpu_cfs_quota_period", null) + pod_pids_limit = lookup(each.value, "pod_pids_limit", null) + } + } dynamic "linux_node_config" { for_each = length(merge( @@ -987,6 +1000,19 @@ resource "google_container_node_pool" "windows_pools" { } } + dynamic "kubelet_config" { + for_each = length(setintersection( + keys(each.value), + ["cpu_manager_policy", "cpu_cfs_quota", "cpu_cfs_quota_period", "pod_pids_limit"] + )) != 0 ? [1] : [] + + content { + cpu_manager_policy = lookup(each.value, "cpu_manager_policy", "static") + cpu_cfs_quota = lookup(each.value, "cpu_cfs_quota", null) + cpu_cfs_quota_period = lookup(each.value, "cpu_cfs_quota_period", null) + pod_pids_limit = lookup(each.value, "pod_pids_limit", null) + } + } boot_disk_kms_key = lookup(each.value, "boot_disk_kms_key", "")