From 2de9e29901c3277abc6dc933c56776577f33f931 Mon Sep 17 00:00:00 2001 From: Fernando Ripoll Date: Thu, 5 Dec 2024 15:43:09 +0100 Subject: [PATCH] Node pool and AZ Tutorials (#2367) --- .../config/vocabularies/docs/accept.txt | 1 + .../cluster-management/_index.md | 8 +- .../cluster-management/multi-az/index.md | 318 ++++++++++++++++++ .../cluster-management/node-pools/index.md | 316 +++++++++++++++++ 4 files changed, 641 insertions(+), 2 deletions(-) create mode 100644 src/content/tutorials/fleet-management/cluster-management/multi-az/index.md create mode 100644 src/content/tutorials/fleet-management/cluster-management/node-pools/index.md diff --git a/.vale/styles/config/vocabularies/docs/accept.txt b/.vale/styles/config/vocabularies/docs/accept.txt index d1f5b86af5..eb26e22b31 100644 --- a/.vale/styles/config/vocabularies/docs/accept.txt +++ b/.vale/styles/config/vocabularies/docs/accept.txt @@ -34,6 +34,7 @@ Grafana gsctl Honeybadger HTTP +impl_title IP[s]? IPAM JSON diff --git a/src/content/tutorials/fleet-management/cluster-management/_index.md b/src/content/tutorials/fleet-management/cluster-management/_index.md index fb133d50c8..9d2522978d 100644 --- a/src/content/tutorials/fleet-management/cluster-management/_index.md +++ b/src/content/tutorials/fleet-management/cluster-management/_index.md @@ -5,9 +5,13 @@ menu: principal: parent: tutorials-fleet-management identifier: tutorials-fleet-management-clusters -last_review_date: 2024-05-02 +last_review_date: 2024-11-29 owner: - - https://github.com/orgs/giantswarm/teams/sig-docs + - https://github.com/orgs/giantswarm/teams/team-phoenix + - https://github.com/orgs/giantswarm/teams/team-rocket user_questions: - How does Giant Swarm manage clusters across different regions and cloud providers? + - How the node pools are managed in Giant Swarm? + - How do I migrate to Cluster API on Giant Swarm? + - How does AWS CNI ENI mode work on Giant Swarm? --- diff --git a/src/content/tutorials/fleet-management/cluster-management/multi-az/index.md b/src/content/tutorials/fleet-management/cluster-management/multi-az/index.md new file mode 100644 index 0000000000..80754158b1 --- /dev/null +++ b/src/content/tutorials/fleet-management/cluster-management/multi-az/index.md @@ -0,0 +1,318 @@ +--- +linkTitle: Multiple AZ +title: Clusters over multiple availability zones +description: Using multiple availability zones both for worker and control plane nodes increases the resilience of the cluster. Here you will see some details regarding support on different cloud providers and releases, plus how to configure workloads to leverage multiple availability zones. +weight: 20 +menu: + principal: + parent: tutorials-fleet-management-clusters + identifier: tutorials-fleet-management-clusters-multi-az +last_review_date: 2024-11-29 +user_questions: +- Does Giant Swarm support multiple availability zones (AZ)? +- What are the benefits of using multiple availability zones (AZ)? +- How do I know which availability zone (AZ) my cluster nodes are running in? +- Can I influence the scheduling of my pods? +- How are availability zones (AZ) selected? +- Can I move standard volumes across availability zones (AZ)? +- What is the alternative to moving volumes across availability zones (AZ)? +- How do I ensure my pods and volumes are on the same nodes? +- Can I spread worker nodes over availability zones? +- How do I create clusters in multiple availability zones (AZ)? +- How do I make sure my workload is distributed evenly over availability zones (AZ)? +owner: + - https://github.com/orgs/giantswarm/teams/team-phoenix + - https://github.com/orgs/giantswarm/teams/team-rocket +--- + +In Giant Swarm platform you can easily launch clusters with worker nodes spread across multiple availability zones (`AZ`). This will lower the risk that your cluster will become unavailable due to an incident in a particular `AWS`, `Azure` or on-premise data center. + +## What availability zones are good for {#benefits} + +Cloud providers, and on-premise installations, supports availability zones within their regions. These zones still have good inter-connectivity but are separated from each other to isolate failures within one zone from the others. As a result, it's more likely that one zone goes down than the whole region. If your clusters are running within multiple availability zones then `Kubernetes` can easily shift your workloads from one zone to another. + +Your cluster nodes have labels that indicate which availability zone they're running in. You can influence the scheduling of your pods via node affinity and/or inter-pod affinity or anti-affinity. + +- [Affinity and anti-affinity](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity) +- [Pod Topology Spread Constraints](https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/) + +This enables use cases such as: + +- Running stateless services balanced over multiple availability zones to increase availability. + +- Grouping certain services that communicate with each other often and make sure they always run together within the same availability zone. + +- Setup replication of your data across multiple availability zones by keeping each instance of your StatefulSet in a different zone. + +## Details you should know about {#details} + +- On AWS and Azure availability zones are randomized across accounts. There is no way to determine which zone you are really in, based on the name. For example, the zone `eu-central-1a` in the account of the cluster isn't necessarily the same `eu-central-1a` in your account. + +- Availability zones get selected randomly by the Giant Swarm management cluster. You only need to specify the required number of availability zones. + +- Nodes will get distributed evenly across availability zones. There is currently no way to determine which or how many nodes should be started in a particular availability zone. But the nodes will have a label `topology.kubernetes.io/zone` (or in Kubernetes before 1.17: `failure-domain.beta.kubernetes.io/zone`) that indicates which availability zone the node is running in. + +- Single availability zone clusters start in a random availability zone too. This is a means to minimize the risk of all your clusters becoming unavailable due to a failure in one particular zone. + +- Standard volumes can not be moved across availability zones. You need to take this into account when designing for high availability. If the zone with your volume goes down, there will be no way to reschedule the pod to another availability zone. You either need to create a new volume from a snapshot or you will have to replicate your data across zones. On AWS, you can also consider using [EFS]({{< relref "/vintage/advanced/storage/efs" >}}) as a storage provider to be able to access volumes from several availability zones. + +- To make sure your pods and volumes end up on the same nodes, our recommendation is to specify `WaitForFirstConsumer` as `volumeBindingMode` in your storage classes. Your clusters come with a default storage class that contains this setting already. See the [Volume Binding Mode](https://kubernetes.io/docs/concepts/storage/storage-classes/#volume-binding-mode) section in the Kubernetes storage documentation for more information. + +Spreading worker nodes over multiple availability zones can be configured per [node pool]({{< relref "/vintage/advanced/cluster-management/node-pools-vintage" >}}) and independent of the choice of a single control plane node vs. using multiple control plane nodes (currently multiple control plane nodes are only supported on AWS). + +## Example pod topology spread constraints and affinity + +To make sure your workload gets scheduled over available worker nodes over availability zones you can make use of [`Pod Topology Spread Constraints`](https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/) and [affinity and anti-affinity](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity). + +In this example, the two constraints make sure, pods with the given labels are distributed across available zones. The first constraint makes sure the number of pods across nodes with the same `topology.kubernetes.io/zone` label is one at max. The second constraint picks the Node with the lowest number of Pods still across all nodes and all zones. This might collide with the first constraint and no schedule would happen at all unless `whenUnsatisfiable` is set to `ScheduleAnyway`. The `affinity` makes sure to not schedule any Pods on `master` Nodes. + +The following fields are part of the `spec.template.spec` of `Deployments`, `StatefulSets`, and `DaemonSets`: + +```yaml +topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app.kubernetes.io/name: hello-world-app + app.kubernetes.io/version: 0.1.0 + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + app.kubernetes.io/name: hello-world-app + app.kubernetes.io/version: 0.1.0 +affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/role + operator: NotIn + values: + - master +``` + +These constraints are checked during scheduling of new pods. In case some dis-balancing happened because of other reasons, [`Descheduler` for `Kubernetes`](https://github.com/kubernetes-sigs/descheduler) can be used to re-schedule pods to be more balanced. + +Please be aware of some undesirable edge-cases and caveats. These might show up on more complex `topologySpreadConstraints` and `affinity` setups. Discussions in issues [kubernetes/kubernetes#106127](https://github.com/kubernetes/kubernetes/issues/106127), [kubernetes/kubernetes#105977](https://github.com/kubernetes/kubernetes/issues/105977) and [kubernetes/kubernetes#105291](https://github.com/kubernetes/kubernetes/issues/105291) touch some of the problems that could arise. + +## Get started + +You can read [`kubectl gs template cluster`]({{< relref "/reference/kubectl-gs/template-cluster" >}}) to learn how to create a cluster with multiple availability zones. + +When inspecting details of such a cluster, you can get the node pool information using the node pool infrastructure resource depending on the provider. + +{{< tabs >}} +{{< tab id="cluster-capa-ec2" for-impl="capa_ec2">}} + +The node pools in Cluster API for AWS are managed by the `AWSMachinePool` resource. When you create a cluster with `kubectl gs template cluster` you rely on `Helm` [`cluster-aws`](https://github.com/giantswarm/cluster-aws) chart. To extend or modify the availability zones for the node pools you can provide a `values.yaml` file as shown below. + +Define the node pool with the availability zones in the `values.yaml` file. + +```sh +cat << EOF > /tmp/values.yaml +data: + values: | + global: + nodePools: + np001: + instanceType: m6i.xlarge + maxSize: 3 + minSize: 1 + availabilityZones: + - eu-west-1a + - eu-west-1b +EOF +``` + +Template the cluster with `kubectl gs template cluster`: + +```sh +kubectl gs template cluster \ + --provider capa \ + --name mycluster \ + --organization testing \ + --release 29.0.1 > /tmp/cluster.yaml +``` + +Append the node pool values configuration manually or using this `yq` command: + +```sh +yq eval-all ' + select(fileIndex == 0) | + ( + select(.kind == "ConfigMap") | + .data.values |= ( + (. | from_yaml) * (load("/tmp/values.yaml").data.values | from_yaml) | to_yaml + ) + ), + select(fileIndex == 0 and .kind == "App") +' /tmp/cluster.yaml > /tmp/merged.yaml +``` + +Now you can apply the merged configuration: + +```sh +kubectl apply -f /tmp/merged.yaml +``` + +To get the node pool information, query the `AWSMachinePool` resource as shown below. + +```text +$ kubectl get AWSManagedMachinePool -l cluster.x-k8s.io/cluster-name=mycluster -oyaml +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: AWSManagedMachinePool +metadata: + name: mycluster-np001 + namespace: org-testing +spec: + availabilityZones: + - eu-west-1a + ... +``` + +{{< /tab >}} +{{< tab id="cluster-capa-eks" for-impl="capa_eks">}} + +The node pools in Cluster API for EKS are managed by the `AWSManagedMachinePool` resource. When you create a cluster with `kubectl gs template cluster` you rely on `Helm` [`cluster-eks`](https://github.com/giantswarm/cluster-eks) chart. To extend or modify the availability zones for the node pools you can provide a `values.yaml` file as shown below. + +Define the node pool with the availability zones in the `values.yaml` file. + +```sh +cat << EOF > /tmp/values.yaml +data: + values: | + global: + nodePools: + np001: + name: "np001" + instanceType: "m5.large" + minSize: 1 + maxSize: 3 + availabilityZones: ["us-west-2a", "us-west-2b", "us-west-2c"] +EOF +``` + +Template the cluster with `kubectl gs template cluster`: + +```sh +kubectl gs template cluster \ + --provider eks \ + --name mycluster \ + --organization testing \ + --release 29.0.1 > /tmp/cluster.yaml +``` + +Append the node pool values configuration manually or using this `yq` command: + +```sh +yq eval-all ' + select(fileIndex == 0) | + ( + select(.metadata.name == "mycluster-userconfig") | + .data.values |= ( + (. | from_yaml) * (load("/tmp/values.yaml").data.values | from_yaml) | to_yaml + ) + ), + select(fileIndex == 0 and .metadata.name == "mycluster"), + select(fileIndex == 0 and .metadata.name == "mycluster-default-apps-userconfig"), + select(fileIndex == 0 and .metadata.name == "mycluster-default-apps") +' /tmp/cluster.yaml > /tmp/merged.yaml +``` + +Now you can apply the merged configuration: + +```sh +kubectl apply -f /tmp/merged.yaml +``` + +To get the node pool information, query the `AWSManagedMachinePool` resource as shown below. + +```text +$ kubectl get AWSManagedMachinePool -l cluster.x-k8s.io/cluster-name=mycluster -oyaml +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: AWSManagedMachinePool +metadata: + name: mycluster-np001 + namespace: org-testing +spec: + availabilityZones: + - us-west-2a + - us-west-2b + ... +``` + +{{< /tab >}} +{{< tab id="cluster-capz-azure-vms" for-impl="capz_vms">}} + +The node pools in Cluster API for Azure are managed by the `MachineDeployment` resource. When you create a cluster with `kubectl gs template cluster` you can define the availability zones for the worker nodes as shown below. + +Define the failure domain for the node pool in the `values.yaml` file. + +```sh +cat << EOF > /tmp/cluster.yaml +data: + values: | + global: + nodePools: + np001: + instanceType: Standard_D4s_v5 + replicas: 4 + failureDomain: "2" +EOF +``` + +Template the cluster with `kubectl gs template cluster`: + +```text +$ kubectl gs template cluster \ + --provider capz \ + --name mycluster \ + --organization testing \ + --region westeurope \ + --azure-subscription-id xxxxxxxxx \ + --release 29.0.1 > /tmp/cluster.yaml +``` + +Append the node pool values configuration manually or using this `yq` command: + +```sh +yq eval-all ' + select(fileIndex == 0) | + ( + select(.metadata.name == "mycluster-userconfig") | + .data.values |= ( + (. | from_yaml) * (load("/tmp/values.yaml").data.values | from_yaml) | to_yaml + ) + ), + select(fileIndex == 0 and .metadata.name == "mycluster") +' /tmp/cluster.yaml > /tmp/merged.yaml +``` + +Now you can apply the merged configuration: + +```sh +kubectl apply -f /tmp/merged.yaml +``` + +Get the node pool information by querying the `MachineDeployment` resource as shown below. + +```text +$ kubectl get MachineDeployment -l cluster.x-k8s.io/cluster-name=mycluster -oyaml +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: mycluster-np001 + namespace: org-testing +spec: + failureDomain: 2 + ... +``` + +{{< /tab >}} +{{< /tabs >}} + +Learn more about [node pools]({{< relref "/tutorials/fleet-management/cluster-management/node-pools" >}}) and how to manage them. diff --git a/src/content/tutorials/fleet-management/cluster-management/node-pools/index.md b/src/content/tutorials/fleet-management/cluster-management/node-pools/index.md new file mode 100644 index 0000000000..18b12aecd8 --- /dev/null +++ b/src/content/tutorials/fleet-management/cluster-management/node-pools/index.md @@ -0,0 +1,316 @@ +--- +linkTitle: Node pools +title: Node pools +description: A general description of node pools as a concept, its benefits, and some details you should be aware of. +weight: 45 +menu: + principal: + identifier: tutorials-fleet-management-clusters-node-pools + parent: tutorials-fleet-management-clusters +user_questions: + - What is a node pool? + - What are node pools? + - In which cloud environments are node pools supported? + - Which workload cluster releases introduced node pools? +owner: + - https://github.com/orgs/giantswarm/teams/team-phoenix +last_review_date: 2023-12-21 +--- + +A node pool is a set of nodes within a `Kubernetes` cluster that share the same configuration (machine type, operating system, etc.). Each node in the pool is labeled by the node pool's name. + +## Advantages + +Prior to the introduction of node pools, a cluster could only comprise one type of worker node. The cluster would have to be scaled as a whole, and the availability zone distribution would apply to all worker nodes of a cluster. This would mean that every worker node would have to be big enough to run the largest possible workload, in terms of memory and CPU resources required. At the same time, all worker nodes in the cluster would have to use the same availability zone distribution, even if some workloads wouldn't require the increased availability. + +Node pools are independent groups of worker nodes belonging to a cluster, where all nodes within a pool share a +common configuration. You can combine any type of node pool within one cluster. Node pools can differ regarding: + +- Machine type +- Availability zone distribution +- Scaling configuration (number of nodes) +- Node labels (the pool name is added as node label by default) + +## Configuration + +Node pools can be created, deleted or updated by changing the configuration used when creating the cluster using [`kubectl-gs`]({{< relref "/getting-started/provision-your-first-workload-cluster" >}}) + +{{< tabs >}} +{{< tab id="nodepool-capa-config" for-impl="capa_ec2" >}} + +```sh +kubectl gs template cluster --provider capa --name mycluster \ + --organization giantswarm \ + --description "my test cluster" \ + --machine-pool-name pool0 \ + --machine-pool-min-size 3 \ + --machine-pool-max-size 5 \ + --machine-pool-instance-type r6i.xlarge +``` + +A node pool is identified by a name that you can pick as a cluster administrator. The name must follow these rules: + +- must be between 5 and 20 characters long +- must start with a lowercase letter or number +- must end with a lowercase letter or number +- must contain only lowercase letters, numbers, and dashes + +For example, `pool0`, `group-1` are valid node pool names. + +The node pool name will be a suffix of the cluster name. In the example above, the node pool name will be `mycluster-pool0`. + +All nodes in the node pool will be labeled with the node pool name, using the `giantswarm.io/machine-pool` label. You can identify the nodes' node pool using that label. + +The example `kubectl` command below will list all nodes with role, node pool name, and node name. + +```text +kubectl get nodes \ + -o=jsonpath='{range .items[*]}{.metadata.labels.kubernetes\.io/role}{"\t"}{.metadata.labels.giantswarm\.io/machine-pool}{"\t"}{.metadata.name}{"\n"}{end}' | sort +master ip-10-1-5-55.eu-central-1.compute.internal +worker mycluster-pool0 ip-10-1-6-225.eu-central-1.compute.internal +worker mycluster-pool0 ip-10-1-6-67.eu-central-1.compute.internal +``` + +{{< /tab >}} +{{< tab id="nodepool-capz-config" for-impl="capz_vms" >}} + +```sh +kubectl gs template cluster --provider capz --name test-cluster \ + --organization giantswarm \ + --description "my test cluster" \ + --machine-pool-name pool0 \ + --machine-pool-min-size 3 \ + --machine-pool-max-size 5 \ + --machine-pool-instance-type Standard_D4s_v5 +``` + +The node pool name will be a suffix of the cluster name. In the example above, the node pool name will be `mycluster-pool0`. + +All nodes in the node pool will be labeled with the node pool name, using the `giantswarm.io/machine-deployment` label. You can identify the nodes' node pool using that label. + +The example `kubectl` command below will list all nodes with role, node pool name, and node name. + +```text +kubectl get nodes \ + -o=jsonpath='{range .items[*]}{.metadata.labels.kubernetes\.io/role}{"\t"}{.metadata.labels.giantswarm\.io/machine-deployment}{"\t"}{.metadata.name}{"\n"}{end}' | sort +master mycluster-control-plane-34c45e6e-rrpnn +worker mycluster-pool0 mycluster-pool0-8268227a-56x9n +worker mycluster-pool0 mycluster-pool0-8268227a-hjf69 +``` + +{{< /tab >}} +{{< /tabs >}} + +## Assigning workloads to node pools {#assigning-workloads} + +Knowing the node pool name of the pool to use, you can use the [`nodeSelector` method of assigning pods](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector) to the node pool. + +Assuming that the node pool name is `pool0`, and the cluster name is `mycluster`, your `nodeSelector` could for example look like this: + +{{< tabs >}} +{{< tab id="nodepool-capa-scheduling" for-impl="capa_ec2" >}} + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: nginx +spec: + containers: + - name: nginx + image: nginx + nodeSelector: + giantswarm.io/machine-pool: mycluster-pool0 +``` + +{{< /tab >}} +{{< tab id="nodepool-capz-scheduling" for-impl="capz_vms" >}} + +```yaml +A similar example for an Azure cluster: + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: nginx +spec: + containers: + - name: nginx + image: nginx + nodeSelector: + giantswarm.io/machine-deployment: mycluster-pool0 +``` + +{{< /tab >}} +{{< /tabs >}} + +You can assign workloads to node pools in a more indirect way too. +[There is a set of labels](https://kubernetes.io/docs/reference/labels-annotations-taints/) that are automatically added by `Kubernetes` to all nodes, that you can use for scheduling. + +For example: you have several node pools with different instance types. Using a `nodeSelector` with the label `node.kubernetes.io/instance-type`, you can assign workloads only to matching nodes. + +Another example: you have different node pools using different availability zones. With a `nodeSelector` using the label `topology.kubernetes.io/zone`, you can assign your workload to the nodes in a particular availability zone. + +## Adding more node pools + +You can add new node pools at any time. You just need to update the cluster configuration and specify the details of the node pools that you want to add. + +For example, if this was the cluster configuration + +```yaml +metadata: + description: "my cluster" + name: test-cluster + organization: giantswarm +nodePools: + nodepool0: + maxSize: 4 + minSize: 3 + instanceTypeOverrides: + - r6i.xlarge + - r5.xlarge + - m5.xlarge +``` + +You can add a new node pool like this: + +```yaml +metadata: + description: "my cluster" + name: test-cluster + organization: giantswarm +nodePools: + nodepool0: + maxSize: 4 + minSize: 3 + instanceTypeOverrides: + - r6i.xlarge + - r5.xlarge + - m5.xlarge + nodepool1: + instanceType: m5.xlarge + maxSize: 3 + minSize: 1 +``` + +## Updating an existing node pool + +Instances in the node pool will be rolled whenever these properties are changed in the node pool definition: + +- `instanceType` +- `additionalSecurityGroups` +- `customNodeLabels` + +Instances will also be rolled if these values are changed: + +- `providerSpecific.ami` + +**Warning:** Please be aware that changing the name of a node pool will result in the deletion of the old node pool and the creation of a new one. + +If you still want to change the name of a node pool, our recommendation is to add a new node pool with the new name, waiting for it to be healthy, and then removing the old one. + +### What happens when a node pool is updated {#what-happens-when-rolling-nodes} + +{{< tabs >}} +{{< tab id="nodepool-update-general" title="General" >}} + +On cluster update, nodes get replaced if their configuration changed (called "rolling nodes"). This means that pods running on the old nodes will be stopped and moved to new nodes automatically. + +Nodes may also get replaced involuntarily, for example if the node becomes unhealthy (for example disk full, out of memory), the cloud provider has a hardware fault, or you are using AWS spot instances that can shut down at any time. Therefore, please make sure that your applications can handle pod restarts. This topic is too large to cover here. Our advise is to research "zero-downtime deployments" and "stateless applications" since with those best practices, typical applications survive pod restarts without any problems. + +{{< /tab >}} +{{< tab id="nodepool-update-capa" for-impl="capa_ec2" >}} + +During a cluster upgrade, it can be necessary to create new EC2 instances (called "rolling nodes"). That only happens if anything changed in the node configuration, such as configuration files or the AMI image (newer version of Kubernetes or Flatcar Linux). For such planned node replacements, our product relies on [instance warmup settings](https://github.com/search?q=repo%3Agiantswarm%2Fcluster-aws%20refreshPreferences&type=code) to ensure that AWS doesn't replace the old nodes too quickly all at once, but rather in steps so a human could still intervene if something goes wrong (for example roll back to previous version). For a small node pool, that would mean that one node would be replaced every 5 minutes. For bigger node pools, small groups of nodes would be replaced every 5 minutes. + +When node pool instances need to be rolled, each instance receives a terminate signal from AWS. +This is propagated as shutdown signal to the OS and then to each running process like the `kubelet`, which will send a `NodeShutDown` event to the pod. + +The `kubelet` process will wait up to 5 minutes for all pods to terminate and after that, it will terminate itself and the shutdown of the AWS EC2 instance will finally proceed. AWS may decide to force-terminate the instance before the 5 minutes. + +{{< /tab >}} +{{< /tabs >}} + +## Node pool deletion + +In a similar way, you can remove a node pool at any time by removing its configuration from the values and updating the cluster. When a node pool is deleted, all instances in the node pool will be terminated, and a similar process [as described above]({{< relref "#what-happens-when-rolling-nodes" >}}) will take place. + +## Using mixed instance types (only {{% impl_title "capa_ec2" %}}) {#mixed-instance-types} + +On {{% impl_title "capa_ec2" %}}, you can override the instance type of the node pool to enable [mixed instances policy on AWS](https://docs.aws.amazon.com/autoscaling/ec2/APIReference/API_LaunchTemplateOverrides.html). + +This can provide `Amazon EC2 Auto Scaling` with a larger selection of instance types to choose from when fulfilling node capacities. + +You can set the `instanceTypeOverrides` value when defining your node pool in the cluster values. For example: + +```yaml +metadata: + description: "my cluster" + name: test-cluster + organization: giantswarm +nodePools: + nodepool0: + maxSize: 4 + minSize: 3 + instanceTypeOverrides: + - r6i.xlarge + - r5.xlarge + - m5.xlarge +``` + +Please notice that when setting `instanceTypeOverrides`, the `instanceType` value will be ignored, and the instance types defined in `instanceTypeOverrides` will be used instead. + +Also, the order in which you define the instance types in `instanceTypeOverrides` is important. +The first instance type in the list that's available in the selected availability zone will be used. + +**Note:** Changing the `instanceTypeOverrides` value won't trigger a rolling update of the node pool. + +Using multiple instance types in a node pool has some benefits: + +- Together with spot instances, using multiple instance types allows better price optimization. Popular instance types tend to have more price adjustments. Allowing older-generation instance types that are less popular tends to result in lower costs and fewer interruptions. + +- Even without spot instances, AWS has a limited number of instances per type in each Availability Zone. It can happen that your selected instance type is temporarily out of stock just in the moment you are in demand of more worker nodes. Allowing the node pool to use multiple instance types reduces this risk and increases the likelihood that your node pool can grow when in need. + +## Node pools and autoscaling {#autoscaling} + +With node pools, you set the autoscaling range per node pool (supported on {{% impl_title "capa_ec2" %}} clusters only). The `Kubernetes` cluster autoscaler has to decide which node pool to scale under which circumstances. + +If you assign workloads to node pools as described [above](#assigning-workloads) and the autoscaler finds pods in `Pending` state, it will decide based on the node selectors which node pools to scale up. + +In case there are workloads not assigned to any node pools, the autoscaler may pick any node pool for scaling. For details on the decision logic, please check the upstream FAQ for [AWS](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md). + +## On-demand and spot instances {#on-demand-spot} + +{{< tabs >}} +{{< tab id="nodepool-capa-spot-instances" for-impl="capa_ec2" >}} + +Node pools can make use of [Amazon EC2 Spot Instances](https://aws.amazon.com/ec2/spot/). On the node pool definition, you can enable it and select the maximum price to pay. + +```yaml +metadata: + description: "my cluster" + name: test-cluster + organization: giantswarm +nodePools: + pool0: + maxSize: 2 + minSize: 2 + spotInstances: + enabled: true + maxPrice: 1.2 +``` + +{{< /tab >}} +{{< tab id="nodepool-capz-spot-instances" for-impl="capz_vms" >}} + +This is currently not supported. + +{{< /tab >}} +{{< /tabs >}} + +## Limitations + +- Clusters without worker nodes (= without node pools) can't be considered fully functional. In order to have all required components scheduled, worker nodes are required. For that reason, it's deactivated any monitoring and alerts for these clusters and don't provide any proactive support. + +Learn more about [how to assign `Pods` to `Nodes`](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/) from the official documentation.