diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9af61e655b..0c542c8442 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.87.1 + rev: v1.88.0 hooks: - id: terraform_fmt - id: terraform_validate diff --git a/README.md b/README.md index 7c6c65a421..1b763ec7de 100644 --- a/README.md +++ b/README.md @@ -113,6 +113,60 @@ On clusters that were created prior to CAM support, there will be an existing ac Setting the `bootstrap_cluster_creator_admin_permissions` is a one time operation when the cluster is created; it cannot be modified later through the EKS API. In this project we are hardcoding this to `false`. If users wish to achieve the same functionality, we will do that through an access entry which can be enabled or disabled at any time of their choosing using the variable `enable_cluster_creator_admin_permissions` +### Enabling EFA Support + +When enabling EFA support via `enable_efa_support = true`, there are two locations this can be specified - one at the cluster level, and one at the nodegroup level. Enabling at the cluster level will add the EFA required ingress/egress rules to the shared security group created for the nodegroup(s). Enabling at the nodegroup level will do the following (per nodegroup where enabled): + +1. All EFA interfaces supported by the instance will be exposed on the launch template used by the nodegroup +2. A placement group with `strategy = "clustered"` per EFA requirements is created and passed to the launch template used by the nodegroup +3. Data sources will reverse lookup the availability zones that support the instance type selected based on the subnets provided, ensuring that only the associated subnets are passed to the launch template and therefore used by the placement group. This avoids the placement group being created in an availability zone that does not support the instance type selected. + +> [!TIP] +> Use the [aws-efa-k8s-device-plugin](https://github.com/aws/eks-charts/tree/master/stable/aws-efa-k8s-device-plugin) Helm chart to expose the EFA interfaces on the nodes as an extended resource, and allow pods to request the interfaces be mounted to their containers. +> +> The EKS AL2 GPU AMI comes with the necessary EFA components pre-installed - you just need to expose the EFA devices on the nodes via their launch templates, ensure the required EFA security group rules are in place, and deploy the `aws-efa-k8s-device-plugin` in order to start utilizing EFA within your cluster. Your application container will need to have the necessary libraries and runtime in order to utilize communication over the EFA interfaces (NCCL, aws-ofi-nccl, hwloc, libfabric, aws-neuornx-collectives, CUDA, etc.). + +If you disable the creation and use of the managed nodegroup custom launch template (`create_launch_template = false` and/or `use_custom_launch_template = false`), this will interfere with the EFA functionality provided. In addition, if you do not supply an `instance_type` for self-managed nodegroup(s), or `instance_types` for the managed nodegroup(s), this will also interfere with the functionality. In order to support the EFA functionality provided by `enable_efa_support = true`, you must utilize the custom launch template created/provided by this module, and supply an `instance_type`/`instance_types` for the respective nodegroup. + +The logic behind supporting EFA uses a data source to lookup the instance type to retrieve the number of interfaces that the instance supports in order to enumerate and expose those interfaces on the launch template created. For managed nodegroups where a list of instance types are supported, the first instance type in the list is used to calculate the number of EFA interfaces supported. Mixing instance types with varying number of interfaces is not recommended for EFA (or in some cases, mixing instance types is not supported - i.e. - p5.48xlarge and p4d.24xlarge). In addition to exposing the EFA interfaces and updating the security group rules, a placement group is created per the EFA requirements and only the availability zones that support the instance type selected are used in the subnets provided to the nodegroup. + +In order to enable EFA support, you will have to specify `enable_efa_support = true` on both the cluster and each nodegroup that you wish to enable EFA support for: + +```hcl +module "eks" { + source = "terraform-aws-modules/eks/aws" + version = "~> 20.0" + + # Truncated for brevity ... + + # Adds the EFA required security group rules to the shared + # security group created for the nodegroup(s) + enable_efa_support = true + + eks_managed_node_groups = { + example = { + instance_types = ["p5.48xlarge"] + + # Exposes all EFA interfaces on the launch template created by the nodegroup(s) + # This would expose all 32 EFA interfaces for the p5.48xlarge instance type + enable_efa_support = true + + pre_bootstrap_user_data = <<-EOT + # Mount NVME instance store volumes since they are typically + # available on instance types that support EFA + setup-local-disks raid0 + EOT + + # EFA should only be enabled when connecting 2 or more nodes + # Do not use EFA on a single node workload + min_size = 2 + max_size = 10 + desired_size = 2 + } + } +} +``` + ## Examples - [EKS Managed Node Group](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/examples/eks_managed_node_group): EKS Cluster using EKS managed node groups @@ -135,7 +189,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3 | -| [aws](#requirement\_aws) | >= 5.34 | +| [aws](#requirement\_aws) | >= 5.38 | | [time](#requirement\_time) | >= 0.9 | | [tls](#requirement\_tls) | >= 3.0 | @@ -143,7 +197,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.34 | +| [aws](#provider\_aws) | >= 5.38 | | [time](#provider\_time) | >= 0.9 | | [tls](#provider\_tls) | >= 3.0 | @@ -240,6 +294,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [eks\_managed\_node\_group\_defaults](#input\_eks\_managed\_node\_group\_defaults) | Map of EKS managed node group default configurations | `any` | `{}` | no | | [eks\_managed\_node\_groups](#input\_eks\_managed\_node\_groups) | Map of EKS managed node group definitions to create | `any` | `{}` | no | | [enable\_cluster\_creator\_admin\_permissions](#input\_enable\_cluster\_creator\_admin\_permissions) | Indicates whether or not to add the cluster creator (the identity used by Terraform) as an administrator via access entry | `bool` | `false` | no | +| [enable\_efa\_support](#input\_enable\_efa\_support) | Determines whether to enable Elastic Fabric Adapter (EFA) support | `bool` | `false` | no | | [enable\_irsa](#input\_enable\_irsa) | Determines whether to create an OpenID Connect Provider for EKS to enable IRSA | `bool` | `true` | no | | [enable\_kms\_key\_rotation](#input\_enable\_kms\_key\_rotation) | Specifies whether key rotation is enabled | `bool` | `true` | no | | [fargate\_profile\_defaults](#input\_fargate\_profile\_defaults) | Map of Fargate Profile default configurations | `any` | `{}` | no | diff --git a/examples/eks_managed_node_group/README.md b/examples/eks_managed_node_group/README.md index e94558cd77..1772d1324a 100644 --- a/examples/eks_managed_node_group/README.md +++ b/examples/eks_managed_node_group/README.md @@ -30,13 +30,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3 | -| [aws](#requirement\_aws) | >= 5.34 | +| [aws](#requirement\_aws) | >= 5.38 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.34 | +| [aws](#provider\_aws) | >= 5.38 | ## Modules diff --git a/examples/eks_managed_node_group/main.tf b/examples/eks_managed_node_group/main.tf index de737d70fb..370bb5755a 100644 --- a/examples/eks_managed_node_group/main.tf +++ b/examples/eks_managed_node_group/main.tf @@ -7,7 +7,7 @@ data "aws_availability_zones" "available" {} locals { name = "ex-${replace(basename(path.cwd), "_", "-")}" - cluster_version = "1.27" + cluster_version = "1.29" region = "eu-west-1" vpc_cidr = "10.0.0.0/16" @@ -37,6 +37,10 @@ module "eks" { enable_cluster_creator_admin_permissions = true + # Enable EFA support by adding necessary security group rules + # to the shared node security group + enable_efa_support = true + cluster_addons = { coredns = { most_recent = true @@ -241,6 +245,26 @@ module "eks" { ExtraTag = "EKS managed node group complete example" } } + + efa = { + # Disabling automatic creation due to instance type/quota availability + # Can be enabled when appropriate for testing/validation + create = false + + instance_types = ["trn1n.32xlarge"] + ami_type = "AL2_x86_64_GPU" + + enable_efa_support = true + pre_bootstrap_user_data = <<-EOT + # Mount NVME instance store volumes since they are typically + # available on instances that support EFA + setup-local-disks raid0 + EOT + + min_size = 2 + max_size = 2 + desired_size = 2 + } } access_entries = { diff --git a/examples/eks_managed_node_group/versions.tf b/examples/eks_managed_node_group/versions.tf index 63e9319d92..fb651ab50d 100644 --- a/examples/eks_managed_node_group/versions.tf +++ b/examples/eks_managed_node_group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.34" + version = ">= 5.38" } } } diff --git a/examples/fargate_profile/README.md b/examples/fargate_profile/README.md index 9ea3bf1871..55e303ce15 100644 --- a/examples/fargate_profile/README.md +++ b/examples/fargate_profile/README.md @@ -20,13 +20,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3 | -| [aws](#requirement\_aws) | >= 5.34 | +| [aws](#requirement\_aws) | >= 5.38 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.34 | +| [aws](#provider\_aws) | >= 5.38 | ## Modules diff --git a/examples/fargate_profile/versions.tf b/examples/fargate_profile/versions.tf index 63e9319d92..fb651ab50d 100644 --- a/examples/fargate_profile/versions.tf +++ b/examples/fargate_profile/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.34" + version = ">= 5.38" } } } diff --git a/examples/karpenter/README.md b/examples/karpenter/README.md index a1303999b0..f36f22c2e5 100644 --- a/examples/karpenter/README.md +++ b/examples/karpenter/README.md @@ -55,7 +55,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3 | -| [aws](#requirement\_aws) | >= 5.34 | +| [aws](#requirement\_aws) | >= 5.38 | | [helm](#requirement\_helm) | >= 2.7 | | [kubectl](#requirement\_kubectl) | >= 2.0 | @@ -63,8 +63,8 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.34 | -| [aws.virginia](#provider\_aws.virginia) | >= 5.34 | +| [aws](#provider\_aws) | >= 5.38 | +| [aws.virginia](#provider\_aws.virginia) | >= 5.38 | | [helm](#provider\_helm) | >= 2.7 | | [kubectl](#provider\_kubectl) | >= 2.0 | diff --git a/examples/karpenter/versions.tf b/examples/karpenter/versions.tf index 96d7b4ea19..711da76a33 100644 --- a/examples/karpenter/versions.tf +++ b/examples/karpenter/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.34" + version = ">= 5.38" } helm = { source = "hashicorp/helm" diff --git a/examples/outposts/README.md b/examples/outposts/README.md index 30fffaf330..87bdafa73a 100644 --- a/examples/outposts/README.md +++ b/examples/outposts/README.md @@ -49,14 +49,14 @@ terraform destroy | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3 | -| [aws](#requirement\_aws) | >= 5.34 | +| [aws](#requirement\_aws) | >= 5.38 | | [kubernetes](#requirement\_kubernetes) | >= 2.20 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.34 | +| [aws](#provider\_aws) | >= 5.38 | | [kubernetes](#provider\_kubernetes) | >= 2.20 | ## Modules diff --git a/examples/outposts/prerequisites/versions.tf b/examples/outposts/prerequisites/versions.tf index 63e9319d92..fb651ab50d 100644 --- a/examples/outposts/prerequisites/versions.tf +++ b/examples/outposts/prerequisites/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.34" + version = ">= 5.38" } } } diff --git a/examples/outposts/versions.tf b/examples/outposts/versions.tf index dff26f6939..73a045bb35 100644 --- a/examples/outposts/versions.tf +++ b/examples/outposts/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.34" + version = ">= 5.38" } kubernetes = { source = "hashicorp/kubernetes" diff --git a/examples/self_managed_node_group/README.md b/examples/self_managed_node_group/README.md index c19999d9b9..2566c3d9cb 100644 --- a/examples/self_managed_node_group/README.md +++ b/examples/self_managed_node_group/README.md @@ -26,13 +26,13 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3 | -| [aws](#requirement\_aws) | >= 5.34 | +| [aws](#requirement\_aws) | >= 5.38 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.34 | +| [aws](#provider\_aws) | >= 5.38 | ## Modules diff --git a/examples/self_managed_node_group/main.tf b/examples/self_managed_node_group/main.tf index dc125e1fbb..433e644993 100644 --- a/examples/self_managed_node_group/main.tf +++ b/examples/self_managed_node_group/main.tf @@ -31,6 +31,12 @@ module "eks" { cluster_version = local.cluster_version cluster_endpoint_public_access = true + enable_cluster_creator_admin_permissions = true + + # Enable EFA support by adding necessary security group rules + # to the shared node security group + enable_efa_support = true + cluster_addons = { coredns = { most_recent = true @@ -252,6 +258,25 @@ module "eks" { ExtraTag = "Self managed node group complete example" } } + + efa = { + # Disabling automatic creation due to instance type/quota availability + # Can be enabled when appropriate for testing/validation + create = false + + instance_type = "trn1n.32xlarge" + + enable_efa_support = true + pre_bootstrap_user_data = <<-EOT + # Mount NVME instance store volumes since they are typically + # available on instances that support EFA + setup-local-disks raid0 + EOT + + min_size = 2 + max_size = 2 + desired_size = 2 + } } tags = local.tags diff --git a/examples/self_managed_node_group/versions.tf b/examples/self_managed_node_group/versions.tf index 63e9319d92..fb651ab50d 100644 --- a/examples/self_managed_node_group/versions.tf +++ b/examples/self_managed_node_group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.34" + version = ">= 5.38" } } } diff --git a/modules/eks-managed-node-group/README.md b/modules/eks-managed-node-group/README.md index ebae013b92..708f7ad1f6 100644 --- a/modules/eks-managed-node-group/README.md +++ b/modules/eks-managed-node-group/README.md @@ -64,13 +64,13 @@ module "eks_managed_node_group" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3 | -| [aws](#requirement\_aws) | >= 5.34 | +| [aws](#requirement\_aws) | >= 5.38 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.34 | +| [aws](#provider\_aws) | >= 5.38 | ## Modules @@ -88,9 +88,13 @@ module "eks_managed_node_group" { | [aws_iam_role_policy_attachment.additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_iam_role_policy_attachment.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_launch_template.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/launch_template) | resource | +| [aws_placement_group.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/placement_group) | resource | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | +| [aws_ec2_instance_type.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ec2_instance_type) | data source | +| [aws_ec2_instance_type_offerings.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ec2_instance_type_offerings) | data source | | [aws_iam_policy_document.assume_role_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | +| [aws_subnets.efa](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/subnets) | data source | ## Inputs @@ -123,6 +127,7 @@ module "eks_managed_node_group" { | [elastic\_gpu\_specifications](#input\_elastic\_gpu\_specifications) | The elastic GPU to attach to the instance | `any` | `{}` | no | | [elastic\_inference\_accelerator](#input\_elastic\_inference\_accelerator) | Configuration block containing an Elastic Inference Accelerator to attach to the instance | `map(string)` | `{}` | no | | [enable\_bootstrap\_user\_data](#input\_enable\_bootstrap\_user\_data) | Determines whether the bootstrap configurations are populated within the user data template. Only valid when using a custom AMI via `ami_id` | `bool` | `false` | no | +| [enable\_efa\_support](#input\_enable\_efa\_support) | Determines whether to enable Elastic Fabric Adapter (EFA) support | `bool` | `false` | no | | [enable\_monitoring](#input\_enable\_monitoring) | Enables/disables detailed monitoring | `bool` | `true` | no | | [enclave\_options](#input\_enclave\_options) | Enable Nitro Enclaves on launched instances | `map(string)` | `{}` | no | | [force\_update\_version](#input\_force\_update\_version) | Force version update if existing pods are unable to be drained due to a pod disruption budget issue | `bool` | `null` | no | diff --git a/modules/eks-managed-node-group/main.tf b/modules/eks-managed-node-group/main.tf index 16ca010ae6..fa74cd3b76 100644 --- a/modules/eks-managed-node-group/main.tf +++ b/modules/eks-managed-node-group/main.tf @@ -24,6 +24,33 @@ module "user_data" { user_data_template_path = var.user_data_template_path } +################################################################################ +# EFA Support +################################################################################ + +data "aws_ec2_instance_type" "this" { + count = var.enable_efa_support ? 1 : 0 + + instance_type = local.efa_instance_type +} + +locals { + efa_instance_type = try(element(var.instance_types, 0), "") + num_network_cards = try(data.aws_ec2_instance_type.this[0].maximum_network_cards, 0) + + efa_network_interfaces = [ + for i in range(local.num_network_cards) : { + associate_public_ip_address = false + delete_on_termination = true + device_index = i == 0 ? 0 : 1 + network_card_index = i + interface_type = "efa" + } + ] + + network_interfaces = var.enable_efa_support ? local.efa_network_interfaces : var.network_interfaces +} + ################################################################################ # Launch template ################################################################################ @@ -31,6 +58,8 @@ module "user_data" { locals { launch_template_name = coalesce(var.launch_template_name, "${var.name}-eks-node-group") security_group_ids = compact(concat([var.cluster_primary_security_group_id], var.vpc_security_group_ids)) + + placement = var.create && var.enable_efa_support ? { group_name = aws_placement_group.this[0].name } : var.placement } resource "aws_launch_template" "this" { @@ -215,7 +244,8 @@ resource "aws_launch_template" "this" { name_prefix = var.launch_template_use_name_prefix ? "${local.launch_template_name}-" : null dynamic "network_interfaces" { - for_each = var.network_interfaces + for_each = local.network_interfaces + content { associate_carrier_ip_address = try(network_interfaces.value.associate_carrier_ip_address, null) associate_public_ip_address = try(network_interfaces.value.associate_public_ip_address, null) @@ -243,14 +273,14 @@ resource "aws_launch_template" "this" { } dynamic "placement" { - for_each = length(var.placement) > 0 ? [var.placement] : [] + for_each = length(local.placement) > 0 ? [local.placement] : [] content { affinity = try(placement.value.affinity, null) - availability_zone = try(placement.value.availability_zone, null) - group_name = try(placement.value.group_name, null) - host_id = try(placement.value.host_id, null) - host_resource_group_arn = try(placement.value.host_resource_group_arn, null) + availability_zone = lookup(placement.value, "availability_zone", null) + group_name = lookup(placement.value, "group_name", null) + host_id = lookup(placement.value, "host_id", null) + host_resource_group_arn = lookup(placement.value, "host_resource_group_arn", null) partition_number = try(placement.value.partition_number, null) spread_domain = try(placement.value.spread_domain, null) tenancy = try(placement.value.tenancy, null) @@ -280,7 +310,7 @@ resource "aws_launch_template" "this" { update_default_version = var.update_launch_template_default_version user_data = module.user_data.user_data - vpc_security_group_ids = length(var.network_interfaces) > 0 ? [] : local.security_group_ids + vpc_security_group_ids = length(local.network_interfaces) > 0 ? [] : local.security_group_ids tags = var.tags @@ -311,7 +341,7 @@ resource "aws_eks_node_group" "this" { # Required cluster_name = var.cluster_name node_role_arn = var.create_iam_role ? aws_iam_role.this[0].arn : var.iam_role_arn - subnet_ids = var.subnet_ids + subnet_ids = var.enable_efa_support ? data.aws_subnets.efa[0].ids : var.subnet_ids scaling_config { min_size = var.min_size @@ -448,6 +478,56 @@ resource "aws_iam_role_policy_attachment" "additional" { role = aws_iam_role.this[0].name } +################################################################################ +# Placement Group +################################################################################ + +resource "aws_placement_group" "this" { + count = var.create && var.enable_efa_support ? 1 : 0 + + name = "${var.cluster_name}-${var.name}" + strategy = "cluster" + + tags = var.tags +} + +################################################################################ +# Instance AZ Lookup + +# Instances usually used in placement groups w/ EFA are only available in +# select availability zones. These data sources will cross reference the availability +# zones supported by the instance type with the subnets provided to ensure only +# AZs/subnets that are supported are used. +################################################################################ + +# Find the availability zones supported by the instance type +data "aws_ec2_instance_type_offerings" "this" { + count = var.create && var.enable_efa_support ? 1 : 0 + + filter { + name = "instance-type" + values = [local.efa_instance_type] + } + + location_type = "availability-zone-id" +} + +# Reverse the lookup to find one of the subnets provided based on the availability +# availability zone ID of the queried instance type (supported) +data "aws_subnets" "efa" { + count = var.create && var.enable_efa_support ? 1 : 0 + + filter { + name = "subnet-id" + values = var.subnet_ids + } + + filter { + name = "availability-zone-id" + values = data.aws_ec2_instance_type_offerings.this[0].locations + } +} + ################################################################################ # Autoscaling Group Schedule ################################################################################ diff --git a/modules/eks-managed-node-group/variables.tf b/modules/eks-managed-node-group/variables.tf index ede9dc4c50..192980888a 100644 --- a/modules/eks-managed-node-group/variables.tf +++ b/modules/eks-managed-node-group/variables.tf @@ -250,6 +250,12 @@ variable "enable_monitoring" { default = true } +variable "enable_efa_support" { + description = "Determines whether to enable Elastic Fabric Adapter (EFA) support" + type = bool + default = false +} + variable "network_interfaces" { description = "Customize network interfaces to be attached at instance boot time" type = list(any) diff --git a/modules/eks-managed-node-group/versions.tf b/modules/eks-managed-node-group/versions.tf index 63e9319d92..fb651ab50d 100644 --- a/modules/eks-managed-node-group/versions.tf +++ b/modules/eks-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.34" + version = ">= 5.38" } } } diff --git a/modules/fargate-profile/README.md b/modules/fargate-profile/README.md index 8656a6f191..1fb59c4ba0 100644 --- a/modules/fargate-profile/README.md +++ b/modules/fargate-profile/README.md @@ -29,13 +29,13 @@ module "fargate_profile" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3 | -| [aws](#requirement\_aws) | >= 5.34 | +| [aws](#requirement\_aws) | >= 5.38 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.34 | +| [aws](#provider\_aws) | >= 5.38 | ## Modules diff --git a/modules/fargate-profile/versions.tf b/modules/fargate-profile/versions.tf index 63e9319d92..fb651ab50d 100644 --- a/modules/fargate-profile/versions.tf +++ b/modules/fargate-profile/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.34" + version = ">= 5.38" } } } diff --git a/modules/karpenter/README.md b/modules/karpenter/README.md index 00f724acc5..d6f27bfd1f 100644 --- a/modules/karpenter/README.md +++ b/modules/karpenter/README.md @@ -85,13 +85,13 @@ module "karpenter" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3 | -| [aws](#requirement\_aws) | >= 5.34 | +| [aws](#requirement\_aws) | >= 5.38 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.34 | +| [aws](#provider\_aws) | >= 5.38 | ## Modules diff --git a/modules/karpenter/versions.tf b/modules/karpenter/versions.tf index 63e9319d92..fb651ab50d 100644 --- a/modules/karpenter/versions.tf +++ b/modules/karpenter/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.34" + version = ">= 5.38" } } } diff --git a/modules/self-managed-node-group/README.md b/modules/self-managed-node-group/README.md index 83e44d0bc5..6fe6e43180 100644 --- a/modules/self-managed-node-group/README.md +++ b/modules/self-managed-node-group/README.md @@ -43,13 +43,13 @@ module "self_managed_node_group" { | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3 | -| [aws](#requirement\_aws) | >= 5.34 | +| [aws](#requirement\_aws) | >= 5.38 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 5.34 | +| [aws](#provider\_aws) | >= 5.38 | ## Modules @@ -69,10 +69,14 @@ module "self_managed_node_group" { | [aws_iam_role_policy_attachment.additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_iam_role_policy_attachment.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_launch_template.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/launch_template) | resource | +| [aws_placement_group.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/placement_group) | resource | | [aws_ami.eks_default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | +| [aws_ec2_instance_type.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ec2_instance_type) | data source | +| [aws_ec2_instance_type_offerings.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ec2_instance_type_offerings) | data source | | [aws_iam_policy_document.assume_role_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | +| [aws_subnets.efa](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/subnets) | data source | ## Inputs @@ -108,6 +112,7 @@ module "self_managed_node_group" { | [ebs\_optimized](#input\_ebs\_optimized) | If true, the launched EC2 instance will be EBS-optimized | `bool` | `null` | no | | [elastic\_gpu\_specifications](#input\_elastic\_gpu\_specifications) | The elastic GPU to attach to the instance | `any` | `{}` | no | | [elastic\_inference\_accelerator](#input\_elastic\_inference\_accelerator) | Configuration block containing an Elastic Inference Accelerator to attach to the instance | `map(string)` | `{}` | no | +| [enable\_efa\_support](#input\_enable\_efa\_support) | Determines whether to enable Elastic Fabric Adapter (EFA) support | `bool` | `false` | no | | [enable\_monitoring](#input\_enable\_monitoring) | Enables/disables detailed monitoring | `bool` | `true` | no | | [enabled\_metrics](#input\_enabled\_metrics) | A list of metrics to collect. The allowed values are `GroupDesiredCapacity`, `GroupInServiceCapacity`, `GroupPendingCapacity`, `GroupMinSize`, `GroupMaxSize`, `GroupInServiceInstances`, `GroupPendingInstances`, `GroupStandbyInstances`, `GroupStandbyCapacity`, `GroupTerminatingCapacity`, `GroupTerminatingInstances`, `GroupTotalCapacity`, `GroupTotalInstances` | `list(string)` | `[]` | no | | [enclave\_options](#input\_enclave\_options) | Enable Nitro Enclaves on launched instances | `map(string)` | `{}` | no | diff --git a/modules/self-managed-node-group/main.tf b/modules/self-managed-node-group/main.tf index eb3192970b..bb1eea0b54 100644 --- a/modules/self-managed-node-group/main.tf +++ b/modules/self-managed-node-group/main.tf @@ -35,6 +35,33 @@ module "user_data" { user_data_template_path = var.user_data_template_path } +################################################################################ +# EFA Support +################################################################################ + +data "aws_ec2_instance_type" "this" { + count = var.enable_efa_support && local.instance_type_provided ? 1 : 0 + + instance_type = var.instance_type +} + +locals { + instance_type_provided = var.instance_type != "" + num_network_cards = try(data.aws_ec2_instance_type.this[0].maximum_network_cards, 0) + + efa_network_interfaces = [ + for i in range(local.num_network_cards) : { + associate_public_ip_address = false + delete_on_termination = true + device_index = i == 0 ? 0 : 1 + network_card_index = i + interface_type = "efa" + } + ] + + network_interfaces = var.enable_efa_support && local.instance_type_provided ? local.efa_network_interfaces : var.network_interfaces +} + ################################################################################ # Launch template ################################################################################ @@ -42,6 +69,8 @@ module "user_data" { locals { launch_template_name = coalesce(var.launch_template_name, "${var.name}-node-group") security_group_ids = compact(concat([var.cluster_primary_security_group_id], var.vpc_security_group_ids)) + + placement = var.create && var.enable_efa_support ? { group_name = aws_placement_group.this[0].name } : var.placement } resource "aws_launch_template" "this" { @@ -321,7 +350,8 @@ resource "aws_launch_template" "this" { name_prefix = var.launch_template_use_name_prefix ? "${local.launch_template_name}-" : null dynamic "network_interfaces" { - for_each = var.network_interfaces + for_each = local.network_interfaces + content { associate_carrier_ip_address = try(network_interfaces.value.associate_carrier_ip_address, null) associate_public_ip_address = try(network_interfaces.value.associate_public_ip_address, null) @@ -347,14 +377,14 @@ resource "aws_launch_template" "this" { } dynamic "placement" { - for_each = length(var.placement) > 0 ? [var.placement] : [] + for_each = length(local.placement) > 0 ? [local.placement] : [] content { affinity = try(placement.value.affinity, null) - availability_zone = try(placement.value.availability_zone, null) - group_name = try(placement.value.group_name, null) - host_id = try(placement.value.host_id, null) - host_resource_group_arn = try(placement.value.host_resource_group_arn, null) + availability_zone = lookup(placement.value, "availability_zone", null) + group_name = lookup(placement.value, "group_name", null) + host_id = lookup(placement.value, "host_id", null) + host_resource_group_arn = lookup(placement.value, "host_resource_group_arn", null) partition_number = try(placement.value.partition_number, null) spread_domain = try(placement.value.spread_domain, null) tenancy = try(placement.value.tenancy, null) @@ -384,7 +414,7 @@ resource "aws_launch_template" "this" { update_default_version = var.update_launch_template_default_version user_data = module.user_data.user_data - vpc_security_group_ids = length(var.network_interfaces) > 0 ? [] : local.security_group_ids + vpc_security_group_ids = length(local.network_interfaces) > 0 ? [] : local.security_group_ids tags = var.tags @@ -664,7 +694,7 @@ resource "aws_autoscaling_group" "this" { target_group_arns = var.target_group_arns termination_policies = var.termination_policies - vpc_zone_identifier = var.subnet_ids + vpc_zone_identifier = var.enable_efa_support ? data.aws_subnets.efa[0].ids : var.subnet_ids wait_for_capacity_timeout = var.wait_for_capacity_timeout wait_for_elb_capacity = var.wait_for_elb_capacity @@ -771,6 +801,56 @@ resource "aws_iam_instance_profile" "this" { } } +################################################################################ +# Placement Group +################################################################################ + +resource "aws_placement_group" "this" { + count = var.create && var.enable_efa_support ? 1 : 0 + + name = "${var.cluster_name}-${var.name}" + strategy = "cluster" + + tags = var.tags +} + +################################################################################ +# Instance AZ Lookup + +# Instances usually used in placement groups w/ EFA are only available in +# select availability zones. These data sources will cross reference the availability +# zones supported by the instance type with the subnets provided to ensure only +# AZs/subnets that are supported are used. +################################################################################ + +# Find the availability zones supported by the instance type +data "aws_ec2_instance_type_offerings" "this" { + count = var.create && var.enable_efa_support ? 1 : 0 + + filter { + name = "instance-type" + values = [var.instance_type] + } + + location_type = "availability-zone-id" +} + +# Reverse the lookup to find one of the subnets provided based on the availability +# availability zone ID of the queried instance type (supported) +data "aws_subnets" "efa" { + count = var.create && var.enable_efa_support ? 1 : 0 + + filter { + name = "subnet-id" + values = var.subnet_ids + } + + filter { + name = "availability-zone-id" + values = data.aws_ec2_instance_type_offerings.this[0].locations + } +} + ################################################################################ # Access Entry ################################################################################ diff --git a/modules/self-managed-node-group/variables.tf b/modules/self-managed-node-group/variables.tf index c6c5086a57..54bded55fe 100644 --- a/modules/self-managed-node-group/variables.tf +++ b/modules/self-managed-node-group/variables.tf @@ -270,6 +270,12 @@ variable "enable_monitoring" { default = true } +variable "enable_efa_support" { + description = "Determines whether to enable Elastic Fabric Adapter (EFA) support" + type = bool + default = false +} + variable "metadata_options" { description = "Customize the metadata options for the instance" type = map(string) diff --git a/modules/self-managed-node-group/versions.tf b/modules/self-managed-node-group/versions.tf index 63e9319d92..fb651ab50d 100644 --- a/modules/self-managed-node-group/versions.tf +++ b/modules/self-managed-node-group/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.34" + version = ">= 5.38" } } } diff --git a/node_groups.tf b/node_groups.tf index ddfc20fa09..a225bb8931 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -179,6 +179,27 @@ locals { ipv6_cidr_blocks = var.cluster_ip_family == "ipv6" ? ["::/0"] : null } } : k => v if var.node_security_group_enable_recommended_rules } + + efa_security_group_rules = { for k, v in + { + ingress_all_self_efa = { + description = "Node to node EFA" + protocol = "-1" + from_port = 0 + to_port = 0 + type = "ingress" + self = true + } + egress_all_self_efa = { + description = "Node to node EFA" + protocol = "-1" + from_port = 0 + to_port = 0 + type = "egress" + self = true + } + } : k => v if var.enable_efa_support + } } resource "aws_security_group" "node" { @@ -205,6 +226,7 @@ resource "aws_security_group" "node" { resource "aws_security_group_rule" "node" { for_each = { for k, v in merge( + local.efa_security_group_rules, local.node_security_group_rules, local.node_security_group_recommended_rules, var.node_security_group_additional_rules, @@ -343,6 +365,7 @@ module "eks_managed_node_group" { license_specifications = try(each.value.license_specifications, var.eks_managed_node_group_defaults.license_specifications, {}) metadata_options = try(each.value.metadata_options, var.eks_managed_node_group_defaults.metadata_options, local.metadata_options) enable_monitoring = try(each.value.enable_monitoring, var.eks_managed_node_group_defaults.enable_monitoring, true) + enable_efa_support = try(each.value.enable_efa_support, var.eks_managed_node_group_defaults.enable_efa_support, false) network_interfaces = try(each.value.network_interfaces, var.eks_managed_node_group_defaults.network_interfaces, []) placement = try(each.value.placement, var.eks_managed_node_group_defaults.placement, {}) maintenance_options = try(each.value.maintenance_options, var.eks_managed_node_group_defaults.maintenance_options, {}) @@ -478,6 +501,7 @@ module "self_managed_node_group" { license_specifications = try(each.value.license_specifications, var.self_managed_node_group_defaults.license_specifications, {}) metadata_options = try(each.value.metadata_options, var.self_managed_node_group_defaults.metadata_options, local.metadata_options) enable_monitoring = try(each.value.enable_monitoring, var.self_managed_node_group_defaults.enable_monitoring, true) + enable_efa_support = try(each.value.enable_efa_support, var.self_managed_node_group_defaults.enable_efa_support, false) network_interfaces = try(each.value.network_interfaces, var.self_managed_node_group_defaults.network_interfaces, []) placement = try(each.value.placement, var.self_managed_node_group_defaults.placement, {}) maintenance_options = try(each.value.maintenance_options, var.self_managed_node_group_defaults.maintenance_options, {}) diff --git a/variables.tf b/variables.tf index 83776d6f4f..fff8895279 100644 --- a/variables.tf +++ b/variables.tf @@ -376,6 +376,12 @@ variable "node_security_group_tags" { default = {} } +variable "enable_efa_support" { + description = "Determines whether to enable Elastic Fabric Adapter (EFA) support" + type = bool + default = false +} + ################################################################################ # IRSA ################################################################################ diff --git a/versions.tf b/versions.tf index a7084a7252..4d006cbfb6 100644 --- a/versions.tf +++ b/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.34" + version = ">= 5.38" } tls = { source = "hashicorp/tls"