From fbc1d135c80fa99f885835953f601b2ca421c13f Mon Sep 17 00:00:00 2001 From: Fernando Miguel Date: Tue, 8 Nov 2022 15:54:21 +0000 Subject: [PATCH 1/5] add new karpenter nth iam policies Signed-off-by: Fernando Miguel --- .../README.md | 1 + .../iam-role-for-service-accounts-eks/main.tf | 2 + .../policies.tf | 47 ++++++++++++++++++- 3 files changed, 49 insertions(+), 1 deletion(-) diff --git a/modules/iam-role-for-service-accounts-eks/README.md b/modules/iam-role-for-service-accounts-eks/README.md index ceafe97c..f1648b36 100644 --- a/modules/iam-role-for-service-accounts-eks/README.md +++ b/modules/iam-role-for-service-accounts-eks/README.md @@ -170,6 +170,7 @@ No modules. | [aws_iam_policy_document.velero](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_iam_policy_document.vpc_cni](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | +| [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source | ## Inputs diff --git a/modules/iam-role-for-service-accounts-eks/main.tf b/modules/iam-role-for-service-accounts-eks/main.tf index ab9b2751..50d87945 100644 --- a/modules/iam-role-for-service-accounts-eks/main.tf +++ b/modules/iam-role-for-service-accounts-eks/main.tf @@ -1,6 +1,8 @@ data "aws_partition" "current" {} data "aws_caller_identity" "current" {} +data "aws_region" "current" {} + locals { account_id = data.aws_caller_identity.current.account_id partition = data.aws_partition.current.partition diff --git a/modules/iam-role-for-service-accounts-eks/policies.tf b/modules/iam-role-for-service-accounts-eks/policies.tf index 72b12e93..4cab3a2e 100644 --- a/modules/iam-role-for-service-accounts-eks/policies.tf +++ b/modules/iam-role-for-service-accounts-eks/policies.tf @@ -504,7 +504,7 @@ resource "aws_iam_role_policy_attachment" "fsx_lustre_csi" { # Karpenter Controller Policy ################################################################################ -# curl -fsSL https://karpenter.sh/v0.6.1/getting-started/cloudformation.yaml +# curl -fsSL https://karpenter.sh/v0.19.0/getting-started/cloudformation.yaml data "aws_iam_policy_document" "karpenter_controller" { count = var.create_role && var.attach_karpenter_controller_policy ? 1 : 0 @@ -578,6 +578,51 @@ data "aws_iam_policy_document" "karpenter_controller" { actions = ["iam:PassRole"] resources = var.karpenter_controller_node_iam_role_arns } + + statement { + sid = "KarpenterEventPolicyEvents" + effect = "Allow" + resources = ["arn:aws:events:${local.account_id}:rule/Karpenter-*"] + + actions = [ + "events:TagResource", + "events:DeleteRule", + "events:PutTargets", + "events:PutRule", + "events:ListTagsForResource", + "events:RemoveTargets", + ] + + condition { + test = "StringEquals" + variable = "aws:ResourceTag/karpenter.sh/discovery" + values = [var.karpenter_controller_cluster_id] + } + } + + statement { + sid = "KarpenterEventPolicyListRules" + effect = "Allow" + resources = ["*"] + actions = ["events:ListRules"] + } + + statement { + sid = "KarpenterEventPolicySQS" + effect = "Allow" + resources = ["arn:aws:sqs:${data.aws_region.current.name}:${local.account_id}:${var.karpenter_controller_cluster_id}"] + + actions = [ + "sqs:DeleteMessage", + "sqs:TagQueue", + "sqs:GetQueueUrl", + "sqs:ReceiveMessage", + "sqs:DeleteQueue", + "sqs:GetQueueAttributes", + "sqs:CreateQueue", + "sqs:SetQueueAttributes", + ] + } } resource "aws_iam_policy" "karpenter_controller" { From 98951566bd050e6833f9e46b55ad33ce7135e8f4 Mon Sep 17 00:00:00 2001 From: Fernando Miguel Date: Thu, 17 Nov 2022 17:28:03 +0000 Subject: [PATCH 2/5] iam reviewed Signed-off-by: Fernando Miguel --- .../policies.tf | 49 ++++--------------- 1 file changed, 9 insertions(+), 40 deletions(-) diff --git a/modules/iam-role-for-service-accounts-eks/policies.tf b/modules/iam-role-for-service-accounts-eks/policies.tf index 6e4dfb2b..0db9d2f3 100644 --- a/modules/iam-role-for-service-accounts-eks/policies.tf +++ b/modules/iam-role-for-service-accounts-eks/policies.tf @@ -509,24 +509,25 @@ resource "aws_iam_role_policy_attachment" "fsx_lustre_csi" { # Karpenter Controller Policy ################################################################################ -# curl -fsSL https://karpenter.sh/v0.19.0/getting-started/cloudformation.yaml +# https://github.com/aws/karpenter/blob/502d275cc330fb0f2435b124935c49632146d945/website/content/en/v0.19.0/getting-started/getting-started-with-eksctl/cloudformation.yaml#L34 data "aws_iam_policy_document" "karpenter_controller" { count = var.create_role && var.attach_karpenter_controller_policy ? 1 : 0 statement { actions = [ - "ec2:CreateLaunchTemplate", "ec2:CreateFleet", + "ec2:CreateLaunchTemplate", "ec2:CreateTags", - "ec2:DescribeLaunchTemplates", + "ec2:DescribeAvailabilityZones", + "ec2:DescribeImages", "ec2:DescribeImages", "ec2:DescribeInstances", - "ec2:DescribeSecurityGroups", - "ec2:DescribeSubnets", - "ec2:DescribeInstanceTypes", "ec2:DescribeInstanceTypeOfferings", - "ec2:DescribeAvailabilityZones", + "ec2:DescribeInstanceTypes", + "ec2:DescribeLaunchTemplates", + "ec2:DescribeSecurityGroups", "ec2:DescribeSpotPriceHistory", + "ec2:DescribeSubnets", "pricing:GetProducts", ] @@ -584,34 +585,6 @@ data "aws_iam_policy_document" "karpenter_controller" { resources = var.karpenter_controller_node_iam_role_arns } - statement { - sid = "KarpenterEventPolicyEvents" - effect = "Allow" - resources = ["arn:aws:events:${local.account_id}:rule/Karpenter-*"] - - actions = [ - "events:TagResource", - "events:DeleteRule", - "events:PutTargets", - "events:PutRule", - "events:ListTagsForResource", - "events:RemoveTargets", - ] - - condition { - test = "StringEquals" - variable = "aws:ResourceTag/karpenter.sh/discovery" - values = [var.karpenter_controller_cluster_id] - } - } - - statement { - sid = "KarpenterEventPolicyListRules" - effect = "Allow" - resources = ["*"] - actions = ["events:ListRules"] - } - statement { sid = "KarpenterEventPolicySQS" effect = "Allow" @@ -619,13 +592,9 @@ data "aws_iam_policy_document" "karpenter_controller" { actions = [ "sqs:DeleteMessage", - "sqs:TagQueue", + "sqs:GetQueueAttributes", "sqs:GetQueueUrl", "sqs:ReceiveMessage", - "sqs:DeleteQueue", - "sqs:GetQueueAttributes", - "sqs:CreateQueue", - "sqs:SetQueueAttributes", ] } } From 934284fc76cb241b6583e9f1f0e532d41ee94c72 Mon Sep 17 00:00:00 2001 From: Fernando Miguel Date: Thu, 17 Nov 2022 18:55:14 +0000 Subject: [PATCH 3/5] add var for karpenter_sqs_queue_arn Signed-off-by: Fernando Miguel --- modules/iam-role-for-service-accounts-eks/policies.tf | 6 +++++- modules/iam-role-for-service-accounts-eks/variables.tf | 6 ++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/modules/iam-role-for-service-accounts-eks/policies.tf b/modules/iam-role-for-service-accounts-eks/policies.tf index 0db9d2f3..4a666b15 100644 --- a/modules/iam-role-for-service-accounts-eks/policies.tf +++ b/modules/iam-role-for-service-accounts-eks/policies.tf @@ -509,6 +509,10 @@ resource "aws_iam_role_policy_attachment" "fsx_lustre_csi" { # Karpenter Controller Policy ################################################################################ +locals { + karpenter_sqs_queue_arn = var.karpenter_sqs_queue_arn != null ? var.karpenter_sqs_queue_arn : "arn:aws:sqs:${data.aws_region.current.name}:${local.account_id}:${var.karpenter_controller_cluster_id}" +} + # https://github.com/aws/karpenter/blob/502d275cc330fb0f2435b124935c49632146d945/website/content/en/v0.19.0/getting-started/getting-started-with-eksctl/cloudformation.yaml#L34 data "aws_iam_policy_document" "karpenter_controller" { count = var.create_role && var.attach_karpenter_controller_policy ? 1 : 0 @@ -588,7 +592,7 @@ data "aws_iam_policy_document" "karpenter_controller" { statement { sid = "KarpenterEventPolicySQS" effect = "Allow" - resources = ["arn:aws:sqs:${data.aws_region.current.name}:${local.account_id}:${var.karpenter_controller_cluster_id}"] + resources = [local.karpenter_sqs_queue_arn] actions = [ "sqs:DeleteMessage", diff --git a/modules/iam-role-for-service-accounts-eks/variables.tf b/modules/iam-role-for-service-accounts-eks/variables.tf index c9192edb..5f80d5a7 100644 --- a/modules/iam-role-for-service-accounts-eks/variables.tf +++ b/modules/iam-role-for-service-accounts-eks/variables.tf @@ -215,6 +215,12 @@ variable "karpenter_subnet_account_id" { default = "" } +variable "karpenter_sqs_queue_arn" { + description = "(Optional) SQS ARN to be used in Karpenter NSTH" + type = string + default = null +} + # AWS Load Balancer Controller variable "attach_load_balancer_controller_policy" { description = "Determines whether to attach the Load Balancer Controller policy to the role" From b98c7c7c83ef28573970ce9aee14f4499fd39ee3 Mon Sep 17 00:00:00 2001 From: Fernando Miguel Date: Thu, 17 Nov 2022 19:00:26 +0000 Subject: [PATCH 4/5] docs Signed-off-by: Fernando Miguel --- modules/iam-role-for-service-accounts-eks/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/iam-role-for-service-accounts-eks/README.md b/modules/iam-role-for-service-accounts-eks/README.md index f1648b36..f18d04bb 100644 --- a/modules/iam-role-for-service-accounts-eks/README.md +++ b/modules/iam-role-for-service-accounts-eks/README.md @@ -207,6 +207,7 @@ No modules. | [karpenter\_controller\_cluster\_id](#input\_karpenter\_controller\_cluster\_id) | Cluster ID where the Karpenter controller is provisioned/managing | `string` | `"*"` | no | | [karpenter\_controller\_node\_iam\_role\_arns](#input\_karpenter\_controller\_node\_iam\_role\_arns) | List of node IAM role ARNs Karpenter can use to launch nodes | `list(string)` | 
[
  "*"
]
| no | | [karpenter\_controller\_ssm\_parameter\_arns](#input\_karpenter\_controller\_ssm\_parameter\_arns) | List of SSM Parameter ARNs that contain AMI IDs launched by Karpenter | `list(string)` | 
[
  "arn:aws:ssm:*:*:parameter/aws/service/*"
]
| no | +| [karpenter\_sqs\_queue\_arn](#input\_karpenter\_sqs\_queue\_arn) | (Optional) SQS ARN to be used in Karpenter NSTH | `string` | `null` | no | | [karpenter\_subnet\_account\_id](#input\_karpenter\_subnet\_account\_id) | Account ID of where the subnets Karpenter will utilize resides. Used when subnets are shared from another account | `string` | `""` | no | | [karpenter\_tag\_key](#input\_karpenter\_tag\_key) | Tag key (`{key = value}`) applied to resources launched by Karpenter through the Karpenter provisioner | `string` | `"karpenter.sh/discovery"` | no | | [max\_session\_duration](#input\_max\_session\_duration) | Maximum CLI/API session duration in seconds between 3600 and 43200 | `number` | `null` | no | From 877ead0203f8501c3b4abba48843a3544ef9439c Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Mon, 21 Nov 2022 12:44:07 -0500 Subject: [PATCH 5/5] fix: Make SQS permissions optional depending on if an SQS ARN is provided --- .../README.md | 3 +-- .../iam-role-for-service-accounts-eks/main.tf | 2 -- .../policies.tf | 25 ++++++++----------- .../variables.tf | 2 +- 4 files changed, 13 insertions(+), 19 deletions(-) diff --git a/modules/iam-role-for-service-accounts-eks/README.md b/modules/iam-role-for-service-accounts-eks/README.md index f18d04bb..fba28cfa 100644 --- a/modules/iam-role-for-service-accounts-eks/README.md +++ b/modules/iam-role-for-service-accounts-eks/README.md @@ -170,7 +170,6 @@ No modules. | [aws_iam_policy_document.velero](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_iam_policy_document.vpc_cni](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | -| [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source | ## Inputs @@ -207,7 +206,7 @@ No modules. | [karpenter\_controller\_cluster\_id](#input\_karpenter\_controller\_cluster\_id) | Cluster ID where the Karpenter controller is provisioned/managing | `string` | `"*"` | no | | [karpenter\_controller\_node\_iam\_role\_arns](#input\_karpenter\_controller\_node\_iam\_role\_arns) | List of node IAM role ARNs Karpenter can use to launch nodes | `list(string)` | 
[
  "*"
]
| no | | [karpenter\_controller\_ssm\_parameter\_arns](#input\_karpenter\_controller\_ssm\_parameter\_arns) | List of SSM Parameter ARNs that contain AMI IDs launched by Karpenter | `list(string)` | 
[
  "arn:aws:ssm:*:*:parameter/aws/service/*"
]
| no | -| [karpenter\_sqs\_queue\_arn](#input\_karpenter\_sqs\_queue\_arn) | (Optional) SQS ARN to be used in Karpenter NSTH | `string` | `null` | no | +| [karpenter\_sqs\_queue\_arn](#input\_karpenter\_sqs\_queue\_arn) | (Optional) ARN of SQS used by Karpenter when native node termination handling is enabled | `string` | `null` | no | | [karpenter\_subnet\_account\_id](#input\_karpenter\_subnet\_account\_id) | Account ID of where the subnets Karpenter will utilize resides. Used when subnets are shared from another account | `string` | `""` | no | | [karpenter\_tag\_key](#input\_karpenter\_tag\_key) | Tag key (`{key = value}`) applied to resources launched by Karpenter through the Karpenter provisioner | `string` | `"karpenter.sh/discovery"` | no | | [max\_session\_duration](#input\_max\_session\_duration) | Maximum CLI/API session duration in seconds between 3600 and 43200 | `number` | `null` | no | diff --git a/modules/iam-role-for-service-accounts-eks/main.tf b/modules/iam-role-for-service-accounts-eks/main.tf index 50d87945..ab9b2751 100644 --- a/modules/iam-role-for-service-accounts-eks/main.tf +++ b/modules/iam-role-for-service-accounts-eks/main.tf @@ -1,8 +1,6 @@ data "aws_partition" "current" {} data "aws_caller_identity" "current" {} -data "aws_region" "current" {} - locals { account_id = data.aws_caller_identity.current.account_id partition = data.aws_partition.current.partition diff --git a/modules/iam-role-for-service-accounts-eks/policies.tf b/modules/iam-role-for-service-accounts-eks/policies.tf index 4a666b15..d4ff8289 100644 --- a/modules/iam-role-for-service-accounts-eks/policies.tf +++ b/modules/iam-role-for-service-accounts-eks/policies.tf @@ -509,10 +509,6 @@ resource "aws_iam_role_policy_attachment" "fsx_lustre_csi" { # Karpenter Controller Policy ################################################################################ -locals { - karpenter_sqs_queue_arn = var.karpenter_sqs_queue_arn != null ? var.karpenter_sqs_queue_arn : "arn:aws:sqs:${data.aws_region.current.name}:${local.account_id}:${var.karpenter_controller_cluster_id}" -} - # https://github.com/aws/karpenter/blob/502d275cc330fb0f2435b124935c49632146d945/website/content/en/v0.19.0/getting-started/getting-started-with-eksctl/cloudformation.yaml#L34 data "aws_iam_policy_document" "karpenter_controller" { count = var.create_role && var.attach_karpenter_controller_policy ? 1 : 0 @@ -589,17 +585,18 @@ data "aws_iam_policy_document" "karpenter_controller" { resources = var.karpenter_controller_node_iam_role_arns } - statement { - sid = "KarpenterEventPolicySQS" - effect = "Allow" - resources = [local.karpenter_sqs_queue_arn] + dynamic "statement" { + for_each = var.karpenter_sqs_queue_arn != null ? [1] : [] - actions = [ - "sqs:DeleteMessage", - "sqs:GetQueueAttributes", - "sqs:GetQueueUrl", - "sqs:ReceiveMessage", - ] + content { + actions = [ + "sqs:DeleteMessage", + "sqs:GetQueueAttributes", + "sqs:GetQueueUrl", + "sqs:ReceiveMessage", + ] + resources = [var.karpenter_sqs_queue_arn] + } } } diff --git a/modules/iam-role-for-service-accounts-eks/variables.tf b/modules/iam-role-for-service-accounts-eks/variables.tf index 5f80d5a7..1eae9778 100644 --- a/modules/iam-role-for-service-accounts-eks/variables.tf +++ b/modules/iam-role-for-service-accounts-eks/variables.tf @@ -216,7 +216,7 @@ variable "karpenter_subnet_account_id" { } variable "karpenter_sqs_queue_arn" { - description = "(Optional) SQS ARN to be used in Karpenter NSTH" + description = "(Optional) ARN of SQS used by Karpenter when native node termination handling is enabled" type = string default = null }