From 6dc4ca68913ee73c39989b770e18ad0e6ab23f83 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Sun, 13 Feb 2022 09:43:06 -0500 Subject: [PATCH 01/17] fix: Restrict `ssm:GetParameter` IAM permissions to only the AWS service parameters --- .../en/preview/getting-started-with-terraform/_index.md | 8 +++++++- .../en/preview/getting-started/cloudformation.yaml | 3 +++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/website/content/en/preview/getting-started-with-terraform/_index.md b/website/content/en/preview/getting-started-with-terraform/_index.md index b76a5e79d04b..553d117eb279 100644 --- a/website/content/en/preview/getting-started-with-terraform/_index.md +++ b/website/content/en/preview/getting-started-with-terraform/_index.md @@ -217,11 +217,17 @@ resource "aws_iam_role_policy" "karpenter_controller" { "ec2:DescribeInstanceTypes", "ec2:DescribeInstanceTypeOfferings", "ec2:DescribeAvailabilityZones", - "ssm:GetParameter" ] Effect = "Allow" Resource = "*" }, + { + Action = [ + "ssm:GetParameter" + ] + Effect = "Allow" + Resource = "arn:aws:ssm:*:*:parameter/aws/service/*" + }, ] }) } diff --git a/website/content/en/preview/getting-started/cloudformation.yaml b/website/content/en/preview/getting-started/cloudformation.yaml index 74af86867cc6..452bbf7dcc88 100644 --- a/website/content/en/preview/getting-started/cloudformation.yaml +++ b/website/content/en/preview/getting-started/cloudformation.yaml @@ -57,4 +57,7 @@ Resources: - ec2:DescribeInstanceTypes - ec2:DescribeInstanceTypeOfferings - ec2:DescribeAvailabilityZones + - Effect: Allow + Resource: "arn:aws:ssm:*:*:parameter/aws/service/*" + Action: - ssm:GetParameter From 91638f7e877ca8b51ee16bb2120d0828dbddd6b0 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Mon, 14 Feb 2022 14:10:11 -0500 Subject: [PATCH 02/17] chore: update permissions and Terraform example --- .../getting-started-with-terraform/_index.md | 190 +++++++++++------- .../getting-started/cloudformation.yaml | 14 +- 2 files changed, 128 insertions(+), 76 deletions(-) diff --git a/website/content/en/preview/getting-started-with-terraform/_index.md b/website/content/en/preview/getting-started-with-terraform/_index.md index 553d117eb279..2d2fd880a485 100644 --- a/website/content/en/preview/getting-started-with-terraform/_index.md +++ b/website/content/en/preview/getting-started-with-terraform/_index.md @@ -18,7 +18,7 @@ Follow the clean-up instructions to reduce any charges. ## Install -Karpenter is installed in clusters with a helm chart. +Karpenter is installed in clusters via a helm chart. Karpenter additionally requires IAM Roles for Service Accounts (IRSA). IRSA permits Karpenter (within the cluster) to make privileged requests to AWS (as @@ -42,19 +42,25 @@ After setting up the tools, set the following environment variables to store commonly used values. ```bash -export CLUSTER_NAME="${USER}-karpenter-demo" +export CLUSTER_NAME="karpenter-ex" export AWS_DEFAULT_REGION="us-east-1" ``` -The first thing we need to do is create our `main.tf` file and place the -following in it. This will let us pass in a cluster name that will be used -throughout the remainder of our config. +The first thing we need to do is create our `main.tf` file and place the following in it. ```hcl -variable "cluster_name" { - description = "The name of the cluster" - type = string +provider "aws" { + region = "us-east-1" } + +locals { + cluster_name = "karpenter-ex" + + # Used to determine correct partition (i.e. - `aws`, `aws-gov`, `aws-cn`, etc.) + partition = data.aws_partition.current.partition +} + +data "aws_partition" "current" {} ``` ### Create a Cluster @@ -66,10 +72,30 @@ that we need to tag the VPC subnets that we want to use for the worker nodes. Place the following Terraform config into your `main.tf` file. ```hcl +provider "aws" { + region = "us-east-1" +} + +locals { + # Change name to suite + cluster_name = "karpenter-ex" + + # Used to determine correct partition (i.e. - `aws`, `aws-gov`, `aws-cn`, etc.) + partition = data.aws_partition.current.partition +} + +data "aws_partition" "current" {} + +################################################################################ +# VPC +################################################################################ + module "vpc" { - source = "terraform-aws-modules/vpc/aws" + # https://registry.terraform.io/modules/terraform-aws-modules/vpc/aws/latest + source = "terraform-aws-modules/vpc/aws" + version = "3.12.0" - name = var.cluster_name + name = local.cluster_name cidr = "10.0.0.0/16" azs = ["us-east-1a", "us-east-1b", "us-east-1c"] @@ -81,56 +107,61 @@ module "vpc" { one_nat_gateway_per_az = false private_subnet_tags = { - "kubernetes.io/cluster/${var.cluster_name}" = "owned" - "karpenter.sh/discovery" = var.cluster_name + "kubernetes.io/cluster/${local.cluster_name}" = "owned" + "karpenter.sh/discovery" = local.cluster_name # for Karpenter auto-discovery } } +################################################################################ +# EKS Cluster +################################################################################ + module "eks" { - source = "terraform-aws-modules/eks/aws" - version = "<18" + # https://registry.terraform.io/modules/terraform-aws-modules/eks/aws/latest + source = "terraform-aws-modules/eks/aws" + version = "18.6.0" + cluster_name = local.cluster_name cluster_version = "1.21" - cluster_name = var.cluster_name - vpc_id = module.vpc.vpc_id - subnets = module.vpc.private_subnets - enable_irsa = true - - # Only need one node to get Karpenter up and running - worker_groups = [ - { - instance_type = "t3a.medium" - asg_max_size = 1 + + vpc_id = module.vpc.vpc_id + subnet_ids = module.vpc.private_subnets + + # Required for Karpenter role below + enable_irsa = true + + # Only need one node to get Karpenter up and running. + # This ensures core services such as VPC CNI, CoreDNS, etc. are up and running + # so that Karpetner can be deployed and start managing compute capacity as required + eks_managed_node_groups = { + initial = { + instance_types = ["t3.medium"] + + min_size = 1 + max_size = 1 + desired_size = 1 + + iam_role_additional_policies = [ + # Required by Karpenter + "arn:${local.partition}:iam::aws:policy/AmazonSSMManagedInstanceCore" + ] } - ] + } tags = { - "karpenter.sh/discovery" = var.cluster_name + "karpenter.sh/discovery" = local.cluster_name # for Karpenter auto-discovery } } +``` At this point, go ahead and apply what we've done to create the VPC and EKS cluster. This may take some time. ```bash terraform init -terraform apply -var "cluster_name=${CLUSTER_NAME}" -``` - -There's a good chance it will fail when trying to configure the aws-auth -ConfigMap. And that's because we need to use the kubeconfig file that was -generated during the cluster install. To use it, run the following. This will -configure both your local CLI and Terraform to use the file. Then try the apply -again. - -```bash -export KUBECONFIG="${PWD}/kubeconfig_${CLUSTER_NAME}" -export KUBE_CONFIG_PATH="${KUBECONFIG}" -terraform apply -var "cluster_name=${CLUSTER_NAME}" +terraform apply ``` -Everything should apply successfully now! - ### Create the EC2 Spot Service Linked Role This step is only necessary if this is the first time you're using EC2 Spot in this account. More details are available [here](https://docs.aws.amazon.com/batch/latest/userguide/spot_fleet_IAM_role.html). @@ -143,33 +174,23 @@ aws iam create-service-linked-role --aws-service-name spot.amazonaws.com ### Configure the KarpenterNode IAM Role -The EKS module creates an IAM role for worker nodes. We'll use that for +The EKS module creates an IAM role for the EKS managed node group nodes. We'll use that for Karpenter (so we don't have to reconfigure the aws-auth ConfigMap), but we need -to add one more policy and create an instance profile. +to create an instance profile we can reference. -Place the following into your `main.tf` to add the policy and create an -instance profile. +Place the following into your `main.tf` to create an instance profile. ```hcl -data "aws_iam_policy" "ssm_managed_instance" { - arn = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" -} - -resource "aws_iam_role_policy_attachment" "karpenter_ssm_policy" { - role = module.eks.worker_iam_role_name - policy_arn = data.aws_iam_policy.ssm_managed_instance.arn -} - resource "aws_iam_instance_profile" "karpenter" { - name = "KarpenterNodeInstanceProfile-${var.cluster_name}" - role = module.eks.worker_iam_role_name + name = "KarpenterNodeInstanceProfile-${local.cluster_name}" + role = module.eks.eks_managed_node_groups["initial"].iam_role_name } ``` Go ahead and apply the changes. ```bash -terraform apply -var "cluster_name=${CLUSTER_NAME}" +terraform apply ``` Now, Karpenter can use this instance profile to launch new EC2 instances and @@ -186,16 +207,17 @@ chart install. ```hcl module "iam_assumable_role_karpenter" { - source = "terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc" - version = "4.7.0" + source = "terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc" + version = "4.11.0" + create_role = true - role_name = "karpenter-controller-${var.cluster_name}" + role_name = "karpenter-controller-${local.cluster_name}" provider_url = module.eks.cluster_oidc_issuer_url oidc_fully_qualified_subjects = ["system:serviceaccount:karpenter:karpenter"] } resource "aws_iam_role_policy" "karpenter_controller" { - name = "karpenter-policy-${var.cluster_name}" + name = "karpenter-policy-${local.cluster_name}" role = module.iam_assumable_role_karpenter.iam_role_name policy = jsonencode({ @@ -205,12 +227,9 @@ resource "aws_iam_role_policy" "karpenter_controller" { Action = [ "ec2:CreateLaunchTemplate", "ec2:CreateFleet", - "ec2:RunInstances", "ec2:CreateTags", "iam:PassRole", - "ec2:TerminateInstances", "ec2:DescribeLaunchTemplates", - "ec2:DeleteLaunchTemplate", "ec2:DescribeInstances", "ec2:DescribeSecurityGroups", "ec2:DescribeSubnets", @@ -221,12 +240,26 @@ resource "aws_iam_role_policy" "karpenter_controller" { Effect = "Allow" Resource = "*" }, + { + Action = [ + "ec2:RunInstances", + "ec2:TerminateInstances", + "ec2:DeleteLaunchTemplate", + ] + Effect = "Allow" + Resource = "*" + Condition = { + "StringEquals" = { + "ec2:ResourceTag/karpenter.sh/discovery" = local.cluster_name + } + } + }, { Action = [ "ssm:GetParameter" ] Effect = "Allow" - Resource = "arn:aws:ssm:*:*:parameter/aws/service/*" + Resource = "arn:${local.partition}:ssm:*:*:parameter/aws/service/*" }, ] }) @@ -238,7 +271,7 @@ Then, apply the changes. ```bash terraform init -terraform apply -var "cluster_name=${CLUSTER_NAME}" +terraform apply ``` ### Install Karpenter Helm Chart @@ -248,15 +281,28 @@ Use helm to deploy Karpenter to the cluster. We are going to use the details and IAM role Karpenter needs to assume. ```hcl +provider "helm" { + kubernetes { + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) + + exec { + api_version = "client.authentication.k8s.io/v1alpha1" + command = "aws" + args = ["eks", "get-token", "--cluster-name", local.cluster_name] + } + } +} + resource "helm_release" "karpenter" { - depends_on = [module.eks.kubeconfig] namespace = "karpenter" create_namespace = true name = "karpenter" repository = "https://charts.karpenter.sh" chart = "karpenter" - version = "{{< param "latest_release_version" >}}" + # Be sure to pull latest version of chart + version = "0.6.1" set { name = "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn" @@ -264,12 +310,12 @@ resource "helm_release" "karpenter" { } set { - name = "clusterName" - value = var.cluster_name + name = "controller.clusterName" + value = module.eks.cluster_id } set { - name = "clusterEndpoint" + name = "controller.clusterEndpoint" value = module.eks.cluster_endpoint } @@ -284,7 +330,7 @@ Now, deploy Karpenter by applying the new Terraform config. ```bash terraform init -terraform apply -var "cluster_name=${CLUSTER_NAME}" +terraform apply ``` diff --git a/website/content/en/preview/getting-started/cloudformation.yaml b/website/content/en/preview/getting-started/cloudformation.yaml index 452bbf7dcc88..a0fea057baff 100644 --- a/website/content/en/preview/getting-started/cloudformation.yaml +++ b/website/content/en/preview/getting-started/cloudformation.yaml @@ -44,11 +44,8 @@ Resources: # Write Operations - ec2:CreateLaunchTemplate - ec2:CreateFleet - - ec2:RunInstances - ec2:CreateTags - iam:PassRole - - ec2:TerminateInstances - - ec2:DeleteLaunchTemplate # Read Operations - ec2:DescribeLaunchTemplates - ec2:DescribeInstances @@ -58,6 +55,15 @@ Resources: - ec2:DescribeInstanceTypeOfferings - ec2:DescribeAvailabilityZones - Effect: Allow - Resource: "arn:aws:ssm:*:*:parameter/aws/service/*" + Resource: "*" + Action: + - ec2:RunInstances + - ec2:TerminateInstances + - ec2:DeleteLaunchTemplate + Condition: + StringEquals: + "ec2:ResourceTag/karpenter.sh/discovery": !Ref ClusterName + - Effect: Allow + Resource: !Sub "arn:${AWS::Partition}:ssm:*:*:parameter/aws/service/*" Action: - ssm:GetParameter From ec47682442fad6592e374a4d92da2a374ae0987e Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Mon, 14 Feb 2022 14:19:22 -0500 Subject: [PATCH 03/17] chore: update doc wording --- .../getting-started-with-terraform/_index.md | 37 +++++-------------- 1 file changed, 9 insertions(+), 28 deletions(-) diff --git a/website/content/en/preview/getting-started-with-terraform/_index.md b/website/content/en/preview/getting-started-with-terraform/_index.md index 2d2fd880a485..67d7f5389277 100644 --- a/website/content/en/preview/getting-started-with-terraform/_index.md +++ b/website/content/en/preview/getting-started-with-terraform/_index.md @@ -69,27 +69,9 @@ We're going to use two different Terraform modules to create our cluster - one to create the VPC and another for the cluster itself. The key part of this is that we need to tag the VPC subnets that we want to use for the worker nodes. -Place the following Terraform config into your `main.tf` file. +Add the following to your `main.tf` to create a VPC and EKS cluster. ```hcl -provider "aws" { - region = "us-east-1" -} - -locals { - # Change name to suite - cluster_name = "karpenter-ex" - - # Used to determine correct partition (i.e. - `aws`, `aws-gov`, `aws-cn`, etc.) - partition = data.aws_partition.current.partition -} - -data "aws_partition" "current" {} - -################################################################################ -# VPC -################################################################################ - module "vpc" { # https://registry.terraform.io/modules/terraform-aws-modules/vpc/aws/latest source = "terraform-aws-modules/vpc/aws" @@ -112,10 +94,6 @@ module "vpc" { } } -################################################################################ -# EKS Cluster -################################################################################ - module "eks" { # https://registry.terraform.io/modules/terraform-aws-modules/eks/aws/latest source = "terraform-aws-modules/eks/aws" @@ -178,7 +156,7 @@ The EKS module creates an IAM role for the EKS managed node group nodes. We'll u Karpenter (so we don't have to reconfigure the aws-auth ConfigMap), but we need to create an instance profile we can reference. -Place the following into your `main.tf` to create an instance profile. +Add the following to your `main.tf` to create the instance profile. ```hcl resource "aws_iam_instance_profile" "karpenter" { @@ -205,6 +183,8 @@ using [IRSA](https://docs.aws.amazon.com/emr/latest/EMR-on-EKS-DevelopmentGuide/ We will create the ServiceAccount and connect it to this role during the Helm chart install. +Add the following to your `main.tf` to create the IAM role for the Karpenter service account. + ```hcl module "iam_assumable_role_karpenter" { source = "terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc" @@ -266,8 +246,7 @@ resource "aws_iam_role_policy" "karpenter_controller" { } ``` -Since we've added a new module, you'll need to run `terraform init` again. -Then, apply the changes. +Since we've added a new module, you'll need to run `terraform init` again before applying the changes. ```bash terraform init @@ -280,6 +259,8 @@ Use helm to deploy Karpenter to the cluster. We are going to use the `helm_release` Terraform resource to do the deploy and pass in the cluster details and IAM role Karpenter needs to assume. +Add the following to your `main.tf` to provision Karpenter via a Helm chart. + ```hcl provider "helm" { kubernetes { @@ -326,14 +307,14 @@ resource "helm_release" "karpenter" { } ``` -Now, deploy Karpenter by applying the new Terraform config. +Since we've added a new provider (helm), you'll need to run `terraform init` again +before applying the changes to deploy Karpenter. ```bash terraform init terraform apply ``` - ### Enable Debug Logging (optional) The global log level can be modified with the `logLevel` chart value (e.g. `--set logLevel=debug`) or the individual components can have their log level set with `controller.logLevel` or `webhook.logLevel` chart values. From d2f8a08f71130c6a5be306741261fe22a4a9b037 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Mon, 14 Feb 2022 14:43:40 -0500 Subject: [PATCH 04/17] chore: one last -var reference --- .../content/en/preview/getting-started-with-terraform/_index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/content/en/preview/getting-started-with-terraform/_index.md b/website/content/en/preview/getting-started-with-terraform/_index.md index 67d7f5389277..4ec4b72f2e7e 100644 --- a/website/content/en/preview/getting-started-with-terraform/_index.md +++ b/website/content/en/preview/getting-started-with-terraform/_index.md @@ -432,7 +432,7 @@ created LaunchTemplates. kubectl delete deployment inflate kubectl delete node -l karpenter.sh/provisioner-name=default helm uninstall karpenter --namespace karpenter -terraform destroy -var "cluster_name=${CLUSTER_NAME}" +terraform destroy aws ec2 describe-launch-templates \ | jq -r ".LaunchTemplates[].LaunchTemplateName" \ | grep -i "Karpenter-${CLUSTER_NAME}" \ From 9ee622449b49fc3d5b54b362ca66241497a56c8b Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Mon, 14 Feb 2022 15:59:38 -0500 Subject: [PATCH 05/17] feat: restrict `iam:PassRole` to only the Karpenter node role --- .../getting-started-with-terraform/_index.md | 20 ++++++++++++++++++- .../getting-started/cloudformation.yaml | 5 ++++- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/website/content/en/preview/getting-started-with-terraform/_index.md b/website/content/en/preview/getting-started-with-terraform/_index.md index 4ec4b72f2e7e..806094089d6e 100644 --- a/website/content/en/preview/getting-started-with-terraform/_index.md +++ b/website/content/en/preview/getting-started-with-terraform/_index.md @@ -208,7 +208,6 @@ resource "aws_iam_role_policy" "karpenter_controller" { "ec2:CreateLaunchTemplate", "ec2:CreateFleet", "ec2:CreateTags", - "iam:PassRole", "ec2:DescribeLaunchTemplates", "ec2:DescribeInstances", "ec2:DescribeSecurityGroups", @@ -234,6 +233,25 @@ resource "aws_iam_role_policy" "karpenter_controller" { } } }, + { + Action = [ + "iam:PassRole", + ] + Effect = "Allow" + Resource = module.eks.eks_managed_node_groups["initial"].iam_role_arn + }, + { + Action = [ + "iam:PassRole", + ] + Effect = "Allow" + Resource = "*" + Condition = { + "StringEquals" = { + "ec2:ResourceTag/karpenter.sh/discovery" = local.cluster_name + } + } + }, { Action = [ "ssm:GetParameter" diff --git a/website/content/en/preview/getting-started/cloudformation.yaml b/website/content/en/preview/getting-started/cloudformation.yaml index a0fea057baff..d915fb4488e9 100644 --- a/website/content/en/preview/getting-started/cloudformation.yaml +++ b/website/content/en/preview/getting-started/cloudformation.yaml @@ -45,7 +45,6 @@ Resources: - ec2:CreateLaunchTemplate - ec2:CreateFleet - ec2:CreateTags - - iam:PassRole # Read Operations - ec2:DescribeLaunchTemplates - ec2:DescribeInstances @@ -63,6 +62,10 @@ Resources: Condition: StringEquals: "ec2:ResourceTag/karpenter.sh/discovery": !Ref ClusterName + - Effect: Allow + Resource: !Ref KarpenterNodeRole + Action: + - iam:PassRole - Effect: Allow Resource: !Sub "arn:${AWS::Partition}:ssm:*:*:parameter/aws/service/*" Action: From 4ebf661cff8ebc7a4bee8cc7fd581d86b6e9231a Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Mon, 14 Feb 2022 16:01:48 -0500 Subject: [PATCH 06/17] fix: remove copy+paste cruft --- .../preview/getting-started-with-terraform/_index.md | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/website/content/en/preview/getting-started-with-terraform/_index.md b/website/content/en/preview/getting-started-with-terraform/_index.md index 806094089d6e..651a058d5667 100644 --- a/website/content/en/preview/getting-started-with-terraform/_index.md +++ b/website/content/en/preview/getting-started-with-terraform/_index.md @@ -240,18 +240,6 @@ resource "aws_iam_role_policy" "karpenter_controller" { Effect = "Allow" Resource = module.eks.eks_managed_node_groups["initial"].iam_role_arn }, - { - Action = [ - "iam:PassRole", - ] - Effect = "Allow" - Resource = "*" - Condition = { - "StringEquals" = { - "ec2:ResourceTag/karpenter.sh/discovery" = local.cluster_name - } - } - }, { Action = [ "ssm:GetParameter" From 684c7471a2d62212d558cbde623d719541bf7588 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Wed, 16 Feb 2022 15:31:21 -0500 Subject: [PATCH 07/17] chore: update to use new sub-module --- .../getting-started-with-terraform/_index.md | 91 +++++-------------- 1 file changed, 23 insertions(+), 68 deletions(-) diff --git a/website/content/en/preview/getting-started-with-terraform/_index.md b/website/content/en/preview/getting-started-with-terraform/_index.md index 651a058d5667..8bf8fe902712 100644 --- a/website/content/en/preview/getting-started-with-terraform/_index.md +++ b/website/content/en/preview/getting-started-with-terraform/_index.md @@ -97,7 +97,7 @@ module "vpc" { module "eks" { # https://registry.terraform.io/modules/terraform-aws-modules/eks/aws/latest source = "terraform-aws-modules/eks/aws" - version = "18.6.0" + version = "18.7.1" cluster_name = local.cluster_name cluster_version = "1.21" @@ -186,69 +186,24 @@ chart install. Add the following to your `main.tf` to create the IAM role for the Karpenter service account. ```hcl -module "iam_assumable_role_karpenter" { - source = "terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc" - version = "4.11.0" - - create_role = true - role_name = "karpenter-controller-${local.cluster_name}" - provider_url = module.eks.cluster_oidc_issuer_url - oidc_fully_qualified_subjects = ["system:serviceaccount:karpenter:karpenter"] -} - -resource "aws_iam_role_policy" "karpenter_controller" { - name = "karpenter-policy-${local.cluster_name}" - role = module.iam_assumable_role_karpenter.iam_role_name - - policy = jsonencode({ - Version = "2012-10-17" - Statement = [ - { - Action = [ - "ec2:CreateLaunchTemplate", - "ec2:CreateFleet", - "ec2:CreateTags", - "ec2:DescribeLaunchTemplates", - "ec2:DescribeInstances", - "ec2:DescribeSecurityGroups", - "ec2:DescribeSubnets", - "ec2:DescribeInstanceTypes", - "ec2:DescribeInstanceTypeOfferings", - "ec2:DescribeAvailabilityZones", - ] - Effect = "Allow" - Resource = "*" - }, - { - Action = [ - "ec2:RunInstances", - "ec2:TerminateInstances", - "ec2:DeleteLaunchTemplate", - ] - Effect = "Allow" - Resource = "*" - Condition = { - "StringEquals" = { - "ec2:ResourceTag/karpenter.sh/discovery" = local.cluster_name - } - } - }, - { - Action = [ - "iam:PassRole", - ] - Effect = "Allow" - Resource = module.eks.eks_managed_node_groups["initial"].iam_role_arn - }, - { - Action = [ - "ssm:GetParameter" - ] - Effect = "Allow" - Resource = "arn:${local.partition}:ssm:*:*:parameter/aws/service/*" - }, - ] - }) +module "karpenter_irsa" { + source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" + version = "4.12.0" + + role_name = "karpenter-controller-${local.cluster_name}" + attach_karpenter_controller_policy = true + + karpenter_controller_cluster_ids = [module.eks.cluster_id] + karpenter_controller_node_iam_role_arns = [ + module.eks.eks_managed_node_groups["initial"].iam_role_arn + ] + + oidc_providers = { + ex = { + provider_arn = module.eks.oidc_provider_arn + namespace_service_accounts = ["karpenter:karpenter"] + } + } } ``` @@ -289,20 +244,20 @@ resource "helm_release" "karpenter" { repository = "https://charts.karpenter.sh" chart = "karpenter" # Be sure to pull latest version of chart - version = "0.6.1" + version = "0.6.3" set { name = "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn" - value = module.iam_assumable_role_karpenter.iam_role_arn + value = module.karpenter_irsa.iam_role_arn } set { - name = "controller.clusterName" + name = "clusterName" value = module.eks.cluster_id } set { - name = "controller.clusterEndpoint" + name = "clusterEndpoint" value = module.eks.cluster_endpoint } From 36d1b418370dcf7e5f9b97d54d12a0a8fa6e106e Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Fri, 18 Mar 2022 09:35:07 -0400 Subject: [PATCH 08/17] chore: re-update and validate --- .../getting-started-with-terraform/_index.md | 207 +++++++++--------- 1 file changed, 100 insertions(+), 107 deletions(-) diff --git a/website/content/en/preview/getting-started/getting-started-with-terraform/_index.md b/website/content/en/preview/getting-started/getting-started-with-terraform/_index.md index 4216c2cc6ff7..e66abd30a05a 100644 --- a/website/content/en/preview/getting-started/getting-started-with-terraform/_index.md +++ b/website/content/en/preview/getting-started/getting-started-with-terraform/_index.md @@ -42,19 +42,25 @@ After setting up the tools, set the following environment variables to store commonly used values. ```bash -export CLUSTER_NAME="${USER}-karpenter-demo" +export CLUSTER_NAME="karpenter-ex" export AWS_DEFAULT_REGION="us-east-1" ``` -The first thing we need to do is create our `main.tf` file and place the -following in it. This will let us pass in a cluster name that will be used -throughout the remainder of our config. +The first thing we need to do is create our `main.tf` file and place the following in it. ```hcl -variable "cluster_name" { - description = "The name of the cluster" - type = string +provider "aws" { + region = "us-east-1" } + +locals { + cluster_name = "karpenter-ex" + + # Used to determine correct partition (i.e. - `aws`, `aws-gov`, `aws-cn`, etc.) + partition = data.aws_partition.current.partition +} + +data "aws_partition" "current" {} ``` ### Create a Cluster @@ -63,13 +69,15 @@ We're going to use two different Terraform modules to create our cluster - one to create the VPC and another for the cluster itself. The key part of this is that we need to tag the VPC subnets that we want to use for the worker nodes. -Place the following Terraform config into your `main.tf` file. +Add the following to your `main.tf` to create a VPC and EKS cluster. ```hcl module "vpc" { - source = "terraform-aws-modules/vpc/aws" + # https://registry.terraform.io/modules/terraform-aws-modules/vpc/aws/latest + source = "terraform-aws-modules/vpc/aws" + version = "3.12.0" - name = var.cluster_name + name = local.cluster_name cidr = "10.0.0.0/16" azs = ["us-east-1a", "us-east-1b", "us-east-1c"] @@ -81,31 +89,45 @@ module "vpc" { one_nat_gateway_per_az = false private_subnet_tags = { - "kubernetes.io/cluster/${var.cluster_name}" = "owned" - "karpenter.sh/discovery" = var.cluster_name + "kubernetes.io/cluster/${local.cluster_name}" = "owned" + "karpenter.sh/discovery" = local.cluster_name # for Karpenter auto-discovery } } module "eks" { - source = "terraform-aws-modules/eks/aws" - version = "<18" + # https://registry.terraform.io/modules/terraform-aws-modules/eks/aws/latest + source = "terraform-aws-modules/eks/aws" + version = "18.11.0" + cluster_name = local.cluster_name cluster_version = "1.21" - cluster_name = var.cluster_name - vpc_id = module.vpc.vpc_id - subnets = module.vpc.private_subnets - enable_irsa = true - - # Only need one node to get Karpenter up and running - worker_groups = [ - { - instance_type = "t3a.medium" - asg_max_size = 1 + + vpc_id = module.vpc.vpc_id + subnet_ids = module.vpc.private_subnets + + # Required for Karpenter role below + enable_irsa = true + + # Only need one node to get Karpenter up and running. + # This ensures core services such as VPC CNI, CoreDNS, etc. are up and running + # so that Karpetner can be deployed and start managing compute capacity as required + eks_managed_node_groups = { + initial = { + instance_types = ["t3.medium"] + + min_size = 1 + max_size = 1 + desired_size = 1 + + iam_role_additional_policies = [ + # Required by Karpenter + "arn:${local.partition}:iam::aws:policy/AmazonSSMManagedInstanceCore" + ] } - ] + } tags = { - "karpenter.sh/discovery" = var.cluster_name + "karpenter.sh/discovery" = local.cluster_name # for Karpenter auto-discovery } } ``` @@ -115,23 +137,9 @@ EKS cluster. This may take some time. ```bash terraform init -terraform apply -var "cluster_name=${CLUSTER_NAME}" -``` - -There's a good chance it will fail when trying to configure the aws-auth -ConfigMap. And that's because we need to use the kubeconfig file that was -generated during the cluster install. To use it, run the following. This will -configure both your local CLI and Terraform to use the file. Then try the apply -again. - -```bash -export KUBECONFIG="${PWD}/kubeconfig_${CLUSTER_NAME}" -export KUBE_CONFIG_PATH="${KUBECONFIG}" -terraform apply -var "cluster_name=${CLUSTER_NAME}" +terraform apply ``` -Everything should apply successfully now! - ### Create the EC2 Spot Service Linked Role This step is only necessary if this is the first time you're using EC2 Spot in this account. More details are available [here](https://docs.aws.amazon.com/batch/latest/userguide/spot_fleet_IAM_role.html). @@ -144,33 +152,23 @@ aws iam create-service-linked-role --aws-service-name spot.amazonaws.com ### Configure the KarpenterNode IAM Role -The EKS module creates an IAM role for worker nodes. We'll use that for +The EKS module creates an IAM role for the EKS managed node group nodes. We'll use that for Karpenter (so we don't have to reconfigure the aws-auth ConfigMap), but we need -to add one more policy and create an instance profile. +to create an instance profile we can reference. -Place the following into your `main.tf` to add the policy and create an -instance profile. +Add the following to your `main.tf` to create the instance profile. ```hcl -data "aws_iam_policy" "ssm_managed_instance" { - arn = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" -} - -resource "aws_iam_role_policy_attachment" "karpenter_ssm_policy" { - role = module.eks.worker_iam_role_name - policy_arn = data.aws_iam_policy.ssm_managed_instance.arn -} - resource "aws_iam_instance_profile" "karpenter" { - name = "KarpenterNodeInstanceProfile-${var.cluster_name}" - role = module.eks.worker_iam_role_name + name = "KarpenterNodeInstanceProfile-${local.cluster_name}" + role = module.eks.eks_managed_node_groups["initial"].iam_role_name } ``` Go ahead and apply the changes. ```bash -terraform apply -var "cluster_name=${CLUSTER_NAME}" +terraform apply ``` Now, Karpenter can use this instance profile to launch new EC2 instances and @@ -185,55 +183,35 @@ using [IRSA](https://docs.aws.amazon.com/emr/latest/EMR-on-EKS-DevelopmentGuide/ We will create the ServiceAccount and connect it to this role during the Helm chart install. +Add the following to your `main.tf` to create the IAM role for the Karpenter service account. + ```hcl -module "iam_assumable_role_karpenter" { - source = "terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc" - version = "4.7.0" - create_role = true - role_name = "karpenter-controller-${var.cluster_name}" - provider_url = module.eks.cluster_oidc_issuer_url - oidc_fully_qualified_subjects = ["system:serviceaccount:karpenter:karpenter"] -} +module "karpenter_irsa" { + source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" + version = "4.14.0" + + role_name = "karpenter-controller-${local.cluster_name}" + attach_karpenter_controller_policy = true + + karpenter_controller_cluster_ids = [module.eks.cluster_id] + karpenter_controller_node_iam_role_arns = [ + module.eks.eks_managed_node_groups["initial"].iam_role_arn + ] -resource "aws_iam_role_policy" "karpenter_controller" { - name = "karpenter-policy-${var.cluster_name}" - role = module.iam_assumable_role_karpenter.iam_role_name - - policy = jsonencode({ - Version = "2012-10-17" - Statement = [ - { - Action = [ - "ec2:CreateLaunchTemplate", - "ec2:CreateFleet", - "ec2:RunInstances", - "ec2:CreateTags", - "iam:PassRole", - "ec2:TerminateInstances", - "ec2:DescribeLaunchTemplates", - "ec2:DeleteLaunchTemplate", - "ec2:DescribeInstances", - "ec2:DescribeSecurityGroups", - "ec2:DescribeSubnets", - "ec2:DescribeInstanceTypes", - "ec2:DescribeInstanceTypeOfferings", - "ec2:DescribeAvailabilityZones", - "ssm:GetParameter" - ] - Effect = "Allow" - Resource = "*" - }, - ] - }) + oidc_providers = { + ex = { + provider_arn = module.eks.oidc_provider_arn + namespace_service_accounts = ["karpenter:karpenter"] + } + } } ``` -Since we've added a new module, you'll need to run `terraform init` again. -Then, apply the changes. +Since we've added a new module, you'll need to run `terraform init` again before applying the changes. ```bash terraform init -terraform apply -var "cluster_name=${CLUSTER_NAME}" +terraform apply ``` ### Install Karpenter Helm Chart @@ -242,25 +220,40 @@ Use helm to deploy Karpenter to the cluster. We are going to use the `helm_release` Terraform resource to do the deploy and pass in the cluster details and IAM role Karpenter needs to assume. +Add the following to your `main.tf` to provision Karpenter via a Helm chart. + ```hcl +provider "helm" { + kubernetes { + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) + + exec { + api_version = "client.authentication.k8s.io/v1alpha1" + command = "aws" + args = ["eks", "get-token", "--cluster-name", local.cluster_name] + } + } +} + resource "helm_release" "karpenter" { - depends_on = [module.eks.kubeconfig] namespace = "karpenter" create_namespace = true name = "karpenter" repository = "https://charts.karpenter.sh" chart = "karpenter" - version = "{{< param "latest_release_version" >}}" + # Be sure to pull latest version of chart + version = "0.7.1" set { name = "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn" - value = module.iam_assumable_role_karpenter.iam_role_arn + value = module.karpenter_irsa.iam_role_arn } set { name = "clusterName" - value = var.cluster_name + value = module.eks.cluster_id } set { @@ -275,14 +268,14 @@ resource "helm_release" "karpenter" { } ``` -Now, deploy Karpenter by applying the new Terraform config. +Since we've added a new provider (helm), you'll need to run `terraform init` again +before applying the changes to deploy Karpenter. ```bash terraform init -terraform apply -var "cluster_name=${CLUSTER_NAME}" +terraform apply ``` - ### Enable Debug Logging (optional) The global log level can be modified with the `logLevel` chart value (e.g. `--set logLevel=debug`) or the individual components can have their log level set with `controller.logLevel` or `webhook.logLevel` chart values. @@ -301,7 +294,7 @@ resources like subnets and security groups using the cluster's name. The `ttlSecondsAfterEmpty` value configures Karpenter to terminate empty nodes. This behavior can be disabled by leaving the value undefined. -Review the [provisioner CRD]({{}}) for more information. For example, +Review the [provisioner CRD](../provisioner) for more information. For example, `ttlSecondsUntilExpired` configures Karpenter to terminate nodes when a maximum age is reached. Note: This provisioner will create capacity as long as the sum of all created capacity is less than the specified limit. @@ -400,7 +393,7 @@ created LaunchTemplates. kubectl delete deployment inflate kubectl delete node -l karpenter.sh/provisioner-name=default helm uninstall karpenter --namespace karpenter -terraform destroy -var "cluster_name=${CLUSTER_NAME}" +terraform destroy aws ec2 describe-launch-templates \ | jq -r ".LaunchTemplates[].LaunchTemplateName" \ | grep -i "Karpenter-${CLUSTER_NAME}" \ From f69fb32224ea505126258b10c8821372ee75a0c1 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Fri, 18 Mar 2022 12:41:47 -0400 Subject: [PATCH 09/17] chore: remove cloudformation/eksctl changes and v0.6.4 changes --- .../getting-started-with-terraform/_index.md | 204 +++++++++--------- .../cloudformation.yaml | 20 +- 2 files changed, 110 insertions(+), 114 deletions(-) diff --git a/website/content/en/v0.6.4/getting-started-with-terraform/_index.md b/website/content/en/v0.6.4/getting-started-with-terraform/_index.md index 3992f4a5a25c..c2ac7275e411 100644 --- a/website/content/en/v0.6.4/getting-started-with-terraform/_index.md +++ b/website/content/en/v0.6.4/getting-started-with-terraform/_index.md @@ -18,7 +18,7 @@ Follow the clean-up instructions to reduce any charges. ## Install -Karpenter is installed in clusters via a helm chart. +Karpenter is installed in clusters with a helm chart. Karpenter additionally requires IAM Roles for Service Accounts (IRSA). IRSA permits Karpenter (within the cluster) to make privileged requests to AWS (as @@ -42,25 +42,19 @@ After setting up the tools, set the following environment variables to store commonly used values. ```bash -export CLUSTER_NAME="karpenter-ex" +export CLUSTER_NAME="${USER}-karpenter-demo" export AWS_DEFAULT_REGION="us-east-1" ``` -The first thing we need to do is create our `main.tf` file and place the following in it. +The first thing we need to do is create our `main.tf` file and place the +following in it. This will let us pass in a cluster name that will be used +throughout the remainder of our config. ```hcl -provider "aws" { - region = "us-east-1" +variable "cluster_name" { + description = "The name of the cluster" + type = string } - -locals { - cluster_name = "karpenter-ex" - - # Used to determine correct partition (i.e. - `aws`, `aws-gov`, `aws-cn`, etc.) - partition = data.aws_partition.current.partition -} - -data "aws_partition" "current" {} ``` ### Create a Cluster @@ -69,15 +63,13 @@ We're going to use two different Terraform modules to create our cluster - one to create the VPC and another for the cluster itself. The key part of this is that we need to tag the VPC subnets that we want to use for the worker nodes. -Add the following to your `main.tf` to create a VPC and EKS cluster. +Place the following Terraform config into your `main.tf` file. ```hcl module "vpc" { - # https://registry.terraform.io/modules/terraform-aws-modules/vpc/aws/latest - source = "terraform-aws-modules/vpc/aws" - version = "3.12.0" + source = "terraform-aws-modules/vpc/aws" - name = local.cluster_name + name = var.cluster_name cidr = "10.0.0.0/16" azs = ["us-east-1a", "us-east-1b", "us-east-1c"] @@ -89,45 +81,31 @@ module "vpc" { one_nat_gateway_per_az = false private_subnet_tags = { - "kubernetes.io/cluster/${local.cluster_name}" = "owned" - "karpenter.sh/discovery" = local.cluster_name # for Karpenter auto-discovery + "kubernetes.io/cluster/${var.cluster_name}" = "owned" + "karpenter.sh/discovery" = var.cluster_name } } module "eks" { - # https://registry.terraform.io/modules/terraform-aws-modules/eks/aws/latest - source = "terraform-aws-modules/eks/aws" - version = "18.7.1" + source = "terraform-aws-modules/eks/aws" + version = "<18" - cluster_name = local.cluster_name cluster_version = "1.21" - - vpc_id = module.vpc.vpc_id - subnet_ids = module.vpc.private_subnets - - # Required for Karpenter role below - enable_irsa = true - - # Only need one node to get Karpenter up and running. - # This ensures core services such as VPC CNI, CoreDNS, etc. are up and running - # so that Karpetner can be deployed and start managing compute capacity as required - eks_managed_node_groups = { - initial = { - instance_types = ["t3.medium"] - - min_size = 1 - max_size = 1 - desired_size = 1 - - iam_role_additional_policies = [ - # Required by Karpenter - "arn:${local.partition}:iam::aws:policy/AmazonSSMManagedInstanceCore" - ] + cluster_name = var.cluster_name + vpc_id = module.vpc.vpc_id + subnets = module.vpc.private_subnets + enable_irsa = true + + # Only need one node to get Karpenter up and running + worker_groups = [ + { + instance_type = "t3a.medium" + asg_max_size = 1 } - } + ] tags = { - "karpenter.sh/discovery" = local.cluster_name # for Karpenter auto-discovery + "karpenter.sh/discovery" = var.cluster_name } } ``` @@ -137,9 +115,23 @@ EKS cluster. This may take some time. ```bash terraform init -terraform apply +terraform apply -var "cluster_name=${CLUSTER_NAME}" +``` + +There's a good chance it will fail when trying to configure the aws-auth +ConfigMap. And that's because we need to use the kubeconfig file that was +generated during the cluster install. To use it, run the following. This will +configure both your local CLI and Terraform to use the file. Then try the apply +again. + +```bash +export KUBECONFIG="${PWD}/kubeconfig_${CLUSTER_NAME}" +export KUBE_CONFIG_PATH="${KUBECONFIG}" +terraform apply -var "cluster_name=${CLUSTER_NAME}" ``` +Everything should apply successfully now! + ### Create the EC2 Spot Service Linked Role This step is only necessary if this is the first time you're using EC2 Spot in this account. More details are available [here](https://docs.aws.amazon.com/batch/latest/userguide/spot_fleet_IAM_role.html). @@ -152,23 +144,33 @@ aws iam create-service-linked-role --aws-service-name spot.amazonaws.com ### Configure the KarpenterNode IAM Role -The EKS module creates an IAM role for the EKS managed node group nodes. We'll use that for +The EKS module creates an IAM role for worker nodes. We'll use that for Karpenter (so we don't have to reconfigure the aws-auth ConfigMap), but we need -to create an instance profile we can reference. +to add one more policy and create an instance profile. -Add the following to your `main.tf` to create the instance profile. +Place the following into your `main.tf` to add the policy and create an +instance profile. ```hcl +data "aws_iam_policy" "ssm_managed_instance" { + arn = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" +} + +resource "aws_iam_role_policy_attachment" "karpenter_ssm_policy" { + role = module.eks.worker_iam_role_name + policy_arn = data.aws_iam_policy.ssm_managed_instance.arn +} + resource "aws_iam_instance_profile" "karpenter" { - name = "KarpenterNodeInstanceProfile-${local.cluster_name}" - role = module.eks.eks_managed_node_groups["initial"].iam_role_name + name = "KarpenterNodeInstanceProfile-${var.cluster_name}" + role = module.eks.worker_iam_role_name } ``` Go ahead and apply the changes. ```bash -terraform apply +terraform apply -var "cluster_name=${CLUSTER_NAME}" ``` Now, Karpenter can use this instance profile to launch new EC2 instances and @@ -183,35 +185,55 @@ using [IRSA](https://docs.aws.amazon.com/emr/latest/EMR-on-EKS-DevelopmentGuide/ We will create the ServiceAccount and connect it to this role during the Helm chart install. -Add the following to your `main.tf` to create the IAM role for the Karpenter service account. - ```hcl -module "karpenter_irsa" { - source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" - version = "4.12.0" - - role_name = "karpenter-controller-${local.cluster_name}" - attach_karpenter_controller_policy = true - - karpenter_controller_cluster_ids = [module.eks.cluster_id] - karpenter_controller_node_iam_role_arns = [ - module.eks.eks_managed_node_groups["initial"].iam_role_arn - ] +module "iam_assumable_role_karpenter" { + source = "terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc" + version = "4.7.0" + create_role = true + role_name = "karpenter-controller-${var.cluster_name}" + provider_url = module.eks.cluster_oidc_issuer_url + oidc_fully_qualified_subjects = ["system:serviceaccount:karpenter:karpenter"] +} - oidc_providers = { - ex = { - provider_arn = module.eks.oidc_provider_arn - namespace_service_accounts = ["karpenter:karpenter"] - } - } +resource "aws_iam_role_policy" "karpenter_controller" { + name = "karpenter-policy-${var.cluster_name}" + role = module.iam_assumable_role_karpenter.iam_role_name + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = [ + "ec2:CreateLaunchTemplate", + "ec2:CreateFleet", + "ec2:RunInstances", + "ec2:CreateTags", + "iam:PassRole", + "ec2:TerminateInstances", + "ec2:DescribeLaunchTemplates", + "ec2:DeleteLaunchTemplate", + "ec2:DescribeInstances", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSubnets", + "ec2:DescribeInstanceTypes", + "ec2:DescribeInstanceTypeOfferings", + "ec2:DescribeAvailabilityZones", + "ssm:GetParameter" + ] + Effect = "Allow" + Resource = "*" + }, + ] + }) } ``` -Since we've added a new module, you'll need to run `terraform init` again before applying the changes. +Since we've added a new module, you'll need to run `terraform init` again. +Then, apply the changes. ```bash terraform init -terraform apply +terraform apply -var "cluster_name=${CLUSTER_NAME}" ``` ### Install Karpenter Helm Chart @@ -220,23 +242,9 @@ Use helm to deploy Karpenter to the cluster. We are going to use the `helm_release` Terraform resource to do the deploy and pass in the cluster details and IAM role Karpenter needs to assume. -Add the following to your `main.tf` to provision Karpenter via a Helm chart. - ```hcl -provider "helm" { - kubernetes { - host = module.eks.cluster_endpoint - cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) - - exec { - api_version = "client.authentication.k8s.io/v1alpha1" - command = "aws" - args = ["eks", "get-token", "--cluster-name", local.cluster_name] - } - } -} - resource "helm_release" "karpenter" { + depends_on = [module.eks.kubeconfig] namespace = "karpenter" create_namespace = true @@ -247,12 +255,12 @@ resource "helm_release" "karpenter" { set { name = "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn" - value = module.karpenter_irsa.iam_role_arn + value = module.iam_assumable_role_karpenter.iam_role_arn } set { name = "clusterName" - value = module.eks.cluster_id + value = var.cluster_name } set { @@ -267,14 +275,14 @@ resource "helm_release" "karpenter" { } ``` -Since we've added a new provider (helm), you'll need to run `terraform init` again -before applying the changes to deploy Karpenter. +Now, deploy Karpenter by applying the new Terraform config. ```bash terraform init -terraform apply +terraform apply -var "cluster_name=${CLUSTER_NAME}" ``` + ### Enable Debug Logging (optional) The global log level can be modified with the `logLevel` chart value (e.g. `--set logLevel=debug`) or the individual components can have their log level set with `controller.logLevel` or `webhook.logLevel` chart values. @@ -392,7 +400,7 @@ created LaunchTemplates. kubectl delete deployment inflate kubectl delete node -l karpenter.sh/provisioner-name=default helm uninstall karpenter --namespace karpenter -terraform destroy +terraform destroy -var "cluster_name=${CLUSTER_NAME}" aws ec2 describe-launch-templates \ | jq -r ".LaunchTemplates[].LaunchTemplateName" \ | grep -i "Karpenter-${CLUSTER_NAME}" \ diff --git a/website/content/en/v0.7.0/getting-started/getting-started-with-eksctl/cloudformation.yaml b/website/content/en/v0.7.0/getting-started/getting-started-with-eksctl/cloudformation.yaml index d915fb4488e9..74af86867cc6 100644 --- a/website/content/en/v0.7.0/getting-started/getting-started-with-eksctl/cloudformation.yaml +++ b/website/content/en/v0.7.0/getting-started/getting-started-with-eksctl/cloudformation.yaml @@ -44,7 +44,11 @@ Resources: # Write Operations - ec2:CreateLaunchTemplate - ec2:CreateFleet + - ec2:RunInstances - ec2:CreateTags + - iam:PassRole + - ec2:TerminateInstances + - ec2:DeleteLaunchTemplate # Read Operations - ec2:DescribeLaunchTemplates - ec2:DescribeInstances @@ -53,20 +57,4 @@ Resources: - ec2:DescribeInstanceTypes - ec2:DescribeInstanceTypeOfferings - ec2:DescribeAvailabilityZones - - Effect: Allow - Resource: "*" - Action: - - ec2:RunInstances - - ec2:TerminateInstances - - ec2:DeleteLaunchTemplate - Condition: - StringEquals: - "ec2:ResourceTag/karpenter.sh/discovery": !Ref ClusterName - - Effect: Allow - Resource: !Ref KarpenterNodeRole - Action: - - iam:PassRole - - Effect: Allow - Resource: !Sub "arn:${AWS::Partition}:ssm:*:*:parameter/aws/service/*" - Action: - ssm:GetParameter From e69ef7a85512017dbc377ce8b3a74afe2c0c288b Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Fri, 18 Mar 2022 13:12:05 -0400 Subject: [PATCH 10/17] chore: align cluster name with other examples --- .../getting-started-with-terraform/_index.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/website/content/en/preview/getting-started/getting-started-with-terraform/_index.md b/website/content/en/preview/getting-started/getting-started-with-terraform/_index.md index e66abd30a05a..a9fbcf235d52 100644 --- a/website/content/en/preview/getting-started/getting-started-with-terraform/_index.md +++ b/website/content/en/preview/getting-started/getting-started-with-terraform/_index.md @@ -42,7 +42,7 @@ After setting up the tools, set the following environment variables to store commonly used values. ```bash -export CLUSTER_NAME="karpenter-ex" +export CLUSTER_NAME="karpenter-demo" export AWS_DEFAULT_REGION="us-east-1" ``` @@ -54,7 +54,7 @@ provider "aws" { } locals { - cluster_name = "karpenter-ex" + cluster_name = "karpenter-demo" # Used to determine correct partition (i.e. - `aws`, `aws-gov`, `aws-cn`, etc.) partition = data.aws_partition.current.partition @@ -243,8 +243,7 @@ resource "helm_release" "karpenter" { name = "karpenter" repository = "https://charts.karpenter.sh" chart = "karpenter" - # Be sure to pull latest version of chart - version = "0.7.1" + version = "{{< param "latest_release_version" >}}" set { name = "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn" @@ -294,7 +293,7 @@ resources like subnets and security groups using the cluster's name. The `ttlSecondsAfterEmpty` value configures Karpenter to terminate empty nodes. This behavior can be disabled by leaving the value undefined. -Review the [provisioner CRD](../provisioner) for more information. For example, +Review the [provisioner CRD]({{}}) for more information. For example, `ttlSecondsUntilExpired` configures Karpenter to terminate nodes when a maximum age is reached. Note: This provisioner will create capacity as long as the sum of all created capacity is less than the specified limit. From c1500af9352cab177220232a05d5603f2f49dc49 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Wed, 23 Mar 2022 17:52:05 -0400 Subject: [PATCH 11/17] chore: updates from testing --- .../getting-started-with-terraform/_index.md | 36 +++++++++++++++---- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/website/content/en/preview/getting-started/getting-started-with-terraform/_index.md b/website/content/en/preview/getting-started/getting-started-with-terraform/_index.md index a9fbcf235d52..78d578350a97 100644 --- a/website/content/en/preview/getting-started/getting-started-with-terraform/_index.md +++ b/website/content/en/preview/getting-started/getting-started-with-terraform/_index.md @@ -90,14 +90,15 @@ module "vpc" { private_subnet_tags = { "kubernetes.io/cluster/${local.cluster_name}" = "owned" - "karpenter.sh/discovery" = local.cluster_name # for Karpenter auto-discovery + # Tags subnets for Karpenter auto-discovery + "karpenter.sh/discovery" = local.cluster_name } } module "eks" { # https://registry.terraform.io/modules/terraform-aws-modules/eks/aws/latest source = "terraform-aws-modules/eks/aws" - version = "18.11.0" + version = "18.13.0" cluster_name = local.cluster_name cluster_version = "1.21" @@ -108,12 +109,21 @@ module "eks" { # Required for Karpenter role below enable_irsa = true + # We will rely only on the cluster security group created by the EKS service + # See note below for `tags` + create_cluster_security_group = false + create_node_security_group = false + # Only need one node to get Karpenter up and running. # This ensures core services such as VPC CNI, CoreDNS, etc. are up and running # so that Karpetner can be deployed and start managing compute capacity as required eks_managed_node_groups = { initial = { instance_types = ["t3.medium"] + # We don't need the node security group since we are using the + # cluster created security group which Karpenter will also use + create_security_group = false + attach_cluster_primary_security_group = true min_size = 1 max_size = 1 @@ -123,12 +133,22 @@ module "eks" { # Required by Karpenter "arn:${local.partition}:iam::aws:policy/AmazonSSMManagedInstanceCore" ] + + tags = { + # Tag node group resources for Karpenter auto-discovery + "karpenter.sh/discovery" = local.cluster_name + } } } +} - tags = { - "karpenter.sh/discovery" = local.cluster_name # for Karpenter auto-discovery - } +resource "aws_ec2_tag" "cluster_primary_security_group" { + # Tag cluster created security group for Karpenter auto-discovery + # NOTE - if creating multiple security groups with this module, only tag the + # security group that Karpenter should utilize with the following tag + resource_id = module.eks.cluster_primary_security_group_id + key = "karpenter.sh/discovery" + value = local.cluster_name } ``` @@ -188,12 +208,12 @@ Add the following to your `main.tf` to create the IAM role for the Karpenter ser ```hcl module "karpenter_irsa" { source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" - version = "4.14.0" + version = "4.15.1" role_name = "karpenter-controller-${local.cluster_name}" attach_karpenter_controller_policy = true - karpenter_controller_cluster_ids = [module.eks.cluster_id] + karpenter_controller_cluster_id = module.eks.cluster_id karpenter_controller_node_iam_role_arns = [ module.eks.eks_managed_node_groups["initial"].iam_role_arn ] @@ -317,6 +337,8 @@ spec: karpenter.sh/discovery: ${CLUSTER_NAME} securityGroupSelector: karpenter.sh/discovery: ${CLUSTER_NAME} + tags: + karpenter.sh/discovery: ${CLUSTER_NAME} ttlSecondsAfterEmpty: 30 EOF ``` From 5f1f22e26be660bb17e98158aa498ef30e51d23a Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Thu, 24 Mar 2022 09:25:05 -0400 Subject: [PATCH 12/17] chore: final update with latest module changes incorporated for Karpenter --- .../getting-started-with-terraform/_index.md | 22 ++++++------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/website/content/en/preview/getting-started/getting-started-with-terraform/_index.md b/website/content/en/preview/getting-started/getting-started-with-terraform/_index.md index 78d578350a97..bd98857ab895 100644 --- a/website/content/en/preview/getting-started/getting-started-with-terraform/_index.md +++ b/website/content/en/preview/getting-started/getting-started-with-terraform/_index.md @@ -98,7 +98,7 @@ module "vpc" { module "eks" { # https://registry.terraform.io/modules/terraform-aws-modules/eks/aws/latest source = "terraform-aws-modules/eks/aws" - version = "18.13.0" + version = "18.14.0" cluster_name = local.cluster_name cluster_version = "1.21" @@ -133,22 +133,15 @@ module "eks" { # Required by Karpenter "arn:${local.partition}:iam::aws:policy/AmazonSSMManagedInstanceCore" ] - - tags = { - # Tag node group resources for Karpenter auto-discovery - "karpenter.sh/discovery" = local.cluster_name - } } } -} -resource "aws_ec2_tag" "cluster_primary_security_group" { - # Tag cluster created security group for Karpenter auto-discovery - # NOTE - if creating multiple security groups with this module, only tag the - # security group that Karpenter should utilize with the following tag - resource_id = module.eks.cluster_primary_security_group_id - key = "karpenter.sh/discovery" - value = local.cluster_name + tags = { + # Tag node group resources for Karpenter auto-discovery + # NOTE - if creating multiple security groups with this module, only tag the + # security group that Karpenter should utilize with the following tag + "karpenter.sh/discovery" = local.cluster_name + } } ``` @@ -413,7 +406,6 @@ created LaunchTemplates. ```bash kubectl delete deployment inflate kubectl delete node -l karpenter.sh/provisioner-name=default -helm uninstall karpenter --namespace karpenter terraform destroy aws ec2 describe-launch-templates \ | jq -r ".LaunchTemplates[].LaunchTemplateName" \ From 3b53b8341cd1987b96e85a7a46c3f133e94f7aaf Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Tue, 29 Mar 2022 14:46:50 -0400 Subject: [PATCH 13/17] chore: update terraform modules to current latest --- .../getting-started/getting-started-with-terraform/_index.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/website/content/en/preview/getting-started/getting-started-with-terraform/_index.md b/website/content/en/preview/getting-started/getting-started-with-terraform/_index.md index bd98857ab895..38fd6d4bc639 100644 --- a/website/content/en/preview/getting-started/getting-started-with-terraform/_index.md +++ b/website/content/en/preview/getting-started/getting-started-with-terraform/_index.md @@ -98,7 +98,7 @@ module "vpc" { module "eks" { # https://registry.terraform.io/modules/terraform-aws-modules/eks/aws/latest source = "terraform-aws-modules/eks/aws" - version = "18.14.0" + version = "18.16.0" cluster_name = local.cluster_name cluster_version = "1.21" @@ -201,7 +201,7 @@ Add the following to your `main.tf` to create the IAM role for the Karpenter ser ```hcl module "karpenter_irsa" { source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" - version = "4.15.1" + version = "4.17.1" role_name = "karpenter-controller-${local.cluster_name}" attach_karpenter_controller_policy = true From b060ee50801db9243a416ce8261723f6d92717b9 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Wed, 30 Mar 2022 09:18:58 -0400 Subject: [PATCH 14/17] feat: create Karpenter provisioner using Terraform+kubectl --- .../getting-started-with-terraform/_index.md | 76 +++++++++++++------ 1 file changed, 52 insertions(+), 24 deletions(-) diff --git a/website/content/en/preview/getting-started/getting-started-with-terraform/_index.md b/website/content/en/preview/getting-started/getting-started-with-terraform/_index.md index 38fd6d4bc639..1c3ada3488c6 100644 --- a/website/content/en/preview/getting-started/getting-started-with-terraform/_index.md +++ b/website/content/en/preview/getting-started/getting-started-with-terraform/_index.md @@ -42,7 +42,6 @@ After setting up the tools, set the following environment variables to store commonly used values. ```bash -export CLUSTER_NAME="karpenter-demo" export AWS_DEFAULT_REGION="us-east-1" ``` @@ -309,31 +308,60 @@ This behavior can be disabled by leaving the value undefined. Review the [provisioner CRD]({{}}) for more information. For example, `ttlSecondsUntilExpired` configures Karpenter to terminate nodes when a maximum age is reached. +Add the following to your `main.tf` to deploy the Karpenter provisioner. + Note: This provisioner will create capacity as long as the sum of all created capacity is less than the specified limit. +```hcl +provider "kubectl" { + apply_retry_count = 5 + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) + load_config_file = false + + exec { + api_version = "client.authentication.k8s.io/v1alpha1" + command = "aws" + args = ["eks", "get-token", "--cluster-name", module.eks.cluster_id] + } +} + +resource "kubectl_manifest" "karpenter_provisioner" { + yaml_body = <<-YAML + apiVersion: karpenter.sh/v1alpha5 + kind: Provisioner + metadata: + name: default + spec: + requirements: + - key: karpenter.sh/capacity-type + operator: In + values: ["spot"] + limits: + resources: + cpu: 1000 + provider: + subnetSelector: + karpenter.sh/discovery: ${local.name} + securityGroupSelector: + karpenter.sh/discovery: ${local.name} + tags: + karpenter.sh/discovery: ${local.name} + ttlSecondsAfterEmpty: 30 + YAML + + depends_on = [ + helm_release.karpenter + ] +} +``` + +Since we've added a new provider (kubectl), you'll need to run `terraform init` again +before applying the changes to deploy the Karpenter provisioner. + ```bash -cat < Date: Thu, 31 Mar 2022 13:18:14 -0400 Subject: [PATCH 15/17] fix: add required provider versions for 3rd party source resolution --- .../getting-started-with-terraform/_index.md | 27 ++++++++++++++++--- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/website/content/en/preview/getting-started/getting-started-with-terraform/_index.md b/website/content/en/preview/getting-started/getting-started-with-terraform/_index.md index 1c3ada3488c6..155be641788b 100644 --- a/website/content/en/preview/getting-started/getting-started-with-terraform/_index.md +++ b/website/content/en/preview/getting-started/getting-started-with-terraform/_index.md @@ -48,6 +48,25 @@ export AWS_DEFAULT_REGION="us-east-1" The first thing we need to do is create our `main.tf` file and place the following in it. ```hcl +terraform { + required_version = "~> 1.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 4.0" + } + helm = { + source = "hashicorp/helm" + version = "~> 2.4" + } + kubectl = { + source = "gavinbunney/kubectl" + version = "~> 1.14" + } + } +} + provider "aws" { region = "us-east-1" } @@ -97,7 +116,7 @@ module "vpc" { module "eks" { # https://registry.terraform.io/modules/terraform-aws-modules/eks/aws/latest source = "terraform-aws-modules/eks/aws" - version = "18.16.0" + version = "18.17.0" cluster_name = local.cluster_name cluster_version = "1.21" @@ -342,11 +361,11 @@ resource "kubectl_manifest" "karpenter_provisioner" { cpu: 1000 provider: subnetSelector: - karpenter.sh/discovery: ${local.name} + karpenter.sh/discovery: ${local.cluster_name} securityGroupSelector: - karpenter.sh/discovery: ${local.name} + karpenter.sh/discovery: ${local.cluster_name} tags: - karpenter.sh/discovery: ${local.name} + karpenter.sh/discovery: ${local.cluster_name} ttlSecondsAfterEmpty: 30 YAML From 67e29501a4055e0ee61b09b36ba2a08ab87c7504 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Tue, 5 Apr 2022 17:22:33 -0400 Subject: [PATCH 16/17] Update website/content/en/preview/getting-started/getting-started-with-terraform/_index.md Co-authored-by: Chris Negus --- .../getting-started/getting-started-with-terraform/_index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/content/en/preview/getting-started/getting-started-with-terraform/_index.md b/website/content/en/preview/getting-started/getting-started-with-terraform/_index.md index 155be641788b..3dc8f9b8ba76 100644 --- a/website/content/en/preview/getting-started/getting-started-with-terraform/_index.md +++ b/website/content/en/preview/getting-started/getting-started-with-terraform/_index.md @@ -139,7 +139,7 @@ module "eks" { initial = { instance_types = ["t3.medium"] # We don't need the node security group since we are using the - # cluster created security group which Karpenter will also use + # cluster-created security group, which Karpenter will also use create_security_group = false attach_cluster_primary_security_group = true From 8576324dad187641339b3da065e7285ee23bfa5e Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Thu, 7 Apr 2022 09:44:29 -0400 Subject: [PATCH 17/17] docs: add note to udate local kubeconfig before running kubectl commands --- .../getting-started-with-terraform/_index.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/website/content/en/preview/getting-started/getting-started-with-terraform/_index.md b/website/content/en/preview/getting-started/getting-started-with-terraform/_index.md index 3dc8f9b8ba76..3b749ff98776 100644 --- a/website/content/en/preview/getting-started/getting-started-with-terraform/_index.md +++ b/website/content/en/preview/getting-started/getting-started-with-terraform/_index.md @@ -388,6 +388,12 @@ terraform apply Karpenter is now active and ready to begin provisioning nodes. Create some pods using a deployment, and watch Karpenter provision nodes in response. +Before we can start interacting with the cluster, we need to update our local kubeconfig: + +```bash +aws eks update-kubeconfig --name karpenter-demo +``` + ### Automatic Node Provisioning This deployment uses the [pause image](https://www.ianlewis.org/en/almighty-pause-container) and starts with zero replicas.