Adds changes for using custom launch template

This commit adds changes for using custom launch template for worker nodes to PVRE and IMDSV2 issues. Signed-off-by: Ashish Ranjan <rnshis@amazon.com>
awslabs · Dec 26, 2023 · 597cd30 · 597cd30
1 parent 04e3a4e
commit 597cd30
Show file tree

Hide file tree

Showing 8 changed files with 214 additions and 5 deletions.
diff --git a/tests/assets/eks_node_group_launch_template.json b/tests/assets/eks_node_group_launch_template.json
@@ -0,0 +1,44 @@
+{
+    "AWSTemplateFormatVersion": "2010-09-09",
+    "Description": "Create an EKS Node Group Launch Template",
+    "Parameters": {
+        "LaunchTemplateName": {
+            "Type": "String",
+            "Description": "Name of the Launch Template"
+        },
+        "ClusterName": {
+            "Type": "String",
+            "Description": "Name of the Cluster"
+        }
+    },
+    "Resources": {
+        "NodeGroupLaunchTemplate": {
+            "Type": "AWS::EC2::LaunchTemplate",
+            "Properties": {
+                "LaunchTemplateName": { "Ref": "LaunchTemplateName" },
+                "LaunchTemplateData": {
+                    "BlockDeviceMappings": [
+                        {
+                            "DeviceName": "/dev/xvda",
+                            "Ebs": {
+                                "VolumeSize": 20,
+                                "VolumeType": "gp2"
+                            }
+                        }
+                    ],
+                    "MetadataOptions": {
+                        "HttpPutResponseHopLimit": 2,
+                        "HttpEndpoint": "enabled",
+                        "HttpTokens": "required"
+                    }
+                }
+            }
+        }
+    },
+    "Outputs": {
+        "NodeGroupLaunchTemplateName": {
+            "Description": "Name of the Node Group Launch Template",
+            "Value": { "Ref": "NodeGroupLaunchTemplate" }
+        }
+    }
+}
diff --git a/tests/pipelines/eks/awscli-cl2-load-with-addons-slos.yaml b/tests/pipelines/eks/awscli-cl2-load-with-addons-slos.yaml
@@ -19,6 +19,8 @@ spec:
       value: $(params.cluster-name)-service-role
     - name: node-role-stack-name
       value: $(params.cluster-name)-node-role
+    - name: launch-template-stack-name
+      value: $(params.cluster-name)-launch-template
     retries: 10
     taskRef:
       kind: Task
@@ -49,7 +51,7 @@ spec:
     default: "https://raw.githubusercontent.com/awslabs/kubernetes-iteration-toolkit/main/tests/assets/amazon-eks-vpc.json"
     type: string
   - name: ng-cfn-url
-    default: "https://raw.githubusercontent.com/awslabs/kubernetes-iteration-toolkit/main/tests/assets/eks_nodeGroup_launch_template.json"
+    default: "https://raw.githubusercontent.com/awslabs/kubernetes-iteration-toolkit/main/tests/assets/eks_node_group_launch_template.json"
     type: string
   - name: kubernetes-version
     type: string
@@ -127,6 +129,26 @@ spec:
     workspaces:
     - name: config
       workspace: config
+  - name: create-launch-template
+    params:
+      - name: cluster-name
+        value: $(params.cluster-name)
+      - name: stack-name
+        value: $(params.cluster-name)-launch-template
+      - name: kubernetes-version
+        value: "$(params.kubernetes-version)"
+      - name: ng-cfn-url
+        value: "$(params.ng-cfn-url)"
+      - name: endpoint
+        value: $(params.endpoint)
+    runAfter:
+      - create-eks-cluster
+    taskRef:
+      kind: Task
+      name: awscli-eks-cfn-launch-template
+    workspaces:
+      - name: config
+        workspace: config
   - name: create-mng-monitoring-nodes
     params:
     - name: cluster-name
@@ -146,7 +168,7 @@ spec:
     - name: nodegroup-prefix
       value: monitoring-
     runAfter:
-    - create-eks-cluster
+    - create-launch-template
     taskRef:
       kind: Task
       name: awscli-eks-nodegroup-create

diff --git a/tests/pipelines/eks/awscli-cl2-load-with-addons.yaml b/tests/pipelines/eks/awscli-cl2-load-with-addons.yaml
@@ -16,6 +16,9 @@ spec:
   - name: slack-message
   - name: amp-workspace-id
   - name: vpc-cfn-url
+  - name: ng-cfn-url
+    default: "https://raw.githubusercontent.com/awslabs/kubernetes-iteration-toolkit/main/tests/assets/eks_node_group_launch_template.json"
+    type: string
   - name: kubernetes-version
   - name: service-role-cfn-url
     default: "https://raw.githubusercontent.com/awslabs/kubernetes-iteration-toolkit/main/tests/assets/eks_service_role.json"
@@ -89,6 +92,26 @@ spec:
     workspaces:
     - name: config    
       workspace: config
+  - name: create-launch-template
+    params:
+      - name: cluster-name
+        value: $(params.cluster-name)
+      - name: stack-name
+        value: $(params.cluster-name)-launch-template
+      - name: kubernetes-version
+        value: "$(params.kubernetes-version)"
+      - name: ng-cfn-url
+        value: "$(params.ng-cfn-url)"
+      - name: endpoint
+        value: $(params.endpoint)
+    runAfter:
+      - create-eks-cluster
+    taskRef:
+      kind: Task
+      name: awscli-eks-cfn-launch-template
+    workspaces:
+      - name: config
+        workspace: config
   - name: create-mng-monitoring-nodes
     params:
     - name: cluster-name
@@ -108,7 +131,7 @@ spec:
     - name: nodegroup-prefix
       value: "monitoring-"
     runAfter:
-    - create-eks-cluster
+    - create-launch-template
     taskRef:
       kind: Task
       name:  awscli-eks-nodegroup-create
@@ -190,6 +213,8 @@ spec:
       value: $(params.cluster-name)-service-role
     - name: node-role-stack-name
       value: $(params.cluster-name)-node-role
+    - name: launch-template-stack-name
+      value: $(params.cluster-name)-launch-template
     taskRef:
       kind: Task
       name:  awscli-eks-cluster-teardown

diff --git a/tests/pipelines/eks/awscli-eks-cl2-load.yaml b/tests/pipelines/eks/awscli-eks-cl2-load.yaml
@@ -18,6 +18,9 @@ spec:
   - name: slack-hook
   - name: slack-message
   - name: vpc-cfn-url
+  - name: ng-cfn-url
+    default: "https://raw.githubusercontent.com/awslabs/kubernetes-iteration-toolkit/main/tests/assets/eks_node_group_launch_template.json"
+    type: string
   - name: kubernetes-version
     default: "1.23"
   - name: amp-workspace-id
@@ -88,6 +91,26 @@ spec:
     workspaces:
     - name: config    
       workspace: config
+  - name: create-cfn-launch-template
+    params:
+      - name: cluster-name
+        value: $(params.cluster-name)
+      - name: stack-name
+        value: $(params.cluster-name)-launch-template
+      - name: kubernetes-version
+        value: "$(params.kubernetes-version)"
+      - name: ng-cfn-url
+        value: "$(params.ng-cfn-url)"
+      - name: endpoint
+        value: $(params.endpoint)
+    runAfter:
+      - create-eks-cluster
+    taskRef:
+      kind: Task
+      name: awscli-eks-cfn-launch-template
+    workspaces:
+      - name: config
+        workspace: config
   - name: create-mng-monitoring-nodes
     params:
     - name: cluster-name
@@ -107,7 +130,7 @@ spec:
     - name: nodegroup-prefix
       value: "monitoring-"
     runAfter:
-    - create-eks-cluster
+    - create-cfn-launch-template
     taskRef:
       kind: Task
       name:  awscli-eks-nodegroup-create
@@ -201,6 +224,8 @@ spec:
       value: $(params.cluster-name)-service-role
     - name: node-role-stack-name
       value: $(params.cluster-name)-node-role
+    - name: launch-template-stack-name
+      value: $(params.cluster-name)-launch-template
     taskRef:
       kind: Task
       name:  awscli-eks-cluster-teardown

diff --git a/tests/tasks/setup/eks/awscli-cfn-lt.yaml b/tests/tasks/setup/eks/awscli-cfn-lt.yaml
@@ -0,0 +1,61 @@
+---
+apiVersion: tekton.dev/v1beta1
+kind: Task
+metadata:
+  name: awscli-eks-cfn-launch-template
+  namespace: scalability
+spec:
+  description: |
+    Create an EKS CFN stack to output a launch template.
+    This Task can be used to create an EKS CFN stack that outputs a launch template.
+  params:
+    - name: cluster-name
+      description: EKS cluster you want to create CFN stack for.
+    - name: stack-name
+      description: Stack name you want to spin.
+    - name: region
+      default: "us-west-2"
+      description: The region where the cluster is in.
+    - name: kubernetes-version
+      default: "1.28"
+      description: The EKS version to install.
+    - name: ng-cfn-url
+      description: The url of the CFN YAML/JSON to create CFN stack for NG launch template
+    - name: endpoint
+      default: ""
+  workspaces:
+    - name: config
+      mountPath: /config/
+  stepTemplate:
+    env:
+      - name: KUBECONFIG
+        value: /config/kubeconfig
+  steps:
+    - name: create-launch-template
+      image: alpine/k8s:1.23.7
+      script: |
+        set -x
+        ENDPOINT_FLAG=""
+        if [ -n "$(params.endpoint)" ]; then
+          ENDPOINT_FLAG="--endpoint $(params.endpoint)"
+        fi
+        
+        curl -s $(params.ng-cfn-url) -o ./amazon-ng-cfn
+        
+        launch_template_name=$(params.cluster-name)-launchTemplate
+        STACK_NAME=$(params.stack-name)
+        STACK_STATUS=$(aws cloudformation describe-stacks --query 'Stacks[?StackName==`'${STACK_NAME}'`].StackStatus' --output text  --region $(params.region))
+
+        if [[ "$STACK_STATUS" == "" ]]; then
+            aws cloudformation create-stack \
+                --stack-name $STACK_NAME \
+                --template-body file://$(pwd)/amazon-ng-cfn \
+                --parameters ParameterKey=LaunchTemplateName,ParameterValue=$launch_template_name\
+                    ParameterKey=ClusterName,ParameterValue=$(params.cluster-name)\
+                --region $(params.region)
+        
+            aws cloudformation wait stack-create-complete --stack-name $STACK_NAME --region $(params.region)
+            echo "CREATED_CFN_STACK=$STACK_NAME"
+        else
+            echo "$STACK_NAME Already exists"
+        fi
diff --git a/tests/tasks/setup/eks/awscli-cp-with-vpc.yaml b/tests/tasks/setup/eks/awscli-cp-with-vpc.yaml
@@ -78,6 +78,33 @@ spec:
       # install csi drivers.
       kubectl apply -k "github.com/kubernetes-sigs/aws-ebs-csi-driver/deploy/kubernetes/overlays/stable/?ref=$(params.aws-ebs-csi-driver-version)"
       # TODO: Calculate replicas based on the cluster size going forward.
+      # Patching the coredns not to get scheduled on the monitoring node.
+      kubectl patch deployment coredns --patch '{
+        "spec": {
+          "template": {
+            "spec": {
+              "affinity": {
+                "podAntiAffinity": {
+                  "requiredDuringSchedulingIgnoredDuringExecution": [
+                    {
+                      "labelSelector": {
+                        "matchExpressions": [
+                          {
+                            "key": "eks.amazonaws.com/nodegroup",
+                            "operator": "In",
+                            "values": ["monitoring-$(params.cluster-name)-nodes-1"]
+                          }
+                        ]
+                      },
+                      "topologyKey": "kubernetes.io/hostname"
+                    }
+                  ]
+                }
+              }
+            }
+          }
+        }
+      }' -n kube-system
       kubectl scale --replicas 1000 deploy coredns -n kube-system
       #ToDo - remove these comments after experimentation
       # Install EKS Pod Identity Agent 

diff --git a/tests/tasks/setup/eks/awscli-mng.yaml b/tests/tasks/setup/eks/awscli-mng.yaml
@@ -69,15 +69,17 @@ spec:
       NG_SUBNETS=$(aws eks $ENDPOINT_FLAG --region $(params.region)  describe-cluster --name $(params.cluster-name) \
       --query cluster.resourcesVpcConfig.subnetIds --output text \
       )
-
+      
       max_nodes=$(params.max-nodes)
       nodes=$(params.desired-nodes)
       asgs=$((nodes/max_nodes))
       echo "asgs: $asgs"
       node_group=$(params.nodegroup-prefix)$(params.cluster-name)-nodes
+    
       create_and_validate_dp_nodes()
       {
         node_group_name=$node_group-$1
+        launch_template_name=$(params.cluster-name)-launchTemplate
         CREATED_NODEGROUP=$(aws eks $ENDPOINT_FLAG --region $(params.region) list-nodegroups --cluster-name $(params.cluster-name)  --query 'nodegroups[?@==`'$node_group_name'`]' --output text)
         EC2_INSTANCES=$3
         if [ "$CREATED_NODEGROUP" == "" ]; then
@@ -86,6 +88,7 @@ spec:
           --cluster-name $(params.cluster-name) \
           --nodegroup-name $node_group_name \
           --node-role $NODE_ROLE_ARN \
+          --launch-template name=$launch_template_name\
           --region $(params.region) \
           --instance-types $EC2_INSTANCES \
           --scaling-config minSize=$(params.min-nodes),maxSize=$2,desiredSize=$2 \

diff --git a/tests/tasks/teardown/awscli-eks.yaml b/tests/tasks/teardown/awscli-eks.yaml
@@ -22,6 +22,7 @@ spec:
     default: "Job is completed"
   - name: service-role-stack-name
   - name: node-role-stack-name
+  - name: launch-template-stack-name
   steps:
   - name: delete-cluster
     image: alpine/k8s:1.23.7
@@ -42,6 +43,7 @@ spec:
     script: |
       aws cloudformation delete-stack --stack-name $(params.service-role-stack-name)
       aws cloudformation delete-stack --stack-name $(params.node-role-stack-name)
+      aws cloudformation delete-stack --stack-name $(params.launch-template-stack-name)
   - name: send-slack-notification
     image: alpine/k8s:1.23.7
     script: |