From d4912a79aae20ecc211ec436f49bf8731a60230d Mon Sep 17 00:00:00 2001 From: jayfranco999 Date: Tue, 15 Oct 2024 12:19:39 +0530 Subject: [PATCH 01/17] feat!(aws):Add Amazon-ebs builder for Linux AMIs Signed-off-by: jayfranco999 --- Jenkinsfile_k8s | 31 +++++++++++- build-jenkins-agent-ubuntu.pkr.hcl | 5 ++ datasources.pkr.hcl | 41 +++++++++++++++ locals.pkr.hcl | 12 +++++ main.pkr.hcl | 4 ++ .../updatecli.d/packer-plugins/amazon.yaml | 50 +++++++++++++++++++ variables.pkr.hcl | 14 +++++- 7 files changed, 155 insertions(+), 2 deletions(-) create mode 100644 datasources.pkr.hcl create mode 100644 updatecli/updatecli.d/packer-plugins/amazon.yaml diff --git a/Jenkinsfile_k8s b/Jenkinsfile_k8s index d591160de..0e8c39d81 100644 --- a/Jenkinsfile_k8s +++ b/Jenkinsfile_k8s @@ -107,10 +107,36 @@ pipeline { axis { name 'compute_type' // "azure-arm" stands for "Azure Resource Manager", unrelated to arm64 CPU - values 'azure-arm', 'docker' + values 'amazon-ebs', 'azure-arm', 'docker' } } excludes { + // Only build Ubuntu images for arm64 CPU in AWS (notValues) + exclude { + axis { + name 'cpu_architecture' + values 'arm64' + } + axis { + name 'agent_type' + notValues 'ubuntu-22.04' + } + axis { + name 'compute_type' + values 'amazon-ebs' + } + } + // Exclude 'amazon-ebs' Windows builds while testing for Linux AMI + exclude { + axis { + name 'agent_type' + values 'windows-2019', 'windows-2022' + } + axis { + name 'compute_type' + values 'amazon-ebs' + } + } // Only build Ubuntu images for arm64 CPU in Azure (notValues) exclude { axis { @@ -149,6 +175,9 @@ pipeline { } } environment { + // Defines the following environment variables: AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY + AWS_ACCESS_KEY_ID = credentials('packer-aws-access-key-id') + AWS_SECRET_ACCESS_KEY = credentials('packer-aws-secret-access-key') // Defines the following environment variables: AZURE_CLIENT_ID, AZURE_CLIENT_SECRET, AZURE_TENANT_ID, AZURE_SUBSCRIPTION_ID // Ref. https://plugins.jenkins.io/azure-credentials/#plugin-content-declarative-pipeline AZURE = credentials('packer-azure-serviceprincipal-sponsorship') diff --git a/build-jenkins-agent-ubuntu.pkr.hcl b/build-jenkins-agent-ubuntu.pkr.hcl index 8868b1f05..5c5f280bd 100644 --- a/build-jenkins-agent-ubuntu.pkr.hcl +++ b/build-jenkins-agent-ubuntu.pkr.hcl @@ -3,6 +3,11 @@ build { name = "ubuntu" } + source "amazon-ebs.base" { + name = "ubuntu" + ssh_username = "ubuntu" + } + source "azure-arm.base" { name = "ubuntu" # List available offers and publishers with the command `az vm image list --output table` diff --git a/datasources.pkr.hcl b/datasources.pkr.hcl new file mode 100644 index 000000000..a17018352 --- /dev/null +++ b/datasources.pkr.hcl @@ -0,0 +1,41 @@ +# # Data sources are always treated BEFORE locals and sources. +data "amazon-ami" "ubuntu-22_04" { + access_key = var.aws_access_key_id + secret_key = var.aws_secret_access_key + filters = { + name = format("ubuntu/images/hvm-ssd/ubuntu-*-22.04-%s-server-*", var.architecture) + root-device-type = "ebs" + virtualization-type = "hvm" + } + most_recent = true + # owners = ["326712726440"] + owners = ["amazon"] + region = var.aws_region +} +//datasources for windows AMI commented for later use +# data "amazon-ami" "windows-2019" { +# access_key = var.aws_access_key_id +# secret_key = var.aws_secret_access_key +# filters = { +# # https://docs.aws.amazon.com/AWSEC2/latest/WindowsGuide/ec2launch-v2.html +# name = "EC2LaunchV2-Windows_Server-2019-English-Core-Base-*" +# root-device-type = "ebs" +# virtualization-type = "hvm" +# } +# most_recent = true +# owners = ["amazon"] +# region = var.aws_region +# } + +# data "amazon-ami" "windows-2022" { +# access_key = var.aws_access_key_id +# secret_key = var.aws_secret_access_key +# filters = { +# name = "Windows_Server-2022-English-Core-Base-*" +# root-device-type = "ebs" +# virtualization-type = "hvm" +# } +# most_recent = true +# owners = ["amazon"] +# region = var.aws_region +# } diff --git a/locals.pkr.hcl b/locals.pkr.hcl index 2ee93addb..7e086bf4c 100644 --- a/locals.pkr.hcl +++ b/locals.pkr.hcl @@ -4,6 +4,18 @@ locals { agent_os_version_safe = replace(var.agent_os_version, ".", "_") image_name = format("jenkins-agent-%s-%s-%s", var.agent_os_type, var.agent_os_version, var.architecture) unique_image_name = format("%s-%s", local.image_name, local.now_unix_timestamp) + + # aws_spot_instance_types = { // for spot instances + # # 4 vCPU x86 / 16 GB / $0.1670 - https://aws.amazon.com/en/ec2/instance-types/t3/#Product_Details + # "amd64" = ["t3.xlarge", "t3a.xlarge", "t2.xlarge", "m6a.xlarge"] + # # 4 vCPU ARM64 (Gravitnb)/ 16 GB / $0.1344 - https://aws.amazon.com/en/ec2/instance-types/t4/#Product_Details + # "arm64" = ["t4g.xlarge", "m7g.xlarge"] + # } + aws_instance_types = { + "amd64" = "t3.xlarge" + "arm64" = "t4g.xlarge" + } + # List available SKUs with the command `az vm image list-skus --offer 0001-com-ubuntu-server-jammy --location eastus --publisher canonical --output table` az_instance_image_sku = { "amd64" = "${local.agent_os_version_safe}-lts-gen2" diff --git a/main.pkr.hcl b/main.pkr.hcl index 533b5872a..bddb93856 100644 --- a/main.pkr.hcl +++ b/main.pkr.hcl @@ -1,6 +1,10 @@ packer { required_version = ">= 1.7.2, < 2" required_plugins { + amazon = { + version = "1.3.3" + source = "github.com/hashicorp/amazon" + } windows-update = { version = "0.16.8" source = "github.com/rgl/windows-update" diff --git a/updatecli/updatecli.d/packer-plugins/amazon.yaml b/updatecli/updatecli.d/packer-plugins/amazon.yaml new file mode 100644 index 000000000..c02b03f20 --- /dev/null +++ b/updatecli/updatecli.d/packer-plugins/amazon.yaml @@ -0,0 +1,50 @@ +--- +name: Bump packer amazon plugin version + +scms: + default: + kind: github + spec: + user: "{{ .github.user }}" + email: "{{ .github.email }}" + owner: "{{ .github.owner }}" + repository: "{{ .github.repository }}" + token: "{{ requiredEnv .github.token }}" + username: "{{ .github.username }}" + branch: "{{ .github.branch }}" + +sources: + lastReleaseVersion: + kind: githubrelease + name: Get the latest packer's amazon plugin version + spec: + owner: "hashicorp" + repository: "packer-plugin-amazon" + token: "{{ requiredEnv .github.token }}" + username: "{{ .github.username }}" + versionfilter: + kind: semver + transformers: + - trimprefix: v + +targets: + updateVersion: + name: "Update the amazon plugin version in main.pkr.hcl" + sourceid: lastReleaseVersion + kind: file + spec: + file: main.pkr.hcl + matchpattern: 'amazon = \{((\r\n|\r|\n)(\s+))version = ".*"' + replacepattern: >- + amazon = {${1}version = "{{ source "lastReleaseVersion" }}" + scmid: default + +actions: + default: + kind: github/pullrequest + title: Bump packer amazon plugin version to {{ source "lastReleaseVersion" }} + scmid: default + spec: + labels: + - dependencies + - packer-amazon-plugin diff --git a/variables.pkr.hcl b/variables.pkr.hcl index f3116ff3d..092d4d1cd 100644 --- a/variables.pkr.hcl +++ b/variables.pkr.hcl @@ -13,6 +13,18 @@ variable "architecture" { description = "CPU architecture ID of the build with the following possible values: [amd64 (default), arm64]" default = "amd64" } +variable "aws_region" { + type = string + default = "us-east-2" +} +variable "aws_access_key_id" { + type = string + default = env("AWS_ACCESS_KEY_ID") +} +variable "aws_secret_access_key" { + type = string + default = env("AWS_SECRET_ACCESS_KEY") +} variable "azure_client_id" { type = string default = env("AZURE_CLIENT_ID") @@ -35,7 +47,7 @@ variable "image_version" { } variable "image_type" { type = string - description = "Which kind of Packer builder to use (e.g. cloud platform): [azure-arm (default), docker]" + description = "Which kind of Packer builder to use (e.g. cloud platform): [amazon-ebs, azure-arm (default), docker]" default = "azure-arm" } variable "build_type" { From 4db47f7600b42a626eece748a35a946500c0bdc3 Mon Sep 17 00:00:00 2001 From: jayfranco999 Date: Tue, 15 Oct 2024 12:53:46 +0530 Subject: [PATCH 02/17] fixup Signed-off-by: jayfranco999 --- sources.pkr.hcl | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/sources.pkr.hcl b/sources.pkr.hcl index bfb2a9adc..e3b07ea31 100644 --- a/sources.pkr.hcl +++ b/sources.pkr.hcl @@ -1,3 +1,47 @@ +# This source defines all the common settings for any AWS AMI (whatever Operating System) +source "amazon-ebs" "base" { + # profile = "terraform-developer" + + # AWS API connection + access_key = var.aws_access_key_id + secret_key = var.aws_secret_access_key + + ami_name = "${local.image_name}-${var.architecture}-${local.now_unix_timestamp}" + # spot_instance_types = local.aws_spot_instance_types[var.architecture] // if spot instances is used + # spot_price = "auto" + instance_type = local.aws_instance_types[var.architecture] + # instance_type = "t2.xlarge" + + + # Define custom rootfs for build to avoid later filesystem extension during agent startups + launch_block_device_mappings { + delete_on_termination = true + device_name = "/dev/sda1" + volume_size = local.windows_disk_size_gb + volume_type = "gp2" + } + + # Where to build the VM + region = var.aws_region + + # Where to export the AMI + ami_regions = [ + var.aws_region + ] + + # Egg-and-chicken: what is the base image to start from (eg. what is my egg)? + source_ami = data.amazon-ami["${var.agent_os_type}-${local.agent_os_version_safe}"].id + # To improve audit and garbage collecting, we provide tags + tags = { + imageplatform = var.architecture + imagetype = local.image_name + timestamp = local.now_unix_timestamp + version = var.image_version + scm_ref = var.scm_ref + build_type = var.build_type + } +} + # This source defines all the common settings for any Azure image (whatever Operating System) source "azure-arm" "base" { managed_image_resource_group_name = local.azure_destination_resource_group From e1d4b002d1aaf60cefa0a1837c082c1eb7b6e37b Mon Sep 17 00:00:00 2001 From: jayfranco999 Date: Tue, 15 Oct 2024 13:05:51 +0530 Subject: [PATCH 03/17] fixup Signed-off-by: jayfranco999 --- datasources.pkr.hcl | 52 ++++++++++++++++++++++----------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/datasources.pkr.hcl b/datasources.pkr.hcl index a17018352..595690f64 100644 --- a/datasources.pkr.hcl +++ b/datasources.pkr.hcl @@ -12,30 +12,30 @@ data "amazon-ami" "ubuntu-22_04" { owners = ["amazon"] region = var.aws_region } -//datasources for windows AMI commented for later use -# data "amazon-ami" "windows-2019" { -# access_key = var.aws_access_key_id -# secret_key = var.aws_secret_access_key -# filters = { -# # https://docs.aws.amazon.com/AWSEC2/latest/WindowsGuide/ec2launch-v2.html -# name = "EC2LaunchV2-Windows_Server-2019-English-Core-Base-*" -# root-device-type = "ebs" -# virtualization-type = "hvm" -# } -# most_recent = true -# owners = ["amazon"] -# region = var.aws_region -# } -# data "amazon-ami" "windows-2022" { -# access_key = var.aws_access_key_id -# secret_key = var.aws_secret_access_key -# filters = { -# name = "Windows_Server-2022-English-Core-Base-*" -# root-device-type = "ebs" -# virtualization-type = "hvm" -# } -# most_recent = true -# owners = ["amazon"] -# region = var.aws_region -# } +data "amazon-ami" "windows-2019" { + access_key = var.aws_access_key_id + secret_key = var.aws_secret_access_key + filters = { + # https://docs.aws.amazon.com/AWSEC2/latest/WindowsGuide/ec2launch-v2.html + name = "EC2LaunchV2-Windows_Server-2019-English-Core-Base-*" + root-device-type = "ebs" + virtualization-type = "hvm" + } + most_recent = true + owners = ["amazon"] + region = var.aws_region +} + +data "amazon-ami" "windows-2022" { + access_key = var.aws_access_key_id + secret_key = var.aws_secret_access_key + filters = { + name = "Windows_Server-2022-English-Core-Base-*" + root-device-type = "ebs" + virtualization-type = "hvm" + } + most_recent = true + owners = ["amazon"] + region = var.aws_region +} From d842e6363ed3b2e9c77046ea3642363dc739ebe3 Mon Sep 17 00:00:00 2001 From: Jaideep <79799016+jayfranco999@users.noreply.github.com> Date: Tue, 15 Oct 2024 17:38:55 +0530 Subject: [PATCH 04/17] Apply suggestions from code review Co-authored-by: Damien Duportal --- datasources.pkr.hcl | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/datasources.pkr.hcl b/datasources.pkr.hcl index 595690f64..cebc671bf 100644 --- a/datasources.pkr.hcl +++ b/datasources.pkr.hcl @@ -13,29 +13,3 @@ data "amazon-ami" "ubuntu-22_04" { region = var.aws_region } -data "amazon-ami" "windows-2019" { - access_key = var.aws_access_key_id - secret_key = var.aws_secret_access_key - filters = { - # https://docs.aws.amazon.com/AWSEC2/latest/WindowsGuide/ec2launch-v2.html - name = "EC2LaunchV2-Windows_Server-2019-English-Core-Base-*" - root-device-type = "ebs" - virtualization-type = "hvm" - } - most_recent = true - owners = ["amazon"] - region = var.aws_region -} - -data "amazon-ami" "windows-2022" { - access_key = var.aws_access_key_id - secret_key = var.aws_secret_access_key - filters = { - name = "Windows_Server-2022-English-Core-Base-*" - root-device-type = "ebs" - virtualization-type = "hvm" - } - most_recent = true - owners = ["amazon"] - region = var.aws_region -} From 541f2d35c788be77399e07b9cbcf3841a640f76a Mon Sep 17 00:00:00 2001 From: jayfranco999 Date: Tue, 15 Oct 2024 17:52:19 +0530 Subject: [PATCH 05/17] fixup Signed-off-by: jayfranco999 --- datasources.pkr.hcl | 1 - locals.pkr.hcl | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/datasources.pkr.hcl b/datasources.pkr.hcl index cebc671bf..b1a0432d3 100644 --- a/datasources.pkr.hcl +++ b/datasources.pkr.hcl @@ -12,4 +12,3 @@ data "amazon-ami" "ubuntu-22_04" { owners = ["amazon"] region = var.aws_region } - diff --git a/locals.pkr.hcl b/locals.pkr.hcl index 7e086bf4c..cf94c264b 100644 --- a/locals.pkr.hcl +++ b/locals.pkr.hcl @@ -24,6 +24,7 @@ locals { windows_winrm_user = { "azure-arm" = "packer" "docker" = "packer" + "amazon-ebs" = "Administrator" } # List available images `az vm image list --location eastus --publisher MicrosoftWindowsServer --offer WindowsServer --sku 2022-datacenter-core-g2 --all --output table` From 318d1b4352d4b72451e992471a0050bc2a94e590 Mon Sep 17 00:00:00 2001 From: jayfranco999 Date: Tue, 15 Oct 2024 18:39:23 +0530 Subject: [PATCH 06/17] fixup Signed-off-by: jayfranco999 --- sources.pkr.hcl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sources.pkr.hcl b/sources.pkr.hcl index e3b07ea31..3d20d85d0 100644 --- a/sources.pkr.hcl +++ b/sources.pkr.hcl @@ -30,7 +30,8 @@ source "amazon-ebs" "base" { ] # Egg-and-chicken: what is the base image to start from (eg. what is my egg)? - source_ami = data.amazon-ami["${var.agent_os_type}-${local.agent_os_version_safe}"].id + source_ami = try(data.amazon-ami["${var.agent_os_type}-${local.agent_os_version_safe}"].id, null) //linux-ami test condition since packer validate will fail without windows source + # source_ami = data.amazon-ami["${var.agent_os_type}-${local.agent_os_version_safe}"].id # To improve audit and garbage collecting, we provide tags tags = { imageplatform = var.architecture From 805ae30b466e8a985358373397e47b127a58e6eb Mon Sep 17 00:00:00 2001 From: jayfranco999 Date: Tue, 15 Oct 2024 19:11:38 +0530 Subject: [PATCH 07/17] fixup Signed-off-by: jayfranco999 --- sources.pkr.hcl | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/sources.pkr.hcl b/sources.pkr.hcl index 3d20d85d0..f8ad55849 100644 --- a/sources.pkr.hcl +++ b/sources.pkr.hcl @@ -30,8 +30,14 @@ source "amazon-ebs" "base" { ] # Egg-and-chicken: what is the base image to start from (eg. what is my egg)? - source_ami = try(data.amazon-ami["${var.agent_os_type}-${local.agent_os_version_safe}"].id, null) //linux-ami test condition since packer validate will fail without windows source - # source_ami = data.amazon-ami["${var.agent_os_type}-${local.agent_os_version_safe}"].id + source_ami { + filters = { + name = format("ubuntu/images/hvm-ssd/ubuntu-*-22.04-%s-server-*", var.architecture) + root-device-type = "ebs" + virtualization-type = "hvm" + } + } + source_ami = data.amazon-ami["${var.agent_os_type}-${local.agent_os_version_safe}"].id # To improve audit and garbage collecting, we provide tags tags = { imageplatform = var.architecture From f702838cf3b45408c573f6b3a6614825b07767d5 Mon Sep 17 00:00:00 2001 From: jayfranco999 Date: Tue, 15 Oct 2024 19:20:20 +0530 Subject: [PATCH 08/17] fixup Signed-off-by: jayfranco999 --- sources.pkr.hcl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sources.pkr.hcl b/sources.pkr.hcl index f8ad55849..5fad56726 100644 --- a/sources.pkr.hcl +++ b/sources.pkr.hcl @@ -30,12 +30,15 @@ source "amazon-ebs" "base" { ] # Egg-and-chicken: what is the base image to start from (eg. what is my egg)? - source_ami { + source_ami_filter { filters = { name = format("ubuntu/images/hvm-ssd/ubuntu-*-22.04-%s-server-*", var.architecture) root-device-type = "ebs" virtualization-type = "hvm" } + most_recent = true + owners = ["amazon"] + region = var.aws_region } source_ami = data.amazon-ami["${var.agent_os_type}-${local.agent_os_version_safe}"].id # To improve audit and garbage collecting, we provide tags From 67bbb16a38e235964bb20d503298742eaa36e667 Mon Sep 17 00:00:00 2001 From: jayfranco999 Date: Tue, 15 Oct 2024 19:22:07 +0530 Subject: [PATCH 09/17] fixup Signed-off-by: jayfranco999 --- sources.pkr.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sources.pkr.hcl b/sources.pkr.hcl index 5fad56726..130b230c6 100644 --- a/sources.pkr.hcl +++ b/sources.pkr.hcl @@ -40,7 +40,7 @@ source "amazon-ebs" "base" { owners = ["amazon"] region = var.aws_region } - source_ami = data.amazon-ami["${var.agent_os_type}-${local.agent_os_version_safe}"].id + # source_ami = data.amazon-ami["${var.agent_os_type}-${local.agent_os_version_safe}"].id # To improve audit and garbage collecting, we provide tags tags = { imageplatform = var.architecture From a5fa046c62fc04d0e9ec4c67a8785d8e416b862d Mon Sep 17 00:00:00 2001 From: jayfranco999 Date: Tue, 15 Oct 2024 19:26:10 +0530 Subject: [PATCH 10/17] fixup Signed-off-by: jayfranco999 --- sources.pkr.hcl | 1 - 1 file changed, 1 deletion(-) diff --git a/sources.pkr.hcl b/sources.pkr.hcl index 130b230c6..1fdc51cff 100644 --- a/sources.pkr.hcl +++ b/sources.pkr.hcl @@ -38,7 +38,6 @@ source "amazon-ebs" "base" { } most_recent = true owners = ["amazon"] - region = var.aws_region } # source_ami = data.amazon-ami["${var.agent_os_type}-${local.agent_os_version_safe}"].id # To improve audit and garbage collecting, we provide tags From a6e7983997171e673f6aaf61ebcb9a152d39005f Mon Sep 17 00:00:00 2001 From: jayfranco999 Date: Wed, 16 Oct 2024 13:44:49 +0530 Subject: [PATCH 11/17] feat(aws): Removed all static credentials Signed-off-by: jayfranco999 --- datasources.pkr.hcl | 2 -- variables.pkr.hcl | 12 ++---------- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/datasources.pkr.hcl b/datasources.pkr.hcl index b1a0432d3..bfa7079aa 100644 --- a/datasources.pkr.hcl +++ b/datasources.pkr.hcl @@ -1,7 +1,5 @@ # # Data sources are always treated BEFORE locals and sources. data "amazon-ami" "ubuntu-22_04" { - access_key = var.aws_access_key_id - secret_key = var.aws_secret_access_key filters = { name = format("ubuntu/images/hvm-ssd/ubuntu-*-22.04-%s-server-*", var.architecture) root-device-type = "ebs" diff --git a/variables.pkr.hcl b/variables.pkr.hcl index 092d4d1cd..d68b88a10 100644 --- a/variables.pkr.hcl +++ b/variables.pkr.hcl @@ -13,17 +13,9 @@ variable "architecture" { description = "CPU architecture ID of the build with the following possible values: [amd64 (default), arm64]" default = "amd64" } -variable "aws_region" { +variable "aws_destination_region" { type = string - default = "us-east-2" -} -variable "aws_access_key_id" { - type = string - default = env("AWS_ACCESS_KEY_ID") -} -variable "aws_secret_access_key" { - type = string - default = env("AWS_SECRET_ACCESS_KEY") + default = env("AWS_DEFAULT_REGION") } variable "azure_client_id" { type = string From aaf0c737477fa5726e2956bda2cc94cd16888a64 Mon Sep 17 00:00:00 2001 From: jayfranco999 Date: Wed, 16 Oct 2024 13:47:34 +0530 Subject: [PATCH 12/17] fixup Signed-off-by: jayfranco999 --- sources.pkr.hcl | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/sources.pkr.hcl b/sources.pkr.hcl index 1fdc51cff..8ccddd9ca 100644 --- a/sources.pkr.hcl +++ b/sources.pkr.hcl @@ -2,10 +2,6 @@ source "amazon-ebs" "base" { # profile = "terraform-developer" - # AWS API connection - access_key = var.aws_access_key_id - secret_key = var.aws_secret_access_key - ami_name = "${local.image_name}-${var.architecture}-${local.now_unix_timestamp}" # spot_instance_types = local.aws_spot_instance_types[var.architecture] // if spot instances is used # spot_price = "auto" @@ -21,12 +17,10 @@ source "amazon-ebs" "base" { volume_type = "gp2" } - # Where to build the VM - region = var.aws_region # Where to export the AMI ami_regions = [ - var.aws_region + var.aws_destination_region ] # Egg-and-chicken: what is the base image to start from (eg. what is my egg)? From 04cc3df72ba49521ad19007afd5b895c0b853866 Mon Sep 17 00:00:00 2001 From: jayfranco999 Date: Wed, 16 Oct 2024 14:20:07 +0530 Subject: [PATCH 13/17] mapping ami ids as per architecture Signed-off-by: jayfranco999 --- build-jenkins-agent-ubuntu.pkr.hcl | 2 ++ datasources.pkr.hcl | 12 ------------ locals.pkr.hcl | 5 ++++- sources.pkr.hcl | 12 +----------- 4 files changed, 7 insertions(+), 24 deletions(-) delete mode 100644 datasources.pkr.hcl diff --git a/build-jenkins-agent-ubuntu.pkr.hcl b/build-jenkins-agent-ubuntu.pkr.hcl index 5c5f280bd..173b8bca4 100644 --- a/build-jenkins-agent-ubuntu.pkr.hcl +++ b/build-jenkins-agent-ubuntu.pkr.hcl @@ -6,6 +6,8 @@ build { source "amazon-ebs.base" { name = "ubuntu" ssh_username = "ubuntu" + # Egg-and-chicken: what is the base image to start from (eg. what is my egg)? + source_ami = local.aws_ubuntu_amis[var.architecture] } source "azure-arm.base" { diff --git a/datasources.pkr.hcl b/datasources.pkr.hcl deleted file mode 100644 index bfa7079aa..000000000 --- a/datasources.pkr.hcl +++ /dev/null @@ -1,12 +0,0 @@ -# # Data sources are always treated BEFORE locals and sources. -data "amazon-ami" "ubuntu-22_04" { - filters = { - name = format("ubuntu/images/hvm-ssd/ubuntu-*-22.04-%s-server-*", var.architecture) - root-device-type = "ebs" - virtualization-type = "hvm" - } - most_recent = true - # owners = ["326712726440"] - owners = ["amazon"] - region = var.aws_region -} diff --git a/locals.pkr.hcl b/locals.pkr.hcl index cf94c264b..f01a94aef 100644 --- a/locals.pkr.hcl +++ b/locals.pkr.hcl @@ -15,7 +15,10 @@ locals { "amd64" = "t3.xlarge" "arm64" = "t4g.xlarge" } - + aws_ubuntu_amis = { + "amd64" = "ami-00eb69d236edcfaf8" + "arm64" = "ami-039e419d24a37cb82" + } # List available SKUs with the command `az vm image list-skus --offer 0001-com-ubuntu-server-jammy --location eastus --publisher canonical --output table` az_instance_image_sku = { "amd64" = "${local.agent_os_version_safe}-lts-gen2" diff --git a/sources.pkr.hcl b/sources.pkr.hcl index 8ccddd9ca..a00f20806 100644 --- a/sources.pkr.hcl +++ b/sources.pkr.hcl @@ -23,17 +23,7 @@ source "amazon-ebs" "base" { var.aws_destination_region ] - # Egg-and-chicken: what is the base image to start from (eg. what is my egg)? - source_ami_filter { - filters = { - name = format("ubuntu/images/hvm-ssd/ubuntu-*-22.04-%s-server-*", var.architecture) - root-device-type = "ebs" - virtualization-type = "hvm" - } - most_recent = true - owners = ["amazon"] - } - # source_ami = data.amazon-ami["${var.agent_os_type}-${local.agent_os_version_safe}"].id + # To improve audit and garbage collecting, we provide tags tags = { imageplatform = var.architecture From 21745d58f1fd7139087164fa277cb78e6e726cf0 Mon Sep 17 00:00:00 2001 From: jayfranco999 Date: Wed, 16 Oct 2024 18:42:59 +0530 Subject: [PATCH 14/17] fixup Signed-off-by: jayfranco999 --- build-jenkins-agent-ubuntu.pkr.hcl | 1 + 1 file changed, 1 insertion(+) diff --git a/build-jenkins-agent-ubuntu.pkr.hcl b/build-jenkins-agent-ubuntu.pkr.hcl index 173b8bca4..f85f4e290 100644 --- a/build-jenkins-agent-ubuntu.pkr.hcl +++ b/build-jenkins-agent-ubuntu.pkr.hcl @@ -8,6 +8,7 @@ build { ssh_username = "ubuntu" # Egg-and-chicken: what is the base image to start from (eg. what is my egg)? source_ami = local.aws_ubuntu_amis[var.architecture] + region = env("AWS_DEFAULT_REGION") } source "azure-arm.base" { From 0ce40f279c6485ad2176efbb926e2f5b2d6cacd3 Mon Sep 17 00:00:00 2001 From: jayfranco999 Date: Wed, 16 Oct 2024 19:00:37 +0530 Subject: [PATCH 15/17] fixup Signed-off-by: jayfranco999 --- Jenkinsfile_k8s | 1 + build-jenkins-agent-ubuntu.pkr.hcl | 1 - .../updatecli.d/packer-plugins/amazon.yaml | 50 ------------------- 3 files changed, 1 insertion(+), 51 deletions(-) delete mode 100644 updatecli/updatecli.d/packer-plugins/amazon.yaml diff --git a/Jenkinsfile_k8s b/Jenkinsfile_k8s index 0e8c39d81..ee635e672 100644 --- a/Jenkinsfile_k8s +++ b/Jenkinsfile_k8s @@ -178,6 +178,7 @@ pipeline { // Defines the following environment variables: AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY AWS_ACCESS_KEY_ID = credentials('packer-aws-access-key-id') AWS_SECRET_ACCESS_KEY = credentials('packer-aws-secret-access-key') + AWS_DEFAULT_REGION = 'us-east-2' // Defines the following environment variables: AZURE_CLIENT_ID, AZURE_CLIENT_SECRET, AZURE_TENANT_ID, AZURE_SUBSCRIPTION_ID // Ref. https://plugins.jenkins.io/azure-credentials/#plugin-content-declarative-pipeline AZURE = credentials('packer-azure-serviceprincipal-sponsorship') diff --git a/build-jenkins-agent-ubuntu.pkr.hcl b/build-jenkins-agent-ubuntu.pkr.hcl index f85f4e290..173b8bca4 100644 --- a/build-jenkins-agent-ubuntu.pkr.hcl +++ b/build-jenkins-agent-ubuntu.pkr.hcl @@ -8,7 +8,6 @@ build { ssh_username = "ubuntu" # Egg-and-chicken: what is the base image to start from (eg. what is my egg)? source_ami = local.aws_ubuntu_amis[var.architecture] - region = env("AWS_DEFAULT_REGION") } source "azure-arm.base" { diff --git a/updatecli/updatecli.d/packer-plugins/amazon.yaml b/updatecli/updatecli.d/packer-plugins/amazon.yaml deleted file mode 100644 index c02b03f20..000000000 --- a/updatecli/updatecli.d/packer-plugins/amazon.yaml +++ /dev/null @@ -1,50 +0,0 @@ ---- -name: Bump packer amazon plugin version - -scms: - default: - kind: github - spec: - user: "{{ .github.user }}" - email: "{{ .github.email }}" - owner: "{{ .github.owner }}" - repository: "{{ .github.repository }}" - token: "{{ requiredEnv .github.token }}" - username: "{{ .github.username }}" - branch: "{{ .github.branch }}" - -sources: - lastReleaseVersion: - kind: githubrelease - name: Get the latest packer's amazon plugin version - spec: - owner: "hashicorp" - repository: "packer-plugin-amazon" - token: "{{ requiredEnv .github.token }}" - username: "{{ .github.username }}" - versionfilter: - kind: semver - transformers: - - trimprefix: v - -targets: - updateVersion: - name: "Update the amazon plugin version in main.pkr.hcl" - sourceid: lastReleaseVersion - kind: file - spec: - file: main.pkr.hcl - matchpattern: 'amazon = \{((\r\n|\r|\n)(\s+))version = ".*"' - replacepattern: >- - amazon = {${1}version = "{{ source "lastReleaseVersion" }}" - scmid: default - -actions: - default: - kind: github/pullrequest - title: Bump packer amazon plugin version to {{ source "lastReleaseVersion" }} - scmid: default - spec: - labels: - - dependencies - - packer-amazon-plugin From 18cf9aaf7fabd97dcc359110420ca6b2e4dcac52 Mon Sep 17 00:00:00 2001 From: jayfranco999 Date: Wed, 16 Oct 2024 19:46:20 +0530 Subject: [PATCH 16/17] fixup Signed-off-by: jayfranco999 --- locals.pkr.hcl | 11 ++++------- main.pkr.hcl | 2 +- sources.pkr.hcl | 9 +++------ variables.pkr.hcl | 2 +- 4 files changed, 9 insertions(+), 15 deletions(-) diff --git a/locals.pkr.hcl b/locals.pkr.hcl index f01a94aef..8a8abe65b 100644 --- a/locals.pkr.hcl +++ b/locals.pkr.hcl @@ -4,17 +4,14 @@ locals { agent_os_version_safe = replace(var.agent_os_version, ".", "_") image_name = format("jenkins-agent-%s-%s-%s", var.agent_os_type, var.agent_os_version, var.architecture) unique_image_name = format("%s-%s", local.image_name, local.now_unix_timestamp) - - # aws_spot_instance_types = { // for spot instances - # # 4 vCPU x86 / 16 GB / $0.1670 - https://aws.amazon.com/en/ec2/instance-types/t3/#Product_Details - # "amd64" = ["t3.xlarge", "t3a.xlarge", "t2.xlarge", "m6a.xlarge"] - # # 4 vCPU ARM64 (Gravitnb)/ 16 GB / $0.1344 - https://aws.amazon.com/en/ec2/instance-types/t4/#Product_Details - # "arm64" = ["t4g.xlarge", "m7g.xlarge"] - # } + + aws_instance_types = { "amd64" = "t3.xlarge" "arm64" = "t4g.xlarge" } + # TODO: track AMI versions with `updatecli` + # Uses aws ec2 describe-images to fetch the ami id as per the architecture aws_ubuntu_amis = { "amd64" = "ami-00eb69d236edcfaf8" "arm64" = "ami-039e419d24a37cb82" diff --git a/main.pkr.hcl b/main.pkr.hcl index bddb93856..c60464e8a 100644 --- a/main.pkr.hcl +++ b/main.pkr.hcl @@ -1,7 +1,7 @@ packer { required_version = ">= 1.7.2, < 2" required_plugins { - amazon = { + amazon = { # TODO: track this version with `updatecli` version = "1.3.3" source = "github.com/hashicorp/amazon" } diff --git a/sources.pkr.hcl b/sources.pkr.hcl index a00f20806..d6a215a37 100644 --- a/sources.pkr.hcl +++ b/sources.pkr.hcl @@ -1,20 +1,17 @@ # This source defines all the common settings for any AWS AMI (whatever Operating System) source "amazon-ebs" "base" { - # profile = "terraform-developer" + ami_name = "${local.image_name}-${var.architecture}-${local.now_unix_timestamp}" - # spot_instance_types = local.aws_spot_instance_types[var.architecture] // if spot instances is used - # spot_price = "auto" instance_type = local.aws_instance_types[var.architecture] - # instance_type = "t2.xlarge" # Define custom rootfs for build to avoid later filesystem extension during agent startups launch_block_device_mappings { delete_on_termination = true device_name = "/dev/sda1" - volume_size = local.windows_disk_size_gb - volume_type = "gp2" + volume_size = local.windows_disk_size_gb # TODO: check if we can rename this local to cover both windows and Ubuntu + volume_type = "gp2" # TODO: check if we can use `gp3` (blocker was ec2 plugin, not packer) } diff --git a/variables.pkr.hcl b/variables.pkr.hcl index d68b88a10..d2c806f27 100644 --- a/variables.pkr.hcl +++ b/variables.pkr.hcl @@ -15,7 +15,7 @@ variable "architecture" { } variable "aws_destination_region" { type = string - default = env("AWS_DEFAULT_REGION") + default = env("AWS_DEFAULT_REGION") # Defaults to the only region we use } variable "azure_client_id" { type = string From 878578ba4d2c2e7950cceabede3a87c19e69e8fd Mon Sep 17 00:00:00 2001 From: Jaideep <79799016+jayfranco999@users.noreply.github.com> Date: Thu, 17 Oct 2024 18:13:45 +0530 Subject: [PATCH 17/17] fixup --- Jenkinsfile_k8s | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile_k8s b/Jenkinsfile_k8s index ee635e672..07afeb1e9 100644 --- a/Jenkinsfile_k8s +++ b/Jenkinsfile_k8s @@ -175,7 +175,7 @@ pipeline { } } environment { - // Defines the following environment variables: AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY + // Defines the following environment variables: AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_DEFAULT_REGION AWS_ACCESS_KEY_ID = credentials('packer-aws-access-key-id') AWS_SECRET_ACCESS_KEY = credentials('packer-aws-secret-access-key') AWS_DEFAULT_REGION = 'us-east-2'