Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

cleanup: empty cik8s and eks-public EKS clusters #555

Merged
merged 1 commit into from
May 21, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
94 changes: 1 addition & 93 deletions cik8s-cluster.tf
Original file line number Diff line number Diff line change
Expand Up @@ -79,8 +79,7 @@ module "cik8s" {
}
# https://github.com/kubernetes-sigs/aws-ebs-csi-driver/blob/master/CHANGELOG.md
aws-ebs-csi-driver = {
addon_version = "v1.28.0-eksbuild.1"
service_account_role_arn = module.cik8s_irsa_ebs.iam_role_arn
addon_version = "v1.28.0-eksbuild.1"
}
}

Expand Down Expand Up @@ -256,97 +255,6 @@ module "cik8s" {
}
}

module "cik8s_iam_role_autoscaler" {
source = "terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc"
version = "5.39.1"
create_role = true
role_name = "${local.autoscaler_account_name}-cik8s"
provider_url = replace(module.cik8s.cluster_oidc_issuer_url, "https://", "")
role_policy_arns = [aws_iam_policy.cluster_autoscaler_cik8s.arn]
oidc_fully_qualified_subjects = ["system:serviceaccount:${local.autoscaler_account_namespace}:${local.autoscaler_account_name}"]

tags = merge(local.common_tags, {
associated_service = "eks/${module.cik8s.cluster_name}"
})
}

module "cik8s_irsa_ebs" {
source = "terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc"
version = "5.30.0"
create_role = true
role_name = "${local.ebs_account_name}-cik8s"
provider_url = replace(module.cik8s.cluster_oidc_issuer_url, "https://", "")
role_policy_arns = [aws_iam_policy.ebs_csi.arn]
oidc_fully_qualified_subjects = ["system:serviceaccount:${local.ebs_account_namespace}:${local.ebs_account_name}"]

tags = merge(local.common_tags, {
associated_service = "eks/${module.cik8s.cluster_name}"
})
}

# Configure the jenkins-infra/kubernetes-management admin service account
module "cik8s_admin_sa" {
providers = {
kubernetes = kubernetes.cik8s
}
source = "./.shared-tools/terraform/modules/kubernetes-admin-sa"
cluster_name = module.cik8s.cluster_name
cluster_hostname = module.cik8s.cluster_endpoint
cluster_ca_certificate_b64 = module.cik8s.cluster_certificate_authority_data
}

output "kubeconfig_cik8s" {
sensitive = true
value = module.cik8s_admin_sa.kubeconfig
}

data "aws_eks_cluster" "cik8s" {
name = local.cik8s_cluster_name
}

data "aws_eks_cluster_auth" "cik8s" {
name = local.cik8s_cluster_name
}

## No restriction on the resources: either managed outside terraform, or already scoped by conditions
#trivy:ignore:aws-iam-no-policy-wildcards
data "aws_iam_policy_document" "cluster_autoscaler_cik8s" {
# Statements as per https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md#full-cluster-autoscaler-features-policy-recommended
statement {
sid = "unrestricted"
effect = "Allow"

actions = [
"autoscaling:DescribeAutoScalingGroups",
"autoscaling:DescribeAutoScalingInstances",
"autoscaling:DescribeLaunchConfigurations",
"autoscaling:DescribeScalingActivities",
"autoscaling:DescribeTags",
"ec2:DescribeInstanceTypes",
"ec2:DescribeLaunchTemplateVersions"
]

resources = ["*"]
}

statement {
sid = "restricted"
effect = "Allow"

actions = [
"autoscaling:SetDesiredCapacity",
"autoscaling:TerminateInstanceInAutoScalingGroup",
"ec2:DescribeImages",
"ec2:GetInstanceTypesFromInstanceRequirements",
"eks:DescribeNodegroup"
]

resources = ["*"]
}
}

resource "aws_iam_policy" "cluster_autoscaler_cik8s" {
name_prefix = "cluster-autoscaler-cik8s"
description = "EKS cluster-autoscaler policy for cluster ${module.cik8s.cluster_name}"
policy = data.aws_iam_policy_document.cluster_autoscaler_cik8s.json
}
171 changes: 1 addition & 170 deletions eks-public-cluster.tf
Original file line number Diff line number Diff line change
Expand Up @@ -68,8 +68,7 @@ module "eks-public" {
}
# https://github.com/kubernetes-sigs/aws-ebs-csi-driver/blob/master/CHANGELOG.md
aws-ebs-csi-driver = {
addon_version = "v1.28.0-eksbuild.1"
service_account_role_arn = module.eks-public_irsa_ebs.iam_role_arn
addon_version = "v1.28.0-eksbuild.1"
}
}

Expand Down Expand Up @@ -108,175 +107,7 @@ module "eks-public" {
]
}

## No restriction on the resources: either managed outside terraform, or already scoped by conditions
#trivy:ignore:aws-iam-no-policy-wildcards
data "aws_iam_policy_document" "cluster_autoscaler_public" {
statement {
sid = "ec2"
effect = "Allow"

actions = [
"ec2:DescribeLaunchTemplateVersions",
"ec2:DescribeInstanceTypes",
]

resources = ["*"]
}

statement {
sid = "ec2AutoScaling"
effect = "Allow"

actions = [
"autoscaling:DescribeAutoScalingGroups",
"autoscaling:DescribeAutoScalingInstances",
"autoscaling:DescribeLaunchConfigurations",
"autoscaling:DescribeTags",
]


resources = ["*"]
}

statement {
sid = "clusterAutoscalerOwn"
effect = "Allow"

actions = [
"autoscaling:SetDesiredCapacity",
"autoscaling:TerminateInstanceInAutoScalingGroup",
"autoscaling:UpdateAutoScalingGroup",
]

resources = ["*"]

condition {
test = "StringEquals"
variable = "autoscaling:ResourceTag/kubernetes.io/cluster/${module.eks-public.cluster_name}"
values = ["owned"]
}

condition {
test = "StringEquals"
variable = "autoscaling:ResourceTag/k8s.io/cluster-autoscaler/enabled"
values = ["true"]
}
}
}

resource "aws_iam_policy" "cluster_autoscaler_public" {
name_prefix = "cluster-autoscaler-public"
description = "EKS cluster-autoscaler policy for cluster ${local.public_cluster_name}"
policy = data.aws_iam_policy_document.cluster_autoscaler_public.json
}

module "eks_iam_assumable_role_autoscaler_eks_public" {
source = "terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc"
version = "5.39.1"
create_role = true
role_name = "${local.autoscaler_account_name}-eks-public"
provider_url = replace(module.eks-public.cluster_oidc_issuer_url, "https://", "")
role_policy_arns = [aws_iam_policy.cluster_autoscaler_public.arn]
oidc_fully_qualified_subjects = ["system:serviceaccount:${local.autoscaler_account_namespace}:${local.autoscaler_account_name}"]

tags = merge(local.common_tags, {
associated_service = "eks/${module.eks-public.cluster_name}"
})
}

module "eks-public_irsa_nlb" {
source = "terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc"
version = "5.30.0"
create_role = true
role_name = "${local.nlb_account_name}-eks-public"
provider_url = replace(module.eks-public.cluster_oidc_issuer_url, "https://", "")
role_policy_arns = [aws_iam_policy.cluster_nlb.arn]
oidc_fully_qualified_subjects = ["system:serviceaccount:${local.nlb_account_namespace}:${local.nlb_account_name}"]

tags = merge(local.common_tags, {
associated_service = "eks/${module.eks-public.cluster_name}"
})
}

module "eks-public_irsa_ebs" {
source = "terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc"
version = "5.30.0"
create_role = true
role_name = "${local.ebs_account_name}-eks-public"
provider_url = replace(module.eks-public.cluster_oidc_issuer_url, "https://", "")
role_policy_arns = [aws_iam_policy.ebs_csi.arn]
oidc_fully_qualified_audiences = ["sts.amazonaws.com"]
oidc_fully_qualified_subjects = ["system:serviceaccount:${local.ebs_account_namespace}:${local.ebs_account_name}"]

tags = merge(local.common_tags, {
associated_service = "eks/${module.eks-public.cluster_name}"
})
}

# Configure the jenkins-infra/kubernetes-management admin service account
module "eks_public_admin_sa" {
providers = {
kubernetes = kubernetes.eks-public
}
source = "./.shared-tools/terraform/modules/kubernetes-admin-sa"
cluster_name = module.eks-public.cluster_name
cluster_hostname = module.eks-public.cluster_endpoint
cluster_ca_certificate_b64 = module.eks-public.cluster_certificate_authority_data
}

output "kubeconfig_eks_public" {
sensitive = true
value = module.eks_public_admin_sa.kubeconfig
}

# Reference to allow configuration of the Terraform's kubernetes provider (in providers.tf)
data "aws_eks_cluster_auth" "public-cluster" {
name = module.eks-public.cluster_name
}

# Elastic IPs used for the Public Load Balancer (so that the addresses never change)
resource "aws_eip" "lb_public" {
count = length(module.vpc.public_subnets)
domain = "vpc"

tags = merge(local.common_tags, {
"Name" = "eks-public-loadbalancer-external-${count.index}"
})
}

# Custom Storage Classes to ensure that EBS PVC are bound to the correct availability zone
resource "kubernetes_storage_class" "ebs_sc" {
metadata {
name = "ebs-sc"
}
storage_provisioner = "ebs.csi.aws.com"
reclaim_policy = "Delete"
volume_binding_mode = "WaitForFirstConsumer"
allow_volume_expansion = true
allowed_topologies {
match_label_expressions {
key = "topology.ebs.csi.aws.com/zone"
values = ["us-east-2a"]
}
}

provider = kubernetes.eks-public
}

resource "kubernetes_storage_class" "ebs_sc_retain" {
metadata {
name = "ebs-sc-retain"
}
storage_provisioner = "ebs.csi.aws.com"
reclaim_policy = "Retain"
volume_binding_mode = "WaitForFirstConsumer"
allow_volume_expansion = true
allowed_topologies {
match_label_expressions {
key = "topology.ebs.csi.aws.com/zone"
values = ["us-east-2a"]
}
}

provider = kubernetes.eks-public
}
4 changes: 2 additions & 2 deletions providers.tf
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ provider "kubernetes" {

provider "kubernetes" {
alias = "cik8s"
host = data.aws_eks_cluster.cik8s.endpoint
cluster_ca_certificate = base64decode(data.aws_eks_cluster.cik8s.certificate_authority[0].data)
host = module.cik8s.cluster_endpoint
cluster_ca_certificate = base64decode(module.cik8s.cluster_certificate_authority_data)
token = data.aws_eks_cluster_auth.cik8s.token
}