Merge pull request #252 from sassoftware/staging

8.0.0 - December 6, 2023

.github/workflows/linter-analysis.yaml

@@ -2,7 +2,7 @@ name: Linter Analysis
 
 on:
   push:
-    branches: ['*'] # '*' will cause the workflow to run on all commits to all branches.
+    branches: [ '**' ] # '**' will cause the workflow to run on all commits to all branches, including those with path separators
 
 jobs:
   # Hadolint: Job-1
@@ -54,8 +54,15 @@ jobs:
         tflint_version: latest
         github_token: ${{ secrets.LINTER_TEST_TOKEN }}
 
+      # Necessary so we can recursively tflint our modules folder
+      # with the plugin, not needed for regular project use.
+    - name: Initializing modules
+      run: |
+       terraform -chdir=modules/aws_autoscaling init
+       terraform -chdir=modules/aws_ebs_csi init
+
     - name: Initializing TFLint
-      run: TFLINT_LOG=info tflint --init -c .tflint.hcl 
+      run: TFLINT_LOG=info tflint --init -c "$(pwd)/linting-configs/.tflint.hcl"
 
     - name: Run TFLint Action
-      run: TFLINT_LOG=info tflint -c .tflint.hcl 
+      run: TFLINT_LOG=info tflint -c "$(pwd)/linting-configs/.tflint.hcl" --recursive

Dockerfile

@@ -1,9 +1,9 @@
-ARG TERRAFORM_VERSION=1.4.5
-ARG AWS_CLI_VERSION=2.11.21
+ARG TERRAFORM_VERSION=1.6.3
+ARG AWS_CLI_VERSION=2.13.33
 FROM hashicorp/terraform:$TERRAFORM_VERSION as terraform
 
 FROM amazon/aws-cli:$AWS_CLI_VERSION
-ARG KUBECTL_VERSION=1.26.7
+ARG KUBECTL_VERSION=1.26.10
 
 WORKDIR /viya4-iac-aws
 

README.md

@@ -47,10 +47,10 @@ The following are also required:
 
 #### Terraform Requirements:
 
-- [Terraform](https://www.terraform.io/downloads.html) v1.4.5
-- [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) - v1.26.7
+- [Terraform](https://www.terraform.io/downloads.html) v1.6.3
+- [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) - v1.26.10
 - [jq](https://stedolan.github.io/jq/) v1.6
-- [AWS CLI](https://aws.amazon.com/cli) (optional; useful as an alternative to the AWS Web Console) v2.7.22
+- [AWS CLI](https://aws.amazon.com/cli) (optional; useful as an alternative to the AWS Web Console) v2.13.33
 
 #### Docker Requirements:
 

container-structure-test.yaml

@@ -17,14 +17,14 @@ commandTests:
   - name: "terraform version"
     command: "terraform"
     args: ["--version"]
-    expectedOutput: ["Terraform v1.4.5"]
+    expectedOutput: ["Terraform v1.6.3"]
   - name: "aws-cli version"
     command: "sh"
     args:
       - -c
       - |
         aws --version
-    expectedOutput: ["aws-cli/2.11.21"]
+    expectedOutput: ["aws-cli/2.13.33"]
 
 metadataTest:
   workdir: "/viya4-iac-aws"

docs/user/BYOnetwork.md

@@ -4,12 +4,14 @@ You have the option to use existing network resources with SAS Viya 4 Terraform
 
 **NOTE:** We refer to the use of existing resources as "bring your own" or "BYO" resources.
 
-| Scenario|Required Variables|Additional Requirements|Resources to be Created|
-| :--- | :--- | :--- | :--- |
-| 1. To work with an existing VPC | `vpc_id` | <ul><li>VPC does not contain any Subnets or other [Network components](https://docs.aws.amazon.com/vpc/latest/userguide/VPC_Networking.html)</li><li>VPC block size must be IPv4 with '/16' netmask (supports 65,536 IP addresses)</li><li>`DNS hostnames` and `DNS resolution` are enabled</li><li>[`subnets`](../CONFIG-VARS.md#networking) CIDR blocks must match with VPC IPv4 CIDR block</li></ul> | Subnets, NAT Gateway and Security Group|
-| 2. To configure all components of your VPC network - Subnets, Routes & associations, Internet and NAT Gateways | `vpc_id`, <br>`subnet_ids` and <br>`nat_id` | <ul><li>all requirements from Scenario #1</li><li>Subnets Availability Zones must be within the [location](../CONFIG-VARS.md#required-variables)</li><li>AWS Tags with `<prefix>` value replaced with the [prefix](../CONFIG-VARS.md#required-variables) input value for <br>- Public Subnets:<ul><li>`{"kubernetes.io/role/elb"="1"}`</li><li>`{"kubernetes.io/cluster/<prefix>-eks"="shared"}`</li></ul>-Private Subnets:<ul><li>`{"kubernetes.io/role/internal-elb"="1"}`</li><li>`{"kubernetes.io/cluster/<prefix>-eks"="shared"}`</li></ul>See [AWS docs](https://docs.aws.amazon.com/eks/latest/userguide/alb-ingress.html) for background on subnet tag requirements to match EKS Cluster name| Security Group |
-| 3. To configure all components of your VPC network and Security Groups | `vpc_id`,<br>`subnet_ids`, <br>`nat_id`, <br>`security_group_id`, <br>`cluster_security_group_id`, and <br>`workers_security_group_id` |<ul><li>all requirements from Scenarios #2 and [these pre-defined Security Groups](#security-groups)</li></ul>| None |
+|Scenario |Description|Required Variables|Optional Variables|Additional Requirements|Resources to be Created|
+| -: | :--- | :--- | :--- | :--- | :---|
+| 0|No existing network resources  | None  | | Not a BYO network scenario | IaC creates the required network resources |
+| 1|To work with an existing VPC | `vpc_id` | | <ul><li>VPC does not contain any Subnets or other [Network components](https://docs.aws.amazon.com/vpc/latest/userguide/VPC_Networking.html)</li><li>VPC block size must be IPv4 with '/16' netmask (supports 65,536 IP addresses)</li><li>`DNS hostnames` and `DNS resolution` are enabled</li><li>[`subnets`](../CONFIG-VARS.md#networking) CIDR blocks must match with VPC IPv4 CIDR block</li></ul> | Subnets, NAT Gateway and Security Groups|
+| 2|To configure all components of your VPC network - Subnets, Routes & associations and optionally Internet and NAT Gateways | `vpc_id`,<br>one `private` subnet and two `control_plane` subnets within the [subnet_ids](../CONFIG-VARS.md#use-existing) map, <br>see [Subnet requirements](../CONFIG-VARS.md#subnet-requirements) | `nat_id`, <br>`public` and `database` subnet lists within the [subnet_ids](../CONFIG-VARS.md#use-existing) map | <ul><li>all requirements from Scenario #1</li><li>Subnets Availability Zones must be within the [location](../CONFIG-VARS.md#required-variables)</li><li>AWS Tags with `<prefix>` value replaced with the [prefix](../CONFIG-VARS.md#required-variables) input value for <br>- Public Subnets:<ul><li>`{"kubernetes.io/role/elb"="1"}`</li><li>`{"kubernetes.io/cluster/<prefix>-eks"="shared"}`</li></ul>-Private Subnets:<ul><li>`{"kubernetes.io/role/internal-elb"="1"}`</li><li>`{"kubernetes.io/cluster/<prefix>-eks"="shared"}`</li></ul>See [AWS docs](https://docs.aws.amazon.com/eks/latest/userguide/network-load-balancing.html) for background on subnet tag requirements to match EKS Cluster name| Security Groups |
+| 3|To configure all components of your VPC network and Security Groups and optionally Internet and NAT Gateways| `vpc_id`,<br>one `private` subnet and two `control_plane` subnets within the [subnet_ids](../CONFIG-VARS.md#use-existing) map, <br>see [Subnet requirements](../CONFIG-VARS.md#subnet-requirements),  <br>`security_group_id`, <br>`cluster_security_group_id`, and <br>`workers_security_group_id` | `nat_id`, <br>`public` and `database` subnet lists within the [subnet_ids](../CONFIG-VARS.md#use-existing) map |<ul><li>all requirements from Scenarios #2 and [these pre-defined Security Groups](#security-groups)</li></ul>| None |
 
+**Note**: The `byo_network_scenario` IAC output value is informational only and is intended to convey the BYO network scenario that IAC has selected according to the [Use Existing](../CONFIG-VARS.md#use-existing) input variable values provided to IAC.
 
 ### Security Groups
 
@@ -50,9 +52,9 @@ For more information on these Security Groups, please see https://docs.aws.amazo
 
 When creating your BYO Network resources you should consult with your Network Administrator and use any of these methods to create a working AWS VPC Network:
 - [AWS QuickStarts for VPC](https://aws.amazon.com/quickstart/architecture/vpc/)
-- See the "simple-vpc" and "complete-vpc" examples in [terraform-aws-vpc module](https://github.com/terraform-aws-modules/terraform-aws-vpc/tree/master/examples) 
+- See the "simple-vpc" and "complete-vpc" examples in [terraform-aws-vpc module](https://github.com/terraform-aws-modules/terraform-aws-vpc/tree/master/examples)
 
-AWS documentation for reference:  
+AWS documentation for reference:
 - [How Amazon VPC works](https://docs.aws.amazon.com/vpc/latest/userguide/how-it-works.html)
 - [VPC and subnet sizing for IPv4](https://docs.aws.amazon.com/vpc/latest/userguide/VPC_Subnets.html#vpc-sizing-ipv4)
 

docs/user/TerraformAWSAuthentication.md

@@ -1,6 +1,6 @@
 # Authenticating Terraform to Access AWS
 
-In order to create and destroy AWS resources on your behalf, Terraform needs a AWS account that has sufficient permissions to perform all the actions defined in the Terraform manifest. You will need an AWS account IAM user that has at a mininum the permissions listed in [this policy](../../files/policies/devops-iac-eks-policy.json).
+In order to create and destroy AWS resources on your behalf, Terraform needs a AWS account that has sufficient permissions to perform all the actions defined in the Terraform manifest. You will need an AWS account IAM user that has at a minimum the permissions listed in [this policy](../../files/policies/devops-iac-eks-policy.json).
 
 You can either use static credentials (including temporary credentials with session token) or a [profile with a credentials file](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html).
 
@@ -28,7 +28,7 @@ TF_VAR_aws_secret_access_key=<your_aws_secret_access_key>
 TF_VAR_aws_session_token=<your_aws_session_token>
 ```
 
-> **NOTE** `AWS_SESSION_TOKEN` is optional and is only required when using you are using temporary AWS credentials. See the [AWS documention](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html) on environment variables for more information.
+> **NOTE** `AWS_SESSION_TOKEN` is optional and is only required when using you are using temporary AWS credentials. See the [AWS documentation](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html) on environment variables for more information.
 
 ## Using AWS Profile with Credentials File
 

docs/user/TerraformUsage.md

@@ -54,7 +54,7 @@ terraform apply
 
 This command can take a few minutes to complete. Once it has completed, Terraform output values are written to the console. The `kubeconfig` file for the cluster is written to `[prefix]-eks-kubeconfig.conf` in the current directory, `$(pwd)`.
 
-### Display Terrafrom Output
+### Display Terraform Output
 
 Once the cloud resources have been created using the `terraform apply` command, Terraform output values can be displayed again later at any time by running the following command:
 
@@ -68,7 +68,7 @@ After provisioning the infrastructure, you can make additional changes by modify
 
 ### Tear Down Cloud Resources
 
-To destroy all the cloud resources created with the previous comamnds, run the following command:
+To destroy all the cloud resources created with the previous commands, run the following command:
 
 ```bash
 terraform destroy

files/custom-data/additional_userdata.sh

@@ -22,12 +22,12 @@ FILESYSTEM_BLOCK_SIZE=${FILESYSTEM_BLOCK_SIZE:-4096}  # Bytes
 STRIDE=$(expr $RAID_CHUNK_SIZE \* 1024 / $FILESYSTEM_BLOCK_SIZE || true)
 STRIPE_WIDTH=$(expr $SSD_NVME_DEVICE_COUNT \* $STRIDE || true)
 
-# Checking if provisioning already happend
+# Checking if provisioning already happened
 if [[ "$(ls -A /pv-disks)" ]]
 then
   echo 'Volumes already present in "/pv-disks"'
   echo -e "\n$(ls -Al /pv-disks | tail -n +2)\n"
-  echo "I assume that provisioning already happend, doing nothing!"
+  echo "I assume that provisioning already happened, doing nothing!"
   exit 0
 fi
 

linting-configs/.tflint.hcl

@@ -14,7 +14,7 @@ config {
 
 plugin "aws" {
   enabled = true
-  version = "0.23.0"
+  version = "0.27.0"
   source  = "github.com/terraform-linters/tflint-ruleset-aws"
 }
 

locals.tf

@@ -7,7 +7,8 @@ locals {
   aws_caller_identity_user_name = element(split("/", data.aws_caller_identity.terraform.arn), length(split("/", data.aws_caller_identity.terraform.arn)) - 1)
 
   # General
-  security_group_id         = var.security_group_id == null ? aws_security_group.sg[0].id : data.aws_security_group.sg[0].id
+  sec_group                 = (length(aws_security_group.sg_a) == 0 && length(aws_security_group.sg_b) == 0) ? null : coalescelist(aws_security_group.sg_a, aws_security_group.sg_b)
+  security_group_id         = var.security_group_id == null ? local.sec_group[0].id : data.aws_security_group.sg[0].id
   cluster_security_group_id = var.cluster_security_group_id == null ? aws_security_group.cluster_security_group[0].id : var.cluster_security_group_id
   workers_security_group_id = var.workers_security_group_id == null ? aws_security_group.workers_security_group[0].id : var.workers_security_group_id
   cluster_name              = "${var.prefix}-eks"
@@ -20,17 +21,32 @@ locals {
   aws_shared_credentials = local.use_aws_shared_credentials_file ? [var.aws_shared_credentials_file] : var.aws_shared_credentials_files
 
   # CIDRs
-  default_public_access_cidrs           = var.default_public_access_cidrs == null ? [] : var.default_public_access_cidrs
-  vm_public_access_cidrs                = var.vm_public_access_cidrs == null ? local.default_public_access_cidrs : var.vm_public_access_cidrs
-  cluster_endpoint_public_access_cidrs  = var.cluster_api_mode == "private" ? [] : (var.cluster_endpoint_public_access_cidrs == null ? local.default_public_access_cidrs : var.cluster_endpoint_public_access_cidrs)
-  cluster_endpoint_private_access_cidrs = var.cluster_endpoint_private_access_cidrs == null ? distinct(concat(module.vpc.public_subnet_cidrs, module.vpc.private_subnet_cidrs)) : var.cluster_endpoint_private_access_cidrs # tflint-ignore: terraform_unused_declarations
-  postgres_public_access_cidrs          = var.postgres_public_access_cidrs == null ? local.default_public_access_cidrs : var.postgres_public_access_cidrs
+  default_public_access_cidrs  = var.default_public_access_cidrs == null ? [] : var.default_public_access_cidrs
+  default_private_access_cidrs = var.default_private_access_cidrs == null ? [] : var.default_private_access_cidrs
+
+  vm_public_access_cidrs  = var.vm_public_access_cidrs == null ? local.default_public_access_cidrs : var.vm_public_access_cidrs
+  vm_private_access_cidrs = var.vm_private_access_cidrs == null ? local.default_private_access_cidrs : var.vm_private_access_cidrs
+
+  cluster_endpoint_public_access_cidrs = var.cluster_api_mode == "private" ? [] : (var.cluster_endpoint_public_access_cidrs == null ? local.default_public_access_cidrs : var.cluster_endpoint_public_access_cidrs)
+
+  cluster_endpoint_private_access_cidrs = var.cluster_endpoint_private_access_cidrs == null ? distinct(concat(module.vpc.public_subnet_cidrs, module.vpc.private_subnet_cidrs, local.default_private_access_cidrs)) : distinct(concat(module.vpc.public_subnet_cidrs, module.vpc.private_subnet_cidrs, local.default_private_access_cidrs, var.cluster_endpoint_private_access_cidrs)) # tflint-ignore: terraform_unused_declarations
+
+  vpc_endpoint_private_access_cidrs = var.vpc_endpoint_private_access_cidrs == null ? distinct(concat(module.vpc.public_subnet_cidrs, module.vpc.private_subnet_cidrs, local.default_private_access_cidrs)) : distinct(concat(module.vpc.public_subnet_cidrs, module.vpc.private_subnet_cidrs, local.default_private_access_cidrs, var.vpc_endpoint_private_access_cidrs))
+
+  postgres_public_access_cidrs = var.postgres_public_access_cidrs == null ? local.default_public_access_cidrs : var.postgres_public_access_cidrs
 
   # Subnets
   jump_vm_subnet   = var.create_jump_public_ip ? module.vpc.public_subnets[0] : module.vpc.private_subnets[0]
   nfs_vm_subnet    = var.create_nfs_public_ip ? module.vpc.public_subnets[0] : module.vpc.private_subnets[0]
   nfs_vm_subnet_az = var.create_nfs_public_ip ? module.vpc.public_subnet_azs[0] : module.vpc.private_subnet_azs[0]
 
+  # Generate list of AZ where created subnets should be placed
+  # If not specified by the user replace with list of all AZs in a region
+  public_subnet_azs        = can(var.subnet_azs["public"]) ? var.subnet_azs["public"] : data.aws_availability_zones.available.names
+  private_subnet_azs       = can(var.subnet_azs["private"]) ? var.subnet_azs["private"] : data.aws_availability_zones.available.names
+  database_subnet_azs      = can(var.subnet_azs["database"]) ? var.subnet_azs["database"] : data.aws_availability_zones.available.names
+  control_plane_subnet_azs = can(var.subnet_azs["control_plane"]) ? var.subnet_azs["control_plane"] : data.aws_availability_zones.available.names
+
   ssh_public_key = (var.create_jump_vm || var.storage_type == "standard"
     ? file(var.ssh_public_key)
     : null
@@ -45,7 +61,7 @@ locals {
   # Kubernetes
   kubeconfig_filename = "${local.cluster_name}-kubeconfig.conf"
   kubeconfig_path     = var.iac_tooling == "docker" ? "/workspace/${local.kubeconfig_filename}" : local.kubeconfig_filename
-  kubeconfig_ca_cert  = data.aws_eks_cluster.cluster.certificate_authority[0].data
+  kubeconfig_ca_cert  = module.eks.cluster_certificate_authority_data
 
   # Mapping node_pools to node_groups
   default_node_pool = {
@@ -87,6 +103,9 @@ locals {
       launch_template_use_name_prefix = true
       launch_template_tags            = { Name = "${local.cluster_name}-default" }
       tags                            = var.autoscaling_enabled ? merge(local.tags, { key = "k8s.io/cluster-autoscaler/${local.cluster_name}", value = "owned", propagate_at_launch = true }, { key = "k8s.io/cluster-autoscaler/enabled", value = "true", propagate_at_launch = true }) : local.tags
+      # Node Pool IAM Configuration
+      iam_role_use_name_prefix = false
+      iam_role_name            = "${var.prefix}-default-eks-node-group"
     }
   }
 
@@ -133,6 +152,9 @@ locals {
       launch_template_use_name_prefix = true
       launch_template_tags            = { Name = "${local.cluster_name}-${key}" }
       tags                            = var.autoscaling_enabled ? merge(local.tags, { key = "k8s.io/cluster-autoscaler/${local.cluster_name}", value = "owned", propagate_at_launch = true }, { key = "k8s.io/cluster-autoscaler/enabled", value = "true", propagate_at_launch = true }) : local.tags
+      # Node Pool IAM Configuration
+      iam_role_use_name_prefix = false
+      iam_role_name            = "${var.prefix}-${key}-eks-node-group"
     }
   }
 
@@ -147,10 +169,10 @@ locals {
 
   postgres_outputs = length(module.postgresql) != 0 ? { for k, v in module.postgresql :
     k => {
-      "server_name" : module.postgresql[k].db_instance_id,
+      "server_name" : module.postgresql[k].db_instance_identifier,
       "fqdn" : module.postgresql[k].db_instance_address,
       "admin" : module.postgresql[k].db_instance_username,
-      "password" : module.postgresql[k].db_instance_password,
+      "password" : local.postgres_servers[k].administrator_password,
       "server_port" : module.postgresql[k].db_instance_port
       "ssl_enforcement_enabled" : local.postgres_servers[k].ssl_enforcement_enabled,
       "internal" : false