Add Terraform Check Module Version with make tsvc_<plan_name> (#33)

* Add Terraform Check Module Version with make tsvc_<plan_name>

* typo

* remove configure.yaml file

* Use gitlab dotenv for creds pass (#45)

* Update gitlab-ci jinja template to use dotenv report artifact and dependencies keywork to pass credentials between jobs

* add dependencies to needs

* Add aws-creds as job dependency for apply_all job

* Update needs to match dependencies

* Add gitlab ci pipeline for testing

* Fix typo in pipeline code

* Uncommit gitlab pipeline

* Change gitlabi place

---------

Co-authored-by: Eddy PEPY <eddy.pepy.ext@orange.com>

* Add Terraform Check Module Version with make tsvc_<plan_name>

* typo

---------

Co-authored-by: arongate <eddy.arrel@gmail.com>
Co-authored-by: Eddy PEPY <eddy.pepy.ext@orange.com>

Makefile

@@ -118,6 +118,11 @@ endif
 ########################################################################################################################
 #  FUNCTIONS
 ########################################################################################################################
+terraform_check_version_commands:
+ifndef CICD_MODE
+	$(DOCKER_COMPOSE_DEV_TOOLS) run terraform_version_check /workdir/${CURRENT_DIR}
+endif
+
 console_commands:
 ifndef CICD_MODE
 	$(TFENV_EXEC) /bin/sh -c "cd ${CURRENT_DIR} && tfenv install"
@@ -364,6 +369,20 @@ console_terraform_demo: ## Connect terraform Docker AWS terraform/demo layer
 console_terraform_demo:
 	@$(MAKE) --no-print-directory CURRENT_DIR=terraform/demo console_commands
 
+tsvc_terraform_demo: ## Check terraform module version terraform/demo
+tsvc_terraform_demo:
+	@$(MAKE) --no-print-directory CURRENT_DIR=terraform/demo terraform_check_version_commands
+
+tsvc_all: ## Install all AWS layers
+tsvc_all: tsvc_terraform_demo 
+
+tsvc_terraform_demo: ## Check terraform module version terraform/demo
+tsvc_terraform_demo:
+	@$(MAKE) --no-print-directory CURRENT_DIR=terraform/demo terraform_check_version_commands
+
+tsvc_all: ## Install all AWS layers
+tsvc_all: tsvc_terraform_demo 
+
 init_terraform_demo: ## Init AWS terraform/demo layer
 init_terraform_demo:
 	@$(MAKE) --no-print-directory CURRENT_DIR=terraform/demo terraform_init_commands
@@ -372,6 +391,14 @@ validate_terraform_demo: ## Validate AWS terraform/demo layer
 validate_terraform_demo:
 	@$(MAKE) --no-print-directory CURRENT_DIR=terraform/demo terraform_validate
 
+validate_terraform_demo: ## Validate AWS terraform/demo layer
+validate_terraform_demo:
+	@$(MAKE) --no-print-directory CURRENT_DIR=terraform/demo terraform_validate
+
+validate_terraform_demo: ## Validate AWS terraform/demo layer
+validate_terraform_demo:
+	@$(MAKE) --no-print-directory CURRENT_DIR=terraform/demo terraform_validate
+
 plan_terraform_demo: ## Plan AWS terraform/demo layer
 plan_terraform_demo:
 	@$(MAKE) --no-print-directory CURRENT_DIR=terraform/demo terraform_plan_commands

automation/jinja2/templates/.env.dist.j2

@@ -36,18 +36,19 @@ CICD_ACCOUNT_ID={{ CICD_ACCOUNT_ID }}
 ########################################################################################################################
 # Docker Compose image tags to use
 ########################################################################################################################
-TFENV_IMAGE_TAG={{ TFENV_IMAGE_TAG }}
-PRECOMMIT_IMAGE_TAG={{ PRECOMMIT_IMAGE_TAG }}
-TFLINT_IMAGE_TAG={{ TFLINT_IMAGE_TAG }}
-DRIFTCTL_IMAGE_TAG={{ DRIFTCTL_IMAGE_TAG }}
-DOTENV_LINTER_IMAGE_TAG={{ DRIFTCTL_IMAGE_TAG }}
-MARKDOWN_LINTER_IMAGE_TAG={{ MARKDOWN_LINTER_IMAGE_TAG }}
-SHELL_LINTER_IMAGE_TAG={{ SHELL_LINTER_IMAGE_TAG }}
-YAML_LINTER_IMAGE_TAG={{ YAML_LINTER_IMAGE_TAG }}
-POWERSHELL_LINTER_IMAGE_TAG={{ POWERSHELL_LINTER_IMAGE_TAG }}
-JSON_LINTER_IMAGE_TAG={{ JSON_LINTER_IMAGE_TAG }}
-TRIVY_IMAGE_TAG={{ TRIVY_IMAGE_TAG }}
-TERRASCAN_IMAGE_TAG={{ TERRASCAN_IMAGE_TAG }}
+TFENV_IMAGE_TAG={{ TFENV_IMAGE_TAG | default("latest", true) }}
+PRECOMMIT_IMAGE_TAG={{ PRECOMMIT_IMAGE_TAG | default("latest", true) }}
+TFLINT_IMAGE_TAG={{ TFLINT_IMAGE_TAG  | default("latest", true)}}
+DRIFTCTL_IMAGE_TAG={{ DRIFTCTL_IMAGE_TAG | default("latest", true) }}
+DOTENV_LINTER_IMAGE_TAG={{ DRIFTCTL_IMAGE_TAG | default("latest", true) }}
+MARKDOWN_LINTER_IMAGE_TAG={{ MARKDOWN_LINTER_IMAGE_TAG | default("latest", true) }}
+SHELL_LINTER_IMAGE_TAG={{ SHELL_LINTER_IMAGE_TAG | default("latest", true) }}
+YAML_LINTER_IMAGE_TAG={{ YAML_LINTER_IMAGE_TAG | default("latest", true) }}
+POWERSHELL_LINTER_IMAGE_TAG={{ POWERSHELL_LINTER_IMAGE_TAG | default("latest", true) }}
+JSON_LINTER_IMAGE_TAG={{ JSON_LINTER_IMAGE_TAG | default("latest", true) }}
+TRIVY_IMAGE_TAG={{ TRIVY_IMAGE_TAG | default("latest", true) }}
+TERRASCAN_IMAGE_TAG={{ TERRASCAN_IMAGE_TAG | default("latest", true) }}
+TERRAFORM_VERSION_CHECK_IMAGE_TAG={{ TERRAFORM_VERSION_CHECK_IMAGE_TAG | default("latest", true) }}
 
 ########################################################################################################################
 # Terraform Logs

automation/jinja2/templates/.gitlab-ci.yml.j2

@@ -5,7 +5,7 @@ variables:
     options:
       - BUILD
       - DESTROY
-    description: When set to DESTROY plan and delete 
+    description: When set to DESTROY plan and delete
   BUILD_TYPE:
     value: CHANGES
     options:
@@ -37,7 +37,7 @@ variables:
   TF_VAR_backend_bucket_access_role: "arn:aws:iam::{{ environ('ACCOUNT_ID') }}:role/{{ environ('CICD_ROLE_NAME') }}"
   {% if  CUSTOM_BACKEND_BUCKET_KEY is sameas false %}
   CUSTOM_BACKEND_BUCKET_KEY: auto
-  {% endif %} 
+  {% endif %}
   PLAN_BINARY_FILE: {{ environ('PLAN_BINARY_FILE') }}
   PLAN_JSON_FILE: {{ environ('PLAN_JSON_FILE') }}
   ACCOUNT_ID: {{ environ('ACCOUNT_ID') }}
@@ -389,7 +389,7 @@ driftctl:
 .validate_job: &validate_job
   extends: .terraform-base
   stage: quality-checks
-  needs: 
+  needs:
     - aws-creds
     - terraform-prepare
   dependencies:

automation/jinja2/templates/make.mk.j2

@@ -46,6 +46,18 @@ console_{{ slug }}:
 	@$(MAKE) --no-print-directory CURRENT_DIR={{ plan_name }} console_commands
 {% endfor %}
 
+{% for plan_name in plans_install %}
+{% set path = plan_name.split('/') %}
+{% set slug = plan_name.replace('/',"_") %}
+tsvc_{{ slug}}: ## Check terraform module version {{ plan_name }}
+tsvc_{{ slug }}:
+	@$(MAKE) --no-print-directory CURRENT_DIR={{ plan_name }} terraform_check_version_commands
+{% endfor %}
+
+tsvc_all: ## Install all AWS layers
+tsvc_all: {% for plan_name in plans_install %}{% set slug = plan_name.replace('/',"_") %}{% set path = plan_name.split('/') %}tsvc_{{ slug }} {% endfor %}
+
+
 {% for plan_name in plans_install %}
 {% set path = plan_name.split('/') %}
 {% set slug = plan_name.replace('/',"_") %}

configure.yaml

@@ -0,0 +1,112 @@
+########################################################################################################################
+# ENV Variables used for local development
+# Use the command $make dotenv_linter to validate this file
+########################################################################################################################
+PROJECT_NAME: starterkit
+COMPOSE_PROJECT_NAME: starterkit
+REGION: eu-west-3
+# Name of the output of the terraform plan
+PLAN_BINARY_FILE: tfplan.binary
+PLAN_JSON_FILE: tfplan.json
+
+########################################################################################################################
+# Docker Compose image tags to use
+########################################################################################################################
+TFENV_IMAGE_TAG: latest
+TFLINT_IMAGE_TAG: v0.39.3
+PRECOMMIT_IMAGE_TAG: v1.74.1
+DRIFTCTL_IMAGE_TAG: latest
+DOTENV_LINTER_IMAGE_TAG: latest
+MARKDOWN_LINTER_IMAGE_TAG: latest
+SHELL_LINTER_IMAGE_TAG: latest
+YAML_LINTER_IMAGE_TAG: latest
+POWERSHELL_LINTER_IMAGE_TAG: latest
+JSON_LINTER_IMAGE_TAG: latest
+TRIVY_IMAGE_TAG: latest
+TERRASCAN_IMAGE_TAG: latest
+TERRAFORM_VERSION_CHECK_IMAGE_TAG: latest
+
+########################################################################################################################
+# GITLAB CI
+# Use to validate the .gitlab-ci.yml file with the command $make gitlab_linter
+########################################################################################################################
+GENERATE_GITLAB_CI: False
+
+CICD_RUNNER_TAGS:
+     - aws
+
+GITLAB_JOBS:
+    aws-creds: True
+    terraform-lint: True
+    precommit: True
+    terraform-format: True
+    terraform-validate: True
+    terraform-terrascan: True
+    md-lint: True
+    shell-lint: True
+    yaml-lint: True
+    terraform-trivy: True
+    driftctl: True
+    plan_all: True
+    apply_all: True
+    delete_all: True
+
+CICD_ROLE_NAME: XXXXXX-CiCd-CrossAccountRole
+CICD_ACCOUNT_ID: 123546789123
+
+# Run Terraform apply only on main branch
+TF_APPLY_ONLY_MAIN: True
+
+# Set Terraform Token key and value to access Terraform Modules stored into private GitLab repo
+TF_TOKEN_MODULE_ACCESS: False
+TF_TOKEN_MODULE_ACCESS_KEY:
+TF_TOKEN_MODULE_ACCESS_VALUE:
+
+########################################################################################################################
+# CUSTOM ENV
+########################################################################################################################
+#CUSTOM_ENV:
+#  MYAPI: test
+
+########################################################################################################################
+# Terraform Logs
+########################################################################################################################
+TF_LOG: TRACE
+TF_LOG_PATH: /tmp/terraform_logs
+
+########################################################################################################################
+# Terraform S3 Backend Configuration
+########################################################################################################################
+# Leave empty if you want to customize the Terraform backend config into the Terraform plans
+TF_VAR_backend_bucket_name: <AWS_S3_BUCKET_NAME>
+TF_VAR_backend_bucket_region: <AWS_REGION_CODE>
+TF_VAR_backend_dynamodb_table: <AWS_DYNAMODB_TABLE_NAME>
+TF_VAR_backend_bucket_access_role: arn:aws:iam::<AWS_ACCOUNT_ID>:role/<AWS_IAM_ROLE_NAME>
+
+# Customize Terraform backend bucket key
+# True: set the key in the Terraform plan remote state confgirutation
+# False: Terraform backend bucket key is automatically generated
+CUSTOM_BACKEND_BUCKET_KEY: False
+
+########################################################################################################################
+# LOCAL DEVELOPMENT
+########################################################################################################################
+LOCAL_ROLE_NAME: <AWS_IAM_ROLE_NAME>
+TF_VAR_assume_role: <AWS_IAM_ROLE_NAME>
+
+########################################################################################################################
+# PLANS
+########################################################################################################################
+# List the folder for each Terraform Plan to install, respect the order of installation
+plans:
+  - terraform/demo
+
+# AWS Account ID in which your plan are deployed and containing the backend bucket
+# SSO Account
+ACCOUNT_ID: <AWS_ACCOUNT_ID>
+
+########################################################################################################################
+# PROXY
+########################################################################################################################
+HTTP_PROXY: ""
+HTTPS_PROXY: ""

configure.yaml.dist

@@ -27,6 +27,7 @@ POWERSHELL_LINTER_IMAGE_TAG: latest
 JSON_LINTER_IMAGE_TAG: latest
 TRIVY_IMAGE_TAG: latest
 TERRASCAN_IMAGE_TAG: latest
+TERRAFORM_VERSION_CHECK_IMAGE_TAG: latest
 
 ########################################################################################################################
 # GITLAB CI

docker-compose-tools.yml

@@ -149,3 +149,17 @@ services:
       - /etc/group:/etc/group:ro
       - ./${PROJECT_ROOT_DIR}:/project
     working_dir: ${DOCKER_WORKDIR}
+
+  terraform_version_check:
+      container_name: ${COMPOSE_PROJECT_NAME}_terraform_version_check
+      image: tfverch/tfvc:${TERRAFORM_VERSION_CHECK_IMAGE_TAG}
+      user: ${MY_UID}:${MY_GID}
+      environment:
+        HOME: /workdir
+      env_file:
+        - .env
+      volumes:
+        - /etc/passwd:/etc/passwd:ro
+        - /etc/group:/etc/group:ro
+        - ./:/workdir
+      working_dir: /workdir

docs/tools.md

@@ -162,6 +162,11 @@ By using Terraform Docs, developers can catch common mistakes such as missing or
 formatting, and incorrect usage examples. It enforces guidelines for maintaining clear, concise, and informative module
 documentation, improving the overall quality of module documentation and enhancing collaboration among team members.
 
+
+- [Terraform Check Version (tsvc)](https://github.com/tfverch/tfvc)
+- Terraform version check (tfvc) is a reporting tool to identify available updates for providers and modules referenced in your Terraform code. 
+- It provides clear warning/failure output and resolution guidance for any issues it detects.
+
 # Adding a Tool
 
 ## New Service in Docker
@@ -198,4 +203,4 @@ the Jinja template in the automation folder.
 
 Launch makefile without stopping on errors `make -k cmd` useful for the `quality-checks` target.
 
-After adding a new Terraform Plan, launch the `make start` to update the `Makefile` and `.gitlab-ci.yml` file.
+After adding a new Terraform Plan, launch the `make start` to update the `Makefile` and `.gitlab-ci.yml` file.

terraform/demo/versions.tf

@@ -1,5 +1,5 @@
 terraform {
-  required_version = "~> 1.5"
+  required_version = "~> 1.4.0"
   required_providers {
     aws = ">= 5.0"
     random = {