Use gitlab dotenv for creds pass (#45)

* Update gitlab-ci jinja template to use dotenv report artifact and dependencies keywork to pass credentials between jobs

* add dependencies to needs

* Add aws-creds as job dependency for apply_all job

* Update needs to match dependencies

* Add gitlab ci pipeline for testing

* Fix typo in pipeline code

* Uncommit gitlab pipeline

* Change gitlabi place

---------

Co-authored-by: Eddy PEPY <eddy.pepy.ext@orange.com>

Makefile

@@ -353,6 +353,10 @@ init_terraform_demo: ## Init AWS terraform/demo layer
 init_terraform_demo:
 	@$(MAKE) --no-print-directory CURRENT_DIR=terraform/demo terraform_init_commands
 
+validate_terraform_demo: ## Validate AWS terraform/demo layer
+validate_terraform_demo:
+	@$(MAKE) --no-print-directory CURRENT_DIR=terraform/demo terraform_validate
+
 plan_terraform_demo: ## Plan AWS terraform/demo layer
 plan_terraform_demo:
 	@$(MAKE) --no-print-directory CURRENT_DIR=terraform/demo terraform_plan_commands

automation/jinja2/templates/.gitlab-ci.yml.j2

@@ -40,12 +40,6 @@ variables:
   {%- set plans_delete = environ('PLAN_DELETE_LIST').split(',') %}
   {%- set runners = environ('CICD_RUNNER_TAGS').split(',') %}
 
-
-cache:
-  key: $CI_COMMIT_REF_SLUG
-  paths:
-    - creds.env
-
 stages:
   - aws-creds
   - quality-checks
@@ -80,9 +74,12 @@ stages:
     AWS_ACCESS_KEY_ID=$(echo $STS_CREDS | jq -r '.Credentials.AccessKeyId')
     AWS_SECRET_ACCESS_KEY=$(echo $STS_CREDS | jq -r '.Credentials.SecretAccessKey')
     AWS_SESSION_TOKEN=$(echo $STS_CREDS | jq -r '.Credentials.SessionToken')
-    echo "export AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID" > creds.env
-    echo "export AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY" >> creds.env
-    echo "export AWS_SESSION_TOKEN=$AWS_SESSION_TOKEN" >> creds.env
+    echo "AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID" > creds.env
+    echo "AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY" >> creds.env
+    echo "AWS_SESSION_TOKEN=$AWS_SESSION_TOKEN" >> creds.env
+  artifacts:
+    reports:
+      dotenv: creds.env
 
 .terraform-base:
   before_script:
@@ -101,7 +98,6 @@ stages:
 .terraform-trivy:
   before_script:
     - rm -rf .terraform
-    - . ./creds.env
     - apk --no-cache --update  add make
   image:
     name: aquasec/trivy:latest
@@ -179,7 +175,6 @@ stages:
       - 'PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'
   before_script:
     - apt-get update && apt-get install -y make
-    - . ./creds.env
   tags:
   {% for runner in runners %}
     - {{ runner }}
@@ -193,8 +188,6 @@ stages:
     entrypoint:
       - '/usr/bin/env'
       - 'PATH=/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'
-  before_script:
-    - . ./creds.env
   tags:
   {% for runner in runners %}
     - {{ runner }}
@@ -210,7 +203,6 @@ stages:
       - 'PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/app/bin'
   before_script:
     - apk --no-cache --update  add make
-    - . ./creds.env
   tags:
   {% for runner in runners %}
     - {{ runner }}
@@ -232,7 +224,11 @@ aws-creds:
 # TESTS
 ########################################################################################################################
 terraform-compliance:
-  needs: [plan_all]
+  needs:
+    - aws-creds
+    - plan_all
+  dependencies:
+    - aws-creds
   extends: .terraform-compliance
   allow_failure: true
   stage: tests
@@ -277,19 +273,23 @@ terraform-format:
 {% endif %}
 {% if  GITLAB_JOBS["terraform-validate"]  %}
 terraform-validate:
-  needs: []
+  needs:
+    - aws-creds
+  dependencies:
+    - aws-creds
   extends: .terraform-base
   allow_failure: true
-  before_script:
-    - . ./creds.env
   stage: quality-checks
   script:
     - make validate
 
 {% endif %}
 {% if  GITLAB_JOBS["terraform-terrascan"]  %}
 terraform-terrascan:
-  needs: []
+  needs:
+    - aws-creds
+  dependencies:
+    - aws-creds
   extends: .terraform-terrascan
   allow_failure: true
   stage: quality-checks
@@ -299,7 +299,10 @@ terraform-terrascan:
 {% endif %}
 {% if  GITLAB_JOBS["md-lint"]  %}
 md-lint:
-  needs: []
+  needs:
+    - aws-creds
+  dependencies:
+    - aws-creds
   extends: .md_lint
   allow_failure: true
   stage: quality-checks
@@ -309,11 +312,13 @@ md-lint:
 {% endif %}
 {% if  GITLAB_JOBS["shell-lint"]  %}
 shell-lint:
-  needs: []
+  needs:
+    - aws-creds
+  dependencies:
+    - aws-creds
   extends: .shelllint
   allow_failure: true
   before_script:
-    - . ./creds.env
     - apk --no-cache --update  add make
   stage: quality-checks
   script:
@@ -332,7 +337,10 @@ yaml-lint:
 {% endif %}
 {% if  GITLAB_JOBS["terraform-trivy"]  %}
 terraform-trivy:
-  needs: []
+  needs:
+    - aws-creds
+  dependencies:
+    - aws-creds
   extends: .terraform-trivy
   allow_failure: true
   stage: quality-checks
@@ -346,7 +354,10 @@ terraform-trivy:
 ########################################################################################################################
 driftctl:
   stage: drift
-  needs: [aws-creds]
+  needs:
+    - aws-creds
+  dependencies:
+    - aws-creds
   allow_failure: true
   image:
     name: snyk/driftctl:$DRIFTCTL_IMAGE_TAG
@@ -356,7 +367,6 @@ driftctl:
     ROLE_TO_ASSUME: ${TF_VAR_backend_bucket_access_role}
     AWS_ROLE_SESSION_NAME: "sessiondrifctl"
   before_script:
-    - . ./creds.env
     - apk add --no-cache aws-cli
     - apk add --no-cache jq
   script: |
@@ -377,10 +387,11 @@ driftctl:
 .plan_job: &plan_job
   extends: .terraform-base
   stage: plan
-  needs: [aws-creds]
+  needs:
+    - aws-creds
+  dependencies:
+    - aws-creds
   allow_failure: false
-  before_script:
-    - . ./creds.env
 {% for plan_name in plans_install +%}
 {% set path = plan_name.split('/') %}
 {% set slug = plan_name.replace('/',"_") %}
@@ -416,14 +427,16 @@ plan_all:
   stage: apply
   allow_failure: false
   when: manual
-  before_script:
-    - . ./creds.env
+  dependencies:
+    - aws-creds
 {% for plan_name in plans_install +%}
 {% set path = plan_name.split('/') %}
 {% set slug = plan_name.replace('/',"_") %}
 apply_{{ slug }}:
   <<: *apply_job
-  needs: [plan_{{ slug }}]
+  needs:
+    - aws-creds
+    - plan_{{ slug }}
   script:
     - make install_{{ slug }}
   only:
@@ -438,7 +451,7 @@ apply_{{ slug }}:
 {% if  GITLAB_JOBS["apply_all"]  %}
 apply_all:
   <<: *apply_job
-  needs: [plan_all]
+  needs: [aws-creds, plan_all]
   script:
     - make install_all
 {% if TF_APPLY_ONLY_MAIN %}
@@ -453,11 +466,13 @@ apply_all:
 ########################################################################################################################
 .delete_job: &delete_job
   extends: .terraform-base
+  needs:
+    - aws-creds
+  dependencies:
+    - aws-creds
   allow_failure: false
   stage: delete
   when: manual
-  before_script:
-    - . ./creds.env
   only:
     variables:
       - $PHASE == "DESTROY"
@@ -475,4 +490,4 @@ delete_all:
   <<: *delete_job
   script:
     - make delete_all
-{% endif %}
+{% endif %}