chore: reorganization of infrastructure as code for better local deve…

…lopment (#532) Co-authored-by: Clément Janin <clement.janin@cds-snc.ca>
cds-snc · Dec 7, 2023 · 6f84917 · 6f84917
1 parent 7440068
commit 6f84917
Show file tree

Hide file tree

Showing 136 changed files with 3,722 additions and 2,849 deletions.
diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml
@@ -12,7 +12,7 @@ services:
       - ..:/workspace:cached
     command: sleep infinity
     environment:
-      AWS_LOCALSTACK: "True"
+      LOCALSTACK: "True"
       DEVCONTAINER: "True"
       TF_VAR_cognito_client_id: ""
       TF_VAR_cognito_endpoint_url: ""
@@ -21,7 +21,7 @@ services:
       TF_VAR_email_address_support: ""
       TF_VAR_localstack_host: "host.docker.internal"
       TF_VAR_region: "ca-central-1"
-      
+
   localstack:
     image: localstack/localstack@sha256:e012933b1122dede2c2ffc686b8ccbf9e6e5db3e20562d340d5917dbf4e019e4
     hostname: localstack

diff --git a/.github/workflows/terragrunt-apply-production.yml b/.github/workflows/terragrunt-apply-production.yml
@@ -11,10 +11,10 @@ env:
   AWS_REGION: ca-central-1
   AWS_ACCESS_KEY_ID: ${{ secrets.PRODUCTION_AWS_ACCESS_KEY_ID }}
   AWS_SECRET_ACCESS_KEY: ${{ secrets.PRODUCTION_AWS_SECRET_ACCESS_KEY }}
-  TERRAFORM_VERSION: 1.4.2
-  TERRAGRUNT_VERSION: 0.46.3
+  TERRAFORM_VERSION: 1.6.5
+  TERRAGRUNT_VERSION: 0.53.8
   TF_INPUT: false
-  TF_VAR_ecs_secret_token_secret: ${{ secrets.PRODUCTION_TOKEN_SECRET }}
+  TF_VAR_ecs_secret_token: ${{ secrets.PRODUCTION_TOKEN_SECRET }}
   TF_VAR_recaptcha_secret: ${{secrets.PRODUCTION_RECAPTCHA_SITE_SECRET}}
   TF_VAR_recaptcha_public: 6LfuLrQnAAAAAK9Df3gem4XLMRVY2Laq6t2fhZhZ
   TF_VAR_notify_api_key: ${{ secrets.PRODUCTION_NOTIFY_API_KEY }}
@@ -68,6 +68,15 @@ jobs:
         working-directory: env/cloud/sqs
         run: terragrunt apply --terragrunt-non-interactive -auto-approve
 
+      - name: Terragrunt apply secrets
+        working-directory: env/cloud/secrets
+        run: terragrunt apply --terragrunt-non-interactive -auto-approve
+
+      - name: Terragrunt apply s3
+        working-directory: env/cloud/s3
+        run: terragrunt apply --terragrunt-non-interactive -auto-approve
+
+      # Depends on kms
       - name: Terragrunt apply sns
         working-directory: env/cloud/sns
         run: terragrunt apply --terragrunt-non-interactive -auto-approve
@@ -82,7 +91,6 @@ jobs:
       - name: Remove Cognito Lambda deps
         run: ./aws/cognito/lambda/deps.sh delete
 
-      # Depends on kms
       - name: Terragrunt apply network
         working-directory: env/cloud/network
         run: terragrunt apply --terragrunt-non-interactive -auto-approve
@@ -105,15 +113,19 @@ jobs:
         run: terragrunt apply --terragrunt-non-interactive -auto-approve
 
       # Depends on everything
-      - name: Install Lambda deps
-        run: ./aws/app/lambda/deps.sh install
-
       - name: Terragrunt apply app
         working-directory: env/cloud/app
         run: terragrunt apply --terragrunt-non-interactive -auto-approve
 
+      - name: Install Lambda deps
+        run: ./aws/lambdas/code/deps.sh install
+
+      - name: Terragrunt apply lambdas
+        working-directory: env/cloud/lambdas
+        run: terragrunt apply --terragrunt-non-interactive -auto-approve
+
       - name: Remove Lambda deps
-        run: ./aws/app/lambda/deps.sh delete
+        run: ./aws/lambdas/code/deps.sh delete
 
       - name: Terragrunt apply alarms
         working-directory: env/cloud/alarms

diff --git a/.github/workflows/terragrunt-apply-staging.yml b/.github/workflows/terragrunt-apply-staging.yml
@@ -17,10 +17,10 @@ env:
   AWS_REGION: ca-central-1
   AWS_ACCESS_KEY_ID: ${{ secrets.STAGING_AWS_ACCESS_KEY_ID }}
   AWS_SECRET_ACCESS_KEY: ${{ secrets.STAGING_AWS_SECRET_ACCESS_KEY }}
-  TERRAFORM_VERSION: 1.4.2
-  TERRAGRUNT_VERSION: 0.46.3
+  TERRAFORM_VERSION: 1.6.5
+  TERRAGRUNT_VERSION: 0.53.8
   TF_INPUT: false
-  TF_VAR_ecs_secret_token_secret: ${{ secrets.STAGING_TOKEN_SECRET }}
+  TF_VAR_ecs_secret_token: ${{ secrets.STAGING_TOKEN_SECRET }}
   TF_VAR_recaptcha_secret: ${{secrets.STAGING_RECAPTCHA_SITE_SECRET}}
   TF_VAR_recaptcha_public: 6LfJDN4eAAAAAGvdRF7ZnQ7ciqdo1RQnQDFmh0VY
   TF_VAR_notify_api_key: ${{ secrets.STAGING_NOTIFY_API_KEY }}
@@ -84,6 +84,9 @@ jobs:
             kms:
               - 'aws/kms/**'
               - 'env/cloud/kms/**'
+            lambdas:
+              - 'aws/lambdas/**'
+              - 'env/cloud/lambdas/**'
             load_balancer:
               - 'aws/load_balancer/**'
               - 'env/cloud/load_balancer/**'
@@ -99,6 +102,12 @@ jobs:
             redis:
               - 'aws/redis/**'
               - 'env/cloud/redis/**'
+            s3:
+              - 'aws/s3/**'
+              - 'env/cloud/s3/**'
+            secrets:
+              - 'aws/secrets/**'
+              - 'env/cloud/secrets/**'
             sns:
               - 'aws/sns/**'
               - 'env/cloud/sns/**'
@@ -127,6 +136,17 @@ jobs:
         working-directory: env/cloud/sqs
         run: terragrunt apply --terragrunt-non-interactive -auto-approve
 
+      - name: Terragrunt apply secrets
+        if: ${{ steps.filter.outputs.secrets == 'true' || steps.filter.outputs.common == 'true' }}
+        working-directory: env/cloud/secrets
+        run: terragrunt apply --terragrunt-non-interactive -auto-approve
+
+      - name: Terragrunt apply s3
+        if: ${{ steps.filter.outputs.s3 == 'true' || steps.filter.outputs.common == 'true' }}
+        working-directory: env/cloud/s3
+        run: terragrunt apply --terragrunt-non-interactive -auto-approve
+
+      # Depends on kms
       - name: Terragrunt apply sns
         if: ${{ steps.filter.outputs.sns == 'true' || steps.filter.outputs.common == 'true' }}
         working-directory: env/cloud/sns
@@ -145,7 +165,6 @@ jobs:
         if: ${{ steps.filter.outputs.cognito == 'true' || steps.filter.outputs.common == 'true' }}
         run: ./aws/cognito/lambda/deps.sh delete
 
-      # Depends on kms
       - name: Terragrunt apply network
         if: ${{ steps.filter.outputs.network == 'true' || steps.filter.outputs.common == 'true' }}
         working-directory: env/cloud/network
@@ -173,18 +192,23 @@ jobs:
         run: terragrunt apply --terragrunt-non-interactive -auto-approve
 
       # Depends on everything
-      - name: Install Lambda deps
-        if: ${{ steps.filter.outputs.app == 'true' || steps.filter.outputs.common == 'true' }}
-        run: ./aws/app/lambda/deps.sh install
-
       - name: Terragrunt apply app
         if: ${{ steps.filter.outputs.app == 'true' || steps.filter.outputs.common == 'true' }}
         working-directory: env/cloud/app
         run: terragrunt apply --terragrunt-non-interactive -auto-approve
 
+      - name: Install Lambda deps
+        if: ${{ steps.filter.outputs.lambdas == 'true' || steps.filter.outputs.common == 'true' }}
+        run: ./aws/lambdas/code/deps.sh install
+
+      - name: Terragrunt apply lambdas
+        if: ${{ steps.filter.outputs.lambdas == 'true' || steps.filter.outputs.common == 'true' }}
+        working-directory: env/cloud/lambdas
+        run: terragrunt apply --terragrunt-non-interactive -auto-approve
+
       - name: Remove Lambda deps
-        if: ${{ steps.filter.outputs.app == 'true' || steps.filter.outputs.common == 'true' }}
-        run: ./aws/app/lambda/deps.sh delete
+        if: ${{ steps.filter.outputs.lambdas == 'true' || steps.filter.outputs.common == 'true' }}
+        run: ./aws/lambdas/code/deps.sh delete
 
       - name: Terragrunt apply alarms
         if: ${{ steps.filter.outputs.alarms == 'true' || steps.filter.outputs.common == 'true' }}

diff --git a/.github/workflows/terragrunt-plan-all-staging.yml b/.github/workflows/terragrunt-plan-all-staging.yml
@@ -11,10 +11,10 @@ env:
   AWS_ACCESS_KEY_ID: ${{ secrets.STAGING_AWS_ACCESS_KEY_ID }}
   AWS_SECRET_ACCESS_KEY: ${{ secrets.STAGING_AWS_SECRET_ACCESS_KEY }}
   CONFTEST_VERSION: 0.36.0
-  TERRAFORM_VERSION: 1.4.2
-  TERRAGRUNT_VERSION: 0.46.3
+  TERRAFORM_VERSION: 1.6.5
+  TERRAGRUNT_VERSION: 0.53.8
   TF_INPUT: false
-  TF_VAR_ecs_secret_token_secret: ${{ secrets.STAGING_TOKEN_SECRET }}
+  TF_VAR_ecs_secret_token: ${{ secrets.STAGING_TOKEN_SECRET }}
   TF_VAR_recaptcha_secret: ${{secrets.STAGING_RECAPTCHA_SITE_SECRET}}
   TF_VAR_recaptcha_public: 6LfJDN4eAAAAAGvdRF7ZnQ7ciqdo1RQnQDFmh0VY
   TF_VAR_notify_api_key: ${{ secrets.STAGING_NOTIFY_API_KEY }}
@@ -72,6 +72,21 @@ jobs:
           comment: "false"
           terragrunt: "true"
 
+      - name: Terragrunt plan secrets
+        uses: cds-snc/terraform-plan@28d2efe5155573489fa5b5816fad20d44d1f274b # v3.0.7
+        with:
+          directory: "env/cloud/secrets"
+          comment: "false"
+          terragrunt: "true"
+
+      - name: Terragrunt plan s3
+        uses: cds-snc/terraform-plan@28d2efe5155573489fa5b5816fad20d44d1f274b # v3.0.7
+        with:
+          directory: "env/cloud/s3"
+          comment: "false"
+          terragrunt: "true"
+
+      # Depends on kms
       - name: Terragrunt plan sns
         uses: cds-snc/terraform-plan@28d2efe5155573489fa5b5816fad20d44d1f274b # v3.0.7
         with:
@@ -92,7 +107,6 @@ jobs:
       - name: Remove Cognito Lambda deps
         run: ./aws/cognito/lambda/deps.sh delete
 
-      # Depends on kms
       - name: Terragrunt plan network
         uses: cds-snc/terraform-plan@28d2efe5155573489fa5b5816fad20d44d1f274b # v3.0.7
         with:
@@ -130,18 +144,25 @@ jobs:
           terragrunt: "true"
 
       # Depends on everything
-      - name: Install Lambda deps
-        run: ./aws/app/lambda/deps.sh install
-
       - name: Terragrunt plan app
         uses: cds-snc/terraform-plan@28d2efe5155573489fa5b5816fad20d44d1f274b # v3.0.7
         with:
           directory: "env/cloud/app"
           comment: "false"
           terragrunt: "true"
 
+      - name: Install Lambda deps
+        run: ./aws/lambdas/code/deps.sh install
+
+      - name: Terragrunt plan lambdas
+        uses: cds-snc/terraform-plan@28d2efe5155573489fa5b5816fad20d44d1f274b # v3.0.7
+        with:
+          directory: "env/cloud/lambdas"
+          comment: "false"
+          terragrunt: "true"
+
       - name: Remove Lambda deps
-        run: ./aws/app/lambda/deps.sh delete
+        run: ./aws/lambdas/code/deps.sh delete
 
       - name: Terragrunt plan alarms
         uses: cds-snc/terraform-plan@28d2efe5155573489fa5b5816fad20d44d1f274b # v3.0.7

diff --git a/.github/workflows/terragrunt-plan-production.yml b/.github/workflows/terragrunt-plan-production.yml
@@ -13,10 +13,10 @@ env:
   AWS_ACCESS_KEY_ID: ${{ secrets.PRODUCTION_AWS_ACCESS_KEY_ID }}
   AWS_SECRET_ACCESS_KEY: ${{ secrets.PRODUCTION_AWS_SECRET_ACCESS_KEY }}
   CONFTEST_VERSION: 0.36.0
-  TERRAFORM_VERSION: 1.4.2
-  TERRAGRUNT_VERSION: 0.46.3
+  TERRAFORM_VERSION: 1.6.5
+  TERRAGRUNT_VERSION: 0.53.8
   TF_INPUT: false
-  TF_VAR_ecs_secret_token_secret: ${{ secrets.PRODUCTION_TOKEN_SECRET }}
+  TF_VAR_ecs_secret_token: ${{ secrets.PRODUCTION_TOKEN_SECRET }}
   TF_VAR_recaptcha_secret: ${{secrets.PRODUCTION_RECAPTCHA_SITE_SECRET}}
   TF_VAR_recaptcha_public: 6LfuLrQnAAAAAK9Df3gem4XLMRVY2Laq6t2fhZhZ
   TF_VAR_notify_api_key: ${{ secrets.PRODUCTION_NOTIFY_API_KEY }}
@@ -83,6 +83,24 @@ jobs:
           github-token: "${{ secrets.GITHUB_TOKEN }}"
           terragrunt: "true"
 
+      - name: Terragrunt plan secrets
+        uses: cds-snc/terraform-plan@28d2efe5155573489fa5b5816fad20d44d1f274b # v3.0.7
+        with:
+          directory: "env/cloud/secrets"
+          comment-delete: "true"
+          comment-title: "Production: secrets"
+          github-token: "${{ secrets.GITHUB_TOKEN }}"
+          terragrunt: "true"
+      - name: Terragrunt plan s3
+        uses: cds-snc/terraform-plan@28d2efe5155573489fa5b5816fad20d44d1f274b # v3.0.7
+        with:
+          directory: "env/cloud/s3"
+          comment-delete: "true"
+          comment-title: "Production: s3"
+          github-token: "${{ secrets.GITHUB_TOKEN }}"
+          terragrunt: "true"
+
+      # Depends on kms
       - name: Terragrunt plan sns
         uses: cds-snc/terraform-plan@28d2efe5155573489fa5b5816fad20d44d1f274b # v3.0.7
         with:
@@ -107,7 +125,6 @@ jobs:
       - name: Remove Cognito Lambda deps
         run: ./aws/cognito/lambda/deps.sh delete
 
-      # Depends on kms
       - name: Terragrunt plan network
         uses: cds-snc/terraform-plan@28d2efe5155573489fa5b5816fad20d44d1f274b # v3.0.7
         with:
@@ -155,8 +172,6 @@ jobs:
           terragrunt: "true"
 
       # Depends on everything
-      - name: Install Lambda deps
-        run: ./aws/app/lambda/deps.sh install
 
       - name: Terragrunt plan app
         uses: cds-snc/terraform-plan@28d2efe5155573489fa5b5816fad20d44d1f274b # v3.0.7
@@ -167,8 +182,20 @@ jobs:
           github-token: "${{ secrets.GITHUB_TOKEN }}"
           terragrunt: "true"
 
+      - name: Install Lambda deps
+        run: ./aws/lambdas/code/deps.sh install
+
+      - name: Terragrunt plan lambdas
+        uses: cds-snc/terraform-plan@28d2efe5155573489fa5b5816fad20d44d1f274b # v3.0.7
+        with:
+          directory: "env/cloud/lambdas"
+          comment-delete: "true"
+          comment-title: "Staging: lambdas"
+          github-token: "${{ secrets.GITHUB_TOKEN }}"
+          terragrunt: "true"
+
       - name: Remove Lambda deps
-        run: ./aws/app/lambda/deps.sh delete
+        run: ./aws/lambdas/code/deps.sh delete
 
       - name: Terragrunt plan alarms
         uses: cds-snc/terraform-plan@28d2efe5155573489fa5b5816fad20d44d1f274b # v3.0.7