feature: deploy redis and postgresql in localstack (#620)

* feature: implement Redis and PostgreSQL in Localstack * feature: make it possible to tweak Localstack persistence settings using environment variables * feature: reintroduce clean terraform state files option in localstack services script * chore: update README.md * Update env/cloud/secrets/terragrunt.hcl Co-authored-by: Bryan Robitaille <bryan.robitaille@cds-snc.ca> --------- Co-authored-by: Bryan Robitaille <bryan.robitaille@cds-snc.ca>
cds-snc · Apr 9, 2024 · 20e0fc1 · 20e0fc1
1 parent 5544ec6
commit 20e0fc1
Show file tree

Hide file tree

Showing 36 changed files with 439 additions and 1,220 deletions.
diff --git a/.env.example b/.env.example
@@ -1,3 +1,12 @@
-APP_ENV="local"
-NOTIFY_API_KEY="<can_be_found_in_1Password>"
-LOCALSTACK_AUTH_TOKEN="your_pro_license_token"
+APP_ENV=local
+
+LOCALSTACK_AUTH_TOKEN=
+
+# Default values will be SNAPSHOT_LOAD_STRATEGY=ON_STARTUP and SNAPSHOT_SAVE_STRATEGY=SCHEDULED 
+# with a SNAPSHOT_FLUSH_INTERVAL=60 if you do not set anything below. The interval variable is only used if the save strategy is set to scheduled.
+# See https://docs.localstack.cloud/user-guide/state-management/persistence/#save-strategies for more information.
+SNAPSHOT_SAVE_STRATEGY=
+SNAPSHOT_FLUSH_INTERVAL=
+SNAPSHOT_LOAD_STRATEGY=
+
+NOTIFY_API_KEY=
diff --git a/README.md b/README.md
diff --git a/aws/alarms/lambda.tf b/aws/alarms/lambda.tf
@@ -28,7 +28,7 @@ resource "aws_lambda_function" "notify_slack" {
 
   environment {
     variables = {
-      ENVIRONMENT      = title(var.env)
+      ENVIRONMENT      = var.env
       SLACK_WEBHOOK    = var.slack_webhook
       OPSGENIE_API_KEY = var.opsgenie_api_key
     }

diff --git a/aws/alarms/lambda/notify_slack/utils.ts b/aws/alarms/lambda/notify_slack/utils.ts
@@ -43,7 +43,7 @@ export const ungzip = (input: Buffer) => {
 
 export const sendToSlack = async (logGroup: string, logMessage: string, logLevel: string) => {
   console.log("Sending to Slack...");
-  const environment = process.env.ENVIRONMENT || "Staging";
+  const environment = process.env.ENVIRONMENT || "staging";
   const logLevelThemeForSlack = logLevelAsEmojiAndColor(logLevel);
   const postData = {
     channel: `#forms-${environment.toLowerCase()}-events`,

diff --git a/aws/ecr/ecr.tf b/aws/ecr/ecr.tf
@@ -45,8 +45,6 @@ resource "aws_ecr_repository" "load_test_repository" {
   image_scanning_configuration {
     scan_on_push = true
   }
-
-
 }
 
 resource "aws_ecr_lifecycle_policy" "load_test_policy" {

diff --git a/aws/lambdas/code/form_archiver/lib/templates.ts b/aws/lambdas/code/form_archiver/lib/templates.ts
@@ -1,16 +1,21 @@
 import { RDSDataClient, ExecuteStatementCommand } from "@aws-sdk/client-rds-data";
-import pg from "pg";
-
-const REGION = process.env.REGION;
 
 /**
  * Delete all form templates that have been marked as archived (has an TTL value that is not null)
  */
-const deleteFormTemplatesMarkedAsArchived = async () => {
+export const deleteFormTemplatesMarkedAsArchived = async () => {
   try {
-    const request = `DELETE FROM "Template" WHERE ttl IS NOT NULL AND ttl < CURRENT_TIMESTAMP`;
-    const database = process.env.LOCALSTACK === "true" ? requestSAM : requestRDS;
-    await database(request, []);
+    const rdsDataClient = new RDSDataClient({ region: process.env.REGION });
+
+    const executeStatementCommand = new ExecuteStatementCommand({
+      database: process.env.DB_NAME,
+      resourceArn: process.env.DB_ARN,
+      secretArn: process.env.DB_SECRET,
+      sql: `DELETE FROM "Template" WHERE ttl IS NOT NULL AND ttl < CURRENT_TIMESTAMP`,
+      includeResultMetadata: false, // set to true if we want metadata like column names
+    });
+
+    await rdsDataClient.send(executeStatementCommand);
   } catch (error) {
     // Warn Message will be sent to slack
     console.warn(
@@ -22,90 +27,3 @@ const deleteFormTemplatesMarkedAsArchived = async () => {
     );
   }
 };
-
-/**
- * Creates and processes request to LOCAL AWS SAM DB
- * @param {string} SQL
- * @param {string[]} parameters
- * @returns PG Client return value
- */
-const requestSAM = async (SQL: string, parameters: Array<string>) => {
-  const dbClient = new pg.Client();
-
-  try {
-    if (
-      process.env.PGHOST &&
-      process.env.PGUSER &&
-      process.env.PGDATABASE &&
-      process.env.PGPASSWORD
-    ) {
-      dbClient.connect();
-    } else {
-      throw new Error("Missing Environment Variables for DB config");
-    }
-
-    const data = await dbClient.query(SQL, parameters);
-
-    return parseConfig(data.rows);
-  } catch (error) {
-    throw new Error(
-      `Error issuing command to Local SAM AWS DB. Reason: ${(error as Error).message}.`
-    );
-  } finally {
-    dbClient.end();
-  }
-};
-
-/**
- * Creates and processes request to RDS
- * @param {string} SQL
- * @param {{name: string, value: {stringValue: string}[]}} parameters
- * @returns RDS client return value
- */
-const requestRDS = async (
-  SQL: string,
-  parameters: { name: string; value: { stringValue: string } }[]
-) => {
-  try {
-    const dbClient = new RDSDataClient({ region: REGION });
-
-    const params = {
-      database: process.env.DB_NAME,
-      resourceArn: process.env.DB_ARN,
-      secretArn: process.env.DB_SECRET,
-      sql: SQL,
-      includeResultMetadata: false, // set to true if we want metadata like column names
-      parameters: parameters,
-    };
-
-    const command = new ExecuteStatementCommand(params);
-
-    const data = await dbClient.send(command);
-
-    return parseConfig(data.records);
-  } catch (error) {
-    throw new Error(`Error issuing command to AWS RDS. Reason: ${(error as Error).message}.`);
-  }
-};
-
-const parseConfig = (records: any[] | undefined) => {
-  if (records) {
-    const parsedRecords = records.map((record) => {
-      let formConfig;
-      if (!(process.env.LOCALSTACK === "true")) {
-        formConfig = JSON.parse(record[0].stringValue.trim(1, -1)) || undefined;
-      } else {
-        formConfig = record.jsonConfig;
-      }
-      return {
-        formConfig,
-      };
-    });
-
-    return { records: parsedRecords };
-  }
-
-  return { records: [] };
-};
-
-export { deleteFormTemplatesMarkedAsArchived };
diff --git a/aws/lambdas/code/form_archiver/package.json b/aws/lambdas/code/form_archiver/package.json
@@ -11,13 +11,11 @@
     "postbuild": "cp package.json dist/package.json && cp yarn.lock dist/yarn.lock && cd ./dist && yarn install --production"
   },
   "dependencies": {
-    "@aws-sdk/client-rds-data": "3.430.0",
-    "pg": "^8.7.3"
+    "@aws-sdk/client-rds-data": "3.430.0"
   },
   "devDependencies": {
     "@types/aws-lambda": "^8.10.126",
     "@types/node": "^20.9.0",
-    "@types/pg": "^8.10.9",
     "typescript": "^5.2.2"
   }
 }
diff --git a/aws/lambdas/code/form_archiver/yarn.lock b/aws/lambdas/code/form_archiver/yarn.lock
@@ -858,179 +858,25 @@
   resolved "https://registry.yarnpkg.com/@types/aws-lambda/-/aws-lambda-8.10.128.tgz#1c26e3e062e33961b250355cb4c9b6b3544b74a9"
   integrity sha512-bw4+ORfyywsj9Tq3WJA9j6HG4EbYy+dz+k8LN73ApmwjTdf82ZCSELz3b7BJQzeG5HE8IaTy9SXTHmruIK5ZlQ==
 
-"@types/node@*", "@types/node@^20.9.0":
+"@types/node@^20.9.0":
   version "20.9.4"
   resolved "https://registry.yarnpkg.com/@types/node/-/node-20.9.4.tgz#cc8f970e869c26834bdb7ed480b30ede622d74c7"
   integrity sha512-wmyg8HUhcn6ACjsn8oKYjkN/zUzQeNtMy44weTJSM6p4MMzEOuKbA3OjJ267uPCOW7Xex9dyrNTful8XTQYoDA==
   dependencies:
     undici-types "~5.26.4"
 
-"@types/pg@^8.10.9":
-  version "8.10.9"
-  resolved "https://registry.yarnpkg.com/@types/pg/-/pg-8.10.9.tgz#d20bb948c6268c5bd847e2bf968f1194c5a2355a"
-  integrity sha512-UksbANNE/f8w0wOMxVKKIrLCbEMV+oM1uKejmwXr39olg4xqcfBDbXxObJAt6XxHbDa4XTKOlUEcEltXDX+XLQ==
-  dependencies:
-    "@types/node" "*"
-    pg-protocol "*"
-    pg-types "^4.0.1"
-
 bowser@^2.11.0:
   version "2.11.0"
   resolved "https://registry.yarnpkg.com/bowser/-/bowser-2.11.0.tgz#5ca3c35757a7aa5771500c70a73a9f91ef420a8f"
   integrity sha512-AlcaJBi/pqqJBIQ8U9Mcpc9i8Aqxn88Skv5d+xBX006BY5u8N3mGLHa5Lgppa7L/HfwgwLgZ6NYs+Ag6uUmJRA==
 
-buffer-writer@2.0.0:
-  version "2.0.0"
-  resolved "https://registry.yarnpkg.com/buffer-writer/-/buffer-writer-2.0.0.tgz#ce7eb81a38f7829db09c873f2fbb792c0c98ec04"
-  integrity sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw==
-
 fast-xml-parser@4.2.5:
   version "4.2.5"
   resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-4.2.5.tgz#a6747a09296a6cb34f2ae634019bf1738f3b421f"
   integrity sha512-B9/wizE4WngqQftFPmdaMYlXoJlJOYxGQOanC77fq9k8+Z0v5dDSVh+3glErdIROP//s/jgb7ZuxKfB8nVyo0g==
   dependencies:
     strnum "^1.0.5"
 
-obuf@~1.1.2:
-  version "1.1.2"
-  resolved "https://registry.yarnpkg.com/obuf/-/obuf-1.1.2.tgz#09bea3343d41859ebd446292d11c9d4db619084e"
-  integrity sha512-PX1wu0AmAdPqOL1mWhqmlOd8kOIZQwGZw6rh7uby9fTc5lhaOWFLX3I6R1hrF9k3zUY40e6igsLGkDXK92LJNg==
-
-packet-reader@1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/packet-reader/-/packet-reader-1.0.0.tgz#9238e5480dedabacfe1fe3f2771063f164157d74"
-  integrity sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ==
-
-pg-cloudflare@^1.1.1:
-  version "1.1.1"
-  resolved "https://registry.yarnpkg.com/pg-cloudflare/-/pg-cloudflare-1.1.1.tgz#e6d5833015b170e23ae819e8c5d7eaedb472ca98"
-  integrity sha512-xWPagP/4B6BgFO+EKz3JONXv3YDgvkbVrGw2mTo3D6tVDQRh1e7cqVGvyR3BE+eQgAvx1XhW/iEASj4/jCWl3Q==
-
-pg-connection-string@^2.6.2:
-  version "2.6.2"
-  resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-2.6.2.tgz#713d82053de4e2bd166fab70cd4f26ad36aab475"
-  integrity sha512-ch6OwaeaPYcova4kKZ15sbJ2hKb/VP48ZD2gE7i1J+L4MspCtBMAx8nMgz7bksc7IojCIIWuEhHibSMFH8m8oA==
-
-pg-int8@1.0.1:
-  version "1.0.1"
-  resolved "https://registry.yarnpkg.com/pg-int8/-/pg-int8-1.0.1.tgz#943bd463bf5b71b4170115f80f8efc9a0c0eb78c"
-  integrity sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==
-
-pg-numeric@1.0.2:
-  version "1.0.2"
-  resolved "https://registry.yarnpkg.com/pg-numeric/-/pg-numeric-1.0.2.tgz#816d9a44026086ae8ae74839acd6a09b0636aa3a"
-  integrity sha512-BM/Thnrw5jm2kKLE5uJkXqqExRUY/toLHda65XgFTBTFYZyopbKjBe29Ii3RbkvlsMoFwD+tHeGaCjjv0gHlyw==
-
-pg-pool@^3.6.1:
-  version "3.6.1"
-  resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-3.6.1.tgz#5a902eda79a8d7e3c928b77abf776b3cb7d351f7"
-  integrity sha512-jizsIzhkIitxCGfPRzJn1ZdcosIt3pz9Sh3V01fm1vZnbnCMgmGl5wvGGdNN2EL9Rmb0EcFoCkixH4Pu+sP9Og==
-
-pg-protocol@*, pg-protocol@^1.6.0:
-  version "1.6.0"
-  resolved "https://registry.yarnpkg.com/pg-protocol/-/pg-protocol-1.6.0.tgz#4c91613c0315349363af2084608db843502f8833"
-  integrity sha512-M+PDm637OY5WM307051+bsDia5Xej6d9IR4GwJse1qA1DIhiKlksvrneZOYQq42OM+spubpcNYEo2FcKQrDk+Q==
-
-pg-types@^2.1.0:
-  version "2.2.0"
-  resolved "https://registry.yarnpkg.com/pg-types/-/pg-types-2.2.0.tgz#2d0250d636454f7cfa3b6ae0382fdfa8063254a3"
-  integrity sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==
-  dependencies:
-    pg-int8 "1.0.1"
-    postgres-array "~2.0.0"
-    postgres-bytea "~1.0.0"
-    postgres-date "~1.0.4"
-    postgres-interval "^1.1.0"
-
-pg-types@^4.0.1:
-  version "4.0.1"
-  resolved "https://registry.yarnpkg.com/pg-types/-/pg-types-4.0.1.tgz#31857e89d00a6c66b06a14e907c3deec03889542"
-  integrity sha512-hRCSDuLII9/LE3smys1hRHcu5QGcLs9ggT7I/TCs0IE+2Eesxi9+9RWAAwZ0yaGjxoWICF/YHLOEjydGujoJ+g==
-  dependencies:
-    pg-int8 "1.0.1"
-    pg-numeric "1.0.2"
-    postgres-array "~3.0.1"
-    postgres-bytea "~3.0.0"
-    postgres-date "~2.0.1"
-    postgres-interval "^3.0.0"
-    postgres-range "^1.1.1"
-
-pg@^8.7.3:
-  version "8.11.3"
-  resolved "https://registry.yarnpkg.com/pg/-/pg-8.11.3.tgz#d7db6e3fe268fcedd65b8e4599cda0b8b4bf76cb"
-  integrity sha512-+9iuvG8QfaaUrrph+kpF24cXkH1YOOUeArRNYIxq1viYHZagBxrTno7cecY1Fa44tJeZvaoG+Djpkc3JwehN5g==
-  dependencies:
-    buffer-writer "2.0.0"
-    packet-reader "1.0.0"
-    pg-connection-string "^2.6.2"
-    pg-pool "^3.6.1"
-    pg-protocol "^1.6.0"
-    pg-types "^2.1.0"
-    pgpass "1.x"
-  optionalDependencies:
-    pg-cloudflare "^1.1.1"
-
-pgpass@1.x:
-  version "1.0.5"
-  resolved "https://registry.yarnpkg.com/pgpass/-/pgpass-1.0.5.tgz#9b873e4a564bb10fa7a7dbd55312728d422a223d"
-  integrity sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==
-  dependencies:
-    split2 "^4.1.0"
-
-postgres-array@~2.0.0:
-  version "2.0.0"
-  resolved "https://registry.yarnpkg.com/postgres-array/-/postgres-array-2.0.0.tgz#48f8fce054fbc69671999329b8834b772652d82e"
-  integrity sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==
-
-postgres-array@~3.0.1:
-  version "3.0.2"
-  resolved "https://registry.yarnpkg.com/postgres-array/-/postgres-array-3.0.2.tgz#68d6182cb0f7f152a7e60dc6a6889ed74b0a5f98"
-  integrity sha512-6faShkdFugNQCLwucjPcY5ARoW1SlbnrZjmGl0IrrqewpvxvhSLHimCVzqeuULCbG0fQv7Dtk1yDbG3xv7Veog==
-
-postgres-bytea@~1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/postgres-bytea/-/postgres-bytea-1.0.0.tgz#027b533c0aa890e26d172d47cf9ccecc521acd35"
-  integrity sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w==
-
-postgres-bytea@~3.0.0:
-  version "3.0.0"
-  resolved "https://registry.yarnpkg.com/postgres-bytea/-/postgres-bytea-3.0.0.tgz#9048dc461ac7ba70a6a42d109221619ecd1cb089"
-  integrity sha512-CNd4jim9RFPkObHSjVHlVrxoVQXz7quwNFpz7RY1okNNme49+sVyiTvTRobiLV548Hx/hb1BG+iE7h9493WzFw==
-  dependencies:
-    obuf "~1.1.2"
-
-postgres-date@~1.0.4:
-  version "1.0.7"
-  resolved "https://registry.yarnpkg.com/postgres-date/-/postgres-date-1.0.7.tgz#51bc086006005e5061c591cee727f2531bf641a8"
-  integrity sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==
-
-postgres-date@~2.0.1:
-  version "2.0.1"
-  resolved "https://registry.yarnpkg.com/postgres-date/-/postgres-date-2.0.1.tgz#638b62e5c33764c292d37b08f5257ecb09231457"
-  integrity sha512-YtMKdsDt5Ojv1wQRvUhnyDJNSr2dGIC96mQVKz7xufp07nfuFONzdaowrMHjlAzY6GDLd4f+LUHHAAM1h4MdUw==
-
-postgres-interval@^1.1.0:
-  version "1.2.0"
-  resolved "https://registry.yarnpkg.com/postgres-interval/-/postgres-interval-1.2.0.tgz#b460c82cb1587507788819a06aa0fffdb3544695"
-  integrity sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==
-  dependencies:
-    xtend "^4.0.0"
-
-postgres-interval@^3.0.0:
-  version "3.0.0"
-  resolved "https://registry.yarnpkg.com/postgres-interval/-/postgres-interval-3.0.0.tgz#baf7a8b3ebab19b7f38f07566c7aab0962f0c86a"
-  integrity sha512-BSNDnbyZCXSxgA+1f5UU2GmwhoI0aU5yMxRGO8CdFEcY2BQF9xm/7MqKnYoM1nJDk8nONNWDk9WeSmePFhQdlw==
-
-postgres-range@^1.1.1:
-  version "1.1.3"
-  resolved "https://registry.yarnpkg.com/postgres-range/-/postgres-range-1.1.3.tgz#9ccd7b01ca2789eb3c2e0888b3184225fa859f76"
-  integrity sha512-VdlZoocy5lCP0c/t66xAfclglEapXPCIVhqqJRncYpvbCgImF0w67aPKfbqUMr72tO2k5q0TdTZwCLjPTI6C9g==
-
-split2@^4.1.0:
-  version "4.2.0"
-  resolved "https://registry.yarnpkg.com/split2/-/split2-4.2.0.tgz#c9c5920904d148bab0b9f67145f245a86aadbfa4"
-  integrity sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==
-
 strnum@^1.0.5:
   version "1.0.5"
   resolved "https://registry.yarnpkg.com/strnum/-/strnum-1.0.5.tgz#5c4e829fe15ad4ff0d20c3db5ac97b73c9b072db"
@@ -1060,8 +906,3 @@ uuid@^8.3.2:
   version "8.3.2"
   resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.2.tgz#80d5b5ced271bb9af6c445f21a1a04c606cefbe2"
   integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==
-
-xtend@^4.0.0:
-  version "4.0.2"
-  resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54"
-  integrity sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==
-Original file line number
+Diff line change
@@ Expand Up / @@ -45,8 +45,6 @@ resource "aws_ecr_repository" "load_test_repository" { @@
       image_scanning_configuration {
         scan_on_push = true
       }
     }
     resource "aws_ecr_lifecycle_policy" "load_test_policy" {
@@ Expand Down @@