From a33cea9e1c217950508027893b403a1d352e12cb Mon Sep 17 00:00:00 2001 From: Bryan Robitaille Date: Tue, 3 Dec 2024 08:14:28 -0500 Subject: [PATCH 1/5] chore: Replace rds data client in reliability and nagware lambdas (#906) --- .devcontainer/devcontainer.json | 2 +- .../workflows/terragrunt-apply-production.yml | 2 +- .../workflows/terragrunt-apply-staging.yml | 2 +- .../workflows/terragrunt-plan-all-staging.yml | 2 +- .../workflows/terragrunt-plan-production.yml | 2 +- .github/workflows/terragrunt-plan-staging.yml | 2 +- README.md | 2 +- aws/lambdas/iam.tf | 3 +- aws/lambdas/inputs.tf | 11 +++- aws/lambdas/nagware.tf | 6 +- aws/lambdas/reliability.tf | 10 ++- aws/network/outputs.tf | 4 +- aws/network/security_groups_lambda.tf | 38 +++++------ docker-compose.yml | 2 +- env/cloud/app/terragrunt.hcl | 4 +- env/cloud/lambdas/terragrunt.hcl | 23 ++++--- env/cloud/pr_review/terragrunt.hcl | 8 ++- env/cloud/redis/terragrunt.hcl | 4 +- env/common/local-provider.tf | 23 ++++++- env/common/provider.tf | 2 +- lambda-code/nagware/lib/rdsConnector.ts | 33 ++++++++++ lambda-code/nagware/lib/templates.ts | 63 ++++++------------- lambda-code/nagware/package.json | 2 +- lambda-code/nagware/yarn.lock | 52 ++------------- lambda-code/reliability/lib/rdsConnector.ts | 33 ++++++++++ lambda-code/reliability/lib/templates.ts | 52 ++++++--------- lambda-code/reliability/package.json | 2 +- lambda-code/reliability/yarn.lock | 52 ++------------- localstack_services.sh | 2 +- 29 files changed, 212 insertions(+), 231 deletions(-) create mode 100644 lambda-code/nagware/lib/rdsConnector.ts create mode 100644 lambda-code/reliability/lib/rdsConnector.ts diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index c7765e602..3c05277ae 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -12,7 +12,7 @@ "terraform": { "version": "1.9.8", "tflint": "latest", - "terragrunt": "0.69.0" + "terragrunt": "0.69.2" }, "aws-cli": { "version": "2.5.6" diff --git a/.github/workflows/terragrunt-apply-production.yml b/.github/workflows/terragrunt-apply-production.yml index 11528ba6a..27fec0087 100644 --- a/.github/workflows/terragrunt-apply-production.yml +++ b/.github/workflows/terragrunt-apply-production.yml @@ -19,7 +19,7 @@ env: AWS_ACCOUNT_ID: ${{ vars.PRODUCTION_AWS_ACCOUNT_ID }} AWS_REGION: ca-central-1 TERRAFORM_VERSION: 1.9.8 - TERRAGRUNT_VERSION: 0.69.0 + TERRAGRUNT_VERSION: 0.69.2 TF_INPUT: false # API TF_VAR_zitadel_application_key: ${{ secrets.PRODUCTION_ZITADEL_APPLICATION_KEY }} diff --git a/.github/workflows/terragrunt-apply-staging.yml b/.github/workflows/terragrunt-apply-staging.yml index 74279de01..7855e255f 100644 --- a/.github/workflows/terragrunt-apply-staging.yml +++ b/.github/workflows/terragrunt-apply-staging.yml @@ -24,7 +24,7 @@ env: AWS_ACCOUNT_ID: ${{ vars.STAGING_AWS_ACCOUNT_ID }} AWS_REGION: ca-central-1 TERRAFORM_VERSION: 1.9.8 - TERRAGRUNT_VERSION: 0.69.0 + TERRAGRUNT_VERSION: 0.69.2 TF_INPUT: false # API TF_VAR_zitadel_application_key: ${{ secrets.STAGING_ZITADEL_APPLICATION_KEY }} diff --git a/.github/workflows/terragrunt-plan-all-staging.yml b/.github/workflows/terragrunt-plan-all-staging.yml index 0583229ab..730937ea1 100644 --- a/.github/workflows/terragrunt-plan-all-staging.yml +++ b/.github/workflows/terragrunt-plan-all-staging.yml @@ -16,7 +16,7 @@ env: AWS_REGION: ca-central-1 CONFTEST_VERSION: 0.46.0 TERRAFORM_VERSION: 1.9.8 - TERRAGRUNT_VERSION: 0.69.0 + TERRAGRUNT_VERSION: 0.69.2 TF_INPUT: false # API TF_VAR_zitadel_application_key: ${{ secrets.STAGING_ZITADEL_APPLICATION_KEY }} diff --git a/.github/workflows/terragrunt-plan-production.yml b/.github/workflows/terragrunt-plan-production.yml index 01e7964ee..af1bc769f 100644 --- a/.github/workflows/terragrunt-plan-production.yml +++ b/.github/workflows/terragrunt-plan-production.yml @@ -21,7 +21,7 @@ env: AWS_REGION: ca-central-1 CONFTEST_VERSION: 0.46.0 TERRAFORM_VERSION: 1.9.8 - TERRAGRUNT_VERSION: 0.69.0 + TERRAGRUNT_VERSION: 0.69.2 TF_INPUT: false # API TF_VAR_zitadel_application_key: ${{ secrets.PRODUCTION_ZITADEL_APPLICATION_KEY }} diff --git a/.github/workflows/terragrunt-plan-staging.yml b/.github/workflows/terragrunt-plan-staging.yml index f78a079e3..6c661d6ce 100644 --- a/.github/workflows/terragrunt-plan-staging.yml +++ b/.github/workflows/terragrunt-plan-staging.yml @@ -26,7 +26,7 @@ env: AWS_REGION: ca-central-1 CONFTEST_VERSION: 0.46.0 TERRAFORM_VERSION: 1.9.8 - TERRAGRUNT_VERSION: 0.69.0 + TERRAGRUNT_VERSION: 0.69.2 TF_INPUT: false # API TF_VAR_zitadel_application_key: ${{ secrets.STAGING_ZITADEL_APPLICATION_KEY }} diff --git a/README.md b/README.md index 948a81a57..d8642dd77 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ Pull Requests in this repository require all commits to be signed before they ca 1. `brew install warrensbox/tap/tfswitch` 1. `tfswitch 1.9.8` 1. `brew install warrensbox/tap/tgswitch` - 1. `tgswitch 0.69.0` + 1. `tgswitch 0.69.2` - Yarn (if you want to deploy the infrastructure locally): diff --git a/aws/lambdas/iam.tf b/aws/lambdas/iam.tf index a92bb3607..c76470978 100644 --- a/aws/lambdas/iam.tf +++ b/aws/lambdas/iam.tf @@ -186,7 +186,8 @@ data "aws_iam_policy_document" "lambda_secrets" { resources = [ var.database_secret_arn, - var.notify_api_key_secret_arn + var.notify_api_key_secret_arn, + var.database_url_secret_arn, ] } } diff --git a/aws/lambdas/inputs.tf b/aws/lambdas/inputs.tf index a2caf2a49..f513d72a0 100644 --- a/aws/lambdas/inputs.tf +++ b/aws/lambdas/inputs.tf @@ -14,6 +14,11 @@ variable "database_secret_arn" { type = string } +variable "database_url_secret_arn" { + description = "Database URL secret version ARN, used by the ECS task" + type = string +} + variable "rds_cluster_arn" { description = "RDS cluster ARN" type = string @@ -199,8 +204,8 @@ variable "ecr_repository_url_vault_integrity_lambda" { type = string } -variable "lambda_nagware_security_group_id" { - description = "Security group ID for the Nagware Lambda" +variable "lambda_security_group_id" { + description = "Security group ID for the Lambdas" type = string } @@ -217,4 +222,4 @@ variable "redis_port" { variable "redis_url" { description = "Redis URL used by the Nagware function. This should not include the protocol or port." type = string -} +} \ No newline at end of file diff --git a/aws/lambdas/nagware.tf b/aws/lambdas/nagware.tf index 71c39c365..cf7f73062 100644 --- a/aws/lambdas/nagware.tf +++ b/aws/lambdas/nagware.tf @@ -10,7 +10,7 @@ resource "aws_lambda_function" "nagware" { timeout = 900 vpc_config { - security_group_ids = [var.lambda_nagware_security_group_id] + security_group_ids = [var.lambda_security_group_id] subnet_ids = var.private_subnet_ids } @@ -24,9 +24,7 @@ resource "aws_lambda_function" "nagware" { REGION = var.region DOMAIN = var.domains[0] DYNAMODB_VAULT_TABLE_NAME = var.dynamodb_vault_table_name - DB_ARN = var.rds_cluster_arn - DB_SECRET = var.database_secret_arn - DB_NAME = var.rds_db_name + DB_URL = var.database_url_secret_arn NOTIFY_API_KEY = var.notify_api_key_secret_arn REDIS_URL = "redis://${var.redis_url}:${var.redis_port}" TEMPLATE_ID = var.gc_template_id diff --git a/aws/lambdas/reliability.tf b/aws/lambdas/reliability.tf index cc745b67d..ad4197db0 100644 --- a/aws/lambdas/reliability.tf +++ b/aws/lambdas/reliability.tf @@ -5,6 +5,12 @@ resource "aws_lambda_function" "reliability" { role = aws_iam_role.lambda.arn timeout = 300 + vpc_config { + security_group_ids = [var.lambda_security_group_id] + subnet_ids = var.private_subnet_ids + } + + lifecycle { ignore_changes = [image_uri] } @@ -15,9 +21,7 @@ resource "aws_lambda_function" "reliability" { REGION = var.region NOTIFY_API_KEY = var.notify_api_key_secret_arn TEMPLATE_ID = var.gc_template_id - DB_ARN = var.rds_cluster_arn - DB_SECRET = var.database_secret_arn - DB_NAME = var.rds_db_name + DB_URL = var.database_url_secret_arn LOCALSTACK = var.localstack_hosted } } diff --git a/aws/network/outputs.tf b/aws/network/outputs.tf index 3e10d19b8..cadb92739 100644 --- a/aws/network/outputs.tf +++ b/aws/network/outputs.tf @@ -33,9 +33,9 @@ output "idp_lb_security_group_id" { value = aws_security_group.idp_lb.id } -output "lambda_nagware_security_group_id" { +output "lambda_security_group_id" { description = "Lambda Nagware function security group ID" - value = aws_security_group.lambda_nagware.id + value = aws_security_group.lambda.id } output "public_subnet_ids" { diff --git a/aws/network/security_groups_lambda.tf b/aws/network/security_groups_lambda.tf index 96994c976..afc4f49e3 100644 --- a/aws/network/security_groups_lambda.tf +++ b/aws/network/security_groups_lambda.tf @@ -1,53 +1,53 @@ # # Nagware # -resource "aws_security_group" "lambda_nagware" { - description = "Lambda Nagware" - name = "lambda_nagware" +resource "aws_security_group" "lambda" { + description = "Lambdas" + name = "lambda" vpc_id = aws_vpc.forms.id } # Internet -resource "aws_security_group_rule" "lambda_nagware_egress_internet" { - description = "Egress to the internet from Nagware Lambda function" +resource "aws_security_group_rule" "lambda_egress_internet" { + description = "Egress to the internet from Lambda function" type = "egress" from_port = 443 to_port = 443 protocol = "tcp" - security_group_id = aws_security_group.lambda_nagware.id + security_group_id = aws_security_group.lambda.id cidr_blocks = ["0.0.0.0/0"] } # PrivateLink -resource "aws_security_group_rule" "privatelink_lambda_nagware_ingress" { - description = "Security group rule for Nagware Lambda function ingress" +resource "aws_security_group_rule" "privatelink_lambda_ingress" { + description = "Security group rule for Lambda function ingress" type = "ingress" from_port = 443 to_port = 443 protocol = "tcp" security_group_id = aws_security_group.privatelink.id - source_security_group_id = aws_security_group.lambda_nagware.id + source_security_group_id = aws_security_group.lambda.id } # Redis -resource "aws_security_group_rule" "redis_ingress_lambda_nagware" { - description = "Ingress to Redis from Nagware Lambda function" +resource "aws_security_group_rule" "lambda_ingress_redis" { + description = "Ingress to Redis from Lambda function" type = "ingress" from_port = 6379 to_port = 6379 protocol = "tcp" security_group_id = aws_security_group.forms_redis.id - source_security_group_id = aws_security_group.lambda_nagware.id + source_security_group_id = aws_security_group.lambda.id } -resource "aws_security_group_rule" "lambda_nagware_egress_redis" { - description = "Egress from Nagware Lambda function to Redis" - type = "egress" - from_port = 6379 - to_port = 6379 +resource "aws_security_group_rule" "lambda_ingress_rds" { + description = "Ingress to database from lambda" + type = "ingress" + from_port = 5432 + to_port = 5432 protocol = "tcp" - security_group_id = aws_security_group.lambda_nagware.id - source_security_group_id = aws_security_group.forms_redis.id + security_group_id = aws_security_group.forms_database.id + source_security_group_id = aws_security_group.lambda.id } # diff --git a/docker-compose.yml b/docker-compose.yml index 624cbff43..ca0215acd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,7 +3,7 @@ volumes: services: localstack: container_name: "GCForms_LocalStack" - image: localstack/localstack-pro:latest + image: localstack/localstack-pro:3 ports: - "127.0.0.1:4566:4566" # LocalStack Gateway - "127.0.0.1:4510-4559:4510-4559" # external services port range diff --git a/env/cloud/app/terragrunt.hcl b/env/cloud/app/terragrunt.hcl index da056d455..34e6af8eb 100644 --- a/env/cloud/app/terragrunt.hcl +++ b/env/cloud/app/terragrunt.hcl @@ -54,7 +54,9 @@ dependency "network" { mock_outputs_merge_strategy_with_state = "shallow" mock_outputs_allowed_terraform_commands = ["init", "fmt", "validate", "plan", "show"] mock_outputs = { - private_subnet_ids = [""] + private_subnet_ids = ["prv-1", "prv-2"] + egress_security_group_id = "sg-1234567890" + ecs_security_group_id = "sg-1234567890" } } diff --git a/env/cloud/lambdas/terragrunt.hcl b/env/cloud/lambdas/terragrunt.hcl index ea93d3fd1..8d4ce355c 100644 --- a/env/cloud/lambdas/terragrunt.hcl +++ b/env/cloud/lambdas/terragrunt.hcl @@ -30,8 +30,8 @@ dependency "network" { mock_outputs_merge_strategy_with_state = "shallow" mock_outputs_allowed_terraform_commands = ["init", "fmt", "validate", "plan", "show"] mock_outputs = { - lambda_nagware_security_group_id = "sg-1234" - private_subnet_ids = ["prv-1", "prv-2"] + lambda_security_group_id = "sg-1234" + private_subnet_ids = ["prv-1", "prv-2"] } } @@ -40,9 +40,10 @@ dependency "rds" { mock_outputs_merge_strategy_with_state = "shallow" mock_outputs_allowed_terraform_commands = ["init", "fmt", "validate", "plan", "show"] mock_outputs = { - rds_cluster_arn = null - rds_db_name = null - database_secret_arn = null + rds_cluster_arn = null + rds_db_name = null + database_secret_arn = null + database_url_secret_arn = null } } @@ -153,8 +154,8 @@ dependency "ecr" { } inputs = { - lambda_nagware_security_group_id = dependency.network.outputs.lambda_nagware_security_group_id - private_subnet_ids = dependency.network.outputs.private_subnet_ids + lambda_security_group_id = dependency.network.outputs.lambda_security_group_id + private_subnet_ids = dependency.network.outputs.private_subnet_ids dynamodb_relability_queue_arn = dependency.dynamodb.outputs.dynamodb_relability_queue_arn dynamodb_vault_arn = dependency.dynamodb.outputs.dynamodb_vault_arn @@ -168,9 +169,10 @@ inputs = { kms_key_cloudwatch_arn = dependency.kms.outputs.kms_key_cloudwatch_arn kms_key_dynamodb_arn = dependency.kms.outputs.kms_key_dynamodb_arn - rds_cluster_arn = dependency.rds.outputs.rds_cluster_arn - rds_db_name = dependency.rds.outputs.rds_db_name - database_secret_arn = dependency.rds.outputs.database_secret_arn + rds_cluster_arn = dependency.rds.outputs.rds_cluster_arn + rds_db_name = dependency.rds.outputs.rds_db_name + database_secret_arn = dependency.rds.outputs.database_secret_arn + database_url_secret_arn = dependency.rds.outputs.database_url_secret_arn redis_port = dependency.redis.outputs.redis_port redis_url = dependency.redis.outputs.redis_url @@ -186,6 +188,7 @@ inputs = { notify_api_key_secret_arn = dependency.secrets.outputs.notify_api_key_secret_arn + reliability_file_storage_arn = dependency.s3.outputs.reliability_file_storage_arn vault_file_storage_arn = dependency.s3.outputs.vault_file_storage_arn vault_file_storage_id = dependency.s3.outputs.vault_file_storage_id diff --git a/env/cloud/pr_review/terragrunt.hcl b/env/cloud/pr_review/terragrunt.hcl index 8baa852a8..1f920b888 100644 --- a/env/cloud/pr_review/terragrunt.hcl +++ b/env/cloud/pr_review/terragrunt.hcl @@ -40,9 +40,11 @@ dependency "network" { mock_outputs_merge_strategy_with_state = "shallow" mock_outputs = { vpc_id = null - privatelink_security_group_id = null - forms_database_security_group_id = null - forms_redis_security_group_id = null + privatelink_security_group_id = "sg-1234567890" + forms_database_security_group_id = "sg-1234567890" + forms_redis_security_group_id = "sg-1234567890" + rds_security_group_id = "sg-1234567890" + redis_security_group_id = "sg-1234567890" } } diff --git a/env/cloud/redis/terragrunt.hcl b/env/cloud/redis/terragrunt.hcl index b372bc695..8b53c46b7 100644 --- a/env/cloud/redis/terragrunt.hcl +++ b/env/cloud/redis/terragrunt.hcl @@ -11,8 +11,8 @@ dependency "network" { mock_outputs_merge_strategy_with_state = "shallow" mock_outputs_allowed_terraform_commands = ["init", "fmt", "validate", "plan", "show"] mock_outputs = { - private_subnet_ids = [""] - redis_security_group_id = null + private_subnet_ids = ["prv-1", "prv-2"] + redis_security_group_id = "sg-1234567890" } } diff --git a/env/common/local-provider.tf b/env/common/local-provider.tf index 811f501b3..0aafee269 100644 --- a/env/common/local-provider.tf +++ b/env/common/local-provider.tf @@ -3,7 +3,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "5.70.0" + version = "5.78.0" } random = { source = "hashicorp/random" @@ -54,6 +54,13 @@ provider "aws" { kms = "http://${var.localstack_host}:4566" ecr = "http://${var.localstack_host}:4566" } + + default_tags { + tags = { + (var.billing_tag_key) = var.billing_tag_value + Terraform = true + } + } } provider "aws" { @@ -92,6 +99,13 @@ provider "aws" { kms = "http://${var.localstack_host}:4566" ecr = "http://${var.localstack_host}:4566" } + + default_tags { + tags = { + (var.billing_tag_key) = var.billing_tag_value + Terraform = true + } + } } provider "aws" { @@ -104,6 +118,13 @@ provider "aws" { skip_metadata_api_check = true skip_requesting_account_id = true + default_tags { + tags = { + (var.billing_tag_key) = var.billing_tag_value + Terraform = true + } + } + endpoints { apigateway = "http://${var.localstack_host}:4566" cloudformation = "http://${var.localstack_host}:4566" diff --git a/env/common/provider.tf b/env/common/provider.tf index 1310fee19..396542851 100644 --- a/env/common/provider.tf +++ b/env/common/provider.tf @@ -3,7 +3,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "5.76.0" + version = "5.78.0" } random = { source = "hashicorp/random" diff --git a/lambda-code/nagware/lib/rdsConnector.ts b/lambda-code/nagware/lib/rdsConnector.ts new file mode 100644 index 000000000..2bd84f3f3 --- /dev/null +++ b/lambda-code/nagware/lib/rdsConnector.ts @@ -0,0 +1,33 @@ +import postgres from "postgres"; +import { SecretsManagerClient, GetSecretValueCommand } from "@aws-sdk/client-secrets-manager"; + +const getConnectionString = async (): Promise => { + try { + const client = new SecretsManagerClient(); + const command = new GetSecretValueCommand({ SecretId: process.env.DB_URL }); + + return client.send(command).then(({ SecretString: secretUrl }) => { + if (secretUrl === undefined) { + throw new Error("RDS Conneciton URL is undefined"); + } + + if (process.env.LOCALSTACK === "true") { + return secretUrl.replace("5432", "4510"); + } + return secretUrl; + }); + } catch (error) { + console.error(error, "[database-connector] Failed to retrieve server-database-url"); + + throw error; + } +}; + +const createDatabaseConnector = async () => { + console.info("[database-connector] Creating new database connector"); + + const connectionString = await getConnectionString(); + return postgres(connectionString); +}; + +export const DatabaseConnectorClient = await createDatabaseConnector(); diff --git a/lambda-code/nagware/lib/templates.ts b/lambda-code/nagware/lib/templates.ts index 5c1cf8f7a..76c4b8da8 100644 --- a/lambda-code/nagware/lib/templates.ts +++ b/lambda-code/nagware/lib/templates.ts @@ -1,4 +1,4 @@ -import { RDSDataClient, ExecuteStatementCommand } from "@aws-sdk/client-rds-data"; +import { DatabaseConnectorClient } from "./rdsConnector.js"; export type TemplateInfo = { formName: string; @@ -11,64 +11,41 @@ export type TemplateInfo = { export async function getTemplateInfo(formID: string): Promise { try { - const rdsDataClient = new RDSDataClient({ region: process.env.REGION }); - // Due to Localstack limitations we have to define aliases for fields that have the same name - const sqlStatement = ` - SELECT usr."name" AS user_name, usr."email", tem."name" AS template_name, tem."jsonConfig", tem."isPublished" + const result = await DatabaseConnectorClient< + { + user_name?: string; + email: string; + template_name?: string; + jsonConfig: Record; + isPublished: boolean; + }[] + >`SELECT usr."name" AS user_name, usr."email", tem."name" AS template_name, tem."jsonConfig", tem."isPublished" FROM "User" usr JOIN "_TemplateToUser" ttu ON usr."id" = ttu."B" JOIN "Template" tem ON tem."id" = ttu."A" - WHERE ttu."A" = :formID + WHERE ttu."A" = ${formID} `; - const executeStatementCommand = new ExecuteStatementCommand({ - database: process.env.DB_NAME, - resourceArn: process.env.DB_ARN, - secretArn: process.env.DB_SECRET, - sql: sqlStatement, - includeResultMetadata: false, // set to true if we want metadata like column names - parameters: [ - { - name: "formID", - value: { - stringValue: formID, - }, - }, - ], - }); - - const response = await rdsDataClient.send(executeStatementCommand); - - if (response.records && response.records.length > 0) { - const firstRecord = response.records[0]; + if (result.length > 0) { + const { template_name, jsonConfig, isPublished } = result[0]; - if ( - firstRecord[2].stringValue === undefined || // template name - firstRecord[3].stringValue === undefined || // template jsonConfig - firstRecord[4].booleanValue === undefined // template isPublished - ) { + if (!template_name || !jsonConfig || isPublished === undefined) { throw new Error( - `Missing required parameters: template name = ${firstRecord[2].stringValue} ; template jsonConfig = ${firstRecord[3].stringValue} ; template isPublished = ${firstRecord[4].stringValue}.` + `Missing required parameters: template name = ${template_name} ; template jsonConfig = ${jsonConfig} ; template isPublished = ${isPublished}.` ); } - const jsonConfig = JSON.parse(firstRecord[3].stringValue.trim()); - - const formName = - firstRecord[2].stringValue !== "" - ? firstRecord[2].stringValue - : `${jsonConfig.titleEn} - ${jsonConfig.titleFr}`; - - const isPublished = firstRecord[4].booleanValue; + // Note we use || instead of ?? to allow for empty strings + const formName = template_name || `${jsonConfig.titleEn} - ${jsonConfig.titleFr}`; - const owners = response.records.map((record) => { + const owners = result.map((record) => { // make sure owner email is defined - if (record[1].stringValue === undefined) { + if (!record.email) { throw new Error(`Missing required parameters: owner email.`); } - return { name: record[0].stringValue, email: record[1].stringValue }; + return { name: record.user_name, email: record.email }; }); return { formName, owners, isPublished }; diff --git a/lambda-code/nagware/package.json b/lambda-code/nagware/package.json index 26425adc3..46b8b9090 100644 --- a/lambda-code/nagware/package.json +++ b/lambda-code/nagware/package.json @@ -12,10 +12,10 @@ }, "dependencies": { "@aws-sdk/client-dynamodb": "3.667.0", - "@aws-sdk/client-rds-data": "3.667.0", "@aws-sdk/client-secrets-manager": "^3.478.0", "@aws-sdk/lib-dynamodb": "3.667.0", "axios": "^1.6.2", + "postgres": "^3.4.5", "redis": "^4.7.0" }, "devDependencies": { diff --git a/lambda-code/nagware/yarn.lock b/lambda-code/nagware/yarn.lock index 3740ea52c..28ee99b23 100644 --- a/lambda-code/nagware/yarn.lock +++ b/lambda-code/nagware/yarn.lock @@ -90,53 +90,6 @@ tslib "^2.6.2" uuid "^9.0.1" -"@aws-sdk/client-rds-data@3.667.0": - version "3.667.0" - resolved "https://registry.yarnpkg.com/@aws-sdk/client-rds-data/-/client-rds-data-3.667.0.tgz#835e49c09fce35af11dc1e7a5f983a8e58fa07d5" - integrity sha512-GjzMohuJW/vUNydds/B2kHtdthutpbgqnNPtK705NfI2HgWAE3EpPoYBgVHVYCJTp7bkCA1SiNeY0rvnUhuIBA== - dependencies: - "@aws-crypto/sha256-browser" "5.2.0" - "@aws-crypto/sha256-js" "5.2.0" - "@aws-sdk/client-sso-oidc" "3.667.0" - "@aws-sdk/client-sts" "3.667.0" - "@aws-sdk/core" "3.667.0" - "@aws-sdk/credential-provider-node" "3.667.0" - "@aws-sdk/middleware-host-header" "3.667.0" - "@aws-sdk/middleware-logger" "3.667.0" - "@aws-sdk/middleware-recursion-detection" "3.667.0" - "@aws-sdk/middleware-user-agent" "3.667.0" - "@aws-sdk/region-config-resolver" "3.667.0" - "@aws-sdk/types" "3.667.0" - "@aws-sdk/util-endpoints" "3.667.0" - "@aws-sdk/util-user-agent-browser" "3.667.0" - "@aws-sdk/util-user-agent-node" "3.667.0" - "@smithy/config-resolver" "^3.0.9" - "@smithy/core" "^2.4.8" - "@smithy/fetch-http-handler" "^3.2.9" - "@smithy/hash-node" "^3.0.7" - "@smithy/invalid-dependency" "^3.0.7" - "@smithy/middleware-content-length" "^3.0.9" - "@smithy/middleware-endpoint" "^3.1.4" - "@smithy/middleware-retry" "^3.0.23" - "@smithy/middleware-serde" "^3.0.7" - "@smithy/middleware-stack" "^3.0.7" - "@smithy/node-config-provider" "^3.1.8" - "@smithy/node-http-handler" "^3.2.4" - "@smithy/protocol-http" "^4.1.4" - "@smithy/smithy-client" "^3.4.0" - "@smithy/types" "^3.5.0" - "@smithy/url-parser" "^3.0.7" - "@smithy/util-base64" "^3.0.0" - "@smithy/util-body-length-browser" "^3.0.0" - "@smithy/util-body-length-node" "^3.0.0" - "@smithy/util-defaults-mode-browser" "^3.0.23" - "@smithy/util-defaults-mode-node" "^3.0.23" - "@smithy/util-endpoints" "^2.1.3" - "@smithy/util-middleware" "^3.0.7" - "@smithy/util-retry" "^3.0.7" - "@smithy/util-utf8" "^3.0.0" - tslib "^2.6.2" - "@aws-sdk/client-secrets-manager@^3.478.0": version "3.670.0" resolved "https://registry.yarnpkg.com/@aws-sdk/client-secrets-manager/-/client-secrets-manager-3.670.0.tgz#fb4c89f0f32bedb1a555ca7a5ac811ceb3295f47" @@ -1347,6 +1300,11 @@ obliterator@^1.6.1: resolved "https://registry.yarnpkg.com/obliterator/-/obliterator-1.6.1.tgz#dea03e8ab821f6c4d96a299e17aef6a3af994ef3" integrity sha512-9WXswnqINnnhOG/5SLimUlzuU1hFJUc8zkwyD59Sd+dPOMf05PmnYG/d6Q7HZ+KmgkZJa1PxRso6QdM3sTNHig== +postgres@^3.4.5: + version "3.4.5" + resolved "https://registry.yarnpkg.com/postgres/-/postgres-3.4.5.tgz#1ef99e51b0ba9b53cbda8a215dd406725f7d15f9" + integrity sha512-cDWgoah1Gez9rN3H4165peY9qfpEo+SA61oQv65O3cRUE1pOEoJWwddwcqKE8XZYjbblOJlYDlLV4h67HrEVDg== + proxy-from-env@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/proxy-from-env/-/proxy-from-env-1.1.0.tgz#e102f16ca355424865755d2c9e8ea4f24d58c3e2" diff --git a/lambda-code/reliability/lib/rdsConnector.ts b/lambda-code/reliability/lib/rdsConnector.ts new file mode 100644 index 000000000..2bd84f3f3 --- /dev/null +++ b/lambda-code/reliability/lib/rdsConnector.ts @@ -0,0 +1,33 @@ +import postgres from "postgres"; +import { SecretsManagerClient, GetSecretValueCommand } from "@aws-sdk/client-secrets-manager"; + +const getConnectionString = async (): Promise => { + try { + const client = new SecretsManagerClient(); + const command = new GetSecretValueCommand({ SecretId: process.env.DB_URL }); + + return client.send(command).then(({ SecretString: secretUrl }) => { + if (secretUrl === undefined) { + throw new Error("RDS Conneciton URL is undefined"); + } + + if (process.env.LOCALSTACK === "true") { + return secretUrl.replace("5432", "4510"); + } + return secretUrl; + }); + } catch (error) { + console.error(error, "[database-connector] Failed to retrieve server-database-url"); + + throw error; + } +}; + +const createDatabaseConnector = async () => { + console.info("[database-connector] Creating new database connector"); + + const connectionString = await getConnectionString(); + return postgres(connectionString); +}; + +export const DatabaseConnectorClient = await createDatabaseConnector(); diff --git a/lambda-code/reliability/lib/templates.ts b/lambda-code/reliability/lib/templates.ts index 96c9081b1..58b92868e 100644 --- a/lambda-code/reliability/lib/templates.ts +++ b/lambda-code/reliability/lib/templates.ts @@ -1,5 +1,5 @@ -import { RDSDataClient, ExecuteStatementCommand } from "@aws-sdk/client-rds-data"; import { FormProperties } from "./types.js"; +import { DatabaseConnectorClient } from "./rdsConnector.js"; export type TemplateInfo = { formConfig: FormProperties; @@ -12,48 +12,34 @@ export type TemplateInfo = { export async function getTemplateInfo(formID: string): Promise { try { - const rdsDataClient = new RDSDataClient({ region: process.env.REGION }); - - const sqlStatement = ` - SELECT t."jsonConfig", deli."emailAddress", deli."emailSubjectEn", deli."emailSubjectFr" + const templates = await DatabaseConnectorClient< + { + jsonConfig?: Record; + emailAddress?: string; + emailSubjectEn?: string; + emailSubjectFr?: string; + }[] + >`SELECT t."jsonConfig", deli."emailAddress", deli."emailSubjectEn", deli."emailSubjectFr" FROM "Template" t LEFT JOIN "DeliveryOption" deli ON t.id = deli."templateId" - WHERE t.id = :formID - `; - - const executeStatementCommand = new ExecuteStatementCommand({ - database: process.env.DB_NAME, - resourceArn: process.env.DB_ARN, - secretArn: process.env.DB_SECRET, - sql: sqlStatement, - includeResultMetadata: false, // set to true if we want metadata like column names - parameters: [ - { - name: "formID", - value: { - stringValue: formID, - }, - }, - ], - }); - - const response = await rdsDataClient.send(executeStatementCommand); + WHERE t.id = ${formID} + `; - if (response.records && response.records.length === 1) { - const firstRecord = response.records[0]; + if (templates.length === 1) { + const { jsonConfig, emailAddress, emailSubjectEn, emailSubjectFr } = templates[0]; // make sure template jsonConfig is defined - if (firstRecord[0].stringValue === undefined) { + if (jsonConfig === undefined) { throw new Error(`Missing required parameters: template jsonConfig.`); } - const formConfig = JSON.parse(firstRecord[0].stringValue!.trim()) as FormProperties; + const formConfig = jsonConfig as FormProperties; - const deliveryOption = firstRecord[1].stringValue + const deliveryOption = emailAddress ? { - emailAddress: firstRecord[1].stringValue, - emailSubjectEn: firstRecord[2].stringValue, - emailSubjectFr: firstRecord[3].stringValue, + emailAddress, + emailSubjectEn, + emailSubjectFr, } : null; diff --git a/lambda-code/reliability/package.json b/lambda-code/reliability/package.json index 6babbfcf3..f417b8867 100644 --- a/lambda-code/reliability/package.json +++ b/lambda-code/reliability/package.json @@ -13,13 +13,13 @@ "dependencies": { "@aws-sdk/client-dynamodb": "3.667.0", "@aws-sdk/client-lambda": "3.667.0", - "@aws-sdk/client-rds-data": "3.667.0", "@aws-sdk/client-s3": "3.667.0", "@aws-sdk/client-secrets-manager": "^3.478.0", "@aws-sdk/client-sqs": "3.667.0", "@aws-sdk/lib-dynamodb": "3.667.0", "axios": "^1.0.0", "json2md": "^1.10.0", + "postgres": "^3.4.5", "uuid": "^8.3.2" }, "devDependencies": { diff --git a/lambda-code/reliability/yarn.lock b/lambda-code/reliability/yarn.lock index 4f9a00246..8b472fe96 100644 --- a/lambda-code/reliability/yarn.lock +++ b/lambda-code/reliability/yarn.lock @@ -172,53 +172,6 @@ "@smithy/util-waiter" "^3.1.6" tslib "^2.6.2" -"@aws-sdk/client-rds-data@3.667.0": - version "3.667.0" - resolved "https://registry.yarnpkg.com/@aws-sdk/client-rds-data/-/client-rds-data-3.667.0.tgz#835e49c09fce35af11dc1e7a5f983a8e58fa07d5" - integrity sha512-GjzMohuJW/vUNydds/B2kHtdthutpbgqnNPtK705NfI2HgWAE3EpPoYBgVHVYCJTp7bkCA1SiNeY0rvnUhuIBA== - dependencies: - "@aws-crypto/sha256-browser" "5.2.0" - "@aws-crypto/sha256-js" "5.2.0" - "@aws-sdk/client-sso-oidc" "3.667.0" - "@aws-sdk/client-sts" "3.667.0" - "@aws-sdk/core" "3.667.0" - "@aws-sdk/credential-provider-node" "3.667.0" - "@aws-sdk/middleware-host-header" "3.667.0" - "@aws-sdk/middleware-logger" "3.667.0" - "@aws-sdk/middleware-recursion-detection" "3.667.0" - "@aws-sdk/middleware-user-agent" "3.667.0" - "@aws-sdk/region-config-resolver" "3.667.0" - "@aws-sdk/types" "3.667.0" - "@aws-sdk/util-endpoints" "3.667.0" - "@aws-sdk/util-user-agent-browser" "3.667.0" - "@aws-sdk/util-user-agent-node" "3.667.0" - "@smithy/config-resolver" "^3.0.9" - "@smithy/core" "^2.4.8" - "@smithy/fetch-http-handler" "^3.2.9" - "@smithy/hash-node" "^3.0.7" - "@smithy/invalid-dependency" "^3.0.7" - "@smithy/middleware-content-length" "^3.0.9" - "@smithy/middleware-endpoint" "^3.1.4" - "@smithy/middleware-retry" "^3.0.23" - "@smithy/middleware-serde" "^3.0.7" - "@smithy/middleware-stack" "^3.0.7" - "@smithy/node-config-provider" "^3.1.8" - "@smithy/node-http-handler" "^3.2.4" - "@smithy/protocol-http" "^4.1.4" - "@smithy/smithy-client" "^3.4.0" - "@smithy/types" "^3.5.0" - "@smithy/url-parser" "^3.0.7" - "@smithy/util-base64" "^3.0.0" - "@smithy/util-body-length-browser" "^3.0.0" - "@smithy/util-body-length-node" "^3.0.0" - "@smithy/util-defaults-mode-browser" "^3.0.23" - "@smithy/util-defaults-mode-node" "^3.0.23" - "@smithy/util-endpoints" "^2.1.3" - "@smithy/util-middleware" "^3.0.7" - "@smithy/util-retry" "^3.0.7" - "@smithy/util-utf8" "^3.0.0" - tslib "^2.6.2" - "@aws-sdk/client-s3@3.667.0": version "3.667.0" resolved "https://registry.yarnpkg.com/@aws-sdk/client-s3/-/client-s3-3.667.0.tgz#06fb8ba8d171c6fdc292fd8544f9b7f9f83d4974" @@ -1725,6 +1678,11 @@ obliterator@^1.6.1: resolved "https://registry.yarnpkg.com/obliterator/-/obliterator-1.6.1.tgz#dea03e8ab821f6c4d96a299e17aef6a3af994ef3" integrity sha512-9WXswnqINnnhOG/5SLimUlzuU1hFJUc8zkwyD59Sd+dPOMf05PmnYG/d6Q7HZ+KmgkZJa1PxRso6QdM3sTNHig== +postgres@^3.4.5: + version "3.4.5" + resolved "https://registry.yarnpkg.com/postgres/-/postgres-3.4.5.tgz#1ef99e51b0ba9b53cbda8a215dd406725f7d15f9" + integrity sha512-cDWgoah1Gez9rN3H4165peY9qfpEo+SA61oQv65O3cRUE1pOEoJWwddwcqKE8XZYjbblOJlYDlLV4h67HrEVDg== + proxy-from-env@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/proxy-from-env/-/proxy-from-env-1.1.0.tgz#e102f16ca355424865755d2c9e8ea4f24d58c3e2" diff --git a/localstack_services.sh b/localstack_services.sh index 99b0d1950..00eaca10c 100755 --- a/localstack_services.sh +++ b/localstack_services.sh @@ -22,7 +22,7 @@ fi # Set proper terraform and terragrunt versions -tgswitch 0.69.0 +tgswitch 0.69.2 tfswitch 1.9.8 basedir=$(pwd) From 6bdacc2f0fc4db1ebad9bdcd719702dcaa561dbd Mon Sep 17 00:00:00 2001 From: Bryan Robitaille Date: Tue, 3 Dec 2024 13:37:58 -0500 Subject: [PATCH 2/5] update sec rules for recommended format --- aws/network/security_groups_lambda.tf | 61 +++++++++++++-------------- 1 file changed, 30 insertions(+), 31 deletions(-) diff --git a/aws/network/security_groups_lambda.tf b/aws/network/security_groups_lambda.tf index afc4f49e3..982ebce43 100644 --- a/aws/network/security_groups_lambda.tf +++ b/aws/network/security_groups_lambda.tf @@ -8,48 +8,47 @@ resource "aws_security_group" "lambda" { } # Internet -resource "aws_security_group_rule" "lambda_egress_internet" { - description = "Egress to the internet from Lambda function" - type = "egress" + +resource "aws_vpc_security_group_ingress_rule" "privatelink" { + security_group_id = aws_security_group.lambda.id + referenced_security_group_id = aws_security_group.privatelink.id + ip_protocol = "tcp" from_port = 443 to_port = 443 - protocol = "tcp" - security_group_id = aws_security_group.lambda.id - cidr_blocks = ["0.0.0.0/0"] + } -# PrivateLink -resource "aws_security_group_rule" "privatelink_lambda_ingress" { - description = "Security group rule for Lambda function ingress" - type = "ingress" - from_port = 443 - to_port = 443 - protocol = "tcp" - security_group_id = aws_security_group.privatelink.id - source_security_group_id = aws_security_group.lambda.id +resource "aws_vpc_security_group_egress_rule" "privatelink" { + security_group_id = aws_security_group.privatelink.id + referenced_security_group_id = aws_security_group.lambda.id + ip_protocol = "tcp" + from_port = 443 + to_port = 443 } + # Redis -resource "aws_security_group_rule" "lambda_ingress_redis" { - description = "Ingress to Redis from Lambda function" - type = "ingress" - from_port = 6379 - to_port = 6379 - protocol = "tcp" - security_group_id = aws_security_group.forms_redis.id - source_security_group_id = aws_security_group.lambda.id +resource "aws_vpc_security_group_ingress_rule" "redis" { + description = "Ingress to Redis from lambda" + security_group_id = aws_security_group.forms_redis.id + referenced_security_group_id = aws_security_group.lambda.id + ip_protocol = "tcp" + from_port = 6379 + to_port = 6379 + } -resource "aws_security_group_rule" "lambda_ingress_rds" { - description = "Ingress to database from lambda" - type = "ingress" - from_port = 5432 - to_port = 5432 - protocol = "tcp" - security_group_id = aws_security_group.forms_database.id - source_security_group_id = aws_security_group.lambda.id +# RDS +resource "aws_vpc_security_group_ingress_rule" "rds" { + description = "Ingress to database from lambda" + security_group_id = aws_security_group.forms_database.id + referenced_security_group_id = aws_security_group.lambda.id + ip_protocol = "tcp" + from_port = 5432 + to_port = 5432 } + # # Athena connector # From aadc527e6f256a943cf45c1219ed983957227f9a Mon Sep 17 00:00:00 2001 From: Bryan Robitaille Date: Tue, 3 Dec 2024 13:43:44 -0500 Subject: [PATCH 3/5] fix lambda egress rule --- aws/network/security_groups_lambda.tf | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/aws/network/security_groups_lambda.tf b/aws/network/security_groups_lambda.tf index 982ebce43..5ad22c5ba 100644 --- a/aws/network/security_groups_lambda.tf +++ b/aws/network/security_groups_lambda.tf @@ -10,20 +10,22 @@ resource "aws_security_group" "lambda" { # Internet resource "aws_vpc_security_group_ingress_rule" "privatelink" { - security_group_id = aws_security_group.lambda.id + description = "Security group rule for Nagware Lambda function ingress" + security_group_id = aws_security_group.lambda.id referenced_security_group_id = aws_security_group.privatelink.id - ip_protocol = "tcp" - from_port = 443 - to_port = 443 - + ip_protocol = "tcp" + from_port = 443 + to_port = 443 + } resource "aws_vpc_security_group_egress_rule" "privatelink" { - security_group_id = aws_security_group.privatelink.id - referenced_security_group_id = aws_security_group.lambda.id + description = "Egress to the internet from Nagware Lambda function" + security_group_id = aws_security_group.lambda.id ip_protocol = "tcp" from_port = 443 to_port = 443 + cidr_ipv4 = "0.0.0.0/0" } From f1952eb2c9c03317a2d52142e5a2ff9715942a09 Mon Sep 17 00:00:00 2001 From: Bryan Robitaille Date: Tue, 3 Dec 2024 13:59:23 -0500 Subject: [PATCH 4/5] adding back legacy sec groups --- aws/network/security_groups_lambda.tf | 53 +++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/aws/network/security_groups_lambda.tf b/aws/network/security_groups_lambda.tf index 5ad22c5ba..368e3734b 100644 --- a/aws/network/security_groups_lambda.tf +++ b/aws/network/security_groups_lambda.tf @@ -1,3 +1,56 @@ +### Everything below this line needs to be deleted +resource "aws_security_group" "lambda_nagware" { + description = "Lambda Nagware" + name = "lambda_nagware" + vpc_id = aws_vpc.forms.id +} + +# Internet +resource "aws_security_group_rule" "lambda_nagware_egress_internet" { + description = "Egress to the internet from Nagware Lambda function" + type = "egress" + from_port = 443 + to_port = 443 + protocol = "tcp" + security_group_id = aws_security_group.lambda_nagware.id + cidr_blocks = ["0.0.0.0/0"] +} + +# PrivateLink +resource "aws_security_group_rule" "privatelink_lambda_nagware_ingress" { + description = "Security group rule for Nagware Lambda function ingress" + type = "ingress" + from_port = 443 + to_port = 443 + protocol = "tcp" + security_group_id = aws_security_group.privatelink.id + source_security_group_id = aws_security_group.lambda_nagware.id +} + +# Redis +resource "aws_security_group_rule" "redis_ingress_lambda_nagware" { + description = "Ingress to Redis from Nagware Lambda function" + type = "ingress" + from_port = 6379 + to_port = 6379 + protocol = "tcp" + security_group_id = aws_security_group.forms_redis.id + source_security_group_id = aws_security_group.lambda_nagware.id +} + +resource "aws_security_group_rule" "lambda_nagware_egress_redis" { + description = "Egress from Nagware Lambda function to Redis" + type = "egress" + from_port = 6379 + to_port = 6379 + protocol = "tcp" + security_group_id = aws_security_group.lambda_nagware.id + source_security_group_id = aws_security_group.forms_redis.id + +} +### Everything above this line needs to be deleted + + # # Nagware # From 8c5f404a157f1a37b82e98ffb104e234812aac9b Mon Sep 17 00:00:00 2001 From: Bryan Robitaille Date: Tue, 3 Dec 2024 14:03:57 -0500 Subject: [PATCH 5/5] rename rule for clarity --- aws/network/security_groups_lambda.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws/network/security_groups_lambda.tf b/aws/network/security_groups_lambda.tf index 368e3734b..789e8be45 100644 --- a/aws/network/security_groups_lambda.tf +++ b/aws/network/security_groups_lambda.tf @@ -72,7 +72,7 @@ resource "aws_vpc_security_group_ingress_rule" "privatelink" { } -resource "aws_vpc_security_group_egress_rule" "privatelink" { +resource "aws_vpc_security_group_egress_rule" "internet" { description = "Egress to the internet from Nagware Lambda function" security_group_id = aws_security_group.lambda.id ip_protocol = "tcp"