diff --git a/infra/terraform/aws/eks.tf b/infra/terraform/aws/eks.tf index fdbc0a579f..9b4dd4c55a 100644 --- a/infra/terraform/aws/eks.tf +++ b/infra/terraform/aws/eks.tf @@ -54,6 +54,8 @@ resource "random_string" "suffix" { resource "aws_security_group" "all_worker_mgmt" { name_prefix = "${var.name_prefix}-worker" vpc_id = module.vpc.vpc_id + + tags = var.tags } module "vpc" { @@ -78,34 +80,32 @@ module "vpc" { "kubernetes.io/cluster/${local.cluster_name}" = "shared" "kubernetes.io/role/internal-elb" = "1" } + + tags = var.tags } module "eks" { - source = "terraform-aws-modules/eks/aws" + source = "terraform-aws-modules/eks/aws" version = "12.2.0" cluster_name = local.cluster_name cluster_version = "1.17" subnets = module.vpc.private_subnets - tags = { - Environment = "test" - GithubRepo = "terraform-aws-eks" - GithubOrg = "terraform-aws-modules" - } + tags = var.tags vpc_id = module.vpc.vpc_id worker_groups = [ { - name = "worker-group-1" - instance_type = "r3.large" - asg_desired_capacity = 2 + name = "worker-group-1" + instance_type = "r3.large" + asg_desired_capacity = 2 }, { - name = "worker-group-2" - instance_type = "r3.large" - asg_desired_capacity = 1 + name = "worker-group-2" + instance_type = "r3.large" + asg_desired_capacity = 1 }, ] @@ -113,5 +113,5 @@ module "eks" { map_roles = var.map_roles map_accounts = var.map_accounts - workers_additional_policies = [aws_iam_policy.worker_policy.id] + workers_additional_policies = [aws_iam_policy.worker_policy.id] } \ No newline at end of file diff --git a/infra/terraform/aws/emr.tf b/infra/terraform/aws/emr.tf index 9fde6fac95..c15e537c3e 100644 --- a/infra/terraform/aws/emr.tf +++ b/infra/terraform/aws/emr.tf @@ -67,4 +67,6 @@ resource "aws_emr_cluster" "persistent_cluster" { step_concurrency_level = 256 log_uri = "s3://${aws_s3_bucket.feast_bucket.id}/logs/${var.name_prefix}-persistent-emr/" + + tags = var.tags } diff --git a/infra/terraform/aws/helm.tf b/infra/terraform/aws/helm.tf index 956b486ad0..00ba3ef741 100644 --- a/infra/terraform/aws/helm.tf +++ b/infra/terraform/aws/helm.tf @@ -12,81 +12,81 @@ provider "helm" { # RDS password is stored in a configmap which is not awesome but that RDS instance is not routable # from the outside anyways so that'll do. locals { - feast_core_config = { - redis = { - enabled = false - } - postgresql = { - enabled = false - } - kafka = { - enabled = false - } + feast_core_config = { + redis = { + enabled = false + } + postgresql = { + enabled = false + } + kafka = { + enabled = false + } - "feast-core" = { - "application-generated.yaml" = { - enabled = false - } + "feast-core" = { + "application-generated.yaml" = { + enabled = false + } - "application-override.yaml" = { - spring = { - datasource = { - url = "jdbc:postgresql://${module.rds_cluster.endpoint}:5432/${module.rds_cluster.database_name}" - username = "${module.rds_cluster.master_username}" - password = "${random_password.db_password.result}" - } - } - feast = { - stream = { - type = "kafka" - options = { - bootstrapServers = ${aws_msk_cluster.msk.bootstrap_brokers} - topic = "feast" - } - } - } - server = { - port = "8080" - } + "application-override.yaml" = { + spring = { + datasource = { + url = "jdbc:postgresql://${module.rds_cluster.endpoint}:5432/${module.rds_cluster.database_name}" + username = "${module.rds_cluster.master_username}" + password = "${random_password.db_password.result}" + } + } + feast = { + stream = { + type = "kafka" + options = { + bootstrapServers = aws_msk_cluster.msk.bootstrap_brokers + topic = "feast" } + } } + server = { + port = "8080" + } + } + } - "feast-online-serving" = { - "application-override.yaml" = { - enabled = true - feast = { - stores = [ - { - name = "online" - type = "REDIS" - config = { - host = module.redis.endpoint - port = 6379 - } - subscriptions = [ - { - name= "*" - project= "*" - version= "*" - } - ] - } - ] - job_store = { - redis_host = module.redis.endpoint - redis_port = 6379 - } + "feast-online-serving" = { + "application-override.yaml" = { + enabled = true + feast = { + stores = [ + { + name = "online" + type = "REDIS" + config = { + host = module.redis.endpoint + port = 6379 + } + subscriptions = [ + { + name = "*" + project = "*" + version = "*" } + ] } + ] + job_store = { + redis_host = module.redis.endpoint + redis_port = 6379 + } } + } } + } } resource "helm_release" "feast" { - name = "feast" - chart = "../../charts/feast" + name = "feast" + chart = "../../charts/feast" - wait = false + wait = false values = [ yamlencode(local.feast_core_config) diff --git a/infra/terraform/aws/msk.tf b/infra/terraform/aws/msk.tf index 0c3fa2daaf..b96aae8fd7 100644 --- a/infra/terraform/aws/msk.tf +++ b/infra/terraform/aws/msk.tf @@ -3,11 +3,11 @@ resource "aws_security_group" "broker" { vpc_id = module.vpc.vpc_id ingress { - description = "Allow connections from the worker group" + description = "Allow connections from the worker group" security_groups = [aws_security_group.all_worker_mgmt.id] - protocol = "tcp" - from_port = 0 - to_port = 65535 + protocol = "tcp" + from_port = 0 + to_port = 65535 } egress { @@ -16,6 +16,8 @@ resource "aws_security_group" "broker" { protocol = "-1" cidr_blocks = ["0.0.0.0/0"] } + + tags = var.tags } resource "aws_msk_cluster" "msk" { @@ -26,7 +28,7 @@ resource "aws_msk_cluster" "msk" { broker_node_group_info { instance_type = "kafka.t3.small" ebs_volume_size = 100 - client_subnets = [module.vpc.private_subnets[0], module.vpc.private_subnets[1]] + client_subnets = [module.vpc.private_subnets[0], module.vpc.private_subnets[1]] security_groups = [aws_security_group.broker.id] } @@ -46,4 +48,5 @@ resource "aws_msk_cluster" "msk" { } } + tags = var.tags } diff --git a/infra/terraform/aws/online_store.tf b/infra/terraform/aws/online_store.tf index f8ea605735..034f69a8c4 100644 --- a/infra/terraform/aws/online_store.tf +++ b/infra/terraform/aws/online_store.tf @@ -1,8 +1,10 @@ - module "redis" { - source = "git::https://github.com/cloudposse/terraform-aws-elasticache-redis.git?ref=tags/0.25.0" - subnets = module.vpc.private_subnets - name = "${var.name_prefix}-online" - vpc_id = module.vpc.vpc_id - allowed_security_groups = [aws_security_group.all_worker_mgmt.id] - availability_zones = module.vpc.azs - } \ No newline at end of file +module "redis" { + source = "git::https://github.com/cloudposse/terraform-aws-elasticache-redis.git?ref=tags/0.25.0" + subnets = module.vpc.private_subnets + name = "${var.name_prefix}-online" + vpc_id = module.vpc.vpc_id + allowed_security_groups = [aws_security_group.all_worker_mgmt.id] + availability_zones = module.vpc.azs + + tags = var.tags +} \ No newline at end of file diff --git a/infra/terraform/aws/rds.tf b/infra/terraform/aws/rds.tf index fdee2107d3..71f338e0fe 100644 --- a/infra/terraform/aws/rds.tf +++ b/infra/terraform/aws/rds.tf @@ -1,25 +1,25 @@ resource "random_password" "db_password" { - length = 16 - special = true + length = 16 + special = true override_special = "!#()-[]<>" } module "rds_cluster" { - source = "git::https://github.com/cloudposse/terraform-aws-rds-cluster.git?ref=tags/0.35.0" - name = "${var.name_prefix}-db" - engine = "aurora-postgresql" - engine_mode = "serverless" - engine_version = "10.7" - cluster_family = "aurora-postgresql10" - cluster_size = 0 - admin_user = var.postgres_db_user - admin_password = random_password.db_password.result - db_name = var.postgres_db_name - db_port = 5432 - instance_type = "db.t2.small" - vpc_id = module.vpc.vpc_id - security_groups = [aws_security_group.all_worker_mgmt.id] - subnets = module.vpc.private_subnets + source = "git::https://github.com/cloudposse/terraform-aws-rds-cluster.git?ref=tags/0.35.0" + name = "${var.name_prefix}-db" + engine = "aurora-postgresql" + engine_mode = "serverless" + engine_version = "10.7" + cluster_family = "aurora-postgresql10" + cluster_size = 0 + admin_user = var.postgres_db_user + admin_password = random_password.db_password.result + db_name = var.postgres_db_name + db_port = 5432 + instance_type = "db.t2.small" + vpc_id = module.vpc.vpc_id + security_groups = [aws_security_group.all_worker_mgmt.id] + subnets = module.vpc.private_subnets scaling_configuration = [ { @@ -30,4 +30,6 @@ module "rds_cluster" { timeout_action = "ForceApplyCapacityChange" } ] + + tags = var.tags } \ No newline at end of file diff --git a/infra/terraform/aws/s3.tf b/infra/terraform/aws/s3.tf index a3af2fe873..489aea0774 100644 --- a/infra/terraform/aws/s3.tf +++ b/infra/terraform/aws/s3.tf @@ -1,7 +1,7 @@ resource "random_string" "s3_suffix" { - length = 8 - lower = true - upper = false + length = 8 + lower = true + upper = false special = false } @@ -17,4 +17,6 @@ resource "aws_s3_bucket" "feast_bucket" { } } } + + tags = var.tags } \ No newline at end of file diff --git a/infra/terraform/aws/variables.tf b/infra/terraform/aws/variables.tf index c8a75c10e8..6fb1a3595b 100644 --- a/infra/terraform/aws/variables.tf +++ b/infra/terraform/aws/variables.tf @@ -5,11 +5,11 @@ variable "name_prefix" { } variable "postgres_db_name" { - default = "feast" + default = "feast" } variable "postgres_db_user" { - default = "feast" + default = "feast" } variable "map_accounts" { @@ -34,5 +34,13 @@ variable "map_roles" { } variable "use_persistent_emr_cluster" { - default = false + description = "Create a persistent EMR cluster." + default = true } + +variable "tags" { + description = "Tags" + type = map(string) + + default = {} +} \ No newline at end of file