From 8694a4cc046fb789afbca6e4837578f257225d4f Mon Sep 17 00:00:00 2001 From: Aidan Russell Date: Mon, 16 Dec 2024 19:31:37 +0000 Subject: [PATCH] Revert "Feat/sagemaker llms update main (#171)" This reverts commit 2293a7aa8f48bd18d3fd3551b50650a9dafc275e. --- infra/airflow_dag_processor.tf | 35 +------- infra/ecr.tf | 144 --------------------------------- infra/main.tf | 2 +- infra/s3_mirrors.tf | 18 ----- infra/s3_notebooks.tf | 20 ----- package-lock.json | 7 +- 6 files changed, 6 insertions(+), 220 deletions(-) diff --git a/infra/airflow_dag_processor.tf b/infra/airflow_dag_processor.tf index 9f5bf3c..aefea3e 100644 --- a/infra/airflow_dag_processor.tf +++ b/infra/airflow_dag_processor.tf @@ -200,28 +200,10 @@ data "aws_iam_policy_document" "airflow_team" { actions = [ "sts:AssumeRole", ] + resources = var.airflow_dag_processors[count.index].assume_roles } - } - dynamic "statement" { - for_each = length(var.airflow_dag_processors[count.index].buckets) > 0 ? [1] : [] - content { - actions = [ - "s3:ListBucket", - ] - resources = var.airflow_dag_processors[count.index].buckets - } - } - - dynamic "statement" { - for_each = length(var.airflow_dag_processors[count.index].buckets) > 0 ? [1] : [] - content { - actions = [ - "s3:GetObject", - ] - resources = [for s in var.airflow_dag_processors[count.index].buckets : "${s}/*"] - } } statement { @@ -253,20 +235,7 @@ data "aws_iam_policy_document" "airflow_team" { ] resources = [ - "arn:aws:secretsmanager:${data.aws_region.aws_region.name}:${data.aws_caller_identity.aws_caller_identity.account_id}:secret:${var.prefix}/airflow/${var.airflow_dag_processors[count.index].name}-*", - "arn:aws:secretsmanager:${data.aws_region.aws_region.name}:${data.aws_caller_identity.aws_caller_identity.account_id}:secret:${var.prefix}/airflow/${var.airflow_dag_processors[count.index].name}_2-*" - ] - } - - # This just gives a permission to call BatchGetSecretValue, but doesn't actually give permission - # to look at any secret values themselves - secretsmanager:GetSecretValue does that - statement { - actions = [ - "secretsmanager:BatchGetSecretValue" - ] - - resources = [ - "*" + "arn:aws:secretsmanager:${data.aws_region.aws_region.name}:${data.aws_caller_identity.aws_caller_identity.account_id}:secret:${var.prefix}/airflow/${var.airflow_dag_processors[count.index].name}-*" ] } diff --git a/infra/ecr.tf b/infra/ecr.tf index 133381d..cf7f450 100644 --- a/infra/ecr.tf +++ b/infra/ecr.tf @@ -1,154 +1,67 @@ -// We only ever deploy tagged images, so all ECR repos have a lifecycle policy to delete untagged -// images. -// -// The current exceptions are the visualisation_base repos, where historically images were used -// using the sha256 hash, although at the time of writing we're moving away from this. - resource "aws_ecr_repository" "user_provided" { name = "${var.prefix}-user-provided" } -resource "aws_ecr_lifecycle_policy" "user_provided_expire_untagged_after_one_day" { - repository = aws_ecr_repository.user_provided.name - policy = data.aws_ecr_lifecycle_policy_document.expire_untagged_after_one_day.json -} - resource "aws_ecr_repository" "admin" { name = "${var.prefix}-admin" } -resource "aws_ecr_lifecycle_policy" "admin_expire_untagged_after_one_day" { - repository = aws_ecr_repository.admin.name - policy = data.aws_ecr_lifecycle_policy_document.expire_untagged_after_one_day.json -} - resource "aws_ecr_repository" "jupyterlab_python" { name = "${var.prefix}-jupyterlab-python" } -resource "aws_ecr_lifecycle_policy" "jupyterlab_python_expire_untagged_after_one_day" { - repository = aws_ecr_repository.jupyterlab_python.name - policy = data.aws_ecr_lifecycle_policy_document.expire_untagged_after_one_day.json -} - resource "aws_ecr_repository" "rstudio" { name = "${var.prefix}-rstudio" } -resource "aws_ecr_lifecycle_policy" "rstudio_expire_untagged_after_one_day" { - repository = aws_ecr_repository.rstudio.name - policy = data.aws_ecr_lifecycle_policy_document.expire_untagged_after_one_day.json -} - resource "aws_ecr_repository" "rstudio_rv4" { name = "${var.prefix}-rstudio-rv4" } -resource "aws_ecr_lifecycle_policy" "rstudio_rv4_expire_untagged_after_one_day" { - repository = aws_ecr_repository.rstudio_rv4.name - policy = data.aws_ecr_lifecycle_policy_document.expire_untagged_after_one_day.json -} - resource "aws_ecr_repository" "pgadmin" { name = "${var.prefix}-pgadmin" } -resource "aws_ecr_lifecycle_policy" "pgadmin_expire_untagged_after_one_day" { - repository = aws_ecr_repository.pgadmin.name - policy = data.aws_ecr_lifecycle_policy_document.expire_untagged_after_one_day.json -} - resource "aws_ecr_repository" "remotedesktop" { name = "${var.prefix}-remotedesktop" } -resource "aws_ecr_lifecycle_policy" "remotedesktop_rv4_expire_untagged_after_one_day" { - repository = aws_ecr_repository.remotedesktop.name - policy = data.aws_ecr_lifecycle_policy_document.expire_untagged_after_one_day.json -} - resource "aws_ecr_repository" "theia" { name = "${var.prefix}-theia" } -resource "aws_ecr_lifecycle_policy" "theia_expire_untagged_after_one_day" { - repository = aws_ecr_repository.theia.name - policy = data.aws_ecr_lifecycle_policy_document.expire_untagged_after_one_day.json -} - resource "aws_ecr_repository" "vscode" { name = "${var.prefix}-vscode" } -resource "aws_ecr_lifecycle_policy" "vscode_expire_untagged_after_one_day" { - repository = aws_ecr_repository.vscode.name - policy = data.aws_ecr_lifecycle_policy_document.expire_untagged_after_one_day.json -} - resource "aws_ecr_repository" "s3sync" { name = "${var.prefix}-s3sync" } -resource "aws_ecr_lifecycle_policy" "s3sync_expire_untagged_after_one_day" { - repository = aws_ecr_repository.s3sync.name - policy = data.aws_ecr_lifecycle_policy_document.expire_untagged_after_one_day.json -} - resource "aws_ecr_repository" "metrics" { name = "${var.prefix}-metrics" } -resource "aws_ecr_lifecycle_policy" "metrics_expire_untagged_after_one_day" { - repository = aws_ecr_repository.metrics.name - policy = data.aws_ecr_lifecycle_policy_document.expire_untagged_after_one_day.json -} - resource "aws_ecr_repository" "sentryproxy" { name = "${var.prefix}-sentryproxy" } - -resource "aws_ecr_lifecycle_policy" "sentryproxy_expire_untagged_after_one_day" { - repository = aws_ecr_repository.sentryproxy.name - policy = data.aws_ecr_lifecycle_policy_document.expire_untagged_after_one_day.json -} - resource "aws_ecr_repository" "dns_rewrite_proxy" { name = "${var.prefix}-dns-rewrite-proxy" } -resource "aws_ecr_lifecycle_policy" "dns_rewrite_proxy_expire_untagged_after_one_day" { - repository = aws_ecr_repository.dns_rewrite_proxy.name - policy = data.aws_ecr_lifecycle_policy_document.expire_untagged_after_one_day.json -} - resource "aws_ecr_repository" "healthcheck" { name = "${var.prefix}-healthcheck" } -resource "aws_ecr_lifecycle_policy" "healthcheck_expire_untagged_after_one_day" { - repository = aws_ecr_repository.healthcheck.name - policy = data.aws_ecr_lifecycle_policy_document.expire_untagged_after_one_day.json -} - resource "aws_ecr_repository" "prometheus" { name = "${var.prefix}-prometheus" } -resource "aws_ecr_lifecycle_policy" "prometheus_expire_untagged_after_one_day" { - repository = aws_ecr_repository.prometheus.name - policy = data.aws_ecr_lifecycle_policy_document.expire_untagged_after_one_day.json -} - resource "aws_ecr_repository" "gitlab" { name = "${var.prefix}-gitlab" } -resource "aws_ecr_lifecycle_policy" "gitlab_expire_untagged_after_one_day" { - repository = aws_ecr_repository.gitlab.name - policy = data.aws_ecr_lifecycle_policy_document.expire_untagged_after_one_day.json -} - resource "aws_ecr_repository" "visualisation_base" { name = "${var.prefix}-visualisation-base" } @@ -165,65 +78,30 @@ resource "aws_ecr_repository" "mirrors_sync" { name = "${var.prefix}-mirrors-sync" } -resource "aws_ecr_lifecycle_policy" "mirrors_sync_expire_untagged_after_one_day" { - repository = aws_ecr_repository.mirrors_sync.name - policy = data.aws_ecr_lifecycle_policy_document.expire_untagged_after_one_day.json -} - resource "aws_ecr_repository" "mirrors_sync_cran_binary" { name = "${var.prefix}-mirrors-sync-cran-binary" } -resource "aws_ecr_lifecycle_policy" "mirrors_sync_cran_binary_expire_untagged_after_one_day" { - repository = aws_ecr_repository.mirrors_sync_cran_binary.name - policy = data.aws_ecr_lifecycle_policy_document.expire_untagged_after_one_day.json -} - resource "aws_ecr_repository" "mirrors_sync_cran_binary_rv4" { name = "${var.prefix}-mirrors-sync-cran-binary-rv4" } -resource "aws_ecr_lifecycle_policy" "mirrors_sync_cran_binary_rv4_expire_untagged_after_one_day" { - repository = aws_ecr_repository.mirrors_sync_cran_binary_rv4.name - policy = data.aws_ecr_lifecycle_policy_document.expire_untagged_after_one_day.json -} - resource "aws_ecr_repository" "superset" { name = "${var.prefix}-superset" } -resource "aws_ecr_lifecycle_policy" "superset_expire_untagged_after_one_day" { - repository = aws_ecr_repository.superset.name - policy = data.aws_ecr_lifecycle_policy_document.expire_untagged_after_one_day.json -} - resource "aws_ecr_repository" "airflow" { name = "${var.prefix}-airflow" } -resource "aws_ecr_lifecycle_policy" "airflow_expire_untagged_after_one_day" { - repository = aws_ecr_repository.airflow.name - policy = data.aws_ecr_lifecycle_policy_document.expire_untagged_after_one_day.json -} - resource "aws_ecr_repository" "flower" { name = "${var.prefix}-flower" } -resource "aws_ecr_lifecycle_policy" "flower_expire_untagged_after_one_day" { - repository = aws_ecr_repository.flower.name - policy = data.aws_ecr_lifecycle_policy_document.expire_untagged_after_one_day.json -} - resource "aws_ecr_repository" "mlflow" { name = "${var.prefix}-mlflow" } -resource "aws_ecr_lifecycle_policy" "mlflow_expire_untagged_after_one_day" { - repository = aws_ecr_repository.mlflow.name - policy = data.aws_ecr_lifecycle_policy_document.expire_untagged_after_one_day.json -} - resource "aws_ecr_repository" "arango" { count = var.arango_on ? 1 : 0 name = "${var.prefix}-arango" @@ -233,28 +111,6 @@ resource "aws_ecr_repository" "sagemaker" { name = "${var.prefix}-sagemaker" } -resource "aws_ecr_lifecycle_policy" "arango_expire_untagged_after_one_day" { - count = var.arango_on ? 1 : 0 - repository = aws_ecr_repository.arango[0].name - policy = data.aws_ecr_lifecycle_policy_document.expire_untagged_after_one_day.json -} - -data "aws_ecr_lifecycle_policy_document" "expire_untagged_after_one_day" { - rule { - priority = 1 - selection { - tag_status = "untagged" - count_type = "sinceImagePushed" - count_unit = "days" - count_number = 1 - } - } -} - -resource "aws_ecr_repository" "sagemaker" { - name = "${var.prefix}-sagemaker" -} - resource "aws_vpc_endpoint" "ecr_dkr" { vpc_id = aws_vpc.main.id service_name = "com.amazonaws.${data.aws_region.aws_region.name}.ecr.dkr" diff --git a/infra/main.tf b/infra/main.tf index 5000efa..8138964 100644 --- a/infra/main.tf +++ b/infra/main.tf @@ -178,7 +178,7 @@ variable "airflow_on" { variable "airflow_db_instance_class" {} variable "airflow_domain" {} variable "airflow_dag_processors" { - type = list(object({ name = string, assume_roles = list(string), buckets = list(string) })) + type = list(object({ name = string, assume_roles = list(string) })) default = [] } variable "airflow_bucket_infix" {} diff --git a/infra/s3_mirrors.tf b/infra/s3_mirrors.tf index ab478bb..c3b4b7d 100644 --- a/infra/s3_mirrors.tf +++ b/infra/s3_mirrors.tf @@ -70,21 +70,3 @@ data "aws_iam_policy_document" "mirrors" { } } } - -resource "aws_s3_bucket_lifecycle_configuration" "intelligent_tier" { - count = var.mirrors_bucket_name != "" ? 1 : 0 - - bucket = aws_s3_bucket.mirrors.*.id[count.index] - - rule { - id = "Move objects >128KB to Intelligent Tier" - # Objects smaller than 128 KB will not transition by default to any storage class - - status = "Enabled" - - transition { - days = 0 - storage_class = "INTELLIGENT_TIERING" - } - } -} diff --git a/infra/s3_notebooks.tf b/infra/s3_notebooks.tf index 9f59706..81ccd72 100644 --- a/infra/s3_notebooks.tf +++ b/infra/s3_notebooks.tf @@ -57,24 +57,4 @@ data "aws_iam_policy_document" "notebooks" { ] } } - statement { - effect = "Allow" - principals { - type = "*" - identifiers = ["*"] - } - actions = [ - "s3:GetObject", - ] - resources = [ - "arn:aws:s3:::${aws_s3_bucket.notebooks.id}/shared/*", - ] - condition { - test = "StringEquals" - variable = "aws:SourceVpce" - values = [ - aws_vpc_endpoint.s3.id - ] - } - } } diff --git a/package-lock.json b/package-lock.json index 784eb28..51b6475 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1107,10 +1107,9 @@ } }, "node_modules/cross-spawn": { - "version": "7.0.6", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", - "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", - "license": "MIT", + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", + "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0",