From 09ad9c8b09146e261b552c8fa1cee308a1e47ce3 Mon Sep 17 00:00:00 2001 From: Jason Ng Date: Tue, 15 Oct 2024 17:19:12 -0700 Subject: [PATCH 1/2] fix: CDI-3452 - Fix references for dbx volumes to allow creating volume on existing catalog and bucket (#656) * wip: add override policy documents * feat: add support for bucket names and split storage credential from catalog * feat: change owner var * ref fix * Update databricks-s3-volume/main.tf Co-authored-by: James Bartolome --------- Co-authored-by: James Bartolome --- databricks-s3-volume/bucket.tf | 2 ++ databricks-s3-volume/main.tf | 16 +++++++++------- databricks-s3-volume/outputs.tf | 12 ++++++++++++ databricks-s3-volume/variables.tf | 22 ++++++++++++++++++++-- 4 files changed, 43 insertions(+), 9 deletions(-) diff --git a/databricks-s3-volume/bucket.tf b/databricks-s3-volume/bucket.tf index 547dcd3f..0f6e4d16 100644 --- a/databricks-s3-volume/bucket.tf +++ b/databricks-s3-volume/bucket.tf @@ -5,6 +5,8 @@ locals { data "aws_iam_policy_document" "databricks-s3" { count = var.volume_bucket != null ? 0 : 1 + override_policy_documents = var.override_policy_documents + # standard UC access statement { sid = "dbxBucketAccess" diff --git a/databricks-s3-volume/main.tf b/databricks-s3-volume/main.tf index 632b0f9c..54cd98e3 100644 --- a/databricks-s3-volume/main.tf +++ b/databricks-s3-volume/main.tf @@ -11,7 +11,9 @@ locals { path = "/databricks/" databricks_aws_account = "414351767826" # Databricks' own AWS account, not CZI's. See https://docs.databricks.com/en/administration-guide/account-settings-e2/credentials.html#step-1-create-a-cross-account-iam-role - bucket_name = var.volume_bucket != null ? var.volume_bucket : replace(var.catalog_name, "_", "-") # buckets don't work with underscores + bucket_name = var.volume_bucket != null ? var.volume_bucket : ( + var.override_bucket_name != null ? var.override_bucket_name : replace(var.catalog_name, "_", "-") # buckets don't work with underscores + ) } ### Databricks storage credential - allows workspace to access an external location. @@ -19,7 +21,7 @@ locals { ### NOTE: resource "databricks_storage_credential" "volume" { - count = var.create_catalog ? 1 : 0 + count = var.create_storage_credential ? 1 : 0 depends_on = [ resource.aws_iam_role.dbx_unity_aws_role, @@ -42,7 +44,7 @@ resource "time_sleep" "wait_30_seconds" { } resource "databricks_external_location" "volume" { - count = var.create_catalog ? 1 : 0 + count = var.create_storage_credential ? 1 : 0 depends_on = [time_sleep.wait_30_seconds] name = local.catalog_name @@ -59,7 +61,7 @@ resource "databricks_catalog" "volume" { depends_on = [databricks_external_location.volume[0]] name = local.catalog_name metastore_id = var.metastore_id - owner = var.catalog_owner + owner = var.owner storage_root = "s3://${local.bucket_name}" comment = "this catalog is managed by terraform - default volume catalog for Databricks workspace ${var.workspace_name}" properties = { @@ -75,7 +77,7 @@ resource "databricks_schema" "volume" { catalog_name = local.catalog_name name = local.schema_name comment = "This schema is managed by Terraform - ${var.volume_comment}" - owner = var.catalog_owner + owner = var.owner properties = var.volume_schema_properties } @@ -85,7 +87,7 @@ resource "databricks_volume" "volume" { catalog_name = local.catalog_name schema_name = local.schema_name volume_type = "EXTERNAL" - storage_location = "s3://${local.bucket_name}/${local.schema_name}" - owner = var.catalog_owner + storage_location = "s3://${local.bucket_name}/${local.schema_name}/${local.volume_name}" + owner = var.owner comment = "This volume is managed by Terraform - ${var.volume_comment}" } \ No newline at end of file diff --git a/databricks-s3-volume/outputs.tf b/databricks-s3-volume/outputs.tf index 1f36f081..d0a58e24 100644 --- a/databricks-s3-volume/outputs.tf +++ b/databricks-s3-volume/outputs.tf @@ -8,4 +8,16 @@ output "volume_specific_bucket_name" { output "volume_path" { value = "${local.catalog_name}.${local.schema_name}.${local.volume_name}" +} + +output "catalog_name" { + value = local.catalog_name +} + +output "schema_name" { + value = local.schema_name +} + +output "volume_name" { + value = local.volume_name } \ No newline at end of file diff --git a/databricks-s3-volume/variables.tf b/databricks-s3-volume/variables.tf index aba69668..e48f8c3c 100644 --- a/databricks-s3-volume/variables.tf +++ b/databricks-s3-volume/variables.tf @@ -9,8 +9,8 @@ variable "catalog_name" { type = string } -variable "catalog_owner" { - description = "User or group name of the catalog owner" +variable "owner" { + description = "User or group name of the owner - will be applied to the catalog, schema, and volume, if applicable" type = string } @@ -112,6 +112,24 @@ variable "additional_rw_bucket_grant_arns" { default = [] } +variable "override_policy_documents" { + description = "(Optional) Additional bucket policies to apply to the bucket. These should already be in JSON" + type = list(string) + default = [] +} + +variable "create_storage_credential" { + description = "(Optional) Flag to create a new Databricks storage credential or look for an existing one for the given bucket_name" + type = bool + default = true +} + +variable "override_bucket_name" { + description = "(Optional) Name of the S3 bucket to create or use for Databricks volume, overriding the default" + type = string + default = null +} + variable "tags" { description = "REQUIRED: Tags to include for this environment." type = object({ From 347931c50c325874b3168d0c30c02b0f69c365c8 Mon Sep 17 00:00:00 2001 From: "czi-github-helper[bot]" <95879977+czi-github-helper[bot]@users.noreply.github.com> Date: Tue, 15 Oct 2024 19:26:52 -0700 Subject: [PATCH 2/2] chore(main): release 0.83.5 (#657) Co-authored-by: czi-github-helper[bot] <95879977+czi-github-helper[bot]@users.noreply.github.com> --- CHANGELOG.md | 7 +++++++ version.txt | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 20fb82d5..1ecf4ed5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [0.83.5](https://github.com/chanzuckerberg/cztack/compare/v0.83.4...v0.83.5) (2024-10-16) + + +### Bug Fixes + +* CDI-3452 - Fix references for dbx volumes to allow creating volume on existing catalog and bucket ([#656](https://github.com/chanzuckerberg/cztack/issues/656)) ([09ad9c8](https://github.com/chanzuckerberg/cztack/commit/09ad9c8b09146e261b552c8fa1cee308a1e47ce3)) + ## [0.83.4](https://github.com/chanzuckerberg/cztack/compare/v0.83.3...v0.83.4) (2024-10-14) diff --git a/version.txt b/version.txt index 3976f649..158e19d3 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.83.4 +0.83.5