feat: add cmek into reidentification (#137)

* fix: add cmek to reidentification

- handle rebase issues

* rotation as default variable

* bump bq version for destroy support

examples/bigquery_sensitive_data/README.md

@@ -8,7 +8,7 @@ This example illustrates how to use the `bigquery_sensitive_data` submodule.
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
 | crypto\_key | The full resource name of the Cloud KMS key that wraps the data crypto key used by DLP. | `string` | n/a | yes |
-| delete\_contents\_on\_destroy | (Optional) If set to true, delete all the tables in the dataset when destroying the resource; otherwise, destroying the resource will fail if tables are present. | `bool` | `null` | no |
+| delete\_contents\_on\_destroy | (Optional) If set to true, delete all the tables in the dataset when destroying the resource; otherwise, destroying the resource will fail if tables are present. | `bool` | `false` | no |
 | non\_sensitive\_project\_id | Project with the de-identified dataset and table. | `string` | n/a | yes |
 | privileged\_data\_project\_id | Project where the privileged datasets and tables are created. | `string` | n/a | yes |
 | subnetwork | The subnetwork used for dataflow. | `string` | n/a | yes |

examples/bigquery_sensitive_data/pipeline.tf

@@ -55,7 +55,7 @@ module "flex_dlp_template" {
   terraform_service_account   = var.terraform_service_account
   image_name                  = "regional_dlp_flex"
   image_tag                   = "0.1.0"
-  kms_key_name                = module.bigquery_sensitive_data.cmek_ingestion_crypto_key
+  kms_key_name                = module.bigquery_sensitive_data.cmek_reidentification_crypto_key
   read_access_members         = ["serviceAccount:${module.bigquery_sensitive_data.dataflow_controller_service_account_email}"]
 
   template_files = {
@@ -88,7 +88,7 @@ module "dataflow_bucket" {
   bucket_policy_only = true
 
   encryption = {
-    default_kms_key_name = module.bigquery_sensitive_data.cmek_ingestion_crypto_key
+    default_kms_key_name = module.bigquery_sensitive_data.cmek_reidentification_crypto_key
   }
 }
 
@@ -109,7 +109,7 @@ resource "google_dataflow_flex_template_job" "regional_dlp" {
     output_table                   = "${var.privileged_data_project_id}:${local.dataset_id}.sample_data"
     service_account_email          = module.bigquery_sensitive_data.dataflow_controller_service_account_email
     subnetwork                     = var.subnetwork
-    dataflow_kms_key               = module.bigquery_sensitive_data.cmek_ingestion_crypto_key
+    dataflow_kms_key               = module.bigquery_sensitive_data.cmek_reidentification_crypto_key
     temp_location                  = "${module.dataflow_bucket.bucket.url}/tmp/"
     no_use_public_ips              = "true"
   }

examples/bigquery_sensitive_data/variables.tf

@@ -52,5 +52,5 @@ variable "terraform_service_account" {
 variable "delete_contents_on_destroy" {
   description = "(Optional) If set to true, delete all the tables in the dataset when destroying the resource; otherwise, destroying the resource will fail if tables are present."
   type        = bool
-  default     = null
+  default     = false
 }

modules/data_warehouse_taxonomy/README.md

@@ -31,9 +31,11 @@ Functional examples are included in the [examples](./examples/bigquery_sensitive
 | cmek\_keyring\_name | The Keyring name for the KMS Customer Managed Encryption Keys. | `string` | n/a | yes |
 | cmek\_location | The location for the KMS Customer Managed Encryption Keys. | `string` | n/a | yes |
 | confidential\_access\_members | List of members in the standard GCP form: user:{email}, serviceAccount:{email}, group:{email} who will have access to confidential information in BigQuery. | `list(string)` | `[]` | no |
+| dataset\_default\_table\_expiration\_ms | TTL of tables using the dataset in MS. The default value is 30 days. | `number` | `2592000000` | no |
 | dataset\_id | The dataset ID to deploy to data warehouse. | `string` | n/a | yes |
 | dataset\_labels | Key value pairs in a map for dataset labels. | `map(string)` | `{}` | no |
 | delete\_contents\_on\_destroy | (Optional) If set to true, delete all the tables in the dataset when destroying the resource; otherwise, destroying the resource will fail if tables are present. | `bool` | `false` | no |
+| key\_rotation\_period\_seconds | Rotation period for keys. The default value is 30 days. | `string` | `"2592000s"` | no |
 | location | Default region to create resources where applicable. | `string` | n/a | yes |
 | non\_sensitive\_project\_id | Project with the de-identified dataset and table. | `string` | n/a | yes |
 | private\_access\_members | List of members in the standard GCP form: user:{email}, serviceAccount:{email}, group:{email} who will have access to private information in BigQuery. | `list(string)` | `[]` | no |
@@ -48,9 +50,9 @@ Functional examples are included in the [examples](./examples/bigquery_sensitive
 | Name | Description |
 |------|-------------|
 | cmek\_bigquery\_crypto\_key | The Customer Managed Crypto Key for the BigQuery service. |
-| cmek\_ingestion\_crypto\_key | The Customer Managed Crypto Key for the Ingestion crypto boundary. |
 | cmek\_keyring\_full\_name | The Keyring full name for the KMS Customer Managed Encryption Keys. |
 | cmek\_keyring\_name | The Keyring name for the KMS Customer Managed Encryption Keys. |
+| cmek\_reidentification\_crypto\_key | The Customer Managed Crypto Key for the reidentification crypto boundary. |
 | dataflow\_controller\_service\_account\_email | The Dataflow controller service account email. See https://cloud.google.com/dataflow/docs/concepts/security-and-permissions#specifying_a_user-managed_controller_service_account. |
 | emails\_list | The service account email addresses by name. |
 | high\_policy\_taxonomy\_id | Content for Policy Tag ID in high policy. |

modules/data_warehouse_taxonomy/kms.tf

@@ -20,24 +20,24 @@ locals {
   compute_sa  = "service-${data.google_project.reid_project.number}@compute-system.iam.gserviceaccount.com"
   bigquery_sa = data.google_bigquery_default_service_account.bigquery_sa.email
 
-  ingestion_key_name = "ingestion_kms_key"
-  bigquery_key_name  = "bigquery_kms_key"
+  reidentification_key_name = "reidentification_kms_key_${random_id.suffix.hex}"
+  bigquery_key_name         = "bigquery_kms_key_${random_id.suffix.hex}"
 
-  ingestion_key_encrypters_decrypters = "serviceAccount:${local.storage_sa},serviceAccount:${local.dataflow_sa},serviceAccount:${local.compute_sa}"
-  bigquery_key_encrypters_decrypters  = "serviceAccount:${local.bigquery_sa}"
+  reidentification_key_encrypters_decrypters = "serviceAccount:${local.storage_sa},serviceAccount:${local.dataflow_sa},serviceAccount:${local.compute_sa}"
+  bigquery_key_encrypters_decrypters         = "serviceAccount:${local.bigquery_sa}"
 
   keys = [
-    local.ingestion_key_name,
+    local.reidentification_key_name,
     local.bigquery_key_name
   ]
 
   encrypters = [
-    local.ingestion_key_encrypters_decrypters,
+    local.reidentification_key_encrypters_decrypters,
     local.bigquery_key_encrypters_decrypters
   ]
 
   decrypters = [
-    local.ingestion_key_encrypters_decrypters,
+    local.reidentification_key_encrypters_decrypters,
     local.bigquery_key_encrypters_decrypters
   ]
 }
@@ -65,13 +65,14 @@ module "cmek" {
   source  = "terraform-google-modules/kms/google"
   version = "~> 2.0.1"
 
-  project_id         = var.taxonomy_project_id
-  location           = var.cmek_location
-  keyring            = var.cmek_keyring_name
-  prevent_destroy    = !var.delete_contents_on_destroy
-  keys               = local.keys
-  set_encrypters_for = local.keys
-  set_decrypters_for = local.keys
-  encrypters         = local.encrypters
-  decrypters         = local.decrypters
+  project_id          = var.taxonomy_project_id
+  location            = var.cmek_location
+  keyring             = var.cmek_keyring_name
+  key_rotation_period = var.key_rotation_period_seconds
+  prevent_destroy     = !var.delete_contents_on_destroy
+  keys                = local.keys
+  set_encrypters_for  = local.keys
+  set_decrypters_for  = local.keys
+  encrypters          = local.encrypters
+  decrypters          = local.decrypters
 }

modules/data_warehouse_taxonomy/main.tf

@@ -15,7 +15,7 @@
  */
 
 resource "random_id" "suffix" {
-  byte_length = 8
+  byte_length = 4
 }
 
 locals {
@@ -56,13 +56,15 @@ module "project-iam-bindings" {
 
 module "bigquery_sensitive_data" {
   source  = "terraform-google-modules/bigquery/google"
-  version = "~> 5.0.0"
-
-  dataset_id                 = var.dataset_id
-  description                = "Dataset for BigQuery Sensitive Data"
-  project_id                 = var.privileged_data_project_id
-  location                   = var.location
-  delete_contents_on_destroy = var.delete_contents_on_destroy
+  version = "~> 5.2.0"
+
+  dataset_id                  = var.dataset_id
+  description                 = "Dataset for BigQuery Sensitive Data"
+  project_id                  = var.privileged_data_project_id
+  location                    = var.location
+  delete_contents_on_destroy  = var.delete_contents_on_destroy
+  encryption_key              = module.cmek.keys[local.bigquery_key_name]
+  default_table_expiration_ms = var.dataset_default_table_expiration_ms
 
   tables = [
     {

modules/data_warehouse_taxonomy/outputs.tf

@@ -64,9 +64,9 @@ output "cmek_keyring_name" {
   value       = module.cmek.keyring_name
 }
 
-output "cmek_ingestion_crypto_key" {
-  description = "The Customer Managed Crypto Key for the Ingestion crypto boundary."
-  value       = module.cmek.keys[local.ingestion_key_name]
+output "cmek_reidentification_crypto_key" {
+  description = "The Customer Managed Crypto Key for the reidentification crypto boundary."
+  value       = module.cmek.keys[local.reidentification_key_name]
 }
 
 output "cmek_bigquery_crypto_key" {

modules/data_warehouse_taxonomy/variables.tf

@@ -88,3 +88,15 @@ variable "cmek_keyring_name" {
   description = "The Keyring name for the KMS Customer Managed Encryption Keys."
   type        = string
 }
+
+variable "dataset_default_table_expiration_ms" {
+  description = "TTL of tables using the dataset in MS. The default value is 30 days."
+  type        = number
+  default     = 2592000000
+}
+
+variable "key_rotation_period_seconds" {
+  description = "Rotation period for keys. The default value is 30 days."
+  type        = string
+  default     = "2592000s"
+}