From f16fd05302179cd2a20485781f3f640a8d5d88ba Mon Sep 17 00:00:00 2001 From: marko7460 Date: Fri, 31 Jul 2020 14:06:52 -0700 Subject: [PATCH] feat: Added shared_vpc_access submodule to enable GKE and Dataproc Service Account access. (#434) BREAKING CHANGE: This change requires that you use the `shared_vpc` submodule to manage service account access. See the upgrade guide for details. --- docs/upgrading_to_project_factory_v9.0.md | 105 ++++++++++++++++++ examples/shared_vpc/main.tf | 17 ++- modules/core_project_factory/main.tf | 55 +-------- modules/core_project_factory/outputs.tf | 5 + modules/project_services/README.md | 1 + modules/project_services/outputs.tf | 5 + modules/shared_vpc/main.tf | 11 ++ modules/shared_vpc/outputs.tf | 2 +- modules/shared_vpc_access/README.md | 41 +++++++ modules/shared_vpc_access/main.tf | 78 +++++++++++++ modules/shared_vpc_access/outputs.tf | 30 +++++ modules/shared_vpc_access/variables.tf | 37 ++++++ modules/shared_vpc_access/versions.tf | 24 ++++ .../dynamic_shared_vpc/controls/svpc.rb | 7 ++ 14 files changed, 360 insertions(+), 58 deletions(-) create mode 100644 docs/upgrading_to_project_factory_v9.0.md create mode 100644 modules/shared_vpc_access/README.md create mode 100644 modules/shared_vpc_access/main.tf create mode 100644 modules/shared_vpc_access/outputs.tf create mode 100644 modules/shared_vpc_access/variables.tf create mode 100644 modules/shared_vpc_access/versions.tf diff --git a/docs/upgrading_to_project_factory_v9.0.md b/docs/upgrading_to_project_factory_v9.0.md new file mode 100644 index 00000000..f5052685 --- /dev/null +++ b/docs/upgrading_to_project_factory_v9.0.md @@ -0,0 +1,105 @@ +# Upgrading to Project Factory v9.0 + +The v9.0 release of Project Factory is a backwards incompatible release for +service projects created with [shared_vpc](../modules/shared_vpc) module that +also have `container.googleapis.com` and/or `dataproc.googleapis.com` API's +enabled. If you don't have these API's enabled on your service projects or you +are creating new projects then there is no action required on your end. + +## Migration Instructions + +If your service projects have the `container.googleapis.com` API enabled then +follow instructions in [GKE API already enabled](#gke-api-already-enabled). + +If your service projects have the `dataproc.googleapis.com` API enabled then +follow instructions in [Dataproc API already enabled](#dataproc-api-already-enabled). + +### GKE API already enabled + +If you have the `container.googleapis.com` API enabled you will see in your +terraform plan that `google_compute_subnetwork_iam_member` +and `google_compute_subnetwork_iam_member` resources will be recreated. This is +a safe operation and you can apply the changes. Example plan can look like this: +```diff +# module.example.module.service-project.module.project-factory.google_compute_subnetwork_iam_member.gke_shared_vpc_subnets[0] will be destroyed + - resource "google_compute_subnetwork_iam_member" "gke_shared_vpc_subnets" { + - etag = "BwWrwEtp6B4=" -> null + - id = "projects/pf-ci-shared2-host-0004-29fd/regions/us-west1/subnetworks/shared-network-subnet-01/roles/compute.networkUser/serviceaccount:service-740499050292@container-engine-robot.iam.gserviceaccount.com" -> null + - member = "serviceAccount:service-740499050292@container-engine-robot.iam.gserviceaccount.com" -> null + - project = "pf-ci-shared2-host-0004-29fd" -> null + - region = "us-west1" -> null + - role = "roles/compute.networkUser" -> null + - subnetwork = "projects/pf-ci-shared2-host-0004-29fd/regions/us-west1/subnetworks/shared-network-subnet-01" -> null + } + + # module.example.module.service-project.module.project-factory.google_compute_subnetwork_iam_member.gke_shared_vpc_subnets[1] will be destroyed + - resource "google_compute_subnetwork_iam_member" "gke_shared_vpc_subnets" { + - etag = "BwWrwEtrwLA=" -> null + - id = "projects/pf-ci-shared2-host-0004-29fd/regions/us-west1/subnetworks/shared-network-subnet-02/roles/compute.networkUser/serviceaccount:service-740499050292@container-engine-robot.iam.gserviceaccount.com" -> null + - member = "serviceAccount:service-740499050292@container-engine-robot.iam.gserviceaccount.com" -> null + - project = "pf-ci-shared2-host-0004-29fd" -> null + - region = "us-west1" -> null + - role = "roles/compute.networkUser" -> null + - subnetwork = "projects/pf-ci-shared2-host-0004-29fd/regions/us-west1/subnetworks/shared-network-subnet-02" -> null + } + + # module.example.module.service-project.module.project-factory.google_project_iam_member.gke_host_agent[0] will be destroyed + - resource "google_project_iam_member" "gke_host_agent" { + - etag = "BwWrwEtQfSY=" -> null + - id = "pf-ci-shared2-host-0004-29fd/roles/container.hostServiceAgentUser/serviceaccount:service-740499050292@container-engine-robot.iam.gserviceaccount.com" -> null + - member = "serviceAccount:service-740499050292@container-engine-robot.iam.gserviceaccount.com" -> null + - project = "pf-ci-shared2-host-0004-29fd" -> null + - role = "roles/container.hostServiceAgentUser" -> null + } + + # module.example.module.service-project.module.shared_vpc_access.google_compute_subnetwork_iam_member.gke_shared_vpc_subnets[0] will be created + + resource "google_compute_subnetwork_iam_member" "gke_shared_vpc_subnets" { + + etag = (known after apply) + + id = (known after apply) + + member = "serviceAccount:service-740499050292@container-engine-robot.iam.gserviceaccount.com" + + project = "pf-ci-shared2-host-0004-29fd" + + region = "us-west1" + + role = "roles/compute.networkUser" + + subnetwork = "shared-network-subnet-01" + } + + # module.example.module.service-project.module.shared_vpc_access.google_compute_subnetwork_iam_member.gke_shared_vpc_subnets[1] will be created + + resource "google_compute_subnetwork_iam_member" "gke_shared_vpc_subnets" { + + etag = (known after apply) + + id = (known after apply) + + member = "serviceAccount:service-740499050292@container-engine-robot.iam.gserviceaccount.com" + + project = "pf-ci-shared2-host-0004-29fd" + + region = "us-west1" + + role = "roles/compute.networkUser" + + subnetwork = "shared-network-subnet-02" + } + + # module.example.module.service-project.module.shared_vpc_access.google_project_iam_member.gke_host_agent[0] will be created + + resource "google_project_iam_member" "gke_host_agent" { + + etag = (known after apply) + + id = (known after apply) + + member = "serviceAccount:service-740499050292@container-engine-robot.iam.gserviceaccount.com" + + project = "pf-ci-shared2-host-0004-29fd" + + role = "roles/container.hostServiceAgentUser" + } +``` + +### Dataproc API already enabled +If you have `dataproc.googleapis.com` API enabled on your projects then terraform +plan will try to bind `roles/compute.networkUser` to +`service-@dataproc-accounts.iam.gserviceaccount.com` at the +project level. Example: +```diff + # module.example.module.service-project.module.shared_vpc_access.google_project_iam_member.dataproc_shared_vpc_network_user[0] will be created + + resource "google_project_iam_member" "dataproc_shared_vpc_network_user" { + + etag = (known after apply) + + id = (known after apply) + + member = "serviceAccount:service-740499050292@dataproc-accounts.iam.gserviceaccount.com" + + project = "pf-ci-shared2-host-0004-29fd" + + role = "roles/compute.networkUser" + } +``` + +If you have already binded the `roles/compute.networkUser` to +`service-@dataproc-accounts.iam.gserviceaccount.com` at the +project level then please remove that binding before running `terraform apply`. diff --git a/examples/shared_vpc/main.tf b/examples/shared_vpc/main.tf index a8ea1aea..eb31a6b6 100644 --- a/examples/shared_vpc/main.tf +++ b/examples/shared_vpc/main.tf @@ -42,12 +42,13 @@ provider "random" { Host Project Creation *****************************************/ module "host-project" { - source = "../../" - random_project_id = true - name = var.host_project_name - org_id = var.organization_id - folder_id = var.folder_id - billing_account = var.billing_account + source = "../../" + random_project_id = true + name = var.host_project_name + org_id = var.organization_id + folder_id = var.folder_id + billing_account = var.billing_account + skip_gcloud_download = true } /****************************************** @@ -119,9 +120,11 @@ module "service-project" { activate_apis = [ "compute.googleapis.com", "container.googleapis.com", + "dataproc.googleapis.com", ] disable_services_on_destroy = "false" + skip_gcloud_download = "true" } /****************************************** @@ -143,9 +146,11 @@ module "service-project-b" { activate_apis = [ "compute.googleapis.com", "container.googleapis.com", + "dataproc.googleapis.com", ] disable_services_on_destroy = "false" + skip_gcloud_download = "true" } /****************************************** diff --git a/modules/core_project_factory/main.tf b/modules/core_project_factory/main.tf index b3545118..28af7aa6 100644 --- a/modules/core_project_factory/main.tf +++ b/modules/core_project_factory/main.tf @@ -42,14 +42,8 @@ locals { "%s@cloudservices.gserviceaccount.com", google_project.main.number, ) - activate_apis = var.impersonate_service_account != "" ? concat(var.activate_apis, ["iamcredentials.googleapis.com"]) : var.activate_apis - api_s_account_fmt = format("serviceAccount:%s", local.api_s_account) - gke_shared_vpc_enabled = var.shared_vpc_enabled && contains(var.activate_apis, "container.googleapis.com") - gke_s_account = format( - "service-%s@container-engine-robot.iam.gserviceaccount.com", - google_project.main.number, - ) - gke_s_account_fmt = local.gke_shared_vpc_enabled ? format("serviceAccount:%s", local.gke_s_account) : "" + activate_apis = var.impersonate_service_account != "" ? concat(var.activate_apis, ["iamcredentials.googleapis.com"]) : var.activate_apis + api_s_account_fmt = format("serviceAccount:%s", local.api_s_account) project_bucket_name = var.bucket_name != "" ? var.bucket_name : format("%s-state", local.temp_project_id) create_bucket = var.bucket_project != "" ? "true" : "false" @@ -58,12 +52,11 @@ locals { local.group_id, local.s_account_fmt, local.api_s_account_fmt, - local.gke_s_account_fmt, ], ) # Workaround for https://github.com/hashicorp/terraform/issues/10857 - shared_vpc_users_length = local.gke_shared_vpc_enabled ? 4 : 3 + shared_vpc_users_length = 3 } resource "null_resource" "preconditions" { @@ -282,8 +275,7 @@ resource "google_service_account_iam_member" "service_account_grant_to_group" { } /****************************************************************************************************************** - compute.networkUser role granted to G Suite group, APIs Service account, Project Service Account, and GKE Service - Account on shared VPC + compute.networkUser role granted to G Suite group, APIs Service account, and Project Service Account *****************************************************************************************************************/ resource "google_project_iam_member" "controlling_group_vpc_membership" { count = var.shared_vpc_enabled && length(var.shared_vpc_subnets) == 0 ? local.shared_vpc_users_length : 0 @@ -437,45 +429,6 @@ resource "google_storage_bucket_iam_member" "api_s_account_storage_admin_on_proj ] } -/****************************************** - compute.networkUser role granted to GKE service account for GKE on shared VPC subnets - *****************************************/ -resource "google_compute_subnetwork_iam_member" "gke_shared_vpc_subnets" { - provider = google-beta - count = local.gke_shared_vpc_enabled && length(var.shared_vpc_subnets) != 0 ? length(var.shared_vpc_subnets) : 0 - subnetwork = element( - split("/", var.shared_vpc_subnets[count.index]), - index( - split("/", var.shared_vpc_subnets[count.index]), - "subnetworks", - ) + 1, - ) - role = "roles/compute.networkUser" - region = element( - split("/", var.shared_vpc_subnets[count.index]), - index(split("/", var.shared_vpc_subnets[count.index]), "regions") + 1, - ) - project = var.shared_vpc - member = local.gke_s_account_fmt - - depends_on = [ - module.project_services, - ] -} - -/****************************************** - container.hostServiceAgentUser role granted to GKE service account for GKE on shared VPC - *****************************************/ -resource "google_project_iam_member" "gke_host_agent" { - count = local.gke_shared_vpc_enabled ? 1 : 0 - project = var.shared_vpc - role = "roles/container.hostServiceAgentUser" - member = local.gke_s_account_fmt - depends_on = [ - module.project_services, - ] -} - /****************************************** Attachment to VPC Service Control Perimeter *****************************************/ diff --git a/modules/core_project_factory/outputs.tf b/modules/core_project_factory/outputs.tf index fef7e6bd..94c2af9a 100644 --- a/modules/core_project_factory/outputs.tf +++ b/modules/core_project_factory/outputs.tf @@ -84,3 +84,8 @@ output "api_s_account_fmt" { value = local.api_s_account_fmt description = "API service account email formatted for terraform use" } + +output "enabled_apis" { + description = "Enabled APIs in the project" + value = module.project_services.enabled_apis +} diff --git a/modules/project_services/README.md b/modules/project_services/README.md index decbf08f..9299cd16 100644 --- a/modules/project_services/README.md +++ b/modules/project_services/README.md @@ -49,6 +49,7 @@ See [examples/project_services](./examples/project_services) for a full example | Name | Description | |------|-------------| +| enabled\_apis | Enabled APIs in the project | | project\_id | The GCP project you want to enable APIs on | diff --git a/modules/project_services/outputs.tf b/modules/project_services/outputs.tf index a53ef375..63248bdf 100644 --- a/modules/project_services/outputs.tf +++ b/modules/project_services/outputs.tf @@ -18,3 +18,8 @@ output "project_id" { description = "The GCP project you want to enable APIs on" value = element(concat([for v in google_project_service.project_services : v.project], [var.project_id]), 0) } + +output "enabled_apis" { + description = "Enabled APIs in the project" + value = [for api in google_project_service.project_services : api.service] +} diff --git a/modules/shared_vpc/main.tf b/modules/shared_vpc/main.tf index 083a1040..b170b43e 100755 --- a/modules/shared_vpc/main.tf +++ b/modules/shared_vpc/main.tf @@ -60,6 +60,17 @@ module "project-factory" { skip_gcloud_download = var.skip_gcloud_download } +/****************************************** + Setting API service accounts for shared VPC + *****************************************/ +module "shared_vpc_access" { + source = "../shared_vpc_access" + host_project_id = var.shared_vpc + service_project_id = module.project-factory.project_id + active_apis = module.project-factory.enabled_apis + shared_vpc_subnets = var.shared_vpc_subnets +} + /****************************************** Billing budget to create if amount is set *****************************************/ diff --git a/modules/shared_vpc/outputs.tf b/modules/shared_vpc/outputs.tf index c18b92bd..be11ee1b 100755 --- a/modules/shared_vpc/outputs.tf +++ b/modules/shared_vpc/outputs.tf @@ -21,7 +21,7 @@ output "project_name" { output "project_id" { description = "If provided, the project uses the given project ID. Mutually exclusive with random_project_id being true." - value = module.project-factory.project_id + value = module.shared_vpc_access.project_id } output "project_number" { diff --git a/modules/shared_vpc_access/README.md b/modules/shared_vpc_access/README.md new file mode 100644 index 00000000..c452a78b --- /dev/null +++ b/modules/shared_vpc_access/README.md @@ -0,0 +1,41 @@ +# Shared VPC Access + +This module grants IAM permissions on host project and subnets to appropriate API service accounts based on activated +APIs. For now only GKE and Dataproc APIs are supported. + +## Example Usage +```hcl +module "shared_vpc_access" { + source = "terraform-google-modules/project-factory/google//modules/shared_vpc_access" + host_project_id = var.shared_vpc + service_project_id = var.service_project + active_apis = [ + "compute.googleapis.com", + "container.googleapis.com", + "dataproc.googleapis.com", + ] + shared_vpc_subnets = [ + "projects/pf-ci-shared2/regions/us-west1/subnetworks/shared-network-subnet-01", + "projects/pf-ci-shared2/regions/us-west1/subnetworks/shared-network-subnet-02", + ] +} +``` + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| active\_apis | The list of active apis on the service project. If api is not active this module will not try to activate it | list(string) | `` | no | +| host\_project\_id | The ID of the host project which hosts the shared VPC | string | n/a | yes | +| service\_project\_id | The ID of the service project | string | n/a | yes | +| shared\_vpc\_subnets | List of subnets fully qualified subnet IDs (ie. projects/$project_id/regions/$region/subnetworks/$subnet_id) | list(string) | `` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| active\_api\_service\_accounts | List of active API service accounts in the service project. | +| project\_id | Service project ID. | + + diff --git a/modules/shared_vpc_access/main.tf b/modules/shared_vpc_access/main.tf new file mode 100644 index 00000000..79a18e44 --- /dev/null +++ b/modules/shared_vpc_access/main.tf @@ -0,0 +1,78 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +data "google_project" "service_project" { + project_id = var.service_project_id +} + +locals { + gke_shared_vpc_enabled = contains(var.active_apis, "container.googleapis.com") + gke_s_account = local.gke_shared_vpc_enabled ? format( + "service-%s@container-engine-robot.iam.gserviceaccount.com", + data.google_project.service_project.number, + ) : "" + dataproc_shared_vpc_enabled = contains(var.active_apis, "dataproc.googleapis.com") + dataproc_s_account = local.dataproc_shared_vpc_enabled ? format( + "service-%s@dataproc-accounts.iam.gserviceaccount.com", + data.google_project.service_project.number + ) : "" + active_api_s_accounts = compact([local.gke_s_account, local.dataproc_s_account]) +} + +/****************************************** + compute.networkUser role granted to GKE service account for GKE on shared VPC subnets + See: https://cloud.google.com/kubernetes-engine/docs/how-to/cluster-shared-vpc + *****************************************/ +resource "google_compute_subnetwork_iam_member" "gke_shared_vpc_subnets" { + provider = google-beta + count = local.gke_shared_vpc_enabled && length(var.shared_vpc_subnets) != 0 ? length(var.shared_vpc_subnets) : 0 + subnetwork = element( + split("/", var.shared_vpc_subnets[count.index]), + index( + split("/", var.shared_vpc_subnets[count.index]), + "subnetworks", + ) + 1, + ) + role = "roles/compute.networkUser" + region = element( + split("/", var.shared_vpc_subnets[count.index]), + index(split("/", var.shared_vpc_subnets[count.index]), "regions") + 1, + ) + project = var.host_project_id + member = format("serviceAccount:%s", local.gke_s_account) +} + +/****************************************** + container.hostServiceAgentUser role granted to GKE service account for GKE on shared VPC + See:https://cloud.google.com/kubernetes-engine/docs/how-to/cluster-shared-vpc + *****************************************/ +resource "google_project_iam_member" "gke_host_agent" { + count = local.gke_shared_vpc_enabled ? 1 : 0 + project = var.host_project_id + role = "roles/container.hostServiceAgentUser" + member = format("serviceAccount:%s", local.gke_s_account) +} + +/****************************************** + compute.networkUser role granted to dataproc service account for dataproc on shared VPC subnets + See: https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/network#creating_a_cluster_that_uses_a_vpc_network_in_another_project + *****************************************/ +resource "google_project_iam_member" "dataproc_shared_vpc_network_user" { + count = local.dataproc_shared_vpc_enabled ? 1 : 0 + project = var.host_project_id + role = "roles/compute.networkUser" + member = format("serviceAccount:%s", local.dataproc_s_account) +} diff --git a/modules/shared_vpc_access/outputs.tf b/modules/shared_vpc_access/outputs.tf new file mode 100644 index 00000000..ff16ab5f --- /dev/null +++ b/modules/shared_vpc_access/outputs.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "active_api_service_accounts" { + description = "List of active API service accounts in the service project." + value = local.active_api_s_accounts +} + +output "project_id" { + description = "Service project ID." + value = var.service_project_id + depends_on = [ + google_compute_subnetwork_iam_member.gke_shared_vpc_subnets, + google_project_iam_member.gke_host_agent, + google_project_iam_member.dataproc_shared_vpc_network_user, + ] +} diff --git a/modules/shared_vpc_access/variables.tf b/modules/shared_vpc_access/variables.tf new file mode 100644 index 00000000..eaa3fa9b --- /dev/null +++ b/modules/shared_vpc_access/variables.tf @@ -0,0 +1,37 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "host_project_id" { + description = "The ID of the host project which hosts the shared VPC" + type = string +} + +variable "service_project_id" { + description = "The ID of the service project" + type = string +} + +variable "shared_vpc_subnets" { + description = "List of subnets fully qualified subnet IDs (ie. projects/$project_id/regions/$region/subnetworks/$subnet_id)" + type = list(string) + default = [] +} + +variable "active_apis" { + description = "The list of active apis on the service project. If api is not active this module will not try to activate it" + type = list(string) + default = [] +} diff --git a/modules/shared_vpc_access/versions.tf b/modules/shared_vpc_access/versions.tf new file mode 100644 index 00000000..2e838fcc --- /dev/null +++ b/modules/shared_vpc_access/versions.tf @@ -0,0 +1,24 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = "~> 0.12.6" + + required_providers { + google = ">= 3.8, < 4.0" + google-beta = ">= 3.8, < 4.0" + } +} diff --git a/test/integration/dynamic_shared_vpc/controls/svpc.rb b/test/integration/dynamic_shared_vpc/controls/svpc.rb index 95f30654..82a45ea9 100644 --- a/test/integration/dynamic_shared_vpc/controls/svpc.rb +++ b/test/integration/dynamic_shared_vpc/controls/svpc.rb @@ -69,6 +69,13 @@ role: "roles/container.hostServiceAgentUser", ) end + + it "includes the dataproc service account in the roles/compute.networkUser IAM binding" do + expect(bindings).to include( + members: including("serviceAccount:service-#{service_project_number}@dataproc-accounts.iam.gserviceaccount.com"), + role: "roles/compute.networkUser", + ) + end end describe command("gcloud beta compute networks subnets get-iam-policy #{shared_vpc_subnet_name_01} --region #{shared_vpc_subnet_region_01} --project #{shared_vpc} --format=json") do