diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml index f8686b2b..3ebbb24a 100644 --- a/build/int.cloudbuild.yaml +++ b/build/int.cloudbuild.yaml @@ -73,6 +73,28 @@ steps: - id: destroy-simple-folder name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy simple-folder-default'] + +- id: init-tfsource + waitFor: + - prepare + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'cft test run TestTFCloudBuildSourceSimple --stage init --verbose'] +- id: apply-tfsource + waitFor: + - init-tfsource + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'cft test run TestTFCloudBuildSourceSimple --stage apply --verbose'] +- id: verify-tfsource + waitFor: + - apply-tfsource + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'cft test run TestTFCloudBuildSourceSimple --stage verify --verbose'] +- id: teardown-tfsource + waitFor: + - verify-tfsource + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'cft test run TestTFCloudBuildSourceSimple --stage teardown --verbose'] + - id: init-tfbuilder waitFor: - prepare @@ -93,6 +115,7 @@ steps: - verify-tfbuilder name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'cft test run TestTFCloudBuildBuilder --stage teardown --verbose'] + - id: init-tfworkspace waitFor: - prepare @@ -113,6 +136,7 @@ steps: - verify-tfworkspace name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'cft test run TestTFCloudBuildWorkspaceSimple --stage teardown --verbose'] + tags: - 'ci' - 'integration' diff --git a/examples/tf_cloudbuild_source_simple/README.md b/examples/tf_cloudbuild_source_simple/README.md new file mode 100644 index 00000000..5fa39196 --- /dev/null +++ b/examples/tf_cloudbuild_source_simple/README.md @@ -0,0 +1,23 @@ +## Overview + +This example demonstrates the simplest usage of the [tf_cloudbuild_source](../../modules/tf_cloudbuild_source/) module. + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| billing\_account | The ID of the billing account to associate projects with. | `string` | n/a | yes | +| group\_org\_admins | Google Group for GCP Organization Administrators | `string` | n/a | yes | +| org\_id | GCP Organization ID | `string` | n/a | yes | +| parent\_folder | The bootstrap parent folder | `string` | `""` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| cloudbuild\_project\_id | Project where CloudBuild configuration and terraform container image will reside. | +| csr\_repos | List of Cloud Source Repos created by the module. | +| gcs\_cloudbuild\_default\_bucket | Bucket used to store temporary files in CloudBuild project. | + + diff --git a/examples/tf_cloudbuild_source_simple/main.tf b/examples/tf_cloudbuild_source_simple/main.tf new file mode 100644 index 00000000..f7e391aa --- /dev/null +++ b/examples/tf_cloudbuild_source_simple/main.tf @@ -0,0 +1,25 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "tf_source" { + source = "../../modules/tf_cloudbuild_source" + + org_id = var.org_id + folder_id = var.parent_folder + billing_account = var.billing_account + group_org_admins = var.group_org_admins + buckets_force_destroy = true +} diff --git a/examples/tf_cloudbuild_source_simple/outputs.tf b/examples/tf_cloudbuild_source_simple/outputs.tf new file mode 100644 index 00000000..a269f986 --- /dev/null +++ b/examples/tf_cloudbuild_source_simple/outputs.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "cloudbuild_project_id" { + description = "Project where CloudBuild configuration and terraform container image will reside." + value = module.tf_source.cloudbuild_project_id +} + +output "csr_repos" { + description = "List of Cloud Source Repos created by the module." + value = module.tf_source.csr_repos +} + +output "gcs_cloudbuild_default_bucket" { + description = "Bucket used to store temporary files in CloudBuild project." + value = module.tf_source.gcs_cloudbuild_default_bucket +} diff --git a/examples/tf_cloudbuild_source_simple/variables.tf b/examples/tf_cloudbuild_source_simple/variables.tf new file mode 100644 index 00000000..e73a4ee1 --- /dev/null +++ b/examples/tf_cloudbuild_source_simple/variables.tf @@ -0,0 +1,36 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "org_id" { + description = "GCP Organization ID" + type = string +} + +variable "parent_folder" { + description = "The bootstrap parent folder" + type = string + default = "" +} + +variable "billing_account" { + description = "The ID of the billing account to associate projects with." + type = string +} + +variable "group_org_admins" { + description = "Google Group for GCP Organization Administrators" + type = string +} diff --git a/modules/tf_cloudbuild_source/README.md b/modules/tf_cloudbuild_source/README.md new file mode 100644 index 00000000..b003bb1f --- /dev/null +++ b/modules/tf_cloudbuild_source/README.md @@ -0,0 +1,70 @@ +## Overview + +## Usage + +Basic usage of this module is as follows: + +```hcl +module "tf-cloudbuild-core" { + source = "terraform-google-modules/bootstrap/google//modules/tf_cloudbuild_source" + version = "~> 6.1" + + org_id = var.org_id + billing_account = var.billing_account + group_org_admins = var.group_org_admins +} +``` + +Functional examples are included in the [examples](../../examples/) directory. + +## Resources created + +This module creates: + +- Project for Cloud Build. +- Default Cloud Build bucket. +- Set of Cloud Source Repos. + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| activate\_apis | List of APIs to enable in the Cloudbuild project. | `list(string)` |
[
"serviceusage.googleapis.com",
"servicenetworking.googleapis.com",
"compute.googleapis.com",
"logging.googleapis.com",
"iam.googleapis.com",
"admin.googleapis.com"
]
| no | +| billing\_account | The ID of the billing account to associate projects with. | `string` | n/a | yes | +| buckets\_force\_destroy | When deleting CloudBuild buckets, this boolean option will delete all contained objects. If false, Terraform will fail to delete buckets which contain objects. | `bool` | `false` | no | +| cloud\_source\_repos | List of Cloud Source Repos to create with CloudBuild triggers. | `list(string)` |
[
"gcp-policies",
"gcp-org",
"gcp-envs",
"gcp-networks",
"gcp-projects"
]
| no | +| folder\_id | The ID of a folder to host this project | `string` | `""` | no | +| group\_org\_admins | Google Group for GCP Organization Administrators | `string` | n/a | yes | +| location | Location for build artifacts bucket | `string` | `"us-central1"` | no | +| org\_id | GCP Organization ID | `string` | n/a | yes | +| project\_id | Custom project ID to use for project created. | `string` | `""` | no | +| project\_labels | Labels to apply to the project. | `map(string)` | `{}` | no | +| storage\_bucket\_labels | Labels to apply to the storage bucket. | `map(string)` | `{}` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| cloudbuild\_project\_id | Project for CloudBuild and Cloud Source Repositories. | +| csr\_repos | List of Cloud Source Repos created by the module. | +| gcs\_cloudbuild\_default\_bucket | Bucket used to store temporary files in CloudBuild project. | + + + +## Requirements + +### Software + +- [Terraform](https://www.terraform.io/downloads.html) >= 0.13.0 +- [terraform-provider-google] plugin >= 3.50.x + +### Permissions + +- `roles/resourcemanager.projectCreator` +- `roles/billing.user` + +## Contributing + +Refer to the [contribution guidelines](../../CONTRIBUTING.md) for +information on contributing to this module. diff --git a/modules/tf_cloudbuild_source/main.tf b/modules/tf_cloudbuild_source/main.tf new file mode 100644 index 00000000..de831281 --- /dev/null +++ b/modules/tf_cloudbuild_source/main.tf @@ -0,0 +1,93 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + cloudbuild_project_id = var.project_id != "" ? var.project_id : "tf-cloudbuild-" + use_random_suffix = var.project_id == "" + + cloudbuild_apis = [ + "cloudbuild.googleapis.com", + "sourcerepo.googleapis.com", + "storage-api.googleapis.com", + "iam.googleapis.com", + "cloudresourcemanager.googleapis.com", + "cloudbilling.googleapis.com" + ] + + activate_apis = distinct(concat(var.activate_apis, local.cloudbuild_apis)) +} + +module "cloudbuild_project" { + source = "terraform-google-modules/project-factory/google" + version = "~> 13.0" + + name = local.cloudbuild_project_id + random_project_id = local.use_random_suffix + disable_services_on_destroy = false + folder_id = var.folder_id + org_id = var.org_id + billing_account = var.billing_account + activate_apis = local.activate_apis + labels = var.project_labels +} + +// On the first run of cloud build submit, a bucket is automaticaly created with name "[PROJECT_ID]_cloudbuild" +// https://cloud.google.com/sdk/gcloud/reference/builds/submit#:~:text=%5BPROJECT_ID%5D_cloudbuild +// This bucket is create in the default region "US" +// https://cloud.google.com/storage/docs/json_api/v1/buckets/insert#:~:text=or%20multi%2Dregion.-,Defaults%20to%20%22US%22,-.%20See%20Cloud%20Storage +// Creating the bucket beforehand make it is possible to define a custom location. +module "cloudbuild_bucket" { + source = "terraform-google-modules/cloud-storage/google//modules/simple_bucket" + version = "~> 3.2" + + name = "${module.cloudbuild_project.project_id}_cloudbuild" + project_id = module.cloudbuild_project.project_id + location = var.location + labels = var.storage_bucket_labels + force_destroy = var.buckets_force_destroy +} + +resource "google_sourcerepo_repository" "gcp_repo" { + for_each = length(var.cloud_source_repos) > 0 ? toset(var.cloud_source_repos) : [] + + project = module.cloudbuild_project.project_id + name = each.value +} + +resource "google_project_iam_member" "org_admins_cloudbuild_editor" { + project = module.cloudbuild_project.project_id + role = "roles/cloudbuild.builds.editor" + member = "group:${var.group_org_admins}" +} + +resource "google_project_iam_member" "org_admins_cloudbuild_viewer" { + project = module.cloudbuild_project.project_id + role = "roles/viewer" + member = "group:${var.group_org_admins}" +} + +resource "google_project_iam_member" "org_admins_source_repo_admin" { + count = length(var.cloud_source_repos) > 0 ? 1 : 0 + project = module.cloudbuild_project.project_id + role = "roles/source.admin" + member = "group:${var.group_org_admins}" +} + +resource "google_storage_bucket_iam_member" "cloudbuild_iam" { + bucket = module.cloudbuild_bucket.bucket.name + role = "roles/storage.admin" + member = "serviceAccount:${module.cloudbuild_project.project_number}@cloudbuild.gserviceaccount.com" +} diff --git a/modules/tf_cloudbuild_source/outputs.tf b/modules/tf_cloudbuild_source/outputs.tf new file mode 100644 index 00000000..95cdc3d8 --- /dev/null +++ b/modules/tf_cloudbuild_source/outputs.tf @@ -0,0 +1,41 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "cloudbuild_project_id" { + description = "Project for CloudBuild and Cloud Source Repositories." + value = module.cloudbuild_project.project_id + + depends_on = [ + google_storage_bucket_iam_member.cloudbuild_iam, + google_project_iam_member.org_admins_cloudbuild_editor, + google_project_iam_member.org_admins_cloudbuild_viewer, + google_project_iam_member.org_admins_source_repo_admin + ] +} + +output "csr_repos" { + description = "List of Cloud Source Repos created by the module." + value = google_sourcerepo_repository.gcp_repo +} + +output "gcs_cloudbuild_default_bucket" { + description = "Bucket used to store temporary files in CloudBuild project." + value = module.cloudbuild_bucket.bucket.name + + depends_on = [ + google_storage_bucket_iam_member.cloudbuild_iam + ] +} diff --git a/modules/tf_cloudbuild_source/variables.tf b/modules/tf_cloudbuild_source/variables.tf new file mode 100644 index 00000000..95f58fe3 --- /dev/null +++ b/modules/tf_cloudbuild_source/variables.tf @@ -0,0 +1,93 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "org_id" { + description = "GCP Organization ID" + type = string +} + +variable "folder_id" { + description = "The ID of a folder to host this project" + type = string + default = "" +} + +variable "project_id" { + description = "Custom project ID to use for project created." + default = "" + type = string +} + +variable "project_labels" { + description = "Labels to apply to the project." + type = map(string) + default = {} +} + +variable "storage_bucket_labels" { + description = "Labels to apply to the storage bucket." + type = map(string) + default = {} +} + +variable "location" { + description = "Location for build artifacts bucket" + type = string + default = "us-central1" +} + +variable "buckets_force_destroy" { + description = "When deleting CloudBuild buckets, this boolean option will delete all contained objects. If false, Terraform will fail to delete buckets which contain objects." + type = bool + default = false +} + +variable "billing_account" { + description = "The ID of the billing account to associate projects with." + type = string +} + +variable "group_org_admins" { + description = "Google Group for GCP Organization Administrators" + type = string +} + +variable "activate_apis" { + description = "List of APIs to enable in the Cloudbuild project." + type = list(string) + + default = [ + "serviceusage.googleapis.com", + "servicenetworking.googleapis.com", + "compute.googleapis.com", + "logging.googleapis.com", + "iam.googleapis.com", + "admin.googleapis.com" + ] +} + +variable "cloud_source_repos" { + description = "List of Cloud Source Repos to create with CloudBuild triggers." + type = list(string) + + default = [ + "gcp-policies", + "gcp-org", + "gcp-envs", + "gcp-networks", + "gcp-projects", + ] +} diff --git a/modules/tf_cloudbuild_source/versions.tf b/modules/tf_cloudbuild_source/versions.tf new file mode 100644 index 00000000..294ce959 --- /dev/null +++ b/modules/tf_cloudbuild_source/versions.tf @@ -0,0 +1,34 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">=0.13.0" + + required_providers { + google = { + source = "hashicorp/google" + version = ">= 3.50, < 5.0" + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 3.50, < 5.0" + } + } + + provider_meta "google" { + module_name = "blueprints/terraform/terraform-google-bootstrap:tf_cloudbuild_source/v6.0.0" + } +} diff --git a/test/integration/tf_cloudbuild_source_simple/tf_cloudbuild_source_simple_test.go b/test/integration/tf_cloudbuild_source_simple/tf_cloudbuild_source_simple_test.go new file mode 100644 index 00000000..2f183fc7 --- /dev/null +++ b/test/integration/tf_cloudbuild_source_simple/tf_cloudbuild_source_simple_test.go @@ -0,0 +1,58 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tf_cloudbuild_source_simple + +import ( + "fmt" + "testing" + + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/gcloud" + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft" + "github.com/stretchr/testify/assert" +) + +func TestTFCloudBuildSourceSimple(t *testing.T) { + bpt := tft.NewTFBlueprintTest(t) + + bpt.DefineVerify(func(assert *assert.Assertions) { + bpt.DefaultVerify(assert) + + projectID := bpt.GetStringOutput("cloudbuild_project_id") + + // cloudbuild buckets + cloudbuildBucket := bpt.GetStringOutput("gcs_cloudbuild_default_bucket") + // we can't use runf since we need to override --format json with --json for alpha storage + bucketOP := gcloud.Run(t, fmt.Sprintf("alpha storage ls --buckets gs://%s", cloudbuildBucket), gcloud.WithCommonArgs([]string{"--project", projectID, "--json"})).Array() + assert.Equalf(1, len(bucketOP), "%s bucket should exist", cloudbuildBucket) + assert.Truef(bucketOP[0].Get("metadata.iamConfiguration.uniformBucketLevelAccess.enabled").Bool(), "%s bucket uniformBucketLevelAccess should be enabled", cloudbuildBucket) + assert.Truef(bucketOP[0].Get("metadata.versioning.enabled").Bool(), "%s bucket versioning should be enabled", cloudbuildBucket) + + //source repos + repos := []string{ + "gcp-policies", + "gcp-org", + "gcp-envs", + "gcp-networks", + "gcp-projects", + } + for _, repo := range repos { + url := fmt.Sprintf("https://source.developers.google.com/p/%s/r/%s", projectID, repo) + repoOP := gcloud.Runf(t, "source repos describe %s --project %s", repo, projectID) + assert.Equalf(url, repoOP.Get("url").String(), "source repo %s should have url %s", repo, url) + } + }) + + bpt.Test() +}