diff --git a/README.md b/README.md index dce693e4..8229bcc9 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ module "cd_pipeline" { project_id = var.project_id primary_location = "us-central1" gar_repo_name = module.ci_pipeline.app_artifact_repo - manifest_wet_repo = "app-wet-manifests" + cloudbuild_cd_repo = "cloudbuild-cd-config-pc" deploy_branch_clusters = { dev = { cluster = "dev-cluster", @@ -119,7 +119,7 @@ service account with the necessary roles applied. ### APIs -A project with the following APIs enabled must be used to host the +Projects with the following APIs enabled must be used to host the resources of this module: CI/CD Project @@ -128,6 +128,8 @@ CI/CD Project - Storage API `storage-api.googleapis.com` - Service Usage API `serviceusage.googleapis.com` - Cloud Build API `cloudbuild.googleapis.com` +- Cloud Deploy API `clouddeploy.googleapis.com` +- Pub/Sub API `pubsub.googleapis.com` - Container Registry API `containerregistry.googleapis.com` - IAM Credentials API `iamcredentials.googleapis.com` - Cloud Source Repositories API `sourcerepo.googleapis.com` diff --git a/build/cloudbuild-cd.yaml b/build/cloudbuild-cd.yaml index 3e1829b7..6905d510 100644 --- a/build/cloudbuild-cd.yaml +++ b/build/cloudbuild-cd.yaml @@ -28,12 +28,6 @@ artifacts: - '/workspace/svcs-endpoints-filtered.json' steps: -############################### Deploy to Cluster ########################### - -- name: "gcr.io/cloud-builders/gke-deploy" - id: "deploy-to-cluster" - args: ['apply', '-f', 'manifests', '-p', '$_CLUSTER_PROJECT', '-c', '$_CLUSTER_NAME', '-l', '$_DEFAULT_REGION'] - ############################### Post-Deploy Checks ########################### # Get endpoint IPs @@ -48,9 +42,6 @@ steps: gcloud container clusters get-credentials $_CLUSTER_NAME --region=$_DEFAULT_REGION kubectl get svc -ojson > /workspace/svcs.json - ### Do we need a version that runs against private IP'd svcs? - # jq '[.items[] | ({svc: .metadata.name, endpoint: "\(.spec.clusterIPs[]):\(.spec.ports[].port)"}), (select(.status.loadBalancer.ingress != null) | {svc: .metadata.name, endpoint: "\(.status.loadBalancer.ingress[].ip):\(.spec.ports[].port)"})]' /workspace/svcs.json > /workspace/svcs-endpoints-filtered.json - ### Below only grabs external IP'd svcs jq '[.items[] | (select(.status.loadBalancer.ingress != null) | {svc: .metadata.name, endpoint: "\(.status.loadBalancer.ingress[].ip):\(.spec.ports[].port)"})]' /workspace/svcs.json > /workspace/svcs-endpoints-filtered.json @@ -134,29 +125,7 @@ steps: - '-xe' - -c - | - # Only run this step if there is a next environment supplied in the trigger substitutions - # This step will be skipped for the final environment, when there are no more promotions to perform - if [ -n "${_NEXT_ENV}" ]; then - git config --global user.email "cicd-agent@secure-cicd.com" # TODO: variable with custom domain group alias - git config --global user.name "Secure CICD Build Agent (automated)" # TODO: app name var - - # Clone the WET repo - cd /workspace - gcloud source repos clone $_MANIFEST_WET_REPO --project=${PROJECT_ID} - git -C /workspace/$_MANIFEST_WET_REPO branch $_NEXT_ENV - git -C /workspace/$_MANIFEST_WET_REPO checkout $_NEXT_ENV - git -C /workspace/$_MANIFEST_WET_REPO branch --set-upstream-to=origin/$_NEXT_ENV || echo "$_NEXT_ENV branch does not yet exist" - git -C /workspace/$_MANIFEST_WET_REPO pull --no-rebase || echo "$_NEXT_ENV branch does not yet exist" - - cd /workspace/$_MANIFEST_WET_REPO - - # Merge manifests up - git status - git merge origin/$BRANCH_NAME - git add . - git push -u origin $_NEXT_ENV - else - echo "_NEXT_ENV not specified. No further environment to promote to" - fi + gcloud deploy releases promote --release=$_RELEASE_ID --delivery-pipeline=$_CLOUDDEPLOY_PIPELINE_NAME --region=$_DEFAULT_REGION + waitFor: - 'binary-authorization-checkpoint' diff --git a/build/cloudbuild-ci.yaml b/build/cloudbuild-ci.yaml index 11086d0f..70dc0a8e 100644 --- a/build/cloudbuild-ci.yaml +++ b/build/cloudbuild-ci.yaml @@ -167,48 +167,16 @@ steps: - 'container-structure' - 'container-scanner' -### Hydrate manifests and push to wet repo +## Create Cloud Deploy Release to trigger render and deploy to first env - name: $_DEFAULT_REGION-docker.pkg.dev/$PROJECT_ID/$_GAR_REPOSITORY/skaffold-builder - id: "commit-upstream" + id: "cloud-deploy-release" entrypoint: "/bin/bash" args: - '-xe' - -c - | - - git config --global user.email "cicd-agent@secure-cicd.com" # TODO: variable with custom domain group alias - git config --global user.name "Secure CICD Build Agent (automated)" # TODO: app name var - - cd /workspace - - # Clone the DRY repo - gcloud config set project ${PROJECT_ID} - gcloud source repos clone $_MANIFEST_DRY_REPO --project=${PROJECT_ID} - - # Clone the WET repo - cd /workspace - gcloud source repos clone $_MANIFEST_WET_REPO --project=${PROJECT_ID} - DEFAULT_BRANCH = ($$(git -C /workspace/$_MANIFEST_WET_REPO remote show origin | grep 'HEAD branch' | cut -d' ' -f5)) - git -C /workspace/$_MANIFEST_WET_REPO branch $_WET_BRANCH_NAME || echo "$_WET_BRANCH_NAME branch already exists" - git -C /workspace/$_MANIFEST_WET_REPO checkout $_WET_BRANCH_NAME - git -C /workspace/$_MANIFEST_WET_REPO branch --set-upstream-to=origin/$_WET_BRANCH_NAME || echo "$_WET_BRANCH_NAME branch does not yet exist" - git -C /workspace/$_MANIFEST_WET_REPO pull --no-rebase || echo "$_WET_BRANCH_NAME branch does not yet exist" - - git -C /workspace/$_MANIFEST_WET_REPO merge $${DEFAULT_BRANCH} - mkdir -p /workspace/$_MANIFEST_WET_REPO/manifests - - skaffold config set --global local-cluster false - skaffold render --offline=true --digest-source='remote' --filename='skaffold.yaml' --build-artifacts=/artifacts/build-artifacts.json --output='/workspace/$_MANIFEST_WET_REPO/manifests/skaffold-render.yaml' --default-repo=${_GAR_REPO_URI} --loud=true - cp /workspace/$_MANIFEST_WET_REPO/manifests/skaffold-render.yaml ./skaffold-render.yaml - - # Go into wet repo and commit changes - cd /workspace/$_MANIFEST_WET_REPO - - # git branch $_WET_BRANCH_NAME || echo "$_WET_BRANCH_NAME branch already exists" - # git checkout $_WET_BRANCH_NAME - git add . - git commit -m 'new deployment of applications' || echo "Nothing to commit" - git push --set-upstream origin $_WET_BRANCH_NAME + RELEASE_NAME='release-$SHORT_SHA' + gcloud deploy releases create $${RELEASE_NAME} --delivery-pipeline=${_CLOUDDEPLOY_PIPELINE_NAME} --build-artifacts=/artifacts/build-artifacts.json --region=${_DEFAULT_REGION} volumes: - path: '/artifacts' name: 'artifacts' diff --git a/examples/app_cicd/README.md b/examples/app_cicd/README.md index 5406d2fc..f961bce9 100644 --- a/examples/app_cicd/README.md +++ b/examples/app_cicd/README.md @@ -140,8 +140,8 @@ done | Name | Description | |------|-------------| -| bin\_auth\_attestor\_names | Names of Attestors | -| bin\_auth\_attestor\_project\_id | Project ID where attestors get created | +| binauth\_attestor\_names | Names of Attestors | +| binauth\_attestor\_project\_id | Project ID where attestors get created | | boa\_artifact\_repo | GAR Repo created to store BoA images | | build\_trigger\_name | The name of the cloud build trigger for the bank of anthos repo. | | cache\_bucket\_name | The name of the storage bucket for cloud build. | diff --git a/examples/app_cicd/cloud-build-builder/Dockerfile b/examples/app_cicd/cloud-build-builder/Dockerfile index 3f3c28d2..88cf6b55 100644 --- a/examples/app_cicd/cloud-build-builder/Dockerfile +++ b/examples/app_cicd/cloud-build-builder/Dockerfile @@ -46,7 +46,7 @@ if [[ ! -e /usr/bin/python ]]; then ln -sf /usr/bin/python3 /usr/bin/python; fi rm -r /root/.cache #### 3. Install gcloud -ENV CLOUD_SDK_VERSION="368.0.0" +ENV CLOUD_SDK_VERSION="391.0.0" ENV CLOUDSDK_INSTALL_DIR /usr/local/gcloud/ RUN wget "https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-${CLOUD_SDK_VERSION}-linux-x86_64.tar.gz" \ && tar -C /usr/local -xzf "google-cloud-sdk-${CLOUD_SDK_VERSION}-linux-x86_64.tar.gz" \ diff --git a/examples/app_cicd/main.tf b/examples/app_cicd/main.tf index 050f2b69..7b32b886 100644 --- a/examples/app_cicd/main.tf +++ b/examples/app_cicd/main.tf @@ -14,19 +14,24 @@ * limitations under the License. */ +locals { + clouddeploy_pipeline_name = "pipeline-01" +} + module "ci_pipeline" { - source = "../../modules/secure-ci" - project_id = var.project_id - app_source_repo = "app-source" - manifest_dry_repo = "app-dry-manifests" - manifest_wet_repo = "app-wet-manifests" - gar_repo_name_suffix = "app-image-repo" - primary_location = "us-central1" - attestor_names_prefix = ["build", "security", "quality"] - app_build_trigger_yaml = "cloudbuild-ci.yaml" - runner_build_folder = "${path.module}/cloud-build-builder" - build_image_config_yaml = "cloudbuild-skaffold-build-image.yaml" - trigger_branch_name = ".*" + source = "../../modules/secure-ci" + project_id = var.project_id + app_source_repo = "app-source" + cloudbuild_cd_repo = "cloudbuild-cd-config" + gar_repo_name_suffix = "app-image-repo" + primary_location = "us-central1" + attestor_names_prefix = ["build", "security", "quality"] + cache_bucket_name = "app-cloudbuild" + app_build_trigger_yaml = "cloudbuild-ci.yaml" + runner_build_folder = "${path.module}/cloud-build-builder" + build_image_config_yaml = "cloudbuild-skaffold-build-image.yaml" + trigger_branch_name = ".*" + clouddeploy_pipeline_name = local.clouddeploy_pipeline_name } module "cd_pipeline" { @@ -34,11 +39,12 @@ module "cd_pipeline" { project_id = var.project_id primary_location = "us-central1" - gar_repo_name = module.ci_pipeline.app_artifact_repo - manifest_wet_repo = "app-wet-manifests" - deploy_branch_clusters = var.deploy_branch_clusters - app_deploy_trigger_yaml = "cloudbuild-cd.yaml" - cache_bucket_name = module.ci_pipeline.cache_bucket_name + gar_repo_name = module.ci_pipeline.app_artifact_repo + cloudbuild_cd_repo = "cloudbuild-cd-config" + deploy_branch_clusters = var.deploy_branch_clusters + app_deploy_trigger_yaml = "cloudbuild-cd.yaml" + cache_bucket_name = module.ci_pipeline.cache_bucket_name + clouddeploy_pipeline_name = local.clouddeploy_pipeline_name depends_on = [ module.ci_pipeline ] diff --git a/examples/app_cicd/outputs.tf b/examples/app_cicd/outputs.tf index 62b98ae6..dd68e5d8 100644 --- a/examples/app_cicd/outputs.tf +++ b/examples/app_cicd/outputs.tf @@ -19,14 +19,14 @@ output "project_id" { description = "The project to run tests against" } -output "bin_auth_attestor_names" { +output "binauth_attestor_names" { description = "Names of Attestors" - value = module.ci_pipeline.bin_auth_attestor_names + value = module.ci_pipeline.binauth_attestor_names } -output "bin_auth_attestor_project_id" { +output "binauth_attestor_project_id" { description = "Project ID where attestors get created" - value = module.ci_pipeline.bin_auth_attestor_project_id + value = module.ci_pipeline.binauth_attestor_project_id } output "boa_artifact_repo" { diff --git a/examples/private_cluster_cicd/README.md b/examples/private_cluster_cicd/README.md index 2996fde7..529b448f 100644 --- a/examples/private_cluster_cicd/README.md +++ b/examples/private_cluster_cicd/README.md @@ -54,8 +54,8 @@ cp -R terraform-google-secure-cicd/examples/private_cluster_cicd/policies bank-o | Name | Description | |------|-------------| -| bin\_auth\_attestor\_names | Names of Attestors | -| bin\_auth\_attestor\_project\_id | Project ID where attestors get created | +| binauth\_attestor\_names | Names of Attestors | +| binauth\_attestor\_project\_id | Project ID where attestors get created | | boa\_artifact\_repo | GAR Repo created to store BoA images | | build\_trigger\_name | The name of the cloud build trigger for the bank of anthos repo. | | cache\_bucket\_name | The name of the storage bucket for cloud build. | diff --git a/examples/private_cluster_cicd/cloud-build-builder/Dockerfile b/examples/private_cluster_cicd/cloud-build-builder/Dockerfile index 3f3c28d2..88cf6b55 100644 --- a/examples/private_cluster_cicd/cloud-build-builder/Dockerfile +++ b/examples/private_cluster_cicd/cloud-build-builder/Dockerfile @@ -46,7 +46,7 @@ if [[ ! -e /usr/bin/python ]]; then ln -sf /usr/bin/python3 /usr/bin/python; fi rm -r /root/.cache #### 3. Install gcloud -ENV CLOUD_SDK_VERSION="368.0.0" +ENV CLOUD_SDK_VERSION="391.0.0" ENV CLOUDSDK_INSTALL_DIR /usr/local/gcloud/ RUN wget "https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-${CLOUD_SDK_VERSION}-linux-x86_64.tar.gz" \ && tar -C /usr/local -xzf "google-cloud-sdk-${CLOUD_SDK_VERSION}-linux-x86_64.tar.gz" \ diff --git a/examples/private_cluster_cicd/main.tf b/examples/private_cluster_cicd/main.tf index 53babe94..189845e4 100644 --- a/examples/private_cluster_cicd/main.tf +++ b/examples/private_cluster_cicd/main.tf @@ -14,22 +14,26 @@ * limitations under the License. */ +locals { + clouddeploy_pipeline_name = "pipeline-private" +} + # Secure-CI module "ci_pipeline" { - source = "../../modules/secure-ci" - project_id = var.project_id - app_source_repo = "app-source-pc" - manifest_dry_repo = "app-dry-manifests-pc" - manifest_wet_repo = "app-wet-manifests-pc" - gar_repo_name_suffix = "app-image-repo-pc" - cache_bucket_name = "private_cluster_cloudbuild" - primary_location = "us-central1" - attestor_names_prefix = ["build-pc", "security-pc", "quality-pc"] - app_build_trigger_yaml = "cloudbuild-ci.yaml" - runner_build_folder = "${path.module}/cloud-build-builder" - build_image_config_yaml = "cloudbuild-skaffold-build-image.yaml" - trigger_branch_name = ".*" - cloudbuild_private_pool = module.cloudbuild_private_pool.workerpool_id + source = "../../modules/secure-ci" + project_id = var.project_id + app_source_repo = "app-source-pc" + cloudbuild_cd_repo = "cloudbuild-cd-config-pc" + gar_repo_name_suffix = "app-image-repo-pc" + cache_bucket_name = "private-cluster-cloudbuild" + primary_location = "us-central1" + attestor_names_prefix = ["build-pc", "security-pc", "quality-pc"] + app_build_trigger_yaml = "cloudbuild-ci.yaml" + runner_build_folder = "${path.module}/cloud-build-builder" + build_image_config_yaml = "cloudbuild-skaffold-build-image.yaml" + trigger_branch_name = ".*" + cloudbuild_private_pool = module.cloudbuild_private_pool.workerpool_id + clouddeploy_pipeline_name = local.clouddeploy_pipeline_name } # Secure-CD @@ -38,12 +42,13 @@ module "cd_pipeline" { project_id = var.project_id primary_location = "us-central1" - gar_repo_name = module.ci_pipeline.app_artifact_repo - manifest_wet_repo = "app-wet-manifests-pc" - deploy_branch_clusters = var.deploy_branch_clusters - app_deploy_trigger_yaml = "cloudbuild-cd.yaml" - cache_bucket_name = module.ci_pipeline.cache_bucket_name - cloudbuild_private_pool = module.cloudbuild_private_pool.workerpool_id + gar_repo_name = module.ci_pipeline.app_artifact_repo + cloudbuild_cd_repo = "cloudbuild-cd-config-pc" + deploy_branch_clusters = var.deploy_branch_clusters + app_deploy_trigger_yaml = "cloudbuild-cd.yaml" + cache_bucket_name = module.ci_pipeline.cache_bucket_name + cloudbuild_private_pool = module.cloudbuild_private_pool.workerpool_id + clouddeploy_pipeline_name = local.clouddeploy_pipeline_name depends_on = [ module.ci_pipeline ] diff --git a/examples/private_cluster_cicd/outputs.tf b/examples/private_cluster_cicd/outputs.tf index 62b98ae6..dd68e5d8 100644 --- a/examples/private_cluster_cicd/outputs.tf +++ b/examples/private_cluster_cicd/outputs.tf @@ -19,14 +19,14 @@ output "project_id" { description = "The project to run tests against" } -output "bin_auth_attestor_names" { +output "binauth_attestor_names" { description = "Names of Attestors" - value = module.ci_pipeline.bin_auth_attestor_names + value = module.ci_pipeline.binauth_attestor_names } -output "bin_auth_attestor_project_id" { +output "binauth_attestor_project_id" { description = "Project ID where attestors get created" - value = module.ci_pipeline.bin_auth_attestor_project_id + value = module.ci_pipeline.binauth_attestor_project_id } output "boa_artifact_repo" { diff --git a/modules/secure-cd/README.md b/modules/secure-cd/README.md index 8e8e758b..389754f8 100644 --- a/modules/secure-cd/README.md +++ b/modules/secure-cd/README.md @@ -4,7 +4,9 @@ This module creates a number of Google Cloud Build triggers to facilitate deploy To securely deploy container images, this pipeline focuses on implementing the "Securing deployed artifacts" and "Securing artifact promotions" sections of the [Shifting left on security report](https://cloud.google.com/solutions/shifting-left-on-security). This module implements security best practices by automating the deployment and promotion of updated container images across multiple GKE clusters, and "failing fast" upon security violation. Users configure a GKE cluster's required attesttions and at which stages a container will receive that attestation based on succesful deployment. The [`cloudbuild-cd.yaml`](../../build/cloudbuild-cd.yaml) contains user-customizable post-deployment security checks to prevent promotion upon discovery of a vulnerability. This module creates: -* [Cloud Build Triggers](https://cloud.google.com/build/docs/automating-builds/create-manage-triggers), one for each specified deployment environment (usually one per cluster) +* A Cloud Deploy [Pipeline and Targets](https://cloud.google.com/deploy/docs/create-pipeline-targets) for each deployment environment. +* A `clouddeploy-operations` [Pub/Sub topic](https://cloud.google.com/deploy/docs/integrating) to integrate with post-deployment processes. +* [Cloud Build Triggers](https://cloud.google.com/build/docs/automating-builds/create-manage-triggers), to execute post-deployment checks, triggered by messages to the `clouddeploy-operations` topic. * A [Binary Authorization Policy](https://cloud.google.com/binary-authorization/docs) in each project with a GKE cluster, specifying which attestations are required to run containers in each cluster ## Usage @@ -16,7 +18,7 @@ module "cd_pipeline" { project_id = var.project_id primary_location = "us-central1" gar_repo_name = - manifest_wet_repo = "app-wet-manifests" + cloudbuild_cd_repo = "cloudbuild-cd-config-pc" deploy_branch_clusters = { dev = { cluster = "dev-cluster", @@ -48,7 +50,7 @@ module "cd_pipeline" { } ``` ### Build Configuration -The template [`cloudbuild-cd.yaml`](../../build/cloudbuild-cd.yaml) build configuration deploys updated containers to specified GKE clusters upon updates to hydrated manifests in the `manifest_wet_repo`. Add the configuration file to the root of the `master` branch of the `manifest_wet_repo` to properly trigger the CD phase. +The template [`cloudbuild-cd.yaml`](../../build/cloudbuild-cd.yaml) build configuration specifies the post-deployment checks to run on a Cloud Deploy target upon successful deployment, triggered by Pub/Sub messages from Cloud Deploy. By default, this runs an OWASP ZAProxy scan of any exposed services. Then, the process will automatically promote the given Release to the next environment in Cloud Deploy. Push the configuration file to the root of the `main` branch of the `cloudbuild_cd_repo` to properly configure the automation. ## Inputs @@ -58,10 +60,11 @@ The template [`cloudbuild-cd.yaml`](../../build/cloudbuild-cd.yaml) build config | additional\_substitutions | Parameters to be substituted in the build specification. All keys should begin with an underscore. | `map(string)` | `{}` | no | | app\_deploy\_trigger\_yaml | Name of application cloudbuild yaml file for deployment | `string` | n/a | yes | | cache\_bucket\_name | cloud build artifact bucket name | `string` | n/a | yes | +| cloudbuild\_cd\_repo | Name of repo that stores the Cloud Build CD phase configs - for post-deployment checks | `string` | n/a | yes | | cloudbuild\_private\_pool | Cloud Build private pool self-link | `string` | `""` | no | +| clouddeploy\_pipeline\_name | Cloud Deploy pipeline name | `string` | n/a | yes | | deploy\_branch\_clusters | mapping of branch names to cluster deployments | 
map(object({
    cluster               = string
    project_id            = string
    location              = string
    required_attestations = list(string)
    env_attestation       = string
    next_env              = string
  }))
| `{}` | no | | gar\_repo\_name | Docker artifact registry repo to store app build images | `string` | n/a | yes | -| manifest\_wet\_repo | Name of repo that contains hydrated K8s manifests files | `string` | n/a | yes | | primary\_location | Region used for key-ring | `string` | n/a | yes | | project\_id | Project ID for CICD Pipeline Project | `string` | n/a | yes | @@ -70,6 +73,8 @@ The template [`cloudbuild-cd.yaml`](../../build/cloudbuild-cd.yaml) build config | Name | Description | |------|-------------| | binauthz\_policy\_required\_attestations | Binary Authorization policy required attestation in GKE projects | +| clouddeploy\_delivery\_pipeline\_id | ID of the Cloud Deploy delivery pipeline | +| clouddeploy\_target\_id | ID(s) of Cloud Deploy targets | | deploy\_trigger\_names | Names of CD Cloud Build triggers | diff --git a/modules/secure-cd/build.tf b/modules/secure-cd/build.tf new file mode 100644 index 00000000..6e299cf2 --- /dev/null +++ b/modules/secure-cd/build.tf @@ -0,0 +1,71 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# Set up Cloud Deploy notifications +# (https://cloud.google.com/deploy/docs/subscribe-deploy-notifications) +resource "google_pubsub_topic" "clouddeploy_topic" { + name = local.clouddeploy_pubsub_topic_name + project = var.project_id +} + +# Trigger post-deploy checks on successful Cloud Deploy rollout +resource "google_cloudbuild_trigger" "deploy_trigger" { + for_each = { + for env, config in var.deploy_branch_clusters : env => config + if config.next_env != "" + } + + project = var.project_id + name = "deploy-trigger-${each.value.cluster}" + + pubsub_config { + topic = google_pubsub_topic.clouddeploy_topic.id + } + + source_to_build { + uri = "https://source.developers.google.com/p/${var.project_id}/r/${var.cloudbuild_cd_repo}" + ref = "main" + repo_type = "CLOUD_SOURCE_REPOSITORIES" + } + + filename = "cloudbuild-cd.yaml" + + substitutions = merge( + { + _GAR_REPOSITORY = var.gar_repo_name + _DEFAULT_REGION = each.value.location + _CLUSTER_NAME = each.value.cluster + _CLUSTER_PROJECT = each.value.project_id + _CLOUDBUILD_FILENAME = var.app_deploy_trigger_yaml + _CACHE_BUCKET_NAME = var.cache_bucket_name + _NEXT_ENV = each.value.next_env + _ATTESTOR_NAME = each.value.env_attestation + _CLOUDBUILD_PRIVATE_POOL = var.cloudbuild_private_pool + _CLOUDDEPLOY_PIPELINE_NAME = var.clouddeploy_pipeline_name + # Create substitutions to parse incoming Pub/sub messages from Cloud Deploy + _ACTION_TYPE = "$(body.message.attributes.Action)" + _RESOURCE_TYPE = "$(body.message.attributes.ResourceType)" + _DELIVERY_PIPELINE_ID = "$(body.message.attributes.DeliveryPipelineId)" + _TARGET_ID = "$(body.message.attributes.TargetId)" + _RELEASE_ID = "$(body.message.attributes.ReleaseId)" + + }, + var.additional_substitutions + ) + + # Only trigger the post-deployment check on relevant Cloud Deploy activity (successful rollout to a target) + filter = "_RESOURCE_TYPE.matches('Rollout') && _ACTION_TYPE.matches('Succeed') && _DELIVERY_PIPELINE_ID.matches('${var.clouddeploy_pipeline_name}') && _TARGET_ID.matches('${google_clouddeploy_target.deploy_target[each.key].name}')" +} diff --git a/modules/secure-cd/iam.tf b/modules/secure-cd/iam.tf new file mode 100644 index 00000000..e4e3708a --- /dev/null +++ b/modules/secure-cd/iam.tf @@ -0,0 +1,104 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + attestor_iam_config = flatten([ + for env_key, env in var.deploy_branch_clusters : [ + for attestor in env.required_attestations : { + env = env_key + attestor = split("/", attestor)[3] + } + ] + ]) + + cd_sa_required_roles = [ + "roles/clouddeploy.jobRunner", + ] +} + +data "google_project" "app_cicd_project" { + project_id = var.project_id +} + +# Cloud Deploy Execution Service Account +# https://cloud.google.com/deploy/docs/cloud-deploy-service-account#execution_service_account +resource "google_service_account" "clouddeploy_execution_sa" { + project = var.project_id + account_id = "clouddeploy-execution-sa" + display_name = "clouddeploy-execution-sa" +} + +resource "google_project_iam_member" "cd_sa_iam" { + for_each = toset(local.cd_sa_required_roles) + + project = var.project_id + role = each.value + member = "serviceAccount:${google_service_account.clouddeploy_execution_sa.email}" +} + +# Cloud Deploy Service Agent +resource "google_project_service_identity" "clouddeploy_service_agent" { + provider = google-beta + + project = var.project_id + service = "clouddeploy.googleapis.com" +} + +resource "google_project_iam_member" "clouddeploy_service_agent_role" { + project = var.project_id + role = "roles/clouddeploy.serviceAgent" + member = "serviceAccount:${google_project_service_identity.clouddeploy_service_agent.email}" +} + +# IAM membership for Cloud Build SA to act as Cloud Deploy Execution SA +resource "google_service_account_iam_member" "cloudbuild_clouddeploy_impersonation" { + service_account_id = google_service_account.clouddeploy_execution_sa.name + role = "roles/iam.serviceAccountUser" + member = "serviceAccount:${data.google_project.app_cicd_project.number}@cloudbuild.gserviceaccount.com" +} + +# IAM membership for Cloud Deploy Execution SA deploy to GKE +resource "google_project_iam_member" "clouddeploy_gke_dev" { + for_each = var.deploy_branch_clusters + project = each.value.project_id + role = "roles/container.developer" + member = "serviceAccount:${google_service_account.clouddeploy_execution_sa.email}" +} + +# IAM membership for Cloud Build SA to deploy to GKE +resource "google_project_iam_member" "gke_dev" { + for_each = var.deploy_branch_clusters + project = each.value.project_id + role = "roles/container.developer" + member = "serviceAccount:${data.google_project.app_cicd_project.number}@cloudbuild.gserviceaccount.com" +} + +# IAM membership for Binary Authorization service agents in GKE projects on attestors +resource "google_project_service_identity" "binauth_service_agent" { + provider = google-beta + for_each = var.deploy_branch_clusters + + project = each.value.project_id + service = "binaryauthorization.googleapis.com" +} + +resource "google_binary_authorization_attestor_iam_member" "binauthz_verifier" { + for_each = { for entry in local.attestor_iam_config : "${entry.env}.${entry.attestor}" => entry } # turn into a map + project = var.project_id + attestor = each.value.attestor + role = "roles/binaryauthorization.attestorsVerifier" + member = "serviceAccount:${google_project_service_identity.binauth_service_agent[each.value.env].email}" +} diff --git a/modules/secure-cd/main.tf b/modules/secure-cd/main.tf index 6ca31242..d8a8879b 100644 --- a/modules/secure-cd/main.tf +++ b/modules/secure-cd/main.tf @@ -15,56 +15,57 @@ */ locals { - attestor_iam_config = flatten([ - for env_key, env in var.deploy_branch_clusters : [ - for attestor in env.required_attestations : { - env = env_key - attestor = split("/", attestor)[3] - } - ] - ]) - deploy_projects = distinct([ for env in var.deploy_branch_clusters : env.project_id ]) + binary_authorization_map = zipmap( local.deploy_projects, [for project_id in local.deploy_projects : [ for env in var.deploy_branch_clusters : env if env.project_id == project_id ]] ) -} -data "google_project" "app_cicd_project" { - project_id = var.project_id + clouddeploy_pubsub_topic_name = "clouddeploy-operations" } -resource "google_cloudbuild_trigger" "deploy_trigger" { +resource "google_clouddeploy_target" "deploy_target" { for_each = var.deploy_branch_clusters - project = var.project_id - name = "deploy-trigger-${each.key}-${each.value.cluster}" - trigger_template { - branch_name = each.key - repo_name = var.manifest_wet_repo + name = "${each.value.cluster}-target" + description = "Target for ${each.key} environment" + location = each.value.location + project = var.project_id + + gke { + cluster = "projects/${each.value.project_id}/locations/${each.value.location}/clusters/${each.value.cluster}" } - substitutions = merge( - { - _GAR_REPOSITORY = var.gar_repo_name - _DEFAULT_REGION = each.value.location - _MANIFEST_WET_REPO = var.manifest_wet_repo - _CLUSTER_NAME = each.value.cluster - _CLUSTER_PROJECT = each.value.project_id - _CLOUDBUILD_FILENAME = var.app_deploy_trigger_yaml - _CACHE_BUCKET_NAME = var.cache_bucket_name - _NEXT_ENV = each.value.next_env - _ATTESTOR_NAME = each.value.env_attestation - _CLOUDBUILD_PRIVATE_POOL = var.cloudbuild_private_pool - }, - var.additional_substitutions - ) - filename = var.app_deploy_trigger_yaml + execution_configs { + usages = ["RENDER", "DEPLOY"] + worker_pool = var.cloudbuild_private_pool + service_account = google_service_account.clouddeploy_execution_sa.email + } + + depends_on = [ + google_project_iam_member.clouddeploy_service_agent_role + ] +} + +resource "google_clouddeploy_delivery_pipeline" "pipeline" { + name = var.clouddeploy_pipeline_name + description = "Pipeline for application" #TODO parameterize + project = var.project_id + location = var.primary_location + + serial_pipeline { + dynamic "stages" { + for_each = var.deploy_branch_clusters + content { + target_id = google_clouddeploy_target.deploy_target[stages.key].name + } + } + } } # Binary Authorization Policy @@ -90,23 +91,4 @@ resource "google_binary_authorization_policy" "deployment_policy" { } } -# IAM membership for Cloud Build SA to allow deployment to GKE -resource "google_project_iam_member" "gke_dev" { - for_each = var.deploy_branch_clusters - project = each.value.project_id - role = "roles/container.developer" - member = "serviceAccount:${data.google_project.app_cicd_project.number}@cloudbuild.gserviceaccount.com" -} -# IAM membership for Binary Authorization service agents in GKE projects on attestors -data "google_project" "gke_projects" { - for_each = var.deploy_branch_clusters - project_id = each.value.project_id -} -resource "google_binary_authorization_attestor_iam_member" "binauthz_verifier" { - for_each = { for entry in local.attestor_iam_config : "${entry.env}.${entry.attestor}" => entry } # turn into a map - project = var.project_id - attestor = each.value.attestor - role = "roles/binaryauthorization.attestorsVerifier" - member = "serviceAccount:service-${data.google_project.gke_projects[each.value.env].number}@gcp-sa-binaryauthorization.iam.gserviceaccount.com" -} diff --git a/modules/secure-cd/outputs.tf b/modules/secure-cd/outputs.tf index e9d60b15..d7af95ff 100644 --- a/modules/secure-cd/outputs.tf +++ b/modules/secure-cd/outputs.tf @@ -23,3 +23,13 @@ output "binauthz_policy_required_attestations" { description = "Binary Authorization policy required attestation in GKE projects" value = [for policy in google_binary_authorization_policy.deployment_policy : policy.cluster_admission_rules.*.require_attestations_by] } + +output "clouddeploy_delivery_pipeline_id" { + description = "ID of the Cloud Deploy delivery pipeline" + value = google_clouddeploy_delivery_pipeline.pipeline.id +} + +output "clouddeploy_target_id" { + description = "ID(s) of Cloud Deploy targets" + value = [for target in google_clouddeploy_target.deploy_target : target.id] +} diff --git a/modules/secure-cd/variables.tf b/modules/secure-cd/variables.tf index 3a197221..e4e8aeea 100644 --- a/modules/secure-cd/variables.tf +++ b/modules/secure-cd/variables.tf @@ -24,9 +24,9 @@ variable "primary_location" { description = "Region used for key-ring" } -variable "manifest_wet_repo" { +variable "cloudbuild_cd_repo" { type = string - description = "Name of repo that contains hydrated K8s manifests files" + description = "Name of repo that stores the Cloud Build CD phase configs - for post-deployment checks" } variable "gar_repo_name" { @@ -68,3 +68,8 @@ variable "cloudbuild_private_pool" { type = string default = "" } + +variable "clouddeploy_pipeline_name" { + description = "Cloud Deploy pipeline name" + type = string +} diff --git a/modules/secure-cd/versions.tf b/modules/secure-cd/versions.tf index d4bf2a86..cff0be38 100644 --- a/modules/secure-cd/versions.tf +++ b/modules/secure-cd/versions.tf @@ -19,11 +19,11 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = ">= 3.45, < 5.0" + version = ">= 4.22, < 5.0" } google-beta = { source = "hashicorp/google-beta" - version = ">= 3.45, < 5.0" + version = ">= 4.22, < 5.0" } } diff --git a/modules/secure-ci/README.md b/modules/secure-ci/README.md index 7e300bb2..6f611d70 100644 --- a/modules/secure-ci/README.md +++ b/modules/secure-ci/README.md @@ -41,25 +41,25 @@ The template [`cloudbuild-ci.yaml`](../../build/cloudbuild-ci.yaml) build config | attestor\_names\_prefix | A list of Binary Authorization attestors to create. The first attestor specified in this list will be used as the build-attestor during the CI phase. | `list(string)` | n/a | yes | | build\_image\_config\_yaml | Name of image builder yaml file | `string` | n/a | yes | | cache\_bucket\_name | Name of cloudbuild artifact and cache GCS bucket | `string` | `""` | no | +| cloudbuild\_cd\_repo | Name of repo that stores the Cloud Build CD phase configs - for post-deployment checks | `string` | `"cloudbuild-cd-config"` | no | | cloudbuild\_private\_pool | Cloud Build private pool self-link | `string` | `""` | no | -| cloudbuild\_service\_account\_roles | IAM roles given to the Cloud Build service account to enable security scanning operations | `list(string)` | 
[
  "roles/artifactregistry.admin",
  "roles/binaryauthorization.attestorsVerifier",
  "roles/cloudbuild.builds.builder",
  "roles/cloudkms.cryptoOperator",
  "roles/containeranalysis.notes.attacher",
  "roles/containeranalysis.notes.occurrences.viewer",
  "roles/source.writer",
  "roles/storage.admin",
  "roles/cloudbuild.workerPoolUser",
  "roles/ondemandscanning.admin"
]
| no | +| cloudbuild\_service\_account\_roles | IAM roles given to the Cloud Build service account to enable security scanning operations | `list(string)` | 
[
  "roles/artifactregistry.admin",
  "roles/binaryauthorization.attestorsVerifier",
  "roles/cloudbuild.builds.builder",
  "roles/clouddeploy.developer",
  "roles/clouddeploy.releaser",
  "roles/cloudkms.cryptoOperator",
  "roles/containeranalysis.notes.attacher",
  "roles/containeranalysis.notes.occurrences.viewer",
  "roles/source.writer",
  "roles/storage.admin",
  "roles/cloudbuild.workerPoolUser",
  "roles/ondemandscanning.admin"
]
| no | +| clouddeploy\_pipeline\_name | Cloud Deploy pipeline name | `string` | `"deploy-pipeline"` | no | | gar\_repo\_name\_suffix | Docker artifact regitery repo to store app build images | `string` | `"app-image-repo"` | no | -| manifest\_dry\_repo | Name of repo that contains template K8s manifests files | `string` | `"app-dry-manifests"` | no | -| manifest\_wet\_repo | Name of repo that will receive hydrated K8s manifests files | `string` | `"app-wet-manifests"` | no | | primary\_location | Region used for key-ring | `string` | n/a | yes | | project\_id | Project ID for CICD Pipeline Project | `string` | n/a | yes | | runner\_build\_folder | Path to the source folder for the cloud builds submit command | `string` | n/a | yes | | trigger\_branch\_name | A regular expression to match one or more branches for the build trigger. | `string` | n/a | yes | | use\_tf\_google\_credentials\_env\_var | Optional GOOGLE\_CREDENTIALS environment variable to be activated. | `bool` | `false` | no | -| wet\_branch\_name | Name of branch in the wet manifest repo that CI pipeline will push to (usually, the name of the first deployed environment) | `string` | `"dev"` | no | ## Outputs | Name | Description | |------|-------------| | app\_artifact\_repo | GAR Repo created to store runner images | -| bin\_auth\_attestor\_names | Names of Attestors | -| bin\_auth\_attestor\_project\_id | Project ID where attestors get created | +| binauth\_attestor\_ids | IDs of Attestors | +| binauth\_attestor\_names | Names of Attestors | +| binauth\_attestor\_project\_id | Project ID where attestors get created | | build\_trigger\_name | The name of the cloud build trigger for the app source repo. | | cache\_bucket\_name | The name of the storage bucket for cloud build. | | source\_repo\_names | Name of the created CSR repos | diff --git a/modules/secure-ci/binauthz.tf b/modules/secure-ci/binauthz.tf index 819b45b4..bbc232a3 100644 --- a/modules/secure-ci/binauthz.tf +++ b/modules/secure-ci/binauthz.tf @@ -17,7 +17,7 @@ resource "random_string" "keyring_name" { length = 4 special = false - number = true + numeric = true upper = false lower = true } @@ -32,9 +32,8 @@ resource "google_kms_key_ring" "keyring" { } module "attestors" { - # temporarily refer to PR for provider version compatibility - source = "github.com/terraform-google-modules/terraform-google-kubernetes-engine//modules/binary-authorization?ref=binauthz-services" - #version = "~> 17.3" + source = "terraform-google-modules/kubernetes-engine/google//modules/binary-authorization" + version = "~> 22.1.0" for_each = toset(var.attestor_names_prefix) project_id = var.project_id diff --git a/modules/secure-ci/main.tf b/modules/secure-ci/main.tf index 10743e21..b77afb56 100644 --- a/modules/secure-ci/main.tf +++ b/modules/secure-ci/main.tf @@ -16,11 +16,11 @@ locals { gar_name = split("/", google_artifact_registry_repository.image_repo.name)[length(split("/", google_artifact_registry_repository.image_repo.name)) - 1] - cache_bucket_name = var.cache_bucket_name == "" ? "${var.project_id}_cloudbuild" : "${var.project_id}_${var.cache_bucket_name}" + cache_bucket_name = var.cache_bucket_name == "" ? "${var.project_id}-cloudbuild" : "${var.project_id}-${var.cache_bucket_name}" } resource "google_sourcerepo_repository" "repos" { - for_each = toset([var.manifest_wet_repo, var.manifest_dry_repo, var.app_source_repo]) + for_each = toset([var.cloudbuild_cd_repo, var.app_source_repo]) name = each.key project = var.project_id } @@ -52,14 +52,12 @@ resource "google_cloudbuild_trigger" "app_build_trigger" { } substitutions = merge( { - _GAR_REPOSITORY = local.gar_name - _DEFAULT_REGION = var.primary_location - _CACHE_BUCKET_NAME = google_storage_bucket.cache_bucket.name - _MANIFEST_DRY_REPO = var.manifest_dry_repo - _MANIFEST_WET_REPO = var.manifest_wet_repo - _WET_BRANCH_NAME = var.wet_branch_name - _ATTESTOR_NAME = module.attestors[var.attestor_names_prefix[0]].attestor - _CLOUDBUILD_PRIVATE_POOL = var.cloudbuild_private_pool + _GAR_REPOSITORY = local.gar_name + _DEFAULT_REGION = var.primary_location + _CACHE_BUCKET_NAME = google_storage_bucket.cache_bucket.name + _ATTESTOR_NAME = module.attestors[var.attestor_names_prefix[0]].attestor + _CLOUDBUILD_PRIVATE_POOL = var.cloudbuild_private_pool + _CLOUDDEPLOY_PIPELINE_NAME = var.clouddeploy_pipeline_name }, var.additional_substitutions ) diff --git a/modules/secure-ci/outputs.tf b/modules/secure-ci/outputs.tf index d85493cd..8acfa5d7 100644 --- a/modules/secure-ci/outputs.tf +++ b/modules/secure-ci/outputs.tf @@ -24,12 +24,17 @@ output "build_trigger_name" { value = google_cloudbuild_trigger.app_build_trigger.name } -output "bin_auth_attestor_names" { +output "binauth_attestor_names" { description = "Names of Attestors" value = [for attestor_name in var.attestor_names_prefix : module.attestors[attestor_name].attestor] } -output "bin_auth_attestor_project_id" { +output "binauth_attestor_ids" { + description = "IDs of Attestors" + value = [for attestor_name in var.attestor_names_prefix : "projects/${var.project_id}/attestors/${module.attestors[attestor_name].attestor}"] +} + +output "binauth_attestor_project_id" { description = "Project ID where attestors get created" value = var.project_id } diff --git a/modules/secure-ci/variables.tf b/modules/secure-ci/variables.tf index ef7d3f83..a795035a 100644 --- a/modules/secure-ci/variables.tf +++ b/modules/secure-ci/variables.tf @@ -50,22 +50,10 @@ variable "app_source_repo" { default = "app-source" } -variable "manifest_dry_repo" { +variable "cloudbuild_cd_repo" { type = string - description = "Name of repo that contains template K8s manifests files" - default = "app-dry-manifests" -} - -variable "manifest_wet_repo" { - type = string - description = "Name of repo that will receive hydrated K8s manifests files" - default = "app-wet-manifests" -} - -variable "wet_branch_name" { - type = string - description = "Name of branch in the wet manifest repo that CI pipeline will push to (usually, the name of the first deployed environment)" - default = "dev" + description = "Name of repo that stores the Cloud Build CD phase configs - for post-deployment checks" + default = "cloudbuild-cd-config" } variable "cache_bucket_name" { @@ -98,6 +86,8 @@ variable "cloudbuild_service_account_roles" { "roles/artifactregistry.admin", "roles/binaryauthorization.attestorsVerifier", "roles/cloudbuild.builds.builder", + "roles/clouddeploy.developer", + "roles/clouddeploy.releaser", "roles/cloudkms.cryptoOperator", "roles/containeranalysis.notes.attacher", "roles/containeranalysis.notes.occurrences.viewer", @@ -119,3 +109,9 @@ variable "cloudbuild_private_pool" { type = string default = "" } + +variable "clouddeploy_pipeline_name" { + description = "Cloud Deploy pipeline name" + type = string + default = "deploy-pipeline" +} diff --git a/modules/workerpool-gke-ha-vpn/main.tf b/modules/workerpool-gke-ha-vpn/main.tf index a786d2cc..c5f36876 100644 --- a/modules/workerpool-gke-ha-vpn/main.tf +++ b/modules/workerpool-gke-ha-vpn/main.tf @@ -17,7 +17,7 @@ # HA VPN module "vpn_ha_1" { source = "terraform-google-modules/vpn/google//modules/vpn_ha" - version = "~> 2.1.0" + version = "~> 2.3.0" project_id = var.project_id region = var.location network = var.workerpool_network @@ -61,7 +61,7 @@ module "vpn_ha_1" { module "vpn_ha_2" { source = "terraform-google-modules/vpn/google//modules/vpn_ha" - version = "~> 1.3.0" + version = "~> 2.3.0" project_id = var.gke_project region = var.gke_location network = var.gke_network diff --git a/test/fixtures/app_cicd/main.tf b/test/fixtures/app_cicd/main.tf index 27b194e5..c7c08c1d 100644 --- a/test/fixtures/app_cicd/main.tf +++ b/test/fixtures/app_cicd/main.tf @@ -20,7 +20,7 @@ module "example" { project_id = var.project_id primary_location = var.primary_location deploy_branch_clusters = { - dev = { + "01-dev" = { cluster = "dev-cluster", project_id = var.gke_project_ids["dev"], location = var.primary_location, @@ -28,7 +28,7 @@ module "example" { env_attestation = "projects/${var.project_id}/attestors/security-attestor" next_env = "qa" }, - qa = { + "02-qa" = { cluster = "qa-cluster", project_id = var.gke_project_ids["qa"], location = var.primary_location, @@ -36,7 +36,7 @@ module "example" { env_attestation = "projects/${var.project_id}/attestors/quality-attestor" next_env = "prod" }, - prod = { + "03-prod" = { cluster = "prod-cluster", project_id = var.gke_project_ids["prod"], location = var.primary_location, @@ -47,60 +47,16 @@ module "example" { } } -# VPCs -module "vpc" { - for_each = var.gke_project_ids - source = "terraform-google-modules/network/google" - version = "~> 4.0" - - project_id = var.gke_project_ids[each.key] - network_name = "gke-vpc-${each.key}" - routing_mode = "REGIONAL" - - subnets = [ - { - subnet_name = "gke-subnet" - subnet_ip = "10.0.0.0/17" - subnet_region = var.primary_location - }, - ] - secondary_ranges = { - gke-subnet = [ - { - range_name = "us-central1-01-gke-01-pods" - ip_cidr_range = "192.168.0.0/18" - }, - { - range_name = "us-central1-01-gke-01-services" - ip_cidr_range = "192.168.64.0/18" - }, - ] - } +resource "google_project_iam_member" "cluster_service_account-gcr" { + for_each = var.gke_service_accounts + project = var.project_id + role = "roles/storage.objectViewer" + member = "serviceAccount:${each.value}" } -module "gke_cluster" { - for_each = var.gke_project_ids - source = "terraform-google-modules/kubernetes-engine/google" - version = "19.0.0" - - project_id = var.gke_project_ids[each.key] - name = "${each.key}-cluster" - regional = true - region = var.primary_location - zones = ["us-central1-a", "us-central1-b", "us-central1-f"] - network = module.vpc[each.key].network_name - subnetwork = module.vpc[each.key].subnets_names[0] - ip_range_pods = "us-central1-01-gke-01-pods" - ip_range_services = "us-central1-01-gke-01-services" - create_service_account = true - enable_binary_authorization = true - skip_provisioners = false - - # Enabled read-access to images in GAR repo in CI/CD project - grant_registry_access = true - registry_project_ids = [var.project_id] - - depends_on = [ - module.vpc - ] +resource "google_project_iam_member" "cluster_service_account-artifact-registry" { + for_each = var.gke_service_accounts + project = var.project_id + role = "roles/artifactregistry.reader" + member = "serviceAccount:${each.value}" } diff --git a/test/fixtures/app_cicd/variables.tf b/test/fixtures/app_cicd/variables.tf index 9688778a..b462f419 100644 --- a/test/fixtures/app_cicd/variables.tf +++ b/test/fixtures/app_cicd/variables.tf @@ -42,3 +42,8 @@ variable "gke_project_ids" { type = map(string) description = "map of env name to GKE project ID" } + +variable "gke_service_accounts" { + type = map(string) + description = "map of env name to GKE service account" +} diff --git a/test/fixtures/cloudbuild_private_pool/main.tf b/test/fixtures/cloudbuild_private_pool/main.tf index 49d7ba2a..a4f768ef 100644 --- a/test/fixtures/cloudbuild_private_pool/main.tf +++ b/test/fixtures/cloudbuild_private_pool/main.tf @@ -16,30 +16,30 @@ locals { deploy_branch_clusters = { - dev = { + "01-dev" = { cluster = "dev-private-cluster", - network = var.gke_vpc_names["dev"] + network = var.gke_private_vpc_names["dev"] project_id = var.gke_project_ids["dev"], location = var.primary_location, - required_attestations = ["projects/${var.project_id}/attestors/build-attestor"] - env_attestation = "projects/${var.project_id}/attestors/security-attestor" - next_env = "qa" + required_attestations = ["projects/${var.project_id}/attestors/build-pc-attestor"] + env_attestation = "projects/${var.project_id}/attestors/security-pc-attestor" + next_env = "02-qa" }, - qa = { + "02-qa" = { cluster = "qa-private-cluster", - network = var.gke_vpc_names["qa"] + network = var.gke_private_vpc_names["qa"] project_id = var.gke_project_ids["qa"], location = var.primary_location, - required_attestations = ["projects/${var.project_id}/attestors/security-attestor", "projects/${var.project_id}/attestors/build-attestor"] - env_attestation = "projects/${var.project_id}/attestors/quality-attestor" - next_env = "prod" + required_attestations = ["projects/${var.project_id}/attestors/security-pc-attestor", "projects/${var.project_id}/attestors/build-pc-attestor"] + env_attestation = "projects/${var.project_id}/attestors/quality-pc-attestor" + next_env = "03-prod" }, - prod = { + "03-prod" = { cluster = "prod-private-cluster", - network = var.gke_vpc_names["prod"] + network = var.gke_private_vpc_names["prod"] project_id = var.gke_project_ids["prod"], location = var.primary_location, - required_attestations = ["projects/${var.project_id}/attestors/quality-attestor", "projects/${var.project_id}/attestors/security-attestor", "projects/${var.project_id}/attestors/build-attestor"] + required_attestations = ["projects/${var.project_id}/attestors/quality-pc-attestor", "projects/${var.project_id}/attestors/security-pc-attestor", "projects/${var.project_id}/attestors/build-pc-attestor"] env_attestation = "" next_env = "" }, diff --git a/test/fixtures/cloudbuild_private_pool/variables.tf b/test/fixtures/cloudbuild_private_pool/variables.tf index eb28b388..78bc11e5 100644 --- a/test/fixtures/cloudbuild_private_pool/variables.tf +++ b/test/fixtures/cloudbuild_private_pool/variables.tf @@ -30,12 +30,12 @@ variable "gke_project_ids" { description = "map of env name to GKE project ID" } -variable "gke_vpc_names" { +variable "gke_private_vpc_names" { type = map(string) description = "map of env name to GKE network name" } -variable "gke_service_accounts" { +variable "gke_private_service_accounts" { type = map(string) description = "map of env name to GKE service account" } diff --git a/test/fixtures/private_cluster_cicd/main.tf b/test/fixtures/private_cluster_cicd/main.tf index 4d7fb4d2..d789dbfd 100644 --- a/test/fixtures/private_cluster_cicd/main.tf +++ b/test/fixtures/private_cluster_cicd/main.tf @@ -16,27 +16,27 @@ locals { deploy_branch_clusters = { - dev = { + "01-dev" = { cluster = "dev-private-cluster", - network = var.gke_vpc_names["dev"] + network = var.gke_private_vpc_names["dev"] project_id = var.gke_project_ids["dev"], location = var.primary_location, required_attestations = ["projects/${var.project_id}/attestors/build-pc-attestor"] env_attestation = "projects/${var.project_id}/attestors/security-pc-attestor" - next_env = "qa" + next_env = "02-qa" }, - qa = { + "02-qa" = { cluster = "qa-private-cluster", - network = var.gke_vpc_names["qa"] + network = var.gke_private_vpc_names["qa"] project_id = var.gke_project_ids["qa"], location = var.primary_location, required_attestations = ["projects/${var.project_id}/attestors/security-pc-attestor", "projects/${var.project_id}/attestors/build-pc-attestor"] env_attestation = "projects/${var.project_id}/attestors/quality-pc-attestor" - next_env = "prod" + next_env = "03-prod" }, - prod = { + "03-prod" = { cluster = "prod-private-cluster", - network = var.gke_vpc_names["prod"] + network = var.gke_private_vpc_names["prod"] project_id = var.gke_project_ids["prod"], location = var.primary_location, required_attestations = ["projects/${var.project_id}/attestors/quality-pc-attestor", "projects/${var.project_id}/attestors/security-pc-attestor", "projects/${var.project_id}/attestors/build-pc-attestor"] @@ -69,14 +69,14 @@ module "example" { } resource "google_project_iam_member" "cluster_service_account-gcr" { - for_each = var.gke_service_accounts + for_each = var.gke_private_service_accounts project = var.project_id role = "roles/storage.objectViewer" member = "serviceAccount:${each.value}" } resource "google_project_iam_member" "cluster_service_account-artifact-registry" { - for_each = var.gke_service_accounts + for_each = var.gke_private_service_accounts project = var.project_id role = "roles/artifactregistry.reader" member = "serviceAccount:${each.value}" diff --git a/test/fixtures/private_cluster_cicd/variables.tf b/test/fixtures/private_cluster_cicd/variables.tf index 0aff0e6b..6b0ecc08 100644 --- a/test/fixtures/private_cluster_cicd/variables.tf +++ b/test/fixtures/private_cluster_cicd/variables.tf @@ -38,7 +38,7 @@ variable "billing_account" { description = "The billing account id associated with the project, e.g. XXXXXX-YYYYYY-ZZZZZZ" } -variable "gke_cluster_names" { +variable "gke_private_cluster_names" { type = map(string) description = "map of env name to GKE cluster name" } @@ -48,12 +48,12 @@ variable "gke_project_ids" { description = "map of env name to GKE project ID" } -variable "gke_vpc_names" { +variable "gke_private_vpc_names" { type = map(string) description = "map of env name to GKE network name" } -variable "gke_service_accounts" { +variable "gke_private_service_accounts" { type = map(string) description = "map of env name to GKE service account" } diff --git a/test/integration/app_cicd/app_cicd_test.go b/test/integration/app_cicd/app_cicd_test.go index cc4498c4..72e641ae 100644 --- a/test/integration/app_cicd/app_cicd_test.go +++ b/test/integration/app_cicd/app_cicd_test.go @@ -36,8 +36,7 @@ func TestAppCICDExample(t *testing.T) { const garRepoNameSuffix = "app-image-repo" const primaryLocation = "us-central1" const appSourceRepoName = "app-source" - const manifestDryRepoName = "app-dry-manifests" - const manifestWetRepoName = "app-wet-manifests" + const cloudbuildCDRepoName = "cloudbuild-cd-config" // initialize Terraform test from the Blueprints test framework appCICDT := tft.NewTFBlueprintTest(t) @@ -59,8 +58,6 @@ func TestAppCICDExample(t *testing.T) { gcbCI := gcloud.Run(t, fmt.Sprintf("beta builds triggers describe %s --project %s", sourceTriggerName, projectID)) assert.Equal(sourceTriggerName, gcbCI.Get("name").String(), "Cloud Build Trigger name is valid") - assert.Equal(manifestDryRepoName, gcbCI.Get("substitutions._MANIFEST_DRY_REPO").String(), "Manifest Dry Repo trigger substitution is valid") - assert.Equal(manifestWetRepoName, gcbCI.Get("substitutions._MANIFEST_WET_REPO").String(), "Manifest Wet Repo trigger substitution is valid") assert.Contains(gcbCI.Get("substitutions._DEFAULT_REGION").String(), primaryLocation, "Default Region trigger substitution is valid") assert.Equal(appSourceRepoName, gcbCI.Get("triggerTemplate.repoName").String(), "Attached CSR repo is valid") @@ -79,7 +76,7 @@ func TestAppCICDExample(t *testing.T) { } // CSR - repos := [3]string{appSourceRepoName, manifestDryRepoName, manifestWetRepoName} + repos := [2]string{appSourceRepoName, cloudbuildCDRepoName} for _, repo := range repos { csr := gcloud.Run(t, fmt.Sprintf("source repos describe %s --project %s", repo, projectID)) @@ -91,14 +88,12 @@ func TestAppCICDExample(t *testing.T) { /////// SECURE-CD /////// // Deploy Triggers - cdTriggers := [3]string{"deploy-trigger-dev-dev-cluster", "deploy-trigger-qa-qa-cluster", "deploy-trigger-prod-prod-cluster"} + cdTriggers := [2]string{"deploy-trigger-dev-cluster", "deploy-trigger-qa-cluster"} for _, cdTrigger := range cdTriggers { gcbCD := gcloud.Run(t, fmt.Sprintf("beta builds triggers describe %s --project %s", cdTrigger, projectID)) assert.Contains(gcbCD.Get("name").String(), cdTrigger, "Trigger name is valid") - assert.Equal(manifestWetRepoName, gcbCD.Get("triggerTemplate.repoName").String(), "repoName triggerTemplate is valid") - assert.Equal(projectID, gcbCD.Get("triggerTemplate.projectId").String(), "Trigger is in correct project") - assert.Equal(manifestWetRepoName, gcbCD.Get("substitutions._MANIFEST_WET_REPO").String(), "_MANIFEST_WET_REPO trigger substitution is valid") + assert.Contains(gcbCD.Get("pubsubConfig.topic").String(), "clouddeploy-operations", "pubsub topic is valid") assert.Contains(gcbCD.Get("substitutions._CLUSTER_PROJECT").String(), "secure-cicd-gke-", "_CLUSTER_PROJECT trigger substitution is valid") } diff --git a/test/integration/private_cluster_cicd/private_cluster_cicd_test.go b/test/integration/private_cluster_cicd/private_cluster_cicd_test.go index a0a48f83..c7fe1fd5 100644 --- a/test/integration/private_cluster_cicd/private_cluster_cicd_test.go +++ b/test/integration/private_cluster_cicd/private_cluster_cicd_test.go @@ -31,6 +31,12 @@ import ( func TestPrivateClusterCICDExample(t *testing.T) { // define constants for all required assertions in the test case + const sourceTriggerName = "app-source-trigger" + const garRepoNameSuffix = "app-image-repo" + const primaryLocation = "us-central1" + const appSourceRepoName = "app-source" + const cloudbuildCDRepoName = "cloudbuild-cd-config" + // initialize Terraform test from the Blueprints test framework privateClusterCICDT := tft.NewTFBlueprintTest(t) diff --git a/test/setup/iam.tf b/test/setup/iam.tf index 35c43a0a..6a285371 100644 --- a/test/setup/iam.tf +++ b/test/setup/iam.tf @@ -16,17 +16,20 @@ locals { int_required_roles = [ - "roles/storage.admin", "roles/artifactregistry.admin", "roles/binaryauthorization.attestorsAdmin", "roles/cloudbuild.builds.builder", "roles/cloudbuild.workerPoolOwner", + "roles/clouddeploy.admin", "roles/cloudkms.admin", "roles/cloudkms.publicKeyViewer", "roles/containeranalysis.notes.editor", "roles/compute.networkAdmin", + "roles/iam.serviceAccountAdmin", + "roles/pubsub.editor", "roles/serviceusage.serviceUsageAdmin", "roles/source.admin", + "roles/storage.admin", "roles/resourcemanager.projectIamAdmin", "roles/viewer" ] diff --git a/test/setup/main.tf b/test/setup/main.tf index 45ead88b..fa194405 100644 --- a/test/setup/main.tf +++ b/test/setup/main.tf @@ -27,6 +27,7 @@ module "project" { activate_apis = [ "cloudresourcemanager.googleapis.com", "cloudbilling.googleapis.com", + "clouddeploy.googleapis.com", "storage-api.googleapis.com", "serviceusage.googleapis.com", "cloudbuild.googleapis.com", @@ -39,7 +40,8 @@ module "project" { "cloudkms.googleapis.com", "binaryauthorization.googleapis.com", "containerscanning.googleapis.com", - "servicenetworking.googleapis.com" + "servicenetworking.googleapis.com", + "pubsub.googleapis.com", ] activate_api_identities = [ { @@ -101,12 +103,70 @@ module "gke_project" { ] } +###### Public Clusters ###### # VPCs module "vpc" { for_each = toset(local.envs) source = "terraform-google-modules/network/google" version = "~> 4.0" + project_id = module.gke_project[each.value].project_id + network_name = "gke-vpc-${each.key}" + routing_mode = "REGIONAL" + + subnets = [ + { + subnet_name = "gke-subnet" + subnet_ip = "10.0.0.0/17" + subnet_region = local.primary_location + }, + ] + secondary_ranges = { + gke-subnet = [ + { + range_name = "us-central1-01-gke-01-pods" + ip_cidr_range = "192.168.0.0/18" + }, + { + range_name = "us-central1-01-gke-01-services" + ip_cidr_range = "192.168.64.0/18" + }, + ] + } +} + +module "gke_cluster" { + for_each = toset(local.envs) + source = "terraform-google-modules/kubernetes-engine/google" + version = "~> 23.0.0" + + + project_id = module.gke_project[each.value].project_id + name = "${each.key}-cluster" + regional = true + region = local.primary_location + zones = ["us-central1-a", "us-central1-b", "us-central1-f"] + network = module.vpc[each.key].network_name + subnetwork = module.vpc[each.key].subnets_names[0] + ip_range_pods = "us-central1-01-gke-01-pods" + ip_range_services = "us-central1-01-gke-01-services" + create_service_account = true + enable_binary_authorization = true + skip_provisioners = false + + depends_on = [ + module.vpc + ] +} + + +###### Private Clusters ###### +# Private Cluster VPCs +module "vpc_private_cluster" { + for_each = toset(local.envs) + source = "terraform-google-modules/network/google" + version = "~> 4.0" + project_id = module.gke_project[each.value].project_id network_name = "gke-private-vpc-${each.value}" routing_mode = "REGIONAL" @@ -136,59 +196,40 @@ resource "google_compute_network_peering_routes_config" "gke_peering_routes_conf for_each = toset(local.envs) project = module.gke_project[each.value].project_id - peering = module.gke_cluster[each.value].peering_name - network = module.vpc[each.value].network_name + peering = module.gke_private_cluster[each.value].peering_name + network = module.vpc_private_cluster[each.value].network_name import_custom_routes = true export_custom_routes = true } -module "gke_cluster" { +module "gke_private_cluster" { for_each = toset(local.envs) source = "terraform-google-modules/kubernetes-engine/google//modules/private-cluster" + version = "~> 23.0.0" project_id = module.gke_project[each.value].project_id name = "${each.value}-private-cluster" regional = true region = local.primary_location zones = ["us-central1-a", "us-central1-b", "us-central1-f"] - network = module.vpc[each.value].network_name - subnetwork = module.vpc[each.value].subnets_names[0] + network = module.vpc_private_cluster[each.value].network_name + subnetwork = module.vpc_private_cluster[each.value].subnets_names[0] ip_range_pods = "us-central1-01-gke-01-pods" ip_range_services = "us-central1-01-gke-01-services" + horizontal_pod_autoscaling = true create_service_account = true enable_binary_authorization = true - skip_provisioners = false - enable_private_endpoint = true - enable_private_nodes = true - master_ipv4_cidr_block = "172.16.${local.ip_increment[each.value]}.0/28" - - remove_default_node_pool = true - node_pools = [ - { - name = "pool-0" - min_count = 1 - max_count = 100 - local_ssd_count = 0 - disk_size_gb = 100 - disk_type = "pd-standard" - image_type = "COS" - auto_repair = true - auto_upgrade = true - preemptible = false - max_pods_per_node = 12 - }, - ] + enable_private_endpoint = true + enable_private_nodes = true + master_ipv4_cidr_block = "172.16.${local.ip_increment[each.value]}.0/28" - ## Can't create these resources before main project is created (after-apply error) - # Enabled read-access to images in GAR repo in CI/CD project - # grant_registry_access = true - # registry_project_ids = [module.project.project_id] + enable_vertical_pod_autoscaling = true master_authorized_networks = [ { - cidr_block = module.vpc[each.value].subnets_ips[0] + cidr_block = module.vpc_private_cluster[each.value].subnets_ips[0] display_name = "VPC" }, { @@ -198,6 +239,6 @@ module "gke_cluster" { ] depends_on = [ - module.vpc + module.vpc_private_cluster ] } diff --git a/test/setup/outputs.tf b/test/setup/outputs.tf index 95d89966..235e1e72 100644 --- a/test/setup/outputs.tf +++ b/test/setup/outputs.tf @@ -51,6 +51,21 @@ output "gke_service_accounts" { } output "gke_cluster_names" { - description = "List of GKE service accounts" + description = "List of GKE clusters" value = zipmap(local.envs, [for env in local.envs : module.gke_cluster[env].name]) } + +output "gke_private_vpc_names" { + description = "List of GKE project IDs" + value = zipmap(local.envs, [for env in local.envs : module.vpc_private_cluster[env].network_name]) +} + +output "gke_private_service_accounts" { + description = "List of GKE private cluster service accounts" + value = zipmap(local.envs, [for env in local.envs : module.gke_private_cluster[env].service_account]) +} + +output "gke_private_cluster_names" { + description = "List of GKE private clusters" + value = zipmap(local.envs, [for env in local.envs : module.gke_private_cluster[env].name]) +}