feat: Cloud Deploy support (#25)

* Replaced Cloud Build trigger based deploys with creating Cloud Deploy targets and pipelines * Trigger post-deployment checks in Cloud Build using Cloud Deploy-originated Pub/Sub notifications * Convert app-wet-manifest repo to cloudbuild-cd-config repo, where the configuration of post-deployment checks will live * various improvements and bug fixes
GoogleCloudPlatform · Aug 23, 2022 · 49402b4 · 49402b4
1 parent 3a7cfb3
commit 49402b4
Show file tree

Hide file tree

Showing 35 changed files with 497 additions and 351 deletions.
diff --git a/README.md b/README.md
@@ -29,7 +29,7 @@ module "cd_pipeline" {
   project_id              = var.project_id
   primary_location        = "us-central1"
   gar_repo_name           = module.ci_pipeline.app_artifact_repo
-  manifest_wet_repo       = "app-wet-manifests"
+  cloudbuild_cd_repo      = "cloudbuild-cd-config-pc"
   deploy_branch_clusters  = {
     dev = {
       cluster               = "dev-cluster",
@@ -119,7 +119,7 @@ service account with the necessary roles applied.
 
 ### APIs
 
-A project with the following APIs enabled must be used to host the
+Projects with the following APIs enabled must be used to host the
 resources of this module:
 
 CI/CD Project
@@ -128,6 +128,8 @@ CI/CD Project
 - Storage API `storage-api.googleapis.com`
 - Service Usage API `serviceusage.googleapis.com`
 - Cloud Build API `cloudbuild.googleapis.com`
+- Cloud Deploy API `clouddeploy.googleapis.com`
+- Pub/Sub API `pubsub.googleapis.com`
 - Container Registry API `containerregistry.googleapis.com`
 - IAM Credentials API `iamcredentials.googleapis.com`
 - Cloud Source Repositories API `sourcerepo.googleapis.com`

diff --git a/build/cloudbuild-cd.yaml b/build/cloudbuild-cd.yaml
@@ -28,12 +28,6 @@ artifacts:
       - '/workspace/svcs-endpoints-filtered.json'
 steps:
 
-############################### Deploy to Cluster ###########################
-
-- name: "gcr.io/cloud-builders/gke-deploy"
-  id: "deploy-to-cluster"
-  args: ['apply', '-f', 'manifests', '-p', '$_CLUSTER_PROJECT', '-c', '$_CLUSTER_NAME', '-l', '$_DEFAULT_REGION']
-
 ############################### Post-Deploy Checks ###########################
 
 # Get endpoint IPs
@@ -48,9 +42,6 @@ steps:
       gcloud container clusters get-credentials $_CLUSTER_NAME --region=$_DEFAULT_REGION
       kubectl get svc -ojson > /workspace/svcs.json
 
-      ### Do we need a version that runs against private IP'd svcs?
-      # jq '[.items[] | ({svc: .metadata.name, endpoint: "\(.spec.clusterIPs[]):\(.spec.ports[].port)"}), (select(.status.loadBalancer.ingress != null) | {svc: .metadata.name, endpoint: "\(.status.loadBalancer.ingress[].ip):\(.spec.ports[].port)"})]' /workspace/svcs.json > /workspace/svcs-endpoints-filtered.json
-
       ### Below only grabs external IP'd svcs
       jq '[.items[] | (select(.status.loadBalancer.ingress != null) | {svc: .metadata.name, endpoint: "\(.status.loadBalancer.ingress[].ip):\(.spec.ports[].port)"})]' /workspace/svcs.json > /workspace/svcs-endpoints-filtered.json
 
@@ -134,29 +125,7 @@ steps:
     - '-xe'
     - -c
     - |
-      # Only run this step if there is a next environment supplied in the trigger substitutions
-      # This step will be skipped for the final environment, when there are no more promotions to perform
-      if [ -n "${_NEXT_ENV}" ]; then
-        git config --global user.email "cicd-agent@secure-cicd.com" # TODO: variable with custom domain group alias
-        git config --global user.name "Secure CICD Build Agent (automated)" # TODO: app name var
-
-        # Clone the WET repo
-        cd /workspace
-        gcloud source repos clone $_MANIFEST_WET_REPO --project=${PROJECT_ID}
-        git -C /workspace/$_MANIFEST_WET_REPO branch $_NEXT_ENV
-        git -C /workspace/$_MANIFEST_WET_REPO checkout $_NEXT_ENV
-        git -C /workspace/$_MANIFEST_WET_REPO branch --set-upstream-to=origin/$_NEXT_ENV || echo "$_NEXT_ENV branch does not yet exist"
-        git -C /workspace/$_MANIFEST_WET_REPO pull --no-rebase || echo "$_NEXT_ENV branch does not yet exist"
-
-        cd /workspace/$_MANIFEST_WET_REPO
-
-        # Merge manifests up
-        git status
-        git merge origin/$BRANCH_NAME
-        git add .
-        git push -u origin $_NEXT_ENV
-      else
-        echo "_NEXT_ENV not specified. No further environment to promote to"
-      fi
+      gcloud deploy releases promote --release=$_RELEASE_ID --delivery-pipeline=$_CLOUDDEPLOY_PIPELINE_NAME --region=$_DEFAULT_REGION
+
   waitFor:
     - 'binary-authorization-checkpoint'
diff --git a/build/cloudbuild-ci.yaml b/build/cloudbuild-ci.yaml
@@ -167,48 +167,16 @@ steps:
     - 'container-structure'
     - 'container-scanner'
 
-### Hydrate manifests and push to wet repo
+## Create Cloud Deploy Release to trigger render and deploy to first env
 - name: $_DEFAULT_REGION-docker.pkg.dev/$PROJECT_ID/$_GAR_REPOSITORY/skaffold-builder
-  id: "commit-upstream"
+  id: "cloud-deploy-release"
   entrypoint: "/bin/bash"
   args:
     - '-xe'
     - -c
     - |
-
-      git config --global user.email "cicd-agent@secure-cicd.com" # TODO: variable with custom domain group alias
-      git config --global user.name "Secure CICD Build Agent (automated)" # TODO: app name var
-
-      cd /workspace
-
-      # Clone the DRY repo
-      gcloud config set project ${PROJECT_ID}
-      gcloud source repos clone $_MANIFEST_DRY_REPO --project=${PROJECT_ID}
-
-      # Clone the WET repo
-      cd /workspace
-      gcloud source repos clone $_MANIFEST_WET_REPO --project=${PROJECT_ID}
-      DEFAULT_BRANCH = ($$(git -C /workspace/$_MANIFEST_WET_REPO remote show origin | grep 'HEAD branch' | cut -d' ' -f5))
-      git -C /workspace/$_MANIFEST_WET_REPO branch $_WET_BRANCH_NAME || echo "$_WET_BRANCH_NAME branch already exists"
-      git -C /workspace/$_MANIFEST_WET_REPO checkout $_WET_BRANCH_NAME
-      git -C /workspace/$_MANIFEST_WET_REPO branch --set-upstream-to=origin/$_WET_BRANCH_NAME || echo "$_WET_BRANCH_NAME branch does not yet exist"
-      git -C /workspace/$_MANIFEST_WET_REPO pull --no-rebase || echo "$_WET_BRANCH_NAME branch does not yet exist"
-
-      git -C /workspace/$_MANIFEST_WET_REPO merge $${DEFAULT_BRANCH}
-      mkdir -p /workspace/$_MANIFEST_WET_REPO/manifests
-
-      skaffold config set --global local-cluster false
-      skaffold render --offline=true --digest-source='remote' --filename='skaffold.yaml' --build-artifacts=/artifacts/build-artifacts.json --output='/workspace/$_MANIFEST_WET_REPO/manifests/skaffold-render.yaml' --default-repo=${_GAR_REPO_URI} --loud=true
-      cp /workspace/$_MANIFEST_WET_REPO/manifests/skaffold-render.yaml ./skaffold-render.yaml
-
-      # Go into wet repo and commit changes
-      cd /workspace/$_MANIFEST_WET_REPO
-
-      # git branch $_WET_BRANCH_NAME || echo "$_WET_BRANCH_NAME branch already exists"
-      # git checkout $_WET_BRANCH_NAME
-      git add .
-      git commit -m 'new deployment of applications' || echo "Nothing to commit"
-      git push --set-upstream origin $_WET_BRANCH_NAME
+      RELEASE_NAME='release-$SHORT_SHA'
+      gcloud deploy releases create $${RELEASE_NAME} --delivery-pipeline=${_CLOUDDEPLOY_PIPELINE_NAME} --build-artifacts=/artifacts/build-artifacts.json --region=${_DEFAULT_REGION}
   volumes:
     - path: '/artifacts'
       name: 'artifacts'

diff --git a/examples/app_cicd/README.md b/examples/app_cicd/README.md
@@ -140,8 +140,8 @@ done
 
 | Name | Description |
 |------|-------------|
-| bin\_auth\_attestor\_names | Names of Attestors |
-| bin\_auth\_attestor\_project\_id | Project ID where attestors get created |
+| binauth\_attestor\_names | Names of Attestors |
+| binauth\_attestor\_project\_id | Project ID where attestors get created |
 | boa\_artifact\_repo | GAR Repo created to store BoA images |
 | build\_trigger\_name | The name of the cloud build trigger for the bank of anthos repo. |
 | cache\_bucket\_name | The name of the storage bucket for cloud build. |

diff --git a/examples/app_cicd/cloud-build-builder/Dockerfile b/examples/app_cicd/cloud-build-builder/Dockerfile
@@ -46,7 +46,7 @@ if [[ ! -e /usr/bin/python ]]; then ln -sf /usr/bin/python3 /usr/bin/python; fi
 rm -r /root/.cache
 
 #### 3. Install gcloud
-ENV CLOUD_SDK_VERSION="368.0.0"
+ENV CLOUD_SDK_VERSION="391.0.0"
 ENV CLOUDSDK_INSTALL_DIR /usr/local/gcloud/
 RUN wget "https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-${CLOUD_SDK_VERSION}-linux-x86_64.tar.gz" \
 && tar -C /usr/local -xzf "google-cloud-sdk-${CLOUD_SDK_VERSION}-linux-x86_64.tar.gz" \

diff --git a/examples/app_cicd/main.tf b/examples/app_cicd/main.tf
@@ -14,31 +14,37 @@
  * limitations under the License.
  */
 
+locals {
+  clouddeploy_pipeline_name = "pipeline-01"
+}
+
 module "ci_pipeline" {
-  source                  = "../../modules/secure-ci"
-  project_id              = var.project_id
-  app_source_repo         = "app-source"
-  manifest_dry_repo       = "app-dry-manifests"
-  manifest_wet_repo       = "app-wet-manifests"
-  gar_repo_name_suffix    = "app-image-repo"
-  primary_location        = "us-central1"
-  attestor_names_prefix   = ["build", "security", "quality"]
-  app_build_trigger_yaml  = "cloudbuild-ci.yaml"
-  runner_build_folder     = "${path.module}/cloud-build-builder"
-  build_image_config_yaml = "cloudbuild-skaffold-build-image.yaml"
-  trigger_branch_name     = ".*"
+  source                    = "../../modules/secure-ci"
+  project_id                = var.project_id
+  app_source_repo           = "app-source"
+  cloudbuild_cd_repo        = "cloudbuild-cd-config"
+  gar_repo_name_suffix      = "app-image-repo"
+  primary_location          = "us-central1"
+  attestor_names_prefix     = ["build", "security", "quality"]
+  cache_bucket_name         = "app-cloudbuild"
+  app_build_trigger_yaml    = "cloudbuild-ci.yaml"
+  runner_build_folder       = "${path.module}/cloud-build-builder"
+  build_image_config_yaml   = "cloudbuild-skaffold-build-image.yaml"
+  trigger_branch_name       = ".*"
+  clouddeploy_pipeline_name = local.clouddeploy_pipeline_name
 }
 
 module "cd_pipeline" {
   source           = "../../modules/secure-cd"
   project_id       = var.project_id
   primary_location = "us-central1"
 
-  gar_repo_name           = module.ci_pipeline.app_artifact_repo
-  manifest_wet_repo       = "app-wet-manifests"
-  deploy_branch_clusters  = var.deploy_branch_clusters
-  app_deploy_trigger_yaml = "cloudbuild-cd.yaml"
-  cache_bucket_name       = module.ci_pipeline.cache_bucket_name
+  gar_repo_name             = module.ci_pipeline.app_artifact_repo
+  cloudbuild_cd_repo        = "cloudbuild-cd-config"
+  deploy_branch_clusters    = var.deploy_branch_clusters
+  app_deploy_trigger_yaml   = "cloudbuild-cd.yaml"
+  cache_bucket_name         = module.ci_pipeline.cache_bucket_name
+  clouddeploy_pipeline_name = local.clouddeploy_pipeline_name
   depends_on = [
     module.ci_pipeline
   ]

diff --git a/examples/app_cicd/outputs.tf b/examples/app_cicd/outputs.tf
@@ -19,14 +19,14 @@ output "project_id" {
   description = "The project to run tests against"
 }
 
-output "bin_auth_attestor_names" {
+output "binauth_attestor_names" {
   description = "Names of Attestors"
-  value       = module.ci_pipeline.bin_auth_attestor_names
+  value       = module.ci_pipeline.binauth_attestor_names
 }
 
-output "bin_auth_attestor_project_id" {
+output "binauth_attestor_project_id" {
   description = "Project ID where attestors get created"
-  value       = module.ci_pipeline.bin_auth_attestor_project_id
+  value       = module.ci_pipeline.binauth_attestor_project_id
 }
 
 output "boa_artifact_repo" {

diff --git a/examples/private_cluster_cicd/README.md b/examples/private_cluster_cicd/README.md
@@ -54,8 +54,8 @@ cp -R terraform-google-secure-cicd/examples/private_cluster_cicd/policies bank-o
 
 | Name | Description |
 |------|-------------|
-| bin\_auth\_attestor\_names | Names of Attestors |
-| bin\_auth\_attestor\_project\_id | Project ID where attestors get created |
+| binauth\_attestor\_names | Names of Attestors |
+| binauth\_attestor\_project\_id | Project ID where attestors get created |
 | boa\_artifact\_repo | GAR Repo created to store BoA images |
 | build\_trigger\_name | The name of the cloud build trigger for the bank of anthos repo. |
 | cache\_bucket\_name | The name of the storage bucket for cloud build. |

diff --git a/examples/private_cluster_cicd/cloud-build-builder/Dockerfile b/examples/private_cluster_cicd/cloud-build-builder/Dockerfile
@@ -46,7 +46,7 @@ if [[ ! -e /usr/bin/python ]]; then ln -sf /usr/bin/python3 /usr/bin/python; fi
 rm -r /root/.cache
 
 #### 3. Install gcloud
-ENV CLOUD_SDK_VERSION="368.0.0"
+ENV CLOUD_SDK_VERSION="391.0.0"
 ENV CLOUDSDK_INSTALL_DIR /usr/local/gcloud/
 RUN wget "https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-${CLOUD_SDK_VERSION}-linux-x86_64.tar.gz" \
 && tar -C /usr/local -xzf "google-cloud-sdk-${CLOUD_SDK_VERSION}-linux-x86_64.tar.gz" \

diff --git a/examples/private_cluster_cicd/main.tf b/examples/private_cluster_cicd/main.tf
@@ -14,22 +14,26 @@
  * limitations under the License.
  */
 
+locals {
+  clouddeploy_pipeline_name = "pipeline-private"
+}
+
 # Secure-CI
 module "ci_pipeline" {
-  source                  = "../../modules/secure-ci"
-  project_id              = var.project_id
-  app_source_repo         = "app-source-pc"
-  manifest_dry_repo       = "app-dry-manifests-pc"
-  manifest_wet_repo       = "app-wet-manifests-pc"
-  gar_repo_name_suffix    = "app-image-repo-pc"
-  cache_bucket_name       = "private_cluster_cloudbuild"
-  primary_location        = "us-central1"
-  attestor_names_prefix   = ["build-pc", "security-pc", "quality-pc"]
-  app_build_trigger_yaml  = "cloudbuild-ci.yaml"
-  runner_build_folder     = "${path.module}/cloud-build-builder"
-  build_image_config_yaml = "cloudbuild-skaffold-build-image.yaml"
-  trigger_branch_name     = ".*"
-  cloudbuild_private_pool = module.cloudbuild_private_pool.workerpool_id
+  source                    = "../../modules/secure-ci"
+  project_id                = var.project_id
+  app_source_repo           = "app-source-pc"
+  cloudbuild_cd_repo        = "cloudbuild-cd-config-pc"
+  gar_repo_name_suffix      = "app-image-repo-pc"
+  cache_bucket_name         = "private-cluster-cloudbuild"
+  primary_location          = "us-central1"
+  attestor_names_prefix     = ["build-pc", "security-pc", "quality-pc"]
+  app_build_trigger_yaml    = "cloudbuild-ci.yaml"
+  runner_build_folder       = "${path.module}/cloud-build-builder"
+  build_image_config_yaml   = "cloudbuild-skaffold-build-image.yaml"
+  trigger_branch_name       = ".*"
+  cloudbuild_private_pool   = module.cloudbuild_private_pool.workerpool_id
+  clouddeploy_pipeline_name = local.clouddeploy_pipeline_name
 }
 
 # Secure-CD
@@ -38,12 +42,13 @@ module "cd_pipeline" {
   project_id       = var.project_id
   primary_location = "us-central1"
 
-  gar_repo_name           = module.ci_pipeline.app_artifact_repo
-  manifest_wet_repo       = "app-wet-manifests-pc"
-  deploy_branch_clusters  = var.deploy_branch_clusters
-  app_deploy_trigger_yaml = "cloudbuild-cd.yaml"
-  cache_bucket_name       = module.ci_pipeline.cache_bucket_name
-  cloudbuild_private_pool = module.cloudbuild_private_pool.workerpool_id
+  gar_repo_name             = module.ci_pipeline.app_artifact_repo
+  cloudbuild_cd_repo        = "cloudbuild-cd-config-pc"
+  deploy_branch_clusters    = var.deploy_branch_clusters
+  app_deploy_trigger_yaml   = "cloudbuild-cd.yaml"
+  cache_bucket_name         = module.ci_pipeline.cache_bucket_name
+  cloudbuild_private_pool   = module.cloudbuild_private_pool.workerpool_id
+  clouddeploy_pipeline_name = local.clouddeploy_pipeline_name
   depends_on = [
     module.ci_pipeline
   ]

diff --git a/examples/private_cluster_cicd/outputs.tf b/examples/private_cluster_cicd/outputs.tf
@@ -19,14 +19,14 @@ output "project_id" {
   description = "The project to run tests against"
 }
 
-output "bin_auth_attestor_names" {
+output "binauth_attestor_names" {
   description = "Names of Attestors"
-  value       = module.ci_pipeline.bin_auth_attestor_names
+  value       = module.ci_pipeline.binauth_attestor_names
 }
 
-output "bin_auth_attestor_project_id" {
+output "binauth_attestor_project_id" {
   description = "Project ID where attestors get created"
-  value       = module.ci_pipeline.bin_auth_attestor_project_id
+  value       = module.ci_pipeline.binauth_attestor_project_id
 }
 
 output "boa_artifact_repo" {

diff --git a/modules/secure-cd/README.md b/modules/secure-cd/README.md
@@ -4,7 +4,9 @@ This module creates a number of Google Cloud Build triggers to facilitate deploy
 To securely deploy container images, this pipeline focuses on implementing the "Securing deployed artifacts" and "Securing artifact promotions" sections of the [Shifting left on security report](https://cloud.google.com/solutions/shifting-left-on-security). This module implements security best practices by automating the deployment and promotion of updated container images across multiple GKE clusters, and "failing fast" upon security violation. Users configure a GKE cluster's required attesttions and at which stages a container will receive that attestation based on succesful deployment. The [`cloudbuild-cd.yaml`](../../build/cloudbuild-cd.yaml) contains user-customizable post-deployment security checks to prevent promotion upon discovery of a vulnerability.
 
 This module creates:
-* [Cloud Build Triggers](https://cloud.google.com/build/docs/automating-builds/create-manage-triggers), one for each specified deployment environment (usually one per cluster)
+* A Cloud Deploy [Pipeline and Targets](https://cloud.google.com/deploy/docs/create-pipeline-targets) for each deployment environment.
+* A `clouddeploy-operations` [Pub/Sub topic](https://cloud.google.com/deploy/docs/integrating) to integrate with post-deployment processes.
+* [Cloud Build Triggers](https://cloud.google.com/build/docs/automating-builds/create-manage-triggers), to execute post-deployment checks, triggered by messages to the `clouddeploy-operations` topic.
 * A [Binary Authorization Policy](https://cloud.google.com/binary-authorization/docs) in each project with a GKE cluster, specifying which attestations are required to run containers in each cluster
 
 ## Usage
@@ -16,7 +18,7 @@ module "cd_pipeline" {
   project_id              = var.project_id
   primary_location        = "us-central1"
   gar_repo_name           = <NAME_OF_ARTIFACT_REGISTRY_REPO>
-  manifest_wet_repo       = "app-wet-manifests"
+  cloudbuild_cd_repo      = "cloudbuild-cd-config-pc"
   deploy_branch_clusters  = {
     dev = {
       cluster               = "dev-cluster",
@@ -48,7 +50,7 @@ module "cd_pipeline" {
 }
 ```
 ### Build Configuration
-The template [`cloudbuild-cd.yaml`](../../build/cloudbuild-cd.yaml) build configuration deploys updated containers to specified GKE clusters upon updates to hydrated manifests in the `manifest_wet_repo`. Add the configuration file to the root of the `master` branch of the `manifest_wet_repo` to properly trigger the CD phase.
+The template [`cloudbuild-cd.yaml`](../../build/cloudbuild-cd.yaml) build configuration specifies the post-deployment checks to run on a Cloud Deploy target upon successful deployment, triggered by Pub/Sub messages from Cloud Deploy. By default, this runs an OWASP ZAProxy scan of any exposed services. Then, the process will automatically promote the given Release to the next environment in Cloud Deploy. Push the configuration file to the root of the `main` branch of the `cloudbuild_cd_repo` to properly configure the automation.
 
 <!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 ## Inputs
@@ -58,10 +60,11 @@ The template [`cloudbuild-cd.yaml`](../../build/cloudbuild-cd.yaml) build config
 | additional\_substitutions | Parameters to be substituted in the build specification. All keys should begin with an underscore. | `map(string)` | `{}` | no |
 | app\_deploy\_trigger\_yaml | Name of application cloudbuild yaml file for deployment | `string` | n/a | yes |
 | cache\_bucket\_name | cloud build artifact bucket name | `string` | n/a | yes |
+| cloudbuild\_cd\_repo | Name of repo that stores the Cloud Build CD phase configs - for post-deployment checks | `string` | n/a | yes |
 | cloudbuild\_private\_pool | Cloud Build private pool self-link | `string` | `""` | no |
+| clouddeploy\_pipeline\_name | Cloud Deploy pipeline name | `string` | n/a | yes |
 | deploy\_branch\_clusters | mapping of branch names to cluster deployments | <pre>map(object({<br>    cluster               = string<br>    project_id            = string<br>    location              = string<br>    required_attestations = list(string)<br>    env_attestation       = string<br>    next_env              = string<br>  }))</pre> | `{}` | no |
 | gar\_repo\_name | Docker artifact registry repo to store app build images | `string` | n/a | yes |
-| manifest\_wet\_repo | Name of repo that contains hydrated K8s manifests files | `string` | n/a | yes |
 | primary\_location | Region used for key-ring | `string` | n/a | yes |
 | project\_id | Project ID for CICD Pipeline Project | `string` | n/a | yes |
 
@@ -70,6 +73,8 @@ The template [`cloudbuild-cd.yaml`](../../build/cloudbuild-cd.yaml) build config
 | Name | Description |
 |------|-------------|
 | binauthz\_policy\_required\_attestations | Binary Authorization policy required attestation in GKE projects |
+| clouddeploy\_delivery\_pipeline\_id | ID of the Cloud Deploy delivery pipeline |
+| clouddeploy\_target\_id | ID(s) of Cloud Deploy targets |
 | deploy\_trigger\_names | Names of CD Cloud Build triggers |
 
 <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->