Nested Virtualization of Staging Env in CI

.circleci/config.yml

@@ -113,75 +113,33 @@ jobs:
             xvfb-run -a pipenv run python3 test_gui.py
 
   staging-test-with-rebase:
-    docker:
-      - image: quay.io/freedomofpress/circleci-docker:latest
-        environment:
-          FPF_CI: true
-          CI_SD_ENV: staging
-          CI_AWS_TYPE: t2.medium
-          FPF_GRSEC: false
-          TEST_REPORTS: /root/sd
-    working_directory: ~/sd
+    machine:
+      enabled: true
 
+    working_directory: ~/sd
     steps:
-
       - checkout
 
       - run:
           name: Rebase on-top of github target
           command: ./devops/scripts/rebase-ci.sh
 
       - run:
-          name: Installation apt pre-reqs
-          command: apt-get install -y enchant
-
-      - run:
-          name: Install pip==18.0 to work pipenv/pip bug
-          command: pip install pip==18.0
-
-      - run:
-          name: Installation pre-reqs
-          command: pip install -U -r securedrop/requirements/develop-requirements.txt
-
-      - run:
-          name: Check Python dependencies for CVEs
-          command: make safety
-
-      - run:
-          name: Run static security testing on source code
-          command: make bandit
-
-      - setup_remote_docker
-
-      - run:
-          name: Run Debian builds
-          command: make build-debs
-
-      - run:
-          name: Provision staging servers and run tests
+          name: Run Staging tests on GCE
           command: make ci-go
-          no_output_timeout: 20m
 
       - run:
           name: Ensure environment torn down
-          command: molecule destroy -s aws
-          when: on_fail
-
-      - run:
-          name: Collect JUnit test result files
-          command: ./devops/scripts/combine-junit-test-results.sh
+          # Always report true, since env should will destroyed already
+          # if all tests passed.
+          command: make ci-teardown || true
+          when: always
 
       - store_test_results:
-          path: /root/sd/junit
-
-      - store_artifacts:
-          path: /root/sd/junit
-
-      - store_artifacts:
-          path: /root/sd/raw-test-output
+          path: ~/sd/junit
 
       - store_artifacts:
-          path: /root/sd/.tor_version
+          path: ~/sd/junit
 
 workflows:
   version: 2

Makefile

@@ -1,24 +1,29 @@
 DEFAULT_GOAL: help
 SHELL := /bin/bash
+GCLOUD_VERSION := 222.0.0-1
 PWD := $(shell pwd)
 TAG ?= $(shell git rev-parse HEAD)
 STABLE_VER := $(shell cat molecule/shared/stable.ver)
 
 .PHONY: ci-spinup
-ci-spinup: ## Creates AWS EC2 hosts for testing staging environment.
-	./devops/scripts/ci-spinup.sh
+ci-spinup: ## Creates GCE host for testing staging environment.
+	./devops/gce-nested/gce-start.sh
 
 .PHONY: ci-teardown
-ci-teardown: ## Destroy AWS EC2 hosts for testing staging environment.
-	./devops/scripts/ci-teardown.sh
+ci-teardown: ## Destroys GCE host for testing staging environment.
+	./devops/gce-nested/gce-stop.sh
 
 .PHONY: ci-run
-ci-run: ## Provisions AWS EC2 hosts for testing staging environment.
-	./devops/scripts/ci-runner.sh
+ci-run: ## Provisions GCE host for testing staging environment.
+	./devops/gce-nested/gce-runner.sh
 
 .PHONY: ci-go
-ci-go: ## Creates, provisions, tests, and destroys AWS EC2 hosts for testing staging environment.
-	@if [[ "${CIRCLE_BRANCH}" != docs-* ]]; then molecule test -s aws; else echo Not running on docs branch...; fi
+ci-go: ## Creates, provisions, tests, and destroys GCE host for testing staging environment.
+	@if [[ "${CIRCLE_BRANCH}" != docs-* ]]; then \
+		make ci-spinup; \
+		make ci-run; \
+		make ci-teardown; \
+	fi
 
 .PHONY: ci-lint
 ci-lint: ## Runs linting in linting container.
@@ -86,14 +91,25 @@ docker-build-ubuntu: ## Builds SD Ubuntu docker container
 
 .PHONY: build-debs
 build-debs: ## Builds and tests debian packages
-	@if [[ "${CIRCLE_BRANCH}" != docs-* ]]; then molecule test -s builder; else echo Not running on docs branch...; fi
+	@./devops/scripts/build-debs.sh
+
+.PHONY: build-debs-notest
+build-debs-notest: ## Builds and tests debian packages (sans tests)
+	@./devops/scripts/build-debs.sh notest
 
 .PHONY: build-debs-xenial
 build-debs-xenial: ## Builds and tests debian packages (includes Xenial overrides, TESTING ONLY)
 	@if [[ "${CIRCLE_BRANCH}" != docs-* ]]; then \
 		molecule converge -s builder -- -e securedrop_build_xenial_support=True; \
 		else echo Not running on docs branch...; fi
 
+.PHONY: build-gcloud-docker
+build-gcloud-docker: ## Build docker container for gcloud sdk
+	echo "${GCLOUD_VERSION}" > devops/gce-nested/gcloud-container.ver && \
+	docker build --build-arg="GCLOUD_VERSION=${GCLOUD_VERSION}" \
+				 -f devops/docker/Dockerfile.gcloud \
+				 -t "quay.io/freedomofpress/gcloud-sdk:${GCLOUD_VERSION}" .
+
 .PHONY: safety
 safety: ## Runs `safety check` to check python dependencies for vulnerabilities
 	pip install --upgrade safety && \
@@ -137,7 +153,7 @@ vagrant-package: ## Package up a vagrant box of the last stable SD release
 
 .PHONY: staging
 staging: ## Creates local staging environment in VM, autodetecting platform
-	@./devops/create-staging-env
+	@./devops/scripts/create-staging-env
 
 .PHONY: staging-xenial
 staging-xenial: ## Creates local staging VMs based on Xenial, autodetecting platform

devops/docker/Dockerfile.gcloud

@@ -0,0 +1,21 @@
+FROM centos:7 as gcloud-downloader
+ARG GCLOUD_VERSION
+LABEL org="Freedom of the Press"
+LABEL image_name="gcloud-sdk-centos7"
+
+COPY devops/docker/google-cloud-sdk.repo /etc/yum.repos.d/google-cloud-sdk.repo
+COPY devops/docker/gce-rpm-key.gpg /etc/pki/rpm-gpg/
+COPY devops/docker/gce-yum-key.gpg /etc/pki/rpm-gpg/
+
+RUN rpm --import /etc/pki/rpm-gpg/gce-rpm-key.gpg && \
+    rpm --import /etc/pki/rpm-gpg/gce-yum-key.gpg
+
+RUN rpm --import /etc/pki/rpm-gpg/gce* && \
+    yum install google-cloud-sdk-${GCLOUD_VERSION}.el7.noarch -y && \
+    yum clean all && rm -rf /var/cache/yum
+
+COPY devops/docker/gcloud-wrapper.sh /usr/bin/gcloud-wrapper
+RUN useradd gcloud && \
+    chmod +x /usr/bin/gcloud-wrapper
+USER gcloud
+ENTRYPOINT ["/usr/bin/gcloud-wrapper"]

devops/docker/gce-rpm-key.gpg

@@ -0,0 +1,19 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+Version: GnuPG v1
+
+mQENBFWKtqgBCADmKQWYQF9YoPxLEQZ5XA6DFVg9ZHG4HIuehsSJETMPQ+W9K5c5
+Us5assCZBjG/k5i62SmWb09eHtWsbbEgexURBWJ7IxA8kM3kpTo7bx+LqySDsSC3
+/8JRkiyibVV0dDNv/EzRQsGDxmk5Xl8SbQJ/C2ECSUT2ok225f079m2VJsUGHG+5
+RpyHHgoMaRNedYP8ksYBPSD6sA3Xqpsh/0cF4sm8QtmsxkBmCCIjBa0B0LybDtdX
+XIq5kPJsIrC2zvERIPm1ez/9FyGmZKEFnBGeFC45z5U//pHdB1z03dYKGrKdDpID
+17kNbC5wl24k/IeYyTY9IutMXvuNbVSXaVtRABEBAAG0Okdvb2dsZSBDbG91ZCBQ
+YWNrYWdlcyBSUE0gU2lnbmluZyBLZXkgPGdjLXRlYW1AZ29vZ2xlLmNvbT6JATgE
+EwECACIFAlWKtqgCGy8GCwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJEPCcOUw+
+G6jV+QwH/0wRH+XovIwLGfkg6kYLEvNPvOIYNQWnrT6zZ+XcV47WkJ+i5SR+QpUI
+udMSWVf4nkv+XVHruxydafRIeocaXY0E8EuIHGBSB2KR3HxG6JbgUiWlCVRNt4Qd
+6udC6Ep7maKEIpO40M8UHRuKrp4iLGIhPm3ELGO6uc8rks8qOBMH4ozU+3PB9a0b
+GnPBEsZdOBI1phyftLyyuEvG8PeUYD+uzSx8jp9xbMg66gQRMP9XGzcCkD+b8w1o
+7v3J3juKKpgvx5Lqwvwv2ywqn/Wr5d5OBCHEw8KtU/tfxycz/oo6XUIshgEbS/+P
+6yKDuYhRp6qxrYXjmAszIT25cftb4d4=
+=/PbX
+-----END PGP PUBLIC KEY BLOCK-----

devops/docker/gce-yum-key.gpg

@@ -0,0 +1,33 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+Version: GnuPG v1
+
+mQENBFUd6rIBCAD6mhKRHDn3UrCeLDp7U5IE7AhhrOCPpqGF7mfTemZYHf/5Jdjx
+cOxoSFlK7zwmFr3lVqJ+tJ9L1wd1K6P7RrtaNwCiZyeNPf/Y86AJ5NJwBe0VD0xH
+TXzPNTqRSByVYtdN94NoltXUYFAAPZYQls0x0nUD1hLMlOlC2HdTPrD1PMCnYq/N
+uL/Vk8sWrcUt4DIS+0RDQ8tKKe5PSV0+PnmaJvdF5CKawhh0qGTklS2MXTyKFoqj
+XgYDfY2EodI9ogT/LGr9Lm/+u4OFPvmN9VN6UG+s0DgJjWvpbmuHL/ZIRwMEn/tp
+uneaLTO7h1dCrXC849PiJ8wSkGzBnuJQUbXnABEBAAG0QEdvb2dsZSBDbG91ZCBQ
+YWNrYWdlcyBBdXRvbWF0aWMgU2lnbmluZyBLZXkgPGdjLXRlYW1AZ29vZ2xlLmNv
+bT6JAT4EEwECACgFAlUd6rICGy8FCQWjmoAGCwkIBwMCBhUIAgkKCwQWAgMBAh4B
+AheAAAoJEDdGwginMXsPcLcIAKi2yNhJMbu4zWQ2tM/rJFovazcY28MF2rDWGOnc
+9giHXOH0/BoMBcd8rw0lgjmOosBdM2JT0HWZIxC/Gdt7NSRA0WOlJe04u82/o3OH
+WDgTdm9MS42noSP0mvNzNALBbQnlZHU0kvt3sV1YsnrxljoIuvxKWLLwren/GVsh
+FLPwONjw3f9Fan6GWxJyn/dkX3OSUGaduzcygw51vksBQiUZLCD2Tlxyr9NvkZYT
+qiaWW78L6regvATsLc9L/dQUiSMQZIK6NglmHE+cuSaoK0H4ruNKeTiQUw/EGFaL
+ecay6Qy/s3Hk7K0QLd+gl0hZ1w1VzIeXLo2BRlqnjOYFX4CZAQ0EWsFo2wEIAOsX
+XwoJuxmWjg2MC9V5xMEKenpZwFAnmhKHv4T3yNf1jOdQKs2uCZ4JwIxS9MNEPF9N
+oMnJtoe6B9trjeeqGRs2knjthewhr5gvp4QT16ZKZC2OtJYiJj7ZgljCwOCyByQX
+d26qRvTY50FCWHohsc+hcHof/9vU+BliyiYH7zjVdbUtIk9iVhsitZ/AN9C+2QVA
+j3Svo2SdVNCWmpCHkYs1Y1ipE2sZA+awH42tRiuSXWdS3UtEa76sJ7htJpKY1vAo
+xAqRE4TiROIHvYM+TvMfgubS6jRgUVYbiqwwi6oSKEn/0o1fwZgGv61aDIuiguWx
+0reX7h1Wp3xyOQkzUTEAEQEAAbRAR29vZ2xlIENsb3VkIFBhY2thZ2VzIEF1dG9t
+YXRpYyBTaWduaW5nIEtleSA8Z2MtdGVhbUBnb29nbGUuY29tPokBPgQTAQIAKAUC
+WsFo2wIbLwUJBaOagAYLCQgHAwIGFQgCCQoLBBYCAwECHgECF4AACgkQagMLIboH
+9Pvx7wf/VYfYs3+dU2GblNLVVgkbwH4hbzNLgGrKjPEL2IkAmpkhUdeXyDxr8e6z
+xF9dHtydgdyDyyNJol9CGo71Fsqd9+K5CAaurBDG4LaMFroz9ArN6NN4/QyCLrun
+Kssk1asUjvVGGuK1BmbNNnY+hbF+/pv5O/m/Ss9ob663Unjumf6RiC1Rop2wnPW6
+aLofMroBpwN/QLQKSwl0obsw5axlwHjF47Eli7Lo247opx0TPz9fIRSMi4g6WFhN
+3SEfwT9IQFtdd+3v9UFALnA2rjSLM+L7pYUr97U7jYMinNDvj2iBhDV6h17E82Ev
+N6QpHdeEas1cn3mvko7XRWuwsU13wg==
+=4CNh
+-----END PGP PUBLIC KEY BLOCK-----

devops/docker/gcloud-wrapper.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+#
+
+set -e
+
+SVC_ACCT_FILE=/gce-svc-acct.json
+
+# Try to authenticate Google tooling
+gcloud auth activate-service-account \
+    --key-file "${SVC_ACCT_FILE}" > /dev/null
+
+# Run the container in background, allows subsequent system calls
+if [ "$1" = "background" ]; then
+    tail -f /dev/null
+else
+    /usr/bin/gcloud "$@"
+fi

devops/docker/google-cloud-sdk.repo

@@ -0,0 +1,8 @@
+[google-cloud-sdk]
+name=Google Cloud SDK
+baseurl=https://packages.cloud.google.com/yum/repos/cloud-sdk-el7-x86_64
+enabled=1
+gpgcheck=1
+repo_gpgcheck=1
+gpgkey=file:///etc/pki/rpm-gpg/gce-rpm-key.gpg
+       file:///etc/pki/rpm-gpg/gce-yum-key.gpg

devops/gce-nested/ci-env.sh

@@ -0,0 +1,74 @@
+# Mimic CI, set up the all the required environment variables to match the
+# nested virtualization tests. This file should be sourced by the GCE CI
+# tooling in order to prepare the env.
+
+# If these scripts are run on developer workstations, the CI env
+# vars populated by CircleCI won't be present; make a sane default.
+if [ -z "${CIRCLE_BUILD_NUM:-}" ]; then
+    export CIRCLE_BUILD_NUM="${USER}"
+fi
+
+# Set common vars we'll need throughout the CI scripts.
+TOPLEVEL="$(git rev-parse --show-toplevel)"
+export TOPLEVEL
+GCE_CREDS_FILE="${TOPLEVEL}/.gce.creds"
+export GCE_CREDS_FILE
+export BUILD_NUM="${CIRCLE_BUILD_NUM}"
+export PROJECT_ID="securedrop-ci"
+export JOB_NAME="sd-ci-nested"
+export GCLOUD_MACHINE_TYPE="n1-highcpu-4"
+GCLOUD_CONTAINER_VER="$(cat "${TOPLEVEL}/devops/gce-nested/gcloud-container.ver")"
+export GCLOUD_CONTAINER_VER
+export CLOUDSDK_COMPUTE_ZONE="us-west1-c"
+export EPHEMERAL_DIRECTORY="/tmp/gce-nested"
+export FULL_JOB_ID="${JOB_NAME}-${BUILD_NUM}"
+export SSH_USER_NAME=sdci
+export SSH_PRIVKEY="${EPHEMERAL_DIRECTORY}/gce"
+export SSH_PUBKEY="${SSH_PRIVKEY}.pub"
+
+# The GCE credentials are stored as an env var on the CI platform,
+# retrievable via GOOGLE_CREDENTIALS. Let's read that value, decode it,
+# and write it to disk in the CI environment so the gcloud tooling
+# can authenticate.
+function generate_gce_creds_file() {
+    # First check if there is an existing cred file
+    if [ ! -f "${GCE_CREDS_FILE}" ]; then
+
+        # Oh there isnt one!? Well do we have a google cred env var?
+        if [ -z "${GOOGLE_CREDENTIALS:-}" ]; then
+            echo "ERROR: Make sure you set env var GOOGLE_CREDENTIALS"
+        # Oh we do!? Well then lets process it
+        else
+            # Does the env var have a google string it in.. assume we are a json
+            if [[ "$GOOGLE_CREDENTIALS" =~ google ]]; then
+                echo "$GOOGLE_CREDENTIALS" > "$GCE_CREDS_FILE"
+            # otherwise assume we are a base64 string. Thats needed for CircleCI
+            else
+                echo "$GOOGLE_CREDENTIALS" | base64 --decode > "$GCE_CREDS_FILE"
+            fi
+        fi
+    fi
+}
+
+# Wrapper function to communicate with the gcloud API. Ensure gcloud-sdk
+# container is running, and if so, pass all args to it.
+function gcloud_call() {
+    if ! (docker ps | grep -q gcloud_tool); then
+        docker run --rm \
+                --env="CLOUDSDK_COMPUTE_ZONE=${CLOUDSDK_COMPUTE_ZONE}" \
+                --volume "${EPHEMERAL_DIRECTORY}/gce.pub:/gce.pub" \
+                --volume "${GCE_CREDS_FILE}:/gce-svc-acct.json" \
+                --name gcloud_tool -d \
+                "quay.io/freedomofpress/gcloud-sdk:${GCLOUD_CONTAINER_VER}" \
+                background >/dev/null 2>&1
+        # Give container a moment for gcloud tooling to authenticate
+        # Kept falling over on first calls without this
+        sleep 3
+    fi
+
+    docker exec -i gcloud_tool \
+        /usr/bin/gcloud --project "${PROJECT_ID}" "$@"
+}
+
+
+generate_gce_creds_file