Reorganizes GCE shell scripts for clarity

Removed the shellcheck exemptions at the top of the files to get
warnings displayed again, then improved based on that output.
Reduced the "source" calls from 2 to 1, and in general made the scripts
a bit more readable.

Sprinkled comments liberally throughout, to bless future maintainers.

.circleci/config.yml

@@ -126,21 +126,15 @@ jobs:
 
       - run:
           name: Start remote GCE instance
-          command: |
-            source devops/gce-nested/ci-env.sh
-            devops/gce-nested/gce-start.sh
+          command: ./devops/gce-nested/gce-start.sh
 
       - run:
           name: Start Test process
-          command: |
-            source devops/gce-nested/ci-env.sh
-            devops/gce-nested/gce-runner.sh
+          command: ./devops/gce-nested/gce-runner.sh
 
       - run:
           name: Ensure environment torn down
-          command: |
-            source devops/gce-nested/ci-env.sh
-            devops/gce-nested/gce-stop.sh
+          command: ./devops/gce-nested/gce-stop.sh
           when: always
 
       - store_test_results:

devops/gce-nested/ci-env.sh

@@ -1,38 +1,74 @@
-#!/bin/bash
-# shellcheck disable=SC2162,SC2034,SC2059,SC2086,SC2155
-#
 # Mimic CI, set up the all the required environment variables to match the
-# nested virtualization tests
-
-ROOTDIR="$(git rev-parse --show-toplevel)"
-GCE_CREDS_FILE="${ROOTDIR}/.gce.creds"
-
-# First check if there is an existing cred file
-if [ ! -f "${GCE_CREDS_FILE}" ]; then
-
-    # Oh there isnt one!? Well do we have a google cred env var?
-    if [ -z "${GOOGLE_CREDENTIALS}" ]; then
-        echo "ERROR: Make sure you set env var GOOGLE_CREDENTIALS"
-    # Oh we do!? Well then lets process it
-    else
-        # Does the env var have a google string it in.. assume we are a json
-        if [[ "${GOOGLE_CREDENTIALS}" =~ google ]]; then
-            printf "${GOOGLE_CREDENTIALS}" > "${GCE_CREDS_FILE}"
-        # otherwise assume we are a base64 string. Thats needed for CircleCI
-        else
-            printf "${GOOGLE_CREDENTIALS}" | base64 --decode > "${GCE_CREDS_FILE}"
-        fi
-    fi
-fi
+# nested virtualization tests. This file should be sourced by the GCE CI
+# tooling in order to prepare the env.
 
+# If these scripts are run on developer workstations, the CI env
+# vars populated by CircleCI won't be present; make a sane default.
 if [ -z "${CIRCLE_BUILD_NUM:-}" ]; then
     export CIRCLE_BUILD_NUM="${USER}"
 fi
 
+# Set common vars we'll need throughout the CI scripts.
+TOPLEVEL="$(git rev-parse --show-toplevel)"
+export TOPLEVEL
+GCE_CREDS_FILE="${TOPLEVEL}/.gce.creds"
+export GCE_CREDS_FILE
 export BUILD_NUM="${CIRCLE_BUILD_NUM}"
-export PROJECT_ID=securedrop-ci
-export JOB_NAME=sd-ci-nested
-export GCLOUD_MACHINE_TYPE=n1-highcpu-4
-export GCLOUD_CONTAINER_VER="$(cat ${ROOTDIR}/devops/gce-nested/gcloud-container.ver)"
-export CLOUDSDK_COMPUTE_ZONE=us-west1-c
+export PROJECT_ID="securedrop-ci"
+export JOB_NAME="sd-ci-nested"
+export GCLOUD_MACHINE_TYPE="n1-highcpu-4"
+GCLOUD_CONTAINER_VER="$(cat "${TOPLEVEL}/devops/gce-nested/gcloud-container.ver")"
+export GCLOUD_CONTAINER_VER
+export CLOUDSDK_COMPUTE_ZONE="us-west1-c"
 export EPHEMERAL_DIRECTORY="/tmp/gce-nested"
+export FULL_JOB_ID="${JOB_NAME}-${BUILD_NUM}"
+export SSH_USER_NAME=sdci
+export SSH_PRIVKEY="${EPHEMERAL_DIRECTORY}/gce"
+export SSH_PUBKEY="${SSH_PRIVKEY}.pub"
+
+# The GCE credentials are stored as an env var on the CI platform,
+# retrievable via GOOGLE_CREDENTIALS. Let's read that value, decode it,
+# and write it to disk in the CI environment so the gcloud tooling
+# can authenticate.
+function generate_gce_creds_file() {
+    # First check if there is an existing cred file
+    if [ ! -f "${GCE_CREDS_FILE}" ]; then
+
+        # Oh there isnt one!? Well do we have a google cred env var?
+        if [ -z "${GOOGLE_CREDENTIALS:-}" ]; then
+            echo "ERROR: Make sure you set env var GOOGLE_CREDENTIALS"
+        # Oh we do!? Well then lets process it
+        else
+            # Does the env var have a google string it in.. assume we are a json
+            if [[ "$GOOGLE_CREDENTIALS" =~ google ]]; then
+                echo "$GOOGLE_CREDENTIALS" > "$GCE_CREDS_FILE"
+            # otherwise assume we are a base64 string. Thats needed for CircleCI
+            else
+                echo "$GOOGLE_CREDENTIALS" | base64 --decode > "$GCE_CREDS_FILE"
+            fi
+        fi
+    fi
+}
+
+# Wrapper function to communicate with the gcloud API. Ensure gcloud-sdk
+# container is running, and if so, pass all args to it.
+function gcloud_call() {
+    if ! (docker ps | grep -q gcloud_tool); then
+        docker run --rm \
+                --env="CLOUDSDK_COMPUTE_ZONE=${CLOUDSDK_COMPUTE_ZONE}" \
+                --volume "${EPHEMERAL_DIRECTORY}/gce.pub:/gce.pub" \
+                --volume "${GCE_CREDS_FILE}:/gce-svc-acct.json" \
+                --name gcloud_tool -d \
+                "quay.io/freedomofpress/gcloud-sdk:${GCLOUD_CONTAINER_VER}" \
+                background >/dev/null 2>&1
+        # Give container a moment for gcloud tooling to authenticate
+        # Kept falling over on first calls without this
+        sleep 3
+    fi
+
+    docker exec -i gcloud_tool \
+        /usr/bin/gcloud --project "${PROJECT_ID}" "$@"
+}
+
+
+generate_gce_creds_file

devops/gce-nested/gce-runner.sh

@@ -1,57 +1,64 @@
 #!/bin/bash
-# shellcheck disable=SC2162,SC2034,SC2059,SC2086,SC1090,SC2145,SC2035
-#
-
-set -u
+# Configure GCE instance to run the SecureDrop staging environment,
+# including configuration tests. Test results will be collected as XML
+# for storage as artifacts on the build, so devs can review via web.
 set -e
+set -u
+
 
 TOPLEVEL="$(git rev-parse --show-toplevel)"
-CURDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
-. "${CURDIR}/gce.source"
+# shellcheck source=devops/gce-nested/ci-env.sh
+. "${TOPLEVEL}/devops/gce-nested/ci-env.sh"
 
-SSH_USER_NAME=sdci
-SSH_PRIV="${EPHEMERAL_DIRECTORY}/gce"
-REMOTE_IP=$(gcloud_call compute instances describe \
+REMOTE_IP="$(gcloud_call compute instances describe \
             "${JOB_NAME}-${BUILD_NUM}" \
-            --format="value(networkInterfaces[0].accessConfigs.natIP)")
+            --format="value(networkInterfaces[0].accessConfigs.natIP)")"
 SSH_TARGET="${SSH_USER_NAME}@${REMOTE_IP}"
-SSH_OPTS="-i ${SSH_PRIV} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
+SSH_OPTS=(-i "$SSH_PRIVKEY" -o "StrictHostKeyChecking=no" -o "UserKnownHostsFile=/dev/null")
+
 
+# Wrapper utility to run commands on remote GCE instance
 function ssh_gce {
-    eval "ssh ${SSH_OPTS} ${SSH_TARGET} \"cd ~/securedrop-source/ && $@\""
+    # We want all args to be evaluated locally, then passed to the remote
+    # host for execution, so we can safely disable shellcheck 2029.
+    # shellcheck disable=SC2029
+    ssh "${SSH_OPTS[@]}" "$SSH_TARGET" "cd ~/securedrop-source/ && $*"
 }
 
-function scp_gce {
-    eval "scp ${SSH_OPTS} ${SSH_TARGET}:~/securedrop-source/$1 $2"
+# Retrieve XML from test results, for posting as build artifact in CI.
+function fetch_junit_test_results() {
+    local remote_src
+    local local_dest
+    remote_src='junit/*xml'
+    local_dest='junit/'
+    scp "${SSH_OPTS[@]}" "${SSH_TARGET}:~/securedrop-source/${remote_src}" "$local_dest"
 }
 
 # Copy up securedrop repo to remote server
-rsync -a -e "ssh ${SSH_OPTS}" \
-    --exclude .git \
-    --exclude admin/.tox \
-    --exclude *.box \
-    --exclude *.deb \
-    --exclude *.pyc \
-    --exclude *.venv \
-    --exclude .python3 \
-    --exclude .mypy_cache \
-    --exclude securedrop/.sass-cache \
-    --exclude .gce.creds \
-    --exclude *.creds \
-    "${TOPLEVEL}/" "${SSH_TARGET}:~/securedrop-source"
-
-# Run staging process
-ssh_gce "make build-debs-notest"
+function copy_securedrop_repo() {
+  rsync -a -e "ssh ${SSH_OPTS[*]}" \
+      --exclude .git \
+      --exclude admin/.tox \
+      --exclude '*.box' \
+      --exclude '*.deb' \
+      --exclude '*.pyc' \
+      --exclude '*.venv' \
+      --exclude .python3 \
+      --exclude .mypy_cache \
+      --exclude securedrop/.sass-cache \
+      --exclude .gce.creds \
+      --exclude '*.creds' \
+      "${TOPLEVEL}/" "${SSH_TARGET}:~/securedrop-source"
+}
 
-# Run staging process
-# This needs to always pass so test collection can be performed
-ssh_gce "make staging" || export EXIT_CODE="$?"
+# Main logic
+copy_securedrop_repo
+
+ssh_gce "make build-debs-notest"
 
-# Pull test results back for analysis
-scp_gce 'junit/*xml' 'junit/'
+# The test results should be collected regardless of pass/fail,
+# so register a trap to ensure the fetch always runs.
+trap fetch_junit_test_results EXIT
 
-# not proficient with bash traps..
-# someone is welcome to hack this back in with traps
-if [ "${EXIT_CODE:-0}" -ne 0 ]; then
-    exit 1
-fi
+# Run staging environment
+ssh_gce "make staging"

devops/gce-nested/gce-start.sh

@@ -1,38 +1,53 @@
 #!/bin/bash
-# shellcheck disable=SC2086,SC1090
-#
-#
-
+# Create the GCE instance that will host the Staging VMs. All this script
+# does is provision the instances; the actual config and tests are
+# handled by the adjacent gce-runner script.
 set -u
 set -e
 
-# Create ephemeral directory
-mkdir -p "${EPHEMERAL_DIRECTORY}" || true
+TOPLEVEL="$(git rev-parse --show-toplevel)"
+# shellcheck source=devops/gce-nested/ci-env.sh
+. "${TOPLEVEL}/devops/gce-nested/ci-env.sh"
+
 
-# Ensure SSH key in-place
-if [ ! -f "${EPHEMERAL_DIRECTORY}/gce.pub" ]; then
-    ssh-keygen -f "${EPHEMERAL_DIRECTORY}/gce" -q -P ""
-fi
+function create_gce_ssh_key() {
+    # Ensure SSH key in-place
+    if [[ ! -f "$SSH_PUBKEY" ]]; then
+        mkdir -p "$EPHEMERAL_DIRECTORY"
+        ssh-keygen -f "$SSH_PRIVKEY" -q -P ""
+    fi
+}
 
-# Ensure docker container is launched
-CURDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
-. "${CURDIR}/gce.source"
+# Lookup the latest GCE image available for use with SD CI.
+# Value will be used in the create call.
+function find_latest_ci_image() {
+    gcloud_call compute images list \
+        --filter="family:fpf-securedrop AND name ~ ^ci-nested-virt" \
+        --sort-by=~Name --limit=1 --format="value(Name)"
+}
 
-# Find latest CI image
-IMG_LOCATE=$(gcloud_call compute images list \
-    --filter="family:fpf-securedrop AND name ~ ^ci-nested-virt" \
-    --sort-by=~Name --limit=1 --format="value(Name)")
+# Call out to GCE API and start a new instance, designating
+# the SD CI network settings.
+function create_sd_ci_gce_instance() {
+  # First check that a suitable instance isn't already running.
+  if ! gcloud_call compute instances describe "${FULL_JOB_ID}" >/dev/null 2>&1; then
+      # Fetch latest image id, for use in create call
+      local ci_image
+      ci_image="$(find_latest_ci_image)"
+      # Fire-up remote instance
+      gcloud_call compute instances create "${FULL_JOB_ID}" \
+          --image="$ci_image" \
+          --network securedropci \
+          --subnet ci-subnet \
+          --boot-disk-type=pd-ssd \
+          --machine-type="${GCLOUD_MACHINE_TYPE}" \
+          --metadata "ssh-keys=${SSH_USER_NAME}:$(cat $SSH_PUBKEY)"
 
-if ! gcloud_call compute instances describe "${FULL_JOB_ID}" >/dev/null 2>&1; then
-    # Fire-up remote instance
-    gcloud_call compute instances create "${FULL_JOB_ID}" \
-        --image="${IMG_LOCATE}" \
-        --network securedropci \
-        --subnet ci-subnet \
-        --boot-disk-type=pd-ssd \
-        --machine-type="${GCLOUD_MACHINE_TYPE}" \
-        --metadata "ssh-keys=sdci:$(cat ${EPHEMERAL_DIRECTORY}/gce.pub)"
+      # Give box a few more seconds for SSH to become available
+      sleep 20
+  fi
+}
 
-    # Give box a few more seconds for SSH to become available
-    sleep 20
-fi
+# Main logic
+create_gce_ssh_key
+create_sd_ci_gce_instance

devops/gce-nested/gce-stop.sh

@@ -1,13 +1,14 @@
 #!/bin/bash
-# shellcheck disable=SC1090
-#
-#
+# Destroys GCE instances used for CI. This script will be run by CI
+# regardless of pass/fail state of tests, to ensure instances don't
+# remain running, incurring additional costs.
 
 set -u
 set -e
 
-CURDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
-. "${CURDIR}/gce.source"
+TOPLEVEL="$(git rev-parse --show-toplevel)"
+# shellcheck source=devops/gce-nested/ci-env.sh
+. "${TOPLEVEL}/devops/gce-nested/ci-env.sh"
 
 # Destroy remote instance
 gcloud_call --quiet compute instances delete "${JOB_NAME}-${BUILD_NUM}"