chore(tests): Add cloud ops data tests

.github/workflows/e2e-latest.yml

@@ -112,7 +112,7 @@ jobs:
       run: |
         set -x
         # install dependencies
-        python3 -m pip install -r tests/requirements.txt
+        python3 -m pip install --user -r tests/requirements.txt
         # authenticate cluster
         CLUSTER_ZONE=$(gcloud container clusters list --filter="name:cloud-ops-sandbox" --project ${{ env.PROJECT_ID }} --format="value(zone)")
         gcloud container clusters get-credentials cloud-ops-sandbox --zone "$CLUSTER_ZONE"
@@ -124,7 +124,7 @@ jobs:
       run: |
         set -x
         # install dependencies
-        python3 -m pip install -r tests/ratingservice/requirements.txt
+        python3 -m pip install --user -r tests/ratingservice/requirements.txt
         # get service URL
         RATING_SERVICE_URL="https://ratingservice-dot-$(gcloud app describe --format='value(defaultHostname)' --project=${{ env.PROJECT_ID }})"
         # run test

.github/workflows/e2e-release.yml

@@ -98,7 +98,7 @@ jobs:
         # change to previous release code state
         cd ./release-code
         # install dependencies
-        python3 -m pip install -r tests/requirements.txt
+        python3 -m pip install --user -r tests/requirements.txt
         # authenticate cluster
         CLUSTER_ZONE=$(gcloud container clusters list --filter="name:cloud-ops-sandbox" --project ${{ env.PROJECT_ID }} --format="value(zone)")
         gcloud container clusters get-credentials cloud-ops-sandbox --zone "$CLUSTER_ZONE"

.github/workflows/e2e-upgrade.yml

@@ -131,7 +131,7 @@ jobs:
       run: |
         set -x
         # install dependencies
-        python3 -m pip install -r tests/requirements.txt
+        python3 -m pip install --user -r tests/requirements.txt
         # authenticate cluster
         CLUSTER_ZONE=$(gcloud container clusters list --filter="name:cloud-ops-sandbox" --project ${{ env.PROJECT_ID }} --format="value(zone)")
         gcloud container clusters get-credentials cloud-ops-sandbox --zone "$CLUSTER_ZONE"
@@ -143,7 +143,7 @@ jobs:
       run: |
         set -x
         # install dependencies
-        python3 -m pip install -r tests/ratingservice/requirements.txt
+        python3 -m pip install --user -r tests/ratingservice/requirements.txt
         # get service URL
         RATING_SERVICE_URL="https://ratingservice-dot-$(gcloud app describe --format='value(defaultHostname)' --project=${{ env.PROJECT_ID }})"
         # run test

.github/workflows/e2e_scripts/clean.sh

@@ -111,7 +111,7 @@ for INSTANCE_NAME in $(gcloud sql instances list --project=$PROJECT_ID --format=
 done
 
 # delete GKE hub clusters
-for MEMBER_NAME in $(gcloud container fleet memberships list --format="value(NAME)"); do
+for MEMBER_NAME in $(gcloud beta container hub memberships list --format="value(NAME)"); do
   echo "deleting GKE hub membership $MEMBER_NAME"
   gcloud beta container hub memberships delete $MEMBER_NAME --quiet
 done

RELEASING.md

@@ -20,6 +20,7 @@ There are two artifacts that make up a Cloud Operations Sandbox release:
   - a new git tag should have been pushed for the new release version
   - updated container images (hipstershop services, loadgenerator, cloud shell container) should be pushed in the [`stackdriver-sandbox-230822` GCR repo](http://console.cloud.google.com/gcr/images/stackdriver-sandbox-230822)
 1. Review the new release pull request.
+  - Hint: use `sandboxctl test` to run tests on a live, newly-created sandbox environment
   - If any issues arise, delete the branch and tag, push new changes to develop, and start the release process again (see [Reverting Faulty Releases](#reverting-faulty-releases))
 1. When the PR has been reviewed and thoroughly tested, merge it into main
    - Don't squash; we want to keep the git history

sre-recipes/sandboxctl

@@ -202,5 +202,26 @@ def destroy():
     destroy_command = "../terraform/destroy.sh"
     utils.run_interactive_shell_command(destroy_command)
 
+@cli.command()
+@click.option('--skip-ratingservice', default=False,  is_flag=True,
+              help="Don't test the ratingservice")
+def test(skip_ratingservice):
+    """Run tests to ensure environment is working as expected"""
+    os.chdir(os.path.abspath(sys.path[0]))
+    command = "../tests/run_tests.sh"
+    if skip_ratingservice:
+        command = f'SKIP_RATINGS_TEST=TRUE {command}'
+    os.setpgrp()
+    complete = False
+    try:
+        # run install.sh in a background shell
+        process = subprocess.Popen(command, bufsize=0, shell=True)
+        process.communicate()
+        complete = True
+    finally:
+        # kill background process if script is terminated
+        if not complete:
+            os.killpg(0, signal.SIGTERM)
+
 if __name__ == "__main__":
     cli()

terraform/monitoring/03_services.tf

@@ -42,7 +42,7 @@ variable "istio_services" {
       availability_goal      = 0.9
       availability_burn_rate = 2
       latency_goal           = 0.9
-      latency_threshold      = 1000
+      latency_threshold      = 2000
       latency_burn_rate      = 2
     },
     {