test: Create BYOID Integration tests

We are adding in regression tests for BYOID.  This PR focuses on
file-based BYOID

CONTRIBUTING.rst

@@ -42,9 +42,9 @@ You can run the system tests with ``nox``::
 To run a single session, specify it with ``nox -s``::
 
     $ nox -f system_tests/noxfile.py -s service_account
-    
-First, set the environemnt variable ``GOOGLE_APPLICATION_CREDENTIALS`` to a valid service account.
-See `Creating and Managing Service Account Keys`_ for how to obtain a service account. 
+
+First, set the environment variable ``GOOGLE_APPLICATION_CREDENTIALS`` to a valid service account.
+See `Creating and Managing Service Account Keys`_ for how to obtain a service account.
 
 Project and Credentials Setup
 -------------------------------
@@ -86,26 +86,40 @@ This will allow the user to impersonate service accounts on the project.
 ``service_account.json``
 ~~~~~~~~~~~~~~~~~~~~~~~~
 
-Follow `Creating and Managing Service Account Keys`_ to create a service account. 
+Follow `Creating and Managing Service Account Keys`_ to create a service account.
 
 Copy the credentials file to ``service_account.json``.
 
 Grant the account associated with ``service_account.json`` the following roles.
 
 - App Engine Admin (for App Engine tests)
-- Service Account Token Creator (for impersonated credentials tests)
+- Service Account Token Creator (for impersonated credentials and BYOID tests)
 - Pub/Sub Viewer (for gRPC tests)
 - Storage Object Viewer (for impersonated credentials tests)
+- DNS Viewer (for BYOID tests)
 
 ``impersonated_service_account.json``
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Follow `Creating and Managing Service Account Keys`_ to create a service account. 
+Follow `Creating and Managing Service Account Keys`_ to create a service account.
 
 Copy the credentials file to ``impersonated_service_account.json``.
 
 .. _Creating and Managing Service Account Keys: https://cloud.google.com/iam/docs/creating-managing-service-account-keys
 
+``setup_byoid``
+~~~~~~~~~~~~~~~~
+
+In order to run the BYOID tests, you will need to set up a Workload Identity Pool,
+as well as attach relevant policy bindings for this new resource to our service account.
+To do this, make sure you have IAM Workload Identity Pool Admin and Service
+Account Admin permissions, and then run:
+
+  $ ./scripts/setup_byoid.sh
+
+and then use the output to replace the variables near
+the top of system_tests/system_tests_sync/test_byoid.py
+
 App Engine System Tests
 ~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -118,16 +132,16 @@ From ``system_tests/app_engine_test_app`` run the following commands ::
     $ pip install --target lib -r requirements.txt
     $ gcloud app deploy -q app.yaml
 
-After the app is deployed, change ``service`` in ``app.yaml`` back to ``google-auth-system-tests``. 
+After the app is deployed, change ``service`` in ``app.yaml`` back to ``google-auth-system-tests``.
 You can now run the App Engine tests: ::
 
     $ nox -f system_tests/noxfile.py -s app_engine
-    
+
 Compute Engine Tests
 ^^^^^^^^^^^^^^^^^^^^
 
 These tests cannot be run locally and will be skipped if they are run outside of Google Compute Engine.
-    
+
 grpc Tests
 ^^^^^^^^^^^^
 

scripts/setup_byoid.sh

@@ -0,0 +1,68 @@
+#!/bin/bash
+# Copyright 2021 Google LLC.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+# This file is a mostly common setup file to ensure all BYOID integration tests
+# are set up in a consistent fashion across the languages in our various client libraries.
+# It assumes that the current user has the relevant permissions to run each of
+# the commands listed.
+
+suffix=""
+
+function generate_random_string () {
+  local valid_chars=abcdefghijklmnopqrstuvwxyz0123456789
+  for i in {1..8} ; do
+    suffix+="${valid_chars:RANDOM%${#valid_chars}:1}"
+    done
+}
+
+generate_random_string
+
+pool_id="pool-"$suffix
+oidc_provider_id="oidc-"$suffix
+aws_provider_id="aws-"$suffix
+
+# Fill in.
+project_id="stellar-day-254222"
+project_number="79992041559"
+aws_account_id="077071391996"
+aws_role_name="ci-python-test"
+service_account_email="kokoro@stellar-day-254222.iam.gserviceaccount.com"
+sub="104692443208068386138"
+
+oidc_aud="//iam.googleapis.com/projects/$project_number/locations/global/workloadIdentityPools/$pool_id/providers/$oidc_provider_id"
+aws_aud="//iam.googleapis.com/projects/$project_number/locations/global/workloadIdentityPools/$pool_id/providers/$aws_provider_id"
+
+gcloud config set project $project_id
+
+# Create the Workload Identity Pool.
+gcloud beta iam workload-identity-pools create $pool_id \
+    --location="global" \
+    --description="Test pool" \
+    --display-name="Test pool for Python"
+
+# Create the OIDC Provider.
+gcloud beta iam workload-identity-pools providers create-oidc $oidc_provider_id \
+    --workload-identity-pool=$pool_id \
+    --issuer-uri="https://accounts.google.com" \
+    --location="global" \
+    --attribute-mapping="google.subject=assertion.sub"
+
+# Create the AWS Provider.
+gcloud beta iam workload-identity-pools providers create-aws $aws_provider_id \
+    --workload-identity-pool=$pool_id \
+    --account-id=$aws_account_id \
+    --location="global"
+
+# Give permission to impersonate the service account.
+gcloud iam service-accounts add-iam-policy-binding $service_account_email \
+--role roles/iam.workloadIdentityUser \
+--member "principal://iam.googleapis.com/projects/$project_number/locations/global/workloadIdentityPools/$pool_id/subject/$sub"
+
+gcloud iam service-accounts add-iam-policy-binding $service_account_email \
+  --role roles/iam.workloadIdentityUser \
+  --member "principalSet://iam.googleapis.com/projects/$project_number/locations/global/workloadIdentityPools/$pool_id/attribute.aws_role/arn:aws:sts::$aws_account_id:assumed-role/$aws_role_name"
+
+echo "OIDC audience: "$oidc_aud
+echo "AWS audience: "$aws_aud

system_tests/noxfile.py

@@ -317,6 +317,12 @@ def mtls_http(session):
     session.run("pytest", "system_tests_sync/test_mtls_http.py")
 
 
+@nox.session(python=["3.7"]) # These tests include our Client Libraries, which are only guaranteed to work in v3.6+
+def byoid(session):
+    session.install(*TEST_DEPENDENCIES_SYNC, "google-auth", "google-api-python-client")
+    session.run("pytest", "system_tests_sync/test_byoid.py")
+
+
 #ASYNC SYSTEM TESTS
 
 @nox.session(python=PYTHON_VERSIONS_ASYNC)

system_tests/system_tests_sync/test_byoid.py

@@ -0,0 +1,80 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import json
+import os
+import tempfile
+
+from google.oauth2 import service_account
+from googleapiclient import discovery
+import pytest
+
+_AUDIENCE_OIDC = "//iam.googleapis.com/projects/79992041559/locations/global/workloadIdentityPools/pool-73wslmxn/providers/oidc-73wslmxn"
+
+
+@pytest.fixture
+def oidc_credentials(service_account_file, http_request):
+    result = service_account.IDTokenCredentials.from_service_account_file(
+        service_account_file,
+        target_audience=_AUDIENCE_OIDC)
+    result.refresh(http_request)
+    yield result
+
+
+@pytest.fixture
+def service_account_info(service_account_file):
+    with open(service_account_file) as f:
+        return json.load(f)
+
+
+# Our BYOID tests involve setting up some preconditions, setting a credential file,
+# and then making sure that our client libraries can work with the set credentials.
+def get_project_dns(project_id, credential_data):
+    fd, credfile_path = tempfile.mkstemp()
+    try:
+        with os.fdopen(fd, 'w') as credfile:
+            credfile.write(json.dumps(credential_data))
+
+        os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = credfile_path
+
+        # If our setup and credential file are correct,
+        # discovery.build should be able to establish these as the default credentials.
+        service = discovery.build('dns', 'v1')
+        request = service.projects().get(project=project_id)
+        return request.execute()
+    finally:
+        os.remove(credfile_path)
+
+
+# This test makes sure that setting an accesible credential file
+# works to allow access to Google resources.
+def test_file_based_byoid(oidc_credentials, service_account_info):
+    fd, tmpfile_path = tempfile.mkstemp()
+    try:
+        with os.fdopen(fd, 'w') as tmpfile:
+            tmpfile.write(oidc_credentials.token)
+
+        assert get_project_dns(service_account_info["project_id"], {
+            "type": "external_account",
+            "audience": _AUDIENCE_OIDC,
+            "subject_token_type": "urn:ietf:params:oauth:token-type:jwt",
+            "token_url": "https://sts.googleapis.com/v1beta/token",
+            "service_account_impersonation_url": "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/{}:generateAccessToken".format(oidc_credentials.service_account_email),
+            "credential_source": {
+                "file": tmpfile_path,
+            },
+        })
+    finally:
+        os.remove(tmpfile_path)