test: Create BYOID Integration tests

CONTRIBUTING.rst

@@ -42,9 +42,9 @@ You can run the system tests with ``nox``::
 To run a single session, specify it with ``nox -s``::
 
     $ nox -f system_tests/noxfile.py -s service_account
-    
-First, set the environemnt variable ``GOOGLE_APPLICATION_CREDENTIALS`` to a valid service account.
-See `Creating and Managing Service Account Keys`_ for how to obtain a service account. 
+
+First, set the environment variable ``GOOGLE_APPLICATION_CREDENTIALS`` to a valid service account.
+See `Creating and Managing Service Account Keys`_ for how to obtain a service account.
 
 Project and Credentials Setup
 -------------------------------
@@ -86,26 +86,40 @@ This will allow the user to impersonate service accounts on the project.
 ``service_account.json``
 ~~~~~~~~~~~~~~~~~~~~~~~~
 
-Follow `Creating and Managing Service Account Keys`_ to create a service account. 
+Follow `Creating and Managing Service Account Keys`_ to create a service account.
 
 Copy the credentials file to ``service_account.json``.
 
 Grant the account associated with ``service_account.json`` the following roles.
 
 - App Engine Admin (for App Engine tests)
-- Service Account Token Creator (for impersonated credentials tests)
+- Service Account Token Creator (for impersonated credentials and BYOID tests)
 - Pub/Sub Viewer (for gRPC tests)
 - Storage Object Viewer (for impersonated credentials tests)
+- DNS Viewer (for BYOID tests)
 
 ``impersonated_service_account.json``
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Follow `Creating and Managing Service Account Keys`_ to create a service account. 
+Follow `Creating and Managing Service Account Keys`_ to create a service account.
 
 Copy the credentials file to ``impersonated_service_account.json``.
 
 .. _Creating and Managing Service Account Keys: https://cloud.google.com/iam/docs/creating-managing-service-account-keys
 
+``setup_byoid``
+~~~~~~~~~~~~~~~~
+
+In order to run the BYOID tests, you will need to set up a Workload Identity Pool,
+as well as attach relevant policy bindings for this new resource to our service account.
+To do this, make sure you have IAM Workload Identity Pool Admin and Service
+Account Admin permissions, and then run:
+
+  $ ./scripts/setup_byoid.sh
+
+and then use the output to replace the variables near
+the top of system_tests/system_tests_sync/test_byoid.py
+
 App Engine System Tests
 ~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -118,16 +132,16 @@ From ``system_tests/app_engine_test_app`` run the following commands ::
     $ pip install --target lib -r requirements.txt
     $ gcloud app deploy -q app.yaml
 
-After the app is deployed, change ``service`` in ``app.yaml`` back to ``google-auth-system-tests``. 
+After the app is deployed, change ``service`` in ``app.yaml`` back to ``google-auth-system-tests``.
 You can now run the App Engine tests: ::
 
     $ nox -f system_tests/noxfile.py -s app_engine
-    
+
 Compute Engine Tests
 ^^^^^^^^^^^^^^^^^^^^
 
 These tests cannot be run locally and will be skipped if they are run outside of Google Compute Engine.
-    
+
 grpc Tests
 ^^^^^^^^^^^^
 

scripts/setup_byoid.sh

@@ -0,0 +1,68 @@
+#!/bin/bash
+# Copyright 2021 Google LLC.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+# This file is a mostly common setup file to ensure all BYOID integration tests
+# are set up in a consistent fashion across the languages in our various client libraries.
+# It assumes that the current user has the relevant permissions to run each of
+# the commands listed.
+
+suffix=""
+
+function generate_random_string () {
+  local valid_chars=abcdefghijklmnopqrstuvwxyz0123456789
+  for i in {1..8} ; do
+    suffix+="${valid_chars:RANDOM%${#valid_chars}:1}"
+    done
+}
+
+generate_random_string
+
+pool_id="pool-"$suffix
+oidc_provider_id="oidc-"$suffix
+aws_provider_id="aws-"$suffix
+
+# Fill in.
+project_id="stellar-day-254222"
+project_number="79992041559"
+aws_account_id="077071391996"
+aws_role_name="ci-python-test"
+service_account_email="kokoro@stellar-day-254222.iam.gserviceaccount.com"
+sub="104692443208068386138"
+
+oidc_aud="//iam.googleapis.com/projects/$project_number/locations/global/workloadIdentityPools/$pool_id/providers/$oidc_provider_id"
+aws_aud="//iam.googleapis.com/projects/$project_number/locations/global/workloadIdentityPools/$pool_id/providers/$aws_provider_id"
+
+gcloud config set project $project_id
+
+# Create the Workload Identity Pool.
+gcloud beta iam workload-identity-pools create $pool_id \
+    --location="global" \
+    --description="Test pool" \
+    --display-name="Test pool for Python"
+
+# Create the OIDC Provider.
+gcloud beta iam workload-identity-pools providers create-oidc $oidc_provider_id \
+    --workload-identity-pool=$pool_id \
+    --issuer-uri="https://accounts.google.com" \
+    --location="global" \
+    --attribute-mapping="google.subject=assertion.sub"
+
+# Create the AWS Provider.
+gcloud beta iam workload-identity-pools providers create-aws $aws_provider_id \
+    --workload-identity-pool=$pool_id \
+    --account-id=$aws_account_id \
+    --location="global"
+
+# Give permission to impersonate the service account.
+gcloud iam service-accounts add-iam-policy-binding $service_account_email \
+--role roles/iam.workloadIdentityUser \
+--member "principal://iam.googleapis.com/projects/$project_number/locations/global/workloadIdentityPools/$pool_id/subject/$sub"
+
+gcloud iam service-accounts add-iam-policy-binding $service_account_email \
+  --role roles/iam.workloadIdentityUser \
+  --member "principalSet://iam.googleapis.com/projects/$project_number/locations/global/workloadIdentityPools/$pool_id/attribute.aws_role/arn:aws:sts::$aws_account_id:assumed-role/$aws_role_name"
+
+echo "OIDC audience: "$oidc_aud
+echo "AWS audience: "$aws_aud

system_tests/noxfile.py

@@ -354,9 +354,14 @@ def mtls_http(session):
     )
 
 
-# ASYNC SYSTEM TESTS
+@nox.session(python=PYTHON_VERSIONS_SYNC)
+def byoid(session):
+    session.install(*TEST_DEPENDENCIES_SYNC, "google-auth", "google-api-python-client", "enum34")
+    default(session, "system_tests_sync/test_byoid.py")
 
 
+# ASYNC SYSTEM TESTS
+
 @nox.session(python=PYTHON_VERSIONS_ASYNC)
 def service_account_async(session):
     session.install(*(TEST_DEPENDENCIES_SYNC + TEST_DEPENDENCIES_ASYNC))
@@ -374,7 +379,7 @@ def default_explicit_service_account_async(session):
     session.install(LIBRARY_DIR)
     default(
         session,
-        "system_tests_async/test_default.py", 
+        "system_tests_async/test_default.py",
         "system_tests_async/test_id_token.py",
     )
 

system_tests/system_tests_sync/test_byoid.py

@@ -0,0 +1,121 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import json
+import os
+from tempfile import NamedTemporaryFile
+
+import sys
+import google.auth
+from google.oauth2 import service_account
+import pytest
+
+_AUDIENCE_OIDC = "//iam.googleapis.com/projects/79992041559/locations/global/workloadIdentityPools/pool-73wslmxn/providers/oidc-73wslmxn"
+
+
+def dns_access_direct(request, project_id):
+    # First, get the default credentials.
+    credentials, _ = google.auth.default(
+        scopes=["https://www.googleapis.com/auth/cloud-platform.read-only"],
+        request=request,
+    )
+
+    # Apply the default credentials to the headers to make the request.
+    headers = {}
+    credentials.apply(headers)
+    response = request(
+        url="https://dns.googleapis.com/dns/v1/projects/{}".format(project_id),
+        headers=headers,
+    )
+
+    if response.status == 200:
+        return response.data
+
+
+def dns_access_client_library(request, project_id):
+    service = discovery.build("dns", "v1")
+    request = service.projects().get(project=project_id)
+    return request.execute()
+
+
+dns_access_funcs = [dns_access_direct]
+try:
+    from googleapiclient import discovery
+
+    dns_access_funcs.append(dns_access_client_library)
+except ImportError as e:
+    if sys.version_info[0] == 3:
+        raise e
+
+
+@pytest.fixture(params=dns_access_funcs)
+def dns_access(request, http_request):
+    def wrapper(project_id):
+        return request.param(http_request, project_id)
+
+    yield wrapper
+
+
+@pytest.fixture
+def oidc_credentials(service_account_file, http_request):
+    result = service_account.IDTokenCredentials.from_service_account_file(
+        service_account_file, target_audience=_AUDIENCE_OIDC
+    )
+    result.refresh(http_request)
+    yield result
+
+
+@pytest.fixture
+def service_account_info(service_account_file):
+    with open(service_account_file) as f:
+        yield json.load(f)
+
+
+# Our BYOID tests involve setting up some preconditions, setting a credential file,
+# and then making sure that our client libraries can work with the set credentials.
+def get_project_dns(dns_access, project_id, credential_data):
+    with NamedTemporaryFile() as credfile:
+        credfile.write(json.dumps(credential_data).encode("utf-8"))
+        credfile.flush()
+        os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = credfile.name
+
+        # If our setup and credential file are correct,
+        # discovery.build should be able to establish these as the default credentials.
+        return dns_access(project_id)
+
+
+# This test makes sure that setting an accesible credential file
+# works to allow access to Google resources.
+def test_file_based_byoid(oidc_credentials, service_account_info, dns_access):
+    with NamedTemporaryFile() as tmpfile:
+        tmpfile.write(oidc_credentials.token.encode("utf-8"))
+        tmpfile.flush()
+
+        assert get_project_dns(
+            dns_access,
+            service_account_info["project_id"],
+            {
+                "type": "external_account",
+                "audience": _AUDIENCE_OIDC,
+                "subject_token_type": "urn:ietf:params:oauth:token-type:jwt",
+                "token_url": "https://sts.googleapis.com/v1beta/token",
+                "service_account_impersonation_url": "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/{}:generateAccessToken".format(
+                    oidc_credentials.service_account_email
+                ),
+                "credential_source": {
+                    "file": tmpfile.name,
+                },
+            },
+        )