From b4c605afe2cbbd48e768c7e1b8db2ac12d0782cd Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Tue, 16 Dec 2014 01:44:26 -0800 Subject: [PATCH 1/3] Changing auth to use implicit environment. --- CONTRIBUTING.rst | 20 ++- README.rst | 8 +- .../_components/datastore-getting-started.rst | 5 +- docs/_components/datastore-quickstart.rst | 13 +- docs/_components/storage-getting-started.rst | 2 +- docs/_components/storage-quickstart.rst | 13 +- docs/index.rst | 14 +- gcloud/credentials.py | 56 ++++++- gcloud/datastore/__init__.py | 43 ++--- gcloud/datastore/connection.py | 4 +- gcloud/datastore/demo/__init__.py | 10 +- gcloud/datastore/query.py | 4 +- gcloud/datastore/test___init__.py | 40 +---- gcloud/datastore/transaction.py | 12 +- gcloud/storage/__init__.py | 44 ++--- gcloud/storage/acl.py | 2 +- gcloud/storage/bucket.py | 15 +- gcloud/storage/connection.py | 101 +++++++++--- gcloud/storage/demo/__init__.py | 9 +- gcloud/storage/test___init__.py | 39 ++--- gcloud/storage/test_connection.py | 151 ++++++++++++++++-- gcloud/test_credentials.py | 29 +++- regression/app_credentials.json.sample | 7 + regression/key.json.enc | Bin 0 -> 1248 bytes regression/key.p12.enc | Bin 1744 -> 0 bytes regression/local_test_setup.sample | 3 +- regression/regression_utils.py | 25 ++- scripts/run_regression.sh | 4 +- 28 files changed, 405 insertions(+), 268 deletions(-) create mode 100644 regression/app_credentials.json.sample create mode 100644 regression/key.json.enc delete mode 100644 regression/key.p12.enc diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 54395707d2c1..86ad3daaa2c6 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -164,12 +164,12 @@ Running Regression Tests bamboo-shift-455). - ``GCLOUD_TESTS_DATASET_ID``: The name of the dataset your tests connect to. This is typically the same as ``GCLOUD_TESTS_PROJECT_ID``. - - ``GCLOUD_TESTS_CLIENT_EMAIL``: The email for the service account you're - authenticating with - - ``GCLOUD_TESTS_KEY_FILE``: The path to an encrypted key file. - See private key + - ``GOOGLE_APPLICATION_CREDENTIALS``: The path to an encrypted JSON file; + see ``regression/app_credentials.json.sample`` as an example. Such a file + can be downloaded directly from the developer's console by clicking + "Generate new JSON key". See private key `docs `__ - for explanation on how to get a private key. + for more details. - Examples of these can be found in ``regression/local_test_setup.sample``. We recommend copying this to ``regression/local_test_setup``, editing the values @@ -177,10 +177,6 @@ Running Regression Tests $ source regression/local_test_setup -- The ``GCLOUD_TESTS_KEY_FILE`` value should point to a valid path (relative or - absolute) on your system where the key file for your service account can - be found. - - For datastore tests, you'll need to create composite `indexes `__ with the ``gcloud`` command line @@ -194,8 +190,10 @@ Running Regression Tests $ export CLOUDSDK_PYTHON_SITEPACKAGES=1 # Authenticate the gcloud tool with your account. - $ gcloud auth activate-service-account $GCLOUD_TESTS_CLIENT_EMAIL \ - > --key-file=$GCLOUD_TESTS_KEY_FILE + $ SERVICE_ACCOUNT_EMAIL="some-account@developer.gserviceaccount.com" + $ P12_CREDENTIALS_FILE="path/to/keyfile.p12" + $ gcloud auth activate-service-account $SERVICE_ACCOUNT_EMAIL \ + > --key-file=$P12_CREDENTIALS_FILE # Create the indexes $ gcloud preview datastore create-indexes regression/data/ \ diff --git a/README.rst b/README.rst index 00036c023f54..072657e0c3d3 100644 --- a/README.rst +++ b/README.rst @@ -48,9 +48,7 @@ Library. .. code:: python from gcloud import datastore - dataset = datastore.get_dataset('dataset-id-here', - 'long-email@googleapis.com', - '/path/to/private.key') + dataset = datastore.get_dataset('dataset-id-here') # Then do other things... query = dataset.query().kind('EntityKind') entity = dataset.entity('EntityKind') @@ -75,9 +73,7 @@ to learn how to connect to the Cloud Storage using this Client Library. .. code:: python import gcloud.storage - bucket = gcloud.storage.get_bucket('bucket-id-here', - 'long-email@googleapis.com', - '/path/to/private.key') + bucket = gcloud.storage.get_bucket('bucket-id-here', 'project-id') # Then do other things... key = bucket.get_key('/remote/path/to/file.txt') print key.get_contents_as_string() diff --git a/docs/_components/datastore-getting-started.rst b/docs/_components/datastore-getting-started.rst index cd1ada33d9a4..c53e1b6b3541 100644 --- a/docs/_components/datastore-getting-started.rst +++ b/docs/_components/datastore-getting-started.rst @@ -38,10 +38,7 @@ Add some data to your dataset Open a Python console and... >>> from gcloud import datastore - >>> dataset = datastore.get_dataset( - >>> '>> '', - >>> '/path/to/.key') + >>> dataset = datastore.get_dataset('') >>> dataset.query().fetch() [] >>> entity = dataset.entity('Person') diff --git a/docs/_components/datastore-quickstart.rst b/docs/_components/datastore-quickstart.rst index 48c4e404f925..88129f2538a5 100644 --- a/docs/_components/datastore-quickstart.rst +++ b/docs/_components/datastore-quickstart.rst @@ -22,12 +22,12 @@ authentication to your project: bamboo-shift-455). - ``GCLOUD_TESTS_DATASET_ID``: The name of the dataset your tests connect to. This is typically the same as ``GCLOUD_TESTS_PROJECT_ID``. - - ``GCLOUD_TESTS_CLIENT_EMAIL``: The email for the service account you're - authenticating with - - ``GCLOUD_TESTS_KEY_FILE``: The path to an encrypted key file. - See private key + - ``GOOGLE_APPLICATION_CREDENTIALS``: The path to an encrypted JSON file; + see ``regression/app_credentials.json.sample`` as an example. Such a file + can be downloaded directly from the developer's console by clicking + "Generate new JSON key". See private key `docs `__ - for explanation on how to get a private key. + for more details. Run the `example script `_ @@ -68,7 +68,6 @@ you can create entities and save them:: >>> from gcloud import datastore >>> from gcloud.datastore import demo - >>> dataset = datastore.get_dataset( - >>> demo.DATASET_ID, demo.CLIENT_EMAIL, demo.PRIVATE_KEY_PATH) + >>> dataset = datastore.get_dataset(demo.DATASET_ID) ---- diff --git a/docs/_components/storage-getting-started.rst b/docs/_components/storage-getting-started.rst index 97ea52405679..47ad25c9169f 100644 --- a/docs/_components/storage-getting-started.rst +++ b/docs/_components/storage-getting-started.rst @@ -45,7 +45,7 @@ The first step in accessing Cloud Storage is to create a connection to the service:: >>> from gcloud import storage - >>> connection = storage.get_connection(project_name, email, key_path) + >>> connection = storage.get_connection(project_name) We're going to use this :class:`connection ` object diff --git a/docs/_components/storage-quickstart.rst b/docs/_components/storage-quickstart.rst index ed2d9cbe303b..aff52c32fbed 100644 --- a/docs/_components/storage-quickstart.rst +++ b/docs/_components/storage-quickstart.rst @@ -22,12 +22,12 @@ authentication to your project: bamboo-shift-455). - ``GCLOUD_TESTS_DATASET_ID``: The name of the dataset your tests connect to. This is typically the same as ``GCLOUD_TESTS_PROJECT_ID``. - - ``GCLOUD_TESTS_CLIENT_EMAIL``: The email for the service account you're - authenticating with - - ``GCLOUD_TESTS_KEY_FILE``: The path to an encrypted key file. - See private key + - ``GOOGLE_APPLICATION_CREDENTIALS``: The path to an encrypted JSON file; + see ``regression/app_credentials.json.sample`` as an example. Such a file + can be downloaded directly from the developer's console by clicking + "Generate new JSON key". See private key `docs `__ - for explanation on how to get a private key. + for more details. Run the `example script `_ @@ -76,7 +76,6 @@ you can create buckets and keys:: >>> from gcloud import storage >>> from gcloud.storage import demo - >>> connection = storage.get_connection( - >>> demo.PROJECT_NAME, demo.CLIENT_EMAIL, demo.PRIVATE_KEY_PATH) + >>> connection = storage.get_connection(demo.PROJECT_ID) ---- diff --git a/docs/index.rst b/docs/index.rst index e4ca5d895406..28865c028fc6 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -29,10 +29,7 @@ Cloud Datastore .. code-block:: python from gcloud import datastore - dataset = datastore.get_dataset( - '', - '', - '/path/to/your/key') + dataset = datastore.get_dataset('') entity = dataset.entity('Person') entity['name'] = 'Your name' entity['age'] = 25 @@ -46,13 +43,8 @@ Cloud Storage .. _Google Cloud Storage: https://developers.google.com/storage/ .. code-block:: python - + from gcloud import storage - bucket = storage.get_bucket( - '', - '', - '', - '/path/to/your/key') + bucket = storage.get_bucket('', '') key = bucket.new_key('my-test-file.txt') key = key.upload_contents_from_string('this is test content!') - \ No newline at end of file diff --git a/gcloud/credentials.py b/gcloud/credentials.py index 12aceca8a025..0744b67f1409 100644 --- a/gcloud/credentials.py +++ b/gcloud/credentials.py @@ -17,16 +17,56 @@ from oauth2client import client -def get_for_service_account(client_email, private_key_path, scope=None): +def get_credentials(): + """Gets credentials implicitly from the current environment. + + .. note:: + You should not need to use this function directly. Instead, use the + helper methods provided in + :func:`gcloud.datastore.__init__.get_connection` and + :func:`gcloud.datastore.__init__.get_dataset` which use this method + under the hood. + + Checks environment in order of precedence: + - Google App Engine (production and testing) + - Environment variable GOOGLE_APPLICATION_CREDENTIALS pointing to + a file with stored credentials information. + - Stored "well known" file associated with `gcloud` command line tool. + - Google Compute Engine production environment. + + The file referred to in GOOGLE_APPLICATION_CREDENTIALS is expected to + contain information about credentials that are ready to use. This means + either service account information or user account information with + a ready-to-use refresh token: + { { + 'type': 'authorized_user', 'type': 'service_account', + 'client_id': '...', 'client_id': '...', + 'client_secret': '...', OR 'client_email': '...', + 'refresh_token': '..., 'private_key_id': '...', + } 'private_key': '...', + } + The second of these is simply a JSON key downloaded from the Google APIs + console. The first is a close cousin of the "client secrets" JSON file + used by `oauth2client.clientsecrets` but differs in formatting. + + :rtype: :class:`oauth2client.client.GoogleCredentials`, + :class:`oauth2client.appengine.AppAssertionCredentials`, + :class:`oauth2client.gce.AppAssertionCredentials`, + :class:`oauth2client.service_account._ServiceAccountCredentials` + :returns: A new credentials instance corresponding to the implicit + environment. + """ + return client.GoogleCredentials.get_application_default() + + +def get_for_service_account_p12(client_email, private_key_path, scope=None): """Gets the credentials for a service account. .. note:: - You should not need to use this function directly. - Instead, use the helper methods provided in - :func:`gcloud.datastore.__init__.get_connection` - and - :func:`gcloud.datastore.__init__.get_dataset` - which use this method under the hood. + This method is not used by default, instead :func:`get_credentials` + is used. This method is intended to be used when the environments is + known explicitly and detecting the environment implicitly would be + superfluous. :type client_email: string :param client_email: The e-mail attached to the service account. @@ -34,7 +74,7 @@ def get_for_service_account(client_email, private_key_path, scope=None): :type private_key_path: string :param private_key_path: The path to a private key file (this file was given to you when you created the service - account). + account). This file must be in P12 format. :type scope: string or tuple of strings :param scope: The scope against which to authenticate. (Different services diff --git a/gcloud/datastore/__init__.py b/gcloud/datastore/__init__.py index 53d94e2e7f5f..b4a8bd0a57ad 100644 --- a/gcloud/datastore/__init__.py +++ b/gcloud/datastore/__init__.py @@ -17,9 +17,7 @@ You'll typically use these to get started with the API: >>> from gcloud import datastore ->>> dataset = datastore.get_dataset('dataset-id-here', -... 'long-email@googleapis.com', -... '/path/to/private.key') +>>> dataset = datastore.get_dataset('dataset-id-here') >>> # Then do other things... >>> query = dataset.query().kind('EntityKind') >>> entity = dataset.entity('EntityKind') @@ -53,37 +51,30 @@ """The scope required for authenticating as a Cloud Datastore consumer.""" -def get_connection(client_email, private_key_path): +from gcloud import credentials +from gcloud.datastore.connection import Connection + + +def get_connection(): """Shortcut method to establish a connection to the Cloud Datastore. Use this if you are going to access several datasets with the same set of credentials (unlikely): >>> from gcloud import datastore - >>> connection = datastore.get_connection(email, key_path) + >>> connection = datastore.get_connection() >>> dataset1 = connection.dataset('dataset1') >>> dataset2 = connection.dataset('dataset2') - :type client_email: string - :param client_email: The e-mail attached to the service account. - - :type private_key_path: string - :param private_key_path: The path to a private key file (this file was - given to you when you created the service - account). - :rtype: :class:`gcloud.datastore.connection.Connection` :returns: A connection defined with the proper credentials. """ - from gcloud import credentials - from gcloud.datastore.connection import Connection + implicit_credentials = credentials.get_credentials() + scoped_credentials = implicit_credentials.create_scoped(SCOPE) + return Connection(credentials=scoped_credentials) - svc_account_credentials = credentials.get_for_service_account( - client_email, private_key_path, scope=SCOPE) - return Connection(credentials=svc_account_credentials) - -def get_dataset(dataset_id, client_email, private_key_path): +def get_dataset(dataset_id): """Establish a connection to a particular dataset in the Cloud Datastore. This is a shortcut method for creating a connection and using it @@ -92,7 +83,7 @@ def get_dataset(dataset_id, client_email, private_key_path): You'll generally use this as the first call to working with the API: >>> from gcloud import datastore - >>> dataset = datastore.get_dataset('dataset-id', email, key_path) + >>> dataset = datastore.get_dataset('dataset-id') >>> # Now you can do things with the dataset. >>> dataset.query().kind('TestKind').fetch() [...] @@ -103,16 +94,8 @@ def get_dataset(dataset_id, client_email, private_key_path): and is usually the same as your Cloud Datastore project name. - :type client_email: string - :param client_email: The e-mail attached to the service account. - - :type private_key_path: string - :param private_key_path: The path to a private key file (this file was - given to you when you created the service - account). - :rtype: :class:`gcloud.datastore.dataset.Dataset` :returns: A dataset with a connection using the provided credentials. """ - connection = get_connection(client_email, private_key_path) + connection = get_connection() return connection.dataset(dataset_id) diff --git a/gcloud/datastore/connection.py b/gcloud/datastore/connection.py index 5124fdcec856..84880b3fbd1e 100644 --- a/gcloud/datastore/connection.py +++ b/gcloud/datastore/connection.py @@ -180,7 +180,7 @@ def lookup(self, dataset_id, key_pbs): >>> from gcloud import datastore >>> from gcloud.datastore.key import Key - >>> connection = datastore.get_connection(email, key_path) + >>> connection = datastore.get_connection() >>> dataset = connection.dataset('dataset-id') >>> key = Key(dataset=dataset).kind('MyKind').id(1234) @@ -248,7 +248,7 @@ def run_query(self, dataset_id, query_pb, namespace=None): uses this method to fetch data: >>> from gcloud import datastore - >>> connection = datastore.get_connection(email, key_path) + >>> connection = datastore.get_connection() >>> dataset = connection.dataset('dataset-id') >>> query = dataset.query().kind('MyKind').filter('property =', 'val') diff --git a/gcloud/datastore/demo/__init__.py b/gcloud/datastore/demo/__init__.py index 62530d1a2216..72907e1b1695 100644 --- a/gcloud/datastore/demo/__init__.py +++ b/gcloud/datastore/demo/__init__.py @@ -12,17 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -# pragma NO COVER import os from gcloud import datastore -__all__ = ['get_dataset', 'CLIENT_EMAIL', 'DATASET_ID', 'KEY_FILENAME'] + +__all__ = ['get_dataset', 'DATASET_ID'] DATASET_ID = os.getenv('GCLOUD_TESTS_DATASET_ID') -CLIENT_EMAIL = os.getenv('GCLOUD_TESTS_CLIENT_EMAIL') -KEY_FILENAME = os.getenv('GCLOUD_TESTS_KEY_FILE') -def get_dataset(): # pragma NO COVER - return datastore.get_dataset(DATASET_ID, CLIENT_EMAIL, KEY_FILENAME) +def get_dataset(): + return datastore.get_dataset(DATASET_ID) diff --git a/gcloud/datastore/query.py b/gcloud/datastore/query.py index 1058b9ee18df..71d6483d68ce 100644 --- a/gcloud/datastore/query.py +++ b/gcloud/datastore/query.py @@ -43,7 +43,7 @@ class Query(object): generates a query that can be executed without any additional work:: >>> from gcloud import datastore - >>> dataset = datastore.get_dataset('dataset-id', email, key_path) + >>> dataset = datastore.get_dataset('dataset-id') >>> query = dataset.query('MyKind') :type kind: string @@ -319,7 +319,7 @@ def fetch(self, limit=None): For example:: >>> from gcloud import datastore - >>> dataset = datastore.get_dataset('dataset-id', email, key_path) + >>> dataset = datastore.get_dataset('dataset-id') >>> query = dataset.query('Person').filter('name =', 'Sally') >>> query.fetch() [, , ...] diff --git a/gcloud/datastore/test___init__.py b/gcloud/datastore/test___init__.py index b20eae78534d..e258cda05003 100644 --- a/gcloud/datastore/test___init__.py +++ b/gcloud/datastore/test___init__.py @@ -17,66 +17,42 @@ class Test_get_connection(unittest2.TestCase): - def _callFUT(self, client_email, private_key_path): + def _callFUT(self): from gcloud.datastore import get_connection - return get_connection(client_email, private_key_path) + return get_connection() def test_it(self): - from tempfile import NamedTemporaryFile from gcloud import credentials - from gcloud.datastore import SCOPE from gcloud.datastore.connection import Connection from gcloud.test_credentials import _Client from gcloud._testing import _Monkey - CLIENT_EMAIL = 'phred@example.com' - PRIVATE_KEY = 'SEEkR1t' client = _Client() with _Monkey(credentials, client=client): - with NamedTemporaryFile() as f: - f.write(PRIVATE_KEY) - f.flush() - found = self._callFUT(CLIENT_EMAIL, f.name) + found = self._callFUT() self.assertTrue(isinstance(found, Connection)) self.assertTrue(found._credentials is client._signed) - expected_called_with = { - 'service_account_name': CLIENT_EMAIL, - 'private_key': PRIVATE_KEY, - 'scope': SCOPE, - } - self.assertEqual(client._called_with, expected_called_with) + self.assertTrue(client._get_app_default_called) class Test_get_dataset(unittest2.TestCase): - def _callFUT(self, dataset_id, client_email, private_key_path): + def _callFUT(self, dataset_id): from gcloud.datastore import get_dataset - return get_dataset(dataset_id, client_email, private_key_path) + return get_dataset(dataset_id) def test_it(self): - from tempfile import NamedTemporaryFile from gcloud import credentials - from gcloud.datastore import SCOPE from gcloud.datastore.connection import Connection from gcloud.datastore.dataset import Dataset from gcloud.test_credentials import _Client from gcloud._testing import _Monkey - CLIENT_EMAIL = 'phred@example.com' - PRIVATE_KEY = 'SEEkR1t' DATASET_ID = 'DATASET' client = _Client() with _Monkey(credentials, client=client): - with NamedTemporaryFile() as f: - f.write(PRIVATE_KEY) - f.flush() - found = self._callFUT(DATASET_ID, CLIENT_EMAIL, f.name) + found = self._callFUT(DATASET_ID) self.assertTrue(isinstance(found, Dataset)) self.assertTrue(isinstance(found.connection(), Connection)) self.assertEqual(found.id(), DATASET_ID) - expected_called_with = { - 'service_account_name': CLIENT_EMAIL, - 'private_key': PRIVATE_KEY, - 'scope': SCOPE, - } - self.assertEqual(client._called_with, expected_called_with) + self.assertTrue(client._get_app_default_called) diff --git a/gcloud/datastore/transaction.py b/gcloud/datastore/transaction.py index 047193d7967a..67de81b7af3c 100644 --- a/gcloud/datastore/transaction.py +++ b/gcloud/datastore/transaction.py @@ -29,7 +29,7 @@ class Transaction(object): mutation, and execute those within a transaction:: >>> from gcloud import datastore - >>> dataset = datastore.get_dataset('dataset-id', email, key_path) + >>> dataset = datastore.get_dataset('dataset-id') >>> with dataset.transaction() ... entity1.save() ... entity2.save() @@ -38,7 +38,7 @@ class Transaction(object): exits with an error:: >>> from gcloud import datastore - >>> dataset = datastore.get_dataset('dataset-id', email, key_path) + >>> dataset = datastore.get_dataset('dataset-id') >>> with dataset.transaction() as t: ... do_some_work() ... raise Exception() # rolls back @@ -96,14 +96,14 @@ class Transaction(object): For example, this is perfectly valid:: >>> from gcloud import datastore - >>> dataset = datastore.get_dataset('dataset-id', email, key_path) + >>> dataset = datastore.get_dataset('dataset-id') >>> with dataset.transaction(): ... dataset.entity('Thing').save() However, this **wouldn't** be acceptable:: >>> from gcloud import datastore - >>> dataset = datastore.get_dataset('dataset-id', email, key_path) + >>> dataset = datastore.get_dataset('dataset-id') >>> with dataset.transaction(): ... dataset.entity('Thing').save() ... with dataset.transaction(): @@ -114,8 +114,8 @@ class Transaction(object): really need to nest transactions, try:: >>> from gcloud import datastore - >>> dataset1 = datastore.get_dataset('dataset-id', email, key_path) - >>> dataset2 = datastore.get_dataset('dataset-id', email, key_path) + >>> dataset1 = datastore.get_dataset('dataset-id1') + >>> dataset2 = datastore.get_dataset('dataset-id2') >>> with dataset1.transaction(): ... dataset1.entity('Thing').save() ... with dataset2.transaction(): diff --git a/gcloud/storage/__init__.py b/gcloud/storage/__init__.py index ff6907ea8215..0d3fdf0262a6 100644 --- a/gcloud/storage/__init__.py +++ b/gcloud/storage/__init__.py @@ -17,10 +17,7 @@ You'll typically use these to get started with the API: >>> import gcloud.storage ->>> bucket = gcloud.storage.get_bucket('bucket-id-here', - 'project-id', - 'long-email@googleapis.com', - '/path/to/private.key') +>>> bucket = gcloud.storage.get_bucket('bucket-id-here', 'project-id') >>> # Then do other things... >>> key = bucket.get_key('/remote/path/to/file.txt') >>> print key.get_contents_as_string() @@ -42,51 +39,44 @@ __version__ = '0.1' +from gcloud import credentials +from gcloud.storage.connection import Connection + + SCOPE = ('https://www.googleapis.com/auth/devstorage.full_control', 'https://www.googleapis.com/auth/devstorage.read_only', 'https://www.googleapis.com/auth/devstorage.read_write') -def get_connection(project, client_email, private_key_path): +def get_connection(project): """Shortcut method to establish a connection to Cloud Storage. Use this if you are going to access several buckets with the same set of credentials: >>> from gcloud import storage - >>> connection = storage.get_connection(project, email, key_path) + >>> connection = storage.get_connection(project) >>> bucket1 = connection.get_bucket('bucket1') >>> bucket2 = connection.get_bucket('bucket2') :type project: string :param project: The name of the project to connect to. - :type client_email: string - :param client_email: The e-mail attached to the service account. - - :type private_key_path: string - :param private_key_path: The path to a private key file (this file was - given to you when you created the service - account). - :rtype: :class:`gcloud.storage.connection.Connection` :returns: A connection defined with the proper credentials. """ - from gcloud import credentials - from gcloud.storage.connection import Connection + implicit_credentials = credentials.get_credentials() + scoped_credentials = implicit_credentials.create_scoped(SCOPE) + return Connection(project=project, credentials=scoped_credentials) - svc_account_credentials = credentials.get_for_service_account( - client_email, private_key_path, scope=SCOPE) - return Connection(project=project, credentials=svc_account_credentials) - -def get_bucket(bucket_name, project, client_email, private_key_path): +def get_bucket(bucket_name, project): """Shortcut method to establish a connection to a particular bucket. You'll generally use this as the first call to working with the API: >>> from gcloud import storage - >>> bucket = storage.get_bucket(project, bucket_name, email, key_path) + >>> bucket = storage.get_bucket(project, bucket_name) >>> # Now you can do things with the bucket. >>> bucket.exists('/path/to/file.txt') False @@ -98,16 +88,8 @@ def get_bucket(bucket_name, project, client_email, private_key_path): :type project: string :param project: The name of the project to connect to. - :type client_email: string - :param client_email: The e-mail attached to the service account. - - :type private_key_path: string - :param private_key_path: The path to a private key file (this file was - given to you when you created the service - account). - :rtype: :class:`gcloud.storage.bucket.Bucket` :returns: A bucket with a connection using the provided credentials. """ - connection = get_connection(project, client_email, private_key_path) + connection = get_connection(project) return connection.get_bucket(bucket_name) diff --git a/gcloud/storage/acl.py b/gcloud/storage/acl.py index e0af9af20b82..9197ca9ab6d8 100644 --- a/gcloud/storage/acl.py +++ b/gcloud/storage/acl.py @@ -19,7 +19,7 @@ :func:`gcloud.storage.bucket.Bucket.acl`:: >>> from gcloud import storage - >>> connection = storage.get_connection(project, email, key_path) + >>> connection = storage.get_connection(project) >>> bucket = connection.get_bucket(bucket_name) >>> acl = bucket.acl diff --git a/gcloud/storage/bucket.py b/gcloud/storage/bucket.py index 104ac884b983..cae8205d667e 100644 --- a/gcloud/storage/bucket.py +++ b/gcloud/storage/bucket.py @@ -150,7 +150,7 @@ def get_key(self, key): This will return None if the key doesn't exist:: >>> from gcloud import storage - >>> connection = storage.get_connection(project, email, key_path) + >>> connection = storage.get_connection(project) >>> bucket = connection.get_bucket('my-bucket') >>> print bucket.get_key('/path/to/key.txt') @@ -276,7 +276,7 @@ def delete_key(self, key): >>> from gcloud import storage >>> from gcloud.storage import exceptions - >>> connection = storage.get_connection(project, email, key_path) + >>> connection = storage.get_connection(project) >>> bucket = connection.get_bucket('my-bucket') >>> print bucket.get_all_keys() [] @@ -358,7 +358,7 @@ def upload_file(self, filename, key=None): For example:: >>> from gcloud import storage - >>> connection = storage.get_connection(project, email, key_path) + >>> connection = storage.get_connection(project) >>> bucket = connection.get_bucket('my-bucket') >>> bucket.upload_file('~/my-file.txt', 'remote-text-file.txt') >>> print bucket.get_all_keys() @@ -369,7 +369,7 @@ def upload_file(self, filename, key=None): path):: >>> from gcloud import storage - >>> connection = storage.get_connection(project, email, key_path) + >>> connection = storage.get_connection(project) >>> bucket = connection.get_bucket('my-bucket') >>> bucket.upload_file('~/my-file.txt') >>> print bucket.get_all_keys() @@ -398,7 +398,7 @@ def upload_file_object(self, file_obj, key=None): For example:: >>> from gcloud import storage - >>> connection = storage.get_connection(project, email, key_path) + >>> connection = storage.get_connection(project) >>> bucket = connection.get_bucket('my-bucket') >>> bucket.upload_file(open('~/my-file.txt'), 'remote-text-file.txt') >>> print bucket.get_all_keys() @@ -409,7 +409,7 @@ def upload_file_object(self, file_obj, key=None): path):: >>> from gcloud import storage - >>> connection = storage.get_connection(project, email, key_path) + >>> connection = storage.get_connection(project) >>> bucket = connection.get_bucket('my-bucket') >>> bucket.upload_file(open('~/my-file.txt')) >>> print bucket.get_all_keys() @@ -655,8 +655,7 @@ def configure_website(self, main_page_suffix=None, not_found_page=None): of an index page and a page to use when a key isn't found:: >>> from gcloud import storage - >>> connection = storage.get_connection(project, email, - private_key_path) + >>> connection = storage.get_connection(project) >>> bucket = connection.get_bucket(bucket_name) >>> bucket.configure_website('index.html', '404.html') diff --git a/gcloud/storage/connection.py b/gcloud/storage/connection.py index 4f1a447c26ed..5e0d52e8d240 100644 --- a/gcloud/storage/connection.py +++ b/gcloud/storage/connection.py @@ -24,6 +24,8 @@ from Crypto.PublicKey import RSA from Crypto.Signature import PKCS1_v1_5 from OpenSSL import crypto +from oauth2client import client +from oauth2client import service_account import pytz from gcloud.connection import Connection as _Base @@ -41,6 +43,71 @@ def _utcnow(): # pragma: NO COVER testing replaces return datetime.datetime.utcnow() +def _get_pem_key(credentials): + """Gets RSA key for a PEM payload from a credentials object. + + :type credentials: :class:`client.SignedJwtAssertionCredentials`, + :class:`service_account._ServiceAccountCredentials` + :param credentials: The credentials used to create an RSA key + for signing text. + + :rtype: :class:`Crypto.PublicKey.RSA._RSAobj` + :returns: An RSA object used to sign text. + :raises: `TypeError` if `credentials` is the wrong type. + """ + if isinstance(credentials, client.SignedJwtAssertionCredentials): + # Take our PKCS12 (.p12) key and make it into a RSA key we can use. + pkcs12 = crypto.load_pkcs12( + base64.b64decode(credentials.private_key), + 'notasecret') + pem_text = crypto.dump_privatekey( + crypto.FILETYPE_PEM, pkcs12.get_privatekey()) + elif isinstance(credentials, service_account._ServiceAccountCredentials): + pem_text = credentials._private_key_pkcs8_text + else: + raise TypeError((credentials, + 'not a valid service account credentials type')) + + return RSA.importKey(pem_text) + + +def _get_signed_query_params(credentials, expiration, signature_string): + """Gets query parameters for creating a signed URL. + + :type credentials: :class:`client.SignedJwtAssertionCredentials`, + :class:`service_account._ServiceAccountCredentials` + :param credentials: The credentials used to create an RSA key + for signing text. + + :type expiration: int or long + :param expiration: When the signed URL should expire. + + :type signature_string: string + :param signature_string: The string to be signed by the credentials. + + :rtype: dict + :returns: Query parameters matching the signing credentials with a + signed payload. + """ + pem_key = _get_pem_key(credentials) + # Sign the string with the RSA key. + signer = PKCS1_v1_5.new(pem_key) + signature_hash = SHA256.new(signature_string) + signature_bytes = signer.sign(signature_hash) + signature = base64.b64encode(signature_bytes) + + if isinstance(credentials, client.SignedJwtAssertionCredentials): + service_account_name = credentials.service_account_name + elif isinstance(credentials, service_account._ServiceAccountCredentials): + service_account_name = credentials._service_account_email + # We know one of the above must occur since `_get_pem_key` fails if not. + return { + 'GoogleAccessId': service_account_name, + 'Expires': str(expiration), + 'Signature': signature, + } + + class Connection(_Base): """A connection to Google Cloud Storage via the JSON REST API. @@ -57,7 +124,7 @@ class Connection(_Base): :class:`gcloud.storage.bucket.Bucket` objects:: >>> from gcloud import storage - >>> connection = storage.get_connection(project, email, key_path) + >>> connection = storage.get_connection(project) >>> bucket = connection.create_bucket('my-bucket-name') You can then delete this bucket:: @@ -276,7 +343,7 @@ def get_all_buckets(self): operations are identical:: >>> from gcloud import storage - >>> connection = storage.get_connection(project, email, key_path) + >>> connection = storage.get_connection(project) >>> for bucket in connection.get_all_buckets(): >>> print bucket >>> # ... is the same as ... @@ -301,7 +368,7 @@ def get_bucket(self, bucket_name): >>> from gcloud import storage >>> from gcloud.storage import exceptions - >>> connection = storage.get_connection(project, email, key_path) + >>> connection = storage.get_connection(project) >>> try: >>> bucket = connection.get_bucket('my-bucket') >>> except exceptions.NotFound: @@ -325,7 +392,7 @@ def lookup(self, bucket_name): than catching an exception:: >>> from gcloud import storage - >>> connection = storage.get_connection(project, email, key_path) + >>> connection = storage.get_connection(project) >>> bucket = connection.get_bucket('doesnt-exist') >>> print bucket None @@ -350,7 +417,7 @@ def create_bucket(self, bucket): For example:: >>> from gcloud import storage - >>> connection = storage.get_connection(project, client, key_path) + >>> connection = storage.get_connection(project) >>> bucket = connection.create_bucket('my-bucket') >>> print bucket @@ -375,7 +442,7 @@ def delete_bucket(self, bucket, force=False): a bucket object:: >>> from gcloud import storage - >>> connection = storage.get_connection(project, email, key_path) + >>> connection = storage.get_connection(project) >>> connection.delete_bucket('my-bucket') True @@ -479,26 +546,10 @@ def generate_signed_url(self, resource, expiration, str(expiration), resource]) - # Take our PKCS12 (.p12) key and make it into a RSA key we can use... - pkcs12 = crypto.load_pkcs12( - base64.b64decode(self.credentials.private_key), - 'notasecret') - pem = crypto.dump_privatekey( - crypto.FILETYPE_PEM, pkcs12.get_privatekey()) - pem_key = RSA.importKey(pem) - - # Sign the string with the RSA key. - signer = PKCS1_v1_5.new(pem_key) - signature_hash = SHA256.new(signature_string) - signature_bytes = signer.sign(signature_hash) - signature = base64.b64encode(signature_bytes) - # Set the right query parameters. - query_params = { - 'GoogleAccessId': self.credentials.service_account_name, - 'Expires': str(expiration), - 'Signature': signature, - } + query_params = _get_signed_query_params(self.credentials, + expiration, + signature_string) # Return the built URL. return '{endpoint}{resource}?{querystring}'.format( diff --git a/gcloud/storage/demo/__init__.py b/gcloud/storage/demo/__init__.py index 3c3470f403ba..c007b779e26b 100644 --- a/gcloud/storage/demo/__init__.py +++ b/gcloud/storage/demo/__init__.py @@ -12,16 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -# pragma NO COVER import os from gcloud import storage -__all__ = ['get_connection', 'CLIENT_EMAIL', 'KEY_FILENAME', 'PROJECT_ID'] +__all__ = ['get_connection', 'PROJECT_ID'] PROJECT_ID = os.getenv('GCLOUD_TESTS_PROJECT_ID') -CLIENT_EMAIL = os.getenv('GCLOUD_TESTS_CLIENT_EMAIL') -KEY_FILENAME = os.getenv('GCLOUD_TESTS_KEY_FILE') -def get_connection(): # pragma NO COVER. - return storage.get_connection(PROJECT_ID, CLIENT_EMAIL, KEY_FILENAME) +def get_connection(): + return storage.get_connection(PROJECT_ID) diff --git a/gcloud/storage/test___init__.py b/gcloud/storage/test___init__.py index 1197af631a27..90c8f6d5ff10 100644 --- a/gcloud/storage/test___init__.py +++ b/gcloud/storage/test___init__.py @@ -22,41 +22,27 @@ def _callFUT(self, *args, **kw): return get_connection(*args, **kw) def test_it(self): - from tempfile import NamedTemporaryFile from gcloud import credentials - from gcloud.storage import SCOPE from gcloud.storage.connection import Connection from gcloud.test_credentials import _Client from gcloud._testing import _Monkey PROJECT = 'project' - CLIENT_EMAIL = 'phred@example.com' - PRIVATE_KEY = 'SEEkR1t' client = _Client() with _Monkey(credentials, client=client): - with NamedTemporaryFile() as f: - f.write(PRIVATE_KEY) - f.flush() - found = self._callFUT(PROJECT, CLIENT_EMAIL, f.name) + found = self._callFUT(PROJECT) self.assertTrue(isinstance(found, Connection)) self.assertEqual(found.project, PROJECT) self.assertTrue(found._credentials is client._signed) - expected_called_with = { - 'service_account_name': CLIENT_EMAIL, - 'private_key': PRIVATE_KEY, - 'scope': SCOPE, - } - self.assertEqual(client._called_with, expected_called_with) + self.assertTrue(client._get_app_default_called) class Test_get_bucket(unittest2.TestCase): def _callFUT(self, *args, **kw): from gcloud.storage import get_bucket - return get_bucket(*args, **kw) def test_it(self): - from tempfile import NamedTemporaryFile from gcloud import storage from gcloud._testing import _Monkey @@ -65,24 +51,21 @@ def test_it(self): class _Connection(object): def get_bucket(self, bucket_name): - self._called_With = bucket_name + self._called_with = bucket_name return bucket + connection = _Connection() - _called_With = [] + _called_with = [] def get_connection(*args, **kw): - _called_With.append((args, kw)) + _called_with.append((args, kw)) return connection + BUCKET = 'bucket' PROJECT = 'project' - CLIENT_EMAIL = 'phred@example.com' - PRIVATE_KEY = 'SEEkR1t' with _Monkey(storage, get_connection=get_connection): - with NamedTemporaryFile() as f: - f.write(PRIVATE_KEY) - f.flush() - found = self._callFUT(BUCKET, PROJECT, CLIENT_EMAIL, f.name) + found = self._callFUT(BUCKET, PROJECT) + self.assertTrue(found is bucket) - self.assertEqual(_called_With, - [((PROJECT, CLIENT_EMAIL, f.name), {})]) - self.assertEqual(connection._called_With, BUCKET) + self.assertEqual(_called_with, [((PROJECT,), {})]) + self.assertEqual(connection._called_with, BUCKET) diff --git a/gcloud/storage/test_connection.py b/gcloud/storage/test_connection.py index 3655fdcb69aa..55b3bc6ae5d2 100644 --- a/gcloud/storage/test_connection.py +++ b/gcloud/storage/test_connection.py @@ -606,6 +606,7 @@ def test_generate_signed_url_w_expiration_int(self): import base64 import urlparse from gcloud._testing import _Monkey + from gcloud.test_credentials import _Credentials from gcloud.storage import connection as MUT ENDPOINT = 'http://api.example.com' @@ -619,8 +620,17 @@ def test_generate_signed_url_w_expiration_int(self): conn = self._makeOne(PROJECT, _Credentials()) conn.API_ACCESS_ENDPOINT = ENDPOINT + def _get_signed_query_params(*args): + credentials, expiration = args[:2] + return { + 'GoogleAccessId': credentials.service_account_name, + 'Expires': str(expiration), + 'Signature': SIGNED, + } + with _Monkey(MUT, crypto=crypto, RSA=rsa, PKCS1_v1_5=pkcs_v1_5, - SHA256=sha256): + SHA256=sha256, + _get_signed_query_params=_get_signed_query_params): url = conn.generate_signed_url(RESOURCE, 1000) scheme, netloc, path, qs, frag = urlparse.urlsplit(url) @@ -753,6 +763,135 @@ def test__get_expiration_seconds_w_timedelta_days(self): self.assertEqual(result, utc_seconds + 86400) +class Test__get_pem_key(unittest2.TestCase): + + def _callFUT(self, credentials): + from gcloud.storage.connection import _get_pem_key + return _get_pem_key(credentials) + + def test_bad_argument(self): + self.assertRaises(TypeError, self._callFUT, None) + + def test_signed_jwt_for_p12(self): + from oauth2client import client + from gcloud._testing import _Monkey + from gcloud.storage import connection as MUT + + scopes = [] + credentials = client.SignedJwtAssertionCredentials( + 'dummy_service_account_name', 'dummy_private_key_text', scopes) + crypto = _Crypto() + rsa = _RSA() + with _Monkey(MUT, crypto=crypto, RSA=rsa): + result = self._callFUT(credentials) + self.assertEqual(result, 'imported:__PEM__') + + def test_service_account_via_json_key(self): + from oauth2client import service_account + from gcloud._testing import _Monkey + from gcloud.storage import connection as MUT + + scopes = [] + + PRIVATE_TEXT = 'dummy_private_key_pkcs8_text' + + def _get_private_key(private_key_pkcs8_text): + return private_key_pkcs8_text + + with _Monkey(service_account, _get_private_key=_get_private_key): + credentials = service_account._ServiceAccountCredentials( + 'dummy_service_account_id', 'dummy_service_account_email', + 'dummy_private_key_id', PRIVATE_TEXT, scopes) + + rsa = _RSA() + with _Monkey(MUT, RSA=rsa): + result = self._callFUT(credentials) + + expected = 'imported:%s' % (PRIVATE_TEXT,) + self.assertEqual(result, expected) + + +class Test__get_signed_query_params(unittest2.TestCase): + + def _callFUT(self, credentials, expiration, signature_string): + from gcloud.storage.connection import _get_signed_query_params + return _get_signed_query_params(credentials, expiration, + signature_string) + + def test_wrong_type(self): + from gcloud._testing import _Monkey + from gcloud.storage import connection as MUT + + crypto = _Crypto() + pkcs_v1_5 = _PKCS1_v1_5() + rsa = _RSA() + sha256 = _SHA256() + + def _get_pem_key(credentials): + return credentials + + BAD_CREDENTIALS = None + EXPIRATION = '100' + SIGNATURE_STRING = 'dummy_signature' + with _Monkey(MUT, crypto=crypto, RSA=rsa, PKCS1_v1_5=pkcs_v1_5, + SHA256=sha256, _get_pem_key=_get_pem_key): + self.assertRaises(NameError, self._callFUT, + BAD_CREDENTIALS, EXPIRATION, SIGNATURE_STRING) + + def _run_test_with_credentials(self, credentials, account_name): + import base64 + from gcloud._testing import _Monkey + from gcloud.storage import connection as MUT + + crypto = _Crypto() + pkcs_v1_5 = _PKCS1_v1_5() + rsa = _RSA() + sha256 = _SHA256() + + EXPIRATION = '100' + SIGNATURE_STRING = 'dummy_signature' + with _Monkey(MUT, crypto=crypto, RSA=rsa, PKCS1_v1_5=pkcs_v1_5, + SHA256=sha256): + result = self._callFUT(credentials, EXPIRATION, SIGNATURE_STRING) + + self.assertEqual(sha256._signature_string, SIGNATURE_STRING) + SIGNED = base64.b64encode('DEADBEEF') + expected_query = { + 'Expires': EXPIRATION, + 'GoogleAccessId': account_name, + 'Signature': SIGNED, + } + self.assertEqual(result, expected_query) + + def test_signed_jwt_for_p12(self): + from oauth2client import client + + scopes = [] + ACCOUNT_NAME = 'dummy_service_account_name' + credentials = client.SignedJwtAssertionCredentials( + ACCOUNT_NAME, 'dummy_private_key_text', scopes) + self._run_test_with_credentials(credentials, ACCOUNT_NAME) + + def test_service_account_via_json_key(self): + from oauth2client import service_account + from gcloud._testing import _Monkey + + scopes = [] + + PRIVATE_TEXT = 'dummy_private_key_pkcs8_text' + + def _get_private_key(private_key_pkcs8_text): + return private_key_pkcs8_text + + ACCOUNT_NAME = 'dummy_service_account_email' + with _Monkey(service_account, _get_private_key=_get_private_key): + credentials = service_account._ServiceAccountCredentials( + 'dummy_service_account_id', ACCOUNT_NAME, + 'dummy_private_key_id', PRIVATE_TEXT, scopes) + + self._run_test_with_credentials(credentials, ACCOUNT_NAME) + + class Http(object): _called_with = None @@ -767,16 +906,6 @@ def request(self, **kw): return self._response, self._content -class _Credentials(object): - - service_account_name = 'testing@example.com' - - @property - def private_key(self): - import base64 - return base64.b64encode('SEEKRIT') - - class _Crypto(object): FILETYPE_PEM = 'pem' diff --git a/gcloud/test_credentials.py b/gcloud/test_credentials.py index 2133ff2b6547..50cb246c2e05 100644 --- a/gcloud/test_credentials.py +++ b/gcloud/test_credentials.py @@ -17,7 +17,7 @@ class TestCredentials(unittest2.TestCase): - def test_get_for_service_account_wo_scope(self): + def test_get_for_service_account_p12_wo_scope(self): from tempfile import NamedTemporaryFile from gcloud import credentials from gcloud._testing import _Monkey @@ -28,8 +28,8 @@ def test_get_for_service_account_wo_scope(self): with NamedTemporaryFile() as file_obj: file_obj.write(PRIVATE_KEY) file_obj.flush() - found = credentials.get_for_service_account(CLIENT_EMAIL, - file_obj.name) + found = credentials.get_for_service_account_p12( + CLIENT_EMAIL, file_obj.name) self.assertTrue(found is client._signed) expected_called_with = { 'service_account_name': CLIENT_EMAIL, @@ -38,7 +38,7 @@ def test_get_for_service_account_wo_scope(self): } self.assertEqual(client._called_with, expected_called_with) - def test_get_for_service_account_w_scope(self): + def test_get_for_service_account_p12_w_scope(self): from tempfile import NamedTemporaryFile from gcloud import credentials from gcloud._testing import _Monkey @@ -50,7 +50,7 @@ def test_get_for_service_account_w_scope(self): with NamedTemporaryFile() as file_obj: file_obj.write(PRIVATE_KEY) file_obj.flush() - found = credentials.get_for_service_account( + found = credentials.get_for_service_account_p12( CLIENT_EMAIL, file_obj.name, SCOPE) self.assertTrue(found is client._signed) expected_called_with = { @@ -61,9 +61,26 @@ def test_get_for_service_account_w_scope(self): self.assertEqual(client._called_with, expected_called_with) +class _Credentials(object): + + service_account_name = 'testing@example.com' + + def create_scoped(self, scopes): + self._scopes = scopes + return self + + class _Client(object): def __init__(self): - self._signed = object() + self._signed = _Credentials() + + class GoogleCredentials(object): + @staticmethod + def get_application_default(): + self._get_app_default_called = True + return self._signed + + self.GoogleCredentials = GoogleCredentials def SignedJwtAssertionCredentials(self, **kw): self._called_with = kw diff --git a/regression/app_credentials.json.sample b/regression/app_credentials.json.sample new file mode 100644 index 000000000000..b37488e19d3b --- /dev/null +++ b/regression/app_credentials.json.sample @@ -0,0 +1,7 @@ +{ + "client_id": "my-project", + "client_email": "some-account@developer.gserviceaccount.com", + "private_key": "-----BEGIN PRIVATE KEY-----\nSome Private Key Text\n-----END PRIVATE KEY-----\n", + "private_key_id": "your-key-id", + "type": "service_account" +} diff --git a/regression/key.json.enc b/regression/key.json.enc new file mode 100644 index 0000000000000000000000000000000000000000..d8cfe76029a6ed40c2ff596ac890723e1b75a54e GIT binary patch literal 1248 zcmV<61Rwj=&Th_l=Y-{S8-dS3{&hLD@ZTigx2xP>KHb|caDk5T2dN3+mbZujXKn4E zWWX_oysE~RZi@S1mjd6i=fhXG|5?FBwfU1`#0A%Vu-n0YEBS_VG&-3*L_jUBv<7AF zElX*8ixWSn;8D{Yt`fx6S!zlIf!M9@}ko$t$uynRGF8As<(sn;!htU;8|kL6oMfPnX-XrRR=;KGrj*e<*F(9#32%UInufg_p>x zFGL;MLRCDv0=NcG1kkCp1U;*QpPX_T$a{gtg6DjFzMXj}NDxu^kK+>$&hi2#{|ec1wBsUGh0U>(ZvKT?mT_J}#J4 z7t^%p54!MCiUOSwAt3h;qHNeHt|lv=>;+wPp$JaDf}8y-ng z;E^5HbY>69H9|;UaBq82;y&dLOFsnn!{J^A!bt|#>%#jrale_Gof@-{a|*PGac%z;X!?(1SAQJ6wViGuhSlw!qJQ`26L+PF0K&wpYX7rs+yOoeh8okDn zQbg#8Hy?;urJcif=;Nk<@F8bxJJI=u=2>Nd`_L*F0X!O+pfEis4b5|%dUrD}Xe6LX ztxA8ERY198S6S+o1t;iallWjM;1=_K(TCH~Ep=`rG|RV-R9}L0%{8GpuO|CMl9P*q zzNzwGK0jeB84DOTyv7-jymMv0Tzg*2jnl?al%xe-Ovdu!*Avd3~FQN1_(+otX%s?U% KNErv&+R>9b%x7Kz literal 0 HcmV?d00001 diff --git a/regression/key.p12.enc b/regression/key.p12.enc deleted file mode 100644 index af827b65625ab4b7039876de239c73dcf346f6e6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1744 zcmV;>1~2)@F_derNwnJ5*N_`^zmig$8?5}JLSe6i)PEHzgjrKUs?1+togFtl71fjS z#yeQ)Sj|UqB_1s6OEhuGQF1qya_mFt<`Vf5t6wvYulQ#_iOPqIY_MRza9b|3QgU-R z#GxxB=3*089p3~sOqjh9^eaR-FD+Jg7a>y;wZ68U`maZl`B0y$&J)kVmgWVZqxP(> zkie`pyuLX+$2gkKKUCNF9z*?bTgdX7=_(b7S{m>{<^Bb{$*a#TFE74YjM^2tQVpM7 z2QMl1EVk~Km4Cx1a={wL%td`!Oc5vnDrLm+-q;jS*Lt>&n#AfTAO>u^{bFsOC-TsN z_=n>C2D zs4Nc{^ZU!PuK(dTwWPx3OLvx6I|H0L!3I9!tjU)=&k*&J5^Oy2Xmy>90CSNnd)v# z7&c#0M?&V-4FE!3sIB=|373HVW`MIvDylw>C~>G_Fqs2rEzLzRhi*d<#q-YT=iwJ> zm63z711Kd+fS}a#NtfUlF$~<^kqnSanw!4Gn3x~(A_wT$W=RJ4Z%wm-IpxI))&*c+ zgc)OdE&$1~*);wS&=@i;OsdjaI4{0@L#J}Y4X*Ob_-N0)9irkWn+uO*a&srwGWYIA zW3uXa%{Pp!U=2kcqEfv%*oif9`0i~R>*rN0Suwo+l#?C^0H{urs*X%4PCE5{-S`|g zO{SMQiwG8N`f!coKE+Lc<5#2mbptCMe zQifQ)bcBgU*E7#m3sQ6z4|%~@ibD6K@bBMK(=?dVU1?h%wGZlxm37!7NAYFa#({t) zEYRpi2ClEPgs2EXcf>~OkcHlh-^`pP4W^ZQ4U-^8-9$HC^kP*HJdmWu(sOpSOhCy{ z8eER1*pFsvzH!E$r61#PvLIzb@~G!Atl~<~sM}AzPT(mwEADDHy_miKEe^NuL>{o1 zNpPe*y8UlLm$b-L)byOK@+i%V!xfl(q;$ZSV(dGcs(e*~$xExHs?sQnW+(gMuas?= zMyRzQ3WPgPn0N+Fni0STEo;7z;*vzq5Z>eV56zMw8X=enFFf`BgOGJ(%)g91sZK6c z@Wj#8y~P~{(8FoTRJM&x#DwpwALQ+(Q|n4$sP46x5+0>a)&ZuohIG_~sa`l_Hr`Fd^GJS* z#n_~J64axko;4;-55IZZF-p@&x zf_NI;r8#@8VIoWQ5K-{MZ(VCI(~Xc{l3l2rQ5MJ~k%RDXDnAb$jD`}&N1Zx-Qj1r} z9v1MnqJGx-GWtQxnL1b(#%PpRJyyoqeznp8D#FSp&;%C`r4`m~1K`s$i+Lfmi;d(!P@eVz7A8Tm9=%y*GSvHgbwhC(a}(%ny7` zQ}zS?c;fCg%Wm}1Fu3}L-G=Ko8cwd#ayp+Ew`#)XsX@`1z6k=6 zSg#TK9~fc}&mRS0`qsS2ehjwnEkkf+q_?0dw znp6bhk&c6d(2$JHX8&@LSrQn&Eyvl^vq~bA%SX%Q!5#%c;OUbe~zlE`}4UOU4HXEPN1r>Z@*6jqTutSNv!vpY#GFAVDa zL;{IhY~+C=GsWGV>6@gov@5upt1U#4sJ#FJOrwj5=Cu-!c}&A~*Meh; m6ZGM!_#*dc;8)60r;J)wr^|q^1an=yLT3Gw93|3kF4u*6HfK2i diff --git a/regression/local_test_setup.sample b/regression/local_test_setup.sample index afb399044b39..870f8f00dc19 100644 --- a/regression/local_test_setup.sample +++ b/regression/local_test_setup.sample @@ -1,4 +1,3 @@ +export GOOGLE_APPLICATION_CREDENTIALS="app_credentials.json.sample" export GCLOUD_TESTS_PROJECT_ID="my-project" export GCLOUD_TESTS_DATASET_ID=${GCLOUD_TESTS_PROJECT_ID} -export GCLOUD_TESTS_CLIENT_EMAIL="some-account@developer.gserviceaccount.com" -export GCLOUD_TESTS_KEY_FILE="path.key" diff --git a/regression/regression_utils.py b/regression/regression_utils.py index fb265b1a32cd..69eea2b1f932 100644 --- a/regression/regression_utils.py +++ b/regression/regression_utils.py @@ -20,11 +20,10 @@ from gcloud import storage -# Defaults from shell environ. May be None. +# From shell environ. May be None. PROJECT_ID = os.getenv('GCLOUD_TESTS_PROJECT_ID') DATASET_ID = os.getenv('GCLOUD_TESTS_DATASET_ID') -CLIENT_EMAIL = os.getenv('GCLOUD_TESTS_CLIENT_EMAIL') -KEY_FILENAME = os.getenv('GCLOUD_TESTS_KEY_FILE') +CREDENTIALS = os.getenv('GOOGLE_APPLICATION_CREDENTIALS') CACHED_RETURN_VALS = {} ENVIRON_ERROR_MSG = """\ @@ -35,40 +34,36 @@ def get_environ(require_datastore=False, require_storage=False): if require_datastore: - if DATASET_ID is None or CLIENT_EMAIL is None or KEY_FILENAME is None: + if DATASET_ID is None or not os.path.isfile(CREDENTIALS): print(ENVIRON_ERROR_MSG, file=sys.stderr) sys.exit(1) if require_storage: - if PROJECT_ID is None or CLIENT_EMAIL is None or KEY_FILENAME is None: + if PROJECT_ID is None or not os.path.isfile(CREDENTIALS): print(ENVIRON_ERROR_MSG, file=sys.stderr) sys.exit(1) return { 'project_id': PROJECT_ID, 'dataset_id': DATASET_ID, - 'client_email': CLIENT_EMAIL, - 'key_filename': KEY_FILENAME, } def get_dataset(): environ = get_environ(require_datastore=True) - get_dataset_args = (environ['dataset_id'], environ['client_email'], - environ['key_filename']) - key = ('get_dataset', get_dataset_args) + dataset_id = environ['dataset_id'] + key = ('get_dataset', dataset_id) if key not in CACHED_RETURN_VALS: # Cache return value for the environment. - CACHED_RETURN_VALS[key] = datastore.get_dataset(*get_dataset_args) + CACHED_RETURN_VALS[key] = datastore.get_dataset(dataset_id) return CACHED_RETURN_VALS[key] def get_storage_connection(): environ = get_environ(require_storage=True) - get_connection_args = (environ['project_id'], environ['client_email'], - environ['key_filename']) - key = ('get_storage_connection', get_connection_args) + project_id = environ['project_id'] + key = ('get_storage_connection', project_id) if key not in CACHED_RETURN_VALS: # Cache return value for the environment. - CACHED_RETURN_VALS[key] = storage.get_connection(*get_connection_args) + CACHED_RETURN_VALS[key] = storage.get_connection(project_id) return CACHED_RETURN_VALS[key] diff --git a/scripts/run_regression.sh b/scripts/run_regression.sh index 038b9f22e446..8d9e49a36ceb 100755 --- a/scripts/run_regression.sh +++ b/scripts/run_regression.sh @@ -26,8 +26,8 @@ if [[ "${TRAVIS}" == "true" ]]; then # Convert encrypted key file into decrypted file to be used. openssl aes-256-cbc -K $encrypted_a1b222e8c14d_key \ -iv $encrypted_a1b222e8c14d_iv \ - -in regression/key.p12.enc \ - -out $GCLOUD_TESTS_KEY_FILE -d + -in regression/key.json.enc \ + -out $GOOGLE_APPLICATION_CREDENTIALS -d else echo "Running in Travis during non-merge to master, doing nothing." exit From 1a1dac0281d10439715a9ba3a5d929e2824f7df9 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Tue, 16 Dec 2014 08:52:21 -0800 Subject: [PATCH 2/3] Removing superfluous patches in test_generate_signed_url_w_expiration_int. --- gcloud/storage/test_connection.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/gcloud/storage/test_connection.py b/gcloud/storage/test_connection.py index 55b3bc6ae5d2..03cfc9c1b565 100644 --- a/gcloud/storage/test_connection.py +++ b/gcloud/storage/test_connection.py @@ -613,10 +613,6 @@ def test_generate_signed_url_w_expiration_int(self): RESOURCE = '/name/key' PROJECT = 'project' SIGNED = base64.b64encode('DEADBEEF') - crypto = _Crypto() - rsa = _RSA() - pkcs_v1_5 = _PKCS1_v1_5() - sha256 = _SHA256() conn = self._makeOne(PROJECT, _Credentials()) conn.API_ACCESS_ENDPOINT = ENDPOINT @@ -628,9 +624,7 @@ def _get_signed_query_params(*args): 'Signature': SIGNED, } - with _Monkey(MUT, crypto=crypto, RSA=rsa, PKCS1_v1_5=pkcs_v1_5, - SHA256=sha256, - _get_signed_query_params=_get_signed_query_params): + with _Monkey(MUT, _get_signed_query_params=_get_signed_query_params): url = conn.generate_signed_url(RESOURCE, 1000) scheme, netloc, path, qs, frag = urlparse.urlsplit(url) From 78adc967f98cdd1c1f384b03031cf8827a3789a5 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Tue, 16 Dec 2014 11:58:19 -0800 Subject: [PATCH 3/3] Changing description of GOOGLE_APPLICATION_CREDENTIALS. --- CONTRIBUTING.rst | 2 +- docs/_components/datastore-quickstart.rst | 2 +- docs/_components/storage-quickstart.rst | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 86ad3daaa2c6..9d74e61450e4 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -164,7 +164,7 @@ Running Regression Tests bamboo-shift-455). - ``GCLOUD_TESTS_DATASET_ID``: The name of the dataset your tests connect to. This is typically the same as ``GCLOUD_TESTS_PROJECT_ID``. - - ``GOOGLE_APPLICATION_CREDENTIALS``: The path to an encrypted JSON file; + - ``GOOGLE_APPLICATION_CREDENTIALS``: The path to a JSON key file; see ``regression/app_credentials.json.sample`` as an example. Such a file can be downloaded directly from the developer's console by clicking "Generate new JSON key". See private key diff --git a/docs/_components/datastore-quickstart.rst b/docs/_components/datastore-quickstart.rst index 88129f2538a5..bb53731f515b 100644 --- a/docs/_components/datastore-quickstart.rst +++ b/docs/_components/datastore-quickstart.rst @@ -22,7 +22,7 @@ authentication to your project: bamboo-shift-455). - ``GCLOUD_TESTS_DATASET_ID``: The name of the dataset your tests connect to. This is typically the same as ``GCLOUD_TESTS_PROJECT_ID``. - - ``GOOGLE_APPLICATION_CREDENTIALS``: The path to an encrypted JSON file; + - ``GOOGLE_APPLICATION_CREDENTIALS``: The path to a JSON key file; see ``regression/app_credentials.json.sample`` as an example. Such a file can be downloaded directly from the developer's console by clicking "Generate new JSON key". See private key diff --git a/docs/_components/storage-quickstart.rst b/docs/_components/storage-quickstart.rst index aff52c32fbed..563055998d7c 100644 --- a/docs/_components/storage-quickstart.rst +++ b/docs/_components/storage-quickstart.rst @@ -22,7 +22,7 @@ authentication to your project: bamboo-shift-455). - ``GCLOUD_TESTS_DATASET_ID``: The name of the dataset your tests connect to. This is typically the same as ``GCLOUD_TESTS_PROJECT_ID``. - - ``GOOGLE_APPLICATION_CREDENTIALS``: The path to an encrypted JSON file; + - ``GOOGLE_APPLICATION_CREDENTIALS``: The path to a JSON key file; see ``regression/app_credentials.json.sample`` as an example. Such a file can be downloaded directly from the developer's console by clicking "Generate new JSON key". See private key