From b35c64f98a812438e77c596c012169805908f958 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Wed, 28 Jan 2015 16:04:41 -0800 Subject: [PATCH 1/3] Using httplib instead of httplib2 to get GCE metadata. Fixes #575. --- gcloud/datastore/_implicit_environ.py | 21 ++++---- gcloud/datastore/test___init__.py | 76 ++++++++++++++++----------- 2 files changed, 58 insertions(+), 39 deletions(-) diff --git a/gcloud/datastore/_implicit_environ.py b/gcloud/datastore/_implicit_environ.py index dab44fe782e2..2ee86d904603 100644 --- a/gcloud/datastore/_implicit_environ.py +++ b/gcloud/datastore/_implicit_environ.py @@ -18,7 +18,7 @@ imply the current dataset ID and connection from the enviroment. """ -import httplib2 +import httplib import socket try: @@ -64,17 +64,20 @@ def compute_engine_id(): :returns: Compute Engine project ID if the metadata service is available, else ``None``. """ - http = httplib2.Http(timeout=0.1) - uri = 'http://169.254.169.254/computeMetadata/v1/project/project-id' + host = '169.254.169.254' + uri_path = '/computeMetadata/v1/project/project-id' headers = {'Metadata-Flavor': 'Google'} + connection = httplib.HTTPConnection(host, timeout=0.1) - response = content = None + content = None try: - response, content = http.request(uri, method='GET', headers=headers) - except socket.timeout: + connection.request('GET', uri_path, headers=headers) + response = connection.getresponse() + if response.status == 200: + content = response.read() + except socket.error: # socket.timeout or socket.error(64, 'Host is down') pass - - if response is None or response['status'] != '200': - return None + finally: + connection.close() return content diff --git a/gcloud/datastore/test___init__.py b/gcloud/datastore/test___init__.py index 754a3399a7a6..16c6d726169e 100644 --- a/gcloud/datastore/test___init__.py +++ b/gcloud/datastore/test___init__.py @@ -109,37 +109,38 @@ def _implicit_compute_engine_helper(self, status): from gcloud.datastore import _implicit_environ COMPUTE_ENGINE_ID = 'GCE' - HTTPLIB2 = _Httplib2(COMPUTE_ENGINE_ID, status=status) - if status == '200': + HTTPLIB = _Httplib(COMPUTE_ENGINE_ID, status=status) + if status == 200: EXPECTED_ID = COMPUTE_ENGINE_ID else: EXPECTED_ID = None with self._monkey(None): - with _Monkey(_implicit_environ, httplib2=HTTPLIB2): + with _Monkey(_implicit_environ, httplib=HTTPLIB): self._callFUT() self.assertEqual(_implicit_environ.DATASET_ID, EXPECTED_ID) - self.assertEqual(len(HTTPLIB2._http_instances), 1) + self.assertEqual(len(HTTPLIB._http_connections), 1) - (timeout, http_instance), = HTTPLIB2._http_instances + (host, timeout, http_connection), = HTTPLIB._http_connections + self.assertEqual(host, '169.254.169.254') self.assertEqual(timeout, 0.1) self.assertEqual( - http_instance._called_uris, - ['http://169.254.169.254/computeMetadata/v1/project/project-id']) + http_connection._called_args, + [('GET', '/computeMetadata/v1/project/project-id')]) expected_kwargs = { - 'method': 'GET', 'headers': { 'Metadata-Flavor': 'Google', }, } - self.assertEqual(http_instance._called_kwargs, [expected_kwargs]) + self.assertEqual(http_connection._called_kwargs, [expected_kwargs]) + self.assertEqual(http_connection._close_count, 1) def test_set_implicit_from_compute_engine(self): - self._implicit_compute_engine_helper('200') + self._implicit_compute_engine_helper(200) def test_set_implicit_from_compute_engine_bad_status(self): - self._implicit_compute_engine_helper('404') + self._implicit_compute_engine_helper(404) def test_set_implicit_from_compute_engine_raise_timeout(self): self._implicit_compute_engine_helper('RAISE') @@ -150,15 +151,15 @@ def test_set_implicit_both_appengine_and_compute(self): APP_ENGINE_ID = 'GAE' APP_IDENTITY = _AppIdentity(APP_ENGINE_ID) - HTTPLIB2 = _Httplib2('GCE') + HTTPLIB = _Httplib('GCE') with self._monkey(None): with _Monkey(_implicit_environ, app_identity=APP_IDENTITY, - httplib2=HTTPLIB2): + httplib=HTTPLIB): self._callFUT() self.assertEqual(_implicit_environ.DATASET_ID, APP_ENGINE_ID) - self.assertEqual(len(HTTPLIB2._http_instances), 0) + self.assertEqual(len(HTTPLIB._http_connections), 0) def test_set_implicit_three_env_appengine_and_compute(self): from gcloud._testing import _Monkey @@ -166,15 +167,15 @@ def test_set_implicit_three_env_appengine_and_compute(self): IMPLICIT_DATASET_ID = 'IMPLICIT' APP_IDENTITY = _AppIdentity('GAE') - HTTPLIB2 = _Httplib2('GCE') + HTTPLIB = _Httplib('GCE') with self._monkey(IMPLICIT_DATASET_ID): with _Monkey(_implicit_environ, app_identity=APP_IDENTITY, - httplib2=HTTPLIB2): + httplib=HTTPLIB): self._callFUT() self.assertEqual(_implicit_environ.DATASET_ID, IMPLICIT_DATASET_ID) - self.assertEqual(len(HTTPLIB2._http_instances), 0) + self.assertEqual(len(HTTPLIB._http_connections), 0) class Test_set_default_connection(unittest2.TestCase): @@ -274,33 +275,48 @@ def get_application_id(self): return self.app_id -class _Http(object): +class _HTTPResponse(object): + + def __init__(self, status, data): + self.status = status + self.data = data + + def read(self): + return self.data + + +class _HTTPConnection(object): def __init__(self, parent): self.parent = parent - self._called_uris = [] + self._close_count = 0 + self._called_args = [] self._called_kwargs = [] - def request(self, uri, **kwargs): - import socket - - self._called_uris.append(uri) + def request(self, method, uri, **kwargs): + self._called_args.append((method, uri)) self._called_kwargs.append(kwargs) + def getresponse(self): + import socket + if self.parent.status == 'RAISE': raise socket.timeout('timed out') else: - return {'status': self.parent.status}, self.parent.project_id + return _HTTPResponse(self.parent.status, self.parent.project_id) + + def close(self): + self._close_count += 1 -class _Httplib2(object): +class _Httplib(object): - def __init__(self, project_id, status='200'): + def __init__(self, project_id, status=200): self.project_id = project_id self.status = status - self._http_instances = [] + self._http_connections = [] - def Http(self, timeout=None): - result = _Http(self) - self._http_instances.append((timeout, result)) + def HTTPConnection(self, host, timeout=None): + result = _HTTPConnection(self) + self._http_connections.append((host, timeout, result)) return result From 8dc13b522e0a0fbfa8c16e278720962b9d35bee5 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Wed, 28 Jan 2015 21:44:20 -0800 Subject: [PATCH 2/3] Adding default mock httplib in Test_set_default_dataset_id. Both - test_set_explicit_None_wo_env_var_set - test_no_env_var_set fall-through to the GCE check, but were actually making real HTTP calls before. This makes sure httplib is always stubbed out but also allows modification of that stub. --- gcloud/datastore/test___init__.py | 46 ++++++++++++++----------------- 1 file changed, 21 insertions(+), 25 deletions(-) diff --git a/gcloud/datastore/test___init__.py b/gcloud/datastore/test___init__.py index 16c6d726169e..852b5f82f5d4 100644 --- a/gcloud/datastore/test___init__.py +++ b/gcloud/datastore/test___init__.py @@ -30,12 +30,19 @@ def _callFUT(self, dataset_id=None): from gcloud.datastore import set_default_dataset_id return set_default_dataset_id(dataset_id=dataset_id) - def _monkey(self, implicit_dataset_id): + def _monkey(self, implicit_dataset_id, httplib=None, app_identity=None): + from contextlib import nested import os - from gcloud.datastore import _DATASET_ENV_VAR_NAME + from gcloud._testing import _Monkey + from gcloud.datastore import _DATASET_ENV_VAR_NAME + from gcloud.datastore import _implicit_environ + environ = {_DATASET_ENV_VAR_NAME: implicit_dataset_id} - return _Monkey(os, getenv=environ.get) + httplib = httplib or _Httplib(None, status=404) + return nested(_Monkey(os, getenv=environ.get), + _Monkey(_implicit_environ, httplib=httplib, + app_identity=app_identity)) def test_no_env_var_set(self): from gcloud.datastore import _implicit_environ @@ -79,33 +86,28 @@ def test_set_explicit_None_w_env_var_set(self): self.assertEqual(_implicit_environ.DATASET_ID, IMPLICIT_DATASET_ID) def test_set_implicit_from_appengine(self): - from gcloud._testing import _Monkey from gcloud.datastore import _implicit_environ APP_ENGINE_ID = 'GAE' APP_IDENTITY = _AppIdentity(APP_ENGINE_ID) - with self._monkey(None): - with _Monkey(_implicit_environ, app_identity=APP_IDENTITY): - self._callFUT() + with self._monkey(None, app_identity=APP_IDENTITY): + self._callFUT() self.assertEqual(_implicit_environ.DATASET_ID, APP_ENGINE_ID) def test_set_implicit_both_env_and_appengine(self): - from gcloud._testing import _Monkey from gcloud.datastore import _implicit_environ IMPLICIT_DATASET_ID = 'IMPLICIT' APP_IDENTITY = _AppIdentity('GAE') - with self._monkey(IMPLICIT_DATASET_ID): - with _Monkey(_implicit_environ, app_identity=APP_IDENTITY): - self._callFUT() + with self._monkey(IMPLICIT_DATASET_ID, app_identity=APP_IDENTITY): + self._callFUT() self.assertEqual(_implicit_environ.DATASET_ID, IMPLICIT_DATASET_ID) def _implicit_compute_engine_helper(self, status): - from gcloud._testing import _Monkey from gcloud.datastore import _implicit_environ COMPUTE_ENGINE_ID = 'GCE' @@ -115,9 +117,8 @@ def _implicit_compute_engine_helper(self, status): else: EXPECTED_ID = None - with self._monkey(None): - with _Monkey(_implicit_environ, httplib=HTTPLIB): - self._callFUT() + with self._monkey(None, httplib=HTTPLIB): + self._callFUT() self.assertEqual(_implicit_environ.DATASET_ID, EXPECTED_ID) self.assertEqual(len(HTTPLIB._http_connections), 1) @@ -146,33 +147,28 @@ def test_set_implicit_from_compute_engine_raise_timeout(self): self._implicit_compute_engine_helper('RAISE') def test_set_implicit_both_appengine_and_compute(self): - from gcloud._testing import _Monkey from gcloud.datastore import _implicit_environ APP_ENGINE_ID = 'GAE' APP_IDENTITY = _AppIdentity(APP_ENGINE_ID) HTTPLIB = _Httplib('GCE') - with self._monkey(None): - with _Monkey(_implicit_environ, app_identity=APP_IDENTITY, - httplib=HTTPLIB): - self._callFUT() + with self._monkey(None, httplib=HTTPLIB, app_identity=APP_IDENTITY): + self._callFUT() self.assertEqual(_implicit_environ.DATASET_ID, APP_ENGINE_ID) self.assertEqual(len(HTTPLIB._http_connections), 0) def test_set_implicit_three_env_appengine_and_compute(self): - from gcloud._testing import _Monkey from gcloud.datastore import _implicit_environ IMPLICIT_DATASET_ID = 'IMPLICIT' APP_IDENTITY = _AppIdentity('GAE') HTTPLIB = _Httplib('GCE') - with self._monkey(IMPLICIT_DATASET_ID): - with _Monkey(_implicit_environ, app_identity=APP_IDENTITY, - httplib=HTTPLIB): - self._callFUT() + with self._monkey(IMPLICIT_DATASET_ID, httplib=HTTPLIB, + app_identity=APP_IDENTITY): + self._callFUT() self.assertEqual(_implicit_environ.DATASET_ID, IMPLICIT_DATASET_ID) self.assertEqual(len(HTTPLIB._http_connections), 0) From 8febd04f764c94e707cc9b70b00fa5c8454bf03d Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Thu, 29 Jan 2015 13:33:06 -0800 Subject: [PATCH 3/3] Returning response value in compute_engine_id() instead of using var. --- gcloud/datastore/_implicit_environ.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/gcloud/datastore/_implicit_environ.py b/gcloud/datastore/_implicit_environ.py index 2ee86d904603..26e6b69382e5 100644 --- a/gcloud/datastore/_implicit_environ.py +++ b/gcloud/datastore/_implicit_environ.py @@ -69,15 +69,12 @@ def compute_engine_id(): headers = {'Metadata-Flavor': 'Google'} connection = httplib.HTTPConnection(host, timeout=0.1) - content = None try: connection.request('GET', uri_path, headers=headers) response = connection.getresponse() if response.status == 200: - content = response.read() + return response.read() except socket.error: # socket.timeout or socket.error(64, 'Host is down') pass finally: connection.close() - - return content