From a19f3e58f2a09f128bfac2a3678c26c4bd741910 Mon Sep 17 00:00:00 2001 From: Nikolay Novik Date: Tue, 7 Nov 2017 23:53:34 +0200 Subject: [PATCH 1/9] update changes --- CHANGES.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index d887a7c4..09f52fad 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,8 +1,9 @@ Changes ------- -0.4.6a0 (XXXX-XX-XX) +0.5.0 (XXXX-XX-XX) ^^^^^^^^^^^^^^^^^^ -* fix #190 +* Fix error botocore error checking #190 +* Update supported botocore requirement to: >=1.7.28, <=1.7.40 0.4.5 (2017-09-05) ^^^^^^^^^^^^^^^^^^ From 9a6ed0beea08b7bbdcd811b612b68cd1a5d5f0b9 Mon Sep 17 00:00:00 2001 From: Alexander Mohr Date: Mon, 17 Jul 2017 18:20:53 -0700 Subject: [PATCH 2/9] work --- aiobotocore/endpoint.py | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/aiobotocore/endpoint.py b/aiobotocore/endpoint.py index 543d57c9..feb51688 100644 --- a/aiobotocore/endpoint.py +++ b/aiobotocore/endpoint.py @@ -259,23 +259,11 @@ def __init__(self, host, conn_timeout=self._conn_timeout, skip_auto_headers={'CONTENT-TYPE'}, response_class=ClientResponseProxy, - loop=self._loop) + loop=self._loop, + auto_decompress=False) @asyncio.coroutine def _request(self, method, url, headers, data): - # Note: When using aiobotocore with dynamodb, requests fail on crc32 - # checksum computation as soon as the response data reaches ~5KB. - # When AWS response is gzip compressed: - # 1. aiohttp is automatically decompressing the data - # (http://aiohttp.readthedocs.io/en/stable/client.html#binary-response-content) - # 2. botocore computes crc32 on the uncompressed data bytes and fails - # cause crc32 has been computed on the compressed data - # The following line forces aws not to use gzip compression, - # if there is a way to configure aiohttp not to perform decompression, - # we can remove the following line and take advantage of - # aws gzip compression. - # See: https://github.com/aio-libs/aiohttp/issues/1992 - headers['Accept-Encoding'] = 'identity' headers_ = MultiDict( (z[0], text_(z[1], encoding='utf-8')) for z in headers.items()) From 839b73758f4d4f97fd6ee4d965e9e43fbd57ca0e Mon Sep 17 00:00:00 2001 From: Alexander Mohr Date: Mon, 17 Jul 2017 18:37:40 -0700 Subject: [PATCH 3/9] add placeholders --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index fb0b21db..a9dc2cfc 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ # If adding requirements make sure to also add to requirements-dev.txt install_requires = [ 'botocore>=1.7.28, <=1.7.40', - 'aiohttp>=2.0.4, <=2.3.0', + 'aiohttp>=2.3.0, <=2.3.2', 'multidict>=2.1.4', 'wrapt>=1.10.10', 'packaging>=16.8', From 5829e6f3554d94e1130d20a49fdd90134344d334 Mon Sep 17 00:00:00 2001 From: Alexander Mohr Date: Mon, 17 Jul 2017 23:22:45 -0700 Subject: [PATCH 4/9] add changes text --- CHANGES.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES.txt b/CHANGES.txt index 09f52fad..72a6acaa 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -4,6 +4,7 @@ Changes ^^^^^^^^^^^^^^^^^^ * Fix error botocore error checking #190 * Update supported botocore requirement to: >=1.7.28, <=1.7.40 +* Bump aiohttp requirement to support compressed responses correctly #298 0.4.5 (2017-09-05) ^^^^^^^^^^^^^^^^^^ From 2a27b965ee8190f23be8f91a9386e7ce20783999 Mon Sep 17 00:00:00 2001 From: Alexander Mohr Date: Tue, 1 Aug 2017 17:44:03 -0700 Subject: [PATCH 5/9] put this section back with updated comment --- aiobotocore/endpoint.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/aiobotocore/endpoint.py b/aiobotocore/endpoint.py index feb51688..01499d2d 100644 --- a/aiobotocore/endpoint.py +++ b/aiobotocore/endpoint.py @@ -264,6 +264,19 @@ def __init__(self, host, @asyncio.coroutine def _request(self, method, url, headers, data): + # botocore itself does not support compressed responses yet: + # https://github.com/boto/botocore/issues/1255. However aiohttp sets + # the accept headers for a variety of compressed encodings...so for + # example when using aiobotocore with dynamodb calls, requests fail on + # crc32 checksum computation as soon as the response ata reaches ~5KB. + # When AWS response is gzip compressed: + # 1. aiohttp is automatically decompressing the data + # (http://aiohttp.readthedocs.io/en/stable/client.html#binary-response-content) + # 2. botocore computes crc32 on the uncompressed data bytes and fails + # cause crc32 has been computed on the compressed data + # The following line forces aws not to use gzip compression. After + # botocore adds compressed response support we can remove this section. + # headers['Accept-Encoding'] = 'identity' headers_ = MultiDict( (z[0], text_(z[1], encoding='utf-8')) for z in headers.items()) From 7298c1265c3a77fa50d26472ed160b72a3e0acfd Mon Sep 17 00:00:00 2001 From: Alexander Mohr Date: Tue, 1 Aug 2017 17:45:53 -0700 Subject: [PATCH 6/9] put back --- aiobotocore/endpoint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aiobotocore/endpoint.py b/aiobotocore/endpoint.py index 01499d2d..ebdbe14c 100644 --- a/aiobotocore/endpoint.py +++ b/aiobotocore/endpoint.py @@ -276,7 +276,7 @@ def _request(self, method, url, headers, data): # cause crc32 has been computed on the compressed data # The following line forces aws not to use gzip compression. After # botocore adds compressed response support we can remove this section. - # headers['Accept-Encoding'] = 'identity' + headers['Accept-Encoding'] = 'identity' headers_ = MultiDict( (z[0], text_(z[1], encoding='utf-8')) for z in headers.items()) From 29ee0d444647b84fcaca6a7dddc8c76ccb34026d Mon Sep 17 00:00:00 2001 From: Nikolay Novik Date: Wed, 8 Nov 2017 00:26:34 +0200 Subject: [PATCH 7/9] update aiohttp --- aiobotocore/endpoint.py | 78 +++-------------------------------------- requirements-dev.txt | 2 +- tests/test_patches.py | 9 +++-- 3 files changed, 10 insertions(+), 79 deletions(-) diff --git a/aiobotocore/endpoint.py b/aiobotocore/endpoint.py index ebdbe14c..57fd6ff1 100644 --- a/aiobotocore/endpoint.py +++ b/aiobotocore/endpoint.py @@ -170,45 +170,6 @@ def read(self): return resp_msg, stream_reader -def _aiohttp_do_redirect(session, method, url, headers, data, resp): - # This is the redirect code from aiohttp, remove once - # https://github.com/aio-libs/aiobotocore/issues/267 is supported - - # For 301 and 302, mimic IE, now changed in RFC - # https://github.com/kennethreitz/requests/pull/269 - - if (resp.status == 303 and - resp.method != hdrs.METH_HEAD) \ - or (resp.status in (301, 302) and - resp.method == hdrs.METH_POST): - method = hdrs.METH_GET - data = None - if headers.get(hdrs.CONTENT_LENGTH): - headers.pop(hdrs.CONTENT_LENGTH) - - r_url = (resp.headers.get(hdrs.LOCATION) or - resp.headers.get(hdrs.URI)) - if r_url is None: - return None - - r_url = URL( - r_url, encoded=not session.requote_redirect_url) - - scheme = r_url.scheme - if scheme not in ('http', 'https', ''): - resp.close() - raise ValueError( - 'Can redirect only to http or https') - elif not scheme: - r_url = url.join(r_url) - - url = r_url - params = None - resp.release() - - return method, url, headers, params, data - - class AioEndpoint(Endpoint): def __init__(self, host, endpoint_prefix, event_emitter, proxies=None, verify=True, @@ -264,19 +225,6 @@ def __init__(self, host, @asyncio.coroutine def _request(self, method, url, headers, data): - # botocore itself does not support compressed responses yet: - # https://github.com/boto/botocore/issues/1255. However aiohttp sets - # the accept headers for a variety of compressed encodings...so for - # example when using aiobotocore with dynamodb calls, requests fail on - # crc32 checksum computation as soon as the response ata reaches ~5KB. - # When AWS response is gzip compressed: - # 1. aiohttp is automatically decompressing the data - # (http://aiohttp.readthedocs.io/en/stable/client.html#binary-response-content) - # 2. botocore computes crc32 on the uncompressed data bytes and fails - # cause crc32 has been computed on the compressed data - # The following line forces aws not to use gzip compression. After - # botocore adds compressed response support we can remove this section. - headers['Accept-Encoding'] = 'identity' headers_ = MultiDict( (z[0], text_(z[1], encoding='utf-8')) for z in headers.items()) @@ -287,27 +235,11 @@ def _request(self, method, url, headers, data): data = _IOBaseWrapper(data) url = URL(url, encoded=True) - - # See https://github.com/aio-libs/aiobotocore/issues/267 for details - for i in range(MAX_REDIRECTS): - resp = yield from self._aio_session.request(method, url=url, - headers=headers_, - data=data, - proxy=proxy, - timeout=None, - allow_redirects=False) - - if resp.status in {301, 302, 303, 307}: - redir_arr = _aiohttp_do_redirect(self._aio_session, method, - url, headers, data, resp) - - if redir_arr is None: - break - - method, url, headers, params, data = redir_arr - else: - break - + resp = yield from self._aio_session.request(method, url=url, + headers=headers_, + data=data, + proxy=proxy, + timeout=None) return resp @asyncio.coroutine diff --git a/requirements-dev.txt b/requirements-dev.txt index daed7cd7..0f47da3d 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,7 +6,7 @@ moto==0.4.31 pytest-cov==2.5.1 pytest==3.2.3 sphinx==1.6.5 -aiohttp==2.2.5 +aiohttp==2.3.2 botocore==1.7.40 multidict==3.3.2 wrapt==1.10.11 diff --git a/tests/test_patches.py b/tests/test_patches.py index 44bfb08d..c327c541 100644 --- a/tests/test_patches.py +++ b/tests/test_patches.py @@ -27,20 +27,19 @@ # github.com/aio-libs/aiobotocore/pull/248 _READ_TIMEOUT_DIGESTS = { # for our replacement of _factory - TCPConnector: {'a153acda1bfc891d01a56597f77b33afbd786d3d'}, + TCPConnector: {'1e6a3c14ce703781253da5cfdc7847b4ae898234'}, # for its inheritance to DataQueue - ResponseHandler: {'1cfdb033cb4b4d586bbebf49bed3c2930c026809'}, + ResponseHandler: {'5f11c28d0075e36dfec4705891f1c90248202ba4'}, # for our replacement of read() DataQueue: {'8ad4d5df1d016547daea6389707bc656630582e5'}, # for our patch of _wait - StreamReader: {'dc5a5a33e86cedf2d2c8413d951d2274a79303f4'}, + StreamReader: {'c0a9a31a8c3e550de5985ab642028983f709b37b'}, # for digging into _protocol ( 2.1.x, 2.2.x ) - ClientResponse: {'d1e0c16dea4fe3426caa1e9b0dc9f5f1992d838e', - 'bc374038ac3bfd7cc13dadb6aebbf0f67ebb7620'}, + ClientResponse: {'1dc0008e88b3f5ec2f59f6f5f03fae601f4a011d'}, } # These are guards to our main patches From 05f2db7eeff43bd6ef559610a4a3b4e4c0e06c89 Mon Sep 17 00:00:00 2001 From: Nikolay Novik Date: Wed, 8 Nov 2017 00:32:06 +0200 Subject: [PATCH 8/9] flake8 --- aiobotocore/endpoint.py | 1 - 1 file changed, 1 deletion(-) diff --git a/aiobotocore/endpoint.py b/aiobotocore/endpoint.py index 57fd6ff1..adb30abd 100644 --- a/aiobotocore/endpoint.py +++ b/aiobotocore/endpoint.py @@ -8,7 +8,6 @@ import aiohttp.http_exceptions from aiohttp.client_proto import ResponseHandler from aiohttp.helpers import CeilTimeout -from aiohttp import hdrs from aiohttp.client import URL from aiohttp.client_reqrep import ClientResponse from botocore.endpoint import EndpointCreator, Endpoint, DEFAULT_TIMEOUT, \ From 786a9a01123d7fcba759cb534d0d458bf1dc2f37 Mon Sep 17 00:00:00 2001 From: Nikolay Novik Date: Thu, 9 Nov 2017 23:40:47 +0200 Subject: [PATCH 9/9] update setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a9dc2cfc..fe13491c 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ # If adding requirements make sure to also add to requirements-dev.txt install_requires = [ 'botocore>=1.7.28, <=1.7.40', - 'aiohttp>=2.3.0, <=2.3.2', + 'aiohttp>=2.3.0, <3.0.0', 'multidict>=2.1.4', 'wrapt>=1.10.10', 'packaging>=16.8',