diff --git a/CHANGES.txt b/CHANGES.txt index d887a7c4..72a6acaa 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,8 +1,10 @@ Changes ------- -0.4.6a0 (XXXX-XX-XX) +0.5.0 (XXXX-XX-XX) ^^^^^^^^^^^^^^^^^^ -* fix #190 +* Fix error botocore error checking #190 +* Update supported botocore requirement to: >=1.7.28, <=1.7.40 +* Bump aiohttp requirement to support compressed responses correctly #298 0.4.5 (2017-09-05) ^^^^^^^^^^^^^^^^^^ diff --git a/aiobotocore/endpoint.py b/aiobotocore/endpoint.py index 543d57c9..adb30abd 100644 --- a/aiobotocore/endpoint.py +++ b/aiobotocore/endpoint.py @@ -8,7 +8,6 @@ import aiohttp.http_exceptions from aiohttp.client_proto import ResponseHandler from aiohttp.helpers import CeilTimeout -from aiohttp import hdrs from aiohttp.client import URL from aiohttp.client_reqrep import ClientResponse from botocore.endpoint import EndpointCreator, Endpoint, DEFAULT_TIMEOUT, \ @@ -170,45 +169,6 @@ def read(self): return resp_msg, stream_reader -def _aiohttp_do_redirect(session, method, url, headers, data, resp): - # This is the redirect code from aiohttp, remove once - # https://github.com/aio-libs/aiobotocore/issues/267 is supported - - # For 301 and 302, mimic IE, now changed in RFC - # https://github.com/kennethreitz/requests/pull/269 - - if (resp.status == 303 and - resp.method != hdrs.METH_HEAD) \ - or (resp.status in (301, 302) and - resp.method == hdrs.METH_POST): - method = hdrs.METH_GET - data = None - if headers.get(hdrs.CONTENT_LENGTH): - headers.pop(hdrs.CONTENT_LENGTH) - - r_url = (resp.headers.get(hdrs.LOCATION) or - resp.headers.get(hdrs.URI)) - if r_url is None: - return None - - r_url = URL( - r_url, encoded=not session.requote_redirect_url) - - scheme = r_url.scheme - if scheme not in ('http', 'https', ''): - resp.close() - raise ValueError( - 'Can redirect only to http or https') - elif not scheme: - r_url = url.join(r_url) - - url = r_url - params = None - resp.release() - - return method, url, headers, params, data - - class AioEndpoint(Endpoint): def __init__(self, host, endpoint_prefix, event_emitter, proxies=None, verify=True, @@ -259,23 +219,11 @@ def __init__(self, host, conn_timeout=self._conn_timeout, skip_auto_headers={'CONTENT-TYPE'}, response_class=ClientResponseProxy, - loop=self._loop) + loop=self._loop, + auto_decompress=False) @asyncio.coroutine def _request(self, method, url, headers, data): - # Note: When using aiobotocore with dynamodb, requests fail on crc32 - # checksum computation as soon as the response data reaches ~5KB. - # When AWS response is gzip compressed: - # 1. aiohttp is automatically decompressing the data - # (http://aiohttp.readthedocs.io/en/stable/client.html#binary-response-content) - # 2. botocore computes crc32 on the uncompressed data bytes and fails - # cause crc32 has been computed on the compressed data - # The following line forces aws not to use gzip compression, - # if there is a way to configure aiohttp not to perform decompression, - # we can remove the following line and take advantage of - # aws gzip compression. - # See: https://github.com/aio-libs/aiohttp/issues/1992 - headers['Accept-Encoding'] = 'identity' headers_ = MultiDict( (z[0], text_(z[1], encoding='utf-8')) for z in headers.items()) @@ -286,27 +234,11 @@ def _request(self, method, url, headers, data): data = _IOBaseWrapper(data) url = URL(url, encoded=True) - - # See https://github.com/aio-libs/aiobotocore/issues/267 for details - for i in range(MAX_REDIRECTS): - resp = yield from self._aio_session.request(method, url=url, - headers=headers_, - data=data, - proxy=proxy, - timeout=None, - allow_redirects=False) - - if resp.status in {301, 302, 303, 307}: - redir_arr = _aiohttp_do_redirect(self._aio_session, method, - url, headers, data, resp) - - if redir_arr is None: - break - - method, url, headers, params, data = redir_arr - else: - break - + resp = yield from self._aio_session.request(method, url=url, + headers=headers_, + data=data, + proxy=proxy, + timeout=None) return resp @asyncio.coroutine diff --git a/requirements-dev.txt b/requirements-dev.txt index e88d80aa..a3c6d80f 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,7 +6,7 @@ moto==0.4.31 pytest-cov==2.5.1 pytest==3.2.3 sphinx==1.6.5 -aiohttp==2.2.5 +aiohttp==2.3.2 botocore==1.7.43 multidict==3.3.2 wrapt==1.10.11 diff --git a/setup.py b/setup.py index fb0b21db..fe13491c 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ # If adding requirements make sure to also add to requirements-dev.txt install_requires = [ 'botocore>=1.7.28, <=1.7.40', - 'aiohttp>=2.0.4, <=2.3.0', + 'aiohttp>=2.3.0, <3.0.0', 'multidict>=2.1.4', 'wrapt>=1.10.10', 'packaging>=16.8', diff --git a/tests/test_patches.py b/tests/test_patches.py index 44bfb08d..c327c541 100644 --- a/tests/test_patches.py +++ b/tests/test_patches.py @@ -27,20 +27,19 @@ # github.com/aio-libs/aiobotocore/pull/248 _READ_TIMEOUT_DIGESTS = { # for our replacement of _factory - TCPConnector: {'a153acda1bfc891d01a56597f77b33afbd786d3d'}, + TCPConnector: {'1e6a3c14ce703781253da5cfdc7847b4ae898234'}, # for its inheritance to DataQueue - ResponseHandler: {'1cfdb033cb4b4d586bbebf49bed3c2930c026809'}, + ResponseHandler: {'5f11c28d0075e36dfec4705891f1c90248202ba4'}, # for our replacement of read() DataQueue: {'8ad4d5df1d016547daea6389707bc656630582e5'}, # for our patch of _wait - StreamReader: {'dc5a5a33e86cedf2d2c8413d951d2274a79303f4'}, + StreamReader: {'c0a9a31a8c3e550de5985ab642028983f709b37b'}, # for digging into _protocol ( 2.1.x, 2.2.x ) - ClientResponse: {'d1e0c16dea4fe3426caa1e9b0dc9f5f1992d838e', - 'bc374038ac3bfd7cc13dadb6aebbf0f67ebb7620'}, + ClientResponse: {'1dc0008e88b3f5ec2f59f6f5f03fae601f4a011d'}, } # These are guards to our main patches