From ef96004d1d84111ca7416da8fcca180be27abbc9 Mon Sep 17 00:00:00 2001 From: Alexander Mohr Date: Fri, 20 Aug 2021 03:09:15 -0700 Subject: [PATCH] fix retries and refactor httpsession (#877) --- CHANGES.rst | 24 ++- README.rst | 32 ++-- aiobotocore/__init__.py | 5 +- aiobotocore/endpoint.py | 128 ++------------ aiobotocore/httpsession.py | 183 +++++++++++++++++++++ aiobotocore/waiter.py | 2 +- docs/index.rst | 9 +- docs/tutorial.rst | 2 +- examples/dynamodb_batch_write.py | 14 +- examples/dynamodb_create_table.py | 14 +- examples/simple.py | 9 +- examples/sqs_queue_consumer.py | 14 +- examples/sqs_queue_create.py | 14 +- examples/sqs_queue_producer.py | 4 +- setup.py | 1 + tests/botocore/test_signers.py | 1 - tests/conftest.py | 8 +- tests/test_basic_s3.py | 4 +- tests/test_patches.py | 4 + tests/test_session.py | 32 ++++ tests/{test-version.py => test_version.py} | 0 21 files changed, 296 insertions(+), 208 deletions(-) create mode 100644 aiobotocore/httpsession.py rename tests/{test-version.py => test_version.py} (100%) diff --git a/CHANGES.rst b/CHANGES.rst index 054f06ed..1c948ba9 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,38 +1,44 @@ Changes ------- +1.4.0 (2021-08-20) +^^^^^^^^^^^^^^^^^^ +* fix retries via config `#877 `_ +* remove AioSession and get_session top level names to match botocore_ +* change exceptions raised to match those of botocore_, see `mappings `_ + 1.3.3 (2021-07-12) ^^^^^^^^^^^^^^^^^^ -* fix AioJSONParser #872 +* fix AioJSONParser `#872 `_ 1.3.2 (2021-07-07) ^^^^^^^^^^^^^^^^^^ -* Bump to botocore 1.20.106 +* Bump to botocore_ to `1.20.106 `_ 1.3.1 (2021-06-11) ^^^^^^^^^^^^^^^^^^ * TCPConnector: change deprecated ssl_context to ssl -* fix non awaited generate presigned url calls #868 +* fix non awaited generate presigned url calls `#868 `_ 1.3.0 (2021-04-09) ^^^^^^^^^^^^^^^^^^ -* Bump to botocore 1.20.49 #856 +* Bump to botocore_ to `1.20.49 `_ `#856 `_ 1.2.2 (2021-03-11) ^^^^^^^^^^^^^^^^^^ -* Await call to async method _load_creds_via_assume_role #851 (thanks @puzza007) +* Await call to async method _load_creds_via_assume_role `#858 `_ (thanks `@puzza007 `_) 1.2.1 (2021-02-10) ^^^^^^^^^^^^^^^^^^ -* verify strings are now correctly passed to aiohttp.TCPConnector #851 (thanks @FHTMitchell) +* verify strings are now correctly passed to aiohttp.TCPConnector `#851 `_ (thanks `@FHTMitchell `_) 1.2.0 (2021-01-11) ^^^^^^^^^^^^^^^^^^ -* bump botocore to 1.19.52 -* use passed in http_session_cls param to create_client (#797) +* bump botocore to `1.19.52 `_ +* use passed in http_session_cls param to create_client `#797 `_ 1.1.2 (2020-10-07) ^^^^^^^^^^^^^^^^^^ -* fix AioPageIterator search method #831 (thanks @joseph-jones) +* fix AioPageIterator search method #831 (thanks `@joseph-jones `_) 1.1.1 (2020-08-31) ^^^^^^^^^^^^^^^^^^ diff --git a/README.rst b/README.rst index 6842cfa1..e40d9ffb 100644 --- a/README.rst +++ b/README.rst @@ -17,10 +17,7 @@ aiobotocore Async client for amazon services using botocore_ and aiohttp_/asyncio_. -Main purpose of this library to support amazon s3 api, but other services -should work (may be with minor fixes). For now we have tested -only upload/download api for s3, other users report that SQS and Dynamo -services work also. More tests coming soon. +This library is a mostly full featured asynchronous version of botocore. Install @@ -36,7 +33,7 @@ Basic Example .. code:: python import asyncio - import aiobotocore + from aiobotocore.session import get_session AWS_ACCESS_KEY_ID = "xxx" AWS_SECRET_ACCESS_KEY = "xxx" @@ -48,7 +45,7 @@ Basic Example folder = 'aiobotocore' key = '{}/{}'.format(folder, filename) - session = aiobotocore.get_session() + session = get_session() async with session.create_client('s3', region_name='us-west-2', aws_secret_access_key=AWS_SECRET_ACCESS_KEY, aws_access_key_id=AWS_ACCESS_KEY_ID) as client: @@ -90,36 +87,36 @@ Context Manager Examples .. code:: python from contextlib import AsyncExitStack - + from aiobotocore.session import AioSession - - + + # How to use in existing context manager class Manager: def __init__(self): self._exit_stack = AsyncExitStack() self._s3_client = None - + async def __aenter__(self): session = AioSession() self._s3_client = await self._exit_stack.enter_async_context(session.create_client('s3')) - + async def __aexit__(self, exc_type, exc_val, exc_tb): await self._exit_stack.__aexit__(exc_type, exc_val, exc_tb) - + # How to use with an external exit_stack async def create_s3_client(session: AioSession, exit_stack: AsyncExitStack): # Create client and add cleanup client = await exit_stack.enter_async_context(session.create_client('s3')) return client - - + + async def non_manager_example(): session = AioSession() - + async with AsyncExitStack() as exit_stack: s3_client = await create_s3_client(session, exit_stack) - + # do work with s3_client @@ -186,8 +183,7 @@ Requirements .. _Python: https://www.python.org .. _asyncio: https://docs.python.org/3/library/asyncio.html .. _botocore: https://github.com/boto/botocore -.. _aiohttp: https://github.com/KeepSafe/aiohttp - +.. _aiohttp: https://github.com/aio-libs/aiohttp awscli ------ diff --git a/aiobotocore/__init__.py b/aiobotocore/__init__.py index 5919ff40..96e3ce8d 100644 --- a/aiobotocore/__init__.py +++ b/aiobotocore/__init__.py @@ -1,4 +1 @@ -from .session import get_session, AioSession - -__all__ = ['get_session', 'AioSession'] -__version__ = '1.3.3' +__version__ = '1.4.0' diff --git a/aiobotocore/endpoint.py b/aiobotocore/endpoint.py index 02b0a4d9..9f2cfba6 100644 --- a/aiobotocore/endpoint.py +++ b/aiobotocore/endpoint.py @@ -1,23 +1,17 @@ import aiohttp import asyncio -import io -import pathlib -import os -import ssl -import sys + import aiohttp.http_exceptions -from aiohttp.client import URL from botocore.endpoint import EndpointCreator, Endpoint, DEFAULT_TIMEOUT, \ MAX_POOL_CONNECTIONS, logger, history_recorder, create_request_object from botocore.exceptions import ConnectionClosedError from botocore.hooks import first_non_none_response from botocore.utils import is_valid_endpoint_url -from multidict import MultiDict -from urllib.parse import urlparse from urllib3.response import HTTPHeaderDict + +from aiobotocore.httpsession import AIOHTTPSession from aiobotocore.response import StreamingBody -from aiobotocore._endpoint_helpers import _text, _IOBaseWrapper, \ - ClientResponseProxy +from aiobotocore._endpoint_helpers import ClientResponseProxy # noqa: F401, E501 lgtm [py/unused-import] async def convert_to_response_dict(http_response, operation_model): @@ -62,10 +56,6 @@ async def convert_to_response_dict(http_response, operation_model): class AioEndpoint(Endpoint): - def __init__(self, *args, proxies=None, **kwargs): - super().__init__(*args, **kwargs) - self.proxies = proxies or {} - async def create_request(self, params, operation_model=None): request = create_request_object(params) if operation_model: @@ -237,53 +227,15 @@ async def _needs_retry(self, attempts, operation_model, request_dict, return True async def _send(self, request): - # Note: When using aiobotocore with dynamodb, requests fail on crc32 - # checksum computation as soon as the response data reaches ~5KB. - # When AWS response is gzip compressed: - # 1. aiohttp is automatically decompressing the data - # (http://aiohttp.readthedocs.io/en/stable/client.html#binary-response-content) - # 2. botocore computes crc32 on the uncompressed data bytes and fails - # cause crc32 has been computed on the compressed data - # The following line forces aws not to use gzip compression, - # if there is a way to configure aiohttp not to perform decompression, - # we can remove the following line and take advantage of - # aws gzip compression. - # https://github.com/boto/botocore/issues/1255 - url = request.url - headers = request.headers - data = request.body - - headers['Accept-Encoding'] = 'identity' - headers_ = MultiDict( - (z[0], _text(z[1], encoding='utf-8')) for z in headers.items()) - - # botocore does this during the request so we do this here as well - # TODO: this should be part of the ClientSession, perhaps make wrapper - proxy = self.proxies.get(urlparse(url.lower()).scheme) - - if isinstance(data, io.IOBase): - data = _IOBaseWrapper(data) - - url = URL(url, encoded=True) - resp = await self.http_session.request( - request.method, url=url, headers=headers_, data=data, proxy=proxy) - - # If we're not streaming, read the content so we can retry any timeout - # errors, see: - # https://github.com/boto/botocore/blob/develop/botocore/vendored/requests/sessions.py#L604 - if not request.stream_output: - await resp.read() - - return resp + return await self.http_session.send(request) class AioEndpointCreator(EndpointCreator): - # TODO: handle socket_options def create_endpoint(self, service_model, region_name, endpoint_url, verify=None, response_parser_factory=None, timeout=DEFAULT_TIMEOUT, max_pool_connections=MAX_POOL_CONNECTIONS, - http_session_cls=aiohttp.ClientSession, + http_session_cls=AIOHTTPSession, proxies=None, socket_options=None, client_cert=None, @@ -297,68 +249,20 @@ def create_endpoint(self, service_model, region_name, endpoint_url, endpoint_prefix = service_model.endpoint_prefix logger.debug('Setting %s timeout as %s', endpoint_prefix, timeout) - - if isinstance(timeout, (list, tuple)): - conn_timeout, read_timeout = timeout - else: - conn_timeout = read_timeout = timeout - - if connector_args is None: - # AWS has a 20 second idle timeout: - # https://forums.aws.amazon.com/message.jspa?messageID=215367 - # aiohttp default timeout is 30s so set something reasonable here - connector_args = dict(keepalive_timeout=12) - - timeout = aiohttp.ClientTimeout( - sock_connect=conn_timeout, - sock_read=read_timeout - ) - - verify = self._get_verify_value(verify) - ssl_context = None - if client_cert: - if isinstance(client_cert, str): - key_file = None - cert_file = client_cert - elif isinstance(client_cert, tuple): - cert_file, key_file = client_cert - else: - raise TypeError("client_cert must be str or tuple, not %s" % - client_cert.__class__.__name__) - - ssl_context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) - ssl_context.load_cert_chain(cert_file, key_file) - elif isinstance(verify, (str, pathlib.Path)): - ssl_context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH, - cafile=str(verify)) - - if ssl_context: - # Enable logging of TLS session keys via defacto standard environment variable # noqa: E501 - # 'SSLKEYLOGFILE', if the feature is available (Python 3.8+). Skip empty values. # noqa: E501 - if hasattr(ssl_context, 'keylog_filename'): - keylogfile = os.environ.get('SSLKEYLOGFILE') - if keylogfile and not sys.flags.ignore_environment: - ssl_context.keylog_filename = keylogfile - - # TODO: add support for proxies_config - - connector = aiohttp.TCPConnector( - limit=max_pool_connections, - verify_ssl=bool(verify), - ssl=ssl_context, - **connector_args) - - aio_session = http_session_cls( - connector=connector, + http_session = http_session_cls( timeout=timeout, - skip_auto_headers={'CONTENT-TYPE'}, - response_class=ClientResponseProxy, - auto_decompress=False) + proxies=proxies, + verify=self._get_verify_value(verify), + max_pool_connections=max_pool_connections, + socket_options=socket_options, + client_cert=client_cert, + proxies_config=proxies_config, + connector_args=connector_args + ) return AioEndpoint( endpoint_url, endpoint_prefix=endpoint_prefix, event_emitter=self._event_emitter, response_parser_factory=response_parser_factory, - http_session=aio_session, - proxies=proxies) + http_session=http_session) diff --git a/aiobotocore/httpsession.py b/aiobotocore/httpsession.py new file mode 100644 index 00000000..21992b8f --- /dev/null +++ b/aiobotocore/httpsession.py @@ -0,0 +1,183 @@ +import asyncio +import io +import socket +from typing import Dict, Optional + +import aiohttp # lgtm [py/import-and-import-from] +from aiohttp import ClientSSLError, ClientConnectorError, ClientProxyConnectionError, \ + ClientHttpProxyError, ServerTimeoutError, ServerDisconnectedError +from aiohttp.client import URL +from multidict import MultiDict + +from botocore.httpsession import ProxyConfiguration, create_urllib3_context, \ + MAX_POOL_CONNECTIONS, InvalidProxiesConfigError, SSLError, \ + EndpointConnectionError, ProxyConnectionError, ConnectTimeoutError, \ + ConnectionClosedError, HTTPClientError, ReadTimeoutError, logger, get_cert_path + +from aiobotocore._endpoint_helpers import _text, _IOBaseWrapper, \ + ClientResponseProxy + + +class AIOHTTPSession: + def __init__( + self, + verify: bool = True, + proxies: Dict[str, str] = None, # {scheme: url} + timeout: float = None, + max_pool_connections: int = MAX_POOL_CONNECTIONS, + socket_options=None, + client_cert=None, + proxies_config=None, + connector_args=None + ): + # TODO: handle socket_options + + self._verify = verify + self._proxy_config = ProxyConfiguration(proxies=proxies, + proxies_settings=proxies_config) + + if isinstance(timeout, (list, tuple)): + conn_timeout, read_timeout = timeout + else: + conn_timeout = read_timeout = timeout + + timeout = aiohttp.ClientTimeout( + sock_connect=conn_timeout, + sock_read=read_timeout + ) + + self._cert_file = None + self._key_file = None + if isinstance(client_cert, str): + self._cert_file = client_cert + elif isinstance(client_cert, tuple): + self._cert_file, self._key_file = client_cert + + self._timeout = timeout + self._connector_args = connector_args + if self._connector_args is None: + # AWS has a 20 second idle timeout: + # https://forums.aws.amazon.com/message.jspa?messageID=215367 + # aiohttp default timeout is 30s so set something reasonable here + self._connector_args = dict(keepalive_timeout=12) + + self._max_pool_connections = max_pool_connections + self._socket_options = socket_options + if socket_options is None: + self._socket_options = [] + + # aiohttp handles 100 continue so we shouldn't need AWSHTTP[S]ConnectionPool + # it also pools by host so we don't need a manager, and can pass proxy via + # request so don't need proxy manager + + if proxies: + proxies_settings = self._proxy_config.settings + ssl_context = self._setup_proxy_ssl_context(proxies_settings) + # TODO: add support for + # proxies_settings.get('proxy_use_forwarding_for_https') + else: + ssl_context = self._get_ssl_context() + + # inline self._setup_ssl_cert + if bool(verify): + ca_certs = get_cert_path(verify) + if ca_certs: + ssl_context.load_verify_locations(ca_certs, None, None) + + self._connector = aiohttp.TCPConnector( + limit=max_pool_connections, + verify_ssl=bool(verify), + ssl=ssl_context, + **connector_args) + + self._session: Optional[aiohttp.ClientSession] = None + + async def __aenter__(self): + self._session = aiohttp.ClientSession( + connector=self._connector, + timeout=self._timeout, + skip_auto_headers={'CONTENT-TYPE'}, + response_class=ClientResponseProxy, + auto_decompress=False) + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + await self._session.__aexit__(exc_type, exc_val, exc_tb) + + def _get_ssl_context(self): + ssl_context = create_urllib3_context() + if self._cert_file: + ssl_context.load_cert_chain(self._cert_file, self._key_file) + return ssl_context + + def _setup_proxy_ssl_context(self, proxies_settings): + proxy_ca_bundle = proxies_settings.get('proxy_ca_bundle') + proxy_cert = proxies_settings.get('proxy_client_cert') + if proxy_ca_bundle is None and proxy_cert is None: + return None + + context = self._get_ssl_context() + try: + # urllib3 disables this by default but we need + # it for proper proxy tls negotiation. + context.check_hostname = True + if proxy_ca_bundle is not None: + context.load_verify_locations(cafile=proxy_ca_bundle) + + if isinstance(proxy_cert, tuple): + context.load_cert_chain(proxy_cert[0], keyfile=proxy_cert[1]) + elif isinstance(proxy_cert, str): + context.load_cert_chain(proxy_cert) + + return context + except IOError as e: + raise InvalidProxiesConfigError(error=e) + + async def send(self, request): + proxy_url = self._proxy_config.proxy_url_for(request.url) + + try: + url = request.url + headers = request.headers + data = request.body + + # https://github.com/boto/botocore/issues/1255 + headers['Accept-Encoding'] = 'identity' + + headers_ = MultiDict( + (z[0], _text(z[1], encoding='utf-8')) for z in headers.items()) + + if isinstance(data, io.IOBase): + data = _IOBaseWrapper(data) + + url = URL(url, encoded=True) + resp = await self._session.request( + request.method, url=url, headers=headers_, data=data, proxy=proxy_url) + + if not request.stream_output: + # Cause the raw stream to be exhausted immediately. We do it + # this way instead of using preload_content because + # preload_content will never buffer chunked responses + await resp.read() + + return resp + except ClientSSLError as e: + raise SSLError(endpoint_url=request.url, error=e) + except (ClientConnectorError, socket.gaierror) as e: + raise EndpointConnectionError(endpoint_url=request.url, error=e) + except (ClientProxyConnectionError, ClientHttpProxyError) as e: + raise ProxyConnectionError(proxy_url=proxy_url, error=e) + except ServerTimeoutError as e: + raise ConnectTimeoutError(endpoint_url=request.url, error=e) + except asyncio.TimeoutError as e: + raise ReadTimeoutError(endpoint_url=request.url, error=e) + except ServerDisconnectedError as e: + raise ConnectionClosedError( + error=e, + request=request, + endpoint_url=request.url + ) + except Exception as e: + message = 'Exception received when sending urllib3 HTTP request' + logger.debug(message, exc_info=True) + raise HTTPClientError(error=e) diff --git a/aiobotocore/waiter.py b/aiobotocore/waiter.py index 4e4d602c..e895cf8a 100644 --- a/aiobotocore/waiter.py +++ b/aiobotocore/waiter.py @@ -2,7 +2,7 @@ # WaiterModel is required for client.py import from botocore.exceptions import ClientError -from botocore.waiter import WaiterModel # noqa: F401, lgtm[py/unused-import] +from botocore.waiter import WaiterModel # noqa: F401, lgtm [py/unused-import] from botocore.waiter import Waiter, xform_name, logger, WaiterError, \ NormalizedOperationMethod as _NormalizedOperationMethod, is_valid_waiter_error from botocore.docs.docstring import WaiterDocstring diff --git a/docs/index.rst b/docs/index.rst index 19f73e45..13404cf0 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -14,10 +14,7 @@ aiobotocore's documentation! Async client for amazon services using botocore_ and aiohttp_/asyncio_. -Main purpose of this library to support amazon S3 API, but other services -should work (may be with minor fixes). For now we have tested -only upload/download API for S3, other users report that SQS and Dynamo -services work also. More tests coming soon. +This library is a mostly full featured asynchronous version of botocore. Features @@ -32,7 +29,7 @@ Basic Example .. code:: python import asyncio - import aiobotocore + from aiobotocore.session import get_session AWS_ACCESS_KEY_ID = "xxx" AWS_SECRET_ACCESS_KEY = "xxx" @@ -44,7 +41,7 @@ Basic Example folder = 'aiobotocore' key = f'{folder}/{filename}' - session = aiobotocore.get_session() + session = get_session() async with session.create_client('s3', region_name='us-west-2', aws_secret_access_key=AWS_SECRET_ACCESS_KEY, aws_access_key_id=AWS_ACCESS_KEY_ID) as client: diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 73ba5a7b..1a572b9d 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -30,7 +30,7 @@ Using Botocore The first step in using aiobotocore is to create a ``Session`` object. ``Session`` objects then allow you to create individual clients:: - session = aiobotocore.get_session() + session = aiobotocore.session.get_session() async with session.create_client('s3', region_name='us-west-2', aws_secret_access_key=AWS_SECRET_ACCESS_KEY, aws_access_key_id=AWS_ACCESS_KEY_ID) as client: diff --git a/examples/dynamodb_batch_write.py b/examples/dynamodb_batch_write.py index ac658d59..3cd444f9 100644 --- a/examples/dynamodb_batch_write.py +++ b/examples/dynamodb_batch_write.py @@ -1,7 +1,7 @@ # Boto should get credentials from ~/.aws/credentials or the environment import asyncio -import aiobotocore +from aiobotocore.session import get_session def get_items(start_num, num_items): @@ -43,7 +43,7 @@ def create_batch_write_structure(table_name, start_num, num_items): async def go(): - session = aiobotocore.get_session() + session = get_session() async with session.create_client('dynamodb', region_name='us-west-2') as client: table_name = 'test' @@ -97,13 +97,5 @@ async def go(): print(f'Response: {response["Item"]}') -def main(): - try: - loop = asyncio.get_event_loop() - loop.run_until_complete(go()) - except KeyboardInterrupt: - pass - - if __name__ == '__main__': - main() + asyncio.run(go()) diff --git a/examples/dynamodb_create_table.py b/examples/dynamodb_create_table.py index 78709765..1e34f17d 100644 --- a/examples/dynamodb_create_table.py +++ b/examples/dynamodb_create_table.py @@ -2,11 +2,11 @@ import uuid import asyncio -import aiobotocore +from aiobotocore.session import get_session async def go(): - session = aiobotocore.get_session() + session = get_session() async with session.create_client('dynamodb', region_name='us-west-2') as client: # Create random table name table_name = f'aiobotocore-{uuid.uuid4()}' @@ -38,13 +38,5 @@ async def go(): print(f"Table {table_name} created") -def main(): - try: - loop = asyncio.get_event_loop() - loop.run_until_complete(go()) - except KeyboardInterrupt: - pass - - if __name__ == '__main__': - main() + asyncio.run(go()) diff --git a/examples/simple.py b/examples/simple.py index 299213c7..977ed8d5 100644 --- a/examples/simple.py +++ b/examples/simple.py @@ -1,5 +1,6 @@ import asyncio -import aiobotocore +from aiobotocore.session import get_session + AWS_ACCESS_KEY_ID = "xxx" AWS_SECRET_ACCESS_KEY = "xxx" @@ -12,7 +13,7 @@ async def go(): folder = 'aiobotocore' key = f'{folder}/{filename}' - session = aiobotocore.get_session() + session = get_session() async with session.create_client( 's3', region_name='us-west-2', aws_secret_access_key=AWS_SECRET_ACCESS_KEY, @@ -33,5 +34,5 @@ async def go(): print(resp) -loop = asyncio.get_event_loop() -loop.run_until_complete(go()) +if __name__ == '__main__': + asyncio.run(go()) diff --git a/examples/sqs_queue_consumer.py b/examples/sqs_queue_consumer.py index fc7b1e66..3ca3dbee 100644 --- a/examples/sqs_queue_consumer.py +++ b/examples/sqs_queue_consumer.py @@ -5,7 +5,7 @@ import asyncio import sys -import aiobotocore +from aiobotocore.session import get_session import botocore.exceptions QUEUE_NAME = 'test_queue12' @@ -13,7 +13,7 @@ async def go(): # Boto should get credentials from ~/.aws/credentials or the environment - session = aiobotocore.get_session() + session = get_session() async with session.create_client('sqs', region_name='us-west-2') as client: try: response = await client.get_queue_url(QueueName=QUEUE_NAME) @@ -54,13 +54,5 @@ async def go(): print('Finished') -def main(): - try: - loop = asyncio.get_event_loop() - loop.run_until_complete(go()) - except KeyboardInterrupt: - pass - - if __name__ == '__main__': - main() + asyncio.run(go()) diff --git a/examples/sqs_queue_create.py b/examples/sqs_queue_create.py index 3bbaf371..aead4925 100644 --- a/examples/sqs_queue_create.py +++ b/examples/sqs_queue_create.py @@ -1,11 +1,11 @@ # Boto should get credentials from ~/.aws/credentials or the environment import asyncio -import aiobotocore +from aiobotocore.session import get_session async def go(): - session = aiobotocore.get_session() + session = get_session() async with session.create_client('sqs', region_name='us-west-2') as client: print('Creating test_queue1') @@ -24,13 +24,5 @@ async def go(): print('Done') -def main(): - try: - loop = asyncio.get_event_loop() - loop.run_until_complete(go()) - except KeyboardInterrupt: - pass - - if __name__ == '__main__': - main() + asyncio.run(go()) diff --git a/examples/sqs_queue_producer.py b/examples/sqs_queue_producer.py index a0ce5a1a..bf66204b 100644 --- a/examples/sqs_queue_producer.py +++ b/examples/sqs_queue_producer.py @@ -6,7 +6,7 @@ import random import sys -import aiobotocore +from aiobotocore.session import get_session import botocore.exceptions QUEUE_NAME = 'test_queue12' @@ -14,7 +14,7 @@ async def go(): # Boto should get credentials from ~/.aws/credentials or the environment - session = aiobotocore.get_session() + session = get_session() async with session.create_client('sqs', region_name='us-west-2') as client: try: response = await client.get_queue_url(QueueName=QUEUE_NAME) diff --git a/setup.py b/setup.py index 216db2bc..b5581e43 100644 --- a/setup.py +++ b/setup.py @@ -55,6 +55,7 @@ def read_version(): version=read_version(), description='Async client for aws services using botocore and aiohttp', long_description='\n\n'.join((read('README.rst'), read('CHANGES.rst'))), + long_description_content_type='text/x-rst', classifiers=classifiers, author="Nikolay Novik", author_email="nickolainovik@gmail.com", diff --git a/tests/botocore/test_signers.py b/tests/botocore/test_signers.py index c70641f8..db087d88 100644 --- a/tests/botocore/test_signers.py +++ b/tests/botocore/test_signers.py @@ -3,7 +3,6 @@ import pytest from unittest import mock -import aiobotocore import aiobotocore.credentials import aiobotocore.signers import botocore.auth diff --git a/tests/conftest.py b/tests/conftest.py index 89131474..22c5b532 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -76,15 +76,15 @@ async def assert_num_uploads_found( @pytest.fixture def aa_fail_proxy_config(monkeypatch): # NOTE: name of this fixture must be alphabetically first to run first - monkeypatch.setenv('HTTP_PROXY', 'http://{}:54321'.format(host)) - monkeypatch.setenv('HTTPS_PROXY', 'http://{}:54321'.format(host)) + monkeypatch.setenv('HTTP_PROXY', f'http://{host}:54321') + monkeypatch.setenv('HTTPS_PROXY', f'http://{host}:54321') @pytest.fixture def aa_succeed_proxy_config(monkeypatch): # NOTE: name of this fixture must be alphabetically first to run first - monkeypatch.setenv('HTTP_PROXY', 'http://{}:54321'.format(host)) - monkeypatch.setenv('HTTPS_PROXY', 'http://{}:54321'.format(host)) + monkeypatch.setenv('HTTP_PROXY', f'http://{host}:54321') + monkeypatch.setenv('HTTPS_PROXY', f'http://{host}:54321') # this will cause us to skip proxying monkeypatch.setenv('NO_PROXY', 'amazonaws.com') diff --git a/tests/test_basic_s3.py b/tests/test_basic_s3.py index 1fd055bd..2e37927c 100644 --- a/tests/test_basic_s3.py +++ b/tests/test_basic_s3.py @@ -2,8 +2,8 @@ from collections import defaultdict import pytest -import aiohttp import aioitertools +from botocore.exceptions import EndpointConnectionError async def fetch_all(pages): @@ -29,7 +29,7 @@ async def test_can_make_request(s3_client): async def test_fail_proxy_request(aa_fail_proxy_config, s3_client): # based on test_can_make_request - with pytest.raises(aiohttp.ClientConnectorError): + with pytest.raises(EndpointConnectionError): await s3_client.list_buckets() diff --git a/tests/test_patches.py b/tests/test_patches.py index 1f6ef7c4..c44edf80 100644 --- a/tests/test_patches.py +++ b/tests/test_patches.py @@ -39,6 +39,7 @@ create_credential_resolver, get_credentials, create_mfa_serial_refresher, \ AssumeRoleWithWebIdentityCredentialFetcher, SSOCredentialFetcher, SSOProvider from botocore.handlers import inject_presigned_url_ec2, inject_presigned_url_rds +from botocore.httpsession import URLLib3Session # This file ensures that our private patches will work going forward. If a @@ -303,6 +304,9 @@ # handlers.py inject_presigned_url_rds: {'5a34e1666d84f6229c54a59bffb69d46e8117b3a'}, inject_presigned_url_ec2: {'37fad2d9c53ca4f1783e32799fa8f70930f44c23'}, + + # httpsession.py + URLLib3Session: {'ecb7c86e8eb07c7f261493de3d08d221a39e8ef4'}, } diff --git a/tests/test_session.py b/tests/test_session.py index 84e536b2..a17ce69d 100644 --- a/tests/test_session.py +++ b/tests/test_session.py @@ -1,4 +1,10 @@ +import logging + import pytest +from botocore.exceptions import EndpointConnectionError + +from aiobotocore.session import AioSession +from aiobotocore.config import AioConfig @pytest.mark.moto @@ -14,3 +20,29 @@ def handler(**kwargs): await session.get_service_data('s3') assert handler_called + + +@pytest.mark.moto +@pytest.mark.asyncio +async def test_retry(session: AioSession, caplog): + caplog.set_level(logging.DEBUG) + + config = AioConfig( + connect_timeout=1, + read_timeout=1, + + # this goes through a slightly different codepath than regular retries + retries={ + "mode": "standard", + "total_max_attempts": 3, + }, + ) + + async with session.create_client( + 's3', config=config, aws_secret_access_key="xxx", aws_access_key_id="xxx", + endpoint_url='http://localhost:7878') as client: + + with pytest.raises(EndpointConnectionError): + await client.get_object(Bucket='foo', Key='bar') + + assert 'sleeping for' in caplog.text diff --git a/tests/test-version.py b/tests/test_version.py similarity index 100% rename from tests/test-version.py rename to tests/test_version.py