From fbfeb1242ab1f378f6d4fe84b5d3dacdd8874189 Mon Sep 17 00:00:00 2001 From: Alexander Mohr Date: Thu, 25 Aug 2022 02:57:44 -0700 Subject: [PATCH] bump to 1.27.59 (#963) --- CHANGES.rst | 4 + Pipfile | 2 +- aiobotocore/__init__.py | 2 +- aiobotocore/client.py | 94 ++++++------ aiobotocore/credentials.py | 189 +++++++++++++++-------- aiobotocore/endpoint.py | 20 ++- aiobotocore/eventstream.py | 16 +- aiobotocore/httpsession.py | 27 ++-- aiobotocore/paginate.py | 28 ++-- aiobotocore/parsers.py | 34 ++-- aiobotocore/response.py | 2 +- aiobotocore/retries/adaptive.py | 2 +- aiobotocore/retries/standard.py | 4 +- aiobotocore/retryhandler.py | 2 +- aiobotocore/session.py | 86 +++++------ aiobotocore/signers.py | 138 ++++++++--------- aiobotocore/utils.py | 13 +- aiobotocore/waiter.py | 112 +++++++------- setup.py | 6 +- tests/boto_tests/test_credentials.py | 11 -- tests/mock_server.py | 2 +- tests/test_patches.py | 222 +++++++++++++-------------- 22 files changed, 537 insertions(+), 479 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 5650559b..f0a40597 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,5 +1,9 @@ Changes ------- +2.4.0 (2022-08-25) +^^^^^^^^^^^^^^^^^^ +* bump botocore to 1.27.59 + 2.3.4 (2022-06-23) ^^^^^^^^^^^^^^^^^^ * fix select_object_content diff --git a/Pipfile b/Pipfile index 837461b9..91456ff2 100644 --- a/Pipfile +++ b/Pipfile @@ -13,7 +13,7 @@ flake8-isort = "== 4.1.1" black = "== 22.6.0" isort = "== 5.10.1" docker = '==5.0.0' -moto = {extras = ["server","s3","sqs","lambda","dynamodb","cloudformation", "sns", "batch", "ec2", "rds"],version = "~=2.2.19"} +moto = {extras = ["server","s3","sqs","lambda","dynamodb","cloudformation", "sns", "batch", "ec2", "rds"],version = "~=4.0.0"} pytest = "==6.2.4" pytest-cov = "==2.11.1" pytest-asyncio = "==0.14.0" diff --git a/aiobotocore/__init__.py b/aiobotocore/__init__.py index 6ec85d61..ba9b9133 100644 --- a/aiobotocore/__init__.py +++ b/aiobotocore/__init__.py @@ -1 +1 @@ -__version__ = '2.3.4' +__version__ = '2.4.0' diff --git a/aiobotocore/client.py b/aiobotocore/client.py index b184b407..a0c8c344 100644 --- a/aiobotocore/client.py +++ b/aiobotocore/client.py @@ -69,6 +69,9 @@ async def create_client( ) service_client = cls(**client_args) self._register_retries(service_client) + self._register_eventbridge_events( + service_client, endpoint_bridge, endpoint_url + ) self._register_s3_events( service_client, endpoint_bridge, @@ -103,37 +106,6 @@ async def _create_client_class(self, service_name, service_model): cls = type(str(class_name), tuple(bases), class_attributes) return cls - def _register_endpoint_discovery(self, client, endpoint_url, config): - if endpoint_url is not None: - # Don't register any handlers in the case of a custom endpoint url - return - # Only attach handlers if the service supports discovery - if client.meta.service_model.endpoint_discovery_operation is None: - return - events = client.meta.events - service_id = client.meta.service_model.service_id.hyphenize() - enabled = False - if config and config.endpoint_discovery_enabled is not None: - enabled = config.endpoint_discovery_enabled - elif self._config_store: - enabled = self._config_store.get_config_variable( - 'endpoint_discovery_enabled' - ) - - enabled = self._normalize_endpoint_discovery_config(enabled) - if enabled and self._requires_endpoint_discovery(client, enabled): - discover = enabled is True - manager = AioEndpointDiscoveryManager( - client, always_discover=discover - ) - handler = AioEndpointDiscoveryHandler(manager) - handler.register(events, service_id) - else: - events.register( - 'before-parameter-build', - block_endpoint_discovery_required_operations, - ) - def _register_retries(self, client): # botocore retry handlers may block. We add our own implementation here. # botocore provides three implementations: @@ -206,9 +178,40 @@ def _register_legacy_retries(self, client): ) unique_id = 'retry-config-%s' % service_event_name client.meta.events.register( - 'needs-retry.%s' % service_event_name, handler, unique_id=unique_id + f"needs-retry.{service_event_name}", handler, unique_id=unique_id ) + def _register_endpoint_discovery(self, client, endpoint_url, config): + if endpoint_url is not None: + # Don't register any handlers in the case of a custom endpoint url + return + # Only attach handlers if the service supports discovery + if client.meta.service_model.endpoint_discovery_operation is None: + return + events = client.meta.events + service_id = client.meta.service_model.service_id.hyphenize() + enabled = False + if config and config.endpoint_discovery_enabled is not None: + enabled = config.endpoint_discovery_enabled + elif self._config_store: + enabled = self._config_store.get_config_variable( + 'endpoint_discovery_enabled' + ) + + enabled = self._normalize_endpoint_discovery_config(enabled) + if enabled and self._requires_endpoint_discovery(client, enabled): + discover = enabled is True + manager = AioEndpointDiscoveryManager( + client, always_discover=discover + ) + handler = AioEndpointDiscoveryHandler(manager) + handler.register(events, service_id) + else: + events.register( + 'before-parameter-build', + block_endpoint_discovery_required_operations, + ) + def _register_s3_events( self, client, @@ -290,6 +293,10 @@ def __getattr__(self, item): ) ) + async def close(self): + """Closes underlying endpoint connections.""" + await self._endpoint.close() + async def _make_api_call(self, operation_name, api_params): operation_model = self._service_model.operation_model(operation_name) service_name = self._service_model.service_name @@ -399,20 +406,15 @@ async def _emit_api_params(self, api_params, operation_model, context): # parameters or return a new set of parameters to use. service_id = self._service_model.service_id.hyphenize() responses = await self.meta.events.emit( - 'provide-client-params.{service_id}.{operation_name}'.format( - service_id=service_id, operation_name=operation_name - ), + f'provide-client-params.{service_id}.{operation_name}', params=api_params, model=operation_model, context=context, ) api_params = first_non_none_response(responses, default=api_params) - event_name = 'before-parameter-build.{service_id}.{operation_name}' await self.meta.events.emit( - event_name.format( - service_id=service_id, operation_name=operation_name - ), + f'before-parameter-build.{service_id}.{operation_name}', params=api_params, model=operation_model, context=context, @@ -462,11 +464,11 @@ def paginate(self, **kwargs): ) # Rename the paginator class based on the type of paginator. - paginator_class_name = str( - '{}.Paginator.{}'.format( - get_service_module_name(self.meta.service_model), - actual_operation_name, - ) + service_module_name = get_service_module_name( + self.meta.service_model + ) + paginator_class_name = ( + f"{service_module_name}.Paginator.{actual_operation_name}" ) # Create the new paginator class @@ -516,7 +518,3 @@ async def __aenter__(self): async def __aexit__(self, exc_type, exc_val, exc_tb): await self._endpoint.http_session.__aexit__(exc_type, exc_val, exc_tb) - - async def close(self): - """Close all http connections.""" - return await self._endpoint.http_session.close() diff --git a/aiobotocore/credentials.py b/aiobotocore/credentials.py index c20cd434..d498e204 100644 --- a/aiobotocore/credentials.py +++ b/aiobotocore/credentials.py @@ -2,16 +2,17 @@ import datetime import json import logging +import os import subprocess from copy import deepcopy from hashlib import sha1 -from typing import Optional import botocore.compat from botocore import UNSIGNED from botocore.compat import compat_shell_split from botocore.config import Config from botocore.credentials import ( + _DEFAULT_ADVISORY_REFRESH_TIMEOUT, AssumeRoleCredentialFetcher, AssumeRoleProvider, AssumeRoleWithWebIdentityProvider, @@ -19,36 +20,36 @@ BotoProvider, CachedCredentialFetcher, CanonicalNameCredentialSourcer, + ConfigNotFound, ConfigProvider, ContainerMetadataFetcher, ContainerProvider, CredentialResolver, + CredentialRetrievalError, Credentials, EnvProvider, InstanceMetadataProvider, + InvalidConfigError, + MetadataRetrievalError, OriginalEC2Provider, + PartialCredentialsError, ProcessProvider, ProfileProviderBuilder, ReadOnlyCredentials, RefreshableCredentials, + RefreshWithMFAUnsupportedError, SharedCredentialProvider, SSOProvider, + SSOTokenLoader, + UnauthorizedSSOTokenError, + UnknownCredentialError, _get_client_creator, _local_now, _parse_if_needed, _serialize_if_needed, + parse, resolve_imds_endpoint_mode, ) -from botocore.exceptions import ( - CredentialRetrievalError, - InvalidConfigError, - MetadataRetrievalError, - PartialCredentialsError, - RefreshWithMFAUnsupportedError, - UnauthorizedSSOTokenError, - UnknownCredentialError, -) -from botocore.utils import SSOTokenLoader from dateutil.tz import tzutc from aiobotocore.config import AioConfig @@ -78,6 +79,7 @@ def create_credential_resolver(session, cache=None, region_name=None): 'ec2_metadata_service_endpoint_mode': resolve_imds_endpoint_mode( session ), + 'ec2_credential_refresh_window': _DEFAULT_ADVISORY_REFRESH_TIMEOUT, } if cache is None: @@ -214,7 +216,7 @@ async def refresh(): return refresh -def create_aio_mfa_serial_refresher(actual_refresh): +def create_mfa_serial_refresher(actual_refresh): class _Refresher: def __init__(self, refresh): self._refresh = refresh @@ -232,41 +234,23 @@ async def call(self): return _Refresher(actual_refresh).call +# TODO: deprecate +create_aio_mfa_serial_refresher = create_mfa_serial_refresher + + class AioCredentials(Credentials): async def get_frozen_credentials(self): return ReadOnlyCredentials( self.access_key, self.secret_key, self.token ) - @classmethod - def from_credentials(cls, obj: Optional[Credentials]): - if obj is None: - return None - return cls(obj.access_key, obj.secret_key, obj.token, obj.method) - class AioRefreshableCredentials(RefreshableCredentials): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._refresh_lock = asyncio.Lock() - @classmethod - def from_refreshable_credentials( - cls, obj: Optional[RefreshableCredentials] - ): - if obj is None: - return None - return cls( # Using interval values here to skip property calling .refresh() - obj._access_key, - obj._secret_key, - obj._token, - obj._expiry_time, - obj._refresh_using, - obj.method, - obj._time_fetcher, - ) - - # Redeclaring the properties so it doesnt call refresh + # Redeclaring the properties so it doesn't call refresh # Have to redeclare setter as we're overriding the getter @property def access_key(self): @@ -529,8 +513,8 @@ async def _retrieve_credentials_using(self, credential_process): raise CredentialRetrievalError( provider=self.METHOD, error_msg=( - "Unsupported version '%s' for credential process " - "provider, supported versions: 1" % version + f"Unsupported version '{version}' for credential process " + f"provider, supported versions: 1" ), ) try: @@ -543,7 +527,7 @@ async def _retrieve_credentials_using(self, credential_process): except KeyError as e: raise CredentialRetrievalError( provider=self.METHOD, - error_msg="Missing required key in response: %s" % e, + error_msg=f"Missing required key in response: {e}", ) @@ -567,49 +551,124 @@ async def load(self): class AioEnvProvider(EnvProvider): async def load(self): - # It gets credentials from an env var, - # so just convert the response to Aio variants - result = super().load() - if isinstance(result, RefreshableCredentials): - return AioRefreshableCredentials.from_refreshable_credentials( - result - ) - elif isinstance(result, Credentials): - return AioCredentials.from_credentials(result) + access_key = self.environ.get(self._mapping['access_key'], '') + + if access_key: + logger.info('Found credentials in environment variables.') + fetcher = self._create_credentials_fetcher() + credentials = fetcher(require_expiry=False) + + expiry_time = credentials['expiry_time'] + if expiry_time is not None: + expiry_time = parse(expiry_time) + return AioRefreshableCredentials( + credentials['access_key'], + credentials['secret_key'], + credentials['token'], + expiry_time, + refresh_using=fetcher, + method=self.METHOD, + ) - return None + return AioCredentials( + credentials['access_key'], + credentials['secret_key'], + credentials['token'], + method=self.METHOD, + ) + else: + return None class AioOriginalEC2Provider(OriginalEC2Provider): async def load(self): - result = super().load() - if isinstance(result, Credentials): - result = AioCredentials.from_credentials(result) - return result + if 'AWS_CREDENTIAL_FILE' in self._environ: + full_path = os.path.expanduser( + self._environ['AWS_CREDENTIAL_FILE'] + ) + creds = self._parser(full_path) + if self.ACCESS_KEY in creds: + logger.info('Found credentials in AWS_CREDENTIAL_FILE.') + access_key = creds[self.ACCESS_KEY] + secret_key = creds[self.SECRET_KEY] + # EC2 creds file doesn't support session tokens. + return AioCredentials( + access_key, secret_key, method=self.METHOD + ) + else: + return None class AioSharedCredentialProvider(SharedCredentialProvider): async def load(self): - result = super().load() - if isinstance(result, Credentials): - result = AioCredentials.from_credentials(result) - return result + try: + available_creds = self._ini_parser(self._creds_filename) + except ConfigNotFound: + return None + if self._profile_name in available_creds: + config = available_creds[self._profile_name] + if self.ACCESS_KEY in config: + logger.info( + "Found credentials in shared credentials file: %s", + self._creds_filename, + ) + access_key, secret_key = self._extract_creds_from_mapping( + config, self.ACCESS_KEY, self.SECRET_KEY + ) + token = self._get_session_token(config) + return AioCredentials( + access_key, secret_key, token, method=self.METHOD + ) class AioConfigProvider(ConfigProvider): async def load(self): - result = super().load() - if isinstance(result, Credentials): - result = AioCredentials.from_credentials(result) - return result + try: + full_config = self._config_parser(self._config_filename) + except ConfigNotFound: + return None + if self._profile_name in full_config['profiles']: + profile_config = full_config['profiles'][self._profile_name] + if self.ACCESS_KEY in profile_config: + logger.info( + "Credentials found in config file: %s", + self._config_filename, + ) + access_key, secret_key = self._extract_creds_from_mapping( + profile_config, self.ACCESS_KEY, self.SECRET_KEY + ) + token = self._get_session_token(profile_config) + return AioCredentials( + access_key, secret_key, token, method=self.METHOD + ) + else: + return None class AioBotoProvider(BotoProvider): async def load(self): - result = super().load() - if isinstance(result, Credentials): - result = AioCredentials.from_credentials(result) - return result + if self.BOTO_CONFIG_ENV in self._environ: + potential_locations = [self._environ[self.BOTO_CONFIG_ENV]] + else: + potential_locations = self.DEFAULT_CONFIG_FILENAMES + for filename in potential_locations: + try: + config = self._ini_parser(filename) + except ConfigNotFound: + # Move on to the next potential config file name. + continue + if 'Credentials' in config: + credentials = config['Credentials'] + if self.ACCESS_KEY in credentials: + logger.info( + "Found credentials in boto config file: %s", filename + ) + access_key, secret_key = self._extract_creds_from_mapping( + credentials, self.ACCESS_KEY, self.SECRET_KEY + ) + return AioCredentials( + access_key, secret_key, method=self.METHOD + ) class AioAssumeRoleProvider(AssumeRoleProvider): @@ -653,7 +712,7 @@ async def _load_creds_via_assume_role(self, profile_name): ) refresher = fetcher.fetch_credentials if mfa_serial is not None: - refresher = create_aio_mfa_serial_refresher(refresher) + refresher = create_mfa_serial_refresher(refresher) # The initial credentials are empty and the expiration time is set # to now so that we can delay the call to assume role until it is diff --git a/aiobotocore/endpoint.py b/aiobotocore/endpoint.py index b7d03d9e..81965b54 100644 --- a/aiobotocore/endpoint.py +++ b/aiobotocore/endpoint.py @@ -65,6 +65,9 @@ async def convert_to_response_dict(http_response, operation_model): class AioEndpoint(Endpoint): + async def close(self): + await self.http_session.close() + async def create_request(self, params, operation_model=None): request = create_request_object(params) if operation_model: @@ -91,7 +94,6 @@ async def _send_request(self, request_dict, operation_model): context = request_dict['context'] self._update_retries_context(context, attempts) request = await self.create_request(request_dict, operation_model) - context = request_dict['context'] success_response, exception = await self._get_response( request, operation_model, context ) @@ -151,7 +153,7 @@ async def _get_response(self, request, operation_model, context): ) service_id = operation_model.service_model.service_id.hyphenize() await self._event_emitter.emit( - f'response-received.{service_id}.{operation_model.name}', + f"response-received.{service_id}.{operation_model.name}", **kwargs_to_emit, ) return success_response, exception @@ -170,9 +172,7 @@ async def _do_get_response(self, request, operation_model, context): }, ) service_id = operation_model.service_model.service_id.hyphenize() - event_name = 'before-send.{}.{}'.format( - service_id, operation_model.name - ) + event_name = f"before-send.{service_id}.{operation_model.name}" responses = await self._event_emitter.emit( event_name, request=request ) @@ -180,12 +180,12 @@ async def _do_get_response(self, request, operation_model, context): if http_response is None: http_response = await self._send(request) except HTTPClientError as e: - return None, e + return (None, e) except Exception as e: logger.debug( "Exception received when sending HTTP request.", exc_info=True ) - return None, e + return (None, e) # This returns the http_response and the parsed_data. response_dict = await convert_to_response_dict( @@ -258,9 +258,7 @@ async def _needs_retry( caught_exception=None, ): service_id = operation_model.service_model.service_id.hyphenize() - event_name = 'needs-retry.{}.{}'.format( - service_id, operation_model.name - ) + event_name = f"needs-retry.{service_id}.{operation_model.name}" responses = await self._event_emitter.emit( event_name, response=response, @@ -277,7 +275,7 @@ async def _needs_retry( # Request needs to be retried, and we need to sleep # for the specified number of times. logger.debug( - "Response received to retry, sleeping for " "%s seconds", + "Response received to retry, sleeping for %s seconds", handler_response, ) await asyncio.sleep(handler_response) diff --git a/aiobotocore/eventstream.py b/aiobotocore/eventstream.py index e1dd6950..7d11ac61 100644 --- a/aiobotocore/eventstream.py +++ b/aiobotocore/eventstream.py @@ -6,13 +6,6 @@ class AioEventStream(EventStream): - async def _create_raw_event_generator(self): - event_stream_buffer = EventStreamBuffer() - async for chunk, _ in self._raw_stream.content.iter_chunks(): - event_stream_buffer.add_data(chunk) - for event in event_stream_buffer: - yield event - def __iter__(self): raise NotImplementedError('Use async-for instead') @@ -25,6 +18,13 @@ async def __anext__(self): if parsed_event: yield parsed_event + async def _create_raw_event_generator(self): + event_stream_buffer = EventStreamBuffer() + async for chunk, _ in self._raw_stream.content.iter_chunks(): + event_stream_buffer.add_data(chunk) + for event in event_stream_buffer: + yield event # unfortunately no yield from async func support + async def get_initial_response(self): try: async for event in self._event_generator: @@ -36,3 +36,5 @@ async def get_initial_response(self): except StopIteration: pass raise NoInitialResponseError() + + # self._raw_stream.close() is sync so no override needed diff --git a/aiobotocore/httpsession.py b/aiobotocore/httpsession.py index aa18f1c5..b23d92f7 100644 --- a/aiobotocore/httpsession.py +++ b/aiobotocore/httpsession.py @@ -22,15 +22,18 @@ EndpointConnectionError, HTTPClientError, InvalidProxiesConfigError, + LocationParseError, ProxyConfiguration, ProxyConnectionError, ReadTimeoutError, SSLError, + _is_ipaddress, create_urllib3_context, ensure_boolean, get_cert_path, logger, mask_proxy_url, + parse_url, urlparse, ) from multidict import MultiDict @@ -132,16 +135,14 @@ async def __aexit__(self, exc_type, exc_val, exc_tb): self._session = None self._connector = None - async def close(self): - await self.__aexit__(None, None, None) - def _get_ssl_context(self): ssl_context = create_urllib3_context() if self._cert_file: ssl_context.load_cert_chain(self._cert_file, self._key_file) return ssl_context - def _setup_proxy_ssl_context(self, proxies_settings): + def _setup_proxy_ssl_context(self, proxy_url): + proxies_settings = self._proxy_config.settings proxy_ca_bundle = proxies_settings.get('proxy_ca_bundle') proxy_cert = proxies_settings.get('proxy_client_cert') if proxy_ca_bundle is None and proxy_cert is None: @@ -149,9 +150,11 @@ def _setup_proxy_ssl_context(self, proxies_settings): context = self._get_ssl_context() try: - # urllib3 disables this by default but we need - # it for proper proxy tls negotiation. - context.check_hostname = True + url = parse_url(proxy_url) + # urllib3 disables this by default but we need it for proper + # proxy tls negotiation when proxy_url is not an IP Address + if not _is_ipaddress(url.host): + context.check_hostname = True if proxy_ca_bundle is not None: context.load_verify_locations(cafile=proxy_ca_bundle) @@ -161,14 +164,16 @@ def _setup_proxy_ssl_context(self, proxies_settings): context.load_cert_chain(proxy_cert) return context - except OSError as e: + except (OSError, LocationParseError) as e: raise InvalidProxiesConfigError(error=e) - async def send(self, request): - proxy_url = self._proxy_config.proxy_url_for(request.url) - proxy_headers = self._proxy_config.proxy_headers_for(request.url) + async def close(self): + await self.__aexit__(None, None, None) + async def send(self, request): try: + proxy_url = self._proxy_config.proxy_url_for(request.url) + proxy_headers = self._proxy_config.proxy_headers_for(request.url) url = request.url headers = request.headers data = request.body diff --git a/aiobotocore/paginate.py b/aiobotocore/paginate.py index 461eb9d1..75d70ff6 100644 --- a/aiobotocore/paginate.py +++ b/aiobotocore/paginate.py @@ -83,13 +83,23 @@ async def __anext__(self): and previous_next_token == next_token ): message = ( - "The same next token was received " - "twice: %s" % next_token + f"The same next token was received " + f"twice: {next_token}" ) raise PaginationError(message=message) self._inject_token_into_kwargs(current_kwargs, next_token) previous_next_token = next_token + async def search(self, expression): + compiled = jmespath.compile(expression) + async for page in self: + results = compiled.search(page) + if isinstance(results, list): + for element in results: + yield element # unfortunately yield from not avail from async f + else: + yield results + def result_key_iters(self): teed_results = aioitertools.tee(self, len(self.result_keys)) return [ @@ -134,7 +144,7 @@ async def build_full_result(self): # Now both result_value and existing_value contain something if isinstance(result_value, list): existing_value.extend(result_value) - elif isinstance(result_value, (int, float, (str,))): + elif isinstance(result_value, (int, float, str)): # Modify the existing result with the sum or concatenation set_value_from_jmespath( complete_result, @@ -146,16 +156,6 @@ async def build_full_result(self): complete_result['NextToken'] = self.resume_token return complete_result - async def search(self, expression): - compiled = jmespath.compile(expression) - async for page in self: - results = compiled.search(page) - if isinstance(results, list): - for element in results: - yield element - else: - yield results - class AioPaginator(Paginator): PAGE_ITERATOR_CLS = AioPageIterator @@ -188,4 +188,4 @@ async def __anext__(self): if results is None: results = [] for result in results: - yield result + yield result # yield from not avail from async func diff --git a/aiobotocore/parsers.py b/aiobotocore/parsers.py index 6925df04..6169bf38 100644 --- a/aiobotocore/parsers.py +++ b/aiobotocore/parsers.py @@ -14,21 +14,24 @@ from .eventstream import AioEventStream -class AioRestXMLParser(RestXMLParser): - def _create_event_stream(self, response, shape): - parser = self._event_stream_parser - name = response['context'].get('operation_name') - return AioEventStream(response['body'], shape, parser, name) +class AioResponseParserFactory(ResponseParserFactory): + def create_parser(self, protocol_name): + parser_cls = PROTOCOL_PARSERS[protocol_name] + return parser_cls(**self._defaults) -class AioEC2QueryParser(EC2QueryParser): +def create_parser(protocol): + return AioResponseParserFactory().create_parser(protocol) + + +class AioQueryParser(QueryParser): def _create_event_stream(self, response, shape): parser = self._event_stream_parser name = response['context'].get('operation_name') return AioEventStream(response['body'], shape, parser, name) -class AioQueryParser(QueryParser): +class AioEC2QueryParser(EC2QueryParser): def _create_event_stream(self, response, shape): parser = self._event_stream_parser name = response['context'].get('operation_name') @@ -109,6 +112,13 @@ def _create_event_stream(self, response, shape): return AioEventStream(response['body'], shape, parser, name) +class AioRestXMLParser(RestXMLParser): + def _create_event_stream(self, response, shape): + parser = self._event_stream_parser + name = response['context'].get('operation_name') + return AioEventStream(response['body'], shape, parser, name) + + PROTOCOL_PARSERS = { 'ec2': AioEC2QueryParser, 'query': AioQueryParser, @@ -116,13 +126,3 @@ def _create_event_stream(self, response, shape): 'rest-json': AioRestJSONParser, 'rest-xml': AioRestXMLParser, } - - -class AioResponseParserFactory(ResponseParserFactory): - def create_parser(self, protocol_name): - parser_cls = PROTOCOL_PARSERS[protocol_name] - return parser_cls(**self._defaults) - - -def create_parser(protocol): - return AioResponseParserFactory().create_parser(protocol) diff --git a/aiobotocore/response.py b/aiobotocore/response.py index e2c24d38..7046a464 100644 --- a/aiobotocore/response.py +++ b/aiobotocore/response.py @@ -118,7 +118,7 @@ async def iter_chunks(self, chunk_size=_DEFAULT_CHUNK_SIZE): def _verify_content_length(self): # See: https://github.com/kennethreitz/requests/issues/1855 # Basically, our http library doesn't do this for us, so we have - # to do this ourself. + # to do this our self. if ( self._self_content_length is not None and self._self_amount_read != int(self._self_content_length) diff --git a/aiobotocore/retries/adaptive.py b/aiobotocore/retries/adaptive.py index 6d59aa62..45f2e130 100644 --- a/aiobotocore/retries/adaptive.py +++ b/aiobotocore/retries/adaptive.py @@ -47,7 +47,7 @@ class AsyncClientRateLimiter: # Most of the code here comes directly from botocore. The main change is making the # callbacks async. - # This doesn't inherits from the botocore ClientRateLimiter for two reasons: + # This doesn't inherit from the botocore ClientRateLimiter for two reasons: # * the interface is slightly changed (methods are now async) # * we rewrote the entirety of the class anyway diff --git a/aiobotocore/retries/standard.py b/aiobotocore/retries/standard.py index 1e4d82ae..ad56d7bf 100644 --- a/aiobotocore/retries/standard.py +++ b/aiobotocore/retries/standard.py @@ -26,7 +26,7 @@ def register_retry_handler(client, max_attempts=DEFAULT_MAX_ATTEMPTS): service_id = client.meta.service_model.service_id service_event_name = service_id.hyphenize() client.meta.events.register( - 'after-call.%s' % service_event_name, retry_quota.release_retry_quota + f'after-call.{service_event_name}', retry_quota.release_retry_quota ) handler = AioRetryHandler( @@ -60,7 +60,7 @@ async def needs_retry(self, **kwargs): if self._retry_quota.acquire_retry_quota(context): retry_delay = self._retry_policy.compute_retry_delay(context) logger.debug( - "Retry needed, retrying request after " "delay of: %s", + "Retry needed, retrying request after delay of: %s", retry_delay, ) else: diff --git a/aiobotocore/retryhandler.py b/aiobotocore/retryhandler.py index c8f6cce1..df7cbb18 100644 --- a/aiobotocore/retryhandler.py +++ b/aiobotocore/retryhandler.py @@ -134,7 +134,7 @@ async def _call( 'MaxAttemptsReached' ] = True logger.debug( - "Reached the maximum number of retry " "attempts: %s", + "Reached the maximum number of retry attempts: %s", attempt_number, ) return False diff --git a/aiobotocore/session.py b/aiobotocore/session.py index 0bb008dc..dfb88a7a 100644 --- a/aiobotocore/session.py +++ b/aiobotocore/session.py @@ -47,9 +47,9 @@ def __init__( session_vars, event_hooks, include_builtin_handlers, profile ) - def _register_response_parser_factory(self): - self._components.register_component( - 'response_parser_factory', AioResponseParserFactory() + def _create_credential_resolver(self): + return create_credential_resolver( + self, region_name=self._last_client_region_used ) def _register_smart_defaults_factory(self): @@ -66,6 +66,46 @@ def create_smart_defaults_factory(): 'smart_defaults_factory', create_smart_defaults_factory ) + def _register_response_parser_factory(self): + self._components.register_component( + 'response_parser_factory', AioResponseParserFactory() + ) + + def set_credentials(self, access_key, secret_key, token=None): + self._credentials = AioCredentials(access_key, secret_key, token) + + async def get_credentials(self): + if self._credentials is None: + self._credentials = await ( + self._components.get_component( + 'credential_provider' + ).load_credentials() + ) + return self._credentials + + async def get_service_model(self, service_name, api_version=None): + service_description = await self.get_service_data( + service_name, api_version + ) + return ServiceModel(service_description, service_name=service_name) + + async def get_service_data(self, service_name, api_version=None): + """ + Retrieve the fully merged data associated with a service. + """ + data_path = service_name + service_data = self.get_component('data_loader').load_service_model( + data_path, type_name='service-2', api_version=api_version + ) + service_id = EVENT_ALIASES.get(service_name, service_name) + await self._events.emit( + 'service-data-loaded.%s' % service_id, + service_data=service_data, + service_name=service_name, + session=self, + ) + return service_data + def create_client(self, *args, **kwargs): return ClientCreatorContext(self._create_client(*args, **kwargs)) @@ -166,46 +206,6 @@ async def _create_client( monitor.register(client.meta.events) return client - def _create_credential_resolver(self): - return create_credential_resolver( - self, region_name=self._last_client_region_used - ) - - async def get_credentials(self): - if self._credentials is None: - self._credentials = await ( - self._components.get_component( - 'credential_provider' - ).load_credentials() - ) - return self._credentials - - def set_credentials(self, access_key, secret_key, token=None): - self._credentials = AioCredentials(access_key, secret_key, token) - - async def get_service_model(self, service_name, api_version=None): - service_description = await self.get_service_data( - service_name, api_version - ) - return ServiceModel(service_description, service_name=service_name) - - async def get_service_data(self, service_name, api_version=None): - """ - Retrieve the fully merged data associated with a service. - """ - data_path = service_name - service_data = self.get_component('data_loader').load_service_model( - data_path, type_name='service-2', api_version=api_version - ) - service_id = EVENT_ALIASES.get(service_name, service_name) - await self._events.emit( - 'service-data-loaded.%s' % service_id, - service_data=service_data, - service_name=service_name, - session=self, - ) - return service_data - async def get_available_regions( self, service_name, partition_name='aws', allow_non_regional=False ): diff --git a/aiobotocore/signers.py b/aiobotocore/signers.py index 3afaacf7..49c15f93 100644 --- a/aiobotocore/signers.py +++ b/aiobotocore/signers.py @@ -80,35 +80,6 @@ async def sign( auth.add_auth(request) - async def get_auth_instance( - self, signing_name, region_name, signature_version=None, **kwargs - ): - if signature_version is None: - signature_version = self._signature_version - - cls = botocore.auth.AUTH_TYPE_MAPS.get(signature_version) - if cls is None: - raise UnknownSignatureVersionError( - signature_version=signature_version - ) - - frozen_credentials = None - if self._credentials is not None: - frozen_credentials = ( - await self._credentials.get_frozen_credentials() - ) - kwargs['credentials'] = frozen_credentials - if cls.REQUIRES_REGION: - if self._region_name is None: - raise botocore.exceptions.NoRegionError() - kwargs['region_name'] = region_name - kwargs['service_name'] = signing_name - auth = cls(**kwargs) - return auth - - # Alias get_auth for backwards compatibility. - get_auth = get_auth_instance - async def _choose_signer(self, operation_name, signing_type, context): signing_type_suffix_map = { 'presign-post': '-presign-post', @@ -145,6 +116,35 @@ async def _choose_signer(self, operation_name, signing_type, context): return signature_version + async def get_auth_instance( + self, signing_name, region_name, signature_version=None, **kwargs + ): + if signature_version is None: + signature_version = self._signature_version + + cls = botocore.auth.AUTH_TYPE_MAPS.get(signature_version) + if cls is None: + raise UnknownSignatureVersionError( + signature_version=signature_version + ) + + frozen_credentials = None + if self._credentials is not None: + frozen_credentials = ( + await self._credentials.get_frozen_credentials() + ) + kwargs['credentials'] = frozen_credentials + if cls.REQUIRES_REGION: + if self._region_name is None: + raise botocore.exceptions.NoRegionError() + kwargs['region_name'] = region_name + kwargs['service_name'] = signing_name + auth = cls(**kwargs) + return auth + + # Alias get_auth for backwards compatibility. + get_auth = get_auth_instance + async def generate_presigned_url( self, request_dict, @@ -227,6 +227,46 @@ async def generate_db_auth_token( return presigned_url[len(scheme) :] +class AioS3PostPresigner(S3PostPresigner): + async def generate_presigned_post( + self, + request_dict, + fields=None, + conditions=None, + expires_in=3600, + region_name=None, + ): + if fields is None: + fields = {} + + if conditions is None: + conditions = [] + + # Create the policy for the post. + policy = {} + + # Create an expiration date for the policy + datetime_now = datetime.datetime.utcnow() + expire_date = datetime_now + datetime.timedelta(seconds=expires_in) + policy['expiration'] = expire_date.strftime(botocore.auth.ISO8601) + + # Append all of the conditions that the user supplied. + policy['conditions'] = [] + for condition in conditions: + policy['conditions'].append(condition) + + # Store the policy and the fields in the request for signing + request = create_request_object(request_dict) + request.context['s3-presign-post-fields'] = fields + request.context['s3-presign-post-policy'] = policy + + await self._request_signer.sign( + 'PutObject', request, region_name, 'presign-post' + ) + # Return the url and the fields for th form to post. + return {'url': request.url, 'fields': fields} + + def add_generate_presigned_url(class_attributes, **kwargs): class_attributes['generate_presigned_url'] = generate_presigned_url @@ -296,46 +336,6 @@ async def generate_presigned_url( ) -class AioS3PostPresigner(S3PostPresigner): - async def generate_presigned_post( - self, - request_dict, - fields=None, - conditions=None, - expires_in=3600, - region_name=None, - ): - if fields is None: - fields = {} - - if conditions is None: - conditions = [] - - # Create the policy for the post. - policy = {} - - # Create an expiration date for the policy - datetime_now = datetime.datetime.utcnow() - expire_date = datetime_now + datetime.timedelta(seconds=expires_in) - policy['expiration'] = expire_date.strftime(botocore.auth.ISO8601) - - # Append all of the conditions that the user supplied. - policy['conditions'] = [] - for condition in conditions: - policy['conditions'].append(condition) - - # Store the policy and the fields in the request for signing - request = create_request_object(request_dict) - request.context['s3-presign-post-fields'] = fields - request.context['s3-presign-post-policy'] = policy - - await self._request_signer.sign( - 'PutObject', request, region_name, 'presign-post' - ) - # Return the url and the fields for th form to post. - return {'url': request.url, 'fields': fields} - - def add_generate_presigned_post(class_attributes, **kwargs): class_attributes['generate_presigned_post'] = generate_presigned_post diff --git a/aiobotocore/utils.py b/aiobotocore/utils.py index 1bd96991..4e64ac11 100644 --- a/aiobotocore/utils.py +++ b/aiobotocore/utils.py @@ -82,7 +82,10 @@ def __init__( ): self._timeout = timeout self._num_attempts = num_attempts + if config is None: + config = {} self._base_url = self._select_base_url(base_url, config) + self._config = config if env is None: env = os.environ.copy() @@ -212,13 +215,15 @@ async def retrieve_iam_role_credentials(self): role_name = await self._get_iam_role(token) credentials = await self._get_credentials(role_name, token) if self._contains_all_credential_fields(credentials): - return { + credentials = { 'role_name': role_name, 'access_key': credentials['AccessKeyId'], 'secret_key': credentials['SecretAccessKey'], 'token': credentials['Token'], 'expiry_time': credentials['Expiration'], } + self._evaluate_expiration(credentials) + return credentials else: if 'Code' in credentials and 'Message' in credentials: logger.debug( @@ -368,10 +373,10 @@ async def redirect_from_error( # we'll get a 400 Bad Request but we won't get a # body saying it's an "AuthorizationHeaderMalformed". is_special_head_object = ( - error_code in ['301', '400'] and operation.name == 'HeadObject' + error_code in ('301', '400') and operation.name == 'HeadObject' ) is_special_head_bucket = ( - error_code in ['301', '400'] + error_code in ('301', '400') and operation.name == 'HeadBucket' and 'x-amz-bucket-region' in response_metadata.get('HTTPHeaders', {}) @@ -381,7 +386,7 @@ async def redirect_from_error( ) is_redirect_status = response[0] is not None and response[ 0 - ].status_code in [301, 302, 307] + ].status_code in (301, 302, 307) is_permanent_redirect = error_code == 'PermanentRedirect' if not any( [ diff --git a/aiobotocore/waiter.py b/aiobotocore/waiter.py index 2a9f7736..92bec299 100644 --- a/aiobotocore/waiter.py +++ b/aiobotocore/waiter.py @@ -18,6 +18,59 @@ ) +def create_waiter_with_client(waiter_name, waiter_model, client): + """ + + :type waiter_name: str + :param waiter_name: The name of the waiter. The name should match + the name (including the casing) of the key name in the waiter + model file (typically this is CamelCasing). + + :type waiter_model: botocore.waiter.WaiterModel + :param waiter_model: The model for the waiter configuration. + + :type client: botocore.client.BaseClient + :param client: The botocore client associated with the service. + + :rtype: botocore.waiter.Waiter + :return: The waiter object. + + """ + single_waiter_config = waiter_model.get_waiter(waiter_name) + operation_name = xform_name(single_waiter_config.operation) + operation_method = NormalizedOperationMethod( + getattr(client, operation_name) + ) + + # Create a new wait method that will serve as a proxy to the underlying + # Waiter.wait method. This is needed to attach a docstring to the + # method. + async def wait(self, **kwargs): + await AIOWaiter.wait(self, **kwargs) + + wait.__doc__ = WaiterDocstring( + waiter_name=waiter_name, + event_emitter=client.meta.events, + service_model=client.meta.service_model, + service_waiter_model=waiter_model, + include_signature=False, + ) + + # Rename the waiter class based on the type of waiter. + waiter_class_name = str( + '%s.Waiter.%s' + % (get_service_module_name(client.meta.service_model), waiter_name) + ) + + # Create the new waiter class + documented_waiter_cls = type(waiter_class_name, (Waiter,), {'wait': wait}) + + # Return an instance of the new waiter class. + return documented_waiter_cls( + waiter_name, single_waiter_config, operation_method + ) + + class NormalizedOperationMethod(_NormalizedOperationMethod): async def __call__(self, **kwargs): try: @@ -54,7 +107,8 @@ async def wait(self, **kwargs): # can just handle here by raising an exception. raise WaiterError( name=self.name, - reason='An error occurred ({}): {}'.format( + reason='An error occurred (%s): %s' + % ( response['Error'].get('Code', 'Unknown'), response['Error'].get('Message', 'Unknown'), ), @@ -88,59 +142,3 @@ async def wait(self, **kwargs): last_response=response, ) await asyncio.sleep(sleep_amount) - - -def create_waiter_with_client(waiter_name, waiter_model, client): - """ - - :type waiter_name: str - :param waiter_name: The name of the waiter. The name should match - the name (including the casing) of the key name in the waiter - model file (typically this is CamelCasing). - - :type waiter_model: botocore.waiter.WaiterModel - :param waiter_model: The model for the waiter configuration. - - :type client: botocore.client.BaseClient - :param client: The botocore client associated with the service. - - :rtype: botocore.waiter.Waiter - :return: The waiter object. - - """ - single_waiter_config = waiter_model.get_waiter(waiter_name) - operation_name = xform_name(single_waiter_config.operation) - operation_method = NormalizedOperationMethod( - getattr(client, operation_name) - ) - - # Create a new wait method that will serve as a proxy to the underlying - # Waiter.wait method. This is needed to attach a docstring to the - # method. - async def wait(self, **kwargs): - await AIOWaiter.wait(self, **kwargs) - - wait.__doc__ = WaiterDocstring( - waiter_name=waiter_name, - event_emitter=client.meta.events, - service_model=client.meta.service_model, - service_waiter_model=waiter_model, - include_signature=False, - ) - - # Rename the waiter class based on the type of waiter. - waiter_class_name = str( - '{}.AIOWaiter.{}'.format( - get_service_module_name(client.meta.service_model), waiter_name - ) - ) - - # Create the new waiter class - documented_waiter_cls = type( - waiter_class_name, (AIOWaiter,), {'wait': wait} - ) - - # Return an instance of the new waiter class. - return documented_waiter_cls( - waiter_name, single_waiter_config, operation_method - ) diff --git a/setup.py b/setup.py index 51332fe3..48abcb96 100644 --- a/setup.py +++ b/setup.py @@ -7,15 +7,15 @@ # NOTE: When updating botocore make sure to update awscli/boto3 versions below install_requires = [ # pegged to also match items in `extras_require` - 'botocore>=1.24.21,<1.24.22', + 'botocore>=1.27.59,<1.27.60', 'aiohttp>=3.3.1', 'wrapt>=1.10.10', 'aioitertools>=0.5.1', ] extras_require = { - 'awscli': ['awscli>=1.22.76,<1.22.77'], - 'boto3': ['boto3>=1.21.21,<1.21.22'], + 'awscli': ['awscli>=1.25.60,<1.25.61'], + 'boto3': ['boto3>=1.24.59,<1.24.60'], } diff --git a/tests/boto_tests/test_credentials.py b/tests/boto_tests/test_credentials.py index 8f2daa70..231b6a9b 100644 --- a/tests/boto_tests/test_credentials.py +++ b/tests/boto_tests/test_credentials.py @@ -878,17 +878,6 @@ async def test_get_credentials(mock_session): assert creds is None -@pytest.mark.moto -@pytest.mark.asyncio -async def test_from_aiocredentials_is_none(): - creds = credentials.AioCredentials.from_credentials(None) - assert creds is None - creds = credentials.AioRefreshableCredentials.from_refreshable_credentials( - None - ) - assert creds is None - - class Self: pass diff --git a/tests/mock_server.py b/tests/mock_server.py index 7b31cd2d..a36b1b05 100644 --- a/tests/mock_server.py +++ b/tests/mock_server.py @@ -104,7 +104,7 @@ async def s3_server(): @pytest.fixture async def dynamodb2_server(): - async with MotoService('dynamodb2') as svc: + async with MotoService('dynamodb') as svc: yield svc.endpoint_url diff --git a/tests/test_patches.py b/tests/test_patches.py index 603cda91..7a584608 100644 --- a/tests/test_patches.py +++ b/tests/test_patches.py @@ -134,21 +134,21 @@ _API_DIGESTS = { # args.py ClientArgsCreator.get_client_args: { - 'a98b0bf9fe62f79b533b87664183c8886bc6816b' + '5e5b18cb0b466d3acb2e0ecacbc8dc78de4022fc' }, # client.py - ClientCreator.create_client: {'5cc47860c371ecd83b2e62c58bef590085cb07e0'}, + ClientCreator.create_client: {'3af567fcde81899a3b722d9cafd6a5c78e8ea08c'}, ClientCreator._create_client_class: { - '5e493d069eedbf314e40e12a7886bbdbcf194335' + 'fcecaf8d4f2c1ac3c5d0eb50c573233ef86d641d' }, ClientCreator._register_endpoint_discovery: { - '2eb9009d83a3999c77ecf2fd3335dab94348182e' + '483c6c8e035810d1b76110fc1956de76943c2f18' }, ClientCreator._get_client_args: { - '555e1e41f93df7558c8305a60466681e3a267ef3' + 'cc8da937425ba78f715304a82cec346dedb6292e' }, ClientCreator._register_s3_events: { - 'accf68c9e3e45b114310e8c635270ccb5fc4926e' + 'e2ada7e2fcc23f62a414a9dc806a50c0fe6c135c' }, ClientCreator._register_retries: { '16d3064142e5f9e45b0094bbfabf7be30183f255' @@ -160,27 +160,27 @@ '9ec4ff68599544b4f46067b3783287862d38fb50' }, ClientCreator._register_legacy_retries: { - '7dbd1a9d045b3d4f5bf830664f17c7bc610ee3a3' + '000b2f2a122602e2e741ec2e89308dc2e2b67329' }, - BaseClient._make_api_call: {'6517c7ead41bf0c70f38bb70666bffd21835ed72'}, - BaseClient._make_request: {'033a386f7d1025522bea7f2bbca85edc5c8aafd2'}, + BaseClient._make_api_call: {'ba323d78c89c292efe7fec6b74fe6c258b63d565'}, + BaseClient._make_request: {'cfd8bbf19ea132134717cdf9c460694ddacdbf58'}, BaseClient._convert_to_request_dict: { - '0071c2a37c3c696d9b0fba5f54b2985489c76b78' + '2e6eb436e95822f993d70d6127ae11e20689f9c4' }, - BaseClient._emit_api_params: {'2bfadaaa70671b63c50b1beed6d6c66e85813e9b'}, - BaseClient.get_paginator: {'c69885f5f73fae048c0b93b43bbfcd1f9c6168b8'}, - BaseClient.get_waiter: {'23d57598555bfbc4c6e7ec93406d05771f108d9e'}, - BaseClient.__getattr__: {'63f8ad095789d47880867f18537a277195845111'}, + BaseClient._emit_api_params: {'abd67874dae8d5cd2788412e4699398cb915a119'}, + BaseClient.get_paginator: {'3531d5988aaaf0fbb3885044ccee1a693ec2608b'}, + BaseClient.get_waiter: {'44f0473d993d49ac7502984a7ccee3240b088404'}, + BaseClient.__getattr__: {'3ec17f468f50789fa633d6041f40b66a2f593e77'}, # config.py Config.merge: {'c3dd8c3ffe0da86953ceba4a35267dfb79c6a2c8'}, - Config: {'1fb5fb546abe4970c98560b9f869339322930cdc'}, + Config: {'df1410e13b577bd3c1affa83309c206478907316'}, # credentials.py - create_mfa_serial_refresher: {'180b81fc40c91d1cf40de1a28e32ae7d601e1d50'}, + create_mfa_serial_refresher: {'9b5e98782fcacdcea5899a6d0d29d1b9de348bb0'}, Credentials.get_frozen_credentials: { - '08af57df08ee9953e440aa7aca58137ed936cdb6' + 'eb247f2884aee311bdabba3435e749c3b8589100' }, RefreshableCredentials.__init__: { - 'c685fd2c62eb60096fdf8bb885fb642df1819f7f' + '1a6b83fc845f05feab117ce4fab73b13baed6e3b' }, # We've overridden some properties RefreshableCredentials.__dict__['access_key'].fset: { @@ -202,15 +202,15 @@ '005c1b44b616f37739ce9276352e4e83644d8220' }, RefreshableCredentials._refresh: { - 'f4759b7ef0d1f0d8af07855dcd9ca49ef12c2e7b' + 'd5731d01db2812d498df19b4bd5d7c17519241fe' }, RefreshableCredentials._protected_refresh: { - 'c1b6bcb5d1d145ef2980037f2c24385151e3acab' + '9f8fdb76f41c3b1c64fd4d03d0701504626939e5' }, RefreshableCredentials.get_frozen_credentials: { 'f661c84a8b759786e011f0b1e8a468a0c6294e36' }, - SSOCredentialFetcher: {'d68353a0d2c291d5742f134d28ae1e1419faa4c6'}, + SSOCredentialFetcher: {'ac0cc1f456392f844c7182af635ee87e9fe9cf02'}, SSOProvider.load: {'f43d79e1520b2a7b7ef85cd537f41e19d4bce806'}, CachedCredentialFetcher._get_credentials: { '02a7d13599d972e3f258d2b53f87eeda4cc3e3a4' @@ -229,7 +229,7 @@ }, # Referenced by AioAssumeRoleWithWebIdentityCredentialFetcher AssumeRoleWithWebIdentityCredentialFetcher.__init__: { - '85c022a7237a3500ca973b2f7f91bffe894e4577' + 'ab270375dfe425c5e21276590dea690fdbfe40a5' }, AssumeRoleWithWebIdentityCredentialFetcher._get_credentials: { '02eba9d4e846474910cb076710070348e395a819' @@ -238,17 +238,17 @@ '8fb4fefe8664b7d82a67e0fd6d6812c1c8d92285' }, # Ensure that the load method doesn't do anything we should asyncify - EnvProvider.load: {'07cff5032b39b568505779774a1ca66efc513abb'}, + EnvProvider.load: {'39871a6ec3b3f5d51bc967122793e86b7ca6ed3c'}, ContainerProvider.__init__: {'ea6aafb2e12730066af930fb5a27f7659c1736a1'}, ContainerProvider.load: {'57c35569050b45c1e9e33fcdb3b49da9e342fdcf'}, ContainerProvider._retrieve_or_fail: { '7c14f1cdee07217f847a71068866bdd10c3fa0fa' }, ContainerProvider._create_fetcher: { - '09a3ffded0fc20a574f3b34fa432a1569d5e729f' + '935ae28fdb1c76f419523d4030265f8c4d9d0b00' }, InstanceMetadataProvider.load: { - '4a27eb94fe220fba2b46c97bdd9e16de199ce004' + '2016921ddb86cab5ec699ab217e847a300745990' }, ProfileProviderBuilder._create_process_provider: { 'c5eea47bcfc449a6d73a9892bd0e1897f6be0c20' @@ -260,19 +260,19 @@ 'f9a40d4211f6e663ba2ae9682fba5306152178c5' }, ProfileProviderBuilder._create_web_identity_provider: { - '0907c1ad5573bc5c0fc87efb601a6c4c3fcf34ae' + '478745fa6779a7c69fe9441d89d3e921438e3a59' }, ProfileProviderBuilder._create_sso_provider: { '258e6d07bdf40ea2c7551bae0cd6e1ab58e4e502' }, - ConfigProvider.load: {'8fb32140086dce65fa28be8edd3ac0d22698c3ae'}, + ConfigProvider.load: {'d0714da9f1f54cebc555df82f181c4913ce97258'}, SharedCredentialProvider.load: { - 'c0be1fe376d25952461ca18d9bef4b4340203441' + '8a17d992e2a90ebc0e07ba5a5dfef2b725367496' }, ProcessProvider.__init__: {'2e870ec0c6b0bc8483fa9b1159ef68bbd7a12c56'}, - ProcessProvider.load: {'aac90e2c8823939f09936b9c883e67503128e438'}, + ProcessProvider.load: {'6866e1d3abbde7a14e83aea28cc49377faaca84b'}, ProcessProvider._retrieve_credentials_using: { - 'ffc27c7cba0e37cf6db3a3eacfd54be8bd99d3a9' + 'c12acda42ddc5dfd73946adce8c155295f8c6b88' }, CredentialResolver.load_credentials: { 'ef31ba8817f84c1f61f36259da1cc6e597b8625a' @@ -285,19 +285,19 @@ }, AssumeRoleProvider.load: {'ee9ddb43e25eb1105185253c0963a2f5add49a95'}, AssumeRoleProvider._load_creds_via_assume_role: { - '9fdba45a8dd16b885dea7c1fafc7d02609870fa7' + '85116d63561c9a8bfdfffdbf837b8a7e61b47ea3' }, AssumeRoleProvider._resolve_source_credentials: { '105c0c011e23d76a3b8bd3d9b91b6d945c8307a1' }, AssumeRoleProvider._resolve_credentials_from_profile: { - '402a1a6b3e0a29c234b7883e5b855110eb655830' + 'a87ece979f8c94c1afd5801156e2b39f0d6d45ab' }, AssumeRoleProvider._resolve_static_credentials_from_profile: { - '58f04986bb1027d548212b7769034e5dae5cc30f' + 'a470795f6ba451cf99ce7456fef24777f8087654' }, AssumeRoleProvider._resolve_credentials_from_source: { - '6f76ae62f477279a2297565f80a5cfbe5ea30eaf' + 'de41138b36bfc74d7f8a21f6002b55279d3de017' }, CanonicalNameCredentialSourcer.source_credentials: { '602930a78e0e64e3b313a046aab5edc3bcf5c2d9' @@ -305,34 +305,34 @@ CanonicalNameCredentialSourcer._get_provider: { 'c028b9776383cc566be10999745b6082f458d902' }, - BotoProvider.load: {'9351b8565c2c969937963fc1d3fbc8b3b6d8ccc1'}, - OriginalEC2Provider.load: {'bde9af019f01acf3848a6eda125338b2c588c1ab'}, - create_credential_resolver: {'177ad331d4b527b9aae765d90e2f17badefeb4a8'}, + BotoProvider.load: {'e84ebfe3d6698dc4683f0f6699d4a9907c87bebb'}, + OriginalEC2Provider.load: {'dc58cd1c79d177105b183a2d20e1154e6f8f0733'}, + create_credential_resolver: {'f3501ad2330afe5e1ef4e71c7537f94885c73821'}, get_credentials: {'ff0c735a388ac8dd7fe300a32c1e36cdf33c0f56'}, # configprovider.py SmartDefaultsConfigStoreFactory.merge_smart_defaults: { - 'e1049d34cba3197b4e70dabbbe59e17686fa90f9' + 'e320299bb739694fefe2f5df6be62cc5321d3dc5' }, SmartDefaultsConfigStoreFactory.resolve_auto_mode: { - '61e749ec045bb0c670bcbc9846b4cfc16cde5718' + '013fa8904b42931c69e3d8623025a1582379ba2a' }, # endpoint.py - convert_to_response_dict: {'2c73c059fa63552115314b079ae8cbf5c4e78da0'}, - Endpoint.create_request: {'4ccc14de2fd52f5c60017e55ff8e5b78bbaabcec'}, - Endpoint._send_request: {'214fa3a0e72f3877cef915c8429d40729775f0cf'}, - Endpoint._get_response: {'6803c16fb6576ea18d9e3d8ffb2e9f3874d9b8ee'}, - Endpoint._do_get_response: {'91370e4a034ec61fa8090fe9442cfafe9b63c6cc'}, - Endpoint._needs_retry: {'0f40f52d8c90c6e10b4c9e1c4a5ca00ef2c72850'}, + convert_to_response_dict: {'5b7701c1f5b3cb2daa6eb307cdbdbbb2e9d33e5f'}, + Endpoint.create_request: {'37d0fbd02f91aef6c0499a2d0a725bf067c3ce8b'}, + Endpoint._send_request: {'5d40748a95c3005728e6548b402b90cb57d6f575'}, + Endpoint._get_response: {'bbf10e6e07147d50e09d7205bf0883bd673a8bf3'}, + Endpoint._do_get_response: {'5afcfe76196406903afb24e05e3dd0feeac1a23d'}, + Endpoint._needs_retry: {'f718e2ff874763a677648fe6f87cc65e4cec2792'}, Endpoint._send: {'644c7e5bb88fecaa0b2a204411f8c7e69cc90bf1'}, Endpoint._add_modeled_error_fields: { - '1eefcfacbe9a2c3700c61982e565ce6c4cf1ea3a' + 'd0390647f2d7a4a325be048dcda4dcc7f42fdd17' }, EndpointCreator.create_endpoint: { - '77a36b0fdc2e4ae7c421849843b93b4dcae5e06f' + '863e17b1299f9fda2cef5be3297d470d1bfa86ae' }, # eventstream.py EventStream._create_raw_event_generator: { - 'cc101f3ca2bca4f14ccd6b385af900a15f96967b' + '1764be20b3abe19b60381756a989794de298ffbb' }, EventStream.__iter__: {'8a9b454943f8ef6e81f5794d641adddd1fdd5248'}, EventStream.get_initial_response: { @@ -344,20 +344,20 @@ '23670e04e0b09a9575c7533442bca1b2972ade82' }, HierarchicalEmitter._verify_and_register: { - 'aa14572fd9d42b83793d4a9d61c680e37761d762' + '41eda968127e35e02e7120ec621240b61639e3dd' }, EventAliaser.emit_until_response: { '0d635bf7ae5022b1fdde891cd9a91cd4c449fd49' }, # paginate.py - PageIterator.__iter__: {'a56ec9b28dba7e48936d7164b5ea0e3a0fc0287d'}, + PageIterator.__iter__: {'a7e83728338e61ff2ca0a26c6f03c67cbabffc32'}, PageIterator.result_key_iters: { - '04d3c647bd98caba3687df80e650fea517a0068e' + 'e8cd36fdc4960e08c9aa50196c4e5d1ee4e39756' }, PageIterator.build_full_result: { - 'afe8cd8daad2cf32ae34f877985ab79501bf7742' + '9051327d350ed5a4843c74d34be74ba2f1732e30' }, - ResultKeyIterator: {'f71d98959ccda5e05e35cf3cf224fbc9310d33bb'}, + ResultKeyIterator: {'3028dde4c4de6029f628f4a9d1fff36986b41591'}, # parsers.py ResponseParserFactory.create_parser: { '5cf11c9acecd1f60a013f6facbe0f294daa3f390' @@ -385,47 +385,47 @@ }, create_parser: {'37e9f1c3b60de17f477a9b79eae8e1acaa7c89d7'}, # response.py - StreamingBody: {'a177edffd0c4ec72f1bb4b9e09ea33bc6d37b248'}, - get_response: {'f31b478792a5e0502f142daca881b69955e5c11d'}, + StreamingBody: {'73cb1276dfb509331b964d3d5ed69e5efa008de5'}, + get_response: {'6515f43730b546419695c26d4bc0d198fde54b10'}, # session.py - Session.__init__: {'ccf156a76beda3425fb54363f3b2718dc0445f6d'}, + Session.__init__: {'d0d3b11d6feb4783d2a7399246ce02c58e2c34e7'}, Session._register_response_parser_factory: { - 'd6cd5a8b1b473b0ec3b71db5f621acfb12cc412c' + 'bb8f7f3cc4d9ff9551f0875604747c4bb5030ff6' }, - Session.create_client: {'7e0c40c06d3fede4ebbd862f1d6e51118c4a1ff0'}, + Session.create_client: {'a179ef6d370020181d99fcb6cb1279e948d72afa'}, Session._create_credential_resolver: { '87e98d201c72d06f7fbdb4ebee2dce1c09de0fb2' }, - Session.get_credentials: {'c0de970743b6b9dd91b5a71031db8a495fde53e4'}, + Session.get_credentials: {'718da08b630569e631f93aedd65f1d9215bfc30b'}, get_session: {'c47d588f5da9b8bde81ccc26eaef3aee19ddd901'}, - Session.get_service_data: {'e28f2de9ebaf13214f1606d33349dfa8e2555923'}, + Session.get_service_data: {'3879b969c0c2b1d5b454006a1025deb4322ae804'}, Session.get_service_model: {'1c8f93e6fb9913e859e43aea9bc2546edbea8365'}, Session.get_available_regions: { - 'bc455d24d98fbc112ff22325ebfd12a6773cb7d4' + '9fb4df0b7d082a74d524a4a15aaf92a2717e0358' }, Session._register_smart_defaults_factory: { - '24ab10e4751ada800dde24d40d1d105be76a0a14' + 'af5fc9cf6837ed119284603ca1086e4113febec0' }, # signers.py RequestSigner.handler: {'371909df136a0964ef7469a63d25149176c2b442'}, - RequestSigner.sign: {'a07e4caab222bf9375036b1fafaf021ccb5b2bf3'}, + RequestSigner.sign: {'d90346d5e066e89cd902c5c936f59b644ecde275'}, RequestSigner.get_auth: {'4f8099bef30f9a72fa3bcaa1bd3d22c4fbd224a8'}, RequestSigner.get_auth_instance: { - '4f8099bef30f9a72fa3bcaa1bd3d22c4fbd224a8' + 'c2c34a0f44cac8819c7e9b74ca52dc82a28a1a08' }, - RequestSigner._choose_signer: {'d1e0e3196ada449d3ae0ec09b8ae9b5868c50d4e'}, + RequestSigner._choose_signer: {'eb82bd279d8c6cb7c93f7330a45544f0dda73170'}, RequestSigner.generate_presigned_url: { - '2acffdfd926b7b6f6cc4b70b90c0587e7f424888' + '417682868eacc10bf4c65f3dfbdba7d20d9250db' }, add_generate_presigned_url: {'5820f74ac46b004eb79e00eea1adc467bcf4defe'}, - generate_presigned_url: {'9c471f957210c0a71a11f5c73be9fed844ecb5bb'}, + generate_presigned_url: {'03cb4e442690f2df47f1580c66345b12764dee0e'}, S3PostPresigner.generate_presigned_post: { - 'b91d50bae4122d7ab540653865ec9294520ac0e1' + '269efc9af054a2fd2728d5b0a27db82c48053d7f' }, add_generate_presigned_post: {'e30360f2bd893fabf47f5cdb04b0de420ccd414d'}, - generate_presigned_post: {'e9756488cf1ceb68d23b36688f3d0767505f3c77'}, + generate_presigned_post: {'56687f0abdf1451951cffabefa9d970256fca420'}, add_generate_db_auth_token: {'f61014e6fac4b5c7ee7ac2d2bec15fb16fa9fbe5'}, - generate_db_auth_token: {'5f5a758458c007107a23124192339f747472dc75'}, + generate_db_auth_token: {'1f37e1e5982d8528841ce6b79f229b3e23a18959'}, # utils.py ContainerMetadataFetcher.__init__: { '46d90a7249ba8389feb487779b0a02e6faa98e57' @@ -437,26 +437,26 @@ '4ee8aa704cf0a378d68ef9a7b375a1aa8840b000' }, ContainerMetadataFetcher._retrieve_credentials: { - 'f5294f9f811cb3cc370e4824ca106269ea1f44f9' + 'b00694931af86ef1a9305ad29328030ee366cea9' }, ContainerMetadataFetcher._get_response: { - '7e5acdd2cf0167a047e3d5ee1439565a2f79f6a6' + '4dc84054db957f2c1fb2fb1b01eb462bd57b1a64' }, - IMDSFetcher.__init__: {'a0766a5ba7dde9c26f3c51eb38d73f8e6087d492'}, - IMDSFetcher._get_request: {'d06ba6890b94c819e79e27ac819454b28f704535'}, + IMDSFetcher.__init__: {'e7e62b79f6a9e4cb14120e61c4516f0a61148100'}, + IMDSFetcher._get_request: {'7f8ad4724ac08300a0c55e307bfeb5abc0579d26'}, IMDSFetcher._fetch_metadata_token: { - 'c162c832ec24082cd2054945382d8dc6a1ec5e7b' + '12225b35a73130632038785a8c2e6fbaaf9de1f4' }, - IMDSFetcher._default_retry: {'d1fa834cedfc7a2bf9957ba528eed24f600f7ef6'}, + IMDSFetcher._default_retry: {'362ce5eff50bfb74e58fbdd3f44146a87958318a'}, IMDSFetcher._is_non_ok_response: { '448b80545b1946ec44ff19ebca8d4993872a6281' }, IMDSFetcher._is_empty: {'241b141c9c352a4ef72964f8399d46cbe9a5aebc'}, IMDSFetcher._log_imds_response: { - 'f1e09ad248feb167f55b11bbae735ea0e2c7b446' + 'dcbe619ce2ddb8b5015f128612d86dd8a5dd31e8' }, InstanceMetadataFetcher.retrieve_iam_role_credentials: { - '76737f6add82a1b9a0dc590cf10bfac0c7026a2e' + '40f31ba06abb9853c2e6fea68846742bd3eda919' }, InstanceMetadataFetcher._get_iam_role: { '80073d7adc9fb604bc6235af87241f5efc296ad7' @@ -468,46 +468,46 @@ '97818b51182a2507c99876a40155adda0451dd82' }, InstanceMetadataFetcher._needs_retry_for_role_name: { - '0f1034c9de5be2d79a584e1e057b8df5b39f4514' + 'ca9557fb8e58d03e09d77f9fb63d21afb4689b58' }, InstanceMetadataFetcher._needs_retry_for_credentials: { - '977be4286b42916779ade4c20472ec3a6a26c90d' + 'e7e5a8ce541110eb79bf98414171d3a1c137e32b' }, S3RegionRedirector.redirect_from_error: { - 'f6f765431145a9bed8e73e6a3dbc7b0d6ae5f738' + '3863b2c6472513b7896bfccc9dfd2567c472f441' }, S3RegionRedirector.get_bucket_region: { 'b5bbc8b010576668dc2812d657c4b48af79e8f99' }, InstanceMetadataRegionFetcher.retrieve_region: { - 'e916aeb4a28a265224a21006dce1d443cd1207c4' + '0134024f0aa2d2b49ec436ea8058c1eca8fac4af' }, InstanceMetadataRegionFetcher._get_region: { - '73f8c60d21aae765db3a473a527c846e0108291f' + '16e8fc546958471650eef233b0fd287758293019' }, IMDSRegionProvider.provide: {'09d1b70bc1dd7a37cb9ffd437acd71283b9142e9'}, IMDSRegionProvider._get_instance_metadata_region: { '4631ced79cff143de5d3fdf03cd69720778f141b' }, IMDSRegionProvider._create_fetcher: { - '28b711326769d03a282558066058cd85b1cb4568' + '7031d7cdaea0244a5d860f2f8f6c013e25578123' }, # waiter.py NormalizedOperationMethod.__call__: { '79723632d023739aa19c8a899bc2b814b8ab12ff' }, - Waiter.wait: {'a9fa6e3b1210929b9e3887abff90aeb451383547'}, - create_waiter_with_client: {'c3d12c9a4293105cc8c2ecfc7e69a2152ad564de'}, + Waiter.wait: {'735608297a2a3d4572e6705daafcf4fc8556fc03'}, + create_waiter_with_client: {'e6ea06674b6fdf9157c95757a12b3c9c35af531c'}, # handlers.py - inject_presigned_url_rds: {'5a34e1666d84f6229c54a59bffb69d46e8117b3a'}, - inject_presigned_url_ec2: {'37fad2d9c53ca4f1783e32799fa8f70930f44c23'}, - parse_get_bucket_location: {'dde31b9fe4447ed6eb9b8c26ab14cc2bd3ae2c64'}, - check_for_200_error: {'94005c964d034e68bb2f079e89c93115c1f11aad'}, + inject_presigned_url_rds: {'b5d45b339686346e81b255d4e8c36e76d3fe6a78'}, + inject_presigned_url_ec2: {'48e09a5e4e95577e716be30f2d2706949261a07f'}, + parse_get_bucket_location: {'64ffbf5c6aa6ebd083f49371000fa046d0de1fc6'}, + check_for_200_error: {'ded7f3aaef7b1a5d047c4dac86692ab55cbd7a13'}, _looks_like_special_case_error: { - 'adcf7c6f77aa123bd94e96ef0beb4ba548e55086' + '86946722d10a72b593483fca0abf30100c609178' }, # httpsession.py - URLLib3Session: {'5adede4ba9d2a80e776bfeb71127656fafff91d7'}, + URLLib3Session: {'c72094afb3aa62db0ade9be09be72ec7a2c3d80a'}, EndpointDiscoveryHandler.discover_endpoint: { 'd87eff9008356a6aaa9b7078f23ba7a9ff0c7a60' }, @@ -520,73 +520,73 @@ # retries/adaptive.py # See comments in AsyncTokenBucket: we completely replace the ClientRateLimiter # implementation from botocore. - adaptive.ClientRateLimiter: {'d4ba74b924cdccf705adeb89f2c1885b4d21ce02'}, + adaptive.ClientRateLimiter: {'e9d99c7921170815f0eaf5c2848765cd6ed90177'}, adaptive.register_retry_handler: { - 'd662512878511e72d1202d880ae181be6a5f9d37' + '96c073719a3d5d41d1ca7ae5f7e31bbb431c75b3' }, # retries/standard.py standard.register_retry_handler: { - '8d464a753335ce7457c5eea73e80d9a224fe7f21' + 'da0ae35712211bc38938e93c4af8b7aeb999084e' }, standard.RetryHandler.needs_retry: { - '2dfc4c2d2efcd5ca00ae84ccdca4ab070d831e22' + '89a4148d7f4af9d2795d1d0189293528aa668b59' }, standard.RetryPolicy.should_retry: { 'b30eadcb94dadcdb90a5810cdeb2e3a0bc0c74c9' }, standard.StandardRetryConditions.__init__: { - '82f00342fb50a681e431f07e63623ab3f1e39577' + 'e17de49a447769160964a2da926b7d72544efd48' }, standard.StandardRetryConditions.is_retryable: { - '4d14d1713bc2806c24b6797b2ec395a29c9b0453' + '558a0f0b4d30f996e046779fe233f587611ca5c7' }, standard.OrRetryChecker.is_retryable: { '5ef0b84b1ef3a49bc193d76a359dbd314682856b' }, # retries/special.py special.RetryDDBChecksumError.is_retryable: { - '0769cca303874f8dce47dcc93980fa0841fbaab6' + '6c6e0945b0989b13fd8e7d78dbfcde307a131eae' }, # retries/bucket.py # See comments in AsyncTokenBucket: we completely replace the TokenBucket # implementation from botocore. - TokenBucket: {'9d543c15de1d582fe99a768fd6d8bde1ed8bb930'}, + TokenBucket: {'bddbf15ba4c2f04cb7fe04992168fa0e839047e5'}, # awsresponse.py AWSResponse.content: {'1d74998e3e0abe52b52c251a1eae4971e65b1053'}, AWSResponse.text: {'a724100ba9f6d51b333b8fe470fac46376d5044a'}, # httpchecksum.py handle_checksum_body: {'4b9aeef18d816563624c66c57126d1ffa6fe1993'}, - _handle_bytes_response: {'76f4f9d1da968dc6dbc24fd9f59b4b8ee86799f4'}, + _handle_bytes_response: {'0761c4590c6addbe8c674e40fca9f7dd375a184b'}, # retryhandler.py retryhandler.create_retry_handler: { - 'fde9dfbc581f3d571f7bf9af1a966f0d28f6d89d' + '8fee36ed89d789194585f56b8dd4f525985a5811' }, retryhandler.create_checker_from_retry_config: { - '3022785da77b62e0df06f048da3bb627a2e59bd5' + 'bc43996b75ab9ffc7a4e8f20fc62805857867109' }, retryhandler._create_single_checker: { - '517aaf8efda4bfe851d8dc024513973de1c5ffde' + 'da29339040ab1faeaf2d80752504e4f8116686f2' }, retryhandler._create_single_response_checker: { - 'f55d841e5afa5ebac6b883edf74a9d656415474b' + 'dda92bb44f295a1f61750c7e1fbc176f66cb8b44' }, retryhandler.RetryHandler.__call__: { - '0ff14b0e97db0d553e8b94a357c11187ca31ea5a' + 'e599399167b1f278e4cd839170f887d60eea5bfa' }, retryhandler.MaxAttemptsDecorator.__call__: { - 'd04ae8ff3ab82940bd7a5ffcd2aa27bf45a4817a' + '24b442126f0ff730be0ae64dc7158929d4d2fca7' }, retryhandler.MaxAttemptsDecorator._should_retry: { - '33af9b4af06372dc2a7985d6cbbf8dfbaee4be2a' + '8ba8bcec27974861ec35a7d91ce96db1c04833fe' }, retryhandler.MultiChecker.__call__: { - 'dae2cc32aae9fa0a527630db5c5d8db96d957633' + 'e8302c52e1bbbb129b6f505633a4bc4ae1e5a34f' }, retryhandler.CRC32Checker.__call__: { - '4f0b55948e05a9039dc0ba62c80eb341682b85ac' + '882a731eaf6b0ddca68ab4032a169a0fa09a4d43' }, retryhandler.CRC32Checker._check_response: { - 'bc371df204ab7138e792b782e83473e6e9b7a620' + '3ee7afd0bb1a3bf53934d77e44f619962c52b0c9' }, }