From 477b838940942572cb4fd3bce8eb0aeb3cb5008f Mon Sep 17 00:00:00 2001 From: Jakob Keller <57402305+jakob-keller@users.noreply.github.com> Date: Tue, 16 Jan 2024 20:19:01 +0100 Subject: [PATCH] fix race condition in S3 Express identity cache (closes #1072) --- CHANGES.rst | 4 ++++ aiobotocore/__init__.py | 2 +- aiobotocore/utils.py | 22 ++++------------------ 3 files changed, 9 insertions(+), 19 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index b5e9c4f9..5470aa8f 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,10 @@ Changes ------- +2.9.1 (2024-01-16) +^^^^^^^^^^^^^^^^^^ +* fix race condition in S3 Express identity cache #1072 + 2.9.0 (2023-12-12) ^^^^^^^^^^^^^^^^^^ * bump botocore dependency specification diff --git a/aiobotocore/__init__.py b/aiobotocore/__init__.py index 387cfacc..b03f5b5d 100644 --- a/aiobotocore/__init__.py +++ b/aiobotocore/__init__.py @@ -1 +1 @@ -__version__ = '2.9.0' +__version__ = '2.9.1' diff --git a/aiobotocore/utils.py b/aiobotocore/utils.py index 3163bafd..f7df88f1 100644 --- a/aiobotocore/utils.py +++ b/aiobotocore/utils.py @@ -367,30 +367,16 @@ async def get_credentials(self, **kwargs): class AioS3ExpressIdentityCache(AioIdentityCache, S3ExpressIdentityCache): - @functools.cached_property - def _aio_credential_cache(self): - """Substitutes upstream credential cache.""" - return {} + @functools.lru_cache(maxsize=100) + def _get_credentials(self, bucket): + return asyncio.create_task(super().get_credentials(bucket=bucket)) async def get_credentials(self, bucket): # upstream uses `@functools.lru_cache(maxsize=100)` to cache credentials. # This is incompatible with async code. # We need to implement custom caching logic. - if (credentials := self._aio_credential_cache.get(bucket)) is None: - # cache miss -> get credentials asynchronously - credentials = await super().get_credentials(bucket=bucket) - - # upstream cache is bounded at 100 entries - if len(self._aio_credential_cache) >= 100: - # drop oldest entry from cache (deviates from lru_cache logic) - self._aio_credential_cache.pop( - next(iter(self._aio_credential_cache)), - ) - - self._aio_credential_cache[bucket] = credentials - - return credentials + return await self._get_credentials(bucket=bucket) def build_refresh_callback(self, bucket): async def refresher():