diff --git a/docs/service_configuration.rst b/docs/service_configuration.rst index 50ab969..0faf21f 100644 --- a/docs/service_configuration.rst +++ b/docs/service_configuration.rst @@ -39,6 +39,7 @@ Common Karton configuration fields are listed below: [s3] address S3 API address [s3] access_key S3 API access key (username) [s3] secret_key S3 API secret key (password) + [s3] iam_auth S3 IAM configuration (default: False) [s3] bucket Default bucket name for storing produced resources [redis] host Redis server hostname [redis] port Redis server port @@ -54,6 +55,8 @@ Common Karton configuration fields are listed below: [signaling] status Turns on producing of 'karton.signaling.status' tasks, signalling the task start and finish events by Karton service (default: 0, off) ============ =============== ======================================================================================================================================= +Note that if both ``iam_auth = True`` and the ``access_key``, ``secret_key`` pair are provided in the configuration file, Karton will first try to load secrets via IAM provider and +will fallback to the provided pair otherwise. More information about credential loading can be found `here `_. Karton System configuration --------------------------- diff --git a/karton/core/backend.py b/karton/core/backend.py index 0de208d..b235c0d 100644 --- a/karton/core/backend.py +++ b/karton/core/backend.py @@ -9,10 +9,16 @@ from typing import IO, Any, Dict, Iterable, Iterator, List, Optional, Set, Tuple, Union import boto3 +from botocore.credentials import ( + ContainerProvider, + InstanceMetadataFetcher, + InstanceMetadataProvider, +) from redis import AuthenticationError, StrictRedis from redis.client import Pipeline from urllib3.response import HTTPResponse +from .config import Config from .exceptions import InvalidIdentityError from .task import Task, TaskPriority, TaskState from .utils import chunks, chunks_iter @@ -99,7 +105,7 @@ def parse_client_name( class KartonBackend: def __init__( self, - config, + config: Config, identity: Optional[str] = None, service_info: Optional[KartonServiceInfo] = None, ) -> None: @@ -113,11 +119,51 @@ def __init__( self.redis = self.make_redis( config, identity=identity, service_info=service_info ) + + session_token = None + endpoint = config.get("s3", "address") + access_key = config.get("s3", "access_key") + secret_key = config.get("s3", "secret_key") + iam_auth = config.getboolean("s3", "iam_auth") + + if not endpoint: + raise RuntimeError("Attempting to get S3 client without an endpoint set") + + if access_key and secret_key and iam_auth: + logger.warning( + "Warning: iam is turned on and both S3 access key and secret key are" + " provided" + ) + + if iam_auth: + iam_providers = [ + ContainerProvider(), + InstanceMetadataProvider( + iam_role_fetcher=InstanceMetadataFetcher( + timeout=1000, num_attempts=2 + ) + ), + ] + + for provider in iam_providers: + creds = provider.load() + if creds: + access_key = creds.access_key + secret_key = creds.secret_key + session_token = creds.token + break + + if access_key is None or secret_key is None: + raise RuntimeError( + "Attempting to get S3 client without an access_key/secret_key set" + ) + self.s3 = boto3.client( "s3", - endpoint_url=config["s3"]["address"], - aws_access_key_id=config["s3"]["access_key"], - aws_secret_access_key=config["s3"]["secret_key"], + endpoint_url=endpoint, + aws_access_key_id=access_key, + aws_secret_access_key=secret_key, + aws_session_token=session_token, ) @staticmethod diff --git a/pyproject.toml b/pyproject.toml index f7b1b60..36e6f50 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,4 +10,4 @@ line_length = 88 lint-version = "2" source = "karton/" # Temporary workaround for Union[Awaitable[T]], T] mess in Redis typing -extra-requirements = "redis<5.0.0" +extra-requirements = "redis<5.0.0 botocore-stubs"