diff --git a/docs/syntax/docker-compose/secrets.rst b/docs/syntax/docker-compose/secrets.rst index 7b8469ef7..8ed26a748 100644 --- a/docs/syntax/docker-compose/secrets.rst +++ b/docs/syntax/docker-compose/secrets.rst @@ -9,31 +9,54 @@ As you might have already used these, docker-compose allows you to define secret To help continue with docker-compose syntax compatibility, you can now declare your secret in docker-compose, and add an extension field which will be a direct mapping to the secret name you have in AWS Secrets Manager. -.. code-block:: yaml +ECS ComposeX will automatically add IAM permissions to **the execution** role of your Task definition and will export the secret +to your container, using the same name as in the compose file. - secrets: - topsecret_info: - x-secrets: - Name: /path/to/my/secret +.. seealso:: - services: - serviceA: - secrets: - - topsecret_info + `docker-compose secrets reference`_ -This will automatically add IAM permissions to **the execution** role of your Task definition and will export the secret -to your container, using the same name as in the compose file. +.. hint:: -.. note:: + For security purposes, the containers **envoy** and **xray-daemon** are not getting assigned the secrets. - Only Fargate 1.4.0+ Platform Version supports secrets JSON Key + +Syntax +====== + +.. code-block:: + + x-secrets: + Name: str + LinksTo: [] + JsonKeys: [] + Lookup: {} + +Name +---- + +Type: String + +The name of the secret in secrets manager to use and import. .. hint:: - If you believe that your service application should have access to the secret via **Task Role**, simply add to the - secret definition as follows: + If you want to put the full ARN, you can. There will be a validation for it. + +LinksTo +------- + +Type: List of Strings + +AllowedValues: + +* EcsExecutionRole +* EcsTaskRole - .. code-block:: yaml +If you believe that your service application should have access to the secret via **Task Role**, simply add to the +secret definition as follows: + +.. code-block:: yaml secret-name: x-secrets: @@ -47,14 +70,89 @@ to your container, using the same name as in the compose file. If you do not specify **EcsExecutionRole** when specifying **LinksTo** then you will not get the secret exposed to your container via AWS ECS Secrets property of your Container Definition -.. hint:: +JsonKeys +-------- - For security purposes, the containers **envoy** and **xray-daemon** are not getting assigned the secrets. +Type: List of objects/dicts +.. note:: -.. seealso:: + Only Fargate 1.4.0+ Platform Version supports secrets JSON Key - `docker-compose secrets reference`_ +.. code-block:: yaml + :caption: JsonKeys objects structure + Key: str + Name: str + +Key +""" + +Name of the JSON Key in your secret. + +Name +"""" + +The Name of the secret specifically for the secret JSON key + + +Examples +======== + +.. code-block:: yaml + :caption: Short example + + secrets: + topsecret_info: + x-secrets: + Name: /path/to/my/secret + + services: + serviceA: + secrets: + - topsecret_info + +.. code-block:: yaml + :caption: Secret with assignment to Task and Execution Role + + secrets: + abcd: {} + john: + x-secrets: + LinksTo: + - EcsExecutionRole + - EcsTaskRole + Name: SFTP/asl-cscs-files-dev + + +.. code-block:: yaml + :caption: Secret Looked up from Tags and Name, also using JsonKeys + + secrets: + zyx: + x-secrets: + Name: secret/with/kmskey + Lookup: + Tags: + - costcentre: lambda + - composexdev: "yes" + JsonKeys: + - Key: username + Name: PSQL_USERNAME + - Key: password + Name: PSQL_PASSWORD + + +.. code-block:: yaml + :caption: Secret with assignment to Task and Execution Role + + secrets: + abcd: {} + john: + x-secrets: + LinksTo: + - EcsExecutionRole + - EcsTaskRole + Name: arn:aws:secretsmanager:eu-west-1:123456789012:secret:/secret/abcd .. _docker-compose secrets reference: https://docs.docker.com/compose/compose-file/#secrets diff --git a/ecs_composex/common/aws.py b/ecs_composex/common/aws.py index fd6fc8ba2..59a4462ac 100644 --- a/ecs_composex/common/aws.py +++ b/ecs_composex/common/aws.py @@ -225,7 +225,7 @@ def find_aws_resource_arn_from_tags_api( """ res_types = { "secretsmanager:secret": { - "regexp": r"(?:^arn:aws(?:-[a-z]+)?:secretsmanager:[\w-]+:[0-9]{12}:secret:)([\S]+)(?:-[A-Za-z0-9]+)$" + "regexp": r"(?:^arn:aws(?:-[a-z]+)?:secretsmanager:[\w-]+:[0-9]{12}:secret:)([\S]+)(?:-[A-Za-z0-9]{1,6})$" }, } if types is not None and isinstance(types, dict): @@ -238,7 +238,10 @@ def find_aws_resource_arn_from_tags_api( resources_r = get_resources_from_tags(session, aws_resource_search, search_tags) LOG.debug(search_tags) - arns = [i["ResourceARN"] for i in resources_r["ResourceTagMappingList"]] + if not resources_r or not keyisset("ResourceTagMappingList", resources_r): + arns = [] + else: + arns = [i["ResourceARN"] for i in resources_r["ResourceTagMappingList"]] return handle_search_results( arns, name, res_types, aws_resource_search, allow_multi=allow_multi ) diff --git a/ecs_composex/common/compose_secrets.py b/ecs_composex/common/compose_secrets.py deleted file mode 100644 index 8afbd4b0d..000000000 --- a/ecs_composex/common/compose_secrets.py +++ /dev/null @@ -1,95 +0,0 @@ -# -*- coding: utf-8 -*- -# ECS ComposeX -# Copyright (C) 2020 John Mille -# # -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# # -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# # -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -""" -Represent a service from the docker-compose services -""" - -from troposphere import Sub, AWS_PARTITION, AWS_REGION, AWS_ACCOUNT_ID -from troposphere.ecs import Secret as EcsSecret - -from ecs_composex.common import LOG, keyisset -from ecs_composex.ecs.ecs_params import TASK_ROLE_T, EXEC_ROLE_T - -RES_KEY = "secrets" -XRES_KEY = "x-secrets" - - -def match_secrets_services_config(service, s_secret, secrets): - """ - Function to match the services and secrets - :param service: - :param s_secret: - :param secrets: - :return: - """ - if isinstance(s_secret, str): - secret_name = s_secret - elif isinstance(s_secret, dict) and keyisset("source", s_secret): - secret_name = s_secret["source"] - else: - raise LookupError("Could not identify the secret source", s_secret) - for gl_secret in secrets: - if gl_secret.name == secret_name: - LOG.info(f"Matched secret {gl_secret.name} with {service.name}") - service.secrets.append(gl_secret) - gl_secret.services.append(service) - - -class ComposeSecret(object): - """ - Class to represent a Compose secret. - """ - - main_key = "secrets" - - def __init__(self, name, definition): - self.services = [] - if not keyisset("Name", definition[XRES_KEY]): - raise KeyError(f"Missing Name in the {XRES_KEY} defintion") - self.name = name - aws_name = definition[XRES_KEY]["Name"] - if aws_name.startswith("arn:"): - self.aws_name = definition[XRES_KEY]["Name"] - self.aws_iam_name = definition[XRES_KEY]["Name"] - else: - self.aws_name = Sub( - f"arn:${{{AWS_PARTITION}}}:secretsmanager:${{{AWS_REGION}}}:${{{AWS_ACCOUNT_ID}}}:secret:{aws_name}" - ) - self.aws_iam_name = Sub( - f"arn:${{{AWS_PARTITION}}}:secretsmanager:${{{AWS_REGION}}}:${{{AWS_ACCOUNT_ID}}}:secret:{aws_name}*" - ) - self.links = ( - definition[XRES_KEY]["LinksTo"] - if keyisset("LinksTo", definition[XRES_KEY]) - else [EXEC_ROLE_T] - ) - self.ecs_secret = EcsSecret(Name=self.name, ValueFrom=self.aws_name) - self.validate_links() - - def validate_links(self): - if not isinstance(self.links, list): - raise TypeError("LinksTo must be of type", list, "Got", type(self.links)) - for link in self.links: - if link not in [EXEC_ROLE_T, TASK_ROLE_T]: - raise ValueError( - "Links in LinksTo can only be one of", - EXEC_ROLE_T, - TASK_ROLE_T, - "Got", - link, - ) diff --git a/ecs_composex/common/compose_services.py b/ecs_composex/common/compose_services.py index a2d298eff..c65756c5e 100644 --- a/ecs_composex/common/compose_services.py +++ b/ecs_composex/common/compose_services.py @@ -38,7 +38,7 @@ from ecs_composex.common import NONALPHANUM, LOG from ecs_composex.common import keyisset, keypresent from ecs_composex.common.cfn_params import ROOT_STACK_NAME -from ecs_composex.common.compose_secrets import ( +from ecs_composex.secrets.compose_secrets import ( ComposeSecret, match_secrets_services_config, ) @@ -410,6 +410,10 @@ def __init__(self, name, definition, volumes=None, secrets=None): self.set_container_definition() def set_container_definition(self): + """ + Function to define the container definition matching the service definition + """ + secrets = [secret for secrets in self.secrets for secret in secrets.ecs_secret] self.container_definition = ContainerDefinition( Image=Ref(self.image_param), Name=self.name, @@ -435,7 +439,7 @@ def set_container_definition(self): HealthCheck=self.ecs_healthcheck, DependsOn=Ref(AWS_NO_VALUE), Essential=True, - Secrets=[secret.ecs_secret for secret in self.secrets], + Secrets=secrets, ) self.container_parameters.update({self.image_param.title: self.image}) @@ -646,20 +650,33 @@ def assign_policy_to_role(role_secrets, role): :param troposphere.iam.Role role: :return: """ + + secrets_list = [secret.arn for secret in role_secrets] + secrets_kms_keys = [secret.kms_key_arn for secret in role_secrets if secret.kms_key] + secrets_statement = { + "Effect": "Allow", + "Action": ["secretsmanager:GetSecretValue"], + "Sid": "AllowSecretsAccess", + "Resource": [secret for secret in secrets_list], + } + secrets_keys_statement = {} + if secrets_kms_keys: + secrets_keys_statement = { + "Effect": "Allow", + "Action": ["kms:Decrypt"], + "Sid": "AllowSecretsKmsKeyDecrypt", + "Resource": [kms_key for kms_key in secrets_kms_keys], + } role_policy = Policy( PolicyName="AccessToPreDefinedSecrets", PolicyDocument={ "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": ["secretsmanager:GetSecretValue"], - "Sid": "AllowSecretsAccess", - "Resource": [secret.aws_iam_name for secret in role_secrets], - } - ], + "Statement": [secrets_statement], }, ) + if secrets_keys_statement: + role_policy.PolicyDocument["Statement"].append(secrets_keys_statement) + if hasattr(role, "Policies") and isinstance(role.Policies, list): existing_policy_names = [ policy.PolicyName for policy in getattr(role, "Policies") @@ -984,7 +1001,6 @@ def set_secrets_access(self): secrets = [] for service in self.services: for secret in service.secrets: - secrets.append(secret) if secrets: assign_secrets_to_roles( diff --git a/ecs_composex/common/settings.py b/ecs_composex/common/settings.py index 17c813258..0176a17db 100644 --- a/ecs_composex/common/settings.py +++ b/ecs_composex/common/settings.py @@ -32,7 +32,7 @@ from ecs_composex.common.aws import get_account_id, get_region_azs from ecs_composex.common.aws import get_cross_role_session from ecs_composex.common.cfn_params import USE_FLEET_T -from ecs_composex.common.compose_secrets import ComposeSecret +from ecs_composex.secrets.compose_secrets import ComposeSecret from ecs_composex.common.compose_services import ( ComposeService, ComposeFamily, @@ -120,8 +120,6 @@ def merge_service_definition(original_def, override_def, nested=False): and key in original_def.keys() and key == "ports" ): - print("Got ports to merge") - print(original_def[key], override_def[key]) original_def[key] = merge_ports(original_def[key], override_def[key]) elif not isinstance(override_def[key], (list, dict)): @@ -381,6 +379,7 @@ def __init__(self, content=None, profile_name=None, session=None, **kwargs): self.volumes = [] self.services = [] self.secrets = [] + self.secrets_mappings = {} self.families = {} self.account_id = None self.output_dir = self.default_output_dir @@ -420,7 +419,7 @@ def set_secrets(self): secret_def["x-secrets"], dict ): LOG.info(f"Adding secret {secret_name} to settings") - secret = ComposeSecret(secret_name, secret_def) + secret = ComposeSecret(secret_name, secret_def, self) self.secrets.append(secret) self.compose_content[ComposeSecret.main_key][secret_name] = secret @@ -456,16 +455,6 @@ def set_services(self): self.compose_content[ComposeService.main_key][service_name] = service self.services.append(service) - def get_family_name(self, family_name): - if family_name != NONALPHANUM.sub("", family_name): - if not NONALPHANUM.sub("", family_name) in self.families.keys(): - LOG.warn( - f"Family name {family_name} must be AlphaNumerical. " - f"Set to {NONALPHANUM.sub('', family_name)}" - ) - family_name = NONALPHANUM.sub("", family_name) - return family_name - def add_new_family(self, family_name, service, assigned_services): if service.name in [service.name for service in assigned_services]: LOG.info( diff --git a/ecs_composex/ecs/ecs_template.py b/ecs_composex/ecs/ecs_template.py index 8d05c0f8a..89e7f47c0 100644 --- a/ecs_composex/ecs/ecs_template.py +++ b/ecs_composex/ecs/ecs_template.py @@ -44,6 +44,7 @@ ) from ecs_composex.ecs.ecs_service_config import ServiceConfig from ecs_composex.vpc import vpc_params +from ecs_composex.secrets.secrets_params import RES_KEY as SECRETS_KEY def initialize_service_template(service_name): @@ -178,11 +179,15 @@ def get_service_family_name(services_families, service_name): def generate_services(settings): """ Function to handle creation of services within the same family. + + :param ecs_composex.common.settings.ComposeXSettings settings: :return: """ for family_name in settings.families: family = settings.families[family_name] family.template = initialize_service_template(family_name) + if settings.secrets_mappings: + family.template.add_mapping(SECRETS_KEY, settings.secrets_mappings) family.init_task_definition() family.set_secrets_access() family.refresh() diff --git a/ecs_composex/kms/kms_params.py b/ecs_composex/kms/kms_params.py index 293bee9f6..c1f117dc5 100644 --- a/ecs_composex/kms/kms_params.py +++ b/ecs_composex/kms/kms_params.py @@ -15,11 +15,19 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import re from os import path from troposphere import Parameter RES_KEY = f"x-{path.basename(path.dirname(path.abspath(__file__)))}" +KMS_KEY_ARN_RE = re.compile( + r"(?:^arn:aws(?:-[a-z]+)?:kms:[\S]+:[0-9]+:)((key/)([\S]+))$" +) +KMS_ALIAS_ARN_RE = re.compile( + r"(?:^arn:aws(?:-[a-z]+)?:kms:[\S]+:[0-9]+:)((alias/)([\S]+))$" +) + KMS_KEY_ARN_T = "Arn" KMS_KEY_ID_T = "KeyId" KMS_KEY_ALIAS_NAME_T = "KmsKeyAliasName" diff --git a/ecs_composex/secrets/compose_secrets.py b/ecs_composex/secrets/compose_secrets.py new file mode 100644 index 000000000..355886232 --- /dev/null +++ b/ecs_composex/secrets/compose_secrets.py @@ -0,0 +1,241 @@ +# -*- coding: utf-8 -*- +# -*- coding: utf-8 -*- +# ECS ComposeX +# Copyright (C) 2020 John Mille +# # +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# # +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# # +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +""" +Represent a service from the docker-compose services +""" + +import re +from copy import deepcopy + +from troposphere import Sub, FindInMap +from troposphere import AWS_PARTITION, AWS_REGION, AWS_ACCOUNT_ID +from troposphere.ecs import Secret as EcsSecret + +from ecs_composex.common import LOG, keyisset, NONALPHANUM +from ecs_composex.ecs.ecs_params import TASK_ROLE_T, EXEC_ROLE_T +from ecs_composex.kms.kms_params import KMS_KEY_ARN_RE +from ecs_composex.secrets.secrets_aws import lookup_secret_config +from ecs_composex.secrets.secrets_params import XRES_KEY, RES_KEY + + +def get_name_from_arn(secret_arn): + secret_re = re.compile( + r"(?:^arn:aws(?:-[a-z]+)?:secretsmanager:[\w-]+:[0-9]{12}:secret:)([\S]+)(?:-[A-Za-z0-9]{1,6})$" + ) + if not secret_re.match(secret_arn): + raise ValueError( + "The secret ARN is invalid", + secret_arn, + "No name cound be found from it via", + r"(?:^arn:aws(?:-[a-z]+)?:secretsmanager:[\w-]+:[0-9]{12}:secret:)([\S]+)(?:-[A-Za-z0-9]{1,6})$", + ) + return secret_re.match(secret_arn).groups()[0] + + +def match_secrets_services_config(service, s_secret, secrets): + """ + Function to match the services and secrets + :param service: + :param s_secret: + :param secrets: + :return: + """ + if isinstance(s_secret, str): + secret_name = s_secret + elif isinstance(s_secret, dict) and keyisset("source", s_secret): + secret_name = s_secret["source"] + else: + raise LookupError("Could not identify the secret source", s_secret) + for gl_secret in secrets: + if gl_secret.name == secret_name: + LOG.info(f"Matched secret {gl_secret.name} with {service.name}") + service.secrets.append(gl_secret) + gl_secret.services.append(service) + + +class ComposeSecret(object): + """ + Class to represent a Compose secret. + """ + + main_key = "secrets" + map_kms_name = "KmsKeyId" + map_arn_name = "Arn" + map_name_name = "Name" + json_keys_key = "JsonKeys" + map_name = RES_KEY + + def __init__(self, name, definition, settings): + self.services = [] + if not keyisset("Name", definition[XRES_KEY]): + raise KeyError(f"Missing Name in the {XRES_KEY} defintion") + self.name = name + self.logical_name = NONALPHANUM.sub("", self.name) + self.definition = deepcopy(definition) + self.links = [EXEC_ROLE_T] + self.arn = None + self.aws_name = None + self.kms_key = None + self.kms_key_arn = None + self.ecs_secret = [] + self.mapping = {} + if not keyisset("Lookup", self.definition[XRES_KEY]): + self.define_names_from_import() + else: + self.define_names_from_lookup(settings.session) + + self.define_links() + self.validate_links() + if self.mapping: + settings.secrets_mappings.update({self.name: self.mapping}) + self.add_json_keys() + + def add_json_keys(self): + """ + Method to add secrets definitions based on JSON secret keys + """ + if not keyisset(self.json_keys_key, self.definition[XRES_KEY]): + return + required_keys = ["Name", "Key"] + for secret_key in self.definition[XRES_KEY][self.json_keys_key]: + if not all(key in required_keys for key in secret_key): + raise KeyError( + "For Secrets JSON Key support, you must specify", + required_keys, + "Got", + secret_key.keys(), + ) + json_key = secret_key["Key"] + secret_name = secret_key["Name"] + if isinstance(self.arn, str): + self.ecs_secret.append( + EcsSecret(Name=secret_name, ValueFrom=f"{self.arn}:{json_key}") + ) + elif isinstance(self.arn, Sub): + self.ecs_secret.append( + EcsSecret( + Name=secret_name, + ValueFrom=Sub( + f"arn:${{{AWS_PARTITION}}}:secretsmanager:${{{AWS_REGION}}}:${{{AWS_ACCOUNT_ID}}}:" + f"secret:${{SecretName}}:{json_key}", + SecretName=FindInMap( + self.map_name, self.name, self.map_name_name + ), + ), + ) + ) + elif isinstance(self.arn, FindInMap): + self.ecs_secret.append( + EcsSecret( + Name=secret_name, + ValueFrom=Sub( + f"${{SecretArn}}:{json_key}", + SecretArn=FindInMap( + self.map_name, self.name, self.map_kms_name + ), + ), + ) + ) + + def define_names_from_import(self): + """ + Method to define the names from docker-compose file content + """ + if not keyisset(self.map_name_name, self.definition[XRES_KEY]): + raise KeyError( + f"Missing {self.map_name_name} when doing non-lookup import for {self.name}" + ) + name_input = self.definition[XRES_KEY][self.map_name_name] + if name_input.startswith("arn:"): + self.aws_name = get_name_from_arn( + self.definition[XRES_KEY][self.map_name_name] + ) + self.mapping = { + self.map_arn_name: name_input, + self.map_name_name: self.aws_name, + } + else: + self.aws_name = name_input + self.mapping = {self.map_name_name: self.aws_name} + self.arn = Sub( + f"arn:${{{AWS_PARTITION}}}:secretsmanager:${{{AWS_REGION}}}:${{{AWS_ACCOUNT_ID}}}:" + "secret:${SecretName}", + SecretName=FindInMap(self.map_name, self.name, self.map_name_name), + ) + self.ecs_secret = [EcsSecret(Name=self.name, ValueFrom=self.arn)] + if keyisset(self.map_kms_name, self.definition): + if not self.definition[self.map_kms_name].startswith( + "arn:" + ) or not KMS_KEY_ARN_RE.match(self.definition[self.map_kms_name]): + LOG.error( + f"When specifying {self.map_kms_name} you must specify the full VALID ARN" + ) + else: + self.mapping[self.map_kms_name] = self.definition[self.map_kms_name] + self.kms_key_arn = FindInMap( + self.map_name, self.name, self.map_kms_name + ) + + def define_names_from_lookup(self, session): + """ + Method to Lookup the secret based on its tags. + :return: + """ + lookup_info = self.definition[XRES_KEY]["Lookup"] + if keyisset("Name", self.definition[XRES_KEY]): + lookup_info["Name"] = self.definition[XRES_KEY]["Name"] + secret_config = lookup_secret_config(self.logical_name, lookup_info, session) + self.aws_name = get_name_from_arn(secret_config[self.logical_name]) + self.arn = secret_config[self.logical_name] + if keyisset("KmsKeyId", secret_config) and not secret_config[ + "KmsKeyId" + ].startswith("alias"): + self.kms_key = secret_config["KmsKeyId"] + elif keyisset("KmsKeyId", secret_config) and secret_config[ + "KmsKeyId" + ].startswith("alias"): + LOG.warn("The KMS Key retrieved is a KMS Key Alias, not importing.") + + self.mapping = { + self.map_arn_name: secret_config[self.logical_name], + self.map_name_name: secret_config[self.map_name_name], + } + if self.kms_key: + self.mapping[self.map_kms_name] = self.kms_key + + self.arn = FindInMap(self.map_name, self.name, self.map_arn_name) + self.kms_key_arn = FindInMap(self.map_name, self.name, self.map_kms_name) + self.ecs_secret = [EcsSecret(Name=self.name, ValueFrom=self.arn)] + + def define_links(self): + if keyisset("LinksTo", self.definition[XRES_KEY]): + self.links = self.definition[XRES_KEY]["LinksTo"] + + def validate_links(self): + if not isinstance(self.links, list): + raise TypeError("LinksTo must be of type", list, "Got", type(self.links)) + for link in self.links: + if link not in [EXEC_ROLE_T, TASK_ROLE_T]: + raise ValueError( + "Links in LinksTo can only be one of", + EXEC_ROLE_T, + TASK_ROLE_T, + "Got", + link, + ) diff --git a/ecs_composex/secrets/secrets_aws.py b/ecs_composex/secrets/secrets_aws.py new file mode 100644 index 000000000..086f2d94f --- /dev/null +++ b/ecs_composex/secrets/secrets_aws.py @@ -0,0 +1,78 @@ +# -*- coding: utf-8 -*- +# ECS ComposeX +# Copyright (C) 2020 John Mille +# # +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# # +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# # +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +""" +Module to find the Secrets from AWS Tags +""" + +import re + +from botocore.exceptions import ClientError + +from ecs_composex.common import LOG, keyisset +from ecs_composex.common.aws import ( + find_aws_resource_arn_from_tags_api, + define_lookup_role_from_info, +) + + +def get_secret_config(logical_name, secret_arn, session): + """ + + :param str secret_arn: + :param boto3.session.Session session: + :return: + """ + + secret_config = {} + client = session.client("secretsmanager") + try: + secret_r = client.describe_secret(SecretId=secret_arn) + secret_config.update({logical_name: secret_r["ARN"], "Name": secret_r["Name"]}) + if keyisset("KmsKeyId", secret_r): + secret_config.update({"KmsKeyId": secret_r["KmsKeyId"]}) + return secret_config + except client.exceptions.ResourceNotFoundException: + return None + except ClientError as error: + LOG.error(error) + raise + + +def lookup_secret_config(logical_name, lookup, session): + """ + Function to find the DB in AWS account + + :param dict lookup: The Lookup definition for DB + :param boto3.session.Session session: Boto3 session for clients + :return: + """ + secrets_types = { + "secretsmanager:secret": { + "regexp": r"(?:^arn:aws(?:-[a-z]+)?:secretsmanager:[\w-]+:[0-9]{12}:secret:)([\S]+)(?:-[A-Za-z0-9]+)$" + }, + } + lookup_session = define_lookup_role_from_info(lookup, session) + secret_arn = find_aws_resource_arn_from_tags_api( + lookup, + lookup_session, + "secretsmanager:secret", + types=secrets_types, + ) + config = get_secret_config(logical_name, secret_arn, lookup_session) + LOG.debug(config) + return config diff --git a/ecs_composex/secrets/secrets_params.py b/ecs_composex/secrets/secrets_params.py index fb453482a..535d20200 100644 --- a/ecs_composex/secrets/secrets_params.py +++ b/ecs_composex/secrets/secrets_params.py @@ -21,6 +21,9 @@ from troposphere import Parameter +RES_KEY = "secrets" +XRES_KEY = "x-secrets" + PASSWORD_LENGTH_T = "PasswordLength" PASSWORD_LENGTH = Parameter( PASSWORD_LENGTH_T, Type="Number", MinValue=8, MaxValue=32, Default=16 diff --git a/pytests/settings/role_arn/sts.AssumeRole_1.json b/pytests/settings/role_arn/sts.AssumeRole_1.json index d54826874..249dd5789 100644 --- a/pytests/settings/role_arn/sts.AssumeRole_1.json +++ b/pytests/settings/role_arn/sts.AssumeRole_1.json @@ -2,32 +2,32 @@ "status_code": 200, "data": { "Credentials": { - "AccessKeyId": "ASIAVOAWVJQOEVY2AYGS", - "SecretAccessKey": "uqF5Ju6SiZONEREDACTED123p5akjxfOJtmbT/V0ncDrcGwJKEta", - "SessionToken": "FwoGZONEREDACTED123vYXdzEKn//////////wEaDOx+Sc+2WVjMjoMl6CKzAV+99cj7E5U7EGjZuLX+kebhSKEA+9cytjwyWAz593lUJWXoF3u7xA5gBUtksMxS/rxYfOU6uvB71vOz5CjfjUNxnpBk+V9yvPvQCRjhDy8+iWa97U123gyOFLXFD0Hs7IUBBAUpUwfXy3FxDhWLIjoHxfp7lVwu7uTV1AXGhR6A3+uHxGiBT/4OAqrw24DjAYlAqlZONEREDACTED123lZvB38g7d7ZgTfsA2lEGpxz2vBOiwkBfMVm6qizfKLK8kP0FMi072TPEoLu5bIj5AgHVQOnhaDhoucxIAP3linG2SrqlEcRQAwPkHQTp4uFIDgd=", + "AccessKeyId": "ASIAVOAWVJQOJRDSSEU4", + "SecretAccessKey": "JoGpd30/O79jMC61B7i0Yesut/fL4551cubvGAEX", + "SessionToken": "FwoGZONEREDACTED123vYXdzELD//////////wEaDFaKguTeO9joa3Af9yK7AUD8O3AA6TauPCLIxcU26tlz/x3mAfnWlBUyJpEoPNzBS900fEb6E7jX0P3szHn4jJWv00zlZONEREDACTED123o+veIdNCFaU84otMTElyMTAdOFcbEq+0I3bpRnARjEO2tqV37NBvKel/wTYHx3FMrL8ceSZpf0KTpphLu1/Df1NdRQrVbNybBTRW6FqCC4rB4jOBrTW0XT4aUijDe/GbHglPWlpgBGkJuUReZONEREDACTED123f3nquIc5DMS4bwgTxqbBB5HuFVRnUQAo7KjK/QUyLSOob11dyj1hDA8FadDKwPT2AtNgsnhXqnorufGF3bHufTFn/dJkdb6VhX4YBg==", "Expiration": { "__class__": "datetime", "year": 2020, "month": 11, - "day": 5, - "hour": 16, - "minute": 0, - "second": 54, + "day": 16, + "hour": 15, + "minute": 17, + "second": 4, "microsecond": 0 } }, "AssumedRoleUser": { - "AssumedRoleId": "AROAVOAWVJQOLB24ILL25:ComposeX@render", - "Arn": "arn:aws:sts::000000000000:assumed-role/testx/ComposeX@render" + "AssumedRoleId": "AROAVOAWVJQOLB24ILL24:ComposeXSettings@render", + "Arn": "arn:aws:sts::000000000000:assumed-role/testx/ComposeXSettings@render" }, "ResponseMetadata": { - "RequestId": "e6ed5816-c773-4e7c-a9f5-567fe35334ad", + "RequestId": "8ad0a722-2255-4949-b308-8c5a0aadaaec", "HTTPStatusCode": 200, "HTTPHeaders": { - "x-amzn-requestid": "e6ed5816-c773-4e7c-a9f5-567fe35334ad", + "x-amzn-requestid": "8ad0a722-2255-4949-b308-8c5a0aadaaec", "content-type": "text/xml", - "content-length": "1062", - "date": "Thu, 05 Nov 2020 15:45:54 GMT" + "content-length": "1090", + "date": "Mon, 16 Nov 2020 15:02:03 GMT" }, "RetryAttempts": 0 } diff --git a/pytests/settings/role_arn/sts.AssumeRole_2.json b/pytests/settings/role_arn/sts.AssumeRole_2.json index 6ee16dc27..93923417d 100644 --- a/pytests/settings/role_arn/sts.AssumeRole_2.json +++ b/pytests/settings/role_arn/sts.AssumeRole_2.json @@ -7,13 +7,13 @@ "Message": "User: arn:aws:sts::000000000000:assumed-role/AWSReservedSSO_PowerUserAccess_e9ddee7954f3b1a9/john@ews-network.net is not authorized to perform: sts:AssumeRole on resource: arn:aws:iam::000000000000:role/test" }, "ResponseMetadata": { - "RequestId": "3232e2a4-3bba-4068-905a-c9eed6aa3e4f", + "RequestId": "a9e087b4-adb1-4029-842b-ef7ec8eef750", "HTTPStatusCode": 403, "HTTPHeaders": { - "x-amzn-requestid": "3232e2a4-3bba-4068-905a-c9eed6aa3e4f", + "x-amzn-requestid": "a9e087b4-adb1-4029-842b-ef7ec8eef750", "content-type": "text/xml", "content-length": "451", - "date": "Thu, 05 Nov 2020 15:47:19 GMT" + "date": "Mon, 16 Nov 2020 15:02:03 GMT" }, "RetryAttempts": 0 } diff --git a/pytests/settings/secrets/secretsmanager.DescribeSecret_1.json b/pytests/settings/secrets/secretsmanager.DescribeSecret_1.json new file mode 100644 index 000000000..0502c25bc --- /dev/null +++ b/pytests/settings/secrets/secretsmanager.DescribeSecret_1.json @@ -0,0 +1,66 @@ +{ + "status_code": 200, + "data": { + "ARN": "arn:aws:secretsmanager:eu-west-1:000000000000:secret:secret/with/kmskey-3MmbWA", + "Name": "secret/with/kmskey", + "Description": "Secret with non default KMS Key encryption", + "KmsKeyId": "arn:aws:kms:eu-west-1:000000000000:key/55431c45-325f-4d5e-828c-58d027a8f26f", + "LastChangedDate": { + "__class__": "datetime", + "year": 2020, + "month": 11, + "day": 16, + "hour": 13, + "minute": 37, + "second": 1, + "microsecond": 127000 + }, + "LastAccessedDate": { + "__class__": "datetime", + "year": 2020, + "month": 11, + "day": 16, + "hour": 0, + "minute": 0, + "second": 0, + "microsecond": 0 + }, + "Tags": [ + { + "Key": "composexdev", + "Value": "yes" + }, + { + "Key": "costcentre", + "Value": "lambda" + } + ], + "VersionIdsToStages": { + "44a02c36-4b87-47f3-9a59-d32e6a98bbb6": [ + "AWSCURRENT" + ] + }, + "CreatedDate": { + "__class__": "datetime", + "year": 2020, + "month": 11, + "day": 16, + "hour": 9, + "minute": 16, + "second": 56, + "microsecond": 556000 + }, + "ResponseMetadata": { + "RequestId": "3338c2c4-b5f1-4645-a9f8-c92a2f43fb46", + "HTTPStatusCode": 200, + "HTTPHeaders": { + "date": "Mon, 16 Nov 2020 14:44:04 GMT", + "content-type": "application/x-amz-json-1.1", + "content-length": "521", + "connection": "keep-alive", + "x-amzn-requestid": "3338c2c4-b5f1-4645-a9f8-c92a2f43fb46" + }, + "RetryAttempts": 0 + } + } +} diff --git a/pytests/settings/secrets/tagging.GetResources_1.json b/pytests/settings/secrets/tagging.GetResources_1.json new file mode 100644 index 000000000..9d2e990c1 --- /dev/null +++ b/pytests/settings/secrets/tagging.GetResources_1.json @@ -0,0 +1,32 @@ +{ + "status_code": 200, + "data": { + "PaginationToken": "", + "ResourceTagMappingList": [ + { + "ResourceARN": "arn:aws:secretsmanager:eu-west-1:000000000000:secret:secret/with/kmskey-3MmbWA", + "Tags": [ + { + "Key": "composexdev", + "Value": "yes" + }, + { + "Key": "costcentre", + "Value": "lambda" + } + ] + } + ], + "ResponseMetadata": { + "RequestId": "155e023f-41fe-44cc-a7e0-57e323235f76", + "HTTPStatusCode": 200, + "HTTPHeaders": { + "x-amzn-requestid": "155e023f-41fe-44cc-a7e0-57e323235f76", + "content-type": "application/x-amz-json-1.1", + "content-length": "229", + "date": "Mon, 16 Nov 2020 14:44:04 GMT" + }, + "RetryAttempts": 0 + } + } +} diff --git a/pytests/test_common_aws.py b/pytests/test_common_aws.py index ce527cc18..f0a4ec339 100644 --- a/pytests/test_common_aws.py +++ b/pytests/test_common_aws.py @@ -36,7 +36,7 @@ def res_types(): "regexp": r"(?:^arn:aws(?:-[a-z]+)?:rds:[\w-]+:[0-9]{12}:cluster:)([\S]+)$" }, "secret": { - "regexp": r"(?:^arn:aws(?:-[a-z]+)?:secretsmanager:[\w-]+:[0-9]{12}:secret:)([\S]+)(?:-[A-Za-z0-9]+)$" + "regexp": r"(?:^arn:aws(?:-[a-z]+)?:secretsmanager:[\w-]+:[0-9]{12}:secret:)([\S]+)(?:-[A-Za-z0-9]{1,6})$" }, } diff --git a/pytests/test_settings.py b/pytests/test_settings.py index 589157b2d..89e0265f4 100644 --- a/pytests/test_settings.py +++ b/pytests/test_settings.py @@ -48,7 +48,13 @@ def test_export_attribute(): generate_export_strings("toto", 123) -def get_content(): +def get_basic_content(): + here = path.abspath(path.dirname(__file__)) + content = load_composex_file(f"{here}/../use-cases/blog.yml") + return deepcopy(content) + + +def get_secrets_content(): here = path.abspath(path.dirname(__file__)) content = load_composex_file(f"{here}/../use-cases/blog.features.yml") return deepcopy(content) @@ -59,30 +65,32 @@ def test_iam_role_arn(): here = path.abspath(path.dirname(__file__)) session = boto3.session.Session() pill = placebo.attach(session, data_path=f"{here}/{case_path}") + # pill.record() pill.playback() - ComposeXSettings( - content=get_content(), + settings = ComposeXSettings( + content=get_basic_content(), session=session, **{ ComposeXSettings.name_arg: "test", ComposeXSettings.command_arg: ComposeXSettings.render_arg, ComposeXSettings.input_file_arg: path.abspath( - f"{here}/../uses-cases/blog.features.yml" + f"{here}/../uses-cases/blog.yml" ), ComposeXSettings.format_arg: "yaml", ComposeXSettings.arn_arg: "arn:aws:iam::012345678912:role/testx", }, ) + print(settings.secrets_mappings) with raises(ValueError): ComposeXSettings( - content=get_content(), + content=get_basic_content(), session=session, **{ ComposeXSettings.name_arg: "test", ComposeXSettings.command_arg: ComposeXSettings.render_arg, ComposeXSettings.input_file_arg: path.abspath( - f"{here}/../uses-cases/blog.features.yml" + f"{here}/../uses-cases/blog.yml" ), ComposeXSettings.format_arg: "yaml", ComposeXSettings.arn_arg: "arn:aws:iam::012345678912:roleX/testx", @@ -90,15 +98,40 @@ def test_iam_role_arn(): ) with raises(ClientError): ComposeXSettings( - content=get_content(), + content=get_basic_content(), session=session, **{ ComposeXSettings.name_arg: "test", ComposeXSettings.command_arg: ComposeXSettings.render_arg, ComposeXSettings.input_file_arg: path.abspath( - f"{here}/../uses-cases/blog.features.yml" + f"{here}/../uses-cases/blog.yml" ), ComposeXSettings.format_arg: "yaml", ComposeXSettings.arn_arg: "arn:aws:iam::012345678912:role/test", }, ) + + +def test_secrets_import(): + """ + Function to test secrets import + """ + case_path = "settings/secrets" + here = path.abspath(path.dirname(__file__)) + session = boto3.session.Session() + pill = placebo.attach(session, data_path=f"{here}/{case_path}") + # pill.record() + pill.playback() + + settings = ComposeXSettings( + content=get_secrets_content(), + session=session, + **{ + ComposeXSettings.name_arg: "test", + ComposeXSettings.command_arg: ComposeXSettings.render_arg, + ComposeXSettings.input_file_arg: path.abspath( + f"{here}/../uses-cases/blog.features.yml" + ), + ComposeXSettings.format_arg: "yaml", + }, + ) diff --git a/use-cases/blog.features.yml b/use-cases/blog.features.yml index 24f334de9..57c1aa940 100644 --- a/use-cases/blog.features.yml +++ b/use-cases/blog.features.yml @@ -8,7 +8,14 @@ secrets: Name: SFTP/asl-cscs-files-dev zyx: x-secrets: - Name: SFTP/smicuser + Name: secret/with/kmskey + Lookup: + Tags: + - costcentre: lambda + - composexdev: "yes" + JsonKeys: + - Name: ZYX_TEST + Key: test services: app01: deploy: