x-secrets::Lookup (#256)

* Making secrets a package of their own

* Lookup working and rework to put secrets into mappings to make it easier to track

* Added KMS KEY Arn validation

* Updated docs and regex

* Adding JSON Keys support

docs/syntax/docker-compose/secrets.rst

@@ -9,31 +9,54 @@ As you might have already used these, docker-compose allows you to define secret
 To help continue with docker-compose syntax compatibility, you can now declare your secret in docker-compose,
 and add an extension field which will be a direct mapping to the secret name you have in AWS Secrets Manager.
 
-.. code-block:: yaml
+ECS ComposeX will automatically add IAM permissions to **the execution** role of your Task definition and will export the secret
+to your container, using the same name as in the compose file.
 
-    secrets:
-      topsecret_info:
-        x-secrets:
-          Name: /path/to/my/secret
+.. seealso::
 
-    services:
-      serviceA:
-        secrets:
-          - topsecret_info
+    `docker-compose secrets reference`_
 
-This will automatically add IAM permissions to **the execution** role of your Task definition and will export the secret
-to your container, using the same name as in the compose file.
+.. hint::
 
-.. note::
+    For security purposes, the containers **envoy** and **xray-daemon** are not getting assigned the secrets.
 
-    Only Fargate 1.4.0+ Platform Version supports secrets JSON Key
+
+Syntax
+======
+
+.. code-block::
+
+    x-secrets:
+      Name: str
+      LinksTo: []
+      JsonKeys: []
+      Lookup: {}
+
+Name
+----
+
+Type: String
+
+The name of the secret in secrets manager to use and import.
 
 .. hint::
 
-    If you believe that your service application should have access to the secret via **Task Role**, simply add to the
-    secret definition as follows:
+    If you want to put the full ARN, you can. There will be a validation for it.
+
+LinksTo
+-------
+
+Type: List of Strings
+
+AllowedValues:
+
+* EcsExecutionRole
+* EcsTaskRole
 
-    .. code-block:: yaml
+If you believe that your service application should have access to the secret via **Task Role**, simply add to the
+secret definition as follows:
+
+.. code-block:: yaml
 
         secret-name:
           x-secrets:
@@ -47,14 +70,89 @@ to your container, using the same name as in the compose file.
     If you do not specify **EcsExecutionRole** when specifying **LinksTo** then you will not get the secret exposed
     to your container via AWS ECS Secrets property of your Container Definition
 
-.. hint::
+JsonKeys
+--------
 
-    For security purposes, the containers **envoy** and **xray-daemon** are not getting assigned the secrets.
+Type: List of objects/dicts
 
+.. note::
 
-.. seealso::
+    Only Fargate 1.4.0+ Platform Version supports secrets JSON Key
 
-    `docker-compose secrets reference`_
+.. code-block:: yaml
+    :caption: JsonKeys objects structure
 
+    Key: str
+    Name: str
+
+Key
+"""
+
+Name of the JSON Key in your secret.
+
+Name
+""""
+
+The Name of the secret specifically for the secret JSON key
+
+
+Examples
+========
+
+.. code-block:: yaml
+    :caption: Short example
+
+    secrets:
+      topsecret_info:
+        x-secrets:
+          Name: /path/to/my/secret
+
+    services:
+      serviceA:
+        secrets:
+          - topsecret_info
+
+.. code-block:: yaml
+    :caption: Secret with assignment to Task and Execution Role
+
+    secrets:
+      abcd: {}
+      john:
+        x-secrets:
+          LinksTo:
+            - EcsExecutionRole
+            - EcsTaskRole
+          Name: SFTP/asl-cscs-files-dev
+
+
+.. code-block:: yaml
+    :caption: Secret Looked up from Tags and Name, also using JsonKeys
+
+    secrets:
+      zyx:
+        x-secrets:
+          Name: secret/with/kmskey
+          Lookup:
+            Tags:
+              - costcentre: lambda
+              - composexdev: "yes"
+          JsonKeys:
+            - Key: username
+              Name: PSQL_USERNAME
+            - Key: password
+              Name: PSQL_PASSWORD
+
+
+.. code-block:: yaml
+    :caption: Secret with assignment to Task and Execution Role
+
+    secrets:
+      abcd: {}
+      john:
+        x-secrets:
+          LinksTo:
+            - EcsExecutionRole
+            - EcsTaskRole
+          Name: arn:aws:secretsmanager:eu-west-1:123456789012:secret:/secret/abcd
 
 .. _docker-compose secrets reference: https://docs.docker.com/compose/compose-file/#secrets

ecs_composex/common/aws.py

@@ -225,7 +225,7 @@ def find_aws_resource_arn_from_tags_api(
     """
     res_types = {
         "secretsmanager:secret": {
-            "regexp": r"(?:^arn:aws(?:-[a-z]+)?:secretsmanager:[\w-]+:[0-9]{12}:secret:)([\S]+)(?:-[A-Za-z0-9]+)$"
+            "regexp": r"(?:^arn:aws(?:-[a-z]+)?:secretsmanager:[\w-]+:[0-9]{12}:secret:)([\S]+)(?:-[A-Za-z0-9]{1,6})$"
         },
     }
     if types is not None and isinstance(types, dict):
@@ -238,7 +238,10 @@ def find_aws_resource_arn_from_tags_api(
 
     resources_r = get_resources_from_tags(session, aws_resource_search, search_tags)
     LOG.debug(search_tags)
-    arns = [i["ResourceARN"] for i in resources_r["ResourceTagMappingList"]]
+    if not resources_r or not keyisset("ResourceTagMappingList", resources_r):
+        arns = []
+    else:
+        arns = [i["ResourceARN"] for i in resources_r["ResourceTagMappingList"]]
     return handle_search_results(
         arns, name, res_types, aws_resource_search, allow_multi=allow_multi
     )

ecs_composex/common/compose_services.py

@@ -38,7 +38,7 @@
 from ecs_composex.common import NONALPHANUM, LOG
 from ecs_composex.common import keyisset, keypresent
 from ecs_composex.common.cfn_params import ROOT_STACK_NAME
-from ecs_composex.common.compose_secrets import (
+from ecs_composex.secrets.compose_secrets import (
     ComposeSecret,
     match_secrets_services_config,
 )
@@ -410,6 +410,10 @@ def __init__(self, name, definition, volumes=None, secrets=None):
         self.set_container_definition()
 
     def set_container_definition(self):
+        """
+        Function to define the container definition matching the service definition
+        """
+        secrets = [secret for secrets in self.secrets for secret in secrets.ecs_secret]
         self.container_definition = ContainerDefinition(
             Image=Ref(self.image_param),
             Name=self.name,
@@ -435,7 +439,7 @@ def set_container_definition(self):
             HealthCheck=self.ecs_healthcheck,
             DependsOn=Ref(AWS_NO_VALUE),
             Essential=True,
-            Secrets=[secret.ecs_secret for secret in self.secrets],
+            Secrets=secrets,
         )
         self.container_parameters.update({self.image_param.title: self.image})
 
@@ -646,20 +650,33 @@ def assign_policy_to_role(role_secrets, role):
     :param troposphere.iam.Role role:
     :return:
     """
+
+    secrets_list = [secret.arn for secret in role_secrets]
+    secrets_kms_keys = [secret.kms_key_arn for secret in role_secrets if secret.kms_key]
+    secrets_statement = {
+        "Effect": "Allow",
+        "Action": ["secretsmanager:GetSecretValue"],
+        "Sid": "AllowSecretsAccess",
+        "Resource": [secret for secret in secrets_list],
+    }
+    secrets_keys_statement = {}
+    if secrets_kms_keys:
+        secrets_keys_statement = {
+            "Effect": "Allow",
+            "Action": ["kms:Decrypt"],
+            "Sid": "AllowSecretsKmsKeyDecrypt",
+            "Resource": [kms_key for kms_key in secrets_kms_keys],
+        }
     role_policy = Policy(
         PolicyName="AccessToPreDefinedSecrets",
         PolicyDocument={
             "Version": "2012-10-17",
-            "Statement": [
-                {
-                    "Effect": "Allow",
-                    "Action": ["secretsmanager:GetSecretValue"],
-                    "Sid": "AllowSecretsAccess",
-                    "Resource": [secret.aws_iam_name for secret in role_secrets],
-                }
-            ],
+            "Statement": [secrets_statement],
         },
     )
+    if secrets_keys_statement:
+        role_policy.PolicyDocument["Statement"].append(secrets_keys_statement)
+
     if hasattr(role, "Policies") and isinstance(role.Policies, list):
         existing_policy_names = [
             policy.PolicyName for policy in getattr(role, "Policies")
@@ -984,7 +1001,6 @@ def set_secrets_access(self):
             secrets = []
             for service in self.services:
                 for secret in service.secrets:
-
                     secrets.append(secret)
             if secrets:
                 assign_secrets_to_roles(