Feature x-s3 (#196)

docs/features.rst

@@ -11,6 +11,8 @@
 
 .. include:: ../ecs_composex/dynamodb/README.rst
 
+.. include:: ../ecs_composex/s3/README.rst
+
 .. include:: ../ecs_composex/vpc/README.rst
 
 .. include:: ../ecs_composex/kms/README.rst

docs/modules_syntax.rst

@@ -15,6 +15,8 @@
 
 .. include:: ../ecs_composex/dynamodb/SYNTAX.rst
 
+.. include:: ../ecs_composex/s3/SYNTAX.rst
+
 .. include:: ../ecs_composex/secrets/SYNTAX.rst
 
 .. include:: ../ecs_composex/sns/SYNTAX.rst

ecs_composex/common/aws.py

@@ -42,19 +42,18 @@ def define_tagsgroups_filter_tags(tags):
     return filters
 
 
-def get_resources_from_tags(session, service_code, res_type, search_tags):
+def get_resources_from_tags(session, aws_resource_search, search_tags):
     """
 
     :param boto3.session.Session session: The boto3 session for API calls
-    :param str service_code: AWS Service short code, ie. rds, ec2
-    :param str res_type: Resource type we are after within the AWS Service, ie. cluster, instance
+    :param str aws_resource_search: AWS Service short code, ie. rds, ec2
     :param list search_tags: The tags to search the resource with.
     :return:
     """
     try:
         client = session.client("resourcegroupstaggingapi")
         resources_r = client.get_resources(
-            ResourceTypeFilters=[f"{service_code}:{res_type}"], TagFilters=search_tags
+            ResourceTypeFilters=[aws_resource_search], TagFilters=search_tags
         )
         return resources_r
     except ClientError as error:
@@ -100,29 +99,30 @@ def handle_multi_results(arns, name, res_type, regexp):
         )
 
 
-def handle_search_results(arns, name, res_types, res_type, service_code):
+def handle_search_results(arns, name, res_types, aws_resource_search):
     """
     Function to parse tag resource search results
 
     :param list arns:
     :param str name:
     :param dict res_types:
-    :param str res_type:
-    :param str service_code:
+    :param str aws_resource_search:
     :return:
     """
     if not arns:
         raise LookupError(
             "No resources were found with the provided tags and information"
         )
     if arns and isinstance(name, str):
-        return handle_multi_results(arns, name, res_type, res_types[res_type]["regexp"])
+        return handle_multi_results(
+            arns, name, aws_resource_search, res_types[aws_resource_search]["regexp"]
+        )
     elif not name and len(arns) == 1:
-        LOG.info(f"Matched {service_code}:{res_type}")
+        LOG.info(f"Matched {aws_resource_search} to AWS Resource")
         return arns[0]
     elif not name and len(arns) != 1:
         raise LookupError(
-            f"More than one {service_code}:{res_type} was found with the current tags."
+            f"More than one resource {name}:{aws_resource_search} was found with the current tags."
             "Found",
             arns,
         )
@@ -132,9 +132,8 @@ def validate_search_input(res_types, res_type):
     """
     Function to validate the search query
 
-    :param info:
-    :param res_types:
-    :param res_type:
+    :param dict res_types:
+    :param str res_type:
     :return:
     """
 
@@ -146,34 +145,32 @@ def validate_search_input(res_types, res_type):
         )
 
 
-def find_aws_resource_arn_from_tags_api(
-    info, session, service_code, res_type, types=None
-):
+def find_aws_resource_arn_from_tags_api(info, session, aws_resource_search, types=None):
     """
     Function to find the RDS DB based on info
 
     :param dict info:
     :param boto3.session.Session session: Boto3 session for clients
-    :param str res_type: Resource type we are after within the AWS Service, ie. cluster, instance
-    :param str service_code: AWS Service short code, ie. rds, ec2
+    :param str aws_resource_search: Resource type we are after within the AWS Service, ie. cluster, instance
     :param dict types: Additional types to match.
     :return:
     """
     res_types = {
-        "secret": {
+        "secretsmanager:secret": {
             "regexp": r"(?:^arn:aws(?:-[a-z]+)?:secretsmanager:[\w-]+:[0-9]{12}:secret:)([\S]+)(?:-[A-Za-z0-9]+)$"
         },
     }
     if types is not None and isinstance(types, dict):
         res_types.update(types)
-    validate_search_input(res_types, res_type)
+    validate_search_input(res_types, aws_resource_search)
     search_tags = (
         define_tagsgroups_filter_tags(info["Tags"]) if keyisset("Tags", info) else ()
     )
     name = info["Name"] if keyisset("Name", info) else None
-    resources_r = get_resources_from_tags(session, service_code, res_type, search_tags)
+
+    resources_r = get_resources_from_tags(session, aws_resource_search, search_tags)
     arns = [i["ResourceARN"] for i in resources_r["ResourceTagMappingList"]]
-    return handle_search_results(arns, name, res_types, res_type, service_code)
+    return handle_search_results(arns, name, res_types, aws_resource_search)
 
 
 def get_region_azs(session):

ecs_composex/common/compose_resources.py

@@ -22,7 +22,7 @@
 from troposphere import Sub
 from troposphere.ecs import Environment
 
-from ecs_composex.common import NONALPHANUM, keyisset
+from ecs_composex.common import LOG, NONALPHANUM, keyisset, keypresent
 from ecs_composex.resource_settings import generate_export_strings
 from ecs_composex.common.cfn_params import ROOT_STACK_NAME
 
@@ -38,9 +38,12 @@ def set_resources(settings, resource_class, res_key):
     if not keyisset(res_key, settings.compose_content):
         return
     for resource_name in settings.compose_content[res_key]:
-        settings.compose_content[res_key][resource_name] = resource_class(
+        new_definition = resource_class(
             resource_name, settings.compose_content[res_key][resource_name]
         )
+        LOG.debug(type(new_definition))
+        LOG.debug(new_definition.__dict__)
+        settings.compose_content[res_key][resource_name] = new_definition
 
 
 class Service(object):
@@ -90,11 +93,14 @@ def __init__(self, name, definition):
             if not keyisset("Settings", self.definition)
             else self.definition["Settings"]
         )
-        self.properties = (
-            None
-            if not keyisset("Properties", self.definition)
-            else self.definition["Properties"]
-        )
+        if keyisset("Properties", self.definition):
+            self.properties = self.definition["Properties"]
+        elif not keyisset("Properties", self.definition) and keypresent(
+            "Properties", self.definition
+        ):
+            self.properties = {}
+        else:
+            self.properties = None
         self.services = (
             []
             if not keyisset("Services", self.definition)
@@ -113,12 +119,14 @@ def __init__(self, name, definition):
     def __repr__(self):
         return self.logical_name
 
-    def generate_resource_envvars(self, attribute):
+    def generate_resource_envvars(self, attribute, arn=None):
         """
         :return: environment key/pairs
         :rtype: list<troposphere.ecs.Environment>
         """
-        export_string = generate_export_strings(self.logical_name, attribute)
+        export_string = (
+            generate_export_strings(self.logical_name, attribute) if not arn else arn
+        )
         if self.settings and keyisset("EnvNames", self.settings):
             for env_name in self.settings["EnvNames"]:
                 self.env_vars.append(

ecs_composex/common/stacks.py

@@ -69,6 +69,7 @@ class ComposeXStack(Stack, object):
         "UpdatePolicy",
         "UpdateReplacePolicy",
     ]
+    is_void = False
 
     def __init__(
         self, title, stack_template, stack_parameters=None, file_name=None, **kwargs

ecs_composex/ecs_composex.py

@@ -77,6 +77,8 @@
     "dynamodb",
     f"{X_KEY}kms",
     "kms",
+    f"{X_KEY}s3",
+    "s3",
 ]
 EXCLUDED_X_KEYS = [
     f"{X_KEY}configs",
@@ -142,6 +144,29 @@ def get_mod_class(module_name):
     return the_class
 
 
+def invoke_x_to_ecs(module, settings, services_stack, services_families, resource):
+    """
+
+    :param str module:
+    :param ecs_composex.common.settings.ComposeXSettings settings: The compose file content
+    :param ecs_composex.ecs.ServicesStack services_stack: root stack for services.
+    :param dict services_families: Families and services mappings
+    :param resource: The XStack resource
+    :return:
+    """
+    composex_key = f"{X_KEY}{module}"
+    ecs_function = get_mod_function(f"{module}.{module}_ecs", f"{module}_to_ecs")
+    if ecs_function:
+        LOG.debug(ecs_function)
+        ecs_function(
+            settings.compose_content[composex_key],
+            services_stack,
+            services_families,
+            resource,
+            settings,
+        )
+
+
 def apply_x_configs_to_ecs(
     settings, root_template, services_stack, services_families, **kwargs
 ):
@@ -161,21 +186,12 @@ def apply_x_configs_to_ecs(
         if (
             issubclass(type(resource), ComposeXStack)
             and resource_name in SUPPORTED_X_MODULES
+            and not resource.is_void
         ):
             module = getattr(resource, "title")
-            composex_key = f"{X_KEY}{module}"
-            ecs_function = get_mod_function(
-                f"{module}.{module}_ecs", f"{module}_to_ecs"
+            invoke_x_to_ecs(
+                module, settings, services_stack, services_families, resource
             )
-            if ecs_function:
-                LOG.debug(ecs_function)
-                ecs_function(
-                    settings.compose_content[composex_key],
-                    services_stack,
-                    services_families,
-                    resource,
-                    settings,
-                )
 
 
 def apply_x_to_x_configs(root_template, settings):
@@ -192,6 +208,7 @@ def apply_x_to_x_configs(root_template, settings):
             issubclass(type(resource), ComposeXStack)
             and resource_name in SUPPORTED_X_MODULES
             and hasattr(resource, "add_xdependencies")
+            and not resource.is_void
         ):
             resource.add_xdependencies(root_template, settings.compose_content)
 
@@ -227,11 +244,39 @@ def add_compute(root_template, settings, vpc_stack):
     return root_template.add_resource(compute_stack)
 
 
-def add_x_resources(root_template, settings, vpc_stack=None):
+def handle_new_xstack(
+    key,
+    res_type,
+    settings,
+    services_families,
+    services_stack,
+    vpc_stack,
+    root_template,
+    xstack,
+):
+    tcp_services = ["x-rds", "x-appmesh"]
+    if vpc_stack and key in tcp_services:
+        xstack.get_from_vpc_stack(vpc_stack)
+    elif not vpc_stack and key in tcp_services:
+        xstack.no_vpc_parameters()
+    LOG.debug(xstack, xstack.is_void)
+    if xstack.is_void:
+        invoke_x_to_ecs(res_type, settings, services_stack, services_families, xstack)
+    elif (
+        hasattr(xstack, "title")
+        and hasattr(xstack, "stack_template")
+        and not xstack.is_void
+    ):
+        root_template.add_resource(xstack)
+
+
+def add_x_resources(
+    root_template, settings, services_stack, services_families, vpc_stack=None
+):
     """
     Function to add each X resource from the compose file
     """
-    tcp_services = ["x-rds", "x-appmesh"]
+
     for key in settings.compose_content:
         if key.startswith(X_KEY) and key not in EXCLUDED_X_KEYS:
             res_type = RES_REGX.sub("", key)
@@ -240,17 +285,23 @@ def add_x_resources(root_template, settings, vpc_stack=None):
             LOG.debug(xclass)
             if not xclass:
                 LOG.info(f"Class for {res_type} not found")
+                xstack = None
             else:
                 xstack = xclass(
                     res_type.strip(),
                     settings=settings,
                     Parameters=parameters,
                 )
-                if vpc_stack and key in tcp_services:
-                    xstack.get_from_vpc_stack(vpc_stack)
-                elif not vpc_stack and key in tcp_services:
-                    xstack.no_vpc_parameters()
-                root_template.add_resource(xstack)
+            handle_new_xstack(
+                key,
+                res_type,
+                settings,
+                services_families,
+                services_stack,
+                vpc_stack,
+                root_template,
+                xstack,
+            )
 
 
 def create_services(root_stack, settings, vpc_stack, dns_params, create_cluster):
@@ -331,7 +382,13 @@ def generate_full_template(settings):
     services_stack = create_services(
         root_stack, settings, vpc_stack, dns_settings.nested_params, create_cluster
     )
-    add_x_resources(root_stack.stack_template, settings, vpc_stack=vpc_stack)
+    add_x_resources(
+        root_stack.stack_template,
+        settings,
+        services_stack,
+        services_families,
+        vpc_stack=vpc_stack,
+    )
     apply_x_configs_to_ecs(
         settings, root_stack.stack_template, services_stack, services_families
     )

ecs_composex/rds/rds_aws.py

@@ -128,7 +128,7 @@ def handle_secret(lookup, db_config, session):
     """
     if keyisset("secret", lookup):
         secret_arn = find_aws_resource_arn_from_tags_api(
-            lookup["secret"], session, "secretsmanager", "secret"
+            lookup["secret"], session, "secretsmanager:secret"
         )
         if secret_arn and db_config:
             db_config.update({"SecretArn": secret_arn})
@@ -159,8 +159,10 @@ def lookup_rds_resource(lookup, session):
     :return:
     """
     rds_types = {
-        "db": {"regexp": r"(?:^arn:aws(?:-[a-z]+)?:rds:[\w-]+:[0-9]{12}:db:)([\S]+)$"},
-        "cluster": {
+        "rds:db": {
+            "regexp": r"(?:^arn:aws(?:-[a-z]+)?:rds:[\w-]+:[0-9]{12}:db:)([\S]+)$"
+        },
+        "rds:cluster": {
             "regexp": r"(?:^arn:aws(?:-[a-z]+)?:rds:[\w-]+:[0-9]{12}:cluster:)([\S]+)$"
         },
     }
@@ -170,7 +172,7 @@ def lookup_rds_resource(lookup, session):
     elif keyisset("db", lookup):
         res_type = "db"
     db_arn = find_aws_resource_arn_from_tags_api(
-        lookup[res_type], session, "rds", res_type, types=rds_types
+        lookup[res_type], session, f"rds:{res_type}", types=rds_types
     )
     if not db_arn:
         return None

ecs_composex/rds/rds_template.py

@@ -21,7 +21,7 @@
 
 from troposphere import Ref, Join
 
-from ecs_composex.common import build_template, validate_kwargs, keyisset, LOG
+from ecs_composex.common import build_template, validate_kwargs, LOG
 from ecs_composex.common.cfn_params import ROOT_STACK_NAME_T, ROOT_STACK_NAME
 from ecs_composex.common.stacks import ComposeXStack
 from ecs_composex.rds.rds_db_template import (