Improved logging settings (#265)

* Small fix on Secrets JSON Keys
* Working logging parameters
* Added docs

docs/modules_syntax.rst

@@ -8,12 +8,18 @@
     syntax/docker-compose/volumes
     syntax/docker-compose/secrets
 
+.. toctree::
+    :caption: Services Extension Fields
+    :maxdepth: 1
+
+    syntax/docker-compose/ecs
+    syntax/docker-compose/ecs.details/logging
+
 .. toctree::
     :caption: ComposeX syntax
     :maxdepth: 1
 
     syntax/composex/common
-    syntax/composex/ecs
     syntax/composex/appmesh
     syntax/composex/elbv2
     syntax/composex/rds

docs/syntax/composex/ecs.details/iam.rst → docs/syntax/docker-compose/ecs.details/iam.rst

@@ -4,11 +4,12 @@
 iam
 ====
 
+.. contents::
 
 This section is the entrypoint to further extension of IAM definition for the IAM roles created throughout.
 
 boundary
-""""""""
+========
 
 This key represents an IAM policy (name or ARN) that needs to be added to the IAM roles in order to represent the IAM
 Permissions Boundary.
@@ -43,7 +44,7 @@ Examples:
     if you specify ony the name, ie. containers, this will resolve into arn:${partition}:iam::${accountId}:policy/containers
 
 policies
-"""""""""
+========
 
 Allows you to define additional IAM policies.
 Follows the same pattern as CFN IAM Policies
@@ -65,32 +66,7 @@ Follows the same pattern as CFN IAM Policies
                   Sid: "AllowDescribeAll"
 
 managed_policies
-""""""""""""""""
+================
 
 Allows you to add additional managed policies. You can specify the full ARN or just a string for the name / path of the
 policy. If will resolve into the same regexp as for `boundary`_
-
-
-xray
-^^^^^
-This section allows to enable X-Ray to run right next to your container.
-It will use the AWS original image for X-Ray Daemon and exposes the ports to the task.
-
-Example:
-
-.. code-block:: yaml
-
-    x-configs:
-      composex:
-        xray:
-          enabled: true
-
-    services:
-      serviceA:
-        x-configs:
-          xray:
-            enabled: True
-
-.. seealso::
-
-    ecs_composex.ecs.ecs_service#set_xray

docs/syntax/docker-compose/ecs.details/logging.rst

@@ -0,0 +1,34 @@
+.. _x_configs_logging_syntax_reference:
+
+=========
+x-logging
+=========
+
+.. contents::
+
+Section to allow passing in arguments for logging.
+
+RetentionInDays
+=====================
+
+Value to indicate how long should the logs be retained for the service.
+
+.. note::
+
+    If the value you enter is not in the allowed values, will set to the closest accepted value.
+
+
+.. hint:: Emulates the CW Logs property `RetentionInDays Property`_
+
+
+
+CreateLogGroup
+===============
+
+Allows you to define whether or not you want ComposeX to create the LogGroup.
+If set to False, it will grant *logs:CreateLogGroup* to the Execution Role.
+It will also define in the *awslogs driver* (`awslogs driver documentation`_) and set **awslogs-create-group** to True
+
+
+.. _RetentionInDays Property: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-logs-loggroup.html#cfn-logs-loggroup-retentionindays
+.. _awslogs driver documentation: https://docs.aws.amazon.com/AmazonECS/latest/developerguide/using_awslogs.html

docs/syntax/docker-compose/ecs.details/xray.rst

@@ -0,0 +1,27 @@
+.. _xray_syntax_reference:
+
+=====
+xray
+=====
+
+This section allows to enable X-Ray to run right next to your container.
+It will use the AWS original image for X-Ray Daemon and exposes the ports to the task.
+
+Example:
+
+.. code-block:: yaml
+
+    x-configs:
+      composex:
+        xray:
+          enabled: true
+
+    services:
+      serviceA:
+        x-configs:
+          xray:
+            enabled: True
+
+.. seealso::
+
+    ecs_composex.ecs.ecs_service#set_xray

docs/syntax/composex/ecs.rst → docs/syntax/docker-compose/ecs.rst

@@ -37,9 +37,8 @@ you can use the native *configs* key of Docker compose.
     ecs.details/network
 
 .. toctree::
-    :caption: Logging
-
-    ecs.details/logging
+    :caption: AWS X-Ray
 
+    ecs.details/xray
 
 :ref:`secrets_syntax_reference`

ecs_composex/common/compose_services.py

@@ -51,7 +51,7 @@
 from ecs_composex.ecs import ecs_params
 from ecs_composex.ecs.docker_tools import find_closest_fargate_configuration
 from ecs_composex.ecs.ecs_iam import add_service_roles
-from ecs_composex.ecs.ecs_params import LOG_GROUP, AWS_XRAY_IMAGE
+from ecs_composex.ecs.ecs_params import LOG_GROUP_NAME, AWS_XRAY_IMAGE
 from ecs_composex.ecs.ecs_params import LOG_GROUP_RETENTION
 from ecs_composex.ecs.ecs_params import NETWORK_MODE, EXEC_ROLE_T, TASK_ROLE_T, TASK_T
 from ecs_composex.iam import define_iam_policy, add_role_boundaries
@@ -235,7 +235,7 @@ def set_memory_to_mb(value):
     else:
         raise ValueError(f"Could not parse {value} to units")
     LOG.debug(f"Computed unit for {value}: {unit}. Results into {final_amount}MB")
-    return final_amount
+    return int(final_amount)
 
 
 def define_ingress_mappings(service_ports):
@@ -321,6 +321,7 @@ class ComposeService(object):
         ("userns_mode", str),
         ("volumes", list),
         ("x-configs", dict),
+        ("x-logging", dict),
     ]
 
     def __init__(self, name, definition, volumes=None, secrets=None):
@@ -392,6 +393,8 @@ def __init__(self, name, definition, volumes=None, secrets=None):
         self.mem_alloc = None
         self.mem_resa = None
         self.cpu_amount = None
+        self.logging = None
+        self.x_logging = {"RetentionInDays": 14, "CreateLogGroup": True}
         self.families = []
         self.my_family = None
         self.container_definition = None
@@ -402,6 +405,7 @@ def __init__(self, name, definition, volumes=None, secrets=None):
         )
         self.ecs_healthcheck = Ref(AWS_NO_VALUE)
         self.set_ecs_healthcheck()
+        self.define_logging()
         self.container_parameters = {}
 
         self.map_volumes(volumes)
@@ -430,7 +434,7 @@ def set_container_definition(self):
             LogConfiguration=LogConfiguration(
                 LogDriver="awslogs",
                 Options={
-                    "awslogs-group": Ref(LOG_GROUP),
+                    "awslogs-group": self.logical_name,
                     "awslogs-region": Ref(AWS_REGION),
                     "awslogs-stream-prefix": self.name,
                 },
@@ -443,6 +447,13 @@ def set_container_definition(self):
         )
         self.container_parameters.update({self.image_param.title: self.image})
 
+    def define_logging(self):
+        """
+        Method to define logging properties
+        """
+        if keyisset("x-logging", self.definition):
+            self.x_logging = self.definition["x-logging"]
+
     def map_volumes(self, volumes=None):
         """
         Method to apply mapping of volumes to the service and define the mapping configuration
@@ -767,6 +778,7 @@ def __init__(self, services, family_name):
         self.service_config = None
         self.scalable_target = None
         self.ecs_service = None
+        self.task_logging_options = {}
         self.stack_parameters = {}
         self.set_xray()
         self.sort_container_configs()
@@ -823,25 +835,21 @@ def reset_logging_retention_period(self, closest_valid):
         :return:
         """
         for service in self.services:
-            if service.x_configs:
-                if keyisset("logging", service.x_configs):
-                    service.x_configs["logging"][
-                        "logs_retention_period"
-                    ] = closest_valid
-                else:
-                    service.x_configs["logging"] = {
-                        "logs_retention_period": closest_valid
-                    }
+            if service.x_logging and keyisset("RetentionInDays", service.x_logging):
+                service.x_logging["RetentionInDays"] = closest_valid
+            else:
+                service.x_logging = {"RetentionInDays": closest_valid}
 
     def handle_logging(self):
-        x_logging = []
-        for service in self.services:
-            if service.x_configs and keyisset("logging", service.x_configs):
-                x_logging.append(service.x_configs["logging"])
         periods = [
-            config["logs_retention_period"]
-            for config in x_logging
-            if keyisset("logs_retention_period", config)
+            service.x_logging["RetentionInDays"]
+            for service in self.services
+            if service.x_logging and keyisset("RetentionInDays", service.x_logging)
+        ]
+        enabled = [
+            service.x_logging["CreateLogGroup"]
+            for service in self.services
+            if service.x_logging and keypresent("CreateLogGroup", service.x_logging)
         ]
         if periods and max(periods) != LOG_GROUP_RETENTION.Default:
             closest_valid = min(
@@ -854,6 +862,19 @@ def handle_logging(self):
                 )
             self.reset_logging_retention_period(closest_valid)
             self.stack_parameters.update({LOG_GROUP_RETENTION.title: closest_valid})
+        if (
+            enabled
+            and not all(enabled)
+            and (
+                not keypresent(ecs_params.CREATE_LOG_GROUP.title, self.stack_parameters)
+                or not self.stack_parameters[ecs_params.CREATE_LOG_GROUP.title]
+                == "False"
+            )
+        ):
+            LOG.warn(
+                "At least one of the services has CreateLogGroup set to False. Disabling new LogsGroups creation"
+            )
+            self.stack_parameters.update({ecs_params.CREATE_LOG_GROUP.title: "False"})
 
     def sort_container_configs(self):
         """
@@ -1076,7 +1097,14 @@ def apply_services_params(self):
             if service.image_param.title not in self.template.parameters:
                 self.template.add_parameter(service.image_param)
 
+    def refresh_container_logging_definition(self):
+        for service in self.services:
+            c_def = service.container_definition
+            logging_def = c_def.LogConfiguration
+            logging_def.Options.update(self.task_logging_options)
+
     def init_task_definition(self):
         add_service_roles(self.template)
         self.set_task_compute_parameter()
         self.set_task_definition()
+        self.refresh_container_logging_definition()

ecs_composex/common/settings.py

@@ -114,7 +114,9 @@ def merge_service_definition(original_def, override_def, nested=False):
                     "with",
                     type(override_def[key]),
                 )
-            handle_lists_merges(original_def[key], override_def[key])
+            original_def[key] = handle_lists_merges(
+                original_def[key], override_def[key]
+            )
         elif (
             isinstance(override_def[key], list)
             and key in original_def.keys()
@@ -177,13 +179,8 @@ def handle_lists_merges(original_list, override_list):
     """
     final_list = []
 
-    original_dict_items = [item for item in original_list if isinstance(item, dict)]
-    override_dict_items = [item for item in override_list if isinstance(item, dict)]
-    unique_dicts = [
-        merge_definitions(original, override)
-        for original, override in zip(original_dict_items, override_dict_items)
-    ]
-    final_list += unique_dicts
+    final_list += [item for item in original_list if isinstance(item, dict)]
+    final_list += [item for item in override_list if isinstance(item, dict)]
 
     original_str_items = [item for item in original_list if isinstance(item, list)]
     final_list += list(
@@ -197,7 +194,7 @@ def handle_lists_merges(original_list, override_list):
     override_list_items = [item for item in override_list if isinstance(item, list)]
 
     if origin_list_items and override_list_items:
-        merged_lists = handle_lists_merges(original_list, override_list)
+        merged_lists = handle_lists_merges(origin_list_items, override_list_items)
         final_list += merged_lists
     elif origin_list_items and not override_list_items:
         final_list += origin_list_items
@@ -415,8 +412,8 @@ def set_secrets(self):
             return
         for secret_name in self.compose_content[ComposeSecret.main_key]:
             secret_def = self.compose_content[ComposeSecret.main_key][secret_name]
-            if keyisset("x-secrets", secret_def) and isinstance(
-                secret_def["x-secrets"], dict
+            if keyisset(ComposeSecret.x_key, secret_def) and isinstance(
+                secret_def[ComposeSecret.x_key], dict
             ):
                 LOG.info(f"Adding secret {secret_name} to settings")
                 secret = ComposeSecret(secret_name, secret_def, self)

ecs_composex/ecs/__init__.py

@@ -38,7 +38,7 @@
 from ecs_composex import __version__ as version
 from ecs_composex.common.stacks import ComposeXStack
 from ecs_composex.ecs import ecs_params
-from ecs_composex.ecs.ecs_conditions import GENERATED_CLUSTER_NAME_CON_T
+from ecs_composex.ecs.ecs_conditions import CREATE_CLUSTER_CON_T
 from ecs_composex.ecs.ecs_params import CLUSTER_NAME, CLUSTER_T
 from ecs_composex.ecs.ecs_template import generate_services
 
@@ -74,7 +74,7 @@ def associate_services_to_root_stack(root_stack, settings, dns_params, vpc_stack
         family.stack_parameters.update(
             {
                 ecs_params.CLUSTER_NAME.title: If(
-                    GENERATED_CLUSTER_NAME_CON_T, Ref(CLUSTER_T), Ref(CLUSTER_NAME)
+                    CREATE_CLUSTER_CON_T, Ref(CLUSTER_T), Ref(CLUSTER_NAME)
                 ),
             }
         )