From c3f076166239147e32dc44cb5712426b2a1d7a9c Mon Sep 17 00:00:00 2001 From: David Goh Date: Thu, 9 Nov 2023 17:02:00 +1100 Subject: [PATCH 01/12] Don't die if access is not defined in target list #696 --- ecs_composex/elbv2/elbv2_stack/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ecs_composex/elbv2/elbv2_stack/helpers.py b/ecs_composex/elbv2/elbv2_stack/helpers.py index ea0a5c85..94b51748 100644 --- a/ecs_composex/elbv2/elbv2_stack/helpers.py +++ b/ecs_composex/elbv2/elbv2_stack/helpers.py @@ -398,7 +398,7 @@ def handle_non_default_services(listener: ComposeListener) -> list: """ left_services = deepcopy(listener.services) for count, service_def in enumerate(listener.services): - if isinstance(service_def["access"], str) and service_def["access"] == "/": + if isinstance(service_def.get("access", None), str) and service_def["access"] == "/": left_services.pop(count) listener.DefaultActions += define_actions(listener, service_def) break From e2787089b05fb48568cb8eaa7acc54d724b739a8 Mon Sep 17 00:00:00 2001 From: David Goh Date: Thu, 9 Nov 2023 17:27:23 +1100 Subject: [PATCH 02/12] make conform formatting #696 --- ecs_composex/elbv2/elbv2_stack/helpers.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ecs_composex/elbv2/elbv2_stack/helpers.py b/ecs_composex/elbv2/elbv2_stack/helpers.py index 94b51748..9a90ed31 100644 --- a/ecs_composex/elbv2/elbv2_stack/helpers.py +++ b/ecs_composex/elbv2/elbv2_stack/helpers.py @@ -398,7 +398,10 @@ def handle_non_default_services(listener: ComposeListener) -> list: """ left_services = deepcopy(listener.services) for count, service_def in enumerate(listener.services): - if isinstance(service_def.get("access", None), str) and service_def["access"] == "/": + if ( + isinstance(service_def.get("access", None), str) + and service_def["access"] == "/" + ): left_services.pop(count) listener.DefaultActions += define_actions(listener, service_def) break From ebcb48f1ecce6aa183c081c339e4fedeabee10f1 Mon Sep 17 00:00:00 2001 From: "John \"Preston\" Mille" Date: Fri, 10 Nov 2023 07:03:28 +0000 Subject: [PATCH 03/12] Bump to 0.23.27-rc2 --- ecs_composex/__init__.py | 2 +- pyproject.toml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ecs_composex/__init__.py b/ecs_composex/__init__.py index afddc38e..f31498d5 100644 --- a/ecs_composex/__init__.py +++ b/ecs_composex/__init__.py @@ -5,4 +5,4 @@ __author__ = """John Preston""" __email__ = "john@compose-x.io" -__version__ = "0.23.27-rc1" +__version__ = "0.23.27-rc2" diff --git a/pyproject.toml b/pyproject.toml index 32568477..188f75c3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "ecs_composex" -version = "0.23.27-rc1" +version = "0.23.27-rc2" description = "Manage, Configure and Deploy your services and AWS services and applications from your docker-compose definition" authors = ["John Preston "] maintainers = ["John Preston "] @@ -82,7 +82,7 @@ sphinx-material = "^0.0.35" github_url = "https://github.com/compose-x/ecs_composex" [tool.tbump.version] -current = "0.23.27-rc1" +current = "0.23.27-rc2" regex = ''' (?P\d+) \. From 06f419d998dcf8b47f8da5fc7f7b8186b7657067 Mon Sep 17 00:00:00 2001 From: "John \"Preston\" Mille" Date: Sun, 12 Nov 2023 14:37:59 +0000 Subject: [PATCH 04/12] Fixing elbv2.Targets no access and conditions set --- .../elbv2/elbv2_stack/elbv2_listener.py | 30 ++++++++++------- ecs_composex/elbv2/elbv2_stack/helpers.py | 33 ++++++++++--------- ecs_composex/elbv2/x-elbv2.spec.json | 12 +++++++ 3 files changed, 48 insertions(+), 27 deletions(-) diff --git a/ecs_composex/elbv2/elbv2_stack/elbv2_listener.py b/ecs_composex/elbv2/elbv2_stack/elbv2_listener.py index fca276ae..1923d768 100644 --- a/ecs_composex/elbv2/elbv2_stack/elbv2_listener.py +++ b/ecs_composex/elbv2/elbv2_stack/elbv2_listener.py @@ -24,11 +24,13 @@ LISTENER_TARGET_RE, add_acm_certs_arn, define_actions, + define_listener_rules_actions, handle_default_actions, handle_non_default_services, import_cognito_pool, import_new_acm_certs, map_service_target, + tea_pot, ) from ecs_composex.resources_import import import_record_properties @@ -92,6 +94,7 @@ def define_default_actions(self, lb: Elbv2, template): If not defined, and there is more than one service, it will fail. If not defined and there is only one service defined, it will skip """ + rules: list = [] if not self.default_actions and not self.services: warnings.warn( f"{self.name} - There are no actions defined or services for listener {self.title}. Skipping" @@ -100,10 +103,15 @@ def define_default_actions(self, lb: Elbv2, template): if self.default_actions: handle_default_actions(self) elif not self.default_actions and self.services and len(self.services) == 1: - LOG.info( - f"{self.title} has no defined DefaultActions and only 1 service. Default all to service." - ) - self.DefaultActions = define_actions(self, self.services[0]) + if not keyisset("Conditions", self.services[0]): + LOG.info( + f"{self.title} has no defined DefaultActions and only 1 service " + "without Conditions. Setting Listener DefaultActions to service." + ) + self.DefaultActions = define_actions(self, self.services[0]) + else: + self.DefaultActions = [tea_pot(True)] + rules = define_listener_rules_actions(self, self.services) elif lb.is_nlb() and self.services and len(self.services) > 1: raise ValueError( f"{lb.module.res_key}.{lb.name} - Listener {self.def_port}" @@ -116,15 +124,15 @@ def define_default_actions(self, lb: Elbv2, template): "If one of the access path is / it will be used as default" ) rules = handle_non_default_services(self) - if rules and lb.is_alb(): - for rule in rules: - template.add_resource(rule) - else: - LOG.warning( - f"{lb.module.res_key}.{lb.name} - LB is NLB. Can't assign Listener Rules." - ) else: raise ValueError(f"Failed to determine any default action for {self.title}") + if rules and lb.is_alb(): + for rule in rules: + template.add_resource(rule) + else: + LOG.warning( + f"{lb.module.res_key}.{lb.name} - LB is NLB. Can't assign Listener Rules." + ) def handle_cognito_pools(self, settings, listener_stack): """ diff --git a/ecs_composex/elbv2/elbv2_stack/helpers.py b/ecs_composex/elbv2/elbv2_stack/helpers.py index 9a90ed31..c18df094 100644 --- a/ecs_composex/elbv2/elbv2_stack/helpers.py +++ b/ecs_composex/elbv2/elbv2_stack/helpers.py @@ -16,7 +16,7 @@ from copy import deepcopy from json import dumps -from compose_x_common.compose_x_common import keyisset +from compose_x_common.compose_x_common import keyisset, set_else_none from troposphere import AWS_NO_VALUE, FindInMap, Ref from troposphere.elasticloadbalancingv2 import ( Action, @@ -198,9 +198,9 @@ def handle_predefined_redirects(listener: ComposeListener, action_name) -> None: f"Redirect {action_name} is not a valid pre-defined setting. Valid values", [r[0] for r in predefined_redirects], ) - for redirect in predefined_redirects: - if action_name == redirect[0]: - action = redirect[1]() + for redirect_key, redirect_function in predefined_redirects: + if action_name == redirect_key: + action = redirect_function() listener.DefaultActions.insert(0, action) @@ -276,7 +276,7 @@ def handle_string_condition_format(access_string) -> list: raise ValueError(f"Could not understand what the access is for {access_string}") -def define_target_conditions(definition) -> list: +def define_target_conditions(definition: dict) -> list: """ Function to create the conditions for forward to target @@ -285,11 +285,16 @@ def define_target_conditions(definition) -> list: :rtype: list """ conditions = [] - if keyisset("Conditions", definition) and isinstance( - definition["Conditions"], list - ): + user_defined_conditions = set_else_none("Conditions", definition, []) + if user_defined_conditions: + if not isinstance(user_defined_conditions, list): + raise TypeError( + "Conditions must be a list. Got {}".format( + type(user_defined_conditions) + ) + ) conditions = import_record_properties( - {"Conditions": definition["Conditions"]}, + {"Conditions": user_defined_conditions}, ListenerRule, set_to_novalue=False, ignore_missing_required=True, @@ -370,13 +375,11 @@ def define_actions(listener, target_def, rule_actions: bool = False) -> list: return actions -def define_listener_rules_actions(listener, left_services) -> list: +def define_listener_rules_actions( + listener: ComposeListener, left_services: list +) -> list[ListenerRule]: """ Function to identify the Target definition and create the resulting rule appropriately. - - :param listener: - :param list left_services: - :return: The action to add or action list for default target """ rules = [] for count, service_def in enumerate(left_services): @@ -528,8 +531,6 @@ def import_cognito_pool(src_name, settings: ComposeXSettings, listener_stack): raise LookupError( f"There is no {COGNITO_KEY} defined in your docker-compose files" ) - print("Cognito SRC Name", src_name) - print(settings.x_resources) pools = [ res for res in settings.x_resources diff --git a/ecs_composex/elbv2/x-elbv2.spec.json b/ecs_composex/elbv2/x-elbv2.spec.json index 7a6acd0f..597c7a6f 100644 --- a/ecs_composex/elbv2/x-elbv2.spec.json +++ b/ecs_composex/elbv2/x-elbv2.spec.json @@ -361,6 +361,18 @@ }, "Listener": { "type": "object", + "oneOf": [ + { + "required": [ + "DefaultActions" + ] + }, + { + "required": [ + "Targets" + ] + } + ], "properties": { "Port": { "$ref": "#/definitions/Port" From e33f7b0cbdaaf2bf774150e4dd903748b264a8f4 Mon Sep 17 00:00:00 2001 From: David Goh Date: Mon, 13 Nov 2023 10:20:13 +1100 Subject: [PATCH 05/12] Fix x-cloudmap Vpc to use Ref: --- ecs_composex/cloudmap/cloudmap_stack.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ecs_composex/cloudmap/cloudmap_stack.py b/ecs_composex/cloudmap/cloudmap_stack.py index 4b4ab863..961313f1 100644 --- a/ecs_composex/cloudmap/cloudmap_stack.py +++ b/ecs_composex/cloudmap/cloudmap_stack.py @@ -18,7 +18,7 @@ from copy import deepcopy from compose_x_common.compose_x_common import keyisset, set_else_none -from troposphere import GetAtt +from troposphere import GetAtt, Ref from ecs_composex.cloudmap.cloudmap_helpers import ( detect_duplicas, @@ -311,13 +311,13 @@ def define_new_namespace(new_namespaces, stack_template): f"{namespace.module.res_key}.{namespace.name} - " "Vpc property was set. Overriding to compose-x x-vpc defined for execution." ) - namespace_props["Vpc"] = f"x-vpc::{VPC_ID.title}" + namespace_props["Vpc"] = Ref(VPC_ID) namespace.cfn_resource = PrivateNamespace( namespace.logical_name, **namespace_props ) elif namespace.uses_default: namespace_props = import_record_properties( - {"Name": namespace.zone_name, "Vpc": f"x-vpc::{VPC_ID.title}"}, + {"Name": namespace.zone_name, "Vpc": Ref(VPC_ID)}, PrivateDnsNamespace, ) namespace.cfn_resource = PrivateDnsNamespace( From 438331bfe63ea4cdaa79fa1bedf3bc66a2efcd6f Mon Sep 17 00:00:00 2001 From: David Goh Date: Mon, 13 Nov 2023 14:34:06 +1100 Subject: [PATCH 06/12] Get secrets, then delete environment variables that are in secrets #707 --- ecs_composex/ecs/ecs_family/__init__.py | 34 ++++++++++++++----------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/ecs_composex/ecs/ecs_family/__init__.py b/ecs_composex/ecs/ecs_family/__init__.py index 8492f8a7..3bd368eb 100644 --- a/ecs_composex/ecs/ecs_family/__init__.py +++ b/ecs_composex/ecs/ecs_family/__init__.py @@ -513,22 +513,26 @@ def set_add_region_when_external(self): def sort_env_vars_alphabetically(self): for service in self.services: - environment = getattr(service.container_definition, "Environment") - if not environment: - pass - original = [_env for _env in environment if isinstance(_env, Environment)] - sorted_env = sorted(original, key=lambda x: x.Name) - for _env in environment: - if not isinstance(_env, Environment): - sorted_env.append(_env) - setattr(service.container_definition, "Environment", sorted_env) secrets = getattr(service.container_definition, "Secrets") - original_secrets = [_env for _env in secrets if isinstance(_env, Secret)] - sorted_secrets = sorted(original_secrets, key=lambda x: x.Name) - for _secret in secrets: - if not isinstance(_secret, Secret): - sorted_secrets.append(_secret) - setattr(service.container_definition, "Secrets", sorted_secrets) + if secrets: + original_secrets = [_env for _env in secrets if isinstance(_env, Secret)] + sorted_secrets = sorted(original_secrets, key=lambda x: x.Name) + for _secret in secrets: + if not isinstance(_secret, Secret): + sorted_secrets.append(_secret) + setattr(service.container_definition, "Secrets", sorted_secrets) + secret_names = frozenset(x.Name for x in sorted_secrets) + else: + secret_names = frozenset() + environment = getattr(service.container_definition, "Environment") + if environment: + remove_secrets = [_env for _env in environment if _env.Name not in secret_names] + original = [_env for _env in remove_secrets if isinstance(_env, Environment)] + sorted_env = sorted(original, key=lambda x: x.Name) + for _env in environment: + if not isinstance(_env, Environment): + sorted_env.append(_env) + setattr(service.container_definition, "Environment", sorted_env) def set_services_to_services_dependencies(self): """ From cc609246a2e1ec44f22f3ccd427ef9e735c51754 Mon Sep 17 00:00:00 2001 From: David Goh Date: Mon, 13 Nov 2023 14:42:16 +1100 Subject: [PATCH 07/12] Don't try to Name filter unless things are Environment #707 --- ecs_composex/ecs/ecs_family/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ecs_composex/ecs/ecs_family/__init__.py b/ecs_composex/ecs/ecs_family/__init__.py index 3bd368eb..217bfae4 100644 --- a/ecs_composex/ecs/ecs_family/__init__.py +++ b/ecs_composex/ecs/ecs_family/__init__.py @@ -526,9 +526,9 @@ def sort_env_vars_alphabetically(self): secret_names = frozenset() environment = getattr(service.container_definition, "Environment") if environment: - remove_secrets = [_env for _env in environment if _env.Name not in secret_names] - original = [_env for _env in remove_secrets if isinstance(_env, Environment)] - sorted_env = sorted(original, key=lambda x: x.Name) + original = [_env for _env in environment if isinstance(_env, Environment)] + remove_secrets = [_env for _env in original if _env.Name not in secret_names] + sorted_env = sorted(remove_secrets, key=lambda x: x.Name) for _env in environment: if not isinstance(_env, Environment): sorted_env.append(_env) From bbff6156bc0d969fce59f61be6d26c89e0a64f65 Mon Sep 17 00:00:00 2001 From: David Goh Date: Mon, 13 Nov 2023 17:33:15 +1100 Subject: [PATCH 08/12] make conform #707 --- ecs_composex/ecs/ecs_family/__init__.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/ecs_composex/ecs/ecs_family/__init__.py b/ecs_composex/ecs/ecs_family/__init__.py index 217bfae4..6d5a758b 100644 --- a/ecs_composex/ecs/ecs_family/__init__.py +++ b/ecs_composex/ecs/ecs_family/__init__.py @@ -515,7 +515,9 @@ def sort_env_vars_alphabetically(self): for service in self.services: secrets = getattr(service.container_definition, "Secrets") if secrets: - original_secrets = [_env for _env in secrets if isinstance(_env, Secret)] + original_secrets = [ + _env for _env in secrets if isinstance(_env, Secret) + ] sorted_secrets = sorted(original_secrets, key=lambda x: x.Name) for _secret in secrets: if not isinstance(_secret, Secret): @@ -526,8 +528,12 @@ def sort_env_vars_alphabetically(self): secret_names = frozenset() environment = getattr(service.container_definition, "Environment") if environment: - original = [_env for _env in environment if isinstance(_env, Environment)] - remove_secrets = [_env for _env in original if _env.Name not in secret_names] + original = [ + _env for _env in environment if isinstance(_env, Environment) + ] + remove_secrets = [ + _env for _env in original if _env.Name not in secret_names + ] sorted_env = sorted(remove_secrets, key=lambda x: x.Name) for _env in environment: if not isinstance(_env, Environment): From 5c1c2040cd474a8fc817a1ac7fc8723fa4ef32b9 Mon Sep 17 00:00:00 2001 From: David Goh Date: Tue, 14 Nov 2023 14:21:11 +1100 Subject: [PATCH 09/12] Rename the environment if it's duplicated by secrets #707 --- ecs_composex/ecs/ecs_family/__init__.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/ecs_composex/ecs/ecs_family/__init__.py b/ecs_composex/ecs/ecs_family/__init__.py index 6d5a758b..7369db21 100644 --- a/ecs_composex/ecs/ecs_family/__init__.py +++ b/ecs_composex/ecs/ecs_family/__init__.py @@ -528,16 +528,17 @@ def sort_env_vars_alphabetically(self): secret_names = frozenset() environment = getattr(service.container_definition, "Environment") if environment: - original = [ - _env for _env in environment if isinstance(_env, Environment) - ] - remove_secrets = [ - _env for _env in original if _env.Name not in secret_names - ] - sorted_env = sorted(remove_secrets, key=lambda x: x.Name) + original = [] + extras = [] for _env in environment: - if not isinstance(_env, Environment): - sorted_env.append(_env) + if isinstance(_env, Environment): + while _env.Name in secret_names: + _env.Name += "_IN_SECRETS" + original.append(_env) + else: + extras.append(_env) + sorted_env = sorted(original, key=lambda x: x.Name) + sorted_env.extend(extras) setattr(service.container_definition, "Environment", sorted_env) def set_services_to_services_dependencies(self): From 7ef7967aeb43609014bcab4b8ca6beb8dbc966a5 Mon Sep 17 00:00:00 2001 From: David Goh Date: Tue, 14 Nov 2023 14:27:14 +1100 Subject: [PATCH 10/12] LOG.warning if we rename environment variables #707 --- ecs_composex/ecs/ecs_family/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ecs_composex/ecs/ecs_family/__init__.py b/ecs_composex/ecs/ecs_family/__init__.py index 7369db21..0053df2e 100644 --- a/ecs_composex/ecs/ecs_family/__init__.py +++ b/ecs_composex/ecs/ecs_family/__init__.py @@ -532,8 +532,11 @@ def sort_env_vars_alphabetically(self): extras = [] for _env in environment: if isinstance(_env, Environment): + orig_env_name = _env.Name while _env.Name in secret_names: _env.Name += "_IN_SECRETS" + if orig_env_name != _env.Name: + LOG.warning(f"Renamed environment {orig_env_name} to {_env.Name}") original.append(_env) else: extras.append(_env) From f33b5700cf2d75ce7e55021f3b162958e2fa02a7 Mon Sep 17 00:00:00 2001 From: David Goh Date: Tue, 21 Nov 2023 10:51:32 +1100 Subject: [PATCH 11/12] Iterate environment in reverse for pop()ing #707 #710 --- ecs_composex/ecs/ecs_family/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ecs_composex/ecs/ecs_family/__init__.py b/ecs_composex/ecs/ecs_family/__init__.py index c215b80c..2290c870 100644 --- a/ecs_composex/ecs/ecs_family/__init__.py +++ b/ecs_composex/ecs/ecs_family/__init__.py @@ -552,7 +552,8 @@ def sort_env_vars( _secret.Name for _secret in getattr(service.container_definition, "Secrets", []) ] - for _index, _env in enumerate(sorted_env): + # Iterate in reverse for popping so we don't mess up indexes + for _index, _env in reversed(enumerate(sorted_env)): if _env.Name in secrets_names: LOG.warning( "services.{}: Environment variable {} overlaps with Secret. Removing.".format( From 8532a874199a6ef6e2343fc1953f5e2b7984f84c Mon Sep 17 00:00:00 2001 From: David Goh Date: Tue, 21 Nov 2023 10:59:48 +1100 Subject: [PATCH 12/12] Bugfix reversed() #707 --- ecs_composex/ecs/ecs_family/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ecs_composex/ecs/ecs_family/__init__.py b/ecs_composex/ecs/ecs_family/__init__.py index 2290c870..3962103e 100644 --- a/ecs_composex/ecs/ecs_family/__init__.py +++ b/ecs_composex/ecs/ecs_family/__init__.py @@ -553,7 +553,7 @@ def sort_env_vars( for _secret in getattr(service.container_definition, "Secrets", []) ] # Iterate in reverse for popping so we don't mess up indexes - for _index, _env in reversed(enumerate(sorted_env)): + for _index, _env in reversed(tuple(enumerate(sorted_env))): if _env.Name in secrets_names: LOG.warning( "services.{}: Environment variable {} overlaps with Secret. Removing.".format(