support for networks: and mapping to additional subnets. (#282)

* Added support for extra subnets definitions and mapping to service families

* Added unit-testing and support from Use

docs/modules_syntax.rst

@@ -7,6 +7,7 @@
     syntax/docker-compose/services
     syntax/docker-compose/volumes
     syntax/docker-compose/secrets
+    syntax/docker-compose/networks
 
 .. toctree::
     :caption: Services Extension Fields

docs/syntax/composex/vpc.rst

@@ -81,12 +81,38 @@ Lookup
     * AppSubnets
     * PublicSubnets
 
-.. note::
 
-    The AppSubnets are the subnets in which will the containers be deployed. Which means, that it requires access to
-    services such as ECR, Secrets Manager etc.
-    You can use any subnet in your existing VPC so long as network connectivity is achieved.
+.. warning::
+
+    When creating newly defined subnets groups, the name must be in the format **^[a-zA-Z0-9]+$**
+
+
+.. hint::
+
+    You can define extra subnet groups based on different tags and map them to your services for override when using
+    **Lookup** or **Use**
+
+    .. code-block:: yaml
+        :caption: Extra subnets definition
+
+        x-vpc:
+          Lookup:
+            VpcId: {}
+            AppSubnets: {}
+            StorageSubnets: {}
+            PublicSubnets: {}
+            Custom01:
+              Tags: {}
+
+        networks:
+          custom01:
+            x-vpc: Custom01
+
 
+        services:
+          serviceA:
+            networks:
+              - custom01
 
 .. tip::
 

docs/syntax/docker-compose/networks.rst

@@ -0,0 +1,53 @@
+.. _compose_networks_syntax_reference:
+
+=========================
+networks
+=========================
+
+In docker-compose one can define diffent subnets which would use different properties, as documented
+`here <https://docs.docker.com/compose/compose-file/#network-configuration-reference>`__
+
+This allows you to logically bind services on different networks etc, very useful in many scenarios.
+
+In ECS ComposeX, we have added support to allow you to define these networks and logically associate them with AWS VPC Subnets.
+
+Refer to :ref:`vpc_syntax_reference` for a full review of ECS ComposeX syntax definition for subnets mappings.
+
+
+You can now define extra subnet groups based on different tags and map them to your services for override when using
+**Lookup** or **Use**
+
+.. code-block:: yaml
+    :caption: Extra subnets definition
+
+    x-vpc:
+      Lookup:
+        VpcId: {}
+          AppSubnets: {}
+          StorageSubnets: {}
+          PublicSubnets: {}
+          Custom01:
+            Tags: {}
+
+.. code-block:: yaml
+    :caption: define compose networks and associate to a Subnet category
+
+    networks:
+      custom01:
+        x-vpc: Custom01
+
+.. code-block:: yaml
+    :caption: Map a compose defined network to a service
+
+    services:
+      serviceA:
+        networks:
+          - custom01
+
+      serviceB:
+        networks:
+          custom01: {}
+
+.. note::
+
+    As per docker-compose config, the rendered networks in a service is a map / object. But it also can be a list.

ecs_composex/common/compose_networks.py

@@ -0,0 +1,76 @@
+#  -*- coding: utf-8 -*-
+#   ECS ComposeX <https://github.com/lambda-my-aws/ecs_composex>
+#   Copyright (C) 2020  John Mille <john@lambda-my-aws.io>
+#  #
+#   This program is free software: you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation, either version 3 of the License, or
+#   (at your option) any later version.
+#  #
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#  #
+#   You should have received a copy of the GNU General Public License
+#   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""
+Class and functions to interact with the networks: defined in compose files.
+"""
+
+import re
+
+from ecs_composex.common import keyisset, LOG
+from ecs_composex.vpc.vpc_params import (
+    APP_SUBNETS,
+    STORAGE_SUBNETS,
+    PUBLIC_SUBNETS,
+    SUBNETS_TYPE,
+    VPC_ID,
+)
+
+
+def match_networks_services_config(service, vol_config, networks):
+    """
+    Function to map network config in services and top-level networks
+
+    :param service:
+    :param vol_config:
+    :param networks:
+    :raises LookupError:
+    """
+    for network in networks:
+        if network.name == vol_config["source"]:
+            network.services.append(service)
+            vol_config["network"] = network
+            service.networks.append(vol_config)
+            LOG.info(f"Mapped {network.name} to {service.name}")
+            return
+    raise LookupError(
+        f"Volume {vol_config['source']} was not found in {[vol.name for vol in networks]}"
+    )
+
+
+class ComposeNetwork(object):
+    """
+    Class to keep track of the Docker-compose Volumes
+    """
+
+    main_key = "networks"
+    driver_opts_key = "driver"
+
+    def __init__(self, name, definition, subnets_list):
+        self.name = name
+        self.subnet_name = name
+        if keyisset("name", definition):
+            self.subnet_name = definition["name"]
+        elif (
+            not keyisset("name", definition)
+            and keyisset("x-vpc", definition)
+            and isinstance(definition["x-vpc"], str)
+        ):
+            self.subnet_name = definition["x-vpc"]
+        subnet_names = [subnet.title for subnet in subnets_list]
+        if self.subnet_name not in subnet_names:
+            raise KeyError(f"No subnet {self.name} defined. Valid options are", subnet_names)

ecs_composex/common/compose_services.py

@@ -25,7 +25,7 @@
     AWS_STACK_NAME,
 )
 from troposphere import Parameter, Tags
-from troposphere import Sub, Ref, GetAtt, ImportValue
+from troposphere import Sub, Ref, GetAtt, ImportValue, Join
 from troposphere.ecs import (
     HealthCheck,
     Environment,
@@ -58,6 +58,7 @@
     ComposeSecret,
     match_secrets_services_config,
 )
+from ecs_composex.vpc.vpc_params import APP_SUBNETS
 
 NUMBERS_REG = r"[^0-9.]"
 MINIMUM_SUPPORTED = 4
@@ -262,7 +263,7 @@ class ComposeService(object):
         ("logging", dict),
         ("links", list),
         ("network_mode", str),
-        ("networks", list),
+        ("networks", (list, dict)),
         ("image", str),
         ("init", bool),
         ("isolation", str),
@@ -332,7 +333,7 @@ def __init__(self, name, definition, volumes=None, secrets=None):
         self.x_logging = {"RetentionInDays": 14, "CreateLogGroup": True}
 
         self.import_x_aws_settings()
-
+        self.networks = {}
         self.replicas = 1
         self.container = None
         self.volumes = []
@@ -375,6 +376,7 @@ def __init__(self, name, definition, volumes=None, secrets=None):
         self.map_secrets(secrets)
         self.set_service_deploy()
         self.set_container_definition()
+        self.set_networks()
 
     def set_container_definition(self):
         """
@@ -410,6 +412,15 @@ def set_container_definition(self):
         )
         self.container_parameters.update({self.image_param.title: self.image})
 
+    def set_networks(self):
+        if not keyisset("networks", self.definition):
+            return
+        if isinstance(self.definition["networks"], list):
+            for name in self.definition["networks"]:
+                self.networks[name] = None
+        elif isinstance(self.definition["networks"], dict):
+            self.networks.update(self.definition["networks"])
+
     def merge_x_aws_role(self, key):
         """
         Method to update the service definition with the x-aws-role information if NOT defined in the composex
@@ -1127,3 +1138,15 @@ def init_task_definition(self):
         self.set_task_compute_parameter()
         self.set_task_definition()
         self.refresh_container_logging_definition()
+
+    def update_family_subnets(self, settings):
+        """
+        Method to update the stack parameters
+
+        :param ecs_composex.common.settings.ComposeXSettings settings:
+        """
+        network_names = list(self.service_config.network.networks.keys())
+        for network in settings.networks:
+            if network.name in network_names:
+                self.stack_parameters.update({APP_SUBNETS.title: Join(",", Ref(network.subnet_name))})
+                LOG.info(f"Set {network.subnet_name} as {APP_SUBNETS.title} for {self.name}")

ecs_composex/common/settings.py

@@ -39,6 +39,7 @@
     ComposeFamily,
 )
 from ecs_composex.common.compose_volumes import ComposeVolume
+from ecs_composex.common.compose_networks import ComposeNetwork
 from ecs_composex.common.envsubst import expandvars
 from ecs_composex.iam import ROLE_ARN_ARG
 from ecs_composex.iam import validate_iam_role_arn
@@ -376,6 +377,8 @@ def __init__(self, content=None, profile_name=None, session=None, **kwargs):
         self.volumes = []
         self.services = []
         self.secrets = []
+        self.networks = []
+        self.subnets_parameters = []
         self.secrets_mappings = {}
         self.families = {}
         self.account_id = None
@@ -435,6 +438,24 @@ def set_volumes(self):
             self.compose_content[ComposeVolume.main_key][volume_name] = volume
             self.volumes.append(volume)
 
+    def set_networks(self, vpc_stack, root_stack):
+        """
+        Method configuring the networks defined at root level
+        :return:
+        """
+        if not keyisset(ComposeNetwork.main_key, self.compose_content):
+            LOG.debug("No networks detected at the root level of compose file")
+            return
+        elif vpc_stack:
+            LOG.info("ComposeX will be creating the VPC, therefore networks are ignored!")
+            return
+        for network_name in self.compose_content[ComposeNetwork.main_key]:
+            network = ComposeNetwork(
+                network_name, self.compose_content[ComposeNetwork.main_key][network_name], self.subnets_parameters
+            )
+            self.compose_content[ComposeNetwork.main_key][network_name] = network
+            self.networks.append(network)
+
     def set_services(self):
         """
         Method to define the ComposeXResource for each service.

ecs_composex/ecs/__init__.py

@@ -85,3 +85,5 @@ def associate_services_to_root_stack(root_stack, settings, dns_params, vpc_stack
             family.stack.get_from_vpc_stack(vpc_stack)
         family.template.set_metadata(metadata)
         root_stack.stack_template.add_resource(family.stack)
+        if settings.networks and family.service_config.network.networks:
+            family.update_family_subnets(settings)

ecs_composex/ecs/ecs_service_network_config.py

@@ -140,13 +140,23 @@ def __init__(self, family):
         :param ecs_composex.common.compose_services.ComposeFamily family:
         """
         self.ports = []
+        self.networks = {}
         self.merge_services_ports(family)
+        self.merge_networks(family)
         self.configuration = merge_services_network(family)
         self.is_public = self.configuration["is_public"]
         self.ingress_from_self = True
         super().__init__(self.configuration[self.master_key], self.ports)
         self.add_self_ingress(family)
 
+    def merge_networks(self, family):
+        """
+        Method to merge network
+        """
+        for svc in family.services:
+            if svc.networks:
+                self.networks.update(svc.networks)
+
     def merge_services_ports(self, family):
         """
         Function to merge two sections of ports

ecs_composex/ecs_composex.py

@@ -360,6 +360,7 @@ def generate_full_template(settings):
     )
     dns_inputs(root_stack)
     vpc_stack = add_vpc_to_root(root_stack, settings)
+    settings.set_networks(vpc_stack, root_stack)
     dns_settings = DnsSettings(root_stack, settings, get_vpc_id(vpc_stack))
     root_stack.Parameters.update(dns_settings.root_params)
     add_ecs_cluster(settings, root_stack)

ecs_composex/vpc/vpc_aws.py

@@ -92,4 +92,17 @@ def lookup_x_vpc_settings(lookup, session):
             re.match(vpc_types[subnet_type]["regexp"], subnet_arn).groups()[0]
             for subnet_arn in subnet_arns
         ]
+    extra_subnets = [key for key in lookup.keys() if key not in required_keys]
+    for subnet_name in extra_subnets:
+        subnet_arns = find_aws_resource_arn_from_tags_api(
+            lookup[subnet_name],
+            lookup_session,
+            subnet_type,
+            types=vpc_types,
+            allow_multi=True
+        )
+        vpc_settings[subnet_name] = [
+            re.match(vpc_types[subnet_type]["regexp"], subnet_arn).groups()[0]
+            for subnet_arn in subnet_arns
+        ]
     return vpc_settings