Refactoring compute and its CLI (#8)

Refactoring compute and its CLI tested and working.

ecs_composex/cli.py

@@ -6,8 +6,10 @@
 import sys
 import argparse
 import warnings
+import json
 from boto3 import session
-from ecs_composex.common import LOG
+from ecs_composex import XFILE_DEST
+from ecs_composex.common import LOG, write_template_to_file
 from ecs_composex.common.aws import (
     BUCKET_NAME, CURATED_AZS
 )
@@ -17,14 +19,13 @@
     APP_SUBNETS_T, PUBLIC_SUBNETS_T, STORAGE_SUBNETS_T,
     VPC_ID_T, VPC_MAP_ID_T
 )
-from ecs_composex.ecs.ecs_params import LAUNCH_TYPE_T
-from ecs_composex.compute.cluster_params import (
-    CLUSTER_NAME_T, USE_FLEET_T
-)
+from ecs_composex.common.cfn_params import USE_FLEET_T
+from ecs_composex.compute.compute_params import CLUSTER_NAME_T
 
 
 def validate_vpc_input(args):
-    """Function to validate the VPC arguments are all present
+    """
+    Function to validate the VPC arguments are all present
 
     :param args: Parser arguments
     :type args: dict
@@ -56,6 +57,7 @@ def validate_cluster_input(args):
     """Function to validate the cluster arguments
 
     :param args: Parser arguments
+    :raise: KeyError
     """
     if not KEYISSET('CreateCluster', args) and not KEYISSET(CLUSTER_NAME_T, args):
         raise KeyError(f"You must provide an ECS Cluster name if you do not want ECS ComposeX to create one for you")
@@ -66,7 +68,7 @@ def main():
     parser = argparse.ArgumentParser()
     #  Generic settings
     parser.add_argument(
-        '-f', '--docker-compose-file', dest='ComposeXFile',
+        '-f', '--docker-compose-file', dest=XFILE_DEST,
         required=True, help="Path to the Docker compose file"
     )
     parser.add_argument(
@@ -131,17 +133,13 @@ def main():
         help='Override/Provide ECS Cluster name'
     )
     # COMPUTE SETTINGS
-    parser.add_argument(
-        '--use-fargate', required=False, default=False, action='store_true',
-        dest=LAUNCH_TYPE_T, help="If you run Fargate only, no EC2 will be created"
-    )
     parser.add_argument(
         '--use-spot-fleet', required=False, default=False, action='store_true',
         dest=USE_FLEET_T, help="Runs spotfleet for EC2. If used in combination "
                                "of --use-fargate, it will create an additional SpotFleet"
     )
     parser.add_argument(
-        '--create-launch-template', dest='CreateLaunchTemplate', action='store_true',
+        '--add-compute-resources', dest='AddComputeResources', action='store_true',
         help='Whether you want to create a launch template to create EC2 resources for'
         ' to expand the ECS Cluster and run containers on EC2 instances you might have access to.'
     )
@@ -158,7 +156,10 @@ def main():
     validate_cluster_input(vars(args))
 
     print("Arguments: " + str(args._))
-    generate_full_template(**vars(args))
+    templates_and_params = generate_full_template(**vars(args))
+    write_template_to_file(templates_and_params[0], args.output_file)
+    with open(f"{args.output_file.split('.')[0]}.params.json", 'w') as params_fd:
+        params_fd.write(json.dumps(templates_and_params[1], indent=4))
 
 
 if __name__ == "__main__":

ecs_composex/common/__init__.py

@@ -21,7 +21,6 @@
 from ecs_composex.common import cfn_params
 from ecs_composex.common import cfn_conditions
 
-
 DATE = dt.utcnow().isoformat()
 DATE_PREFIX = dt.utcnow().strftime('%Y/%m/%d/%H%M')
 NONALPHANUM = re.compile(r'[\W]+')
@@ -237,6 +236,56 @@ def setup_logging():
     return the_logger
 
 
+def write_template_to_file(template, file_path):
+    """
+    Function to write the template to a specific path
+    :param template: the troposphere template
+    :type template: troposphere.Template
+    :param file_path: file path where to write the template rendered
+    :type file_path: str
+    """
+    regex = re.compile(r'(.yml|.yaml)$')
+    with open(file_path, 'w') as template_fd:
+        if regex.findall(file_path):
+            template_fd.write(template.to_yaml())
+        else:
+            template_fd.write(template.to_json())
+
+
+def build_default_stack_parameters(stack_params, **kwargs):
+    """
+    Function to check and define default parameters for the root stack from the CLI options
+    :param stack_params: list of parameters to add to to use for the root stack
+    :type stack_params: list
+    :param kwargs: extended arguments
+    :type kwargs: dict
+    """
+    if KEYISSET(cfn_params.USE_FLEET_T, kwargs):
+        build_parameters_file(stack_params, cfn_params.USE_FLEET_T, kwargs[cfn_params.USE_FLEET_T])
+
+
+def build_parameters_file(params: list, parameter_name: str, parameter_value):
+    """
+    Function to build arguments file to pass onto CFN.
+    Adds the parameter key/value so it can be written to file afterwards
+
+    :param params: list of parameters
+    :type params: list
+    :param parameter_name: key of the parameter
+    :type parameter_name: str
+    :param parameter_value: value of the parameter
+    :type parameter_value: str or int or list
+    """
+    if params is None:
+        params = []
+    if isinstance(parameter_value, (int, float)):
+        parameter_value = str(parameter_value)
+    params.append({
+        "ParameterKey": parameter_name,
+        "ParameterValue": parameter_value
+    })
+
+
 def load_composex_file(file_path):
     """File to load and read the docker compose file
 

ecs_composex/common/cfn_conditions.py

@@ -58,3 +58,9 @@
     Condition(USE_SSM_EXPORTS_T),
     Condition(NOT_USE_CFN_EXPORTS_T)
 )
+
+USE_SPOT_CON_T = "UseSpotFleetHostsCondition"
+USE_SPOT_CON = Equals(Ref(cfn_params.USE_FLEET), "True")
+
+NOT_USE_SPOT_CON_T = "NotUseSpotFleetHostsCondition"
+NOT_USE_SPOT_CON = Not(Condition(USE_SPOT_CON_T))

ecs_composex/common/cfn_params.py

@@ -50,3 +50,13 @@
     USE_SSM_EXPORTS_T, Type='String',
     AllowedValues=['True', 'False'], Default='False'
 )
+
+USE_FLEET_T = "UseSpotFleetHosts"
+USE_FLEET = Parameter(
+    USE_FLEET_T, Type="String", Default="False", AllowedValues=["True", "False"]
+)
+
+USE_ONDEMAND_T = "UseOnDemandHosts"
+USE_ONDEMAND = Parameter(
+    USE_ONDEMAND_T, Type="String", Default="False", AllowedValues=["True", "False"]
+)

ecs_composex/compute/__init__.py

@@ -9,12 +9,14 @@
 
 import boto3
 
-from ecs_composex.common import KEYISSET, load_composex_file
+from ecs_composex import XFILE_DEST
+from ecs_composex.common import KEYISSET, load_composex_file, build_default_stack_parameters
 from ecs_composex.common.aws import get_curated_azs
-from ecs_composex.compute.cluster_template import generate_cluster_template
+from ecs_composex.common.tagging import generate_tags_parameters
+from ecs_composex.compute.compute_template import generate_compute_template
 
 
-def create_cluster_template(session=None, **kwargs):
+def create_compute_stack(session=None, **kwargs):
     """
     Function entrypoint for CLI.
     :param session: boto3 session to override API calls with
@@ -23,20 +25,22 @@ def create_cluster_template(session=None, **kwargs):
     :return: cluster template
     :rtype: troposphere.Template
     """
+    tags_params = ()
+    stack_params = []
     compose_content = None
-    if KEYISSET('ComposeXFile', kwargs):
-        compose_content = load_composex_file(kwargs['ComposeXFile'])
-
-    azs = []
-    if not KEYISSET('AwsAzs', kwargs):
-        if KEYISSET('AwsRegion', kwargs):
-            azs = get_curated_azs(region=kwargs['AwsRegion'])
+    if KEYISSET(XFILE_DEST, kwargs):
+        compose_content = load_composex_file(kwargs[XFILE_DEST])
+        tags_params = generate_tags_parameters(compose_content)
+    if not KEYISSET("AwsAzs", kwargs):
+        if KEYISSET("AwsRegion", kwargs):
+            azs = get_curated_azs(region=kwargs["AwsRegion"])
         elif session is None:
             session = boto3.session.Session()
             azs = get_curated_azs(session=session)
         else:
             azs = get_curated_azs()
-
     else:
-        azs = kwargs['AwsAzs']
-    return generate_cluster_template(azs, compose_content, **kwargs)
+        azs = kwargs["AwsAzs"]
+    template = generate_compute_template(azs, compose_content, tags_params, **kwargs)
+    build_default_stack_parameters(stack_params, **kwargs)
+    return template, stack_params

ecs_composex/compute/cli.py

@@ -8,16 +8,16 @@
 
 import argparse
 import sys
+import json
 
 from boto3 import session
-
-from ecs_composex.compute import create_cluster_template
+from ecs_composex import XFILE_DEST
+from ecs_composex.common import write_template_to_file
 from ecs_composex.common.aws import CURATED_AZS, BUCKET_NAME
 from ecs_composex.ecs.ecs_params import CLUSTER_NAME_T
-from ecs_composex.vpc.vpc_params import (
-    VPC_ID_T, APP_SUBNETS_T,
-    PUBLIC_SUBNETS_T
-)
+from ecs_composex.vpc.vpc_params import VPC_ID_T, APP_SUBNETS_T
+from ecs_composex.compute import create_compute_stack
+from ecs_composex.common.cfn_params import USE_FLEET_T
 
 
 def root_parser():
@@ -26,80 +26,83 @@ def root_parser():
     """
     parser = argparse.ArgumentParser()
     parser.add_argument(
-        '-f', '--docker-compose-file', required=False
+        "-f",
+        "--docker-compose-file",
+        required=False,
+        dest=XFILE_DEST,
+        help="Optionally use the YAML ComposeX file to add options and settings",
     )
     parser.add_argument(
-        '-o', '--output-file', required=True, help="Output file"
+        "-o",
+        "--output-file",
+        required=True,
+        help="Output file. Extension determines the file format",
     )
     #  AWS SETTINGS
     parser.add_argument(
-        '--region', required=False, default=session.Session().region_name,
-        dest='AwsRegion',
+        "--region",
+        required=False,
+        default=session.Session().region_name,
+        dest="AwsRegion",
         help="Specify the region you want to build for"
-             "default use default region from config or environment vars"
+        "default use default region from config or environment vars",
     )
     parser.add_argument(
-        '--az', dest='AwsAzs', action='append', required=False, default=CURATED_AZS,
-        help="List AZs you want to deploy to specifically within the region"
+        "--az",
+        dest="AwsAzs",
+        action="append",
+        required=False,
+        default=CURATED_AZS,
+        help="List AZs you want to deploy to specifically within the region",
     )
     parser.add_argument(
-        '-b', '--bucket-name', type=str, required=False, default=BUCKET_NAME,
-        help='Bucket name to upload the templates to', dest='BucketName'
+        "-b",
+        "--bucket-name",
+        type=str,
+        required=False,
+        default=BUCKET_NAME,
+        help="Bucket name to upload the templates to",
+        dest="BucketName",
     )
     # VPC SETTINGS
     parser.add_argument(
-        '--create-vpc', required=False, default=False, action='store_true',
-        help="Create a VPC for this deployment", dest='CreateVpc'
-    )
-    parser.add_argument(
-        '--vpc-cidr', required=False, default='192.168.36.0/22', dest='VpcCidr',
-        help="Specify the VPC CIDR if you use --create-vpc"
-    )
-    parser.add_argument(
-        '--vpc-id', dest=VPC_ID_T, required=False, type=str,
-        help='Specify VPC ID when not creating one'
-    )
-    parser.add_argument(
-        '--public-subnets', required=False, dest=PUBLIC_SUBNETS_T, action='append',
-        help="List of Subnet IDs to use for the cluster when not creating VPC"
+        "--vpc-id",
+        dest=VPC_ID_T,
+        required=False,
+        type=str,
+        help="Specify VPC ID when not creating one",
     )
     parser.add_argument(
-        '--app-subnets', required=False, dest=APP_SUBNETS_T, action='append',
-        help="List of Subnet IDs to use for the cluster when not creating VPC"
-    )
-    parser.add_argument(
-        '--storage-subnets', required=False, dest=APP_SUBNETS_T, action='append',
-        help="List of Subnet IDs to use for the cluster when not creating VPC"
+        "--hosts-subnets",
+        required=False,
+        dest=APP_SUBNETS_T,
+        action="append",
+        help="List of Subnet IDs to use for the cluster when not creating VPC",
     )
     # CLUSTER SETTINGS
+    parser.add_argument("--cluster-name", dest=CLUSTER_NAME_T, required=False)
     parser.add_argument(
-        '--create-cluster', required=False, default=False, action='store_true',
-        help="Create an ECS Cluster for this deployment", dest='CreateCluster'
-    )
-    parser.add_argument(
-        '--cluster-name', dest=CLUSTER_NAME_T, required=False
-    )
-    parser.add_argument(
-        '--use-spot-fleet', required=False, default=False, action='store_true',
-        dest='UseSpotFleet',
+        "--use-spot-fleet",
+        required=False,
+        default=False,
+        action="store_true",
+        dest=USE_FLEET_T,
         help="Runs spotfleet for EC2. If used in combination "
-             "of --use-fargate, it will create an additional SpotFleet"
+        "of --use-fargate, it will create an additional SpotFleet",
     )
     return parser
 
 
 def main():
     """Console script for ecs_composex."""
     parser = root_parser()
-    parser.add_argument('_', nargs='*')
+    parser.add_argument("_", nargs="*")
     args = parser.parse_args()
 
-    template = create_cluster_template(**vars(args))
-    with open(args.output_file, 'w') as tpl_fd:
-        if args.output_file.endswith('.yml') or args.output_file.endswith('.yaml'):
-            tpl_fd.write(template.to_yaml())
-        else:
-            tpl_fd.write(template.to_json())
+    template_params = create_compute_stack(**vars(args))
+    write_template_to_file(template_params[0], args.output_file)
+    with open(f"{args.output_file.split('.')[0]}.params.json", 'w') as params_fd:
+        params_fd.write(json.dumps(template_params[1], indent=4))
     return 0