diff --git a/ecs_composex/__init__.py b/ecs_composex/__init__.py index 611bab9ce..90d7b821b 100644 --- a/ecs_composex/__init__.py +++ b/ecs_composex/__init__.py @@ -5,3 +5,5 @@ __version__ = "0.1.3" XFILE_DEST = "ComposeXFile" +DIR_DEST = "OutputDirectory" +FILE_DEST = "OutputFile" diff --git a/ecs_composex/cli.py b/ecs_composex/cli.py index 2732e0d24..a93be587b 100644 --- a/ecs_composex/cli.py +++ b/ecs_composex/cli.py @@ -3,19 +3,21 @@ """Console script for ecs_composex.""" -import sys -import argparse +import os import warnings -import json + +import argparse +import sys from boto3 import session -from ecs_composex import XFILE_DEST -from ecs_composex.common import LOG, write_template_to_file -from ecs_composex.common.aws import BUCKET_NAME, CURATED_AZS + +from ecs_composex import XFILE_DEST, DIR_DEST from ecs_composex.common import KEYISSET -from ecs_composex.common.cfn_tools import ( - build_config_template_file, - write_config_template_file, -) +from ecs_composex.common import LOG +from ecs_composex.common.aws import BUCKET_NAME, CURATED_AZS +from ecs_composex.common.cfn_params import USE_FLEET_T +from ecs_composex.common.cfn_tools import build_config_template_file +from ecs_composex.common.templates import FileArtifact +from ecs_composex.compute.compute_params import CLUSTER_NAME_T from ecs_composex.root import generate_full_template from ecs_composex.vpc.vpc_params import ( APP_SUBNETS_T, @@ -24,8 +26,6 @@ VPC_ID_T, VPC_MAP_ID_T, ) -from ecs_composex.common.cfn_params import USE_FLEET_T -from ecs_composex.compute.compute_params import CLUSTER_NAME_T def validate_vpc_input(args): @@ -88,10 +88,17 @@ def main(): parser.add_argument( "-o", "--output-file", + required=False, + default=f"{os.path.basename(os.path.dirname(__file__))}.yml", + help="Output file. Extension determines the file format", + ) + parser.add_argument( + "-d", + "--output-dir", + required=False, + help="Output directory to write all the templates to.", type=str, - required=True, - help="The name and path of the main output file. If you specify extra arguments, it will create a parameters" - " file as well for creating your CFN Stack", + dest=DIR_DEST, ) parser.add_argument( "--cfn-config-file", @@ -239,15 +246,24 @@ def main(): print("Arguments: " + str(args._)) templates_and_params = generate_full_template(**vars(args)) - write_template_to_file(templates_and_params[0], args.output_file) + + template_file = FileArtifact( + args.output_file, template=templates_and_params[0], **vars(args) + ) + template_file.create() cfn_config = build_config_template_file(templates_and_params[1]) if KEYISSET("CfnConfigFile", vars(args)): - config_file_path = args.CfnConfigFile + config_file_name = args.CfnConfigFile else: - config_file_path = f"{args.output_file.split('.')[0]}.config.json" - write_config_template_file(cfn_config, config_file_path) - with open(f"{args.output_file.split('.')[0]}.params.json", "w") as params_fd: - params_fd.write(json.dumps(templates_and_params[1], indent=4)) + config_file_name = f"{args.output_file.split('.')[0]}.config.json" + config_file = FileArtifact(config_file_name, content=cfn_config, **vars(args)) + config_file.create() + params_file = FileArtifact( + f"{args.output_file.split('.')[0]}.params.json", + content=templates_and_params[1], + **vars(args), + ) + params_file.create() if __name__ == "__main__": diff --git a/ecs_composex/common/templates.py b/ecs_composex/common/templates.py index dbfc66574..f1ba54912 100644 --- a/ecs_composex/common/templates.py +++ b/ecs_composex/common/templates.py @@ -3,11 +3,22 @@ """ Functions to manage a template and wheter it should be stored in S3 """ +import yaml + +try: + from yaml import CDumper as Dumper +except ImportError: + from yaml import Dumper +from os import path, mkdir import boto3 +import json from botocore.exceptions import ClientError +from datetime import datetime as dt +from troposphere import Template -from ecs_composex.common import DATE_PREFIX +from ecs_composex import DIR_DEST +from ecs_composex.common import DATE_PREFIX, KEYISSET from ecs_composex.common import LOG @@ -100,6 +111,7 @@ def upload_template( prefix=None, session=None, client=None, + mime=None, **kwargs, ): """Upload template_body to a file in s3 with given prefix and bucket_name @@ -125,7 +137,8 @@ def upload_template( if session is None: session = boto3.session.Session() assert check_bucket(bucket_name=bucket_name, session=session) - + if mime is None: + mime = "application/json" if prefix is None: prefix = DATE_PREFIX @@ -138,7 +151,7 @@ def upload_template( Key=key, Bucket=bucket_name, ContentEncoding="utf-8", - ContentType="application/json", + ContentType=mime, ServerSideEncryption="AES256", **kwargs, ) @@ -154,3 +167,130 @@ def upload_template( except Exception as error: LOG.debug(error) return None + + +def write_file_to_directory(file_content, directory, file_name=None, **kwargs): + """ + Function to write a file to a directory. + :param file_content: + :param directory: + :param file_name: + :return: + """ + + +class FileArtifact(object): + """ + Class for a template file built. + """ + + url = None + body = None + template = None + file_name = None + mime = "text/plain" + session = boto3.session.Session() + bucket = None + uploadable = False + validated = False + output_dir = f"/tmp/{dt.utcnow().strftime('%s')}" + uploaded = False + + def upload(self): + if not self.uploadable: + LOG.error("BucketName was not specified, not attempting upload") + elif not self.validated: + LOG.error("The template was not correctly validated. Not uploading") + else: + self.url = upload_template( + self.body, self.bucket, self.file_name, mime=self.mime, validate=False + ) + LOG.info(f"Template {self.file_name} uploaded successfully to {self.url}") + self.uploaded = True + + def write(self): + try: + mkdir(self.output_dir) + LOG.info(f"Created directory {self.output_dir} to store files") + except FileExistsError: + LOG.debug(f"Output directory {self.output_dir} already exists") + with open(f"{self.output_dir}/{self.file_name}", "w") as template_fd: + template_fd.write(self.body) + LOG.info( + f"Template {self.file_name} written successfully at {self.output_dir}/{self.file_name}" + ) + + def validate(self): + try: + self.session.client("cloudformation").validate_template( + TemplateBody=self.template.to_json() + ) + LOG.debug(f"Template {self.file_name} was validated successfully by CFN") + self.validated = True + except ClientError as error: + LOG.error(error) + + def define_file_specs(self): + """ + Function to set the file body from template if self.template is Template + :return: + """ + if self.file_name.endswith(".json"): + self.mime = "application/json" + elif self.file_name.endswith(".yml") or self.file_name.endswith(".yaml"): + self.mime = "application/x-yaml" + if isinstance(self.template, Template): + if self.mime == "application/x-yaml": + self.body = self.template.to_yaml() + else: + self.body = self.template.to_json() + elif isinstance(self.content, (list, dict, tuple)): + self.validated = True + self.uploadable = True + if self.mime == "application/x-yaml": + self.body = yaml.dump(self.content, Dumper=Dumper) + elif self.mime == "application/json": + self.body = json.dumps(self.content, indent=4) + + def set_from_kwargs(self, **kwargs): + """ + Function to set internal settings based on kwargs keys + :param kwargs: + """ + if KEYISSET(DIR_DEST, kwargs): + self.output_dir = path.abspath(kwargs[DIR_DEST]) + if KEYISSET("BucketName", kwargs): + self.bucket = kwargs["BucketName"] + self.uploadable = True + + def create(self): + """ + Function to write to file and upload in a single function + """ + self.write() + self.upload() + + def __init__(self, file_name, template=None, content=None, session=None, **kwargs): + """ + Init function for our template file object + :param file_name: Name of the file. Mandatory + :param template: If you are providing a template to generate + :param body: raw content to write + :param session: + :param kwargs: + """ + self.file_name = file_name + if template is not None and not isinstance(template, Template): + raise TypeError("template must be of type", Template, "got", type(template)) + elif isinstance(template, Template): + self.template = template + self.validate() + if session is not None: + self.session = session + self.set_from_kwargs(**kwargs) + if content is not None and isinstance(content, (tuple, dict, str, list)): + self.content = content + self.define_file_specs() + + def __repr__(self): + return f"{self.output_dir}/{self.file_name}" diff --git a/ecs_composex/compute/cli.py b/ecs_composex/compute/cli.py index a23d6d9ac..459107039 100644 --- a/ecs_composex/compute/cli.py +++ b/ecs_composex/compute/cli.py @@ -8,11 +8,11 @@ import argparse import sys -import json +import os from boto3 import session -from ecs_composex import XFILE_DEST -from ecs_composex.common import write_template_to_file +from ecs_composex import XFILE_DEST, DIR_DEST +from ecs_composex.common.templates import FileArtifact from ecs_composex.common.aws import CURATED_AZS, BUCKET_NAME from ecs_composex.ecs.ecs_params import CLUSTER_NAME_T from ecs_composex.vpc.vpc_params import VPC_ID_T, APP_SUBNETS_T @@ -35,9 +35,18 @@ def root_parser(): parser.add_argument( "-o", "--output-file", - required=True, + required=False, + default=f"{os.path.basename(os.path.dirname(__file__))}.yml", help="Output file. Extension determines the file format", ) + parser.add_argument( + "-d", + "--output-dir", + required=False, + help="Output directory to write all the templates to.", + type=str, + dest=DIR_DEST, + ) # AWS SETTINGS parser.add_argument( "--region", @@ -90,6 +99,12 @@ def root_parser(): help="Runs spotfleet for EC2. If used in combination " "of --use-fargate, it will create an additional SpotFleet", ) + parser.add_argument( + "--no-upload", + action="store_true", + default=False, + help="Do not upload the file to S3.", + ) return parser @@ -100,9 +115,19 @@ def main(): args = parser.parse_args() template_params = create_compute_stack(**vars(args)) - write_template_to_file(template_params[0], args.output_file) - with open(f"{args.output_file.split('.')[0]}.params.json", "w") as params_fd: - params_fd.write(json.dumps(template_params[1], indent=4)) + template_file = FileArtifact( + args.output_file, template=template_params[0], **vars(args) + ) + params_file_name = f"{args.output_file.split('.')[0]}.params.json" + params_file = FileArtifact( + params_file_name, content=template_params[1], **vars(args) + ) + if args.no_upload: + template_file.write() + params_file.write() + else: + template_file.create() + params_file.create() return 0 diff --git a/ecs_composex/compute/compute_template.py b/ecs_composex/compute/compute_template.py index 2ab34cff7..9fc3f1f79 100644 --- a/ecs_composex/compute/compute_template.py +++ b/ecs_composex/compute/compute_template.py @@ -15,13 +15,8 @@ generate_spot_fleet_template, DEFAULT_SPOT_CONFIG, ) -from ecs_composex.common import ( - build_template, - KEYISSET, - LOG, - add_parameters, - write_template_to_file, -) +from ecs_composex.common.templates import FileArtifact +from ecs_composex.common import build_template, KEYISSET, LOG, add_parameters from ecs_composex.common import cfn_conditions from ecs_composex.common.cfn_params import ( ROOT_STACK_NAME, @@ -29,7 +24,6 @@ USE_FLEET, USE_ONDEMAND, ) -from ecs_composex.common.templates import upload_template from ecs_composex.vpc import vpc_params from ecs_composex.ecs.ecs_params import CLUSTER_NAME from ecs_composex.common.tagging import add_object_tags @@ -84,20 +78,19 @@ def add_spotfleet_stack( parameters.update({tag.title: Ref(tag.title)}) for resource in fleet_template.resources: add_object_tags(fleet_template.resources[resource], tags[1]) - fleet_template_url = upload_template( - fleet_template.to_json(), kwargs["BucketName"], "spot_fleet.json" - ) - if not fleet_template_url: + fleet_file = FileArtifact("spot_fleet.yml", template=fleet_template, **kwargs) + fleet_file.write() + fleet_file.upload() + if not fleet_file.url: LOG.warn( "Fleet template URL not returned. Not adding SpotFleet to Cluster stack" ) return - write_template_to_file(fleet_template, "/tmp/spot_fleet.yml") template.add_resource( Stack( "SpotFleet", Condition=cfn_conditions.USE_SPOT_CON_T, - TemplateURL=fleet_template_url, + TemplateURL=fleet_file.url, Parameters=parameters, ) ) diff --git a/ecs_composex/ecs/ecs_service.py b/ecs_composex/ecs/ecs_service.py index e72101268..1d995f843 100644 --- a/ecs_composex/ecs/ecs_service.py +++ b/ecs_composex/ecs/ecs_service.py @@ -12,11 +12,12 @@ DeploymentController, ) -from ecs_composex.common import LOG, cfn_conditions from ecs_composex.common import build_template, cfn_params, add_parameters +from ecs_composex.common import cfn_conditions from ecs_composex.common.cfn_params import ROOT_STACK_NAME_T from ecs_composex.common.outputs import formatted_outputs -from ecs_composex.common.templates import upload_template +from ecs_composex.common.tagging import add_object_tags +from ecs_composex.common.templates import FileArtifact from ecs_composex.ecs import ecs_conditions from ecs_composex.ecs import ecs_params from ecs_composex.ecs.ecs_iam import add_service_roles, assign_x_resources_to_service @@ -28,7 +29,6 @@ from ecs_composex.ecs.ecs_networking_ingress import define_service_to_service_ingress from ecs_composex.ecs.ecs_task import add_task_defnition from ecs_composex.vpc import vpc_params -from ecs_composex.common.tagging import add_object_tags STATIC = 0 @@ -221,11 +221,8 @@ def generate_service_template( if tags and tags[1]: for resource in service_tpl.resources: add_object_tags(service_tpl.resources[resource], tags[1]) - service_tpl_url = upload_template( - service_tpl.to_json(), - kwargs["BucketName"], - f"{service_name}.json", - session=session, + service_tpl_file = FileArtifact( + f"{service_name}.yml", service_tpl, session=session, **kwargs ) - LOG.debug(service_tpl_url) - return service_tpl_url, parameters, stack_dependencies + service_tpl_file.create() + return service_tpl_file.url, parameters, stack_dependencies diff --git a/ecs_composex/ecs/ecs_template.py b/ecs_composex/ecs/ecs_template.py index e4a3b504f..283f19da9 100644 --- a/ecs_composex/ecs/ecs_template.py +++ b/ecs_composex/ecs/ecs_template.py @@ -133,7 +133,4 @@ def generate_services_templates(compose_content, session=None, **kwargs): ) for resource in template.resources: add_object_tags(template.resources[resource], tags_params[1]) - if KEYISSET("Debug", kwargs): - with open("/tmp/ecs_root.json", "w") as services_fd: - services_fd.write(template.to_json()) return template diff --git a/ecs_composex/root.py b/ecs_composex/root.py index dffea87f0..56fa2b221 100644 --- a/ecs_composex/root.py +++ b/ecs_composex/root.py @@ -23,7 +23,7 @@ USE_ONDEMAND_T, ) from ecs_composex.common.tagging import generate_tags_parameters, add_object_tags -from ecs_composex.common.templates import upload_template +from ecs_composex.common.templates import upload_template, FileArtifact from ecs_composex.compute import create_compute_stack from ecs_composex.compute.compute_params import ( TARGET_CAPACITY_T, @@ -92,17 +92,14 @@ def add_vpc_to_root(root_template, session, tags_params=None, **kwargs): if tags_params is None: tags_params = () vpc_template = create_vpc_template(session=session, **kwargs) - vpc_template_url = upload_template( - vpc_template.to_json(), kwargs["BucketName"], "vpc.json", session=session - ) - LOG.debug(vpc_template_url) - with open(f"{kwargs['output_file']}.vpc.json", "w") as vpc_fd: - vpc_fd.write(vpc_template.to_json()) + vpc_file = FileArtifact("vpc.yml", template=vpc_template, **kwargs) + vpc_file.create() + LOG.debug(vpc_file.url) parameters = {ROOT_STACK_NAME_T: Ref("AWS::StackName")} for param in tags_params: parameters.update({param.title: Ref(param.title)}) vpc_stack = root_template.add_resource( - Stack("Vpc", TemplateURL=vpc_template_url, Parameters=parameters) + Stack("Vpc", TemplateURL=vpc_file.url, Parameters=parameters) ) return vpc_stack @@ -147,6 +144,8 @@ def add_compute( depends_on = [] root_template.add_parameter(TARGET_CAPACITY) compute_template = create_compute_stack(session, **kwargs) + compute_file = FileArtifact("compute.yml", template=compute_template[0], **kwargs) + compute_file.create() parameters = { ROOT_STACK_NAME_T: Ref("AWS::StackName"), TARGET_CAPACITY_T: Ref(TARGET_CAPACITY), @@ -181,16 +180,12 @@ def add_compute( build_parameters_file( params, vpc_params.APP_SUBNETS_T, kwargs[vpc_params.APP_SUBNETS_T] ) - + params_file = FileArtifact("compute.params.json", content=parameters) + params_file.create() compute_stack = root_template.add_resource( Stack( "Compute", - TemplateURL=upload_template( - template_body=compute_template[0].to_json(), - bucket_name=kwargs["BucketName"], - file_name="compute.json", - session=session, - ), + TemplateURL=compute_file.url, Parameters=parameters, DependsOn=dependencies, ) @@ -215,12 +210,8 @@ def add_x_resources(template, session, tags=None, **kwargs): create_function = get_mod_function(res_type, function_name) if create_function: x_template = create_function(session=session, **kwargs) - x_template_url = upload_template( - x_template.to_json(), - kwargs["BucketName"], - f"{res_type}.json", - session=session, - ) + x_file = FileArtifact(f"{res_type}.yml", template=x_template, **kwargs) + x_file.create() depends_on.append(res_type.title().strip()) parameters = {ROOT_STACK_NAME_T: Ref("AWS::StackName")} for tag in tags: @@ -228,7 +219,7 @@ def add_x_resources(template, session, tags=None, **kwargs): template.add_resource( Stack( res_type.title().strip(), - TemplateURL=x_template_url, + TemplateURL=x_file.url, Parameters=parameters, ) ) @@ -249,11 +240,10 @@ def add_services(template, depends, session, vpc_stack=None, **kwargs): :param kwargs: optional parameters """ services_template = create_services_templates(session=session, **kwargs) - services_template_url = upload_template( - template_body=services_template.to_json(), - bucket_name=kwargs["BucketName"], - file_name="services.json", + services_root_file = FileArtifact( + "services.yml", template=services_template, **kwargs ) + services_root_file.create() parameters = {ROOT_STACK_NAME_T: Ref("AWS::StackName")} if KEYISSET("CreateCluster", kwargs): parameters[ecs_params.CLUSTER_NAME_T] = Ref(ROOT_CLUSTER_NAME) @@ -284,7 +274,7 @@ def add_services(template, depends, session, vpc_stack=None, **kwargs): return Stack( "Services", template=template, - TemplateURL=services_template_url, + TemplateURL=services_root_file.url, Parameters=parameters, DependsOn=depends, ) diff --git a/ecs_composex/sqs/cli.py b/ecs_composex/sqs/cli.py index 66590b26e..27d2925c5 100644 --- a/ecs_composex/sqs/cli.py +++ b/ecs_composex/sqs/cli.py @@ -3,15 +3,33 @@ """Console script for ecs_composex.sqs""" import sys +import os import argparse +from ecs_composex import DIR_DEST from ecs_composex.common.aws import BUCKET_NAME from ecs_composex.sqs import create_sqs_template +from ecs_composex.common.templates import FileArtifact def main(): """Console script for ecs_composex.""" parser = argparse.ArgumentParser() + parser.add_argument( + "-o", + "--output-file", + required=False, + default=f"{os.path.basename(os.path.dirname(__file__))}.yml", + help="Output file. Extension determines the file format", + ) + parser.add_argument( + "-d", + "--output-dir", + required=False, + help="Output directory to write all the templates to.", + type=str, + dest=DIR_DEST, + ) parser.add_argument( "-b", "--bucket-name", @@ -28,18 +46,13 @@ def main(): dest="ComposeXFile", help="Path to the Docker Compose / ComposeX file", ) - parser.add_argument( - "-o", "--output-file", required=True, help="Output file for the template body" - ) parser.add_argument("_", nargs="*") args = parser.parse_args() template = create_sqs_template(**vars(args)) - with open(args.output_file, "w") as tpl_fd: - if args.output_file.endswith(".yml") or args.output_file.endswith(".yaml"): - tpl_fd.write(template.to_yaml()) - else: - tpl_fd.write(template.to_json()) + template_file = FileArtifact(args.output_file, template=template, **vars(args)) + template_file.create() + return 0 diff --git a/ecs_composex/sqs/sqs_template.py b/ecs_composex/sqs/sqs_template.py index 720006f00..1a51a39e8 100644 --- a/ecs_composex/sqs/sqs_template.py +++ b/ecs_composex/sqs/sqs_template.py @@ -26,7 +26,7 @@ ) from ecs_composex.common.cfn_params import ROOT_STACK_NAME, ROOT_STACK_NAME_T from ecs_composex.common.outputs import formatted_outputs -from ecs_composex.common.templates import upload_template +from ecs_composex.common.templates import FileArtifact from ecs_composex.sqs.sqs_params import ( SQS_NAME_T, SQS_NAME, @@ -132,10 +132,14 @@ def generate_queue_template(queue_name, properties, redrive_queue=None, tags=Non """ Function that generates a single queue template + :param redrive_queue: SQS Redrive queue for DLQ + :type redrive_queue: str :param queue_name: Name of the Queue as defined in ComposeX File :type queue_name: str :param properties: The queue properties :type properties: dict + :param tags: tags to add to the queue + :type tags: troposphere.Tags :returns: queue_template :rtype: troposphere.Template @@ -220,17 +224,15 @@ def add_queue_stack(queue_name, queue, queues, session, tags, **kwargs): parameters.update({tag.title: Ref(tag.title)}) LOG.debug(parameters) LOG.debug(session) - template_url = upload_template( - template_body=queue_tpl.to_json(), - bucket_name=kwargs["BucketName"], - file_name=f"{queue_name}.json", - session=session, + template_file = FileArtifact( + f"{queue_name}.yml", template=queue_tpl, session=session, **kwargs ) + template_file.create() queue_stack = Stack( queue_name, Parameters=parameters, DependsOn=depends_on, - TemplateURL=template_url, + TemplateURL=template_file.url, ) return queue_stack diff --git a/ecs_composex/vpc/cli.py b/ecs_composex/vpc/cli.py index 12547ff6c..b1d3cf6ff 100644 --- a/ecs_composex/vpc/cli.py +++ b/ecs_composex/vpc/cli.py @@ -7,9 +7,10 @@ import argparse from boto3 import session -from ecs_composex.common.aws import CURATED_AZS +from ecs_composex.common.aws import CURATED_AZS, BUCKET_NAME from ecs_composex.vpc import create_vpc_template -from ecs_composex import XFILE_DEST +from ecs_composex import XFILE_DEST, DIR_DEST +from ecs_composex.common.templates import FileArtifact def main(): @@ -31,6 +32,14 @@ def main(): help="Specify the region you want to build for" "default use default region from config or environment vars", ) + parser.add_argument( + "-d", + "--output-dir", + required=False, + help="Output directory to write all the templates to.", + type=str, + dest=DIR_DEST, + ) parser.add_argument( "--az", dest="AwsAzs", @@ -48,16 +57,32 @@ def main(): action="store_true", help="Whether you want a single NAT for your application subnets or not. Not recommended for production", ) + parser.add_argument( + "--no-upload", + action="store_true", + default=False, + help="Do not upload the file to S3.", + ) + parser.add_argument( + "-b", + "--bucket-name", + type=str, + required=False, + default=BUCKET_NAME, + help="Bucket name to upload the templates to", + dest="BucketName", + ) parser.add_argument("_", nargs="*") args = parser.parse_args() template = create_vpc_template(**vars(args)) + file_name = "vpc.yml" if args.output_file: - with open(args.output_file, "w") as tpl_fd: - if args.output_file.endswith(".yml") or args.output_file.endswith(".yaml"): - tpl_fd.write(template.to_yaml()) - else: - tpl_fd.write(template.to_json()) + file_name = args.output_file + template_file = FileArtifact(file_name, template=template, **vars(args)) + template_file.write() + if not args.no_upload: + template_file.upload() return 0