Merge pull request #11 from jsauter/feature/scaling_group_steps

Initial support of asg and alb.

.gitignore

@@ -8,6 +8,7 @@ __pycache__/
 
 # Distribution / packaging
 .Python
+.vscode
 env/
 build/
 develop-eggs/

cumulus/__init__.py

@@ -1,7 +1,6 @@
-# -*- coding: utf-8 -*-
-
 """Top-level package for Cumulus."""
 
 __author__ = """Brett Swift"""
 __email__ = 'brettswift@gmail.com'
 __version__ = '0.1.5'
+

cumulus/components/userdata/linux.py

@@ -0,0 +1,38 @@
+from troposphere import (
+    Ref,
+    Join,
+)
+
+
+def user_data_for_cfn_init(launch_config_name, asg_name, configsets):
+    """
+    :return: A troposphere Join object that contains userdata for use with cfn-init
+
+    :param configsets: The single 'key' value set in the cfn-init Metadata parameter: cloudformation.InitConfigSets
+    :type asg_name: String name of the ASG cloudformation resource
+    :type launch_config_name: String name of the launch config cloudformation resource
+    """
+    default_userdata_asg_signal = (
+        Join('',
+             [
+                 "#!/bin/bash -xe\n",
+                 "yum update aws-cfn-bootstrap\n",
+                 "# Install the files and packages from the metadata\n",
+                 "/opt/aws/bin/cfn-init ",
+                 "         --stack ", Ref("AWS::StackName"),
+                 "         --resource ", launch_config_name,
+                 "         --configsets %s " % configsets,
+                 "         --region ", Ref("AWS::Region"), "\n",
+                 # "# Get exit code of cfn init to use in cfn-signal\n",
+                 # "export init_status=$?", "\n"
+                 # Signal regardless of existence of an update policy.
+                 # An error will pop up in the logs but I don't think this causes a problem.
+                 "# Signal the ASG we are ready\n\n",
+                 "/opt/aws/bin/cfn-signal -e 0",
+                 # "/opt/aws/bin/cfn-signal -e $init_status",
+                 "    --resource %s" % asg_name,
+                 "    --stack ", Ref("AWS::StackName"),
+                 "    --region ", Ref("AWS::Region"),
+                 "\n"
+             ]))
+    return default_userdata_asg_signal

cumulus/components/userdata/windows.py

@@ -0,0 +1,33 @@
+from troposphere import (
+    Ref,
+    Join,
+)
+
+
+def user_data_for_cfn_init(launch_config_name, asg_name, configsets):
+    """
+    :return: A troposphere Join object that contains userdata for use with cfn-init
+    :param configsets: The single 'key' value set in the cfn-init Metadata parameter: cloudformation.InitConfigSets
+    :type asg_name: String name of the ASG cloudformation resource
+    :type launch_config_name: String name of the launch config cloudformation resource
+    """
+    default_userdata_asg_signal = (
+        Join('',
+             [
+                 "<powershell>\n",
+                 "& ", "$env:ProgramFiles\Amazon\cfn-bootstrap\cfn-init.exe",
+                 "         --stack ", Ref("AWS::StackName"),
+                 "         --resource ", launch_config_name,
+                 "         --configsets %s " % configsets,
+                 "         --region ", Ref("AWS::Region"), "\n",
+                 "# Signal the ASG we are ready\n",
+                 "&", "$env:ProgramFiles\Amazon\cfn-signal",
+                 " -e ",
+                 " $LastExitCode",
+                 "    --resource %s" % asg_name,
+                 "    --stack ", Ref("AWS::StackName"),
+                 "    --region ", Ref("AWS::Region"),
+                 "\n",
+                 "</powershell>"
+             ]))
+    return default_userdata_asg_signal

cumulus/steps/dev_tools/pipeline.py

@@ -192,7 +192,6 @@ def handle(self, chain_context):
                 pipeline_policy
             ]
         )
-
         generic_pipeline = codepipeline.Pipeline(
             "Pipeline",
             # Name=chain_context.instance_name,

cumulus/steps/ec2/__init__.py

@@ -0,0 +1,3 @@
+
+META_SECURITY_GROUP_REF = 'security_group_ref'
+META_TARGET_GROUP_NAME = 'target_group_ref'

cumulus/steps/ec2/alb.py

@@ -0,0 +1,127 @@
+from cumulus.chain import step
+from troposphere import (
+    Ref, Not, Equals, Join, ec2,
+    If, Output
+)
+from troposphere import elasticloadbalancingv2 as alb
+
+CLUSTER_SG_NAME = "%sSG"
+ALB_SG_NAME = "%sAlbSG"
+ALB_NAME = "%sLoadBalancer"
+TARGET_GROUP_DEFAULT = "%sTargetGroup"
+
+
+class Alb(step.Step):
+
+    def __init__(self,
+                 ):
+        step.Step.__init__(self)
+
+    def handle(self, chain_context):
+        print(chain_context.instance_name)
+        self.create_conditions(chain_context.template)
+        self.create_security_groups(chain_context.template, chain_context.instance_name)
+        self.create_default_target_group(chain_context.template, chain_context.instance_name)
+        self.create_load_balancer_alb(chain_context.template, chain_context.instance_name)
+        self.add_listener(chain_context.template, chain_context.instance_name)
+
+    def create_conditions(self, template):
+        template.add_condition(
+            "UseSSL",
+            Not(Equals(Ref("ALBCertName"), "")))
+        template.add_condition(
+            "UseIAMCert",
+            Not(Equals(Ref("ALBCertType"), "acm")))
+
+    def create_security_groups(self, template, instance_name):
+        asg_sg = CLUSTER_SG_NAME % instance_name
+        alb_sg = ALB_SG_NAME % instance_name
+
+        # ALB Security group
+        template.add_resource(
+            ec2.SecurityGroup(
+                alb_sg,
+                GroupDescription=alb_sg,
+                VpcId=Ref("VpcId")
+            ))
+
+        template.add_output(
+            Output("InternalAlbSG", Value=Ref(alb_sg))
+        )
+
+        # TODO: take a list of Cidr's
+        # Allow Internet to connect to ALB
+        template.add_resource(ec2.SecurityGroupIngress(
+            "LocalNetworkTo%sAlbPort443" % instance_name,
+            IpProtocol="tcp", FromPort="443", ToPort="443",
+            CidrIp="10.0.0.0/0",
+            GroupId=Ref(alb_sg),
+        ))
+
+    def create_load_balancer_alb(self, template, instance_name):
+        alb_name = ALB_NAME % instance_name
+        alb_sg = ALB_SG_NAME % instance_name
+
+        load_balancer = template.add_resource(alb.LoadBalancer(
+            alb_name,
+            # Name=alb_name,
+            Scheme="internal",
+            Subnets=Ref("PrivateSubnets"),
+            SecurityGroups=[Ref(alb_sg)]
+        ))
+
+        template.add_output(
+            Output(
+                "CanonicalHostedZoneID",
+                Value=load_balancer.GetAtt("CanonicalHostedZoneID")
+            )
+        )
+        template.add_output(
+            Output("DNSName", Value=load_balancer.GetAtt("DNSName"))
+        )
+
+    def add_listener(self, template, instance_name):
+        # Choose proper certificate source ?-> always acm?
+        acm_cert = Join("", [
+            "arn:aws:acm:",
+            Ref("AWS::Region"),
+            ":",
+            Ref("AWS::AccountId"),
+            ":certificate/", Ref("ALBCertName")])
+        # We probably don't need this code for an IAM Cert
+        iam_cert = Join("", [
+            "arn:aws:iam::",
+            Ref("AWS::AccountId"),
+            ":server-certificate/",
+            Ref("ALBCertName")])
+        cert_id = If("UseIAMCert", iam_cert, acm_cert)
+        alb_name = ALB_NAME % instance_name
+
+        with_ssl = alb.Listener(
+            "Listener",
+            Port="443",
+            Protocol="HTTPS",
+            LoadBalancerArn=Ref(alb_name),
+            DefaultActions=[alb.Action(
+                Type="forward",
+                TargetGroupArn=Ref(TARGET_GROUP_DEFAULT % instance_name)
+            )],
+            Certificates=[alb.Certificate(
+                CertificateArn=cert_id
+            )]
+        )
+
+        template.add_resource(with_ssl)
+
+        template.add_output(
+            Output("IAlbListener", Value=with_ssl.Ref())
+        )
+
+    def create_default_target_group(self, template, instance_name):
+        print(template.__dict__)
+        template.add_resource(alb.TargetGroup(
+            TARGET_GROUP_DEFAULT % instance_name,
+            Port='80',
+            Protocol="HTTP",
+            VpcId=Ref("VpcId"),
+        ))

cumulus/steps/ec2/alb_port.py

@@ -0,0 +1,31 @@
+from troposphere import (
+    Ref, ec2, Join)
+from cumulus.chain import step
+from cumulus.steps.ec2 import META_SECURITY_GROUP_REF
+
+
+class AlbPort(step.Step):
+
+    def __init__(self,
+                 port_to_open,
+                 alb_sg_name):
+
+        step.Step.__init__(self)
+
+        self.port_to_open = port_to_open
+        self.alb_sg_name = alb_sg_name
+
+    def handle(self, chain_context):
+        template = chain_context.template
+
+        name = '%sElbToASGPort%s' % (chain_context.instance_name, self.port_to_open)
+
+        template.add_resource(ec2.SecurityGroupIngress(
+            name,
+            IpProtocol="tcp",
+            FromPort=self.port_to_open,
+            ToPort=self.port_to_open,
+            SourceSecurityGroupId=Ref(self.alb_sg_name),
+            GroupId=chain_context.metadata[META_SECURITY_GROUP_REF]
+        ))
+

cumulus/steps/ec2/block_device_data.py

@@ -0,0 +1,17 @@
+from cumulus.chain import step
+from cumulus.util.tropo import TemplateQuery
+from troposphere import autoscaling
+
+
+
+class BlockDeviceData(step.Step):
+
+    def __init__(self,
+                 volume):
+        step.Step.__init__(self)
+        self.volume = volume
+
+    def handle(self, chain_context):
+        launchConfig = TemplateQuery.get_resource_by_type(template=chain_context.template,
+                                                          type_to_find=autoscaling.LaunchConfiguration)[0]
+        launchConfig.properties['BlockDeviceMappings'] = [self.volume]

cumulus/steps/ec2/dns.py

@@ -0,0 +1,50 @@
+from troposphere import route53
+from troposphere import (
+    Ref, Join, ec2)
+from cumulus.chain import step
+
+
+class Dns(step.Step):
+
+    def __init__(self,
+                 base_domain,
+                 hosted_zone_id,
+                 dns_name,
+                 ):
+
+        step.Step.__init__(self)
+
+        self.base_domain = base_domain
+        self.hosted_zone_id = hosted_zone_id
+        self.dns_name = dns_name
+
+    def handle(self, chain_context):
+        template = chain_context.template
+
+        name = 'AlbAlias%s' % chain_context.instance_name
+
+        template.add_resource(route53.RecordSetGroup(
+            "Route53Records",
+            RecordSets=[
+                route53.RecordSet(
+                    name,
+                    Weight=1,
+                    SetIdentifier="original",
+                    AliasTarget=route53.AliasTarget(
+                        HostedZoneId=self.hosted_zone_id,
+                        DNSName=self.dns_name,
+                        EvaluateTargetHealth=False,
+                    ),
+                    Name=Join("", [
+                        Ref("namespace"),
+                        "-",
+                        Ref("env"),
+                        ".",
+                        self.base_domain,
+                        "."
+                    ]),
+                    Type="A",
+                )
+            ],
+            HostedZoneName=Join("", [self.base_domain, "."])
+        ))

cumulus/steps/ec2/ingress_rule.py

@@ -0,0 +1,33 @@
+from troposphere import elasticloadbalancingv2 as alb
+from troposphere import (
+    Ref, ec2)
+import re
+from cumulus.chain import step
+from cumulus.steps.ec2 import META_SECURITY_GROUP_REF
+
+
+class IngressRule(step.Step):
+
+    def __init__(self,
+                 port_to_open,
+                 cidr):
+
+        step.Step.__init__(self)
+
+        self.port_to_open = port_to_open
+        self.cidr = cidr
+
+    def handle(self, chain_context):
+        template = chain_context.template
+
+        clean_cidr = re.compile('[\W_]+').sub('', self.cidr)
+
+        template.add_resource(ec2.SecurityGroupIngress(
+            "Cidr%sToASGPort%s" % (clean_cidr, self.port_to_open),
+            IpProtocol="tcp",
+            FromPort=self.port_to_open,
+            ToPort=self.port_to_open,
+            CidrIp=self.cidr,
+            GroupId=chain_context.metadata[META_SECURITY_GROUP_REF]
+        ))
+

cumulus/steps/ec2/instance_profile_role.py

@@ -0,0 +1,32 @@
+from troposphere import Ref
+
+from cumulus.chain import step
+from troposphere.iam import InstanceProfile
+
+from cumulus.util.tropo import TemplateQuery
+
+
+class InstanceProfileRole(step.Step):
+
+    def __init__(self,
+                 instance_profile_name,
+                 role):
+        step.Step.__init__(self)
+        self.instance_profile_name = instance_profile_name
+        self.role = role
+
+    def handle(self, chain_context):
+        template = chain_context.template
+
+        template.add_resource(self.role)
+
+        try:
+            instanceProfile = TemplateQuery.get_resource_by_title(template, self.instance_profile_name)
+            instanceProfile.properties['Roles'].append(Ref(self.role))
+        except ValueError:
+            print('Adding new Instance Profile')
+            template.add_resource(InstanceProfile(
+                self.instance_profile_name,
+                Roles=[Ref(self.role)]
+            ))
+