Support for injectable cloudformation roles

Add validation check for pipeline when code build is added out of order
Give codebuild access to codepipeline
Revert name of pipeline
Add missing package
Add user parameters to lambda pipeline step

cumulus/policies/codebuild.py

@@ -31,6 +31,7 @@ def get_policy_code_build_general_access(policy_name):
                         awacs.aws.Action("apigateway", "*"),
                         awacs.aws.Action("cloudwatch", "*"),
                         awacs.aws.Action("cloudfront", "*"),
+                        awacs.aws.Action("codepipeline", "*"),
                         awacs.aws.Action("rds", "*"),
                         awacs.aws.Action("dynamodb", "*"),
                         awacs.aws.Action("lambda", "*"),

cumulus/steps/dev_tools/cloud_formation_action.py

@@ -25,6 +25,7 @@ def __init__(self,
                  output_artifact_name=None,
                  cfn_action_role_arn=None,
                  cfn_action_config_role_arn=None,
+                 cfn_param_overrides=None,
                  ):
         """
         :type cfn_action_config_role_arn: [troposphere.iam.Policy]
@@ -37,6 +38,7 @@ def __init__(self,
         :type action_mode: cumulus.types.cloudformation.action_mode.ActionMode The actual CloudFormation action to execute
         """
         step.Step.__init__(self)
+        self.cfn_param_overrides = cfn_param_overrides
         self.action_name = action_name
         self.input_artifact_names = input_artifact_names
         self.input_template_path = input_template_path
@@ -95,6 +97,9 @@ def handle(self, chain_context):
         if self.cfn_action_role_arn:
             cloud_formation_action.RoleArn = self.cfn_action_role_arn
 
+        if self.cfn_param_overrides:
+            cloud_formation_action.Configuration['ParameterOverrides'] = self.cfn_param_overrides
+
         stage = cumulus.util.template_query.TemplateQuery.get_pipeline_stage_by_name(
             template=chain_context.template,
             stage_name=self.stage_name_to_add,

cumulus/steps/dev_tools/code_build_action.py

@@ -1,23 +1,19 @@
 import awacs
-import troposphere
-
 import awacs.aws
+import awacs.ec2
+import awacs.iam
 import awacs.logs
 import awacs.s3
-import awacs.iam
-import awacs.ec2
 import awacs.sts
+import troposphere
+from troposphere import iam, \
+    codebuild, codepipeline, Ref, ec2
 
 import cumulus.policies
 import cumulus.policies.codebuild
 import cumulus.types.codebuild.buildaction
 import cumulus.util.template_query
-
-from troposphere import iam,\
-    codebuild, codepipeline, Ref, ec2
-
 from cumulus.chain import step
-
 from cumulus.steps.dev_tools import META_PIPELINE_BUCKET_POLICY_REF, \
     META_PIPELINE_BUCKET_NAME
 
@@ -26,11 +22,13 @@ class CodeBuildAction(step.Step):
 
     def __init__(self,
                  action_name,
-                 input_artifact_name,
                  stage_name_to_add,
+                 input_artifact_name,
                  environment=None,
                  vpc_config=None,
-                 buildspec='buildspec.yml'):
+                 buildspec='buildspec.yml',
+                 role_arn=None,
+                 ):
         """
         :type buildspec: basestring path to buildspec.yml or text containing the buildspec.
         :type input_artifact_name: basestring The artifact name in the pipeline. Must contain a buildspec.yml
@@ -39,6 +37,7 @@ def __init__(self,
         :type vpc_config.Vpc_Config: Only required if the codebuild step requires access to the VPC
         """
         step.Step.__init__(self)
+        self.role_arn = role_arn
         self.buildspec = buildspec
         self.environment = environment
         self.input_artifact_name = input_artifact_name
@@ -48,34 +47,22 @@ def __init__(self,
 
     def handle(self, chain_context):
 
+        self.validate(chain_context)
+
         print("Adding action %s Stage." % self.action_name)
         full_action_name = "%s%s" % (self.stage_name_to_add, self.action_name)
 
         policy_name = "%sCodeBuildPolicy" % chain_context.instance_name
         role_name = "CodeBuildRole%s" % full_action_name
 
-        codebuild_role = iam.Role(
-            role_name,
-            Path="/",
-            AssumeRolePolicyDocument=awacs.aws.Policy(
-                Statement=[
-                    awacs.aws.Statement(
-                        Effect=awacs.aws.Allow,
-                        Action=[awacs.sts.AssumeRole],
-                        Principal=awacs.aws.Principal(
-                            'Service',
-                            "codebuild.amazonaws.com"
-                        )
-                    )]
-            ),
-            Policies=[
-                cumulus.policies.codebuild.get_policy_code_build_general_access(policy_name)
-            ],
-            ManagedPolicyArns=[
-                chain_context.metadata[META_PIPELINE_BUCKET_POLICY_REF]
-            ]
+        codebuild_role = self.get_default_code_build_role(
+            chain_context=chain_context,
+            policy_name=policy_name,
+            role_name=role_name,
         )
 
+        codebuild_role_arn = self.role_arn if self.role_arn else troposphere.GetAtt(codebuild_role, 'Arn')
+
         if not self.environment:
             self.environment = codebuild.Environment(
                 ComputeType='BUILD_GENERAL1_SMALL',
@@ -89,7 +76,7 @@ def handle(self, chain_context):
 
         project = self.create_project(
             chain_context=chain_context,
-            codebuild_role=codebuild_role,
+            codebuild_role_arn=codebuild_role_arn,
             codebuild_environment=self.environment,
             name=full_action_name,
         )
@@ -117,7 +104,31 @@ def handle(self, chain_context):
         code_build_action.RunOrder = next_run_order
         stage.Actions.append(code_build_action)
 
-    def create_project(self, chain_context, codebuild_role, codebuild_environment, name):
+    def get_default_code_build_role(self, chain_context, policy_name, role_name):
+        codebuild_role = iam.Role(
+            role_name,
+            Path="/",
+            AssumeRolePolicyDocument=awacs.aws.Policy(
+                Statement=[
+                    awacs.aws.Statement(
+                        Effect=awacs.aws.Allow,
+                        Action=[awacs.sts.AssumeRole],
+                        Principal=awacs.aws.Principal(
+                            'Service',
+                            "codebuild.amazonaws.com"
+                        )
+                    )]
+            ),
+            Policies=[
+                cumulus.policies.codebuild.get_policy_code_build_general_access(policy_name)
+            ],
+            ManagedPolicyArns=[
+                chain_context.metadata[META_PIPELINE_BUCKET_POLICY_REF]
+            ]
+        )
+        return codebuild_role
+
+    def create_project(self, chain_context, codebuild_role_arn, codebuild_environment, name):
 
         artifacts = codebuild.Artifacts(Type='CODEPIPELINE')
 
@@ -152,11 +163,10 @@ def create_project(self, chain_context, codebuild_role, codebuild_environment, n
 
         project = codebuild.Project(
             project_name,
-            DependsOn=codebuild_role,
             Artifacts=artifacts,
             Environment=codebuild_environment,
             Name="%s-%s" % (chain_context.instance_name, project_name),
-            ServiceRole=troposphere.GetAtt(codebuild_role, 'Arn'),
+            ServiceRole=codebuild_role_arn,
             Source=codebuild.Source(
                 "Deploy",
                 Type='CODEPIPELINE',
@@ -166,3 +176,8 @@ def create_project(self, chain_context, codebuild_role, codebuild_environment, n
         )
 
         return project
+
+    def validate(self, chain_context):
+        if META_PIPELINE_BUCKET_POLICY_REF not in chain_context.metadata:
+            raise AssertionError("Could not find expected 'META_PIPELINE_BUCKET_POLICY_REF' in metadata. "
+                                 "Maybe you added the code build step to the chain before the pipeline step?")

cumulus/steps/dev_tools/lambda_action.py

@@ -11,7 +11,7 @@
 import cumulus.policies
 import cumulus.policies.codebuild
 import cumulus.types.codebuild.buildaction
-import cumulus.util.tropo
+import cumulus.util.template_query
 from cumulus.chain import step
 from cumulus.steps.dev_tools import META_PIPELINE_BUCKET_POLICY_REF
 
@@ -74,14 +74,15 @@ def handle(self, chain_context):
                 codepipeline.InputArtifacts(Name=self.input_artifact_name)
             ],
             Configuration={
-                'FunctionName': self.function_name
+                'FunctionName': self.function_name,
+                'UserParameters': self.user_parameters,
             },
             RunOrder="1"
         )
 
         chain_context.template.add_resource(lambda_role)
 
-        stage = cumulus.util.tropo.TemplateQuery.get_pipeline_stage_by_name(
+        stage = cumulus.util.template_query.TemplateQuery.get_pipeline_stage_by_name(
             template=chain_context.template,
             stage_name=self.stage_name_to_add,
         )

cumulus/steps/dev_tools/pipeline.py

@@ -1,35 +1,36 @@
 
 import awacs
-import troposphere
-
-import awacs.iam
 import awacs.aws
-import awacs.sts
-import awacs.s3
-import awacs.logs
+import awacs.awslambda
+import awacs.codecommit
 import awacs.ec2
 import awacs.iam
-import awacs.codecommit
-import awacs.awslambda
-
-from cumulus.chain import step
-import cumulus.steps.dev_tools
-
+import awacs.logs
+import awacs.s3
+import awacs.sts
+import awacs.kms
+import troposphere
 from troposphere import codepipeline, Ref, iam
 from troposphere.s3 import Bucket, VersioningConfiguration
 
+import cumulus.steps.dev_tools
+from cumulus.chain import step
+
 
 class Pipeline(step.Step):
     def __init__(self,
                  name,
                  bucket_name,
+                 pipeline_service_role_arn=None,
                  create_bucket=True,
                  pipeline_policies=None,
                  bucket_policy_statements=None,
                  bucket_kms_key_arn=None,
                  ):
         """
 
+        :type pipeline_service_role_arn: basestring Override the pipeline service role. If you pass this
+                                                     the pipeline_policies is not used.
         :type create_bucket: bool if False, will not create the bucket. Will attach policies either way.
         :type bucket_name: the name of the bucket that will be created suffixed with the chaincontext instance name
         :type bucket_policy_statements: [awacs.aws.Statement]
@@ -40,6 +41,7 @@ def __init__(self,
         self.name = name
         self.bucket_name = bucket_name
         self.create_bucket = create_bucket
+        self.pipeline_service_role_arn = pipeline_service_role_arn
         self.bucket_policy_statements = bucket_policy_statements
         self.pipeline_policies = pipeline_policies or []
         self.bucket_kms_key_arn = bucket_kms_key_arn
@@ -53,7 +55,6 @@ def handle(self, chain_context):
         :param chain_context:
         :return:
         """
-
         if self.create_bucket:
             pipeline_bucket = Bucket(
                 "PipelineBucket%s" % chain_context.instance_name,
@@ -87,6 +88,48 @@ def handle(self, chain_context):
         chain_context.metadata[cumulus.steps.dev_tools.META_PIPELINE_BUCKET_POLICY_REF] = Ref(
             pipeline_bucket_access_policy)
 
+        default_pipeline_role = self.get_default_pipeline_role()
+        pipeline_service_role_arn = self.pipeline_service_role_arn or troposphere.GetAtt(default_pipeline_role, "Arn")
+
+        generic_pipeline = codepipeline.Pipeline(
+            "Pipeline",
+            RoleArn=pipeline_service_role_arn,
+            Stages=[],
+            ArtifactStore=codepipeline.ArtifactStore(
+                Type="S3",
+                Location=self.bucket_name,
+            )
+        )
+
+        if self.bucket_kms_key_arn:
+            encryption_config = codepipeline.EncryptionKey(
+                "ArtifactBucketKmsKey",
+                Id=self.bucket_kms_key_arn,
+                Type='KMS',
+            )
+            generic_pipeline.ArtifactStore.EncryptionKey = encryption_config
+
+        pipeline_output = troposphere.Output(
+            "PipelineName",
+            Description="Code Pipeline",
+            Value=Ref(generic_pipeline),
+        )
+        pipeline_bucket_output = troposphere.Output(
+            "PipelineBucket",
+            Description="Name of the input artifact bucket for the pipeline",
+            Value=self.bucket_name,
+        )
+
+        chain_context.template.add_resource(pipeline_bucket_access_policy)
+
+        if not self.pipeline_service_role_arn:
+            chain_context.template.add_resource(default_pipeline_role)
+
+        chain_context.template.add_resource(generic_pipeline)
+        chain_context.template.add_output(pipeline_output)
+        chain_context.template.add_output(pipeline_bucket_output)
+
+    def get_default_pipeline_role(self):
         # TODO: this can be cleaned up by using a policytype and passing in the pipeline role it should add itself to.
         pipeline_policy = iam.Policy(
             PolicyName="%sPolicy" % self.name,
@@ -148,7 +191,7 @@ def handle(self, chain_context):
                             awacs.aws.Action("lambda", "*")
                         ],
                         Resource=["*"]
-                    )
+                    ),
                 ],
             )
         )
@@ -169,42 +212,7 @@ def handle(self, chain_context):
             ),
             Policies=[pipeline_policy] + self.pipeline_policies
         )
-
-        generic_pipeline = codepipeline.Pipeline(
-            "Pipeline",
-            RoleArn=troposphere.GetAtt(pipeline_service_role, "Arn"),
-            Stages=[],
-            ArtifactStore=codepipeline.ArtifactStore(
-                Type="S3",
-                Location=self.bucket_name,
-            )
-            # TODO: optionally add kms key here
-        )
-
-        if self.bucket_kms_key_arn:
-            encryption_config = codepipeline.EncryptionKey(
-                "ArtifactBucketKmsKey",
-                Id=self.bucket_kms_key_arn,
-                Type='KMS',
-            )
-            generic_pipeline.ArtifactStore.EncryptionKey = encryption_config
-
-        pipeline_output = troposphere.Output(
-            "PipelineName",
-            Description="Code Pipeline",
-            Value=Ref(generic_pipeline),
-        )
-        pipeline_bucket_output = troposphere.Output(
-            "PipelineBucket",
-            Description="Name of the input artifact bucket for the pipeline",
-            Value=self.bucket_name,
-        )
-
-        chain_context.template.add_resource(pipeline_bucket_access_policy)
-        chain_context.template.add_resource(pipeline_service_role)
-        chain_context.template.add_resource(generic_pipeline)
-        chain_context.template.add_output(pipeline_output)
-        chain_context.template.add_output(pipeline_bucket_output)
+        return pipeline_service_role
 
     def get_default_bucket_policy_statements(self, pipeline_bucket):
         bucket_policy_statements = [