diff --git a/cumulus/policies/codebuild.py b/cumulus/policies/codebuild.py index b1b8b0b..8e24046 100644 --- a/cumulus/policies/codebuild.py +++ b/cumulus/policies/codebuild.py @@ -31,6 +31,7 @@ def get_policy_code_build_general_access(policy_name): awacs.aws.Action("apigateway", "*"), awacs.aws.Action("cloudwatch", "*"), awacs.aws.Action("cloudfront", "*"), + awacs.aws.Action("codepipeline", "*"), awacs.aws.Action("rds", "*"), awacs.aws.Action("dynamodb", "*"), awacs.aws.Action("lambda", "*"), diff --git a/cumulus/steps/dev_tools/cloud_formation_action.py b/cumulus/steps/dev_tools/cloud_formation_action.py index 8ca0493..c5dc75c 100644 --- a/cumulus/steps/dev_tools/cloud_formation_action.py +++ b/cumulus/steps/dev_tools/cloud_formation_action.py @@ -25,6 +25,7 @@ def __init__(self, output_artifact_name=None, cfn_action_role_arn=None, cfn_action_config_role_arn=None, + cfn_param_overrides=None, ): """ :type cfn_action_config_role_arn: [troposphere.iam.Policy] @@ -37,6 +38,7 @@ def __init__(self, :type action_mode: cumulus.types.cloudformation.action_mode.ActionMode The actual CloudFormation action to execute """ step.Step.__init__(self) + self.cfn_param_overrides = cfn_param_overrides self.action_name = action_name self.input_artifact_names = input_artifact_names self.input_template_path = input_template_path @@ -95,6 +97,9 @@ def handle(self, chain_context): if self.cfn_action_role_arn: cloud_formation_action.RoleArn = self.cfn_action_role_arn + if self.cfn_param_overrides: + cloud_formation_action.Configuration['ParameterOverrides'] = self.cfn_param_overrides + stage = cumulus.util.template_query.TemplateQuery.get_pipeline_stage_by_name( template=chain_context.template, stage_name=self.stage_name_to_add, diff --git a/cumulus/steps/dev_tools/code_build_action.py b/cumulus/steps/dev_tools/code_build_action.py index 8b10104..9f52b1f 100644 --- a/cumulus/steps/dev_tools/code_build_action.py +++ b/cumulus/steps/dev_tools/code_build_action.py @@ -1,23 +1,19 @@ import awacs -import troposphere - import awacs.aws +import awacs.ec2 +import awacs.iam import awacs.logs import awacs.s3 -import awacs.iam -import awacs.ec2 import awacs.sts +import troposphere +from troposphere import iam, \ + codebuild, codepipeline, Ref, ec2 import cumulus.policies import cumulus.policies.codebuild import cumulus.types.codebuild.buildaction import cumulus.util.template_query - -from troposphere import iam,\ - codebuild, codepipeline, Ref, ec2 - from cumulus.chain import step - from cumulus.steps.dev_tools import META_PIPELINE_BUCKET_POLICY_REF, \ META_PIPELINE_BUCKET_NAME @@ -26,11 +22,13 @@ class CodeBuildAction(step.Step): def __init__(self, action_name, - input_artifact_name, stage_name_to_add, + input_artifact_name, environment=None, vpc_config=None, - buildspec='buildspec.yml'): + buildspec='buildspec.yml', + role_arn=None, + ): """ :type buildspec: basestring path to buildspec.yml or text containing the buildspec. :type input_artifact_name: basestring The artifact name in the pipeline. Must contain a buildspec.yml @@ -39,6 +37,7 @@ def __init__(self, :type vpc_config.Vpc_Config: Only required if the codebuild step requires access to the VPC """ step.Step.__init__(self) + self.role_arn = role_arn self.buildspec = buildspec self.environment = environment self.input_artifact_name = input_artifact_name @@ -48,34 +47,22 @@ def __init__(self, def handle(self, chain_context): + self.validate(chain_context) + print("Adding action %s Stage." % self.action_name) full_action_name = "%s%s" % (self.stage_name_to_add, self.action_name) policy_name = "%sCodeBuildPolicy" % chain_context.instance_name role_name = "CodeBuildRole%s" % full_action_name - codebuild_role = iam.Role( - role_name, - Path="/", - AssumeRolePolicyDocument=awacs.aws.Policy( - Statement=[ - awacs.aws.Statement( - Effect=awacs.aws.Allow, - Action=[awacs.sts.AssumeRole], - Principal=awacs.aws.Principal( - 'Service', - "codebuild.amazonaws.com" - ) - )] - ), - Policies=[ - cumulus.policies.codebuild.get_policy_code_build_general_access(policy_name) - ], - ManagedPolicyArns=[ - chain_context.metadata[META_PIPELINE_BUCKET_POLICY_REF] - ] + codebuild_role = self.get_default_code_build_role( + chain_context=chain_context, + policy_name=policy_name, + role_name=role_name, ) + codebuild_role_arn = self.role_arn if self.role_arn else troposphere.GetAtt(codebuild_role, 'Arn') + if not self.environment: self.environment = codebuild.Environment( ComputeType='BUILD_GENERAL1_SMALL', @@ -89,7 +76,7 @@ def handle(self, chain_context): project = self.create_project( chain_context=chain_context, - codebuild_role=codebuild_role, + codebuild_role_arn=codebuild_role_arn, codebuild_environment=self.environment, name=full_action_name, ) @@ -117,7 +104,31 @@ def handle(self, chain_context): code_build_action.RunOrder = next_run_order stage.Actions.append(code_build_action) - def create_project(self, chain_context, codebuild_role, codebuild_environment, name): + def get_default_code_build_role(self, chain_context, policy_name, role_name): + codebuild_role = iam.Role( + role_name, + Path="/", + AssumeRolePolicyDocument=awacs.aws.Policy( + Statement=[ + awacs.aws.Statement( + Effect=awacs.aws.Allow, + Action=[awacs.sts.AssumeRole], + Principal=awacs.aws.Principal( + 'Service', + "codebuild.amazonaws.com" + ) + )] + ), + Policies=[ + cumulus.policies.codebuild.get_policy_code_build_general_access(policy_name) + ], + ManagedPolicyArns=[ + chain_context.metadata[META_PIPELINE_BUCKET_POLICY_REF] + ] + ) + return codebuild_role + + def create_project(self, chain_context, codebuild_role_arn, codebuild_environment, name): artifacts = codebuild.Artifacts(Type='CODEPIPELINE') @@ -152,11 +163,10 @@ def create_project(self, chain_context, codebuild_role, codebuild_environment, n project = codebuild.Project( project_name, - DependsOn=codebuild_role, Artifacts=artifacts, Environment=codebuild_environment, Name="%s-%s" % (chain_context.instance_name, project_name), - ServiceRole=troposphere.GetAtt(codebuild_role, 'Arn'), + ServiceRole=codebuild_role_arn, Source=codebuild.Source( "Deploy", Type='CODEPIPELINE', @@ -166,3 +176,8 @@ def create_project(self, chain_context, codebuild_role, codebuild_environment, n ) return project + + def validate(self, chain_context): + if META_PIPELINE_BUCKET_POLICY_REF not in chain_context.metadata: + raise AssertionError("Could not find expected 'META_PIPELINE_BUCKET_POLICY_REF' in metadata. " + "Maybe you added the code build step to the chain before the pipeline step?") diff --git a/cumulus/steps/dev_tools/lambda_action.py b/cumulus/steps/dev_tools/lambda_action.py index 9bab5a9..9e0ab31 100644 --- a/cumulus/steps/dev_tools/lambda_action.py +++ b/cumulus/steps/dev_tools/lambda_action.py @@ -11,7 +11,7 @@ import cumulus.policies import cumulus.policies.codebuild import cumulus.types.codebuild.buildaction -import cumulus.util.tropo +import cumulus.util.template_query from cumulus.chain import step from cumulus.steps.dev_tools import META_PIPELINE_BUCKET_POLICY_REF @@ -74,14 +74,15 @@ def handle(self, chain_context): codepipeline.InputArtifacts(Name=self.input_artifact_name) ], Configuration={ - 'FunctionName': self.function_name + 'FunctionName': self.function_name, + 'UserParameters': self.user_parameters, }, RunOrder="1" ) chain_context.template.add_resource(lambda_role) - stage = cumulus.util.tropo.TemplateQuery.get_pipeline_stage_by_name( + stage = cumulus.util.template_query.TemplateQuery.get_pipeline_stage_by_name( template=chain_context.template, stage_name=self.stage_name_to_add, ) diff --git a/cumulus/steps/dev_tools/pipeline.py b/cumulus/steps/dev_tools/pipeline.py index 64a4c7f..a9d8f74 100644 --- a/cumulus/steps/dev_tools/pipeline.py +++ b/cumulus/steps/dev_tools/pipeline.py @@ -1,28 +1,27 @@ import awacs -import troposphere - -import awacs.iam import awacs.aws -import awacs.sts -import awacs.s3 -import awacs.logs +import awacs.awslambda +import awacs.codecommit import awacs.ec2 import awacs.iam -import awacs.codecommit -import awacs.awslambda - -from cumulus.chain import step -import cumulus.steps.dev_tools - +import awacs.logs +import awacs.s3 +import awacs.sts +import awacs.kms +import troposphere from troposphere import codepipeline, Ref, iam from troposphere.s3 import Bucket, VersioningConfiguration +import cumulus.steps.dev_tools +from cumulus.chain import step + class Pipeline(step.Step): def __init__(self, name, bucket_name, + pipeline_service_role_arn=None, create_bucket=True, pipeline_policies=None, bucket_policy_statements=None, @@ -30,6 +29,8 @@ def __init__(self, ): """ + :type pipeline_service_role_arn: basestring Override the pipeline service role. If you pass this + the pipeline_policies is not used. :type create_bucket: bool if False, will not create the bucket. Will attach policies either way. :type bucket_name: the name of the bucket that will be created suffixed with the chaincontext instance name :type bucket_policy_statements: [awacs.aws.Statement] @@ -40,6 +41,7 @@ def __init__(self, self.name = name self.bucket_name = bucket_name self.create_bucket = create_bucket + self.pipeline_service_role_arn = pipeline_service_role_arn self.bucket_policy_statements = bucket_policy_statements self.pipeline_policies = pipeline_policies or [] self.bucket_kms_key_arn = bucket_kms_key_arn @@ -53,10 +55,9 @@ def handle(self, chain_context): :param chain_context: :return: """ - if self.create_bucket: pipeline_bucket = Bucket( - "PipelineBucket%s" % chain_context.instance_name, + "PipelineBucket%s" % self.name, BucketName=self.bucket_name, VersioningConfiguration=VersioningConfiguration( Status="Enabled" @@ -87,6 +88,47 @@ def handle(self, chain_context): chain_context.metadata[cumulus.steps.dev_tools.META_PIPELINE_BUCKET_POLICY_REF] = Ref( pipeline_bucket_access_policy) + default_pipeline_role = self.get_default_pipeline_role() + pipeline_service_role_arn = self.pipeline_service_role_arn or troposphere.GetAtt(default_pipeline_role, "Arn") + + generic_pipeline = codepipeline.Pipeline( + "Pipeline", + RoleArn=pipeline_service_role_arn, + Stages=[], + ArtifactStore=codepipeline.ArtifactStore( + Type="S3", + Location=self.bucket_name, + ) + ) + + if self.bucket_kms_key_arn: + encryption_config = codepipeline.EncryptionKey( + "ArtifactBucketKmsKey", + Id=self.bucket_kms_key_arn, + Type='KMS', + ) + generic_pipeline.ArtifactStore.EncryptionKey = encryption_config + + pipeline_output = troposphere.Output( + "PipelineName", + Description="Code Pipeline", + Value=Ref(generic_pipeline), + ) + pipeline_bucket_output = troposphere.Output( + "PipelineBucket", + Description="Name of the input artifact bucket for the pipeline", + Value=self.bucket_name, + ) + + if not self.pipeline_service_role_arn: + chain_context.template.add_resource(default_pipeline_role) + + chain_context.template.add_resource(pipeline_bucket_access_policy) + chain_context.template.add_resource(generic_pipeline) + chain_context.template.add_output(pipeline_output) + chain_context.template.add_output(pipeline_bucket_output) + + def get_default_pipeline_role(self): # TODO: this can be cleaned up by using a policytype and passing in the pipeline role it should add itself to. pipeline_policy = iam.Policy( PolicyName="%sPolicy" % self.name, @@ -148,7 +190,7 @@ def handle(self, chain_context): awacs.aws.Action("lambda", "*") ], Resource=["*"] - ) + ), ], ) ) @@ -169,42 +211,7 @@ def handle(self, chain_context): ), Policies=[pipeline_policy] + self.pipeline_policies ) - - generic_pipeline = codepipeline.Pipeline( - "Pipeline", - RoleArn=troposphere.GetAtt(pipeline_service_role, "Arn"), - Stages=[], - ArtifactStore=codepipeline.ArtifactStore( - Type="S3", - Location=self.bucket_name, - ) - # TODO: optionally add kms key here - ) - - if self.bucket_kms_key_arn: - encryption_config = codepipeline.EncryptionKey( - "ArtifactBucketKmsKey", - Id=self.bucket_kms_key_arn, - Type='KMS', - ) - generic_pipeline.ArtifactStore.EncryptionKey = encryption_config - - pipeline_output = troposphere.Output( - "PipelineName", - Description="Code Pipeline", - Value=Ref(generic_pipeline), - ) - pipeline_bucket_output = troposphere.Output( - "PipelineBucket", - Description="Name of the input artifact bucket for the pipeline", - Value=self.bucket_name, - ) - - chain_context.template.add_resource(pipeline_bucket_access_policy) - chain_context.template.add_resource(pipeline_service_role) - chain_context.template.add_resource(generic_pipeline) - chain_context.template.add_output(pipeline_output) - chain_context.template.add_output(pipeline_bucket_output) + return pipeline_service_role def get_default_bucket_policy_statements(self, pipeline_bucket): bucket_policy_statements = [ diff --git a/setup.py b/setup.py index 336e19b..b304421 100644 --- a/setup.py +++ b/setup.py @@ -15,6 +15,7 @@ 'troposphere', 'awacs', 'termcolor', + 'enum34', ] setup_requirements = ['pytest-runner', ] @@ -28,7 +29,7 @@ 'pytest-cov', 'coveralls', 'awscli', - 'mock' + 'mock', ] extras = { diff --git a/tests/stacker_test/run-integration.sh b/tests/stacker_test/run-integration.sh index 8cf548e..37845d1 100755 --- a/tests/stacker_test/run-integration.sh +++ b/tests/stacker_test/run-integration.sh @@ -34,6 +34,8 @@ set +e # turn off error mode, ie don't exit with a failure, let the loop continu end=$((SECONDS+600)) pipeline_result=1 while [ $SECONDS -lt ${end} ]; do +# uncomment to nuke without waiting. Do not check in the break uncommented. +# break; sleep 15 # Get status from each stage in the pipeline pipeline_state=$(aws codepipeline get-pipeline-state --name ${PIPELINE_NAME} | jq -r '.stageStates[] | "\(.stageName) \(.latestExecution.status)"') diff --git a/tests/unit/steps/dev_tools/test_code_build_action.py b/tests/unit/steps/dev_tools/test_code_build_action.py new file mode 100644 index 0000000..a1896a0 --- /dev/null +++ b/tests/unit/steps/dev_tools/test_code_build_action.py @@ -0,0 +1,48 @@ +try: + # python 3 + from unittest.mock import patch # noqa + from unittest.mock import MagicMock +except: # noqa + # python 2 + from mock import patch, MagicMock # noqa + +import unittest + +import troposphere +from troposphere import codepipeline # noqa + +from cumulus.chain import chaincontext +from cumulus.steps.dev_tools import META_PIPELINE_BUCKET_POLICY_REF, META_PIPELINE_BUCKET_NAME + + +class TestCodeBuildAction(unittest.TestCase): + + def setUp(self): + self.context = chaincontext.ChainContext( + template=troposphere.Template(), + instance_name='justtestin' + ) + self.context.metadata[META_PIPELINE_BUCKET_POLICY_REF] = "blah" + self.context.metadata[META_PIPELINE_BUCKET_NAME] = troposphere.Ref("notabucket") + + self.pipeline_name = "ThatPipeline" + self.deploy_stage_name = "DeployIt" + self.source_stage_name = "SourceIt" + + TestCodeBuildAction._add_pipeline_and_stage_to_template(self.context.template, self.pipeline_name, self.deploy_stage_name) + + def tearDown(self): + del self.context + + @staticmethod + def _add_pipeline_and_stage_to_template(template, pipeline_name, deploy_stage_name): + pipeline = template.add_resource(troposphere.codepipeline.Pipeline( + pipeline_name, + Stages=[] + )) + + deploy_stage = template.add_resource(troposphere.codepipeline.Stages( + Name=deploy_stage_name, + Actions=[] + )) + pipeline.properties['Stages'].append(deploy_stage) diff --git a/tests/unit/steps/test_pipeline.py b/tests/unit/steps/test_pipeline.py index 7842f0d..ba7f337 100644 --- a/tests/unit/steps/test_pipeline.py +++ b/tests/unit/steps/test_pipeline.py @@ -49,7 +49,8 @@ def tearDown(self): def test_pipeline_has_two_stages(self): sut = pipeline.Pipeline( - name='test', bucket_name='testbucket' + name='test', + bucket_name='testbucket', ) sut.handle(self.context) t = self.context.template @@ -72,7 +73,7 @@ def test_pipeline_creates_default_bucket(self): def test_pipeline_uses_non_default_bucket(self): sut = pipeline.Pipeline( name='test', - bucket_name='testbucket', + bucket_name='ahhjustbucket', create_bucket=False, ) sut.handle(self.context) @@ -107,7 +108,7 @@ def test_code_build_should_not_add_vpc_config(self): project = action.create_project( chain_context=self.context, - codebuild_role='dummy-role', + codebuild_role_arn='dummy-role', codebuild_environment=self.environment, name='test', ) @@ -130,7 +131,7 @@ def test_code_build_should_add_vpc_config(self): project = action.create_project( chain_context=self.context, - codebuild_role='dummy-role', + codebuild_role_arn='dummy-role', codebuild_environment=self.environment, name='test', )