From 7e7a00c3b5f3a4579af77d302d97e4c447df8348 Mon Sep 17 00:00:00 2001 From: Brett Swift Date: Wed, 3 Oct 2018 08:09:23 -0600 Subject: [PATCH] Support for injectable cloudformation roles Add validation check for pipeline when code build is added out of order Give codebuild access to codepipeline Revert name of pipeline Add missing package Add user parameters to lambda pipeline step --- cumulus/policies/codebuild.py | 1 + .../steps/dev_tools/cloud_formation_action.py | 5 + cumulus/steps/dev_tools/code_build_action.py | 85 ++++++++------ cumulus/steps/dev_tools/lambda_action.py | 7 +- cumulus/steps/dev_tools/pipeline.py | 109 ++++++++++-------- setup.py | 3 +- tests/stacker_test/run-integration.sh | 2 + .../steps/dev_tools/test_code_build_action.py | 48 ++++++++ tests/unit/steps/test_pipeline.py | 9 +- 9 files changed, 175 insertions(+), 94 deletions(-) create mode 100644 tests/unit/steps/dev_tools/test_code_build_action.py diff --git a/cumulus/policies/codebuild.py b/cumulus/policies/codebuild.py index b1b8b0b..8e24046 100644 --- a/cumulus/policies/codebuild.py +++ b/cumulus/policies/codebuild.py @@ -31,6 +31,7 @@ def get_policy_code_build_general_access(policy_name): awacs.aws.Action("apigateway", "*"), awacs.aws.Action("cloudwatch", "*"), awacs.aws.Action("cloudfront", "*"), + awacs.aws.Action("codepipeline", "*"), awacs.aws.Action("rds", "*"), awacs.aws.Action("dynamodb", "*"), awacs.aws.Action("lambda", "*"), diff --git a/cumulus/steps/dev_tools/cloud_formation_action.py b/cumulus/steps/dev_tools/cloud_formation_action.py index 8ca0493..c5dc75c 100644 --- a/cumulus/steps/dev_tools/cloud_formation_action.py +++ b/cumulus/steps/dev_tools/cloud_formation_action.py @@ -25,6 +25,7 @@ def __init__(self, output_artifact_name=None, cfn_action_role_arn=None, cfn_action_config_role_arn=None, + cfn_param_overrides=None, ): """ :type cfn_action_config_role_arn: [troposphere.iam.Policy] @@ -37,6 +38,7 @@ def __init__(self, :type action_mode: cumulus.types.cloudformation.action_mode.ActionMode The actual CloudFormation action to execute """ step.Step.__init__(self) + self.cfn_param_overrides = cfn_param_overrides self.action_name = action_name self.input_artifact_names = input_artifact_names self.input_template_path = input_template_path @@ -95,6 +97,9 @@ def handle(self, chain_context): if self.cfn_action_role_arn: cloud_formation_action.RoleArn = self.cfn_action_role_arn + if self.cfn_param_overrides: + cloud_formation_action.Configuration['ParameterOverrides'] = self.cfn_param_overrides + stage = cumulus.util.template_query.TemplateQuery.get_pipeline_stage_by_name( template=chain_context.template, stage_name=self.stage_name_to_add, diff --git a/cumulus/steps/dev_tools/code_build_action.py b/cumulus/steps/dev_tools/code_build_action.py index 8b10104..9f52b1f 100644 --- a/cumulus/steps/dev_tools/code_build_action.py +++ b/cumulus/steps/dev_tools/code_build_action.py @@ -1,23 +1,19 @@ import awacs -import troposphere - import awacs.aws +import awacs.ec2 +import awacs.iam import awacs.logs import awacs.s3 -import awacs.iam -import awacs.ec2 import awacs.sts +import troposphere +from troposphere import iam, \ + codebuild, codepipeline, Ref, ec2 import cumulus.policies import cumulus.policies.codebuild import cumulus.types.codebuild.buildaction import cumulus.util.template_query - -from troposphere import iam,\ - codebuild, codepipeline, Ref, ec2 - from cumulus.chain import step - from cumulus.steps.dev_tools import META_PIPELINE_BUCKET_POLICY_REF, \ META_PIPELINE_BUCKET_NAME @@ -26,11 +22,13 @@ class CodeBuildAction(step.Step): def __init__(self, action_name, - input_artifact_name, stage_name_to_add, + input_artifact_name, environment=None, vpc_config=None, - buildspec='buildspec.yml'): + buildspec='buildspec.yml', + role_arn=None, + ): """ :type buildspec: basestring path to buildspec.yml or text containing the buildspec. :type input_artifact_name: basestring The artifact name in the pipeline. Must contain a buildspec.yml @@ -39,6 +37,7 @@ def __init__(self, :type vpc_config.Vpc_Config: Only required if the codebuild step requires access to the VPC """ step.Step.__init__(self) + self.role_arn = role_arn self.buildspec = buildspec self.environment = environment self.input_artifact_name = input_artifact_name @@ -48,34 +47,22 @@ def __init__(self, def handle(self, chain_context): + self.validate(chain_context) + print("Adding action %s Stage." % self.action_name) full_action_name = "%s%s" % (self.stage_name_to_add, self.action_name) policy_name = "%sCodeBuildPolicy" % chain_context.instance_name role_name = "CodeBuildRole%s" % full_action_name - codebuild_role = iam.Role( - role_name, - Path="/", - AssumeRolePolicyDocument=awacs.aws.Policy( - Statement=[ - awacs.aws.Statement( - Effect=awacs.aws.Allow, - Action=[awacs.sts.AssumeRole], - Principal=awacs.aws.Principal( - 'Service', - "codebuild.amazonaws.com" - ) - )] - ), - Policies=[ - cumulus.policies.codebuild.get_policy_code_build_general_access(policy_name) - ], - ManagedPolicyArns=[ - chain_context.metadata[META_PIPELINE_BUCKET_POLICY_REF] - ] + codebuild_role = self.get_default_code_build_role( + chain_context=chain_context, + policy_name=policy_name, + role_name=role_name, ) + codebuild_role_arn = self.role_arn if self.role_arn else troposphere.GetAtt(codebuild_role, 'Arn') + if not self.environment: self.environment = codebuild.Environment( ComputeType='BUILD_GENERAL1_SMALL', @@ -89,7 +76,7 @@ def handle(self, chain_context): project = self.create_project( chain_context=chain_context, - codebuild_role=codebuild_role, + codebuild_role_arn=codebuild_role_arn, codebuild_environment=self.environment, name=full_action_name, ) @@ -117,7 +104,31 @@ def handle(self, chain_context): code_build_action.RunOrder = next_run_order stage.Actions.append(code_build_action) - def create_project(self, chain_context, codebuild_role, codebuild_environment, name): + def get_default_code_build_role(self, chain_context, policy_name, role_name): + codebuild_role = iam.Role( + role_name, + Path="/", + AssumeRolePolicyDocument=awacs.aws.Policy( + Statement=[ + awacs.aws.Statement( + Effect=awacs.aws.Allow, + Action=[awacs.sts.AssumeRole], + Principal=awacs.aws.Principal( + 'Service', + "codebuild.amazonaws.com" + ) + )] + ), + Policies=[ + cumulus.policies.codebuild.get_policy_code_build_general_access(policy_name) + ], + ManagedPolicyArns=[ + chain_context.metadata[META_PIPELINE_BUCKET_POLICY_REF] + ] + ) + return codebuild_role + + def create_project(self, chain_context, codebuild_role_arn, codebuild_environment, name): artifacts = codebuild.Artifacts(Type='CODEPIPELINE') @@ -152,11 +163,10 @@ def create_project(self, chain_context, codebuild_role, codebuild_environment, n project = codebuild.Project( project_name, - DependsOn=codebuild_role, Artifacts=artifacts, Environment=codebuild_environment, Name="%s-%s" % (chain_context.instance_name, project_name), - ServiceRole=troposphere.GetAtt(codebuild_role, 'Arn'), + ServiceRole=codebuild_role_arn, Source=codebuild.Source( "Deploy", Type='CODEPIPELINE', @@ -166,3 +176,8 @@ def create_project(self, chain_context, codebuild_role, codebuild_environment, n ) return project + + def validate(self, chain_context): + if META_PIPELINE_BUCKET_POLICY_REF not in chain_context.metadata: + raise AssertionError("Could not find expected 'META_PIPELINE_BUCKET_POLICY_REF' in metadata. " + "Maybe you added the code build step to the chain before the pipeline step?") diff --git a/cumulus/steps/dev_tools/lambda_action.py b/cumulus/steps/dev_tools/lambda_action.py index 9bab5a9..9e0ab31 100644 --- a/cumulus/steps/dev_tools/lambda_action.py +++ b/cumulus/steps/dev_tools/lambda_action.py @@ -11,7 +11,7 @@ import cumulus.policies import cumulus.policies.codebuild import cumulus.types.codebuild.buildaction -import cumulus.util.tropo +import cumulus.util.template_query from cumulus.chain import step from cumulus.steps.dev_tools import META_PIPELINE_BUCKET_POLICY_REF @@ -74,14 +74,15 @@ def handle(self, chain_context): codepipeline.InputArtifacts(Name=self.input_artifact_name) ], Configuration={ - 'FunctionName': self.function_name + 'FunctionName': self.function_name, + 'UserParameters': self.user_parameters, }, RunOrder="1" ) chain_context.template.add_resource(lambda_role) - stage = cumulus.util.tropo.TemplateQuery.get_pipeline_stage_by_name( + stage = cumulus.util.template_query.TemplateQuery.get_pipeline_stage_by_name( template=chain_context.template, stage_name=self.stage_name_to_add, ) diff --git a/cumulus/steps/dev_tools/pipeline.py b/cumulus/steps/dev_tools/pipeline.py index 64a4c7f..a9d8f74 100644 --- a/cumulus/steps/dev_tools/pipeline.py +++ b/cumulus/steps/dev_tools/pipeline.py @@ -1,28 +1,27 @@ import awacs -import troposphere - -import awacs.iam import awacs.aws -import awacs.sts -import awacs.s3 -import awacs.logs +import awacs.awslambda +import awacs.codecommit import awacs.ec2 import awacs.iam -import awacs.codecommit -import awacs.awslambda - -from cumulus.chain import step -import cumulus.steps.dev_tools - +import awacs.logs +import awacs.s3 +import awacs.sts +import awacs.kms +import troposphere from troposphere import codepipeline, Ref, iam from troposphere.s3 import Bucket, VersioningConfiguration +import cumulus.steps.dev_tools +from cumulus.chain import step + class Pipeline(step.Step): def __init__(self, name, bucket_name, + pipeline_service_role_arn=None, create_bucket=True, pipeline_policies=None, bucket_policy_statements=None, @@ -30,6 +29,8 @@ def __init__(self, ): """ + :type pipeline_service_role_arn: basestring Override the pipeline service role. If you pass this + the pipeline_policies is not used. :type create_bucket: bool if False, will not create the bucket. Will attach policies either way. :type bucket_name: the name of the bucket that will be created suffixed with the chaincontext instance name :type bucket_policy_statements: [awacs.aws.Statement] @@ -40,6 +41,7 @@ def __init__(self, self.name = name self.bucket_name = bucket_name self.create_bucket = create_bucket + self.pipeline_service_role_arn = pipeline_service_role_arn self.bucket_policy_statements = bucket_policy_statements self.pipeline_policies = pipeline_policies or [] self.bucket_kms_key_arn = bucket_kms_key_arn @@ -53,10 +55,9 @@ def handle(self, chain_context): :param chain_context: :return: """ - if self.create_bucket: pipeline_bucket = Bucket( - "PipelineBucket%s" % chain_context.instance_name, + "PipelineBucket%s" % self.name, BucketName=self.bucket_name, VersioningConfiguration=VersioningConfiguration( Status="Enabled" @@ -87,6 +88,47 @@ def handle(self, chain_context): chain_context.metadata[cumulus.steps.dev_tools.META_PIPELINE_BUCKET_POLICY_REF] = Ref( pipeline_bucket_access_policy) + default_pipeline_role = self.get_default_pipeline_role() + pipeline_service_role_arn = self.pipeline_service_role_arn or troposphere.GetAtt(default_pipeline_role, "Arn") + + generic_pipeline = codepipeline.Pipeline( + "Pipeline", + RoleArn=pipeline_service_role_arn, + Stages=[], + ArtifactStore=codepipeline.ArtifactStore( + Type="S3", + Location=self.bucket_name, + ) + ) + + if self.bucket_kms_key_arn: + encryption_config = codepipeline.EncryptionKey( + "ArtifactBucketKmsKey", + Id=self.bucket_kms_key_arn, + Type='KMS', + ) + generic_pipeline.ArtifactStore.EncryptionKey = encryption_config + + pipeline_output = troposphere.Output( + "PipelineName", + Description="Code Pipeline", + Value=Ref(generic_pipeline), + ) + pipeline_bucket_output = troposphere.Output( + "PipelineBucket", + Description="Name of the input artifact bucket for the pipeline", + Value=self.bucket_name, + ) + + if not self.pipeline_service_role_arn: + chain_context.template.add_resource(default_pipeline_role) + + chain_context.template.add_resource(pipeline_bucket_access_policy) + chain_context.template.add_resource(generic_pipeline) + chain_context.template.add_output(pipeline_output) + chain_context.template.add_output(pipeline_bucket_output) + + def get_default_pipeline_role(self): # TODO: this can be cleaned up by using a policytype and passing in the pipeline role it should add itself to. pipeline_policy = iam.Policy( PolicyName="%sPolicy" % self.name, @@ -148,7 +190,7 @@ def handle(self, chain_context): awacs.aws.Action("lambda", "*") ], Resource=["*"] - ) + ), ], ) ) @@ -169,42 +211,7 @@ def handle(self, chain_context): ), Policies=[pipeline_policy] + self.pipeline_policies ) - - generic_pipeline = codepipeline.Pipeline( - "Pipeline", - RoleArn=troposphere.GetAtt(pipeline_service_role, "Arn"), - Stages=[], - ArtifactStore=codepipeline.ArtifactStore( - Type="S3", - Location=self.bucket_name, - ) - # TODO: optionally add kms key here - ) - - if self.bucket_kms_key_arn: - encryption_config = codepipeline.EncryptionKey( - "ArtifactBucketKmsKey", - Id=self.bucket_kms_key_arn, - Type='KMS', - ) - generic_pipeline.ArtifactStore.EncryptionKey = encryption_config - - pipeline_output = troposphere.Output( - "PipelineName", - Description="Code Pipeline", - Value=Ref(generic_pipeline), - ) - pipeline_bucket_output = troposphere.Output( - "PipelineBucket", - Description="Name of the input artifact bucket for the pipeline", - Value=self.bucket_name, - ) - - chain_context.template.add_resource(pipeline_bucket_access_policy) - chain_context.template.add_resource(pipeline_service_role) - chain_context.template.add_resource(generic_pipeline) - chain_context.template.add_output(pipeline_output) - chain_context.template.add_output(pipeline_bucket_output) + return pipeline_service_role def get_default_bucket_policy_statements(self, pipeline_bucket): bucket_policy_statements = [ diff --git a/setup.py b/setup.py index 336e19b..b304421 100644 --- a/setup.py +++ b/setup.py @@ -15,6 +15,7 @@ 'troposphere', 'awacs', 'termcolor', + 'enum34', ] setup_requirements = ['pytest-runner', ] @@ -28,7 +29,7 @@ 'pytest-cov', 'coveralls', 'awscli', - 'mock' + 'mock', ] extras = { diff --git a/tests/stacker_test/run-integration.sh b/tests/stacker_test/run-integration.sh index 8cf548e..37845d1 100755 --- a/tests/stacker_test/run-integration.sh +++ b/tests/stacker_test/run-integration.sh @@ -34,6 +34,8 @@ set +e # turn off error mode, ie don't exit with a failure, let the loop continu end=$((SECONDS+600)) pipeline_result=1 while [ $SECONDS -lt ${end} ]; do +# uncomment to nuke without waiting. Do not check in the break uncommented. +# break; sleep 15 # Get status from each stage in the pipeline pipeline_state=$(aws codepipeline get-pipeline-state --name ${PIPELINE_NAME} | jq -r '.stageStates[] | "\(.stageName) \(.latestExecution.status)"') diff --git a/tests/unit/steps/dev_tools/test_code_build_action.py b/tests/unit/steps/dev_tools/test_code_build_action.py new file mode 100644 index 0000000..a1896a0 --- /dev/null +++ b/tests/unit/steps/dev_tools/test_code_build_action.py @@ -0,0 +1,48 @@ +try: + # python 3 + from unittest.mock import patch # noqa + from unittest.mock import MagicMock +except: # noqa + # python 2 + from mock import patch, MagicMock # noqa + +import unittest + +import troposphere +from troposphere import codepipeline # noqa + +from cumulus.chain import chaincontext +from cumulus.steps.dev_tools import META_PIPELINE_BUCKET_POLICY_REF, META_PIPELINE_BUCKET_NAME + + +class TestCodeBuildAction(unittest.TestCase): + + def setUp(self): + self.context = chaincontext.ChainContext( + template=troposphere.Template(), + instance_name='justtestin' + ) + self.context.metadata[META_PIPELINE_BUCKET_POLICY_REF] = "blah" + self.context.metadata[META_PIPELINE_BUCKET_NAME] = troposphere.Ref("notabucket") + + self.pipeline_name = "ThatPipeline" + self.deploy_stage_name = "DeployIt" + self.source_stage_name = "SourceIt" + + TestCodeBuildAction._add_pipeline_and_stage_to_template(self.context.template, self.pipeline_name, self.deploy_stage_name) + + def tearDown(self): + del self.context + + @staticmethod + def _add_pipeline_and_stage_to_template(template, pipeline_name, deploy_stage_name): + pipeline = template.add_resource(troposphere.codepipeline.Pipeline( + pipeline_name, + Stages=[] + )) + + deploy_stage = template.add_resource(troposphere.codepipeline.Stages( + Name=deploy_stage_name, + Actions=[] + )) + pipeline.properties['Stages'].append(deploy_stage) diff --git a/tests/unit/steps/test_pipeline.py b/tests/unit/steps/test_pipeline.py index 7842f0d..ba7f337 100644 --- a/tests/unit/steps/test_pipeline.py +++ b/tests/unit/steps/test_pipeline.py @@ -49,7 +49,8 @@ def tearDown(self): def test_pipeline_has_two_stages(self): sut = pipeline.Pipeline( - name='test', bucket_name='testbucket' + name='test', + bucket_name='testbucket', ) sut.handle(self.context) t = self.context.template @@ -72,7 +73,7 @@ def test_pipeline_creates_default_bucket(self): def test_pipeline_uses_non_default_bucket(self): sut = pipeline.Pipeline( name='test', - bucket_name='testbucket', + bucket_name='ahhjustbucket', create_bucket=False, ) sut.handle(self.context) @@ -107,7 +108,7 @@ def test_code_build_should_not_add_vpc_config(self): project = action.create_project( chain_context=self.context, - codebuild_role='dummy-role', + codebuild_role_arn='dummy-role', codebuild_environment=self.environment, name='test', ) @@ -130,7 +131,7 @@ def test_code_build_should_add_vpc_config(self): project = action.create_project( chain_context=self.context, - codebuild_role='dummy-role', + codebuild_role_arn='dummy-role', codebuild_environment=self.environment, name='test', )