Skip to content

Commit

Permalink
feat(aws-codebuild): run script from asset (#677)
Browse files Browse the repository at this point in the history 
Add a feature to CodeBuild projects to allow reading the build script
from an asset.

This is useful to decouple (potentially complex) build scripts from the
artifacts being processed, or for simply running a long-running script
in a serverless fashion.

Fixes #639.

ALSO IN THIS COMMIT

* Add support for NO_SOURCE, which we didn't use to have.
* Fix the use of a caching bucket, which wasn't working properly before.
  • Loading branch information
@rix0rrr
rix0rrr authored Sep 7, 2018
1 parent f70394c commit bd97fd2
Show file tree
Hide file tree
Showing 9 changed files with 718 additions and 28 deletions.
178 changes: 154 additions & 24 deletions packages/@aws-cdk/aws-codebuild/lib/project.ts
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import assets = require('@aws-cdk/assets');
import cloudwatch = require('@aws-cdk/aws-cloudwatch');
import codepipeline = require('@aws-cdk/aws-codepipeline-api');
import events = require('@aws-cdk/aws-events');
Expand All @@ -8,9 +9,11 @@ import cdk = require('@aws-cdk/cdk');
import { BuildArtifacts, CodePipelineBuildArtifacts, NoBuildArtifacts } from './artifacts';
import { cloudformation, ProjectArn, ProjectName } from './codebuild.generated';
import { CommonPipelineBuildActionProps, PipelineBuildAction } from './pipeline-actions';
import { BuildSource } from './source';
import { BuildSource, NoSource } from './source';

const CODEPIPELINE_TYPE = 'CODEPIPELINE';
const S3_BUCKET_ENV = 'SCRIPT_S3_BUCKET';
const S3_KEY_ENV = 'SCRIPT_S3_KEY';

/**
* Properties of a reference to a CodeBuild Project.
Expand Down Expand Up @@ -322,6 +325,26 @@ export interface CommonProjectProps {
*/
buildSpec?: any;

/**
* Run a script from an asset as build script
*
* If supplied together with buildSpec, the asset script will be run
* _after_ the existing commands in buildspec.
*
* This feature can also be used without a source, to simply run an
* arbitrary script in a serverless way.
*
* @default No asset build script
*/
buildScriptAsset?: assets.Asset;

/**
* The script in the asset to run.
*
* @default build.sh
*/
buildScriptAssetEntrypoint?: string;

/**
* Service Role to assume while running the build.
* If not specified, a role will be created.
Expand Down Expand Up @@ -378,8 +401,10 @@ export interface CommonProjectProps {
export interface ProjectProps extends CommonProjectProps {
/**
* The source of the build.
*
* @default NoSource
*/
source: BuildSource;
source?: BuildSource;

/**
* Defines where build artifacts will be stored.
Expand Down Expand Up @@ -409,49 +434,70 @@ export class Project extends ProjectRef {
*/
public readonly projectName: ProjectName;

private readonly source: BuildSource;
private readonly buildImage: IBuildImage;

constructor(parent: cdk.Construct, name: string, props: ProjectProps) {
super(parent, name);

if (props.buildScriptAssetEntrypoint && !props.buildScriptAsset) {
throw new Error('To use buildScriptAssetEntrypoint, supply buildScriptAsset as well.');
}

this.role = props.role || new iam.Role(this, 'Role', {
assumedBy: new cdk.ServicePrincipal('codebuild.amazonaws.com')
});

const environment = this.renderEnvironment(props.environment, props.environmentVariables);

let cache: cloudformation.ProjectResource.ProjectCacheProperty | undefined;
if (props.cacheBucket) {
const cacheDir = props.cacheDir != null ? props.cacheDir : '';
const cacheDir = props.cacheDir != null ? props.cacheDir : new cdk.AwsNoValue();
cache = {
type: 'S3',
location: props.cacheBucket.arnForObjects(cacheDir)
location: new cdk.FnJoin('/', [props.cacheBucket.bucketName, cacheDir]),
};

props.cacheBucket.grantReadWrite(this.role);
}

this.buildImage = (props.environment && props.environment.buildImage) || LinuxBuildImage.UBUNTU_14_04_BASE;

// let source "bind" to the project. this usually involves granting permissions
// for the code build role to interact with the source.
const source = props.source;
source.bind(this);
this.source = props.source || new NoSource();
this.source.bind(this);

const artifacts = this.parseArtifacts(props);
artifacts.bind(this);

const sourceJson = source.toSourceJSON();
if (typeof props.buildSpec === 'string') {
sourceJson.buildSpec = props.buildSpec;
} else {
sourceJson.buildSpec = JSON.stringify(props.buildSpec);
// Inject download commands for asset if requested
const environmentVariables = props.environmentVariables || {};
const buildSpec = props.buildSpec || {};

if (props.buildScriptAsset) {
environmentVariables[S3_BUCKET_ENV] = { value: props.buildScriptAsset.s3BucketName };
environmentVariables[S3_KEY_ENV] = { value: props.buildScriptAsset.s3ObjectKey };
extendBuildSpec(buildSpec, this.buildImage.runScriptBuildspec(props.buildScriptAssetEntrypoint || 'build.sh'));
props.buildScriptAsset.grantRead(this.role);
}

this.validateCodePipelineSettings(source, artifacts);
// Render the source and add in the buildspec
const sourceJson = this.source.toSourceJSON();
if (typeof buildSpec === 'string') {
sourceJson.buildSpec = buildSpec; // Filename to buildspec file
} else if (Object.keys(buildSpec).length > 0) {
// We have to pretty-print the buildspec, otherwise
// CodeBuild will not recognize it as an inline buildspec.
sourceJson.buildSpec = JSON.stringify(buildSpec, undefined, 2); // Literal buildspec
}

this.validateCodePipelineSettings(artifacts);

const resource = new cloudformation.ProjectResource(this, 'Resource', {
description: props.description,
source: sourceJson,
artifacts: artifacts.toArtifactsJSON(),
serviceRole: this.role.roleArn,
environment,
environment: this.renderEnvironment(props.environment, environmentVariables),
encryptionKey: props.encryptionKey && props.encryptionKey.keyArn,
badgeEnabled: props.badge,
cache,
Expand Down Expand Up @@ -513,17 +559,16 @@ export class Project extends ProjectRef {

const hasEnvironmentVars = Object.keys(vars).length > 0;

const buildImage = env.buildImage || LinuxBuildImage.UBUNTU_14_04_BASE;
const errors = buildImage.validate(env);
const errors = this.buildImage.validate(env);
if (errors.length > 0) {
throw new Error("Invalid CodeBuild environment: " + errors.join('\n'));
}

return {
type: buildImage.type,
image: buildImage.imageId,
type: this.buildImage.type,
image: this.buildImage.imageId,
privilegedMode: env.priviledged || false,
computeType: env.computeType || buildImage.defaultComputeType,
computeType: env.computeType || this.buildImage.defaultComputeType,
environmentVariables: !hasEnvironmentVars ? undefined : Object.keys(vars).map(name => ({
name,
type: vars[name].type || BuildEnvironmentVariableType.PlainText,
Expand All @@ -536,15 +581,15 @@ export class Project extends ProjectRef {
if (props.artifacts) {
return props.artifacts;
}
if (props.source.toSourceJSON().type === CODEPIPELINE_TYPE) {
if (this.source.toSourceJSON().type === CODEPIPELINE_TYPE) {
return new CodePipelineBuildArtifacts();
} else {
return new NoBuildArtifacts();
}
}

private validateCodePipelineSettings(source: BuildSource, artifacts: BuildArtifacts) {
const sourceType = source.toSourceJSON().type;
private validateCodePipelineSettings(artifacts: BuildArtifacts) {
const sourceType = this.source.toSourceJSON().type;
const artifactsType = artifacts.toArtifactsJSON().type;

if ((sourceType === CODEPIPELINE_TYPE || artifactsType === CODEPIPELINE_TYPE) &&
Expand Down Expand Up @@ -627,6 +672,11 @@ export interface IBuildImage {
* @param buildEnvironment the current build environment
*/
validate(buildEnvironment: BuildEnvironment): string[];

/**
* Make a buildspec to run the indicated script
*/
runScriptBuildspec(entrypoint: string): any;
}

/**
Expand Down Expand Up @@ -671,6 +721,34 @@ export class LinuxBuildImage implements IBuildImage {
public validate(_: BuildEnvironment): string[] {
return [];
}

public runScriptBuildspec(entrypoint: string): any {
return {
version: '0.2',
phases: {
pre_build: {
commands: [
// Better echo the location here; if this fails, the error message only contains
// the unexpanded variables by default. It might fail if you're running an old
// definition of the CodeBuild project--the permissions will have been changed
// to only allow downloading the very latest version.
`echo "Downloading scripts from s3://\${${S3_BUCKET_ENV}}/\${${S3_KEY_ENV}}"`,
`aws s3 cp s3://\${${S3_BUCKET_ENV}}/\${${S3_KEY_ENV}} /tmp`,
`mkdir -p /tmp/scriptdir`,
`unzip /tmp/$(basename \$${S3_KEY_ENV}) -d /tmp/scriptdir`,
]
},
build: {
commands: [
'export SCRIPT_DIR=/tmp/scriptdir',
`echo "Running ${entrypoint}"`,
`chmod +x /tmp/scriptdir/${entrypoint}`,
`/tmp/scriptdir/${entrypoint}`,
]
}
}
};
}
}

/**
Expand All @@ -697,6 +775,31 @@ export class WindowsBuildImage implements IBuildImage {
}
return ret;
}

public runScriptBuildspec(entrypoint: string): any {
return {
version: '0.2',
phases: {
pre_build: {
// Would love to do downloading here and executing in the next step,
// but I don't know how to propagate the value of $TEMPDIR.
//
// Punting for someone who knows PowerShell well enough.
commands: []
},
build: {
commands: [
`Set-Variable -Name TEMPDIR -Value (New-TemporaryFile).DirectoryName`,
`aws s3 cp s3://$env:${S3_BUCKET_ENV}/$env:${S3_KEY_ENV} $TEMPDIR\\scripts.zip`,
'New-Item -ItemType Directory -Path $TEMPDIR\\scriptdir',
'Expand-Archive -Path $TEMPDIR/scripts.zip -DestinationPath $TEMPDIR\\scriptdir',
'$env:SCRIPT_DIR = "$TEMPDIR\\scriptdir"',
`& $TEMPDIR\\scriptdir\\${entrypoint}`
]
}
}
};
}
}

export interface BuildEnvironmentVariable {
Expand All @@ -723,4 +826,31 @@ export enum BuildEnvironmentVariableType {
* An environment variable stored in Systems Manager Parameter Store.
*/
ParameterStore = 'PARAMETER_STORE'
}
}

/**
* Extend buildSpec phases with the contents of another one
*/
function extendBuildSpec(buildSpec: any, extend: any) {
if (typeof buildSpec === 'string') {
throw new Error('Cannot extend buildspec that is given as a string. Pass the buildspec as a structure instead.');
}
if (buildSpec.version === '0.1') {
throw new Error('Cannot extend buildspec at version "0.1". Set the version to "0.2" or higher instead.');
}
if (buildSpec.version === undefined) {
buildSpec.version = extend.version;
}

if (!buildSpec.phases) {
buildSpec.phases = {};
}

for (const phaseName of Object.keys(extend.phases)) {
if (!(phaseName in buildSpec.phases)) { buildSpec.phases[phaseName] = {}; }
const phase = buildSpec.phases[phaseName];

if (!(phase.commands)) { phase.commands = []; }
phase.commands.push(...extend.phases[phaseName].commands);
}
}
15 changes: 14 additions & 1 deletion packages/@aws-cdk/aws-codebuild/lib/source.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,18 @@ export abstract class BuildSource {
public abstract toSourceJSON(): cloudformation.ProjectResource.SourceProperty;
}

export class NoSource extends BuildSource {
constructor() {
super();
}

public toSourceJSON(): cloudformation.ProjectResource.SourceProperty {
return {
type: SourceType.None,
};
}
}

/**
* CodeCommit Source definition for a CodeBuild project
*/
Expand Down Expand Up @@ -135,10 +147,11 @@ export class S3BucketSource extends BuildSource {
* Source types for CodeBuild Project
*/
export enum SourceType {
None = 'NO_SOURCE',
CodeCommit = 'CODECOMMIT',
CodePipeline = 'CODEPIPELINE',
GitHub = 'GITHUB',
GitHubEnterPrise = 'GITHUB_ENTERPRISE',
BitBucket = 'BITBUCKET',
S3 = 'S3'
S3 = 'S3',
}
1 change: 1 addition & 0 deletions packages/@aws-cdk/aws-codebuild/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@
"pkglint": "^0.8.2"
},
"dependencies": {
"@aws-cdk/assets": "^0.8.2",
"@aws-cdk/aws-cloudwatch": "^0.8.2",
"@aws-cdk/aws-codecommit": "^0.8.2",
"@aws-cdk/aws-codepipeline-api": "^0.8.2",
Expand Down
Loading

0 comments on commit bd97fd2

Please sign in to comment.