Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(core): democratize synthesis and introduce artifacts #1889

Merged
merged 28 commits into from
Mar 1, 2019
Merged
Show file tree
Hide file tree
Changes from 19 commits
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
af3c695
feat(core): democratize synthesis
Feb 26, 2019
f00f5af
move "prepare" before "validate"
Feb 26, 2019
c23f0c6
Use stack's uniqueId as artifact name instead of just id
Feb 26, 2019
25a96f9
Move manifest creation to `run` and don't use synthesizeStacks
Feb 26, 2019
1920c9b
fix visibility modifiers of Stack methods
Feb 26, 2019
89827fd
add "mkdir", "readdir", "exists" and "list" to synth session and add …
Feb 26, 2019
7a35a98
read synthesized stacks in a uniform way
Feb 26, 2019
82622e7
fix applet tests
Feb 26, 2019
6aea527
identify stacks by id and not uniqueid (for now)
Feb 26, 2019
ab0daa3
change findAll API to PreOrder/PostOrder instead of Depth/Breadth
Feb 26, 2019
64bb0d6
Merge remote-tracking branch 'origin/master' into benisrae/construct-…
Feb 26, 2019
3912e28
remove unused bundled deps
Feb 26, 2019
297b66b
fix assert test
Feb 26, 2019
dca97c6
start establishing the concept of "artifacts"
Feb 27, 2019
b4b3847
only write "stacks" in legacy cdk.out
Feb 27, 2019
87165f5
Merge remote-tracking branch 'origin/master' into benisrae/construct-…
Feb 27, 2019
362faa5
ISynthesizable
Feb 27, 2019
d8a7e95
Rename "finalize" to "close", because Java
Feb 27, 2019
9536a17
allow disabling legacy manifest in new versions
Feb 27, 2019
38f402c
comment changes
Feb 28, 2019
60528dd
Merge remote-tracking branch 'origin/master' into benisrae/construct-…
Feb 28, 2019
b526fac
update package-lock.json
Feb 28, 2019
2b3c595
fix how "disable version reporting" is evaluated
Feb 28, 2019
0e12e2a
interm
Feb 28, 2019
5f27239
stack.setParameterValue can be used to assign values to CFN parameter…
Feb 28, 2019
b3bdd3b
build.json and addBuildStep
Feb 28, 2019
f923ad5
revert asset.ts change
Feb 28, 2019
d2d29ed
Merge remote-tracking branch 'origin/master' into benisrae/construct-…
Feb 28, 2019
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
473 changes: 473 additions & 0 deletions design/cloud-assembly.md

Large diffs are not rendered by default.

5 changes: 3 additions & 2 deletions packages/@aws-cdk/applet-js/test/test.applets.ts
Original file line number Diff line number Diff line change
Expand Up @@ -80,16 +80,17 @@ function synthesizeApplet(yamlFile: string, direct = false) {

const command = direct ? yamlFile : 'cdk-applet-js';
const args = direct ? [] : [yamlFile];
const outdir = fs.mkdtempSync(path.join(os.tmpdir(), 'cdk-applet-tests'));

child_process.execFileSync(command, args, {
env: {
...process.env,
CDK_OUTDIR: os.tmpdir(),
CDK_OUTDIR: outdir,
PATH: 'bin:' + process.env.PATH
}
});

return JSON.parse(fs.readFileSync(path.join(os.tmpdir(), 'cdk.out'), { encoding: 'utf-8' }));
return JSON.parse(fs.readFileSync(path.join(outdir, 'cdk.out'), { encoding: 'utf-8' }));
}

function getStack(stackName: string, allStacks: any) {
Expand Down
2 changes: 1 addition & 1 deletion packages/@aws-cdk/assert/lib/expect.ts
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ function isStackClassInstance(x: api.SynthesizedStack | cdk.Stack): x is cdk.Sta

function collectStackMetadata(root: cdk.ConstructNode): api.StackMetadata {
const result: api.StackMetadata = {};
for (const construct of root.findAll(cdk.ConstructOrder.DepthFirst)) {
for (const construct of root.findAll(cdk.ConstructOrder.PreOrder)) {
const path = `/${root.id}/${construct.node.path}`;
for (const entry of construct.node.metadata) {
result[path] = result[path] || [];
Expand Down
264 changes: 71 additions & 193 deletions packages/@aws-cdk/cdk/lib/app.ts
Original file line number Diff line number Diff line change
@@ -1,245 +1,123 @@
import cxapi = require('@aws-cdk/cx-api');
import fs = require('fs');
import path = require('path');
import { Stack } from './cloudformation/stack';
import { IConstruct, MetadataEntry, PATH_SEP, Root } from './core/construct';
import { ConstructOrder, Root } from './core/construct';
import { FileSystemStore, InMemoryStore, ISynthesisSession, SynthesisSession } from './synthesis';

/**
* Represents a CDK program.
*/
export class App extends Root {
private prepared = false;
private _session?: ISynthesisSession;
private readonly legacyManifest: boolean;
private readonly runtimeInformation: boolean;

/**
* Initializes a CDK application.
* @param request Optional toolkit request (e.g. for tests)
*/
constructor() {
constructor(context?: { [key: string]: string }) {
super();
this.loadContext();
}

private get stacks() {
const out: { [name: string]: Stack } = { };
for (const child of this.node.children) {
if (!Stack.isStack(child)) {
throw new Error(`The child ${child.toString()} of App must be a Stack`);
}
this.loadContext(context);

out[child.node.id] = child as Stack;
}
return out;
// both are reverse logic
this.legacyManifest = this.node.getContext(cxapi.DISABLE_LEGACY_MANIFEST_CONTEXT) === 'true' ? false : true;
this.runtimeInformation = this.node.getContext(cxapi.DISABLE_RUNTIME_INFO_CONTEXT) === 'true' ? false : true;
}

/**
* Runs the program. Output is written to output directory as specified in the request.
*/
public run(): void {
public run(): ISynthesisSession {
// this app has already been executed, no-op for you
eladb marked this conversation as resolved.
Show resolved Hide resolved
if (this._session) {
return this._session;
}

const outdir = process.env[cxapi.OUTDIR_ENV];
if (!outdir) {
process.stderr.write(`ERROR: The environment variable "${cxapi.OUTDIR_ENV}" is not defined\n`);
process.stderr.write('AWS CDK Toolkit (>= 0.11.0) is required in order to interact with this program.\n');
process.exit(1);
return;
let store;
if (outdir) {
store = new FileSystemStore({ outdir });
} else {
store = new InMemoryStore();
}

const session = this._session = new SynthesisSession({
store,
legacyManifest: this.legacyManifest,
runtimeInformation: this.runtimeInformation
});

// the three holy phases of synthesis: prepare, validate and synthesize

// prepare
this.node.prepareTree();

// validate
const errors = this.node.validateTree();
if (errors.length > 0) {
const errorList = errors.map(e => `[${e.source.node.path}] ${e.message}`).join('\n ');
throw new Error(`Validation failed with the following errors:\n ${errorList}`);
}

// synthesize (leaves first)
for (const c of this.node.findAll(ConstructOrder.PostOrder)) {
if (SynthesisSession.isSynthesizable(c)) {
c.synthesize(session);
}
}

const result: cxapi.SynthesizeResponse = {
version: cxapi.PROTO_RESPONSE_VERSION,
stacks: this.synthesizeStacks(Object.keys(this.stacks)),
runtime: this.collectRuntimeInformation()
};
// write session manifest and lock store
session.close();

const outfile = path.join(outdir, cxapi.OUTFILE_NAME);
fs.writeFileSync(outfile, JSON.stringify(result, undefined, 2));
return session;
}

/**
* Synthesize and validate a single stack
* Synthesize and validate a single stack.
* @param stackName The name of the stack to synthesize
* @deprecated This method is going to be deprecated in a future version of the CDK
*/
public synthesizeStack(stackName: string): cxapi.SynthesizedStack {
eladb marked this conversation as resolved.
Show resolved Hide resolved
const stack = this.getStack(stackName);

if (!this.prepared) {
// Maintain the existing contract that the tree will be prepared even if
// 'synthesizeStack' is called by itself. But only prepare the tree once.
this.node.prepareTree();
this.prepared = true;
if (!this.legacyManifest) {
throw new Error('No legacy manifest available, return an old-style stack output');
}

// first, validate this stack and stop if there are errors.
const errors = stack.node.validateTree();
if (errors.length > 0) {
const errorList = errors.map(e => `[${e.source.node.path}] ${e.message}`).join('\n ');
throw new Error(`Stack validation failed with the following errors:\n ${errorList}`);
const session = this.run();
const legacy: cxapi.SynthesizeResponse = session.store.readJson(cxapi.OUTFILE_NAME);

const res = legacy.stacks.find(s => s.name === stackName);
if (!res) {
throw new Error(`Stack "${stackName}" not found`);
}

const account = stack.env.account || 'unknown-account';
const region = stack.env.region || 'unknown-region';

const environment: cxapi.Environment = {
name: `${account}/${region}`,
account,
region
};

const missing = Object.keys(stack.missingContext).length ? stack.missingContext : undefined;
return {
name: stack.node.id,
environment,
missing,
template: stack.toCloudFormation(),
metadata: this.collectMetadata(stack),
dependsOn: noEmptyArray(stack.dependencies().map(s => s.node.id)),
};
return res;
}

/**
* Synthesizes multiple stacks
* @deprecated This method is going to be deprecated in a future version of the CDK
*/
public synthesizeStacks(stackNames: string[]): cxapi.SynthesizedStack[] {
this.node.prepareTree();
this.prepared = true;

const ret: cxapi.SynthesizedStack[] = [];
for (const stackName of stackNames) {
ret.push(this.synthesizeStack(stackName));
}
return ret;
}

/**
* Returns metadata for all constructs in the stack.
*/
public collectMetadata(stack: Stack) {
const output: { [id: string]: MetadataEntry[] } = { };

visit(stack);

// add app-level metadata under "."
if (this.node.metadata.length > 0) {
output[PATH_SEP] = this.node.metadata;
private loadContext(defaults: { [key: string]: string } = { }) {
// primve with defaults passed through constructor
eladb marked this conversation as resolved.
Show resolved Hide resolved
for (const [ k, v ] of Object.entries(defaults)) {
this.node.setContext(k, v);
}

return output;

function visit(node: IConstruct) {
if (node.node.metadata.length > 0) {
// Make the path absolute
output[PATH_SEP + node.node.path] = node.node.metadata.map(md => node.node.resolve(md) as MetadataEntry);
}

for (const child of node.node.children) {
visit(child);
}
}
}

private collectRuntimeInformation(): cxapi.AppRuntime {
const libraries: { [name: string]: string } = {};

for (const fileName of Object.keys(require.cache)) {
const pkg = findNpmPackage(fileName);
if (pkg && !pkg.private) {
libraries[pkg.name] = pkg.version;
}
}

// include only libraries that are in the @aws-cdk npm scope
for (const name of Object.keys(libraries)) {
if (!name.startsWith('@aws-cdk/')) {
delete libraries[name];
}
}

// add jsii runtime version
libraries['jsii-runtime'] = getJsiiAgentVersion();

return { libraries };
}

private getStack(stackname: string) {
if (stackname == null) {
throw new Error('Stack name must be defined');
}

const stack = this.stacks[stackname];
if (!stack) {
throw new Error(`Cannot find stack ${stackname}`);
}
return stack;
}

private loadContext() {
// read from environment
const contextJson = process.env[cxapi.CONTEXT_ENV];
const context = !contextJson ? { } : JSON.parse(contextJson);
for (const key of Object.keys(context)) {
this.node.setContext(key, context[key]);
}
}
}

/**
* Determines which NPM module a given loaded javascript file is from.
*
* The only infromation that is available locally is a list of Javascript files,
* and every source file is associated with a search path to resolve the further
* ``require`` calls made from there, which includes its own directory on disk,
* and parent directories - for example:
*
* [ '...repo/packages/aws-cdk-resources/lib/cfn/node_modules',
* '...repo/packages/aws-cdk-resources/lib/node_modules',
* '...repo/packages/aws-cdk-resources/node_modules',
* '...repo/packages/node_modules',
* // etc...
* ]
*
* We are looking for ``package.json`` that is anywhere in the tree, except it's
* in the parent directory, not in the ``node_modules`` directory. For this
* reason, we strip the ``/node_modules`` suffix off each path and use regular
* module resolution to obtain a reference to ``package.json``.
*
* @param fileName a javascript file name.
* @returns the NPM module infos (aka ``package.json`` contents), or
* ``undefined`` if the lookup was unsuccessful.
*/
function findNpmPackage(fileName: string): { name: string, version: string, private?: boolean } | undefined {
const mod = require.cache[fileName];
const paths = mod.paths.map(stripNodeModules);

try {
const packagePath = require.resolve('package.json', { paths });
return require(packagePath);
} catch (e) {
return undefined;
}
const contextFromEnvironment = contextJson
? JSON.parse(contextJson)
: { };

/**
* @param s a path.
* @returns ``s`` with any terminating ``/node_modules``
* (or ``\\node_modules``) stripped off.)
*/
function stripNodeModules(s: string): string {
if (s.endsWith('/node_modules') || s.endsWith('\\node_modules')) {
// /node_modules is 13 characters
return s.substr(0, s.length - 13);
for (const [ k, v ] of Object.entries(contextFromEnvironment)) {
this.node.setContext(k, v);
}
return s;
}
}

function getJsiiAgentVersion() {
let jsiiAgent = process.env.JSII_AGENT;

// if JSII_AGENT is not specified, we will assume this is a node.js runtime
// and plug in our node.js version
if (!jsiiAgent) {
jsiiAgent = `node.js/${process.version}`;
}

return jsiiAgent;
}

function noEmptyArray<T>(xs: T[]): T[] | undefined {
return xs.length > 0 ? xs : undefined;
}
Loading