feat(core): democratize synthesis and introduce artifacts

packages/@aws-cdk/cdk/lib/app.ts

@@ -1,14 +1,13 @@
 import cxapi = require('@aws-cdk/cx-api');
-import fs = require('fs');
-import path = require('path');
 import { Stack } from './cloudformation/stack';
-import { IConstruct, MetadataEntry, PATH_SEP, Root } from './core/construct';
+import { Root } from './core/construct';
+import { InMemorySynthesisSession, ISynthesisSession, SynthesisSession } from './synthesis';
 
 /**
  * Represents a CDK program.
  */
 export class App extends Root {
-  private prepared = false;
+  private _session?: ISynthesisSession;
 
   /**
    * Initializes a CDK application.
@@ -34,73 +33,49 @@ export class App extends Root {
   /**
    * Runs the program. Output is written to output directory as specified in the request.
    */
-  public run(): void {
+  public run(): ISynthesisSession {
+    // this app has already been executed, no-op for you
+    if (this._session) {
+      return this._session;
+    }
+
     const outdir = process.env[cxapi.OUTDIR_ENV];
-    if (!outdir) {
-      process.stderr.write(`ERROR: The environment variable "${cxapi.OUTDIR_ENV}" is not defined\n`);
-      process.stderr.write('AWS CDK Toolkit (>= 0.11.0) is required in order to interact with this program.\n');
-      process.exit(1);
-      return;
+    if (outdir) {
+      this._session = new SynthesisSession({ outdir });
+    } else {
+      this._session = new InMemorySynthesisSession();
     }
 
-    const result: cxapi.SynthesizeResponse = {
-      version: cxapi.PROTO_RESPONSE_VERSION,
-      stacks: this.synthesizeStacks(Object.keys(this.stacks)),
-      runtime: this.collectRuntimeInformation()
-    };
+    // the three holy phases of synthesis: validate, prepare and synthesize
+    const errors = this.node.validateTree();
+    if (errors.length > 0) {
+      const errorList = errors.map(e => `[${e.source.node.path}] ${e.message}`).join('\n  ');
+      throw new Error(`Validation failed with the following errors:\n  ${errorList}`);
+    }
 
-    const outfile = path.join(outdir, cxapi.OUTFILE_NAME);
-    fs.writeFileSync(outfile, JSON.stringify(result, undefined, 2));
+    this.node.prepareTree();
+    this.node.synthesizeTree(this.run());
+
+    this._session.finalize(); // lock session - cannot emit more artifacts
+
+    return this._session;
   }
 
   /**
    * Synthesize and validate a single stack
    * @param stackName The name of the stack to synthesize
    */
   public synthesizeStack(stackName: string): cxapi.SynthesizedStack {
-    const stack = this.getStack(stackName);
-
-    if (!this.prepared) {
-      // Maintain the existing contract that the tree will be prepared even if
-      // 'synthesizeStack' is called by itself. But only prepare the tree once.
-      this.node.prepareTree();
-      this.prepared = true;
-    }
-
-    // first, validate this stack and stop if there are errors.
-    const errors = stack.node.validateTree();
-    if (errors.length > 0) {
-      const errorList = errors.map(e => `[${e.source.node.path}] ${e.message}`).join('\n  ');
-      throw new Error(`Stack validation failed with the following errors:\n  ${errorList}`);
-    }
-
-    const account = stack.env.account || 'unknown-account';
-    const region = stack.env.region || 'unknown-region';
+    this.getStack(stackName); // just make sure stack exists
 
-    const environment: cxapi.Environment = {
-      name: `${account}/${region}`,
-      account,
-      region
-    };
-
-    const missing = Object.keys(stack.missingContext).length ? stack.missingContext : undefined;
-    return {
-      name: stack.node.id,
-      environment,
-      missing,
-      template: stack.toCloudFormation(),
-      metadata: this.collectMetadata(stack),
-      dependsOn: noEmptyArray(stack.dependencies().map(s => s.node.id)),
-    };
+    const artifact = this.run().readFile(Stack.artifactIdForStack(stackName));
+    return JSON.parse(artifact);
   }
 
   /**
    * Synthesizes multiple stacks
    */
   public synthesizeStacks(stackNames: string[]): cxapi.SynthesizedStack[] {
-    this.node.prepareTree();
-    this.prepared = true;
-
     const ret: cxapi.SynthesizedStack[] = [];
     for (const stackName of stackNames) {
       ret.push(this.synthesizeStack(stackName));
@@ -109,30 +84,16 @@ export class App extends Root {
   }
 
   /**
-   * Returns metadata for all constructs in the stack.
+   * Synthesize the app manifest (the root file which the toolkit reads)
    */
-  public collectMetadata(stack: Stack) {
-    const output: { [id: string]: MetadataEntry[] } = { };
-
-    visit(stack);
-
-    // add app-level metadata under "."
-    if (this.node.metadata.length > 0) {
-      output[PATH_SEP] = this.node.metadata;
-    }
-
-    return output;
-
-    function visit(node: IConstruct) {
-      if (node.node.metadata.length > 0) {
-        // Make the path absolute
-        output[PATH_SEP + node.node.path] = node.node.metadata.map(md => node.node.resolve(md) as MetadataEntry);
-      }
+  protected synthesize(session: ISynthesisSession) {
+    const manifest: cxapi.SynthesizeResponse = {
+      version: cxapi.PROTO_RESPONSE_VERSION,
+      stacks: this.synthesizeStacks(Object.keys(this.stacks)),
+      runtime: this.collectRuntimeInformation()
+    };
 
-      for (const child of node.node.children) {
-        visit(child);
-      }
-    }
+    session.writeFile(cxapi.OUTFILE_NAME, JSON.stringify(manifest, undefined, 2));
   }
 
   private collectRuntimeInformation(): cxapi.AppRuntime {
@@ -239,7 +200,3 @@ function getJsiiAgentVersion() {
 
   return jsiiAgent;
 }
-
-function noEmptyArray<T>(xs: T[]): T[] | undefined {
-  return xs.length > 0 ? xs : undefined;
-}

packages/@aws-cdk/cdk/lib/cloudformation/stack.ts

@@ -1,7 +1,8 @@
 import cxapi = require('@aws-cdk/cx-api');
 import { App } from '../app';
-import { Construct, IConstruct } from '../core/construct';
+import { Construct, IConstruct, PATH_SEP } from '../core/construct';
 import { Environment } from '../environment';
+import { ISynthesisSession } from '../synthesis';
 import { CfnReference } from './cfn-tokens';
 import { HashedAddressingScheme, IAddressingScheme, LogicalIDs } from './logical-id';
 
@@ -48,6 +49,10 @@ export class Stack extends Construct {
     return (construct as any)._isStack;
   }
 
+  public static artifactIdForStack(stackName: string) {
+    return `${stackName}.template.json`;
+  }
+
   private static readonly VALID_STACK_NAME_REGEX = /^[A-Za-z][A-Za-z0-9-]*$/;
 
   /**
@@ -125,6 +130,54 @@ export class Stack extends Construct {
     return r as Resource;
   }
 
+  public synthesize(session: ISynthesisSession): void {
+    const account = this.env.account || 'unknown-account';
+    const region = this.env.region || 'unknown-region';
+
+    const environment: cxapi.Environment = {
+      name: `${account}/${region}`,
+      account,
+      region
+    };
+
+    const missing = Object.keys(this.missingContext).length ? this.missingContext : undefined;
+
+    const output: cxapi.SynthesizedStack = {
+      name: this.node.id,
+      template: this.toCloudFormation(),
+      environment,
+      missing,
+      metadata: this.collectMetadata(),
+      dependsOn: noEmptyArray(this.dependencies().map(s => s.node.id)),
+    };
+
+    session.writeFile(Stack.artifactIdForStack(this.node.id), JSON.stringify(output, undefined, 2));
+  }
+
+  public collectMetadata() {
+    const output: { [id: string]: cxapi.MetadataEntry[] } = { };
+
+    visit(this);
+
+    const app = this.parentApp();
+    if (app && app.node.metadata.length > 0) {
+      output[PATH_SEP] = app.node.metadata;
+    }
+
+    return output;
+
+    function visit(node: IConstruct) {
+      if (node.node.metadata.length > 0) {
+        // Make the path absolute
+        output[PATH_SEP + node.node.path] = node.node.metadata.map(md => node.node.resolve(md) as cxapi.MetadataEntry);
+      }
+
+      for (const child of node.node.children) {
+        visit(child);
+      }
+    }
+  }
+
   /**
    * Returns the CloudFormation template for this stack by traversing
    * the tree and invoking toCloudFormation() on all Entity objects.
@@ -530,3 +583,7 @@ function findResources(roots: Iterable<IConstruct>): Resource[] {
   }
   return ret;
 }
+
+function noEmptyArray<T>(xs: T[]): T[] | undefined {
+  return xs.length > 0 ? xs : undefined;
+}

packages/@aws-cdk/cdk/lib/core/construct.ts

@@ -1,10 +1,12 @@
 import cxapi = require('@aws-cdk/cx-api');
 import { IAspect } from '../aspects/aspect';
 import { CloudFormationJSON } from '../cloudformation/cloudformation-json';
+import { ISynthesisSession } from '../synthesis';
 import { makeUniqueId } from '../util/uniqueid';
 import { IDependable } from './dependency';
 import { Token, unresolved } from './tokens';
 import { resolve } from './tokens/resolve';
+
 export const PATH_SEP = '/';
 
 /**
@@ -190,15 +192,22 @@ export class ConstructNode {
    */
   public findAll(order: ConstructOrder = ConstructOrder.DepthFirst): IConstruct[] {
     const ret = new Array<IConstruct>();
-    const queue: IConstruct[] = [this.host];
+    visit(this.host);
+    return ret;
 
-    while (queue.length > 0) {
-      const next = order === ConstructOrder.BreadthFirst ? queue.splice(0, 1)[0] : queue.pop()!;
-      ret.push(next);
-      queue.push(...next.node.children);
-    }
+    function visit(node: IConstruct) {
+      if (order === ConstructOrder.BreadthFirst) {
+        ret.push(node);
+      }
 
-    return ret;
+      for (const child of node.node.children) {
+        visit(child);
+      }
+
+      if (order === ConstructOrder.DepthFirst) {
+        ret.push(node);
+      }
+    }
   }
 
   /**
@@ -332,6 +341,19 @@ export class ConstructNode {
     }
   }
 
+  /**
+   * Synthesizes the entire subtree by writing artifacts into a synthesis session.
+   */
+  public synthesizeTree(session: ISynthesisSession) {
+    const constructs = this.host.node.findAll(ConstructOrder.DepthFirst);
+
+    for (const construct of constructs) {
+      if (Construct.isConstruct(construct)) {
+        (construct as any).synthesize(session);
+      }
+    }
+  }
+
   /**
    * Applies the aspect to this Constructs node
    */
@@ -626,9 +648,23 @@ export class Construct implements IConstruct {
    * understand the implications.
    */
   protected prepare(): void {
-    // Intentionally left blank
+    return;
   }
 
+  /**
+   * Synthesizes this construct into artifacts.
+   *
+   * This method can be overloaded by any construct that wishes to emit artifacts during
+   * the tree synthesis. For example, the `Stack` construct overrides this and produces
+   * CloudFormation templates, `Asset` overrides this to produce asset artifacts, etc.
+   *
+   * To emit artifacts, use the API of the `Session` argument.
+   *
+   * @param _session synthesis session
+   */
+  protected synthesize(_session: ISynthesisSession): void {
+    return;
+  }
 }
 
 /**

packages/@aws-cdk/cdk/lib/index.ts

@@ -32,3 +32,5 @@ export * from './context';
 export * from './environment';
 
 export * from './runtime';
+
+export * from './synthesis';