feat(core): democratize synthesis and introduce artifacts

packages/@aws-cdk/applet-js/test/test.applets.ts

@@ -80,16 +80,17 @@ function synthesizeApplet(yamlFile: string, direct = false) {
 
   const command = direct ? yamlFile : 'cdk-applet-js';
   const args = direct ? [] : [yamlFile];
+  const outdir = fs.mkdtempSync(path.join(os.tmpdir(), 'cdk-applet-tests'));
 
   child_process.execFileSync(command, args, {
     env: {
       ...process.env,
-      CDK_OUTDIR: os.tmpdir(),
+      CDK_OUTDIR: outdir,
       PATH: 'bin:' + process.env.PATH
     }
   });
 
-  return JSON.parse(fs.readFileSync(path.join(os.tmpdir(), 'cdk.out'), { encoding: 'utf-8' }));
+  return JSON.parse(fs.readFileSync(path.join(outdir, 'cdk.out'), { encoding: 'utf-8' }));
 }
 
 function getStack(stackName: string, allStacks: any) {

packages/@aws-cdk/assert/lib/expect.ts

@@ -41,7 +41,7 @@ function isStackClassInstance(x: api.SynthesizedStack | cdk.Stack): x is cdk.Sta
 
 function collectStackMetadata(root: cdk.ConstructNode): api.StackMetadata {
   const result: api.StackMetadata = {};
-  for (const construct of root.findAll(cdk.ConstructOrder.DepthFirst)) {
+  for (const construct of root.findAll(cdk.ConstructOrder.PreOrder)) {
     const path = `/${root.id}/${construct.node.path}`;
     for (const entry of construct.node.metadata) {
       result[path] = result[path] || [];

packages/@aws-cdk/cdk/lib/app.ts

@@ -1,14 +1,13 @@
 import cxapi = require('@aws-cdk/cx-api');
-import fs = require('fs');
-import path = require('path');
 import { Stack } from './cloudformation/stack';
-import { IConstruct, MetadataEntry, PATH_SEP, Root } from './core/construct';
+import { IConstruct, Root } from './core/construct';
+import { InMemorySynthesisSession, ISynthesisSession, SynthesisSession } from './synthesis';
 
 /**
  * Represents a CDK program.
  */
 export class App extends Root {
-  private prepared = false;
+  private _session?: ISynthesisSession;
 
   /**
    * Initializes a CDK application.
@@ -21,118 +20,95 @@ export class App extends Root {
 
   private get stacks() {
     const out: { [name: string]: Stack } = { };
-    for (const child of this.node.children) {
-      if (!Stack.isStack(child)) {
-        throw new Error(`The child ${child.toString()} of App must be a Stack`);
-      }
+    collectStacks(this);
+    return out;
+
+    function collectStacks(c: IConstruct) {
+      for (const child of c.node.children) {
+        if (Stack.isStack(child)) {
+          out[child.node.id] = child; // TODO: this should probably be changed to uniqueId
+        }
 
-      out[child.node.id] = child as Stack;
+        collectStacks(child);
+      }
     }
-    return out;
   }
 
   /**
    * Runs the program. Output is written to output directory as specified in the request.
    */
-  public run(): void {
+  public run(): ISynthesisSession {
+    // this app has already been executed, no-op for you
+    if (this._session) {
+      return this._session;
+    }
+
     const outdir = process.env[cxapi.OUTDIR_ENV];
-    if (!outdir) {
-      process.stderr.write(`ERROR: The environment variable "${cxapi.OUTDIR_ENV}" is not defined\n`);
-      process.stderr.write('AWS CDK Toolkit (>= 0.11.0) is required in order to interact with this program.\n');
-      process.exit(1);
-      return;
+    if (outdir) {
+      this._session = new SynthesisSession({ outdir });
+    } else {
+      this._session = new InMemorySynthesisSession();
     }
 
-    const result: cxapi.SynthesizeResponse = {
+    const session = this._session;
+
+    // the three holy phases of synthesis: prepare, validate and synthesize
+
+    // prepare
+    this.node.prepareTree();
+
+    // validate
+    const errors = this.node.validateTree();
+    if (errors.length > 0) {
+      const errorList = errors.map(e => `[${e.source.node.path}] ${e.message}`).join('\n  ');
+      throw new Error(`Validation failed with the following errors:\n  ${errorList}`);
+    }
+
+    // synthesize
+    this.node.synthesizeTree(session);
+
+    // write the entrypoint/manifest of this app. It includes a *copy* of the
+    // synthesized stack output for backwards compatibility
+
+    const manifest: cxapi.SynthesizeResponse = {
       version: cxapi.PROTO_RESPONSE_VERSION,
-      stacks: this.synthesizeStacks(Object.keys(this.stacks)),
+      stacks: Object.values(this.stacks).map(s => this.readSynthesizedStack(session, s.artifactName)),
       runtime: this.collectRuntimeInformation()
     };
 
-    const outfile = path.join(outdir, cxapi.OUTFILE_NAME);
-    fs.writeFileSync(outfile, JSON.stringify(result, undefined, 2));
+    session.writeFile(cxapi.OUTFILE_NAME, JSON.stringify(manifest, undefined, 2));
+
+    // lock session - cannot emit more artifacts
+    session.finalize();
+
+    return session;
   }
 
   /**
    * Synthesize and validate a single stack
    * @param stackName The name of the stack to synthesize
+   * @deprecated This method is going to be deprecated in a future version of the CDK
    */
   public synthesizeStack(stackName: string): cxapi.SynthesizedStack {
     const stack = this.getStack(stackName);
-
-    if (!this.prepared) {
-      // Maintain the existing contract that the tree will be prepared even if
-      // 'synthesizeStack' is called by itself. But only prepare the tree once.
-      this.node.prepareTree();
-      this.prepared = true;
-    }
-
-    // first, validate this stack and stop if there are errors.
-    const errors = stack.node.validateTree();
-    if (errors.length > 0) {
-      const errorList = errors.map(e => `[${e.source.node.path}] ${e.message}`).join('\n  ');
-      throw new Error(`Stack validation failed with the following errors:\n  ${errorList}`);
-    }
-
-    const account = stack.env.account || 'unknown-account';
-    const region = stack.env.region || 'unknown-region';
-
-    const environment: cxapi.Environment = {
-      name: `${account}/${region}`,
-      account,
-      region
-    };
-
-    const missing = Object.keys(stack.missingContext).length ? stack.missingContext : undefined;
-    return {
-      name: stack.node.id,
-      environment,
-      missing,
-      template: stack.toCloudFormation(),
-      metadata: this.collectMetadata(stack),
-      dependsOn: noEmptyArray(stack.dependencies().map(s => s.node.id)),
-    };
+    const session = this.run();
+    return this.readSynthesizedStack(session, stack.artifactName);
   }
 
   /**
    * Synthesizes multiple stacks
+   * @deprecated This method is going to be deprecated in a future version of the CDK
    */
   public synthesizeStacks(stackNames: string[]): cxapi.SynthesizedStack[] {
-    this.node.prepareTree();
-    this.prepared = true;
-
     const ret: cxapi.SynthesizedStack[] = [];
     for (const stackName of stackNames) {
       ret.push(this.synthesizeStack(stackName));
     }
     return ret;
   }
 
-  /**
-   * Returns metadata for all constructs in the stack.
-   */
-  public collectMetadata(stack: Stack) {
-    const output: { [id: string]: MetadataEntry[] } = { };
-
-    visit(stack);
-
-    // add app-level metadata under "."
-    if (this.node.metadata.length > 0) {
-      output[PATH_SEP] = this.node.metadata;
-    }
-
-    return output;
-
-    function visit(node: IConstruct) {
-      if (node.node.metadata.length > 0) {
-        // Make the path absolute
-        output[PATH_SEP + node.node.path] = node.node.metadata.map(md => node.node.resolve(md) as MetadataEntry);
-      }
-
-      for (const child of node.node.children) {
-        visit(child);
-      }
-    }
+  private readSynthesizedStack(session: ISynthesisSession, artifactName: string) {
+    return JSON.parse(session.readFile(artifactName).toString());
   }
 
   private collectRuntimeInformation(): cxapi.AppRuntime {
@@ -164,6 +140,7 @@ export class App extends Root {
     }
 
     const stack = this.stacks[stackname];
+
     if (!stack) {
       throw new Error(`Cannot find stack ${stackname}`);
     }
@@ -239,7 +216,3 @@ function getJsiiAgentVersion() {
 
   return jsiiAgent;
 }
-
-function noEmptyArray<T>(xs: T[]): T[] | undefined {
-  return xs.length > 0 ? xs : undefined;
-}

packages/@aws-cdk/cdk/lib/cloudformation/stack.ts

@@ -1,7 +1,8 @@
 import cxapi = require('@aws-cdk/cx-api');
 import { App } from '../app';
-import { Construct, IConstruct } from '../core/construct';
+import { Construct, IConstruct, PATH_SEP } from '../core/construct';
 import { Environment } from '../environment';
+import { ISynthesisSession } from '../synthesis';
 import { CfnReference } from './cfn-tokens';
 import { HashedAddressingScheme, IAddressingScheme, LogicalIDs } from './logical-id';
 
@@ -77,6 +78,11 @@ export class Stack extends Construct {
    */
   public readonly name: string;
 
+  /**
+   * The name of the CDK artifact produced by this stack.
+   */
+  public readonly artifactName: string;
+
   /*
    * Used to determine if this construct is a stack.
    */
@@ -106,6 +112,8 @@ export class Stack extends Construct {
 
     this.logicalIds = new LogicalIDs(props && props.namingScheme ? props.namingScheme : new HashedAddressingScheme());
     this.name = this.node.id;
+
+    this.artifactName = `${this.node.uniqueId}.stack.json`;
   }
 
   /**
@@ -417,6 +425,30 @@ export class Stack extends Construct {
     }
   }
 
+  protected synthesize(session: ISynthesisSession): void {
+    const account = this.env.account || 'unknown-account';
+    const region = this.env.region || 'unknown-region';
+
+    const environment: cxapi.Environment = {
+      name: `${account}/${region}`,
+      account,
+      region
+    };
+
+    const missing = Object.keys(this.missingContext).length ? this.missingContext : undefined;
+
+    const output: cxapi.SynthesizedStack = {
+      name: this.node.id,
+      template: this.toCloudFormation(),
+      environment,
+      missing,
+      metadata: this.collectMetadata(),
+      dependsOn: noEmptyArray(this.dependencies().map(s => s.node.id)),
+    };
+
+    session.writeFile(this.artifactName, JSON.stringify(output, undefined, 2));
+  }
+
   /**
    * Applied defaults to environment attributes.
    */
@@ -447,6 +479,30 @@ export class Stack extends Construct {
     }
     return false;
   }
+
+  private collectMetadata() {
+    const output: { [id: string]: cxapi.MetadataEntry[] } = { };
+
+    visit(this);
+
+    const app = this.parentApp();
+    if (app && app.node.metadata.length > 0) {
+      output[PATH_SEP] = app.node.metadata;
+    }
+
+    return output;
+
+    function visit(node: IConstruct) {
+      if (node.node.metadata.length > 0) {
+        // Make the path absolute
+        output[PATH_SEP + node.node.path] = node.node.metadata.map(md => node.node.resolve(md) as cxapi.MetadataEntry);
+      }
+
+      for (const child of node.node.children) {
+        visit(child);
+      }
+    }
+  }
 }
 
 function merge(template: any, part: any) {
@@ -530,3 +586,7 @@ function findResources(roots: Iterable<IConstruct>): Resource[] {
   }
   return ret;
 }
+
+function noEmptyArray<T>(xs: T[]): T[] | undefined {
+  return xs.length > 0 ? xs : undefined;
+}