fix(benchmark): compilation fails due to missing dependencies (#3610)

Also, adding a reference benchmark building the same code using the TypeScript compiler only. --- By submitting this pull request, I confirm that my contribution is made under the terms of the [Apache 2.0 license]. [Apache 2.0 license]: https://www.apache.org/licenses/LICENSE-2.0
aws · Jun 27, 2022 · 28db6e3 · 28db6e3
1 parent 6cb83eb
commit 28db6e3
Show file tree

Hide file tree

Showing 9 changed files with 672 additions and 75 deletions.
diff --git a/packages/@jsii/benchmarks/bin/benchmark.ts b/packages/@jsii/benchmarks/bin/benchmark.ts
@@ -96,6 +96,28 @@ interface ResultsJson {
       Promise.resolve([]),
     );
 
+    // If we are running in GitHub Actions, emit a summary document.
+    if (process.env.GITHUB_STEP_SUMMARY != null) {
+      await fs.writeFile(
+        process.env.GITHUB_STEP_SUMMARY,
+        [
+          '## Benchmark Results',
+          '',
+          'Suite | Avg | StdDev',
+          '------|-----|-------',
+          ...resultsJson
+            .sort((l, r) => l.name.localeCompare(r.name))
+            .map(
+              ({ name, value, range }) =>
+                `${name} | ${value.toFixed(1)} | ${Math.sqrt(range).toFixed(
+                  2,
+                )}`,
+            ),
+        ].join('\n'),
+        'utf-8',
+      );
+    }
+
     if (argv.output) {
       await fs.writeJson(argv.output, resultsJson, { spaces: 2 });
       console.log(`results written to ${argv.output}`);

diff --git a/packages/@jsii/benchmarks/fixtures/aws-cdk-lib@v2-21-1.tgz b/packages/@jsii/benchmarks/fixtures/aws-cdk-lib@v2-21-1.tgz
diff --git a/packages/@jsii/benchmarks/lib/benchmark.ts b/packages/@jsii/benchmarks/lib/benchmark.ts
@@ -186,10 +186,15 @@ export class Benchmark<C> {
       if (this.#profile) {
         profiler = await this.startProfiler();
       }
-      wrapped(ctx);
-      const profile = await this.killProfiler(profiler);
-      const perf = await observer;
-      this.#afterEach(ctx);
+      let profile: Profiler.Profile | undefined;
+      let perf: PerformanceEntry;
+      try {
+        wrapped(ctx);
+        profile = await this.killProfiler(profiler);
+        perf = await observer;
+      } finally {
+        this.#afterEach(ctx);
+      }
 
       i++;
       yield { profile, performance: perf };
@@ -204,19 +209,52 @@ export class Benchmark<C> {
     const iterations: Iteration[] = [];
     const c = await this.#setup?.();
 
+    const durations = new Array<number>();
+    let min = Number.POSITIVE_INFINITY;
+    let max = Number.NEGATIVE_INFINITY;
+    let sum = 0;
+    let average = 0;
+
+    let id = 0;
+    const padding = this.#iterations.toString().length;
     for await (const result of this.runIterations(c)) {
+      id += 1;
+      const duration = result.performance.duration;
+      durations.push(duration);
+      if (min > duration) {
+        min = duration;
+      }
+      if (max < duration) {
+        max = duration;
+      }
+      sum += duration;
+      average = sum / id;
+
+      const idStr = id.toString().padStart(padding, ' ');
+      const durStr = duration.toFixed(0);
+      const eta = new Date(Date.now() + average * (this.#iterations - id));
+      const pct = (100 * id) / this.#iterations;
+
+      this.log(
+        `Iteration ${idStr}/${this.#iterations} (${pct.toFixed(
+          0,
+        )}%) | Duration: ${durStr}ms | ETA ${eta.toISOString()}`,
+      );
       iterations.push(result);
     }
 
     this.#teardown(c);
 
-    const durations = iterations.map((i) => i.performance.duration);
-    const max = Math.max(...durations);
-    const min = Math.min(...durations);
-    const variance = max - min;
-    const average =
-      durations.reduce((accum, duration) => accum + duration, 0) /
-      durations.length;
+    // Variance is the average of the squared differences from the mean
+    const variance =
+      durations
+        .map((duration) => Math.pow(duration - average, 2))
+        .reduce((accum, squareDev) => accum + squareDev) / durations.length;
+
+    // Standard deviation is the square root of variance
+    const stdDev = Math.sqrt(variance);
+
+    this.log(`Completed: ${average.toFixed(0)}±${stdDev.toFixed(0)}ms`);
 
     return {
       name: this.name,
@@ -227,4 +265,8 @@ export class Benchmark<C> {
       iterations,
     };
   }
+
+  private log(message: string) {
+    console.log(`${new Date().toISOString()} | ${this.name} | ${message}`);
+  }
 }
diff --git a/packages/@jsii/benchmarks/lib/index.ts b/packages/@jsii/benchmarks/lib/index.ts
@@ -4,43 +4,172 @@ import { Compiler } from 'jsii/lib/compiler';
 import { loadProjectInfo } from 'jsii/lib/project-info';
 import * as os from 'os';
 import * as path from 'path';
+import * as ts from 'typescript-3.9';
 
 import { Benchmark } from './benchmark';
 import { cdkv2_21_1, cdkTagv2_21_1 } from './constants';
-import { streamUntar } from './util';
-
-// Always run against the same version of CDK source
-const cdk = new Benchmark(`Compile aws-cdk-lib@${cdkTagv2_21_1}`)
-  .setup(async () => {
-    const sourceDir = fs.mkdtempSync(
-      path.join(os.tmpdir(), 'jsii-cdk-bench-snapshot'),
-    );
-    await streamUntar(cdkv2_21_1, { cwd: sourceDir });
-    cp.execSync('npm ci', { cwd: sourceDir });
-
-    // Working directory for benchmark
-    const workingDir = fs.mkdtempSync(
-      path.join(os.tmpdir(), `jsii-cdk-bench@${cdkTagv2_21_1}`),
-    );
-
-    return {
-      workingDir,
-      sourceDir,
-    } as const;
-  })
-  .beforeEach(({ workingDir, sourceDir }) => {
-    fs.removeSync(workingDir);
-    fs.copySync(sourceDir, workingDir);
-  })
-  .subject(({ workingDir }) => {
-    const { projectInfo } = loadProjectInfo(workingDir);
-    const compiler = new Compiler({ projectInfo });
-
-    compiler.emit();
-  })
-  .teardown(({ workingDir, sourceDir }) => {
-    fs.removeSync(workingDir);
-    fs.removeSync(sourceDir);
-  });
-
-export const benchmarks = [cdk];
+import { inDirectory, streamUntar } from './util';
+
+// Using the local `npm` package (from dependencies)
+const npm = path.resolve(__dirname, '..', 'node_modules', '.bin', 'npm');
+
+export const benchmarks = [
+  // Reference comparison using the TypeScript compiler
+  new Benchmark(`Compile aws-cdk-lib@${cdkTagv2_21_1} (tsc)`)
+    .setup(async () => {
+      const sourceDir = fs.mkdtempSync(
+        path.join(os.tmpdir(), 'jsii-cdk-bench-snapshot'),
+      );
+      await streamUntar(cdkv2_21_1, { cwd: sourceDir });
+      cp.execSync(`${npm} ci`, { cwd: sourceDir });
+
+      // Working directory for benchmark
+      const workingDir = fs.mkdtempSync(
+        path.join(os.tmpdir(), `tsc-cdk-bench@${cdkTagv2_21_1}`),
+      );
+
+      return {
+        workingDir,
+        sourceDir,
+      } as const;
+    })
+    .beforeEach(({ workingDir, sourceDir }) => {
+      fs.removeSync(workingDir);
+      fs.copySync(sourceDir, workingDir);
+    })
+    .subject(({ workingDir }) =>
+      inDirectory(workingDir, () => {
+        const { host, options, rootNames } = (function () {
+          const parsed = ts.parseJsonConfigFileContent(
+            fs.readJsonSync(path.join(workingDir, 'tsconfig.json')),
+            ts.sys,
+            workingDir,
+            {
+              module: ts.ModuleKind.CommonJS,
+              moduleResolution: ts.ModuleResolutionKind.NodeJs,
+              newLine: ts.NewLineKind.LineFeed,
+              tsBuildInfoFile: 'tsconfig.tsbuildinfo',
+            },
+            'tsconfig.json',
+          );
+
+          const host = ts.createIncrementalCompilerHost(parsed.options, ts.sys);
+
+          return {
+            host,
+            options: parsed.options,
+            rootNames: [
+              ...parsed.fileNames,
+              ...(parsed.options.lib && host.getDefaultLibLocation != null
+                ? parsed.options.lib.map((lib) =>
+                    path.join(host.getDefaultLibLocation!(), lib),
+                  )
+                : []),
+            ],
+          };
+        })();
+
+        const program = ts
+          .createIncrementalProgram({
+            createProgram: ts.createEmitAndSemanticDiagnosticsBuilderProgram,
+            host,
+            options,
+            rootNames,
+          })
+          .getProgram();
+
+        const preEmitDiagnostics = ts.getPreEmitDiagnostics(program);
+        if (
+          preEmitDiagnostics.some(
+            (diag) => diag.category === ts.DiagnosticCategory.Error,
+          )
+        ) {
+          console.error(
+            ts.formatDiagnosticsWithColorAndContext(
+              preEmitDiagnostics
+                .filter((diag) => diag.category === ts.DiagnosticCategory.Error)
+                .slice(0, 10),
+              host,
+            ),
+          );
+          throw new Error(`TypeScript compiler emitted pre-emit errors!`);
+        }
+
+        const emitResult = program.emit();
+        if (
+          emitResult.diagnostics.some(
+            (diag) => diag.category === ts.DiagnosticCategory.Error,
+          )
+        ) {
+          console.error(
+            ts.formatDiagnosticsWithColorAndContext(
+              emitResult.diagnostics.filter(
+                (diag) => diag.category === ts.DiagnosticCategory.Error,
+              ),
+              host,
+            ),
+          );
+          throw new Error(`TypeScript compiler emitted errors!`);
+        }
+      }),
+    )
+    .teardown(({ workingDir, sourceDir }) => {
+      fs.removeSync(workingDir);
+      fs.removeSync(sourceDir);
+    }),
+
+  // Always run against the same version of CDK source
+  new Benchmark(`Compile aws-cdk-lib@${cdkTagv2_21_1}`)
+    .setup(async () => {
+      const sourceDir = fs.mkdtempSync(
+        path.join(os.tmpdir(), 'jsii-cdk-bench-snapshot'),
+      );
+      await streamUntar(cdkv2_21_1, { cwd: sourceDir });
+      cp.execSync(`${npm} ci`, { cwd: sourceDir });
+
+      // Working directory for benchmark
+      const workingDir = fs.mkdtempSync(
+        path.join(os.tmpdir(), `jsii-cdk-bench@${cdkTagv2_21_1}`),
+      );
+
+      return {
+        workingDir,
+        sourceDir,
+      } as const;
+    })
+    .beforeEach(({ workingDir, sourceDir }) => {
+      fs.removeSync(workingDir);
+      fs.copySync(sourceDir, workingDir);
+    })
+    .subject(({ workingDir }) =>
+      inDirectory(workingDir, () => {
+        const { projectInfo } = loadProjectInfo(workingDir);
+        const compiler = new Compiler({ projectInfo });
+
+        const result = compiler.emit();
+        if (
+          result.diagnostics.some(
+            (diag) => diag.category === ts.DiagnosticCategory.Error,
+          )
+        ) {
+          console.error(
+            ts.formatDiagnosticsWithColorAndContext(
+              result.diagnostics.filter(
+                (diag) => diag.category === ts.DiagnosticCategory.Error,
+              ),
+              {
+                getCurrentDirectory: () => workingDir,
+                getCanonicalFileName: path.resolve,
+                getNewLine: () => ts.sys.newLine,
+              },
+            ),
+          );
+          throw new Error(`jsii compiler emitted errors!`);
+        }
+      }),
+    )
+    .teardown(({ workingDir, sourceDir }) => {
+      fs.removeSync(workingDir);
+      fs.removeSync(sourceDir);
+    }),
+];
diff --git a/packages/@jsii/benchmarks/lib/util.ts b/packages/@jsii/benchmarks/lib/util.ts
@@ -9,3 +9,13 @@ export async function streamUntar(file: string, config: tar.ExtractOptions) {
     stream.on('error', (error: Error) => ko(error));
   });
 }
+
+export function inDirectory<T>(newWorkDir: string, cb: () => T) {
+  const cwd = process.cwd();
+  try {
+    process.chdir(newWorkDir);
+    return cb();
+  } finally {
+    process.chdir(cwd);
+  }
+}
diff --git a/packages/@jsii/benchmarks/package.json b/packages/@jsii/benchmarks/package.json
@@ -7,6 +7,7 @@
   "dependencies": {
     "fs-extra": "^10.1.0",
     "jsii": "^0.0.0",
+    "npm": "^8.12.1",
     "tar": "^6.1.11",
     "typescript-3.9": "npm:typescript@~3.9.10",
     "yargs": "^16.2.0"