diff --git a/packages/angular_devkit/build_angular/src/tools/esbuild/angular/compilation/angular-compilation.ts b/packages/angular_devkit/build_angular/src/tools/esbuild/angular/compilation/angular-compilation.ts index 164a5807aa4b..3cf4bdae354c 100644 --- a/packages/angular_devkit/build_angular/src/tools/esbuild/angular/compilation/angular-compilation.ts +++ b/packages/angular_devkit/build_angular/src/tools/esbuild/angular/compilation/angular-compilation.ts @@ -10,7 +10,7 @@ import type ng from '@angular/compiler-cli'; import type { PartialMessage } from 'esbuild'; import ts from 'typescript'; import { loadEsmModule } from '../../../../utils/load-esm'; -import { profileSync } from '../../profiling'; +import { profileAsync, profileSync } from '../../profiling'; import type { AngularHostOptions } from '../angular-host'; import { convertTypeScriptDiagnostic } from '../diagnostics'; @@ -26,9 +26,8 @@ export abstract class AngularCompilation { static async loadCompilerCli(): Promise { // This uses a wrapped dynamic import to load `@angular/compiler-cli` which is ESM. // Once TypeScript provides support for retaining dynamic imports this workaround can be dropped. - AngularCompilation.#angularCompilerCliModule ??= await loadEsmModule( - '@angular/compiler-cli', - ); + AngularCompilation.#angularCompilerCliModule ??= + await loadEsmModule('@angular/compiler-cli'); return AngularCompilation.#angularCompilerCliModule; } @@ -63,15 +62,17 @@ export abstract class AngularCompilation { referencedFiles: readonly string[]; }>; - abstract emitAffectedFiles(): Iterable; + abstract emitAffectedFiles(): Iterable | Promise>; - protected abstract collectDiagnostics(): Iterable; + protected abstract collectDiagnostics(): + | Iterable + | Promise>; async diagnoseFiles(): Promise<{ errors?: PartialMessage[]; warnings?: PartialMessage[] }> { const result: { errors?: PartialMessage[]; warnings?: PartialMessage[] } = {}; - profileSync('NG_DIAGNOSTICS_TOTAL', () => { - for (const diagnostic of this.collectDiagnostics()) { + await profileAsync('NG_DIAGNOSTICS_TOTAL', async () => { + for (const diagnostic of await this.collectDiagnostics()) { const message = convertTypeScriptDiagnostic(diagnostic); if (diagnostic.category === ts.DiagnosticCategory.Error) { (result.errors ??= []).push(message); @@ -83,4 +84,8 @@ export abstract class AngularCompilation { return result; } + + update?(files: Set): Promise; + + close?(): Promise; } diff --git a/packages/angular_devkit/build_angular/src/tools/esbuild/angular/compilation/factory.ts b/packages/angular_devkit/build_angular/src/tools/esbuild/angular/compilation/factory.ts new file mode 100644 index 000000000000..fe6b648f73f0 --- /dev/null +++ b/packages/angular_devkit/build_angular/src/tools/esbuild/angular/compilation/factory.ts @@ -0,0 +1,35 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ + +import { useParallelTs } from '../../../../utils/environment-options'; +import type { AngularCompilation } from './angular-compilation'; + +/** + * Creates an Angular compilation object that can be used to perform Angular application + * compilation either for AOT or JIT mode. By default a parallel compilation is created + * that uses a Node.js worker thread. + * @param jit True, for Angular JIT compilation; False, for Angular AOT compilation. + * @returns An instance of an Angular compilation object. + */ +export async function createAngularCompilation(jit: boolean): Promise { + if (useParallelTs) { + const { ParallelCompilation } = await import('./parallel-compilation'); + + return new ParallelCompilation(jit); + } + + if (jit) { + const { JitCompilation } = await import('./jit-compilation'); + + return new JitCompilation(); + } else { + const { AotCompilation } = await import('./aot-compilation'); + + return new AotCompilation(); + } +} diff --git a/packages/angular_devkit/build_angular/src/tools/esbuild/angular/compilation/index.ts b/packages/angular_devkit/build_angular/src/tools/esbuild/angular/compilation/index.ts index 3e7eed152a4e..cd79025ab5e1 100644 --- a/packages/angular_devkit/build_angular/src/tools/esbuild/angular/compilation/index.ts +++ b/packages/angular_devkit/build_angular/src/tools/esbuild/angular/compilation/index.ts @@ -7,6 +7,5 @@ */ export { AngularCompilation } from './angular-compilation'; -export { AotCompilation } from './aot-compilation'; -export { JitCompilation } from './jit-compilation'; +export { createAngularCompilation } from './factory'; export { NoopCompilation } from './noop-compilation'; diff --git a/packages/angular_devkit/build_angular/src/tools/esbuild/angular/compilation/parallel-compilation.ts b/packages/angular_devkit/build_angular/src/tools/esbuild/angular/compilation/parallel-compilation.ts new file mode 100644 index 000000000000..b23ed97d0fca --- /dev/null +++ b/packages/angular_devkit/build_angular/src/tools/esbuild/angular/compilation/parallel-compilation.ts @@ -0,0 +1,140 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ + +import type { CompilerOptions } from '@angular/compiler-cli'; +import type { PartialMessage } from 'esbuild'; +import { createRequire } from 'node:module'; +import { MessageChannel } from 'node:worker_threads'; +import Piscina from 'piscina'; +import type { SourceFile } from 'typescript'; +import type { AngularHostOptions } from '../angular-host'; +import { AngularCompilation, EmitFileResult } from './angular-compilation'; + +/** + * An Angular compilation which uses a Node.js Worker thread to load and execute + * the TypeScript and Angular compilers. This allows for longer synchronous actions + * such as semantic and template diagnostics to be calculated in parallel to the + * other aspects of the application bundling process. The worker thread also has + * a separate memory pool which significantly reduces the need for adjusting the + * main Node.js CLI process memory settings with large application code sizes. + */ +export class ParallelCompilation extends AngularCompilation { + readonly #worker: Piscina; + + constructor(readonly jit: boolean) { + super(); + + // TODO: Convert to import.meta usage during ESM transition + const localRequire = createRequire(__filename); + + this.#worker = new Piscina({ + minThreads: 1, + maxThreads: 1, + idleTimeout: Infinity, + filename: localRequire.resolve('./parallel-worker'), + }); + } + + override initialize( + tsconfig: string, + hostOptions: AngularHostOptions, + compilerOptionsTransformer?: + | ((compilerOptions: CompilerOptions) => CompilerOptions) + | undefined, + ): Promise<{ + affectedFiles: ReadonlySet; + compilerOptions: CompilerOptions; + referencedFiles: readonly string[]; + }> { + const stylesheetChannel = new MessageChannel(); + // The request identifier is required because Angular can issue multiple concurrent requests + stylesheetChannel.port1.on('message', ({ requestId, data, containingFile, stylesheetFile }) => { + hostOptions + .transformStylesheet(data, containingFile, stylesheetFile) + .then((value) => stylesheetChannel.port1.postMessage({ requestId, value })) + .catch((error) => stylesheetChannel.port1.postMessage({ requestId, error })); + }); + + // The web worker processing is a synchronous operation and uses shared memory combined with + // the Atomics API to block execution here until a response is received. + const webWorkerChannel = new MessageChannel(); + const webWorkerSignal = new Int32Array(new SharedArrayBuffer(4)); + webWorkerChannel.port1.on('message', ({ workerFile, containingFile }) => { + try { + const workerCodeFile = hostOptions.processWebWorker(workerFile, containingFile); + webWorkerChannel.port1.postMessage({ workerCodeFile }); + } catch (error) { + webWorkerChannel.port1.postMessage({ error }); + } finally { + Atomics.store(webWorkerSignal, 0, 1); + Atomics.notify(webWorkerSignal, 0); + } + }); + + // The compiler options transformation is a synchronous operation and uses shared memory combined + // with the Atomics API to block execution here until a response is received. + const optionsChannel = new MessageChannel(); + const optionsSignal = new Int32Array(new SharedArrayBuffer(4)); + optionsChannel.port1.on('message', (compilerOptions) => { + try { + const transformedOptions = compilerOptionsTransformer?.(compilerOptions) ?? compilerOptions; + optionsChannel.port1.postMessage({ transformedOptions }); + } catch (error) { + webWorkerChannel.port1.postMessage({ error }); + } finally { + Atomics.store(optionsSignal, 0, 1); + Atomics.notify(optionsSignal, 0); + } + }); + + // Execute the initialize function in the worker thread + return this.#worker.run( + { + fileReplacements: hostOptions.fileReplacements, + tsconfig, + jit: this.jit, + stylesheetPort: stylesheetChannel.port2, + optionsPort: optionsChannel.port2, + optionsSignal, + webWorkerPort: webWorkerChannel.port2, + webWorkerSignal, + }, + { + name: 'initialize', + transferList: [stylesheetChannel.port2, optionsChannel.port2, webWorkerChannel.port2], + }, + ); + } + + /** + * This is not needed with this compilation type since the worker will already send a response + * with the serializable esbuild compatible diagnostics. + */ + protected override collectDiagnostics(): never { + throw new Error('Not implemented in ParallelCompilation.'); + } + + override diagnoseFiles(): Promise<{ errors?: PartialMessage[]; warnings?: PartialMessage[] }> { + return this.#worker.run(undefined, { name: 'diagnose' }); + } + + override emitAffectedFiles(): Promise> { + return this.#worker.run(undefined, { name: 'emit' }); + } + + override update(files: Set): Promise { + return this.#worker.run(files, { name: 'update' }); + } + + override close() { + // Workaround piscina bug where a worker thread will be recreated after destroy to meet the minimum. + this.#worker.options.minThreads = 0; + + return this.#worker.destroy(); + } +} diff --git a/packages/angular_devkit/build_angular/src/tools/esbuild/angular/compilation/parallel-worker.ts b/packages/angular_devkit/build_angular/src/tools/esbuild/angular/compilation/parallel-worker.ts new file mode 100644 index 000000000000..96bce2f19972 --- /dev/null +++ b/packages/angular_devkit/build_angular/src/tools/esbuild/angular/compilation/parallel-worker.ts @@ -0,0 +1,119 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ + +import assert from 'node:assert'; +import { randomUUID } from 'node:crypto'; +import { type MessagePort, receiveMessageOnPort } from 'node:worker_threads'; +import { SourceFileCache } from '../source-file-cache'; +import type { AngularCompilation } from './angular-compilation'; +import { AotCompilation } from './aot-compilation'; +import { JitCompilation } from './jit-compilation'; + +export interface InitRequest { + jit: boolean; + tsconfig: string; + fileReplacements?: Record; + stylesheetPort: MessagePort; + optionsPort: MessagePort; + optionsSignal: Int32Array; + webWorkerPort: MessagePort; + webWorkerSignal: Int32Array; +} + +let compilation: AngularCompilation | undefined; + +const sourceFileCache = new SourceFileCache(); + +export async function initialize(request: InitRequest) { + compilation ??= request.jit ? new JitCompilation() : new AotCompilation(); + + const stylesheetRequests = new Map void, (reason: Error) => void]>(); + request.stylesheetPort.on('message', ({ requestId, value, error }) => { + if (error) { + stylesheetRequests.get(requestId)?.[1](error); + } else { + stylesheetRequests.get(requestId)?.[0](value); + } + }); + + const { compilerOptions, referencedFiles } = await compilation.initialize( + request.tsconfig, + { + fileReplacements: request.fileReplacements, + sourceFileCache, + modifiedFiles: sourceFileCache.modifiedFiles, + transformStylesheet(data, containingFile, stylesheetFile) { + const requestId = randomUUID(); + const resultPromise = new Promise((resolve, reject) => + stylesheetRequests.set(requestId, [resolve, reject]), + ); + + request.stylesheetPort.postMessage({ + requestId, + data, + containingFile, + stylesheetFile, + }); + + return resultPromise; + }, + processWebWorker(workerFile, containingFile) { + Atomics.store(request.webWorkerSignal, 0, 0); + request.webWorkerPort.postMessage({ workerFile, containingFile }); + + Atomics.wait(request.webWorkerSignal, 0, 0); + const result = receiveMessageOnPort(request.webWorkerPort)?.message; + + if (result?.error) { + throw result.error; + } + + return result?.workerCodeFile ?? workerFile; + }, + }, + (compilerOptions) => { + Atomics.store(request.optionsSignal, 0, 0); + request.optionsPort.postMessage(compilerOptions); + + Atomics.wait(request.optionsSignal, 0, 0); + const result = receiveMessageOnPort(request.optionsPort)?.message; + + if (result?.error) { + throw result.error; + } + + return result?.transformedOptions ?? compilerOptions; + }, + ); + + return { + referencedFiles, + // TODO: Expand? `allowJs` is the only field needed currently. + compilerOptions: { allowJs: compilerOptions.allowJs }, + }; +} + +export async function diagnose() { + assert(compilation); + + const diagnostics = await compilation.diagnoseFiles(); + + return diagnostics; +} + +export async function emit() { + assert(compilation); + + const files = await compilation.emitAffectedFiles(); + + return [...files]; +} + +export function update(files: Set): void { + sourceFileCache.invalidate(files); +} diff --git a/packages/angular_devkit/build_angular/src/tools/esbuild/angular/compiler-plugin.ts b/packages/angular_devkit/build_angular/src/tools/esbuild/angular/compiler-plugin.ts index 52de66407345..81621af63053 100644 --- a/packages/angular_devkit/build_angular/src/tools/esbuild/angular/compiler-plugin.ts +++ b/packages/angular_devkit/build_angular/src/tools/esbuild/angular/compiler-plugin.ts @@ -23,15 +23,10 @@ import ts from 'typescript'; import { maxWorkers } from '../../../utils/environment-options'; import { JavaScriptTransformer } from '../javascript-transformer'; import { LoadResultCache } from '../load-result-cache'; -import { - logCumulativeDurations, - profileAsync, - profileSync, - resetCumulativeDurations, -} from '../profiling'; +import { logCumulativeDurations, profileAsync, resetCumulativeDurations } from '../profiling'; import { BundleStylesheetOptions } from '../stylesheets/bundle-options'; import { AngularHostOptions } from './angular-host'; -import { AngularCompilation, AotCompilation, JitCompilation, NoopCompilation } from './compilation'; +import { AngularCompilation, NoopCompilation, createAngularCompilation } from './compilation'; import { SharedTSCompilationState, getSharedCompilationState } from './compilation-state'; import { ComponentStylesheetBundler } from './component-stylesheets'; import { FileReferenceTracker } from './file-reference-tracker'; @@ -95,9 +90,7 @@ export function createCompilerPlugin( // Create new reusable compilation for the appropriate mode based on the `jit` plugin option const compilation: AngularCompilation = pluginOptions.noopTypeScriptCompilation ? new NoopCompilation() - : pluginOptions.jit - ? new JitCompilation() - : new AotCompilation(); + : await createAngularCompilation(!!pluginOptions.jit); // Determines if TypeScript should process JavaScript files based on tsconfig `allowJs` option let shouldTsIgnoreJs = true; @@ -141,6 +134,14 @@ export function createCompilerPlugin( pluginOptions.sourceFileCache.invalidate(modifiedFiles); } + if ( + !pluginOptions.noopTypeScriptCompilation && + compilation.update && + pluginOptions.sourceFileCache?.modifiedFiles.size + ) { + await compilation.update(modifiedFiles ?? pluginOptions.sourceFileCache.modifiedFiles); + } + // Create Angular compiler host options const hostOptions: AngularHostOptions = { fileReplacements: pluginOptions.fileReplacements, @@ -298,8 +299,8 @@ export function createCompilerPlugin( } // Update TypeScript file output cache for all affected files - profileSync('NG_EMIT_TS', () => { - for (const { filename, contents } of compilation.emitAffectedFiles()) { + await profileAsync('NG_EMIT_TS', async () => { + for (const { filename, contents } of await compilation.emitAffectedFiles()) { typeScriptFileCache.set(pathToFileURL(filename).href, contents); } }); @@ -426,6 +427,7 @@ export function createCompilerPlugin( build.onDispose(() => { sharedTSCompilationState?.dispose(); void stylesheetBundler.dispose(); + void compilation.close?.(); }); }, }; diff --git a/packages/angular_devkit/build_angular/src/tools/esbuild/angular/diagnostics.ts b/packages/angular_devkit/build_angular/src/tools/esbuild/angular/diagnostics.ts index c1269ee80d34..d43e0a23025d 100644 --- a/packages/angular_devkit/build_angular/src/tools/esbuild/angular/diagnostics.ts +++ b/packages/angular_devkit/build_angular/src/tools/esbuild/angular/diagnostics.ts @@ -83,11 +83,10 @@ export function convertTypeScriptDiagnostic(diagnostic: Diagnostic): PartialMess code = code.slice(3); } - const message: PartialMessage = { - ...convertTypeScriptDiagnosticInfo(diagnostic, `${codePrefix}${code}: `), - // Store original diagnostic for reference if needed downstream - detail: diagnostic, - }; + const message: PartialMessage = convertTypeScriptDiagnosticInfo( + diagnostic, + `${codePrefix}${code}: `, + ); if (diagnostic.relatedInformation?.length) { message.notes = diagnostic.relatedInformation.map((info) => diff --git a/packages/angular_devkit/build_angular/src/utils/environment-options.ts b/packages/angular_devkit/build_angular/src/utils/environment-options.ts index ff82810d74da..548b3a617cfa 100644 --- a/packages/angular_devkit/build_angular/src/utils/environment-options.ts +++ b/packages/angular_devkit/build_angular/src/utils/environment-options.ts @@ -78,6 +78,9 @@ export const allowMinify = debugOptimize.minify; const maxWorkersVariable = process.env['NG_BUILD_MAX_WORKERS']; export const maxWorkers = isPresent(maxWorkersVariable) ? +maxWorkersVariable : 4; +const parallelTsVariable = process.env['NG_BUILD_PARALLEL_TS']; +export const useParallelTs = !isPresent(parallelTsVariable) || !isDisabled(parallelTsVariable); + const legacySassVariable = process.env['NG_BUILD_LEGACY_SASS']; export const useLegacySass: boolean = (() => { if (!isPresent(legacySassVariable)) {