Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Make initial incremental/watch builds as fast normal builds #42960

Closed
wants to merge 7 commits into from
Closed

Make initial incremental/watch builds as fast normal builds #42960

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
3c5d4a7
lazy compute shapes and exported modules for better performance
sokra Feb 26, 2021
90bba5c
use global avoidInitializingSignatures flag instead of NOT_COMPUTED_Y…
sokra Mar 5, 2021
13f153e
avoid disableLazyShapeComputation in compileOnSave test
sokra Mar 5, 2021
94d3aa5
avoid disableLazyShapeComputation in emitAndErrorUpdates test
sokra Mar 5, 2021
d0b4580
avoid disableLazyShapeComputation in tscWatch incremental test
sokra Mar 5, 2021
2165ffd
avoid disableLazyShapeComputation in project reference compile on sav…
sokra Mar 5, 2021
c4fb4f3
remove displayLazyShapeCompilation option
sokra Mar 5, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 10 additions & 5 deletions src/compiler/builder.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ namespace ts {
* Map of file signatures, with key being file path, calculated while getting current changed file's affected files
* These will be committed whenever the iteration through affected files of current changed file is complete
*/
currentAffectedFilesSignatures?: ReadonlyESMap<Path, string> | undefined;
currentAffectedFilesSignatures?: ReadonlyESMap<Path, string | undefined> | undefined;
/**
* Newly computed visible to outside referencedSet
*/
Expand Down Expand Up @@ -110,7 +110,7 @@ namespace ts {
* Map of file signatures, with key being file path, calculated while getting current changed file's affected files
* These will be committed whenever the iteration through affected files of current changed file is complete
*/
currentAffectedFilesSignatures: ESMap<Path, string> | undefined;
currentAffectedFilesSignatures: ESMap<Path, string | undefined> | undefined;
/**
* Newly computed visible to outside referencedSet
*/
Expand Down Expand Up @@ -245,6 +245,9 @@ namespace ts {
}
});

// If there are a lot changed files, avoid intializing siguratures this time for performance reasons
if(state.changedFilesSet.size > 3) state.avoidInitializingSignatures = true;

// If the global file is removed, add all files as changed
if (useOldState && forEachEntry(oldState!.fileInfos, (info, sourceFilePath) => info.affectsGlobalScope && !state.fileInfos.has(sourceFilePath))) {
BuilderState.getAllFilesExcludingDefaultLibraryFile(state, newProgram, /*firstSourceFile*/ undefined)
Expand Down Expand Up @@ -448,7 +451,8 @@ namespace ts {
Debug.checkDefined(state.currentAffectedFilesSignatures),
cancellationToken,
computeHash,
state.currentAffectedFilesExportedModulesMap
state.currentAffectedFilesExportedModulesMap,
/*avoidInitializingSignature*/ true
);
return;
}
Expand Down Expand Up @@ -483,7 +487,8 @@ namespace ts {
Debug.checkDefined(state.currentAffectedFilesSignatures),
cancellationToken,
computeHash,
state.currentAffectedFilesExportedModulesMap
state.currentAffectedFilesExportedModulesMap,
/*avoidInitializingSignatures*/ true
);
// If not dts emit, nothing more to do
if (getEmitDeclarations(state.compilerOptions)) {
Expand Down Expand Up @@ -511,7 +516,7 @@ namespace ts {
function isChangedSignature(state: BuilderProgramState, path: Path) {
const newSignature = Debug.checkDefined(state.currentAffectedFilesSignatures).get(path);
const oldSignature = Debug.checkDefined(state.fileInfos.get(path)).signature;
return newSignature !== oldSignature;
return !oldSignature || newSignature !== oldSignature;
}

/**
Expand Down
55 changes: 43 additions & 12 deletions src/compiler/builderState.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,8 +59,15 @@ namespace ts {
* Cache of all the file names
*/
allFileNames?: readonly string[];
/**
* Avoid computing signatures when none is computed yet.
* It will still compute and compare signature when there is already an old one.
*/
avoidInitializingSignatures: boolean;
}

export const SHAPE_CHANGE_UNKNOWN = Symbol("shape change unknown");

export namespace BuilderState {
/**
* Information about the source file: Its version and optional signature from last emit
Expand Down Expand Up @@ -212,6 +219,9 @@ namespace ts {
// Ensure source files have parent pointers set
newProgram.getTypeChecker();

// Track how many files have been added
let newFiles = 0;

// Create the reference map, and set the file infos
for (const sourceFile of newProgram.getSourceFiles()) {
const version = Debug.checkDefined(sourceFile.version, "Program intended to be used with Builder should have source files with versions set");
Expand All @@ -230,13 +240,16 @@ namespace ts {
}
}
fileInfos.set(sourceFile.resolvedPath, { version, signature: oldInfo && oldInfo.signature, affectsGlobalScope: isFileAffectingGlobalScope(sourceFile) });

if(!oldInfo) newFiles++;
}

return {
fileInfos,
referencedMap,
exportedModulesMap,
hasCalledUpdateShapeSignature
hasCalledUpdateShapeSignature,
avoidInitializingSignatures: !useOldState || newFiles > 3,
};
}

Expand All @@ -258,13 +271,14 @@ namespace ts {
referencedMap: state.referencedMap && new Map(state.referencedMap),
exportedModulesMap: state.exportedModulesMap && new Map(state.exportedModulesMap),
hasCalledUpdateShapeSignature: new Set(state.hasCalledUpdateShapeSignature),
avoidInitializingSignatures: state.avoidInitializingSignatures,
};
}

/**
* Gets the files affected by the path from the program
*/
export function getFilesAffectedBy(state: BuilderState, programOfThisState: Program, path: Path, cancellationToken: CancellationToken | undefined, computeHash: ComputeHash, cacheToUpdateSignature?: ESMap<Path, string>, exportedModulesMapCache?: ComputingExportedModulesMap): readonly SourceFile[] {
export function getFilesAffectedBy(state: BuilderState, programOfThisState: Program, path: Path, cancellationToken: CancellationToken | undefined, computeHash: ComputeHash, cacheToUpdateSignature?: ESMap<Path, string | undefined>, exportedModulesMapCache?: ComputingExportedModulesMap): readonly SourceFile[] {
// Since the operation could be cancelled, the signatures are always stored in the cache
// They will be committed once it is safe to use them
// eg when calling this api from tsserver, if there is no cancellation of the operation
Expand All @@ -275,11 +289,12 @@ namespace ts {
return emptyArray;
}

if (!updateShapeSignature(state, programOfThisState, sourceFile, signatureCache, cancellationToken, computeHash, exportedModulesMapCache)) {
const updateResult = updateShapeSignature(state, programOfThisState, sourceFile, signatureCache, cancellationToken, computeHash, exportedModulesMapCache, state.avoidInitializingSignatures);
if (!updateResult) {
return [sourceFile];
}

const result = (state.referencedMap ? getFilesAffectedByUpdatedShapeWhenModuleEmit : getFilesAffectedByUpdatedShapeWhenNonModuleEmit)(state, programOfThisState, sourceFile, signatureCache, cancellationToken, computeHash, exportedModulesMapCache);
const result = (state.referencedMap ? getFilesAffectedByUpdatedShapeWhenModuleEmit : getFilesAffectedByUpdatedShapeWhenNonModuleEmit)(state, programOfThisState, sourceFile, signatureCache, cancellationToken, computeHash, exportedModulesMapCache, state.avoidInitializingSignatures || updateResult !== true);
if (!cacheToUpdateSignature) {
// Commit all the signatures in the signature cache
updateSignaturesFromCache(state, signatureCache);
Expand All @@ -291,7 +306,7 @@ namespace ts {
* Updates the signatures from the cache into state's fileinfo signatures
* This should be called whenever it is safe to commit the state of the builder
*/
export function updateSignaturesFromCache(state: BuilderState, signatureCache: ESMap<Path, string>) {
export function updateSignaturesFromCache(state: BuilderState, signatureCache: ESMap<Path, string | undefined>) {
signatureCache.forEach((signature, path) => updateSignatureOfFile(state, signature, path));
}

Expand All @@ -303,7 +318,7 @@ namespace ts {
/**
* Returns if the shape of the signature has changed since last emit
*/
export function updateShapeSignature(state: Readonly<BuilderState>, programOfThisState: Program, sourceFile: SourceFile, cacheToUpdateSignature: ESMap<Path, string>, cancellationToken: CancellationToken | undefined, computeHash: ComputeHash, exportedModulesMapCache?: ComputingExportedModulesMap) {
export function updateShapeSignature(state: Readonly<BuilderState>, programOfThisState: Program, sourceFile: SourceFile, cacheToUpdateSignature: ESMap<Path, string | undefined>, cancellationToken: CancellationToken | undefined, computeHash: ComputeHash, exportedModulesMapCache: ComputingExportedModulesMap | undefined, avoidInitializingSignature: boolean): (boolean | typeof SHAPE_CHANGE_UNKNOWN) {
Debug.assert(!!sourceFile);
Debug.assert(!exportedModulesMapCache || !!state.exportedModulesMap, "Compute visible to outside map only if visibleToOutsideReferencedMap present in the state");

Expand All @@ -326,6 +341,14 @@ namespace ts {
}
}
else {
if (prevSignature === undefined && avoidInitializingSignature) {
if (exportedModulesMapCache) {
const references = state.referencedMap ? state.referencedMap.get(sourceFile.resolvedPath) : undefined;
exportedModulesMapCache.set(sourceFile.resolvedPath, references || false);
}
cacheToUpdateSignature.set(sourceFile.resolvedPath, undefined);
return SHAPE_CHANGE_UNKNOWN;
}
const emitOutput = getFileEmitOutput(
programOfThisState,
sourceFile,
Expand All @@ -352,7 +375,7 @@ namespace ts {
}
cacheToUpdateSignature.set(sourceFile.resolvedPath, latestSignature);

return !prevSignature || latestSignature !== prevSignature;
return prevSignature ? latestSignature !== prevSignature : SHAPE_CHANGE_UNKNOWN;
}

/**
Expand Down Expand Up @@ -524,7 +547,7 @@ namespace ts {
/**
* When program emits modular code, gets the files affected by the sourceFile whose shape has changed
*/
function getFilesAffectedByUpdatedShapeWhenModuleEmit(state: BuilderState, programOfThisState: Program, sourceFileWithUpdatedShape: SourceFile, cacheToUpdateSignature: ESMap<Path, string>, cancellationToken: CancellationToken | undefined, computeHash: ComputeHash, exportedModulesMapCache: ComputingExportedModulesMap | undefined) {
function getFilesAffectedByUpdatedShapeWhenModuleEmit(state: BuilderState, programOfThisState: Program, sourceFileWithUpdatedShape: SourceFile, cacheToUpdateSignature: ESMap<Path, string | undefined>, cancellationToken: CancellationToken | undefined, computeHash: ComputeHash, exportedModulesMapCache: ComputingExportedModulesMap | undefined, avoidInitializingSignatures: boolean) {
if (isFileAffectingGlobalScope(sourceFileWithUpdatedShape)) {
return getAllFilesExcludingDefaultLibraryFile(state, programOfThisState, sourceFileWithUpdatedShape);
}
Expand All @@ -541,14 +564,22 @@ namespace ts {

// Start with the paths this file was referenced by
seenFileNamesMap.set(sourceFileWithUpdatedShape.resolvedPath, sourceFileWithUpdatedShape);
const queue = getReferencedByPaths(state, sourceFileWithUpdatedShape.resolvedPath);
const queue = getReferencedByPaths(state, sourceFileWithUpdatedShape.resolvedPath).map(path => ({ path, avoidInitializingSignatures }));
while (queue.length > 0) {
const currentPath = queue.pop()!;
const { path: currentPath, avoidInitializingSignatures } = queue.pop()!;
// avoidInitializingSignatures: false will be always before avoidInitializingSignatures: true in queue
// so avoidInitializingSignatures: false is preferred over avoidInitializingSignatures: true
if (!seenFileNamesMap.has(currentPath)) {
const currentSourceFile = programOfThisState.getSourceFileByPath(currentPath)!;
seenFileNamesMap.set(currentPath, currentSourceFile);
if (currentSourceFile && updateShapeSignature(state, programOfThisState, currentSourceFile, cacheToUpdateSignature, cancellationToken, computeHash, exportedModulesMapCache)) {
queue.push(...getReferencedByPaths(state, currentSourceFile.resolvedPath));
if (currentSourceFile) {
const updateResult = updateShapeSignature(state, programOfThisState, currentSourceFile, cacheToUpdateSignature, cancellationToken, computeHash, exportedModulesMapCache, avoidInitializingSignatures);
if (updateResult) {
const avoidInitializingSignaturesForReferencedPaths = avoidInitializingSignatures || updateResult !== true;
getReferencedByPaths(state, currentSourceFile.resolvedPath).forEach(path => {
queue.push({ path, avoidInitializingSignatures: avoidInitializingSignaturesForReferencedPaths });
});
}
}
}
}
Expand Down
56 changes: 49 additions & 7 deletions src/testRunner/unittests/tsbuild/helpers.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -359,8 +359,10 @@ interface Symbol {
}
else if (incrementalBuildText !== cleanBuildText) {
// Verify build info without affectedFilesPendingEmit
const { buildInfo: incrementalBuildInfo, affectedFilesPendingEmit: incrementalBuildAffectedFilesPendingEmit } = getBuildInfoForIncrementalCorrectnessCheck(incrementalBuildText);
const { buildInfo: cleanBuildInfo, affectedFilesPendingEmit: incrementalAffectedFilesPendingEmit } = getBuildInfoForIncrementalCorrectnessCheck(cleanBuildText);
const { buildInfo: incrementalBuildInfo, affectedFilesPendingEmit: incrementalBuildAffectedFilesPendingEmit, signatures: incrementalSignatures, exportedModulesMap: incrementalExportedModulesMap } =
getBuildInfoForIncrementalCorrectnessCheck(incrementalBuildText);
const { buildInfo: cleanBuildInfo, affectedFilesPendingEmit: incrementalAffectedFilesPendingEmit, signatures: cleanSignatures, exportedModulesMap: cleanExportedModulesMap } =
getBuildInfoForIncrementalCorrectnessCheck(cleanBuildText);
verifyTextEqual(incrementalBuildInfo, cleanBuildInfo, descrepancyInClean, `TsBuild info text without affectedFilesPendingEmit ${subScenario}:: ${outputFile}::\nIncremental buildInfoText:: ${incrementalBuildText}\nClean buildInfoText:: ${cleanBuildText}`);
// Verify that incrementally pending affected file emit are in clean build since clean build can contain more files compared to incremental depending of noEmitOnError option
if (incrementalBuildAffectedFilesPendingEmit && descrepancyInClean === undefined) {
Expand All @@ -372,6 +374,24 @@ interface Symbol {
expectedIndex++;
});
}
if (incrementalSignatures && cleanSignatures) {
for (let i = 0; i < incrementalSignatures.length; i++) {
if (!incrementalSignatures[i]) cleanSignatures[i] = undefined;
}
}
assert.deepEqual(incrementalSignatures, cleanSignatures);
if (incrementalExportedModulesMap && cleanExportedModulesMap) {
const keys = Object.keys(incrementalExportedModulesMap);
if (keys.length > 0) {
assert.containsAllKeys(cleanExportedModulesMap, keys);
for (const key of keys) {
assert.includeMembers(cleanExportedModulesMap[key], incrementalExportedModulesMap[key]);
}
}
}
else {
assert.deepEqual(incrementalExportedModulesMap, cleanExportedModulesMap);
}
}
}

Expand All @@ -397,21 +417,43 @@ interface Symbol {
});
}

function getBuildInfoForIncrementalCorrectnessCheck(text: string | undefined): { buildInfo: string | undefined; affectedFilesPendingEmit?: ProgramBuildInfo["affectedFilesPendingEmit"]; } {
function getBuildInfoForIncrementalCorrectnessCheck(text: string | undefined): {
buildInfo: string | undefined;
affectedFilesPendingEmit?: ProgramBuildInfo["affectedFilesPendingEmit"];
signatures?: BuilderState.FileInfo["signature"][];
exportedModulesMap?: MapLike<string[]>;
} {
const buildInfo = text ? getBuildInfo(text) : undefined;
if (!buildInfo?.program) return { buildInfo: text };
// Ignore noEmit since that shouldnt be reason to emit the tsbuild info and presence of it in the buildinfo file does not matter
const { program: { affectedFilesPendingEmit, options: { noEmit, ...optionsRest}, ...programRest }, ...rest } = buildInfo;
const { program: { affectedFilesPendingEmit, options: { noEmit, ...optionsRest}, fileInfos, referencedMap, exportedModulesMap, fileIdsList, fileNames, ...programRest }, ...rest } = buildInfo;
const signatures: BuilderState.FileInfo["signature"][] = [];
return {
buildInfo: getBuildInfoText({
buildInfo: JSON.stringify({
...rest,
program: {
fileInfos: fileInfos.map(fileInfo => {
const { signature, ...remainingFileInfo } = fileInfo;
return remainingFileInfo;
}),
options: optionsRest,
fileNames,
referencedMap: referencedMap && expandMap(referencedMap),
...programRest
}
},
}),
affectedFilesPendingEmit
affectedFilesPendingEmit,
signatures,
exportedModulesMap: exportedModulesMap && expandMap(exportedModulesMap)
};

function expandMap(map: ProgramBuildInfoReferencedMap) {
const result: MapLike<string[]> = {};
for(const [fileId, listId] of map) {
result[fileNames[fileId]] = fileIdsList![listId].map(id => fileNames[id]);
}
return result;
}
}

export enum CleanBuildDescrepancy {
Expand Down
Loading