Skip to content

Commit

Permalink
Refactor and improve dep graph naming/typing
Browse files Browse the repository at this point in the history
  • Loading branch information
@webpro
webpro committed Jun 8, 2024
1 parent deb3b9c commit 28f05f0
Show file tree
Hide file tree
Showing 13 changed files with 223 additions and 228 deletions.
18 changes: 6 additions & 12 deletions packages/knip/src/ProjectPrincipal.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,7 @@ import type { ReferencedDependencies } from './WorkspaceWorker.js';
import { getCompilerExtensions } from './compilers/index.js';
import type { AsyncCompilers, SyncCompilers } from './compilers/types.js';
import { ANONYMOUS, DEFAULT_EXTENSIONS, FOREIGN_FILE_EXTENSIONS } from './constants.js';
import type {
SerializableExport,
SerializableExportMember,
SerializableFile,
SerializableMap,
UnresolvedImport,
} from './types/serializable-map.js';
import type { DependencyGraph, Export, ExportMember, FileNode, UnresolvedImport } from './types/dependency-graph.js';
import type { BoundSourceFile } from './typescript/SourceFile.js';
import type { SourceFileManager } from './typescript/SourceFileManager.js';
import { createHosts } from './typescript/createHosts.js';
Expand Down Expand Up @@ -74,7 +68,7 @@ export class ProjectPrincipal {
isSkipLibs: boolean;
isWatch: boolean;

cache: CacheConsultant<SerializableFile>;
cache: CacheConsultant<FileNode>;

// @ts-expect-error Don't want to ignore this, but we're not touching this until after init()
backend: {
Expand Down Expand Up @@ -308,7 +302,7 @@ export class ProjectPrincipal {
this.backend.fileManager.sourceFileCache.delete(filePath);
}

public findUnusedMembers(filePath: string, members: SerializableExportMember[]) {
public findUnusedMembers(filePath: string, members: ExportMember[]) {
if (!this.findReferences) {
const languageService = ts.createLanguageService(this.backend.languageServiceHost, ts.createDocumentRegistry());
this.findReferences = timerify(languageService.findReferences);
Expand All @@ -322,7 +316,7 @@ export class ProjectPrincipal {
});
}

public hasExternalReferences(filePath: string, exportedItem: SerializableExport) {
public hasExternalReferences(filePath: string, exportedItem: Export) {
if (exportedItem.jsDocTags.has('@public')) return false;

if (!this.findReferences) {
Expand All @@ -341,8 +335,8 @@ export class ProjectPrincipal {
return externalRefs.length > 0;
}

reconcileCache(serializableMap: SerializableMap) {
for (const [filePath, file] of serializableMap.entries()) {
reconcileCache(graph: DependencyGraph) {
for (const [filePath, file] of graph.entries()) {
const fd = this.cache.getFileDescriptor(filePath);
if (!fd?.meta) continue;
fd.meta.data = _serialize(file);
Expand Down
105 changes: 47 additions & 58 deletions packages/knip/src/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,20 +14,20 @@ import { getFilteredScripts } from './manifest/helpers.js';
import watchReporter from './reporters/watch.js';
import type { CommandLineOptions } from './types/cli.js';
import type {
SerializableExport,
SerializableExportMember,
SerializableFile,
SerializableImportMap,
SerializableImports,
SerializableMap,
} from './types/serializable-map.js';
DependencyGraph,
Export,
ExportMember,
FileNode,
ImportDetails,
ImportMap,
} from './types/dependency-graph.js';
import { debugLog, debugLogArray, debugLogObject } from './util/debug.js';
import { addNsValues, addValues, createFileNode } from './util/dependency-graph.js';
import { isFile } from './util/fs.js';
import { _glob, negate } from './util/glob.js';
import { getGitIgnoredFn } from './util/globby.js';
import { getHandler } from './util/handle-dependency.js';
import { getIsIdentifierReferencedHandler } from './util/is-identifier-referenced.js';
import { addNsValues, addValues } from './util/map.js';
import { getEntryPathFromManifest, getPackageNameFromModuleSpecifier } from './util/modules.js';
import { dirname, join, toPosix } from './util/path.js';
import { findMatch } from './util/regex.js';
Expand Down Expand Up @@ -243,47 +243,35 @@ export const main = async (unresolvedConfiguration: CommandLineOptions) => {

debugLog('*', `Created ${principals.length} programs for ${workspaces.length} workspaces`);

const serializableMap: SerializableMap = new Map();
const graph: DependencyGraph = new Map();
const analyzedFiles = new Set<string>();
const unreferencedFiles = new Set<string>();
const entryPaths = new Set<string>();

const getFile = (filePath: string) => serializableMap.get(filePath) ?? ({} as Partial<SerializableFile>);
const getFileNode = (filePath: string) => graph.get(filePath) ?? createFileNode();

const handleReferencedDependency = getHandler(collector, deputy, chief);

const updateImports = (importedModule: SerializableImports, importItems: SerializableImports) => {
const updateImportDetails = (importedModule: ImportDetails, importItems: ImportDetails) => {
for (const id of importItems.refs) importedModule.refs.add(id);
for (const [id, v] of importItems.imported.entries()) addValues(importedModule.imported, id, v);
for (const [id, v] of importItems.importedNs.entries()) addValues(importedModule.importedNs, id, v);
for (const [id, v] of importItems.importedAs.entries()) addNsValues(importedModule.importedAs, id, v);
for (const [id, v] of importItems.reExportedNs.entries()) addValues(importedModule.reExportedNs, id, v);
for (const [id, v] of importItems.importedNs.entries()) addValues(importedModule.importedNs, id, v);
for (const [id, v] of importItems.reExported.entries()) addValues(importedModule.reExported, id, v);
for (const [id, v] of importItems.reExportedAs.entries()) addNsValues(importedModule.reExportedAs, id, v);
for (const [id, v] of importItems.reExportedBy.entries()) addValues(importedModule.reExportedBy, id, v);
};

const updateImported = (filePath: string, importItems: SerializableImports) => {
const file = getFile(filePath);

if (!file.imported) file.imported = importItems;
else updateImports(file.imported, importItems);

serializableMap.set(filePath, file);
for (const [id, v] of importItems.reExportedNs.entries()) addValues(importedModule.reExportedNs, id, v);
};

const setInternalImports = (filePath: string, internalImports: SerializableImportMap) => {
for (const [specifierFilePath, importItems] of internalImports.entries()) {
// Update imports for current module
const file = getFile(filePath);

if (file.imports) {
const importedFile = file.imports.internal.get(specifierFilePath);
if (!importedFile) file.imports.internal.set(specifierFilePath, importItems);
else updateImports(importedFile, importItems);
}
const updateImportMap = (file: FileNode, importMap: ImportMap) => {
for (const [importedFilePath, importDetails] of importMap.entries()) {
const importedFileImports = file.imports.internal.get(importedFilePath);
if (!importedFileImports) file.imports.internal.set(importedFilePath, importDetails);
else updateImportDetails(importedFileImports, importDetails);

// Update import stats for imported module
updateImported(specifierFilePath, importItems);
const importedFile = getFileNode(importedFilePath);
if (!importedFile.imported) importedFile.imported = importDetails;
else updateImportDetails(importedFile.imported, importDetails);
graph.set(importedFilePath, importedFile);
}
};

Expand Down Expand Up @@ -315,19 +303,17 @@ export const main = async (unresolvedConfiguration: CommandLineOptions) => {
getPrincipalByFilePath
);

const file = getFile(filePath);
const file = getFileNode(filePath);

file.imports = imports;
file.exports = exports;
file.scripts = scripts;
file.traceRefs = traceRefs;

if (imports?.internal) {
file.internalImportCache = imports.internal;
setInternalImports(filePath, imports.internal);
}
updateImportMap(file, imports.internal);
file.internalImportCache = imports.internal;

serializableMap.set(filePath, file);
graph.set(filePath, file);

// Handle scripts here since they might lead to more entry files
if (scripts && scripts.size > 0) {
Expand Down Expand Up @@ -377,17 +363,17 @@ export const main = async (unresolvedConfiguration: CommandLineOptions) => {
for (const filePath of principal.getUnreferencedFiles()) unreferencedFiles.add(filePath);
for (const filePath of principal.entryPaths) entryPaths.add(filePath);

principal.reconcileCache(serializableMap);
principal.reconcileCache(graph);

// Delete principals including TS programs for GC, except when we still need its `LS.findReferences`
if (!isIsolateWorkspaces && isSkipLibs && !isWatch) factory.deletePrincipal(principal);
}

if (isIsolateWorkspaces) for (const principal of principals) factory.deletePrincipal(principal);

const isIdentifierReferenced = getIsIdentifierReferencedHandler(serializableMap, entryPaths);
const isIdentifierReferenced = getIsIdentifierReferencedHandler(graph, entryPaths);

const isExportedItemReferenced = (exportedItem: SerializableExport | SerializableExportMember) =>
const isExportedItemReferenced = (exportedItem: Export | ExportMember) =>
exportedItem.refs > 0 &&
(typeof chief.config.ignoreExportsUsedInFile === 'object'
? exportedItem.type !== 'unknown' && !!chief.config.ignoreExportsUsedInFile[exportedItem.type]
Expand All @@ -397,7 +383,7 @@ export const main = async (unresolvedConfiguration: CommandLineOptions) => {
if (isReportValues || isReportTypes) {
streamer.cast('Connecting the dots...');

for (const [filePath, file] of serializableMap.entries()) {
for (const [filePath, file] of graph.entries()) {
const exportItems = file.exports?.exported;

if (!exportItems || exportItems.size === 0) continue;
Expand Down Expand Up @@ -490,7 +476,7 @@ export const main = async (unresolvedConfiguration: CommandLineOptions) => {
}
}

const [hasStrictlyNsReferences, namespace] = getHasStrictlyNsReferences(serializableMap, importsForExport);
const [hasStrictlyNsReferences, namespace] = getHasStrictlyNsReferences(graph, importsForExport);

const isType = ['enum', 'type', 'interface'].includes(exportedItem.type);

Expand Down Expand Up @@ -521,7 +507,7 @@ export const main = async (unresolvedConfiguration: CommandLineOptions) => {
}
}

for (const [filePath, file] of serializableMap.entries()) {
for (const [filePath, file] of graph.entries()) {
const ws = chief.findWorkspaceByFilePath(filePath);

if (ws) {
Expand Down Expand Up @@ -573,15 +559,19 @@ export const main = async (unresolvedConfiguration: CommandLineOptions) => {
};

if (isWatch) {
const cacheLocation = CacheConsultant.getCacheLocation();

watch('.', { recursive: true }, async (eventType, filename) => {
debugLog('*', `(raw) ${eventType} ${filename}`);

if (filename) {
const startTime = performance.now();
const filePath = join(cwd, toPosix(filename));

if (filename.startsWith(CacheConsultant.getCacheLocation())) return;
if (isGitIgnored(filePath)) return;
if (filename.startsWith(cacheLocation) || filename.startsWith('.git/') || isGitIgnored(filePath)) {
debugLog('*', `ignoring ${eventType} ${filename}`);
return;
}

const workspace = chief.findWorkspaceByFilePath(filePath);
if (workspace) {
Expand Down Expand Up @@ -615,31 +605,30 @@ export const main = async (unresolvedConfiguration: CommandLineOptions) => {

if (event === 'added' || event === 'deleted') {
// Flush, any file might contain (un)resolved imports to added/deleted files
serializableMap.clear();
graph.clear();
for (const filePath of filePaths) analyzeSourceFile(filePath, principal);
} else {
for (const filePath in serializableMap) {
for (const [filePath, file] of graph) {
if (filePaths.includes(filePath)) {
// Reset dep graph
const file = serializableMap.get(filePath);
if (file) file.imported = undefined;
file.imported = undefined;
} else {
// Remove files no longer referenced
serializableMap.delete(filePath);
graph.delete(filePath);
analyzedFiles.delete(filePath);
cachedUnusedFiles.add(filePath);
}
}

// Add existing files that were not yet part of the program
for (const filePath of filePaths)
if (!serializableMap.has(filePath)) analyzeSourceFile(filePath, principal);
for (const filePath of filePaths) if (!graph.has(filePath)) analyzeSourceFile(filePath, principal);

if (!cachedUnusedFiles.has(filePath)) analyzeSourceFile(filePath, principal);

// Rebuild dep graph
for (const filePath of filePaths) {
const cache = serializableMap.get(filePath)?.internalImportCache;
if (cache) setInternalImports(filePath, cache);
const file = graph.get(filePath);
if (file?.internalImportCache) updateImportMap(file, file.internalImportCache);
}
}

Expand Down
74 changes: 74 additions & 0 deletions packages/knip/src/types/dependency-graph.ts
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
import type ts from 'typescript';
import type { Fix, Fixes } from './exports.js';
import type { IssueSymbol, SymbolType } from './issues.js';

type Identifier = string;
type FilePath = string;
type NamespaceOrAlias = string;

type Reference = string;
type References = Set<Reference>;

type Tags = Set<string>;

export type IdToFileMap = Map<Identifier, Set<FilePath>>;
export type IdToNsToFileMap = Map<Identifier, Map<NamespaceOrAlias, Set<FilePath>>>;

export type ImportDetails = {
refs: References;
imported: IdToFileMap;
importedAs: IdToNsToFileMap;
importedNs: IdToFileMap;
reExported: IdToFileMap;
reExportedAs: IdToNsToFileMap;
reExportedNs: IdToFileMap;
};

export type ImportMap = Map<FilePath, ImportDetails>;

export type UnresolvedImport = { specifier: string; pos?: number; line?: number; col?: number };

export interface Export {
identifier: Identifier;
pos: number;
line: number;
col: number;
type: SymbolType;
members: ExportMember[];
jsDocTags: Tags;
refs: number;
fixes: Fixes;
symbol?: ts.Symbol;
}

export type ExportMember = {
identifier: Identifier;
pos: number;
line: number;
col: number;
type: SymbolType;
refs: number;
fix: Fix;
symbol?: ts.Symbol;
jsDocTags: Tags;
};

export type ExportMap = Map<Identifier, Export>;

export type FileNode = {
imports: {
internal: ImportMap;
external: Set<string>;
unresolved: Set<UnresolvedImport>;
};
exports: {
exported: ExportMap;
duplicate: Iterable<Array<IssueSymbol>>;
};
scripts: Set<string>;
imported?: ImportDetails;
internalImportCache?: ImportMap;
traceRefs: References;
};

export type DependencyGraph = Map<FilePath, FileNode>;
Loading

0 comments on commit 28f05f0

Please sign in to comment.