From 2d4612fe3765410f029fe8c18f465d1d4be1c0f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Mangeonjean?= Date: Wed, 6 Nov 2024 19:54:59 +0100 Subject: [PATCH] feat: generate combination packages instead of putting all common stuff in the main package, leading to it depending on xterm --- rollup/rollup-metadata-plugin.ts | 104 +++++++++++--- rollup/rollup.config.ts | 238 ++++++++++++++++++++++++++++--- 2 files changed, 305 insertions(+), 37 deletions(-) diff --git a/rollup/rollup-metadata-plugin.ts b/rollup/rollup-metadata-plugin.ts index 804af608..e6654fb2 100644 --- a/rollup/rollup-metadata-plugin.ts +++ b/rollup/rollup-metadata-plugin.ts @@ -3,12 +3,14 @@ import { builtinModules } from 'module' interface Group { name: string + publicName?: string modules: Set entrypoints: Set dependencies: Set groupDependencies: Set // A greater priority means the that this group will be chosen for module present in multiple groups priority: number + isCombination: boolean } interface GroupResult { @@ -16,6 +18,7 @@ interface GroupResult { directDependencies: Set exclusiveModules: Set entrypoints: Set + isCombination: boolean } interface Options { @@ -30,16 +33,25 @@ interface Options { group: GroupResult moduleGroupName: Map otherDependencies: Set + otherModules: Set options: OutputOptions bundle: OutputBundle } ): void | Promise + // Should shared modules be put in new combination groups + // By default, they are put in the group with the highest priority + generateCombinationGroups?: boolean + getCombinedGroup?: (names: string[]) => { name: string; publicName?: string } + minCompinedGroupSize?: number } export default ({ handle, getGroup = () => ({ name: 'main' }), - stage = 'generateBundle' + stage = 'generateBundle', + generateCombinationGroups = false, + getCombinedGroup = (names) => ({ name: names.reduce((a, b) => `${a}_${b}`) }), + minCompinedGroupSize = 10 }: Options): Plugin => ({ name: 'generate-metadata', [stage]: async function (this: PluginContext, options: OutputOptions, bundle: OutputBundle) { @@ -104,11 +116,13 @@ export default ({ if (!groups.has(groupName)) { groups.set(groupName, { entrypoints: new Set(), + publicName, name: groupName, modules: new Set(), dependencies: new Set(), groupDependencies: new Set(), - priority: priority ?? 0 + priority: priority ?? 0, + isCombination: false }) } const group = groups.get(groupName)! @@ -121,14 +135,6 @@ export default ({ } } - for (const group of groups.values()) { - group.groupDependencies = new Set( - Array.from(group.dependencies) - .map((d) => groupByPublicName.get(d)?.name) - .filter((g): g is string => g != null) - ) - } - const moduleGroups = new Map() for (const [_, group] of groups.entries()) { for (const module of group.modules) { @@ -140,18 +146,68 @@ export default ({ } const moduleGroup = new Map() - for (const [id, groups] of moduleGroups.entries()) { + const combinedModuleGroup = new Map() + for (const [id, currentModuleGroups] of moduleGroups.entries()) { // Find a group that everyone depends on - const greatestPriority = Math.max(...groups.map((g) => g.priority)) - const priorityGroups = groups.filter((g) => g.priority >= greatestPriority) - moduleGroup.set( - id, - priorityGroups.find((group) => - priorityGroups - .filter((ogroup) => ogroup !== group) - .every((ogroup) => ogroup.groupDependencies.has(group.name)) - ) ?? null + const greatestPriority = Math.max(...currentModuleGroups.map((g) => g.priority)) + const priorityGroups = currentModuleGroups.filter((g) => g.priority >= greatestPriority) + + const groupThatEveryOneDependsOn = priorityGroups.find((group) => + priorityGroups + .filter((ogroup) => ogroup !== group) + .every((ogroup) => ogroup.groupDependencies.has(group.name)) ) + moduleGroup.set(id, groupThatEveryOneDependsOn ?? null) + + if (generateCombinationGroups && groupThatEveryOneDependsOn == null) { + const newGroup = getCombinedGroup(priorityGroups.map((g) => g.name)) + let group = groups.get(newGroup.name) + if (group == null) { + // The combination group doesn't exists yet + group = { + entrypoints: new Set(), + name: newGroup.name, + publicName: newGroup.publicName, + modules: new Set(), + dependencies: new Set(), + groupDependencies: new Set(), + priority: 0, + isCombination: true + } + groups.set(newGroup.name, group) + if (newGroup.publicName != null) { + groupByPublicName.set(newGroup.publicName, group) + } + } + group.modules.add(id) + combinedModuleGroup.set(id, group) + } + } + + for (const group of groups.values()) { + group.groupDependencies = new Set( + Array.from(group.dependencies) + .map((d) => groupByPublicName.get(d)?.name) + .filter((g): g is string => g != null) + ) + } + + for (const [id, group] of combinedModuleGroup) { + if (group.modules.size < minCompinedGroupSize) { + // if the combined group is too small and if it doesn't have direct dependencies, remove it + groups.delete(group.name) + } else { + moduleGroup.set(id, group) + + if (group.publicName != null) { + const previousGroups = moduleGroups.get(id) + if (previousGroups != null) { + for (const previousGroup of previousGroups) { + previousGroup.dependencies.add(group.publicName) + } + } + } + } } const moduleGroupName = new Map( @@ -172,7 +228,8 @@ export default ({ directDependencies, entrypoints: group.entrypoints, exclusiveModules, - name + name, + isCombination: group.isCombination } }) @@ -181,10 +238,14 @@ export default ({ .map((set) => Array.from(set)) .flat() ) + const otherModules = new Set(Array.from(moduleGroups.keys())) for (const group of groupResults) { for (const directDependency of group.directDependencies) { otherDependencies.delete(directDependency) } + for (const exclusiveModule of group.exclusiveModules) { + otherModules.delete(exclusiveModule) + } } await Promise.all( @@ -193,6 +254,7 @@ export default ({ group, moduleGroupName, otherDependencies, + otherModules, options, bundle }) diff --git a/rollup/rollup.config.ts b/rollup/rollup.config.ts index b2035538..bb3c4308 100644 --- a/rollup/rollup.config.ts +++ b/rollup/rollup.config.ts @@ -832,6 +832,25 @@ export default (args: Record): rollup.RollupOptions[] => { } }, metadataPlugin({ + generateCombinationGroups: true, + getCombinedGroup(names) { + const name = names + .slice() + .sort() + .map((groupName) => { + const match = /^(.*):(.*)$/.exec(groupName) + if (match == null) { + return groupName + } + const [_, _category, name] = match + return name + }) + .join('-') + return { + name: `common:${name}`, + publicName: `@codingame/monaco-vscode-${name}-common` + } + }, // generate package.json and service-override packages getGroup(id: string, options) { const serviceOverrideDir = nodePath.resolve(options.dir!, 'service-override') @@ -864,11 +883,13 @@ export default (args: Record): rollup.RollupOptions[] => { priority: 1 } }, - async handle({ group, moduleGroupName, otherDependencies, bundle }) { + async handle({ group, moduleGroupName, otherDependencies, otherModules, bundle }) { const customResolutionPlugin = ({ - customLoad + customLoad, + forMain = false }: { customLoad: (id: string) => string | undefined + forMain?: boolean }) => { name: 'custom-resolution', @@ -899,6 +920,9 @@ export default (args: Record): rollup.RollupOptions[] => { nodePath.relative(DIST_DIR_MAIN, resolvedWithExtension) ) if (pathFromRoot.startsWith('external/') && !isExclusive) { + if (forMain) { + return undefined + } return { external: true, id: `vscode/${pathFromRoot}` @@ -916,6 +940,10 @@ export default (args: Record): rollup.RollupOptions[] => { const importFromGroup = isVscodeFile ? (moduleGroupName.get(resolved) ?? 'main') : 'main' + if (importFromGroup === 'main' && forMain) { + return undefined + } + const importFromModule = getPackageFromGroupName(importFromGroup) // Those modules will be imported from external monaco-vscode-api let externalResolved = resolved.startsWith(VSCODE_SRC_DIST_DIR) @@ -964,7 +992,7 @@ export default (args: Record): rollup.RollupOptions[] => { (d) => !ALLOWED_MAIN_DEPENDENCIES.has(d) ) if (notAllowedDependencies.length > 0) { - this.error( + this.warn( `Not allowed dependencies detected in main package: ${notAllowedDependencies.join(', ')}` ) } @@ -1067,6 +1095,52 @@ export default (args: Record): rollup.RollupOptions[] => { source: JSON.stringify(packageJson, null, 2), type: 'asset' }) + + const groupBundle = await rollup.rollup({ + input: Array.from([...group.exclusiveModules, ...otherModules]), + external, + treeshake: false, + plugins: [ + importMetaAssets({ + include: ['**/*.ts', '**/*.js'] + }), + nodeResolve({ + extensions: EXTENSIONS + }), + customResolutionPlugin({ + forMain: true, + customLoad() { + return undefined + } + }) + ] + }) + const output = await groupBundle.generate({ + preserveModules: true, + preserveModulesRoot: nodePath.resolve(DIST_DIR, 'main'), + minifyInternalExports: false, + assetFileNames: 'assets/[name][extname]', + format: 'esm', + dir: nodePath.resolve(DIST_DIR, 'main'), + entryFileNames: '[name].js', + chunkFileNames: '[name].js', + hoistTransitiveImports: false + }) + for (const chunkOrAsset of output.output) { + if (chunkOrAsset.type === 'chunk') { + this.emitFile({ + type: 'prebuilt-chunk', + code: chunkOrAsset.code, + fileName: chunkOrAsset.fileName, + exports: chunkOrAsset.exports, + map: chunkOrAsset.map ?? undefined, + sourcemapFileName: chunkOrAsset.sourcemapFileName ?? undefined + }) + } + bundle[chunkOrAsset.fileName] = chunkOrAsset + } + + await groupBundle.close() } else if (group.name === 'editor.api') { const directory = nodePath.resolve(DIST_DIR, 'editor-api') @@ -1181,6 +1255,123 @@ export default (args: Record): rollup.RollupOptions[] => { for (const exclusiveModule of group.exclusiveModules) { delete bundle[nodePath.relative(DIST_DIR_MAIN, exclusiveModule)] } + } else if (group.isCombination) { + const [_, category, name] = /^(.*):(.*)$/.exec(group.name)! + + const directory = nodePath.resolve(DIST_DIR, `${category}-${name}`) + + await fs.promises.mkdir(directory, { + recursive: true + }) + + const packageJson: PackageJson = { + name: `@codingame/monaco-vscode-${name}-${category}`, + ...Object.fromEntries( + Object.entries(pkg).filter(([key]) => + ['version', 'keywords', 'author', 'license', 'repository', 'type'].includes(key) + ) + ), + private: false, + description: `${pkg.description} - ${name} ${category}`, + exports: { + '.': { + default: './empty.js' + }, + './vscode/*': { + default: './src/*.js' + } + } + } + + const groupBundle = await rollup.rollup({ + input: Array.from(group.exclusiveModules), + external, + treeshake: false, + plugins: [ + importMetaAssets({ + include: ['**/*.ts', '**/*.js'] + // assets are externals and this plugin is not able to ignore external assets + }), + nodeResolve({ + extensions: EXTENSIONS + }), + customResolutionPlugin({ + customLoad() { + return undefined + } + }), + { + name: 'bundle-generator', + generateBundle() { + const externalDependencies = Array.from(this.getModuleIds()).filter( + (id) => this.getModuleInfo(id)!.isExternal + ) + + const uniqueExternalDependencies = new Set( + externalDependencies.flatMap((dep) => { + const match = /((?:@[^/]+?\/)?[^/]*)(?:\/.*)?/.exec(dep) + if (match == null) { + return [] + } + return [match[1]!] + }) + ) + packageJson.dependencies = { + vscode: `npm:${pkg.name}@^${pkg.version}`, + ...Object.fromEntries( + Object.entries(pkg.dependencies).filter(([key]) => + uniqueExternalDependencies.has(key) + ) + ), + ...Object.fromEntries( + Array.from(uniqueExternalDependencies) + .filter((dep) => dep.startsWith('@codingame/monaco-vscode-')) + .map((dep) => { + return [dep, pkg.version] + }) + ) + } + + this.emitFile({ + fileName: 'empty.js', + needsCodeReference: false, + source: 'export {}', + type: 'asset' + }) + this.emitFile({ + fileName: 'package.json', + needsCodeReference: false, + source: JSON.stringify(packageJson, null, 2), + type: 'asset' + }) + } + } + ] + }) + const output = await groupBundle.write({ + preserveModules: true, + preserveModulesRoot: nodePath.resolve(DIST_DIR, 'main/vscode'), + minifyInternalExports: false, + assetFileNames: 'assets/[name][extname]', + format: 'esm', + dir: directory, + entryFileNames: '[name].js', + chunkFileNames: '[name].js', + hoistTransitiveImports: false + }) + await groupBundle.close() + + // remove exclusive files from main bundle to prevent them from being duplicated + for (const exclusiveModule of group.exclusiveModules) { + delete bundle[nodePath.relative(DIST_DIR_MAIN, exclusiveModule)] + } + + const assets = output.output + .filter((file): file is rollup.OutputAsset => file.type === 'asset') + .filter((file) => file.fileName !== 'package.json') + for (const asset of assets) { + delete bundle[asset.fileName] + } } else { const [_, category, name] = /^(.*):(.*)$/.exec(group.name)! @@ -1222,19 +1413,6 @@ export default (args: Record): rollup.RollupOptions[] => { } } : {}) - }, - dependencies: { - vscode: `npm:${pkg.name}@^${pkg.version}`, - ...Object.fromEntries( - Object.entries(pkg.dependencies).filter(([key]) => - group.directDependencies.has(key) - ) - ), - ...Object.fromEntries( - Array.from(group.directDependencies) - .filter((dep) => dep.startsWith('@codingame/monaco-vscode-')) - .map((dep) => [dep, pkg.version]) - ) } } @@ -1287,6 +1465,34 @@ export default (args: Record): rollup.RollupOptions[] => { { name: 'bundle-generator', generateBundle() { + const externalDependencies = Array.from(this.getModuleIds()).filter( + (id) => this.getModuleInfo(id)!.isExternal + ) + + const uniqueExternalDependencies = new Set( + externalDependencies.flatMap((dep) => { + const match = /((?:@[^/]+?\/)?[^/]*)(?:\/.*)?/.exec(dep) + if (match == null) { + return [] + } + return [match[1]!] + }) + ) + packageJson.dependencies = { + vscode: `npm:${pkg.name}@^${pkg.version}`, + ...Object.fromEntries( + Object.entries(pkg.dependencies).filter(([key]) => + uniqueExternalDependencies.has(key) + ) + ), + ...Object.fromEntries( + Array.from(uniqueExternalDependencies) + .filter((dep) => dep.startsWith('@codingame/monaco-vscode-')) + .map((dep) => { + return [dep, pkg.version] + }) + ) + } this.emitFile({ fileName: 'package.json', needsCodeReference: false,