cli: filter Theia extensions

Our generators currently collect any Theia extension installed in
node_modules and mount them into your application without leaving you
much of a choice.

An alternative could be to create your own generators, but this is a lot
of maintenance work.

This commit simplifies the process of controlling what ends up in your
applications by adding new loading strategies:

- all: Like before, use everything found in node_modules
- explicitDependenciesOnly: only use what is defined in your
  dependencies

This is configurable from an application package.json through the
theia.extensions.loading.strategy field.

In addition to those strategies, you can also specify an includes list
and an excludes list. Using those, you can define regular expressions to
filter extensions even more precisely. The matching will be done on the
candidate inversify modules path.

When using the `explicitDependenciesOnly` the includes list will be
evaluated as "in addition to explicit dependencies". If you want the
includes list to be absolute again, use the default 'all' strategy so
that only what's defined in includes is included.

Note that preventing an extension from having its inversify modules
loaded won't prevent it from being included in your bundles. Bundling
should also mostly work no matter what you exclude, but if another
extension was relying on a given Symbol it will most likely break at
runtime. In such a case it is your responsability to bind the missing
symbols using a custom Theia extension, specific to your use-cases.

Signed-off-by: Paul Maréchal <paul.marechal@ericsson.com>

dev-packages/application-manager/src/generator/abstract-generator.ts

@@ -54,13 +54,13 @@ export abstract class AbstractGenerator {
         if (modules.size === 0) {
             return '';
         }
-        const lines = Array.from(modules.keys()).map(moduleName => {
-            const invocation = `${fn}('${modules.get(moduleName)}')`;
-            if (fn === 'require') {
-                return `Promise.resolve(${invocation})`;
-            }
-            return invocation;
-        }).map(statement => `    .then(function () { return ${statement}.then(load) })`);
+        const lines = Array.from(modules.values(), modulePath => {
+            const invocation = `${fn}('${modulePath}')`;
+            const promiseStatement = fn === 'require'
+                ? `Promise.resolve(${invocation})`
+                : invocation; // the `import` function already returns a Promise
+            return `    .then(function () { return ${promiseStatement}.then(load); })`;
+        });
         return os.EOL + lines.join(os.EOL);
     }
 

dev-packages/application-manager/src/generator/backend-generator.ts

@@ -19,7 +19,7 @@ import { AbstractGenerator } from './abstract-generator';
 export class BackendGenerator extends AbstractGenerator {
 
     async generate(): Promise<void> {
-        const backendModules = this.pck.targetBackendModules;
+        const backendModules = this.pck.targetBackendModulesFiltered;
         await this.write(this.pck.backend('server.js'), this.compileServer(backendModules));
         await this.write(this.pck.backend('main.js'), this.compileMain(backendModules));
     }

dev-packages/application-manager/src/generator/frontend-generator.ts

@@ -22,11 +22,11 @@ import { existsSync, readFileSync } from 'fs';
 export class FrontendGenerator extends AbstractGenerator {
 
     async generate(): Promise<void> {
-        const frontendModules = this.pck.targetFrontendModules;
+        const frontendModules = this.pck.targetFrontendModulesFiltered;
         await this.write(this.pck.frontend('index.html'), this.compileIndexHtml(frontendModules));
         await this.write(this.pck.frontend('index.js'), this.compileIndexJs(frontendModules));
         if (this.pck.isElectron()) {
-            const electronMainModules = this.pck.targetElectronMainModules;
+            const electronMainModules = this.pck.targetElectronMainModulesFiltered;
             await this.write(this.pck.frontend('electron-main.js'), this.compileElectronMain(electronMainModules));
         }
     }

dev-packages/application-package/src/application-package.ts

@@ -240,15 +240,76 @@ export class ApplicationPackage {
     }
 
     get targetBackendModules(): Map<string, string> {
-        return this.ifBrowser(this.backendModules, this.backendElectronModules);
+        return this.isBrowser() ? this.backendModules : this.backendElectronModules;
     }
 
     get targetFrontendModules(): Map<string, string> {
-        return this.ifBrowser(this.frontendModules, this.frontendElectronModules);
+        return this.isBrowser() ? this.frontendModules : this.frontendElectronModules;
     }
 
     get targetElectronMainModules(): Map<string, string> {
-        return this.ifElectron(this.electronMainModules, new Map());
+        return this.isElectron() ? this.electronMainModules : new Map();
+    }
+
+    protected _targetBackendModulesFiltered: Map<string, string>;
+    get targetBackendModulesFiltered(): Map<string, string> {
+        if (!this._targetBackendModulesFiltered) {
+            this._targetBackendModulesFiltered = this.filterModules(this.targetBackendModules);
+        }
+        return this._targetBackendModulesFiltered;
+    }
+
+    protected _targetFrontendModulesFiltered: Map<string, string>;
+    get targetFrontendModulesFiltered(): Map<string, string> {
+        if (!this._targetFrontendModulesFiltered) {
+            this._targetFrontendModulesFiltered = this.filterModules(this.targetFrontendModules);
+        }
+        return this._targetFrontendModulesFiltered;
+    }
+
+    protected _targetElectronMainModulesFiltered: Map<string, string>;
+    get targetElectronMainModulesFiltered(): Map<string, string> {
+        if (!this._targetElectronMainModulesFiltered) {
+            this._targetElectronMainModulesFiltered = this.filterModules(this.electronMainModules);
+        }
+        return this._targetElectronMainModulesFiltered;
+    }
+
+    protected filterModules(modules: Map<string, string>): Map<string, string> {
+        const { strategy, includes = [], excludes = [] } = this.props.extensions.loading;
+        const { dependencies = {} } = this.pck;
+        const all = strategy === 'all';
+        const explicitDependenciesOnly = strategy === 'explicitDependenciesOnly';
+        if (!all && !explicitDependenciesOnly) {
+            throw new Error(`unknown theia.extensions.loading.strategy: ${strategy}`);
+        }
+        const filtered = new Map<string, string>();
+        for (const [name, path] of modules.entries()) {
+            if (excludes.some(exclude => new RegExp(exclude).test(path))) {
+                continue;
+            } else if (
+                // When using 'all' the include list has precedent:
+                all && (includes.length === 0 || includes.some(include => new RegExp(include).test(path)))
+                // Using 'explicitDependenciesOnly' is complementary to the include list, else
+                // setting 'explicitDependenciesOnly' would be nulled by the include list.
+                || explicitDependenciesOnly && (this.getPackageName(path) in dependencies || includes.length > 0 && includes.some(include => new RegExp(include).test(path)))
+            ) {
+                filtered.set(name, path);
+            }
+        }
+        return filtered;
+    }
+
+    /**
+     * Only keep the first two parts of the package name e.g.,
+     * - `@a/b/c/...` => `@a/b`
+     * - `a/b/c/...` => `a`
+     */
+    protected getPackageName(mod: string): string {
+        const slice = mod.startsWith('@') ? 2 : 1;
+        return mod.split('/', slice + 1)
+            .slice(0, slice)
+            .join('/');
     }
 
     setDependency(name: string, version: string | undefined): boolean {

dev-packages/application-package/src/application-props.ts

@@ -49,6 +49,11 @@ export interface ApplicationProps extends NpmRegistryProps {
      */
     readonly target: ApplicationProps.Target;
 
+    /**
+     * Theia extensions related properties.
+     */
+    readonly extensions: Readonly<ExtensionsConfig>;
+
     /**
      * Frontend related properties.
      */
@@ -75,6 +80,11 @@ export namespace ApplicationProps {
     export const DEFAULT: ApplicationProps = {
         ...NpmRegistryProps.DEFAULT,
         target: 'browser',
+        extensions: {
+            loading: {
+                strategy: 'all'
+            }
+        },
         backend: {
             config: {}
         },
@@ -94,6 +104,41 @@ export namespace ApplicationProps {
 
 }
 
+/**
+ * Configure how Theia handles extensions.
+ */
+export interface ExtensionsConfig {
+
+    /**
+     * Specify which inversify modules to mount in your application.
+     *
+     * Note that the `loadingStrategy` only affects which inversify module will be mounted
+     * to your application, but an ignored extension might still end up in your bundles.
+     */
+    loading: ExtensionLoadingStrategy;
+}
+
+export interface ExtensionLoadingStrategy {
+
+    /**
+     * - `all`: Use all Theia extensions found installed under `node_modules`.
+     * - `explicitDependenciesOnly`: Only use what's defined as your application dependencies.
+     */
+    strategy: 'all' | 'explicitDependenciesOnly';
+
+    /**
+     * List of regexes to filter in Theia extensions, will be matched against the modules file path.
+     *
+     * Note that matched files will be added to what's matched by `explicitDependenciesOnly`.
+     */
+    includes?: string[]
+
+    /**
+     * List of regexes to filter out Theia extensions, will be matched against the modules file path.
+     */
+    excludes?: string[]
+}
+
 /**
  * Base configuration for the Theia application.
  */