Enable translations for extension code for the web (#155355)

* Initial attempt

* alex feedback

build/lib/extensions.js

@@ -351,12 +351,20 @@ function scanBuiltinExtensions(extensionsRoot, exclude = []) {
             const children = fs.readdirSync(path.join(extensionsRoot, extensionFolder));
             const packageNLSPath = children.filter(child => child === 'package.nls.json')[0];
             const packageNLS = packageNLSPath ? JSON.parse(fs.readFileSync(path.join(extensionsRoot, extensionFolder, packageNLSPath)).toString()) : undefined;
+            let browserNlsMetadataPath;
+            if (packageJSON.browser) {
+                const browserEntrypointFolderPath = path.join(extensionFolder, path.dirname(packageJSON.browser));
+                if (fs.existsSync(path.join(extensionsRoot, browserEntrypointFolderPath, 'nls.metadata.json'))) {
+                    browserNlsMetadataPath = path.join(browserEntrypointFolderPath, 'nls.metadata.json');
+                }
+            }
             const readme = children.filter(child => /^readme(\.txt|\.md|)$/i.test(child))[0];
             const changelog = children.filter(child => /^changelog(\.txt|\.md|)$/i.test(child))[0];
             scannedExtensions.push({
                 extensionPath: extensionFolder,
                 packageJSON,
                 packageNLS,
+                browserNlsMetadataPath,
                 readmePath: readme ? path.join(extensionFolder, readme) : undefined,
                 changelogPath: changelog ? path.join(extensionFolder, changelog) : undefined,
             });

build/lib/extensions.ts

@@ -412,6 +412,7 @@ export interface IScannedBuiltinExtension {
 	extensionPath: string;
 	packageJSON: any;
 	packageNLS?: any;
+	browserNlsMetadataPath?: string;
 	readmePath?: string;
 	changelogPath?: string;
 }
@@ -436,13 +437,21 @@ export function scanBuiltinExtensions(extensionsRoot: string, exclude: string[]
 			const children = fs.readdirSync(path.join(extensionsRoot, extensionFolder));
 			const packageNLSPath = children.filter(child => child === 'package.nls.json')[0];
 			const packageNLS = packageNLSPath ? JSON.parse(fs.readFileSync(path.join(extensionsRoot, extensionFolder, packageNLSPath)).toString()) : undefined;
+			let browserNlsMetadataPath: string | undefined;
+			if (packageJSON.browser) {
+				const browserEntrypointFolderPath = path.join(extensionFolder, path.dirname(packageJSON.browser));
+				if (fs.existsSync(path.join(extensionsRoot, browserEntrypointFolderPath, 'nls.metadata.json'))) {
+					browserNlsMetadataPath = path.join(browserEntrypointFolderPath, 'nls.metadata.json');
+				}
+			}
 			const readme = children.filter(child => /^readme(\.txt|\.md|)$/i.test(child))[0];
 			const changelog = children.filter(child => /^changelog(\.txt|\.md|)$/i.test(child))[0];
 
 			scannedExtensions.push({
 				extensionPath: extensionFolder,
 				packageJSON,
 				packageNLS,
+				browserNlsMetadataPath,
 				readmePath: readme ? path.join(extensionFolder, readme) : undefined,
 				changelogPath: changelog ? path.join(extensionFolder, changelog) : undefined,
 			});

extensions/shared.webpack.config.js

@@ -70,6 +70,10 @@ function withNodeDefaults(/**@type WebpackConfig*/extConfig) {
 	return merge(defaultConfig, extConfig);
 }
 
+/**
+ *
+ * @param {string} context
+ */
 function nodePlugins(context) {
 	// Need to find the top-most `package.json` file
 	const folderName = path.relative(__dirname, context).split(/[\\\/]/)[0];
@@ -109,6 +113,13 @@ function withBrowserDefaults(/**@type WebpackConfig*/extConfig, /** @type Additi
 				test: /\.ts$/,
 				exclude: /node_modules/,
 				use: [{
+					// vscode-nls-dev loader:
+					// * rewrite nls-calls
+					loader: 'vscode-nls-dev/lib/webpack-loader',
+					options: {
+						base: path.join(extConfig.context, 'src')
+					}
+				}, {
 					// configure TypeScript loader:
 					// * enable sources maps for end-to-end source maps
 					loader: 'ts-loader',
@@ -123,6 +134,7 @@ function withBrowserDefaults(/**@type WebpackConfig*/extConfig, /** @type Additi
 		},
 		externals: {
 			'vscode': 'commonjs vscode', // ignored because it doesn't exist,
+			'vscode-nls-web-data': 'commonjs vscode-nls-web-data', // ignored because this is injected by the webworker extension host
 			'applicationinsights-native-metrics': 'commonjs applicationinsights-native-metrics', // ignored because we don't ship native module
 			'@opentelemetry/tracing': 'commonjs @opentelemetry/tracing' // ignored because we don't ship this module
 		},
@@ -138,30 +150,39 @@ function withBrowserDefaults(/**@type WebpackConfig*/extConfig, /** @type Additi
 		},
 		// yes, really source maps
 		devtool: 'source-map',
-		plugins: browserPlugins
+		plugins: browserPlugins(extConfig.context)
 	};
 
 	return merge(defaultConfig, extConfig);
 }
 
-const browserPlugins = [
-	new optimize.LimitChunkCountPlugin({
-		maxChunks: 1
-	}),
-	new CopyWebpackPlugin({
-		patterns: [
-			{ from: 'src', to: '.', globOptions: { ignore: ['**/test/**', '**/*.ts'] }, noErrorOnMissing: true }
-		]
-	}),
-	new DefinePlugin({
-		'process.platform': JSON.stringify('web'),
-		'process.env': JSON.stringify({}),
-		'process.env.BROWSER_ENV': JSON.stringify('true')
-	})
-];
-
-
-
+/**
+ *
+ * @param {string} context
+ */
+function browserPlugins(context) {
+	// Need to find the top-most `package.json` file
+	const folderName = path.relative(__dirname, context).split(/[\\\/]/)[0];
+	const pkgPath = path.join(__dirname, folderName, 'package.json');
+	const pkg = JSON.parse(fs.readFileSync(pkgPath, 'utf8'));
+	const id = `${pkg.publisher}.${pkg.name}`;
+	return [
+		new optimize.LimitChunkCountPlugin({
+			maxChunks: 1
+		}),
+		new CopyWebpackPlugin({
+			patterns: [
+				{ from: 'src', to: '.', globOptions: { ignore: ['**/test/**', '**/*.ts'] }, noErrorOnMissing: true }
+			]
+		}),
+		new DefinePlugin({
+			'process.platform': JSON.stringify('web'),
+			'process.env': JSON.stringify({}),
+			'process.env.BROWSER_ENV': JSON.stringify('true')
+		}),
+		new NLSBundlePlugin(id)
+	];
+}
 
 module.exports = withNodeDefaults;
 module.exports.node = withNodeDefaults;

extensions/typescript-language-features/extension-browser.webpack.config.js

@@ -37,7 +37,7 @@ module.exports = withBrowserDefaults({
 		extension: './src/extension.browser.ts',
 	},
 	plugins: [
-		...browserPlugins, // add plugins, don't replace inherited
+		...browserPlugins(__dirname), // add plugins, don't replace inherited
 
 		// @ts-ignore
 		new CopyPlugin({

src/vs/platform/extensions/common/extensions.ts

@@ -323,6 +323,7 @@ export interface IExtension {
 	readonly changelogUrl?: URI;
 	readonly isValid: boolean;
 	readonly validations: readonly [Severity, string][];
+	readonly browserNlsBundleUris?: { [language: string]: URI };
 }
 
 /**
@@ -389,6 +390,7 @@ export interface IRelaxedExtensionDescription extends IRelaxedExtensionManifest
 	isUserBuiltin: boolean;
 	isUnderDevelopment: boolean;
 	extensionLocation: URI;
+	browserNlsBundleUris?: { [language: string]: URI };
 }
 
 export type IExtensionDescription = Readonly<IRelaxedExtensionDescription>;

src/vs/workbench/api/common/extHostExtensionService.ts

@@ -280,10 +280,18 @@ export abstract class AbstractExtHostExtensionService extends Disposable impleme
 
 	public async getExtension(extensionId: string): Promise<IExtensionDescription | undefined> {
 		const ext = await this._mainThreadExtensionsProxy.$getExtension(extensionId);
+		let browserNlsBundleUris: { [language: string]: URI } | undefined;
+		if (ext?.browserNlsBundleUris) {
+			browserNlsBundleUris = {};
+			for (const language of Object.keys(ext.browserNlsBundleUris)) {
+				browserNlsBundleUris[language] = URI.revive(ext.browserNlsBundleUris[language]);
+			}
+		}
 		return ext && {
 			...ext,
 			identifier: new ExtensionIdentifier(ext.identifier.value),
 			extensionLocation: URI.revive(ext.extensionLocation),
+			browserNlsBundleUris
 		};
 	}
 
@@ -465,7 +473,7 @@ export abstract class AbstractExtHostExtensionService extends Disposable impleme
 
 		const activationTimesBuilder = new ExtensionActivationTimesBuilder(reason.startup);
 		return Promise.all([
-			this._loadCommonJSModule<IExtensionModule>(extensionDescription.identifier, joinPath(extensionDescription.extensionLocation, entryPoint), activationTimesBuilder),
+			this._loadCommonJSModule<IExtensionModule>(extensionDescription, joinPath(extensionDescription.extensionLocation, entryPoint), activationTimesBuilder),
 			this._loadExtensionContext(extensionDescription)
 		]).then(values => {
 			performance.mark(`code/extHost/willActivateExtension/${extensionDescription.identifier.value}`);
@@ -923,7 +931,7 @@ export abstract class AbstractExtHostExtensionService extends Disposable impleme
 
 	protected abstract _beforeAlmostReadyToRunExtensions(): Promise<void>;
 	protected abstract _getEntryPoint(extensionDescription: IExtensionDescription): string | undefined;
-	protected abstract _loadCommonJSModule<T extends object | undefined>(extensionId: ExtensionIdentifier | null, module: URI, activationTimesBuilder: ExtensionActivationTimesBuilder): Promise<T>;
+	protected abstract _loadCommonJSModule<T extends object | undefined>(extensionId: IExtensionDescription | null, module: URI, activationTimesBuilder: ExtensionActivationTimesBuilder): Promise<T>;
 	public abstract $setRemoteEnvironment(env: { [key: string]: string | null }): Promise<void>;
 }
 

src/vs/workbench/api/common/extensionHostMain.ts

@@ -120,7 +120,14 @@ export class ExtensionHostMain {
 	}
 
 	private static _transform(initData: IExtensionHostInitData, rpcProtocol: RPCProtocol): IExtensionHostInitData {
-		initData.allExtensions.forEach((ext) => (<any>ext).extensionLocation = URI.revive(rpcProtocol.transformIncomingURIs(ext.extensionLocation)));
+		initData.allExtensions.forEach((ext) => {
+			(<any>ext).extensionLocation = URI.revive(rpcProtocol.transformIncomingURIs(ext.extensionLocation));
+			const browserNlsBundleUris: { [language: string]: URI } = {};
+			if (ext.browserNlsBundleUris) {
+				Object.keys(ext.browserNlsBundleUris).forEach(lang => browserNlsBundleUris[lang] = URI.revive(rpcProtocol.transformIncomingURIs(ext.browserNlsBundleUris![lang])));
+				(<any>ext).browserNlsBundleUris = browserNlsBundleUris;
+			}
+		});
 		initData.environment.appRoot = URI.revive(rpcProtocol.transformIncomingURIs(initData.environment.appRoot));
 		const extDevLocs = initData.environment.extensionDevelopmentLocationURI;
 		if (extDevLocs) {

src/vs/workbench/api/node/extHostExtensionService.ts

@@ -12,7 +12,7 @@ import { AbstractExtHostExtensionService } from 'vs/workbench/api/common/extHost
 import { ExtHostDownloadService } from 'vs/workbench/api/node/extHostDownloadService';
 import { URI } from 'vs/base/common/uri';
 import { Schemas } from 'vs/base/common/network';
-import { ExtensionIdentifier, IExtensionDescription } from 'vs/platform/extensions/common/extensions';
+import { IExtensionDescription } from 'vs/platform/extensions/common/extensions';
 import { ExtensionRuntime } from 'vs/workbench/api/common/extHostTypes';
 import { CLIServer } from 'vs/workbench/api/node/extHostCLIServer';
 import { realpathSync } from 'vs/base/node/extpath';
@@ -89,24 +89,25 @@ export class ExtHostExtensionService extends AbstractExtHostExtensionService {
 		return extensionDescription.main;
 	}
 
-	protected _loadCommonJSModule<T>(extensionId: ExtensionIdentifier | null, module: URI, activationTimesBuilder: ExtensionActivationTimesBuilder): Promise<T> {
+	protected _loadCommonJSModule<T>(extension: IExtensionDescription | null, module: URI, activationTimesBuilder: ExtensionActivationTimesBuilder): Promise<T> {
 		if (module.scheme !== Schemas.file) {
 			throw new Error(`Cannot load URI: '${module}', must be of file-scheme`);
 		}
 		let r: T | null = null;
 		activationTimesBuilder.codeLoadingStart();
 		this._logService.trace(`ExtensionService#loadCommonJSModule ${module.toString(true)}`);
 		this._logService.flush();
+		const extensionId = extension?.identifier.value;
 		try {
 			if (extensionId) {
-				performance.mark(`code/extHost/willLoadExtensionCode/${extensionId.value}`);
+				performance.mark(`code/extHost/willLoadExtensionCode/${extensionId}`);
 			}
 			r = require.__$__nodeRequire<T>(module.fsPath);
 		} catch (e) {
 			return Promise.reject(e);
 		} finally {
 			if (extensionId) {
-				performance.mark(`code/extHost/didLoadExtensionCode/${extensionId.value}`);
+				performance.mark(`code/extHost/didLoadExtensionCode/${extensionId}`);
 			}
 			activationTimesBuilder.codeLoadingStop();
 		}

src/vs/workbench/api/worker/extHostExtensionService.ts

@@ -8,10 +8,11 @@ import { ExtensionActivationTimesBuilder } from 'vs/workbench/api/common/extHost
 import { AbstractExtHostExtensionService } from 'vs/workbench/api/common/extHostExtensionService';
 import { URI } from 'vs/base/common/uri';
 import { RequireInterceptor } from 'vs/workbench/api/common/extHostRequireInterceptor';
-import { ExtensionIdentifier, IExtensionDescription } from 'vs/platform/extensions/common/extensions';
+import { IExtensionDescription } from 'vs/platform/extensions/common/extensions';
 import { ExtensionRuntime } from 'vs/workbench/api/common/extHostTypes';
 import { timeout } from 'vs/base/common/async';
 import { ExtHostConsoleForwarder } from 'vs/workbench/api/worker/extHostConsoleForwarder';
+import { Language } from 'vs/base/common/platform';
 
 class WorkerRequireInterceptor extends RequireInterceptor {
 
@@ -55,10 +56,11 @@ export class ExtHostExtensionService extends AbstractExtHostExtensionService {
 		return extensionDescription.browser;
 	}
 
-	protected async _loadCommonJSModule<T extends object | undefined>(extensionId: ExtensionIdentifier | null, module: URI, activationTimesBuilder: ExtensionActivationTimesBuilder): Promise<T> {
+	protected async _loadCommonJSModule<T extends object | undefined>(extension: IExtensionDescription | null, module: URI, activationTimesBuilder: ExtensionActivationTimesBuilder): Promise<T> {
 		module = module.with({ path: ensureSuffix(module.path, '.js') });
+		const extensionId = extension?.identifier.value;
 		if (extensionId) {
-			performance.mark(`code/extHost/willFetchExtensionCode/${extensionId.value}`);
+			performance.mark(`code/extHost/willFetchExtensionCode/${extensionId}`);
 		}
 
 		// First resolve the extension entry point URI to something we can load using `fetch`
@@ -67,7 +69,7 @@ export class ExtHostExtensionService extends AbstractExtHostExtensionService {
 		const browserUri = URI.revive(await this._mainThreadExtensionsProxy.$asBrowserUri(module));
 		const response = await fetch(browserUri.toString(true));
 		if (extensionId) {
-			performance.mark(`code/extHost/didFetchExtensionCode/${extensionId.value}`);
+			performance.mark(`code/extHost/didFetchExtensionCode/${extensionId}`);
 		}
 
 		if (response.status !== 200) {
@@ -85,7 +87,7 @@ export class ExtHostExtensionService extends AbstractExtHostExtensionService {
 			initFn = new Function('module', 'exports', 'require', fullSource);
 		} catch (err) {
 			if (extensionId) {
-				console.error(`Loading code for extension ${extensionId.value} failed: ${err.message}`);
+				console.error(`Loading code for extension ${extensionId} failed: ${err.message}`);
 			} else {
 				console.error(`Loading code failed: ${err.message}`);
 			}
@@ -94,10 +96,17 @@ export class ExtHostExtensionService extends AbstractExtHostExtensionService {
 			throw err;
 		}
 
+		const strings: { [key: string]: string[] } = await this.fetchTranslatedStrings(extension);
+
 		// define commonjs globals: `module`, `exports`, and `require`
 		const _exports = {};
 		const _module = { exports: _exports };
 		const _require = (request: string) => {
+			// In order to keep vscode-nls synchronous, we prefetched the translations above
+			// and then return them here when the extension is loaded.
+			if (request === 'vscode-nls-web-data') {
+				return strings;
+			}
 			const result = this._fakeModules!.getModule(request, module);
 			if (result === undefined) {
 				throw new Error(`Cannot load module '${request}'`);
@@ -108,13 +117,13 @@ export class ExtHostExtensionService extends AbstractExtHostExtensionService {
 		try {
 			activationTimesBuilder.codeLoadingStart();
 			if (extensionId) {
-				performance.mark(`code/extHost/willLoadExtensionCode/${extensionId.value}`);
+				performance.mark(`code/extHost/willLoadExtensionCode/${extensionId}`);
 			}
 			initFn(_module, _exports, _require);
 			return <T>(_module.exports !== _exports ? _module.exports : _exports);
 		} finally {
 			if (extensionId) {
-				performance.mark(`code/extHost/didLoadExtensionCode/${extensionId.value}`);
+				performance.mark(`code/extHost/didLoadExtensionCode/${extensionId}`);
 			}
 			activationTimesBuilder.codeLoadingStop();
 		}
@@ -135,6 +144,44 @@ export class ExtHostExtensionService extends AbstractExtHostExtensionService {
 			await timeout(10);
 		}
 	}
+
+	private async fetchTranslatedStrings(extension: IExtensionDescription | null): Promise<{ [key: string]: string[] }> {
+		let strings: { [key: string]: string[] } = {};
+		if (!extension) {
+			return {};
+		}
+		const translationsUri = Language.isDefaultVariant()
+			// If we are in the default variant, load the translations for en only.
+			? extension.browserNlsBundleUris?.en
+			// Otherwise load the translations for the current locale with English as a fallback.
+			: extension.browserNlsBundleUris?.[Language.value()] ?? extension.browserNlsBundleUris?.en;
+		if (extension && translationsUri) {
+			try {
+				const response = await fetch(translationsUri.toString(true));
+				if (!response.ok) {
+					throw new Error(await response.text());
+				}
+				strings = await response.json();
+			} catch (e) {
+				try {
+					console.error(`Failed to load translations for ${extension.identifier.value} from ${translationsUri}: ${e.message}`);
+					const englishStrings = extension.browserNlsBundleUris?.en;
+					if (englishStrings) {
+						const response = await fetch(englishStrings.toString(true));
+						if (!response.ok) {
+							throw new Error(await response.text());
+						}
+						strings = await response.json();
+					}
+					throw new Error('No English strings found');
+				} catch (e) {
+					// TODO what should this do? We really shouldn't ever be here...
+					console.error(e);
+				}
+			}
+		}
+		return strings;
+	}
 }
 
 function ensureSuffix(path: string, suffix: string): string {