Fix for issue #2906 (semantic highlighting styles overruled by TM sco…

…pes)

The idea for the fix comes from the observation that there are VSCode
plugins that implement semantic highlighting using
TextEditor.setDecorations(), and do not suffer from this problem.

We don't want to use setDecorations() itself because it's not really
suited for incremental updates to the highlighting (which the LSP
protocol extension that Theia implements allows for).

However, setDecorations() is implemented in terms of deltaDecorations()
(which is what Theia uses), and changing Theia's use of
deltaDecorations() to be more like the implementation of
setDecorations() seems to fix this bug.

Specifically, instead of getting an inlineClassName directly from the
TokenMetadata (which seems to produce the problematic inlineClassName
that coflicts with TM), we:

 * Get the token color from the TokenMetadata

 * Construct an IDecorationRenderOptions from the token color
   (as if we were going to call setDecorations())

 * Use ICodeEditorService.registerDecorationType() and
   ICodeEditorService.resolveDecorationOptions() to massage the
   IDecorationRenderOptions into an IModelDecorationOptions. This
   appears to cause monaco to allocate a new inlineClassName for
   the color which doesn't conflict with TM.

 * Call deltaDecorations() with IModelDecorationOptions obtained in
   this way

packages/monaco/src/browser/monaco-semantic-highlighting-service.ts

@@ -22,6 +22,16 @@ import { Disposable, DisposableCollection } from '@theia/core/lib/common/disposa
 import { EditorDecoration, EditorDecorationOptions } from '@theia/editor/lib/browser/decorations';
 import { SemanticHighlightingService, SemanticHighlightingRange, Range } from '@theia/editor/lib/browser/semantic-highlight/semantic-highlighting-service';
 import { MonacoEditor } from './monaco-editor';
+import { MonacoEditorService } from './monaco-editor-service';
+
+/**
+ * A helper class for grouping information about a decoration type that has
+ * been registered with the editor service.
+ */
+class DecorationTypeInfo {
+    key: string;
+    options: monaco.editor.IModelDecorationOptions;
+}
 
 @injectable()
 export class MonacoSemanticHighlightingService extends SemanticHighlightingService {
@@ -32,9 +42,93 @@ export class MonacoSemanticHighlightingService extends SemanticHighlightingServi
     @inject(EditorManager)
     protected readonly editorManager: EditorManager;
 
+    @inject(MonacoEditorService)
+    protected readonly monacoEditorService: MonacoEditorService;
+
     protected readonly decorations = new Map<string, Set<string>>();
     protected readonly toDisposeOnEditorClose = new Map<string, Disposable>();
 
+    // laguage id -> (scope index -> decoration type)
+    protected readonly decorationTypes = new Map<string, Map<number, DecorationTypeInfo>>();
+
+    private lastDecorationTypeId: number = 0;
+
+    private nextDecorationTypeKey(): string {
+        return 'MonacoSemanticHighlighting' + (++this.lastDecorationTypeId);
+    }
+
+    protected registerDecorationTypesForLanguage(languageId: string): void {
+        const scopes = this.scopes.get(languageId);
+        if (scopes) {
+            const decorationTypes = new Map<number, DecorationTypeInfo>();
+            for (let index = 0; index < scopes.length; index++) {
+                const modelDecoration = this.toDecorationType(scopes[index]);
+                if (modelDecoration) {
+                    decorationTypes.set(index, modelDecoration);
+                }
+            }
+            this.decorationTypes.set(languageId, decorationTypes);
+        }
+    }
+
+    protected removeDecorationTypesForLanguage(languageId: string): void {
+        const decorationTypes = this.decorationTypes.get(languageId);
+        if (!decorationTypes) {
+            this.logger.warn(`No decoration types are registered for language: ${languageId}`);
+            return;
+        }
+        for (const [, decorationType] of decorationTypes) {
+            this.monacoEditorService.removeDecorationType(decorationType.key);
+        }
+    }
+
+    register(languageId: string, scopes: string[][] | undefined): Disposable {
+        const result = super.register(languageId, scopes);
+        this.registerDecorationTypesForLanguage(languageId);
+        this.themeService().onThemeChange(() => {
+            // When the theme changes, remove the decoration types for the old
+            // colors and create new ones for the new colors.
+            this.removeDecorationTypesForLanguage(languageId);
+            this.registerDecorationTypesForLanguage(languageId);
+        });
+        return result;
+    }
+
+    protected unregister(languageId: string): void {
+        super.unregister(languageId);
+        this.removeDecorationTypesForLanguage(languageId);
+        this.decorationTypes.delete(languageId);
+    }
+
+    protected toDecorationType(scopes: string[]): DecorationTypeInfo | undefined {
+        const key = this.nextDecorationTypeKey();
+        // TODO: why for-of? How to pick the right scope? Is it fine to get the first element (with the narrowest scope)?
+        for (const scope of scopes) {
+            const tokenTheme = this.tokenTheme();
+            const metadata = tokenTheme.match(undefined, scope);
+            // Don't use the inlineClassName from the TokenMetadata, because this
+            // will conflict with styles used for TM scopes
+            // (https://github.com/Microsoft/monaco-editor/issues/1070).
+            // Instead, get the token color, use registerDecorationType() to cause
+            // monaco to allocate a new inlineClassName for that color, and use
+            // resolveDecorationOptions() to get an IModelDecorationOptions
+            // containing that inlineClassName.
+            const colorIndex = monaco.modes.TokenMetadata.getForeground(metadata);
+            const color = tokenTheme.getColorMap()[colorIndex];
+            // If we wanted to support other decoration options such as font style,
+            // we could include them here.
+            const options: monaco.editor.IDecorationRenderOptions = {
+                color: color.toString(),
+            };
+            this.monacoEditorService.registerDecorationType(key, options);
+            return {
+                key,
+                options: this.monacoEditorService.resolveDecorationOptions(key, false)
+            };
+        }
+        return undefined;
+    }
+
     async decorate(languageId: string, uri: URI, ranges: SemanticHighlightingRange[]): Promise<void> {
         const editor = await this.editor(uri);
         if (!editor) {
@@ -116,28 +210,33 @@ export class MonacoSemanticHighlightingService extends SemanticHighlightingServi
 
     protected toDecoration(languageId: string, range: SemanticHighlightingRange): EditorDecoration {
         const { start, end } = range;
-        const scopes = range.scope !== undefined ? this.scopesFor(languageId, range.scope) : [];
-        const options = this.toOptions(scopes);
+        const options = this.toOptions(languageId, range.scope);
         return {
             range: Range.create(start, end),
             options
         };
     }
 
-    protected toOptions(scopes: string[]): EditorDecorationOptions {
-        // TODO: why for-of? How to pick the right scope? Is it fine to get the first element (with the narrowest scope)?
-        for (const scope of scopes) {
-            const metadata = this.tokenTheme().match(undefined, scope);
-            const inlineClassName = monaco.modes.TokenMetadata.getClassNameFromMetadata(metadata);
-            return {
-                inlineClassName
-            };
+    protected toOptions(languageId: string, scope: number | undefined): EditorDecorationOptions {
+        if (scope) {
+            const decorationTypes = this.decorationTypes.get(languageId);
+            if (decorationTypes) {
+                const decoration = decorationTypes.get(scope);
+                if (decoration) {
+                    return {
+                        inlineClassName: decoration.options.inlineClassName
+                    };
+                }
+            }
         }
         return {};
     }
 
-    protected tokenTheme(): monaco.services.TokenTheme {
-        return monaco.services.StaticServices.standaloneThemeService.get().getTheme().tokenTheme;
+    protected themeService(): monaco.services.IStandaloneThemeService {
+        return monaco.services.StaticServices.standaloneThemeService.get();
     }
 
+    protected tokenTheme(): monaco.services.TokenTheme {
+        return this.themeService().getTheme().tokenTheme;
+    }
 }