GH-1845: Extended the LSP with the semantic highlighting capabilities.

Closes #1845

Signed-off-by: Akos Kitta <kittaakos@typefox.io>

examples/browser/package.json

@@ -56,4 +56,4 @@
   "devDependencies": {
     "@theia/cli": "^0.3.13"
   }
-}
+}

packages/core/src/common/disposable.ts

@@ -32,9 +32,14 @@ export namespace Disposable {
 }
 
 export class DisposableCollection implements Disposable {
+
     protected readonly disposables: Disposable[] = [];
     protected readonly onDisposeEmitter = new Emitter<void>();
 
+    constructor(...toDispose: Disposable[]) {
+        toDispose.forEach(d => this.push(d));
+    }
+
     get onDispose(): Event<void> {
         return this.onDisposeEmitter.event;
     }

packages/editor/package.json

@@ -5,7 +5,9 @@
   "dependencies": {
     "@theia/core": "^0.3.13",
     "@theia/languages": "^0.3.13",
-    "@theia/variable-resolver": "^0.3.13"
+    "@theia/variable-resolver": "^0.3.13",
+    "@types/base64-arraybuffer": "0.1.0",
+    "base64-arraybuffer": "^0.1.5"
   },
   "publishConfig": {
     "access": "public"
@@ -45,4 +47,4 @@
   "nyc": {
     "extends": "../../configs/nyc.json"
   }
-}
+}

packages/editor/src/browser/editor-frontend-module.ts

@@ -31,6 +31,7 @@ import { NavigationLocationUpdater } from './navigation/navigation-location-upda
 import { NavigationLocationService } from './navigation/navigation-location-service';
 import { NavigationLocationSimilarity } from './navigation/navigation-location-similarity';
 import { EditorVariableContribution } from './editor-variable-contribution';
+import { SemanticHighlightingService } from './semantic-highlight/semantic-highlighting-service';
 
 export default new ContainerModule(bind => {
     bindEditorPreferences(bind);
@@ -58,4 +59,6 @@ export default new ContainerModule(bind => {
     bind(NavigationLocationSimilarity).toSelf().inSingletonScope();
 
     bind(VariableContribution).to(EditorVariableContribution).inSingletonScope();
+
+    bind(SemanticHighlightingService).toSelf().inSingletonScope();
 });

packages/editor/src/browser/editor.ts

@@ -89,17 +89,25 @@ export interface TextEditor extends Disposable, TextEditorSelection, Navigatable
      */
     deltaDecorations(params: DeltaDecorationParams): string[];
 
+    /**
+     * Gets all the decorations for the lines between `startLineNumber` and `endLineNumber` as an array.
+     * @param startLineNumber The start line number.
+     * @param endLineNumber The end line number.
+     * @return An array with the decorations.
+     */
+    getLinesDecorations(startLineNumber: number, endLineNumber: number): EditorDecoration[];
+
     getVisibleColumn(position: Position): number;
 
     /**
-     * Replaces the text of source given in ReplacetextParams.
+     * Replaces the text of source given in ReplaceTextParams.
      * @param params: ReplaceTextParams
      */
     replaceText(params: ReplaceTextParams): Promise<boolean>;
 
     /**
      * Execute edits on the editor.
-     * @param edits: edits created with `lsp.TextEdit.replace`, `lsp.TextEdit.instert`, `lsp.TextEdit.del`
+     * @param edits: edits created with `lsp.TextEdit.replace`, `lsp.TextEdit.insert`, `lsp.TextEdit.del`
      */
     executeEdits(edits: lsp.TextEdit[]): boolean;
 }

packages/editor/src/browser/semantic-highlight/semantic-highlighting-service.spec.ts

@@ -0,0 +1,30 @@
+/********************************************************************************
+ * Copyright (C) 2018 TypeFox and others.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v. 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
+ *
+ * This Source Code may also be made available under the following Secondary
+ * Licenses when the conditions for such availability set forth in the Eclipse
+ * Public License v. 2.0 are satisfied: GNU General Public License, version 2
+ * with the GNU Classpath Exception which is available at
+ * https://www.gnu.org/software/classpath/license.html.
+ *
+ * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0
+ ********************************************************************************/
+
+import { expect } from 'chai';
+import { SemanticHighlightingService } from './semantic-highlighting-service';
+
+describe('semantic-highlighting-service', () => {
+
+    it('encode-decode', () => {
+        const input = [2, 5, 0, 12, 15, 1, 7, 1000, 1];
+        const expected = SemanticHighlightingService.Token.fromArray(input);
+        const encoded = SemanticHighlightingService.encode(expected);
+        const actual = SemanticHighlightingService.decode(encoded);
+        expect(actual).to.be.deep.equal(expected);
+    });
+
+});

packages/editor/src/browser/semantic-highlight/semantic-highlighting-service.ts

@@ -0,0 +1,210 @@
+/********************************************************************************
+ * Copyright (C) 2018 TypeFox and others.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v. 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
+ *
+ * This Source Code may also be made available under the following Secondary
+ * Licenses when the conditions for such availability set forth in the Eclipse
+ * Public License v. 2.0 are satisfied: GNU General Public License, version 2
+ * with the GNU Classpath Exception which is available at
+ * https://www.gnu.org/software/classpath/license.html.
+ *
+ * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0
+ ********************************************************************************/
+
+import { inject, injectable } from 'inversify';
+import { decode as base64Decode, encode as base64Encode } from 'base64-arraybuffer';
+import { Position, Range } from 'vscode-languageserver-types';
+import URI from '@theia/core/lib/common/uri';
+import { Disposable } from '@theia/core/lib/common/disposable';
+import { ILogger } from '@theia/core/lib/common/logger';
+
+/**
+ * Service for registering and managing semantic highlighting decorations in the code editors for multiple languages.
+ *
+ * The current, default implementation does nothing at all, because the unique identifier of the `EditorDecoration` is not
+ * exposed via the API. A working example is available as the `MonacoSemanticHighlightingService` from the `@theia/monaco` extension.
+ */
+@injectable()
+export class SemanticHighlightingService implements Disposable {
+
+    @inject(ILogger)
+    protected readonly logger: ILogger;
+    protected readonly scopes: Map<string, string[][]> = new Map();
+
+    /**
+     * Registers the supported highlighting scopes for the given language. Returns with a disposable that will unregister the scopes from this service on `dispose`.
+     * @param languageId the unique identifier of the language.
+     * @param scopes a lookup table of the supported (TextMate) scopes received from the server. Semantic highlighting will be be supported for a language if the `scopes` is empty.
+     */
+    register(languageId: string, scopes: string[][] | undefined): Disposable {
+        if (scopes && scopes.length > 0) {
+            this.logger.info(`Registering scopes for language: ${languageId}.`);
+            if (this.scopes.has(languageId)) {
+                const error = new Error(`The scopes are already registered for language: ${languageId}.`);
+                this.logger.error(error.message);
+                throw error;
+            }
+            this.scopes.set(languageId, scopes.map(scope => scope.slice(0)));
+            this.logger.info(`The scopes have been successfully registered for ${languageId}.`);
+            const unregister: (id: string) => void = this.unregister.bind(this);
+            return Disposable.create(() => unregister(languageId));
+        }
+        return Disposable.NULL;
+    }
+
+    protected unregister(languageId: string): void {
+        this.logger.info(`Unregistering scopes for language: ${languageId}.`);
+        if (!this.scopes.has(languageId)) {
+            const error = new Error(`No scopes were registered for language: ${languageId}.`);
+            this.logger.error(error.message);
+            throw error;
+        }
+        this.scopes.delete(languageId);
+        this.logger.info(`The scopes have been successfully unregistered for ${languageId}.`);
+    }
+
+    /**
+     * An array for TextMate scopes for the language.
+     * @param languageId the unique ID of the language.
+     * @param index the index of the TextMate scopes for the language.
+     */
+    protected scopesFor(languageId: string, index: number): string[] {
+        if (index < 0) {
+            throw new Error(`index >= 0. ${index}`);
+        }
+        const scopes = this.scopes.get(languageId);
+        if (!scopes) {
+            throw new Error(`No scopes were registered for language: ${languageId}.`);
+        }
+        if (scopes.length <= index) {
+            throw new Error(`Cannot find scopes by index. Language ID: ${languageId}. Index: ${index}. Scopes: ${scopes}`);
+        }
+        return scopes[index];
+    }
+
+    /**
+     * Decorates the editor with the semantic highlighting scopes.
+     * @param languageId the unique identifier of the language the resource belongs to.
+     * @param uri the URI of the resource to decorate in the editor.
+     * @param ranges the decoration ranges.
+     */
+    async decorate(languageId: string, uri: URI, ranges: SemanticHighlightingRange[]): Promise<void> {
+        // NOOP
+    }
+
+    /**
+     * Disposes the service.
+     */
+    dispose(): void {
+        // NOOP
+    }
+
+}
+
+export namespace SemanticHighlightingService {
+
+    const LENGTH_SHIFT = 0x0000010;
+    const SCOPE_MASK = 0x000FFFF;
+
+    /**
+     * The bare minimum representation of an individual semantic highlighting token.
+     */
+    export interface Token {
+
+        /**
+         * The offset of the token.
+         */
+        readonly character: number;
+
+        /**
+         * The length of the token.
+         */
+        readonly length: number;
+
+        /**
+         * The unique scope index per language.
+         */
+        readonly scope: number;
+    }
+
+    export namespace Token {
+
+        export function fromArray(tokens: number[]): Token[] {
+            if (tokens.length % 3 !== 0) {
+                throw new Error(`"Invalid tokens. 'tokens.length % 3 !== 0' Tokens length was: " + ${tokens.length}.`);
+            }
+            const result: Token[] = [];
+            for (let i = 0; i < tokens.length; i = i + 3) {
+                result.push({
+                    character: tokens[i],
+                    length: tokens[i + 1],
+                    scope: tokens[i + 2]
+                });
+            }
+            return result;
+        }
+
+    }
+
+    /**
+     * Converts the compact, `base64` string token into an array of tokens.
+     */
+    export function decode(tokens: string | undefined): SemanticHighlightingService.Token[] {
+        if (!tokens) {
+            return [];
+        }
+
+        const buffer = base64Decode(tokens);
+        const dataView = new DataView(buffer);
+        const result: SemanticHighlightingService.Token[] = [];
+
+        for (let i = 0; i < buffer.byteLength / Uint32Array.BYTES_PER_ELEMENT; i = i + 2) {
+            const character = dataView.getUint32(i * Uint32Array.BYTES_PER_ELEMENT);
+            const lengthAndScope = dataView.getUint32((i + 1) * Uint32Array.BYTES_PER_ELEMENT);
+            const length = lengthAndScope >> LENGTH_SHIFT;
+            const scope = lengthAndScope & SCOPE_MASK;
+            result.push({
+                character,
+                length,
+                scope
+            });
+        }
+        return result;
+    }
+
+    /**
+     * Encodes the array of tokens into a compact `base64` string representation.
+     */
+    export function encode(tokens: SemanticHighlightingService.Token[] | undefined): string {
+        if (!tokens || tokens.length === 0) {
+            return '';
+        }
+
+        const buffer = new ArrayBuffer(tokens.length * 2 * Uint32Array.BYTES_PER_ELEMENT);
+        const dataView = new DataView(buffer);
+        let j = 0;
+
+        for (let i = 0; i < tokens.length; i++) {
+            const { character, length, scope } = tokens[i];
+            let lengthAndScope = length;
+            lengthAndScope = lengthAndScope << LENGTH_SHIFT;
+            lengthAndScope |= scope;
+            dataView.setUint32(j++ * Uint32Array.BYTES_PER_ELEMENT, character);
+            dataView.setUint32(j++ * Uint32Array.BYTES_PER_ELEMENT, lengthAndScope);
+        }
+        return base64Encode(buffer);
+    }
+
+}
+
+export interface SemanticHighlightingRange extends Range {
+    /**
+     * The unique, internal index of the TextMate scope for the range.
+     */
+    readonly scope?: number;
+}
+
+export { Position, Range };

packages/filesystem/src/node/nsfw-watcher/nsfw-filesystem-watcher.ts

@@ -130,7 +130,7 @@ export class NsfwFileSystemWatcherServer implements FileSystemWatcherServer {
             this.watchers.delete(watcherId);
             this.debug('Stopping watching:', basePath);
             watcher.stop();
-            this.options.info('Stopped watching.');
+            this.options.info('Stopped watching:', basePath);
         });
         this.watcherOptions.set(watcherId, {
             ignored: options.ignored.map(pattern => new Minimatch(pattern))

packages/java/package.json

@@ -9,10 +9,12 @@
     "@theia/monaco": "^0.3.13",
     "@types/glob": "^5.0.30",
     "@types/tar": "4.0.0",
+    "@types/uuid": "^3.4.3",
     "glob": "^7.1.2",
     "mkdirp": "^0.5.0",
     "sha1": "^1.1.1",
-    "tar": "^4.0.0"
+    "tar": "^4.0.0",
+    "uuid": "^3.2.1"
   },
   "devDependencies": {
     "@theia/ext-scripts": "^0.3.13"
@@ -58,6 +60,6 @@
     "extends": "../../configs/nyc.json"
   },
   "ls": {
-    "downloadUrl": "https://www.eclipse.org/downloads/download.php?file=/che/che-ls-jdt/snapshots/che-jdt-language-server-latest.tar.gz&r=1"
+    "downloadUrl": "https://github.com/kittaakos/eclipse.jdt.ls-gh-715/raw/master/jdt-language-server-latest.tar.gz"
   }
 }

packages/java/scripts/download-jdt-ls.js

@@ -56,7 +56,6 @@ function downloadJavaServer() {
                 const statusCode = response.statusCode;
                 const redirectLocation = response.headers.location;
                 if (statusCode >= 300 && statusCode < 400 && redirectLocation) {
-                    console.log("redirect location: " + redirectLocation)
                     downloadWithRedirect(redirectLocation);
                 } else if (statusCode === 200) {
                     response.on('end', () => resolve());
@@ -92,4 +91,4 @@ downloadJavaServer().then(() => {
 }).catch(error => {
     console.error(error);
     process.exit(1);
-});
+});