Support semantic tokens

org.eclipse.jdt.ls.core/META-INF/MANIFEST.MF

@@ -57,7 +57,8 @@ Export-Package: org.eclipse.jdt.ls.core.internal;x-friends:="org.eclipse.jdt.ls.
  org.eclipse.jdt.ls.core.internal.managers;x-friends:="org.eclipse.jdt.ls.tests,org.eclipse.jdt.ls.tests.syntaxserver",
  org.eclipse.jdt.ls.core.internal.preferences;x-friends:="org.eclipse.jdt.ls.tests,org.eclipse.jdt.ls.tests.syntaxserver",
  org.eclipse.jdt.ls.core.internal.syntaxserver;x-friends:="org.eclipse.jdt.ls.tests.syntaxserver",
- org.eclipse.jdt.ls.core.internal.text.correction;x-friends:="org.eclipse.jdt.ls.tests"
+ org.eclipse.jdt.ls.core.internal.text.correction;x-friends:="org.eclipse.jdt.ls.tests",
+ org.eclipse.jdt.ls.core.internal.semantictokens;x-friends:="org.eclipse.jdt.ls.tests"
 Bundle-ClassPath: lib/jsoup-1.9.2.jar,
  lib/remark-1.2.0.jar,
  .

org.eclipse.jdt.ls.core/plugin.xml

@@ -81,6 +81,12 @@
             <command
                   id="java.project.refreshDiagnostics">
             </command>
+            <command
+                  id="java.project.provideSemanticTokens">
+            </command>
+            <command
+                  id="java.project.getSemanticTokensLegend">
+            </command>
       </delegateCommandHandler>
    </extension>
    <extension
@@ -89,7 +95,7 @@
          <importer
             id = "gradleProjectImporter"
             order ="300"
-            class = "org.eclipse.jdt.ls.core.internal.managers.GradleProjectImporter"/> 
+            class = "org.eclipse.jdt.ls.core.internal.managers.GradleProjectImporter"/>
          <importer
             id = "mavenProjectImporter"
             order = "400"

org.eclipse.jdt.ls.core/src/org/eclipse/jdt/ls/core/internal/JDTDelegateCommandHandler.java

@@ -21,6 +21,7 @@
 import org.eclipse.jdt.ls.core.internal.commands.DiagnosticsCommand;
 import org.eclipse.jdt.ls.core.internal.commands.OrganizeImportsCommand;
 import org.eclipse.jdt.ls.core.internal.commands.ProjectCommand;
+import org.eclipse.jdt.ls.core.internal.commands.SemanticTokensCommand;
 import org.eclipse.jdt.ls.core.internal.commands.SourceAttachmentCommand;
 import org.eclipse.jdt.ls.core.internal.commands.ProjectCommand.ClasspathOptions;
 import org.eclipse.lsp4j.WorkspaceEdit;
@@ -68,6 +69,10 @@ public Object executeCommand(String commandId, List<Object> arguments, IProgress
 					return ProjectCommand.isTestFile((String) arguments.get(0));
 				case "java.project.refreshDiagnostics":
 					return DiagnosticsCommand.refreshDiagnostics((String) arguments.get(0), (String) arguments.get(1), (boolean) arguments.get(2));
+				case "java.project.provideSemanticTokens":
+					return SemanticTokensCommand.provide((String) arguments.get(0));
+				case "java.project.getSemanticTokensLegend":
+					return SemanticTokensCommand.getLegend();
 				default:
 					break;
 			}

org.eclipse.jdt.ls.core/src/org/eclipse/jdt/ls/core/internal/commands/SemanticTokensCommand.java

@@ -0,0 +1,61 @@
+/*******************************************************************************
+ * Copyright (c) 2020 Microsoft Corporation and others.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License 2.0
+ * which accompanies this distribution, and is available at
+ * https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *
+ * Contributors:
+ *     Microsoft Corporation - initial API and implementation
+ *******************************************************************************/
+
+package org.eclipse.jdt.ls.core.internal.commands;
+
+import java.util.Collections;
+
+import org.eclipse.core.runtime.NullProgressMonitor;
+import org.eclipse.jdt.core.ICompilationUnit;
+import org.eclipse.jdt.core.JavaModelException;
+import org.eclipse.jdt.core.dom.CompilationUnit;
+import org.eclipse.jdt.core.manipulation.CoreASTProvider;
+import org.eclipse.jdt.ls.core.internal.JDTUtils;
+import org.eclipse.jdt.ls.core.internal.JavaLanguageServerPlugin;
+import org.eclipse.jdt.ls.core.internal.handlers.JsonRpcHelpers;
+import org.eclipse.jdt.ls.core.internal.semantictokens.SemanticTokenManager;
+import org.eclipse.jdt.ls.core.internal.semantictokens.SemanticTokens;
+import org.eclipse.jdt.ls.core.internal.semantictokens.SemanticTokensLegend;
+import org.eclipse.jdt.ls.core.internal.semantictokens.SemanticTokensVisitor;
+import org.eclipse.jface.text.IDocument;
+
+public class SemanticTokensCommand {
+
+    public static SemanticTokens provide(String uri) {
+
+        IDocument document = null;
+
+        ICompilationUnit cu = JDTUtils.resolveCompilationUnit(uri);
+        if (cu != null) {
+            try {
+                document = JsonRpcHelpers.toDocument(cu.getBuffer());
+            } catch (JavaModelException e) {
+                JavaLanguageServerPlugin.logException("Failed to provide semantic tokens for " + uri, e);
+            }
+        }
+        if (document == null) {
+            return new SemanticTokens(Collections.emptyList());
+        }
+
+        SemanticTokensVisitor collector = new SemanticTokensVisitor(document, SemanticTokenManager.getInstance());
+        CompilationUnit root = CoreASTProvider.getInstance().getAST(cu, CoreASTProvider.WAIT_YES, new NullProgressMonitor());
+        root.accept(collector);
+        return collector.getSemanticTokens();
+    }
+
+    public static SemanticTokensLegend getLegend() {
+        return SemanticTokenManager.getInstance().getLegend();
+    }
+
+
+}

org.eclipse.jdt.ls.core/src/org/eclipse/jdt/ls/core/internal/semantictokens/ITokenModifier.java

@@ -0,0 +1,31 @@
+/*******************************************************************************
+ * Copyright (c) 2020 Microsoft Corporation and others.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License 2.0
+ * which accompanies this distribution, and is available at
+ * https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *
+ * Contributors:
+ *     Microsoft Corporation - initial API and implementation
+ *******************************************************************************/
+
+package org.eclipse.jdt.ls.core.internal.semantictokens;
+
+import org.eclipse.jdt.core.dom.IBinding;
+
+public interface ITokenModifier {
+    /**
+     * Determine whether this modifier applies to a named entity.
+     * @param binding corresponding binding of the named entity.
+     * @return <code>true</code> if this modifier applies to the binding and
+	 *    <code>false</code> otherwise
+     */
+    public boolean applies(IBinding binding);
+
+    /**
+     * identifier of the modifier
+     */
+    public String toString();
+}

org.eclipse.jdt.ls.core/src/org/eclipse/jdt/ls/core/internal/semantictokens/SemanticTokenManager.java

@@ -0,0 +1,53 @@
+/*******************************************************************************
+ * Copyright (c) 2020 Microsoft Corporation and others.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License 2.0
+ * which accompanies this distribution, and is available at
+ * https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *
+ * Contributors:
+ *     Microsoft Corporation - initial API and implementation
+ *******************************************************************************/
+
+package org.eclipse.jdt.ls.core.internal.semantictokens;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Collectors;
+
+public class SemanticTokenManager {
+    private TokenModifiers tokenModifiers;
+    private List<TokenType> tokenTypes;
+    private SemanticTokensLegend legend;
+
+    private SemanticTokenManager() {
+        this.tokenModifiers = new TokenModifiers();
+        this.tokenTypes = Arrays.asList(TokenType.values());
+        List<String> modifiers = tokenModifiers.list().stream().map(mod -> mod.toString()).collect(Collectors.toList());
+        List<String> types = tokenTypes.stream().map(TokenType::toString).collect(Collectors.toList());
+        this.legend = new SemanticTokensLegend(types, modifiers);
+    }
+
+    private static class SingletonHelper{
+        private static final SemanticTokenManager INSTANCE = new SemanticTokenManager();
+    }
+
+    public static SemanticTokenManager getInstance(){
+        return SingletonHelper.INSTANCE;
+    }
+
+    public SemanticTokensLegend getLegend() {
+        return this.legend;
+    }
+
+    public TokenModifiers getTokenModifiers() {
+        return tokenModifiers;
+    }
+
+    public List<TokenType> getTokenTypes() {
+        return tokenTypes;
+    }
+
+}

org.eclipse.jdt.ls.core/src/org/eclipse/jdt/ls/core/internal/semantictokens/SemanticTokens.java

@@ -0,0 +1,102 @@
+/*******************************************************************************
+ * Copyright (c) 2020 Microsoft Corporation and others.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License 2.0
+ * which accompanies this distribution, and is available at
+ * https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *
+ * Contributors:
+ *     Microsoft Corporation - initial API and implementation
+ *******************************************************************************/
+
+package org.eclipse.jdt.ls.core.internal.semantictokens;
+
+import java.util.List;
+
+import org.eclipse.lsp4j.util.Preconditions;
+
+public class SemanticTokens {
+
+    /**
+     * Tokens in a file are represented as an array of integers. The position of each token is expressed relative to
+     * the token before it, because most tokens remain stable relative to each other when edits are made in a file.
+     *
+     * ---
+     * In short, each token takes 5 integers to represent, so a specific token `i` in the file consists of the following array indices:
+     *  - at index `5*i`   - `deltaLine`: token line number, relative to the previous token
+     *  - at index `5*i+1` - `deltaStart`: token start character, relative to the previous token (relative to 0 or the previous token's start if they are on the same line)
+     *  - at index `5*i+2` - `length`: the length of the token. A token cannot be multiline.
+     *  - at index `5*i+3` - `tokenType`: will be looked up in `SemanticTokensLegend.tokenTypes`. We currently ask that `tokenType` < 65536.
+     *  - at index `5*i+4` - `tokenModifiers`: each set bit will be looked up in `SemanticTokensLegend.tokenModifiers`
+     *
+     * ---
+     * ### How to encode tokens
+     *
+     * Here is an example for encoding a file with 3 tokens in a uint32 array:
+     * ```
+     *    { line: 2, startChar:  5, length: 3, tokenType: "property",  tokenModifiers: ["private", "static"] },
+     *    { line: 2, startChar: 10, length: 4, tokenType: "type",      tokenModifiers: [] },
+     *    { line: 5, startChar:  2, length: 7, tokenType: "class",     tokenModifiers: [] }
+     * ```
+     *
+     * 1. First of all, a legend must be devised. This legend must be provided up-front and capture all possible token types.
+     * For this example, we will choose the following legend which must be passed in when registering the provider:
+     * ```
+     *    tokenTypes: ['property', 'type', 'class'],
+     *    tokenModifiers: ['private', 'static']
+     * ```
+     *
+     * 2. The first transformation step is to encode `tokenType` and `tokenModifiers` as integers using the legend. Token types are looked
+     * up by index, so a `tokenType` value of `1` means `tokenTypes[1]`. Multiple token modifiers can be set by using bit flags,
+     * so a `tokenModifier` value of `3` is first viewed as binary `0b00000011`, which means `[tokenModifiers[0], tokenModifiers[1]]` because
+     * bits 0 and 1 are set. Using this legend, the tokens now are:
+     * ```
+     *    { line: 2, startChar:  5, length: 3, tokenType: 0, tokenModifiers: 3 },
+     *    { line: 2, startChar: 10, length: 4, tokenType: 1, tokenModifiers: 0 },
+     *    { line: 5, startChar:  2, length: 7, tokenType: 2, tokenModifiers: 0 }
+     * ```
+     *
+     * 3. The next step is to represent each token relative to the previous token in the file. In this case, the second token
+     * is on the same line as the first token, so the `startChar` of the second token is made relative to the `startChar`
+     * of the first token, so it will be `10 - 5`. The third token is on a different line than the second token, so the
+     * `startChar` of the third token will not be altered:
+     * ```
+     *    { deltaLine: 2, deltaStartChar: 5, length: 3, tokenType: 0, tokenModifiers: 3 },
+     *    { deltaLine: 0, deltaStartChar: 5, length: 4, tokenType: 1, tokenModifiers: 0 },
+     *    { deltaLine: 3, deltaStartChar: 2, length: 7, tokenType: 2, tokenModifiers: 0 }
+     * ```
+     *
+     * 4. Finally, the last step is to inline each of the 5 fields for a token in a single array, which is a memory friendly representation:
+     * ```
+     *    // 1st token,  2nd token,  3rd token
+     *    [  2,5,3,0,3,  0,5,4,1,0,  3,2,7,2,0 ]
+     * ```
+     */
+    private final List<Integer> data;
+
+    /**
+     * The result id of the tokens (optional).
+     *
+     * This is the id that will be passed to incrementally calculate semantic tokens.
+     */
+    private final String resultId;
+
+    public SemanticTokens(List<Integer> data) {
+        this(data, null);
+    }
+
+    public SemanticTokens(List<Integer> data, String resultId) {
+        this.data = Preconditions.<List<Integer>>checkNotNull(data, "data");
+        this.resultId = resultId;
+    }
+
+    public String getResultId() {
+        return resultId;
+    }
+
+    public List<Integer> getData() {
+        return data;
+    }
+}

org.eclipse.jdt.ls.core/src/org/eclipse/jdt/ls/core/internal/semantictokens/SemanticTokensLegend.java

@@ -0,0 +1,34 @@
+/*******************************************************************************
+ * Copyright (c) 2020 Microsoft Corporation and others.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License 2.0
+ * which accompanies this distribution, and is available at
+ * https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *
+ * Contributors:
+ *     Microsoft Corporation - initial API and implementation
+ *******************************************************************************/
+
+package org.eclipse.jdt.ls.core.internal.semantictokens;
+
+import java.util.List;
+
+public class SemanticTokensLegend {
+    private final List<String> tokenTypes;
+    private final List<String> tokenModifiers;
+
+    public SemanticTokensLegend(List<String> tokenTypes, List<String> tokenModifiers){
+        this.tokenTypes = tokenTypes;
+        this.tokenModifiers = tokenModifiers;
+    };
+
+    public List<String> getTokenTypes() {
+        return this.tokenTypes;
+    }
+
+    public List<String> getTokenModifiers() {
+        return this.tokenModifiers;
+    }
+}