feat: Extend TextMate Info Hover with scope list

org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/model/TMModel.java

@@ -20,6 +20,7 @@
 import java.lang.System.Logger;
 import java.time.Duration;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 import java.util.concurrent.BlockingQueue;
@@ -255,7 +256,7 @@ private void revalidateTokens() {
 				// check if complete line was tokenized
 				if (r.stoppedEarly) {
 					// treat the rest of the line as one default token
-					r.tokens.add(new TMToken(r.actualStopOffset, ""));
+					r.tokens.add(new TMToken(r.actualStopOffset, "", Collections.emptyList()));
 					// Use the line's starting state as end state in case of incomplete tokenization
 					r.endState = currLineTokens.startState;
 				}

org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/model/TMToken.java

@@ -16,6 +16,8 @@
  */
 package org.eclipse.tm4e.core.model;
 
+import java.util.List;
+
 import org.eclipse.jdt.annotation.Nullable;
 
 /**
@@ -28,10 +30,12 @@ public final class TMToken {
 	public final int startIndex;
 	public final String type;
 	// public readonly language: string
+	public final List<String> scopes;
 
-	public TMToken(final int startIndex, final String type) {
+	public TMToken(final int startIndex, final String type, final List<String> scopes) {
 		this.startIndex = startIndex;
 		this.type = type;
+		this.scopes = scopes;
 	}
 
 	@Override

org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/model/TMTokenizationSupport.java

@@ -29,6 +29,7 @@
 import org.eclipse.jdt.annotation.Nullable;
 import org.eclipse.tm4e.core.grammar.IGrammar;
 import org.eclipse.tm4e.core.grammar.IStateStack;
+import org.eclipse.tm4e.core.grammar.IToken;
 import org.eclipse.tm4e.core.internal.grammar.StateStack;
 import org.eclipse.tm4e.core.internal.utils.MoreCollections;
 import org.eclipse.tm4e.core.internal.utils.StringUtils;
@@ -83,13 +84,12 @@ public TokenizationResult tokenize(final String line,
 		// Create the result early and fill in the tokens later
 		final var tmTokens = new ArrayList<TMToken>(tokens.length < 10 ? tokens.length : 10);
 		String lastTokenType = null;
-		for (final var token : tokens) {
+		for (final IToken token : tokens) {
 			final String tokenType = decodeTextMateTokenCached.apply(decodeMap, token.getScopes());
 
 			// do not push a new token if the type is exactly the same (also helps with ligatures)
 			if (!tokenType.equals(lastTokenType)) {
-				final int tokenStartIndex = token.getStartIndex();
-				tmTokens.add(new TMToken(tokenStartIndex + offsetDelta, tokenType));
+				tmTokens.add(new TMToken(token.getStartIndex() + offsetDelta, tokenType, token.getScopes()));
 				lastTokenType = tokenType;
 			}
 		}

org.eclipse.tm4e.ui/src/main/java/org/eclipse/tm4e/ui/internal/hover/TMTokenTextHover.java

@@ -35,9 +35,11 @@ public class TMTokenTextHover implements ITextHover, ITextHoverExtension {
 
 	private static final class RegionWithTMToken extends Region {
 		final TMToken token;
+		final String text;
 
-		RegionWithTMToken(final int offset, final int length, final TMToken token) {
+		RegionWithTMToken(final int offset, final int length, final String text, final TMToken token) {
 			super(offset, length);
+			this.text = text;
 			this.token = token;
 		}
 	}
@@ -57,7 +59,12 @@ protected IInformationControl doCreateInformationControl(final @NonNullByDefault
 	public @Nullable String getHoverInfo(final @NonNullByDefault({}) ITextViewer textViewer,
 			final @NonNullByDefault({}) IRegion hoverRegion) {
 		if (hoverRegion instanceof final RegionWithTMToken regionWithToken) {
-			return "<b>TextMate scope:</b> " + regionWithToken.token.type;
+			final var text = regionWithToken.text.replace(' ', '·').replace('\t', '→');
+			return "<b>" + text + "</b> (" + text.length()
+					+ " chars)<br>"
+					+ "<br>"
+					+ "<b>Token Type:</b> " + regionWithToken.token.type + "<br>"
+					+ "<b>TextMate Scopes:</b> <li>" + String.join("<li>", regionWithToken.token.scopes);
 		}
 		return null;
 	}
@@ -97,12 +104,11 @@ protected IInformationControl doCreateInformationControl(final @NonNullByDefault
 			if (hoveredToken == null)
 				return null;
 
-			return new RegionWithTMToken(
-					lineStartOffset + hoveredToken.startIndex,
-					nextToken == null
-							? doc.getLineLength(lineIndex) - hoveredToken.startIndex
-							: nextToken.startIndex - hoveredToken.startIndex,
-					hoveredToken);
+			final int regionOffset = lineStartOffset + hoveredToken.startIndex;
+			final int regionLength = nextToken == null
+					? doc.getLineLength(lineIndex) - hoveredToken.startIndex
+					: nextToken.startIndex - hoveredToken.startIndex;
+			return new RegionWithTMToken(regionOffset, regionLength, doc.get(regionOffset, regionLength), hoveredToken);
 		} catch (final BadLocationException e) {
 			return null;
 		}