backport upstream changes (stable ruleIds)

org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/grammar/AttributedScopeStack.java

@@ -11,9 +11,14 @@
  */
 package org.eclipse.tm4e.core.internal.grammar;
 
+import static org.eclipse.tm4e.core.internal.utils.NullSafetyHelper.castNonNull;
+
+import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 
+import org.eclipse.jdt.annotation.NonNullByDefault;
 import org.eclipse.jdt.annotation.Nullable;
 import org.eclipse.tm4e.core.internal.grammar.tokenattrs.EncodedTokenAttributes;
 import org.eclipse.tm4e.core.internal.theme.FontStyle;
@@ -23,13 +28,30 @@
 
 /**
  * @see <a href=
- *      "https://github.com/microsoft/vscode-textmate/blob/e8d1fc5d04b2fc91384c7a895f6c9ff296a38ac8/src/grammar.ts#L417">
- *      github.com/microsoft/vscode-textmate/blob/main/src/grammar.ts</a>
+ *      "https://github.com/microsoft/vscode-textmate/blob/5c3f08bea898b354a60a37900a33c5437aa72f5a/src/grammar/grammar.ts#L418">
+ *      github.com/microsoft/vscode-textmate/blob/main/src/grammar/grammar.ts</a>
  */
 public final class AttributedScopeStack {
 
+	@NonNullByDefault({}) // https://github.com/eclipse-jdt/eclipse.jdt.core/issues/233
+	record Frame(int encodedTokenAttributes, List<String> scopeNames) {
+	}
+
 	private static final Splitter BY_SPACE_SPLITTER = Splitter.on(' ');
 
+	@Nullable
+	static AttributedScopeStack fromExtension(final @Nullable AttributedScopeStack namesScopeList,
+		final List<AttributedScopeStack.Frame> contentNameScopesList) {
+		var current = namesScopeList;
+		@Nullable
+		ScopeStack scopeNames = namesScopeList != null ? namesScopeList.scopePath : null;
+		for (final var frame : contentNameScopesList) {
+			scopeNames = ScopeStack.push(scopeNames, frame.scopeNames);
+			current = new AttributedScopeStack(current, castNonNull(scopeNames), frame.encodedTokenAttributes);
+		}
+		return current;
+	}
+
 	public static AttributedScopeStack createRoot(final String scopeName,
 		final int /*EncodedTokenAttributes*/ tokenAttributes) {
 		return new AttributedScopeStack(null, new ScopeStack(null, scopeName), tokenAttributes);
@@ -53,13 +75,12 @@ public String scopeName() {
 		return this.scopePath.scopeName;
 	}
 
-	@Nullable
-	private final AttributedScopeStack parent;
+	private final @Nullable AttributedScopeStack parent;
 	private final ScopeStack scopePath;
 	final int tokenAttributes;
 
 	public AttributedScopeStack(
-		@Nullable final AttributedScopeStack parent,
+		final @Nullable AttributedScopeStack parent,
 		final ScopeStack scopePath,
 		final int tokenAttributes) {
 		this.parent = parent;
@@ -68,10 +89,10 @@ public AttributedScopeStack(
 	}
 
 	public boolean equals(final AttributedScopeStack other) {
-		return _equals(this, other);
+		return equals(this, other);
 	}
 
-	private static boolean _equals(
+	public static boolean equals(
 		@Nullable AttributedScopeStack a,
 		@Nullable AttributedScopeStack b) {
 		do {
@@ -102,7 +123,7 @@ private static boolean _equals(
 	public static int mergeAttributes(
 		final int existingTokenAttributes,
 		final BasicScopeAttributes basicScopeAttributes,
-		@Nullable final StyleAttributes styleAttributes) {
+		final @Nullable StyleAttributes styleAttributes) {
 		var fontStyle = FontStyle.NotSet;
 		var foreground = 0;
 		var background = 0;
@@ -123,7 +144,7 @@ public static int mergeAttributes(
 			background);
 	}
 
-	AttributedScopeStack pushAttributed(@Nullable final String scopePath, final Grammar grammar) {
+	AttributedScopeStack pushAttributed(final @Nullable String scopePath, final Grammar grammar) {
 		if (scopePath == null) {
 			return this;
 		}
@@ -159,4 +180,22 @@ private AttributedScopeStack _pushAttributed(
 	List<String> getScopeNames() {
 		return this.scopePath.getSegments();
 	}
+
+	public List<AttributedScopeStack.Frame> getExtensionIfDefined(final @Nullable AttributedScopeStack base) {
+		final var result = new ArrayList<AttributedScopeStack.Frame>();
+		var self = this;
+
+		while (self != null && self != base) {
+			final var parent = self.parent;
+			result.add(new AttributedScopeStack.Frame(
+				self.tokenAttributes,
+				self.scopePath.getExtensionIfDefined(parent != null ? parent.scopePath : null)));
+			self = self.parent;
+		}
+		if (self == base) {
+			Collections.reverse(result);
+			return result;
+		}
+		return Collections.emptyList();
+	}
 }

org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/grammar/BalancedBracketSelectors.java

@@ -24,8 +24,8 @@
 
 /**
  * @see <a href=
- *      "https://github.com/microsoft/vscode-textmate/blob/e8d1fc5d04b2fc91384c7a895f6c9ff296a38ac8/src/grammar.ts#L808">
- *      github.com/microsoft/vscode-textmate/blob/main/src/grammar.ts</a>
+ *      "https://github.com/microsoft/vscode-textmate/blob/5c3f08bea898b354a60a37900a33c5437aa72f5a/src/grammar/grammar.ts#L898">
+ *      github.com/microsoft/vscode-textmate/blob/main/src/grammar/grammar.ts</a>
  */
 public class BalancedBracketSelectors {
 	private final Matcher<List<String>>[] balancedBracketScopes;

org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/grammar/Grammar.java

@@ -53,7 +53,7 @@
  * TextMate grammar implementation.
  *
  * @see <a href=
- *      "https://github.com/microsoft/vscode-textmate/blob/e8d1fc5d04b2fc91384c7a895f6c9ff296a38ac8/src/grammar/grammar.ts#L99">
+ *      "https://github.com/microsoft/vscode-textmate/blob/5c3f08bea898b354a60a37900a33c5437aa72f5a/src/grammar/grammar.ts#L98">
  *      github.com/microsoft/vscode-textmate/blob/main/src/grammar/grammar.ts</a>
  */
 public final class Grammar implements IGrammar, IRuleFactoryHelper {
@@ -284,6 +284,8 @@ private <T> T _tokenize(
 				this._grammar.getRepository().getSelf(),
 				this,
 				this._grammar.getRepository());
+			// This ensures ids are deterministic, and thus equal in renderer and webworker.
+			this.getInjections();
 		}
 
 		boolean isFirstLine;

org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/grammar/Injection.java

@@ -24,8 +24,8 @@
 
 /**
  * @see <a href=
- *      "https://github.com/microsoft/vscode-textmate/blob/e8d1fc5d04b2fc91384c7a895f6c9ff296a38ac8/src/grammar.ts#L50">
- *      github.com/microsoft/vscode-textmate/blob/main/src/grammar.ts</a>
+ *      "https://github.com/microsoft/vscode-textmate/blob/5c3f08bea898b354a60a37900a33c5437aa72f5a/src/grammar/grammar.ts#L49">
+ *      github.com/microsoft/vscode-textmate/blob/main/src/grammar/grammar.ts</a>
  */
 final class Injection {
 

org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/grammar/LineTokenizer.java

@@ -17,13 +17,14 @@
 package org.eclipse.tm4e.core.internal.grammar;
 
 import static java.lang.System.Logger.Level.*;
-import static org.eclipse.tm4e.core.internal.utils.NullSafetyHelper.*;
+import static org.eclipse.tm4e.core.internal.utils.NullSafetyHelper.castNonNull;
 
 import java.lang.System.Logger;
 import java.lang.System.Logger.Level;
 import java.time.Duration;
 import java.util.ArrayDeque;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 
 import org.eclipse.jdt.annotation.NonNull;
@@ -192,7 +193,7 @@ private void scanNext() {
 			final StateStack beforePush = stack;
 			// push it on the stack rule
 			final var scopeName = rule.getName(lineText.content, captureIndices);
-			final var nameScopesList = stack.contentNameScopesList.pushAttributed(
+			final var nameScopesList = castNonNull(stack.contentNameScopesList).pushAttributed(
 				scopeName,
 				grammar);
 			stack = stack.push(
@@ -387,7 +388,8 @@ private MatchInjectionsResult matchInjections(final List<Injection> injections,
 		var bestMatchRuleId = RuleId.END_RULE;
 		var bestMatchResultPriority = 0;
 
-		final var scopes = stack.contentNameScopesList.getScopeNames();
+		final List<String> scopes = stack.contentNameScopesList != null ? stack.contentNameScopesList.getScopeNames()
+			: Collections.emptyList();
 
 		for (int i = 0, len = injections.size(); i < len; i++) {
 			final var injection = injections.get(i);
@@ -484,7 +486,7 @@ private void handleCaptures(final Grammar grammar, final OnigString lineText, fi
 			if (retokenizeCapturedWithRuleId.notEquals(RuleId.NO_RULE)) {
 				// the capture requires additional matching
 				final var scopeName = captureRule.getName(lineTextContent, captureIndices);
-				final var nameScopesList = stack.contentNameScopesList.pushAttributed(scopeName, grammar);
+				final var nameScopesList = castNonNull(stack.contentNameScopesList).pushAttributed(scopeName, grammar);
 				final var contentName = captureRule.getContentName(lineTextContent, captureIndices);
 				final var contentNameScopesList = nameScopesList.pushAttributed(contentName, grammar);
 
@@ -501,7 +503,7 @@ private void handleCaptures(final Grammar grammar, final OnigString lineText, fi
 			if (captureRuleScopeName != null) {
 				// push
 				final var base = localStack.isEmpty() ? stack.contentNameScopesList : localStack.getLast().scopes;
-				final var captureRuleScopesList = base.pushAttributed(captureRuleScopeName, grammar);
+				final var captureRuleScopesList = castNonNull(base).pushAttributed(captureRuleScopeName, grammar);
 				localStack.add(new LocalStackElement(captureRuleScopesList, captureIndex.end));
 			}
 		}

org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/grammar/LineTokens.java

@@ -34,8 +34,8 @@
 
 /**
  * @see <a href=
- *      "https://github.com/microsoft/vscode-textmate/blob/e8d1fc5d04b2fc91384c7a895f6c9ff296a38ac8/src/grammar.ts#L855">
- *      github.com/microsoft/vscode-textmate/blob/main/src/grammar.ts</a>
+ *      "https://github.com/microsoft/vscode-textmate/blob/5c3f08bea898b354a60a37900a33c5437aa72f5a/src/grammar/grammar.ts#L945">
+ *      github.com/microsoft/vscode-textmate/blob/main/src/grammar/grammar.ts</a>
  */
 final class LineTokens {
 
@@ -89,13 +89,13 @@ void produce(final StateStack stack, final int endIndex) {
 		this.produceFromScopes(stack.contentNameScopesList, endIndex);
 	}
 
-	void produceFromScopes(final AttributedScopeStack scopesList, final int endIndex) {
+	void produceFromScopes(@Nullable final AttributedScopeStack scopesList, final int endIndex) {
 		if (this._lastTokenEndIndex >= endIndex) {
 			return;
 		}
 
 		if (this._emitBinaryTokens) {
-			int metadata = scopesList.tokenAttributes;
+			int metadata = scopesList != null ? scopesList.tokenAttributes : 0;
 			var containsBalancedBrackets = false;
 			final var balancedBracketSelectors = this.balancedBracketSelectors;
 			if (balancedBracketSelectors != null && balancedBracketSelectors.matchesAlways()) {
@@ -106,7 +106,7 @@ void produceFromScopes(final AttributedScopeStack scopesList, final int endIndex
 				|| balancedBracketSelectors != null
 					&& !balancedBracketSelectors.matchesAlways() && !balancedBracketSelectors.matchesNever()) {
 				// Only generate scope array when required to improve performance
-				final var scopes = scopesList.getScopeNames();
+				final List<String> scopes = scopesList != null ? scopesList.getScopeNames() : Collections.emptyList();
 				for (final var tokenType : _tokenTypeOverrides) {
 					if (tokenType.matcher.matches(scopes)) {
 						metadata = EncodedTokenAttributes.set(
@@ -142,7 +142,7 @@ void produceFromScopes(final AttributedScopeStack scopesList, final int endIndex
 			}
 
 			if (LOGGER.isLoggable(TRACE)) {
-				final var scopes = scopesList.getScopeNames();
+				final List<String> scopes = scopesList != null ? scopesList.getScopeNames() : Collections.emptyList();
 				LOGGER.log(TRACE, "  token: |" + this._lineText
 					.substring(this._lastTokenEndIndex >= 0 ? this._lastTokenEndIndex : 0, endIndex)
 					.replace("\n", "\\n")
@@ -159,7 +159,7 @@ void produceFromScopes(final AttributedScopeStack scopesList, final int endIndex
 			return;
 		}
 
-		final List<String> scopes = scopesList.getScopeNames();
+		final List<String> scopes = scopesList != null ? scopesList.getScopeNames() : Collections.emptyList();
 
 		if (LOGGER.isLoggable(TRACE)) {
 			LOGGER.log(TRACE, "  token: |" +

org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/grammar/ScopeStack.java

@@ -24,11 +24,19 @@
 
 /**
  * @see <a href=
- *      "https://github.com/microsoft/vscode-textmate/blob/e8d1fc5d04b2fc91384c7a895f6c9ff296a38ac8/src/theme.ts#L101">
+ *      "https://github.com/microsoft/vscode-textmate/blob/5c3f08bea898b354a60a37900a33c5437aa72f5a/src/theme.ts#L101">
  *      github.com/microsoft/vscode-textmate/blob/main/src/theme.ts</a>
  */
 public class ScopeStack {
 
+	@Nullable
+	static ScopeStack push(@Nullable ScopeStack path, final List<String> scopeNames) {
+		for (final var name : scopeNames) {
+			path = new ScopeStack(path, name);
+		}
+		return path;
+	}
+
 	public static ScopeStack from(final String first) {
 		return new ScopeStack(null, first);
 	}
@@ -80,4 +88,31 @@ public List<String> getSegments() {
 	public String toString() {
 		return String.join(" ", getSegments());
 	}
+
+	public boolean isExtending(final ScopeStack other) {
+		if (this == other) {
+			return true;
+		}
+
+		final var parent = this.parent;
+		if (parent == null) {
+			return false;
+		}
+		return parent.isExtending(other);
+	}
+
+	public List<String> getExtensionIfDefined(@Nullable final ScopeStack base) {
+		final var result = new ArrayList<String>();
+		@Nullable
+		ScopeStack item = this;
+		while (item != null && item != base) {
+			result.add(item.scopeName);
+			item = item.parent;
+		}
+		if (item == base) {
+			Collections.reverse(result);
+			return result;
+		}
+		return Collections.emptyList();
+	}
 }