Fix eclipse-tm4e#654 cannot reference grammar from 3rd party plugin

org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/registry/SyncRegistry.java

@@ -29,13 +29,14 @@
 import org.eclipse.tm4e.core.internal.grammar.raw.IRawGrammar;
 import org.eclipse.tm4e.core.internal.theme.StyleAttributes;
 import org.eclipse.tm4e.core.internal.theme.Theme;
+import org.eclipse.tm4e.core.internal.utils.ScopeNames;
 
 /**
  * @see <a href=
  *      "https://github.com/microsoft/vscode-textmate/blob/88baacf1a6637c5ec08dce18cea518d935fcf0a0/src/registry.ts">
  *      github.com/microsoft/vscode-textmate/blob/main/src/registry.ts</a>
  */
-public final class SyncRegistry implements IGrammarRepository, IThemeProvider {
+public class SyncRegistry implements IGrammarRepository, IThemeProvider {
 
 	private final Map<String, Grammar> _grammars = new HashMap<>();
 	private final Map<String, @Nullable IRawGrammar> _rawGrammars = new HashMap<>();
@@ -66,9 +67,18 @@ public void addGrammar(final IRawGrammar grammar, @Nullable final Collection<Str
 	}
 
 	@Override
-	@Nullable
-	public IRawGrammar lookup(final String scopeName) {
-		return this._rawGrammars.get(scopeName);
+	public @Nullable IRawGrammar lookup(final String scopeName) {
+		IRawGrammar grammar = this._rawGrammars.get(scopeName);
+		if (grammar == null) {
+			// this code is specific to the tm4e project and not from upstream:
+			// if no grammar was found for the given scopeName, check if the scopeName is qualified, e.g. "source.mylang@com.example.mylang.plugin"
+			// and if so try the unqualified scopeName "source.mylang" as fallback. See org.eclipse.tm4e.registry.internal.TMScope and
+			// for more details org.eclipse.tm4e.registry.internal.AbstractGrammarRegistryManager.getGrammarFor(IContentType...)
+			final var scopeNameWithoutContributor = ScopeNames.withoutContributor(scopeName);
+			if (!scopeNameWithoutContributor.equals(scopeName))
+				grammar = this._rawGrammars.get(scopeNameWithoutContributor);
+		}
+		return grammar;
 	}
 
 	@Override

org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/utils/ScopeNames.java

@@ -0,0 +1,49 @@
+/**
+ * Copyright (c) 2024 Sebastian Thomschke and others.
+ *
+ * This program and the accompanying materials are made
+ * available under the terms of the Eclipse Public License 2.0
+ * which is available at https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *
+ * Contributors:
+ * - Sebastian Thomschke - initial API and implementation
+ */
+package org.eclipse.tm4e.core.internal.utils;
+
+import org.eclipse.jdt.annotation.Nullable;
+
+/**
+ * Utility class to deal with plugin scoped TextMate Scope Names, e.g. "source.batchfile@com.example.myplugin"
+ */
+public final class ScopeNames {
+
+	public static final char CONTRIBUTOR_SEPARATOR = '@';
+
+	public static @Nullable String getContributor(final String scopeName) {
+		final int separatorAt = scopeName.indexOf(CONTRIBUTOR_SEPARATOR);
+		if (separatorAt == -1) {
+			return "";
+		}
+		return scopeName.substring(separatorAt + 1);
+	}
+
+	/**
+	 * @return true if a scope name is suffixed by the contributing plugin id, e.g. "source.batchfile@com.example.myplugin"
+	 */
+	public static boolean hasContributor(final String scopeName) {
+		return scopeName.indexOf(CONTRIBUTOR_SEPARATOR) > -1;
+	}
+
+	public static String withoutContributor(final String scopeName) {
+		final int separatorAt = scopeName.indexOf(CONTRIBUTOR_SEPARATOR);
+		if (separatorAt == -1) {
+			return scopeName;
+		}
+		return scopeName.substring(0, separatorAt);
+	}
+
+	private ScopeNames() {
+	}
+}

org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/registry/Registry.java

@@ -35,6 +35,7 @@
 import org.eclipse.tm4e.core.internal.registry.SyncRegistry;
 import org.eclipse.tm4e.core.internal.theme.Theme;
 import org.eclipse.tm4e.core.internal.theme.raw.RawThemeReader;
+import org.eclipse.tm4e.core.internal.utils.ScopeNames;
 
 /**
  * The registry that will hold all grammars.
@@ -133,6 +134,10 @@ private IGrammar _loadGrammar(
 			@Nullable final Map<String, Integer> embeddedLanguages,
 			@Nullable final Map<String, Integer> tokenTypes,
 			@Nullable final BalancedBracketSelectors balancedBracketSelectors) {
+
+		if (!_loadSingleGrammar(initialScopeName))
+			return null;
+
 		final var dependencyProcessor = new ScopeDependencyProcessor(this._syncRegistry, initialScopeName);
 		while (!dependencyProcessor.Q.isEmpty()) {
 			dependencyProcessor.Q.forEach(request -> this._loadSingleGrammar(request.scopeName));
@@ -147,12 +152,17 @@ private IGrammar _loadGrammar(
 				balancedBracketSelectors);
 	}
 
-	private void _loadSingleGrammar(final String scopeName) {
-		this._ensureGrammarCache.computeIfAbsent(scopeName, this::_doLoadSingleGrammar);
+	private boolean _loadSingleGrammar(final String scopeName) {
+		return this._ensureGrammarCache.computeIfAbsent(scopeName, this::_doLoadSingleGrammar);
 	}
 
 	private boolean _doLoadSingleGrammar(final String scopeName) {
-		final var grammarSource = this._options.getGrammarSource(scopeName);
+		var grammarSource = this._options.getGrammarSource(scopeName);
+		if (grammarSource == null) {
+			final var scopeNameWithoutContributor = ScopeNames.withoutContributor(scopeName);
+			if (!scopeNameWithoutContributor.equals(scopeName))
+				grammarSource = this._options.getGrammarSource(scopeNameWithoutContributor);
+		}
 		if (grammarSource == null) {
 			LOGGER.log(WARNING, "No grammar source for scope [{0}]", scopeName);
 			return false;

org.eclipse.tm4e.core/src/test/java/org/eclipse/tm4e/core/registry/RegistryTest.java

@@ -53,4 +53,11 @@ public FileVisitResult visitFile(final Path file, final @Nullable BasicFileAttri
 		System.out.println("Successfully parsed " + count.intValue() + " grammars.");
 		assertTrue(count.intValue() > 10, "Only " + count.intValue() + " grammars found, expected more than 10!");
 	}
+
+	@Test
+	void testLoadingUnknownGrammar() {
+		final var reg = new Registry();
+		assertNull(reg.grammarForScopeName("undefined"));
+		assertNull(reg.loadGrammar("undefined"));
+	}
 }

org.eclipse.tm4e.registry/src/main/java/org/eclipse/tm4e/registry/internal/TMScope.java

@@ -12,17 +12,17 @@
  */
 package org.eclipse.tm4e.registry.internal;
 
+import static org.eclipse.tm4e.core.internal.utils.ScopeNames.CONTRIBUTOR_SEPARATOR;
 import org.eclipse.jdt.annotation.Nullable;
 import org.eclipse.tm4e.registry.ITMScope;
 
 public final class TMScope implements ITMScope {
-	private static final char SEPARATOR = '@';
 
 	/**
 	 * @param scopeName fully qualified ("source.batchfile@com.example.myplugin") or unqualified scopeName ("source.batchfile")
 	 */
 	public static TMScope parse(final String scopeName) {
-		final int separatorAt = scopeName.indexOf(SEPARATOR);
+		final int separatorAt = scopeName.indexOf(CONTRIBUTOR_SEPARATOR);
 		if (separatorAt == -1) {
 			return new TMScope(scopeName, null, scopeName);
 		}
@@ -40,7 +40,7 @@ public static TMScope parse(final String scopeName) {
 	public TMScope(String scopeName, @Nullable String pluginId) {
 		this.name = scopeName;
 		this.pluginId = pluginId;
-		qualifiedName = pluginId == null ? scopeName : scopeName + SEPARATOR + pluginId;
+		qualifiedName = pluginId == null ? scopeName : scopeName + CONTRIBUTOR_SEPARATOR + pluginId;
 	}
 
 	private TMScope(final String scopeName, @Nullable final String pluginId, final String qualifiedName) {