feat: add support for loading .tmTheme and VSCode JSON themes

org.eclipse.tm4e.core/META-INF/MANIFEST.MF

@@ -17,12 +17,12 @@ Import-Package: org.w3c.css.sac;resolution:=optional,
 Bundle-RequiredExecutionEnvironment: JavaSE-17
 Export-Package: org.eclipse.tm4e.core,
  org.eclipse.tm4e.core.grammar,
- org.eclipse.tm4e.core.internal.grammar;x-friends:="org.eclipse.tm4e.core.tests",
+ org.eclipse.tm4e.core.internal.grammar;x-friends:="org.eclipse.tm4e.core.tests,org.eclipse.tm4e.ui",
  org.eclipse.tm4e.core.internal.grammar.tokenattrs;x-friends:="org.eclipse.tm4e.core.tests",
  org.eclipse.tm4e.core.internal.matcher;x-friends:="org.eclipse.tm4e.core.tests",
  org.eclipse.tm4e.core.internal.oniguruma;x-friends:="org.eclipse.tm4e.languageconfiguration",
- org.eclipse.tm4e.core.internal.theme;x-friends:="org.eclipse.tm4e.core.tests",
- org.eclipse.tm4e.core.internal.theme.raw;x-friends:="org.eclipse.tm4e.core.tests",
+ org.eclipse.tm4e.core.internal.theme;x-friends:="org.eclipse.tm4e.core.tests,org.eclipse.tm4e.ui",
+ org.eclipse.tm4e.core.internal.theme.raw;x-friends:="org.eclipse.tm4e.core.tests,org.eclipse.tm4e.ui",
  org.eclipse.tm4e.core.internal.utils;x-friends:="org.eclipse.tm4e.core.tests,org.eclipse.tm4e.registry,org.eclipse.tm4e.languageconfiguration,org.eclipse.tm4e.markdown,org.eclipse.tm4e.ui,org.eclipse.tm4e.ui.tests",
  org.eclipse.tm4e.core.model,
  org.eclipse.tm4e.core.registry,

org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/theme/ColorMap.java

@@ -12,65 +12,68 @@
 package org.eclipse.tm4e.core.internal.theme;
 
 import java.util.ArrayList;
-import java.util.LinkedHashMap;
+import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
 import org.eclipse.jdt.annotation.Nullable;
 import org.eclipse.tm4e.core.TMException;
 
 /**
- * @see <a href=
- *      "https://github.com/microsoft/vscode-textmate/blob/09effd8b7429b71010e0fa34ea2e16e622692946/src/theme.ts#L385">
- *      github.com/microsoft/vscode-textmate/blob/main/src/theme.ts</a>
+ * Based on <a href="https://github.com/microsoft/vscode-textmate/blob/09effd8b7429b71010e0fa34ea2e16e622692946/src/theme.ts#L385">
+ * github.com/microsoft/vscode-textmate/blob/main/src/theme.ts#ColorMap</a>.
+ * <p>
+ * See also <a href=
+ * "https://github.com/microsoft/vscode/blob/ba2cf46e20df3edf77bdd905acde3e175d985f70/src/vs/editor/common/languages/supports/tokenization.ts#L155">
+ * github.com/microsoft/vscode/blob/main/src/vs/editor/common/languages/supports/tokenization.ts#ColorMap</a>
  */
 public final class ColorMap {
 
 	private final boolean _isFrozen;
-	private int _lastColorId = -1; // -1 and not 0 as in upstream project on purpose
+	private int _lastColorId = 0;
 	private final List<String> _id2color = new ArrayList<>();
-	private final Map<String /*color*/, @Nullable Integer /*ID color*/> _color2id = new LinkedHashMap<>();
+	private final List<String> _id2colorUnmodifiable = Collections.unmodifiableList(_id2color);
+	private final Map<String /*color*/, @Nullable Integer /*ID color*/> _color2id = new HashMap<>();
 
 	public ColorMap() {
 		this(null);
 	}
 
 	public ColorMap(@Nullable final List<String> _colorMap) {
+		_id2color.add(""); // required since the upstream impl works with 1-based indexes
 		if (_colorMap != null) {
 			this._isFrozen = true;
-			for (int i = 0, len = _colorMap.size(); i < len; i++) {
-				this._color2id.put(_colorMap.get(i), i);
-				this._id2color.add(_colorMap.get(i));
+			for (final String color : _colorMap) {
+				final String color_upper = color.toUpperCase();
+				this._color2id.put(color_upper, _id2color.size());
+				this._id2color.add(color_upper);
 			}
 		} else {
 			this._isFrozen = false;
 		}
 	}
 
-	public int getId(@Nullable final String _color) {
-		if (_color == null) {
+	public int getId(final @Nullable String color) {
+		if (color == null)
 			return 0;
-		}
-		final var color = _color.toUpperCase();
-		Integer value = _color2id.get(color);
-		if (value != null) {
+
+		final String color_upper = color.toUpperCase();
+		Integer value = this._color2id.get(color_upper);
+		if (value != null)
 			return value;
-		}
-		if (this._isFrozen) {
-			throw new TMException("Missing color in color map - " + color);
-		}
+
+		if (this._isFrozen)
+			throw new TMException("Missing color in frozen color map:" + color_upper);
+
 		value = ++this._lastColorId;
-		_color2id.put(color, value);
-		if (value >= _id2color.size()) {
-			_id2color.add(color);
-		} else {
-			_id2color.set(value, color);
-		}
+		_color2id.put(color_upper, value);
+		_id2color.add(color_upper);
 		return value;
 	}
 
 	public List<String> getColorMap() {
-		return new ArrayList<>(_color2id.keySet());
+		return _id2colorUnmodifiable;
 	}
 
 	@Override

org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/theme/FontStyle.java

@@ -7,7 +7,8 @@
  * SPDX-License-Identifier: EPL-2.0
  *
  * Contributors:
- * Angelo Zerr <angelo.zerr@gmail.com> - initial API and implementation
+ * - Angelo Zerr <angelo.zerr@gmail.com> - initial API and implementation
+ * - Sebastian Thomschke - added isBold/isItalic/isStrikethrough/isUnderline
  */
 package org.eclipse.tm4e.core.internal.theme;
 
@@ -38,16 +39,16 @@ public static String fontStyleToString(final int fontStyle) {
 		}
 
 		final var style = new StringBuilder();
-		if ((fontStyle & Italic) == Italic) {
+		if (isItalic(fontStyle)) {
 			style.append("italic ");
 		}
-		if ((fontStyle & Bold) == Bold) {
+		if (isBold(fontStyle)) {
 			style.append("bold ");
 		}
-		if ((fontStyle & Underline) == Underline) {
+		if (isUnderline(fontStyle)) {
 			style.append("underline ");
 		}
-		if ((fontStyle & Strikethrough) == Strikethrough) {
+		if (isStrikethrough(fontStyle)) {
 			style.append("strikethrough ");
 		}
 		if (style.isEmpty()) {
@@ -57,6 +58,22 @@ public static String fontStyleToString(final int fontStyle) {
 		return style.toString();
 	}
 
+	public static boolean isBold(final int fontStyle) {
+		return (fontStyle & Bold) == Bold;
+	}
+
+	public static boolean isItalic(final int fontStyle) {
+		return (fontStyle & Italic) == Italic;
+	}
+
+	public static boolean isUnderline(final int fontStyle) {
+		return (fontStyle & Underline) == Underline;
+	}
+
+	public static boolean isStrikethrough(final int fontStyle) {
+		return (fontStyle & Strikethrough) == Strikethrough;
+	}
+
 	private FontStyle() {
 	}
 }

org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/theme/StyleAttributes.java

@@ -24,7 +24,7 @@
  *      github.com/microsoft/vscode-textmate/blob/main/src/theme.ts</a>
  */
 public class StyleAttributes {
-	private static final StyleAttributes NO_STYLE = new StyleAttributes(-1, 0, 0);
+	public static final StyleAttributes NO_STYLE = new StyleAttributes(-1, 0, 0);
 
 	/** @see FontStyle */
 	public final int fontStyle;

org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/theme/Theme.java

@@ -40,7 +40,14 @@
 public final class Theme {
 
 	public static Theme createFromRawTheme(@Nullable final IRawTheme source, @Nullable final List<String> colorMap) {
-		return createFromParsedTheme(parseTheme(source), colorMap);
+		final var theme = createFromParsedTheme(parseTheme(source), colorMap);
+
+		// custom tm4e code, not from upstream
+		if (source != null) {
+			theme.editorColors = source.getEditorColors();
+		}
+
+		return theme;
 	}
 
 	public static Theme createFromParsedTheme(final List<ParsedThemeRule> source, @Nullable final List<String> colorMap) {
@@ -52,6 +59,7 @@ public static Theme createFromParsedTheme(final List<ParsedThemeRule> source, @N
 	private final ColorMap _colorMap;
 	private final StyleAttributes _defaults;
 	private final ThemeTrieElement _root;
+	private Map<String, String> editorColors = Collections.emptyMap(); // custom tm4e code, not from upstream
 
 	public Theme(final ColorMap colorMap, final StyleAttributes defaults, final ThemeTrieElement root) {
 		this._colorMap = colorMap;
@@ -67,6 +75,10 @@ public StyleAttributes getDefaults() {
 		return this._defaults;
 	}
 
+	public Map<String, String> getEditorColors() { // custom tm4e code, not from upstream
+		return editorColors;
+	}
+
 	public @Nullable StyleAttributes match(@Nullable final ScopeStack scopePath) {
 		if (scopePath == null) {
 			return this._defaults;
@@ -153,11 +165,11 @@ public static List<ParsedThemeRule> parseTheme(@Nullable final IRawTheme source)
 			}
 
 			int fontStyle = FontStyle.NotSet;
-			final var settingsFontStyle = entrySetting.getFontStyle();
-			if (settingsFontStyle instanceof final String style) {
+			final String settingsFontStyle = entrySetting.getFontStyle();
+			if (settingsFontStyle != null) {
 				fontStyle = FontStyle.None;
 
-				final var segments = StringUtils.splitToArray(style, ' ');
+				final var segments = StringUtils.splitToArray(settingsFontStyle, ' ');
 				for (final var segment : segments) {
 					switch (segment) {
 						case "italic":

org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/theme/raw/IRawTheme.java

@@ -12,6 +12,7 @@
 package org.eclipse.tm4e.core.internal.theme.raw;
 
 import java.util.Collection;
+import java.util.Map;
 
 import org.eclipse.jdt.annotation.Nullable;
 
@@ -27,4 +28,5 @@ public interface IRawTheme {
 	@Nullable
 	Collection<IRawThemeSetting> getSettings();
 
+	Map<String, String> getEditorColors(); // custom tm4e code, not from upstream
 }

org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/theme/raw/RawTheme.java

@@ -12,6 +12,9 @@
 package org.eclipse.tm4e.core.internal.theme.raw;
 
 import java.util.Collection;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Objects;
 
 import org.eclipse.jdt.annotation.Nullable;
 import org.eclipse.tm4e.core.internal.parser.PropertySettable;
@@ -32,7 +35,30 @@ public final class RawTheme extends PropertySettable.HashMap<@Nullable Object>
 	@Override
 	@SuppressWarnings("unchecked")
 	public @Nullable Collection<IRawThemeSetting> getSettings() {
-		return (Collection<IRawThemeSetting>) super.get("settings");
+		// vscode themes only
+		if (get("tokenColors") instanceof final Collection settings)
+			return settings;
+
+		return (Collection<IRawThemeSetting>) get("settings");
+	}
+
+	// custom tm4e code, not from upstream
+	@Override
+	@SuppressWarnings("unchecked")
+	public Map<String, String> getEditorColors() {
+		// vscode themes only
+		if (get("colors") instanceof final Map colors)
+			return colors;
+
+		final var settings = getSettings();
+		return settings == null
+				? Collections.emptyMap()
+				: settings.stream()
+						.filter(s -> s.getScope() == null)
+						.map(s -> ((Map<String, Map<String, String>>) s).get("settings"))
+						.filter(Objects::nonNull)
+						.findFirst()
+						.orElse(Collections.emptyMap());
 	}
 
 	/*

org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/utils/MoreCollections.java

@@ -13,6 +13,7 @@
 package org.eclipse.tm4e.core.internal.utils;
 
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
@@ -75,6 +76,10 @@ public static <T> T getLastElement(final List<T> list) {
 		return list.get(list.size() - 1);
 	}
 
+	public static <T> Collection<T> nullToEmpty(@Nullable final Collection<T> coll) {
+		return coll == null ? Collections.emptyList() : coll;
+	}
+
 	public static <T> List<T> nullToEmpty(@Nullable final List<T> list) {
 		return list == null ? Collections.emptyList() : list;
 	}

org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/theme/RGB.java

@@ -8,14 +8,26 @@
  *
  * Contributors:
  * - Angelo Zerr <angelo.zerr@gmail.com> - initial API and implementation
- * - Sebastian Thomschke (Vegard IT) - add hashCode/equals methods
+ * - Sebastian Thomschke (Vegard IT) - add methods hashCode/equals, fromHex(String)
  */
 package org.eclipse.tm4e.core.theme;
 
 import org.eclipse.jdt.annotation.Nullable;
 
 public class RGB {
 
+	public static @Nullable RGB fromHex(final @Nullable String hex) {
+		if (hex == null || hex.isBlank())
+			return null;
+
+		final var offset = hex.startsWith("#") ? 1 : 0;
+		final int r = Integer.parseInt(hex.substring(offset + 0, offset + 2), 16);
+		final int g = Integer.parseInt(hex.substring(offset + 2, offset + 4), 16);
+		final int b = Integer.parseInt(hex.substring(offset + 4, offset + 6), 16);
+
+		return new RGB(r, g, b);
+	}
+
 	public final int red;
 	public final int green;
 	public final int blue;