diff --git a/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/grammar/raw/RawCaptures.java b/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/grammar/raw/RawCaptures.java index fbab7a954..d148dd603 100644 --- a/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/grammar/raw/RawCaptures.java +++ b/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/grammar/raw/RawCaptures.java @@ -9,11 +9,9 @@ */ package org.eclipse.tm4e.core.internal.grammar.raw; -import java.util.HashMap; - import org.eclipse.tm4e.core.internal.parser.PropertySettable; -public final class RawCaptures extends HashMap implements IRawCaptures, PropertySettable { +public final class RawCaptures extends PropertySettable.HashMap implements IRawCaptures { private static final long serialVersionUID = 1L; @@ -26,9 +24,4 @@ public IRawRule getCapture(final String captureId) { public Iterable getCaptureIds() { return keySet(); } - - @Override - public void setProperty(final String name, final IRawRule value) { - put(name, value); - } } diff --git a/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/grammar/raw/RawGrammar.java b/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/grammar/raw/RawGrammar.java index 688f517fb..f57c360f4 100644 --- a/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/grammar/raw/RawGrammar.java +++ b/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/grammar/raw/RawGrammar.java @@ -12,9 +12,7 @@ */ package org.eclipse.tm4e.core.internal.grammar.raw; -import java.util.ArrayList; import java.util.Collection; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.NoSuchElementException; @@ -23,7 +21,7 @@ import org.eclipse.jdt.annotation.Nullable; import org.eclipse.tm4e.core.internal.parser.PropertySettable; -public final class RawGrammar extends HashMap implements IRawGrammar, PropertySettable { +public final class RawGrammar extends PropertySettable.HashMap<@Nullable Object> implements IRawGrammar { private static final String FILE_TYPES = "fileTypes"; private static final String FIRST_LINE_MATCH = "firstLineMatch"; @@ -131,11 +129,6 @@ public void putAll(@Nullable final Map OBJECT_FACTORY = path -> { - if (path.size() == 0) { + public static final ObjectFactory OBJECT_FACTORY = new ObjectFactory<>() { + @Override + public RawGrammar createRoot() { return new RawGrammar(); } - return switch (path.last()) { - case RawRule.REPOSITORY -> new RawRepository(); - case RawRule.BEGIN_CAPTURES, RawRule.CAPTURES, RawRule.END_CAPTURES, RawRule.WHILE_CAPTURES -> new RawCaptures(); - default -> new RawRule(); - }; - }; - private static final PListParser JSON_PARSER = new PListParserJSON<>(OBJECT_FACTORY); - private static final PListParser XML_PARSER = new PListParserXML<>(OBJECT_FACTORY); - private static final PListParser YAML_PARSER = new PListParserYAML<>(OBJECT_FACTORY); + @Override + public PropertySettable createChild(final TMParser.PropertyPath path, final Class sourceType) { + return switch (path.last().toString()) { + case RawRule.REPOSITORY -> new RawRepository(); + case RawRule.BEGIN_CAPTURES, RawRule.CAPTURES, RawRule.END_CAPTURES, RawRule.WHILE_CAPTURES -> new RawCaptures(); + default -> List.class.isAssignableFrom(sourceType) + ? new PropertySettable.ArrayList<>() + : new RawRule(); + }; + } + }; public static IRawGrammar readGrammar(final IGrammarSource source) throws Exception { try (var reader = source.getReader()) { - switch (source.getContentType()) { - case JSON: - return JSON_PARSER.parse(reader); - case YAML: - return YAML_PARSER.parse(reader); - case XML: - default: - return XML_PARSER.parse(reader); - } + return switch (source.getContentType()) { + case JSON -> TMParserJSON.INSTANCE.parse(reader, OBJECT_FACTORY); + case YAML -> TMParserYAML.INSTANCE.parse(reader, OBJECT_FACTORY); + default -> TMParserPList.INSTANCE.parse(reader, OBJECT_FACTORY); + }; } } diff --git a/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/grammar/raw/RawRepository.java b/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/grammar/raw/RawRepository.java index e44bf8543..8f430a521 100644 --- a/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/grammar/raw/RawRepository.java +++ b/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/grammar/raw/RawRepository.java @@ -11,13 +11,12 @@ */ package org.eclipse.tm4e.core.internal.grammar.raw; -import java.util.HashMap; import java.util.NoSuchElementException; import org.eclipse.jdt.annotation.Nullable; import org.eclipse.tm4e.core.internal.parser.PropertySettable; -public final class RawRepository extends HashMap implements IRawRepository, PropertySettable { +public final class RawRepository extends PropertySettable.HashMap implements IRawRepository { private static final long serialVersionUID = 1L; @@ -65,9 +64,4 @@ public void putEntries(final PropertySettable target) { target.setProperty(entry.getKey(), entry.getValue()); } } - - @Override - public void setProperty(final String name, final IRawRule value) { - put(name, value); - } } diff --git a/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/grammar/raw/RawRule.java b/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/grammar/raw/RawRule.java index d7f124c3e..7a48d7be8 100644 --- a/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/grammar/raw/RawRule.java +++ b/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/grammar/raw/RawRule.java @@ -12,14 +12,13 @@ package org.eclipse.tm4e.core.internal.grammar.raw; import java.util.Collection; -import java.util.HashMap; import java.util.List; import org.eclipse.jdt.annotation.Nullable; import org.eclipse.tm4e.core.internal.parser.PropertySettable; import org.eclipse.tm4e.core.internal.rule.RuleId; -public class RawRule extends HashMap implements IRawRule, PropertySettable { +public class RawRule extends PropertySettable.HashMap<@Nullable Object> implements IRawRule { private static final String APPLY_END_PATTERN_LAST = "applyEndPatternLast"; private static final String BEGIN = "begin"; @@ -175,9 +174,4 @@ public boolean isApplyEndPatternLast() { } return false; } - - @Override - public void setProperty(final String name, final Object value) { - put(name, value); - } } diff --git a/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/parser/PListContentHandler.java b/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/parser/PListContentHandler.java deleted file mode 100644 index a40c23d56..000000000 --- a/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/parser/PListContentHandler.java +++ /dev/null @@ -1,230 +0,0 @@ -/** - * Copyright (c) 2015-2017 Angelo ZERR. - * This program and the accompanying materials are made - * available under the terms of the Eclipse Public License 2.0 - * which is available at https://www.eclipse.org/legal/epl-2.0/ - * - * SPDX-License-Identifier: EPL-2.0 - * - * Contributors: - * Angelo Zerr - initial API and implementation - */ -package org.eclipse.tm4e.core.internal.parser; - -import static java.lang.System.Logger.Level.WARNING; - -import java.lang.System.Logger; -import java.time.ZonedDateTime; -import java.time.format.DateTimeParseException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; - -import org.eclipse.jdt.annotation.Nullable; -import org.xml.sax.Attributes; -import org.xml.sax.SAXException; -import org.xml.sax.helpers.DefaultHandler; - -final class PListContentHandler extends DefaultHandler { - - private static final Logger LOGGER = System.getLogger(PListContentHandler.class.getName()); - - private static final class PListPathImpl implements PListPath { - final LinkedList keys = new LinkedList<>(); - final List keysUnmodifiable = Collections.unmodifiableList(keys); - final List keysDepths = new ArrayList<>(); - int depth = 0; - - void add(final String key) { - trim(); - keysDepths.add(depth); - keys.add(key); - } - - void trim() { - for (int i = keysDepths.size() - 1; i >= 0; i--) { - if (keysDepths.get(i) >= depth) { - keysDepths.remove(i); - keys.remove(i); - } - } - } - - @Override - public String get(final int index) { - return keys.get(index); - } - - @Override - public String first() { - return keys.getFirst(); - } - - @Override - public String last() { - return keys.getLast(); - } - - @Override - public Iterator iterator() { - return keysUnmodifiable.iterator(); - } - - @Override - public int size() { - return keys.size(); - } - - @Override - public String toString() { - return String.join("/", keys.toArray(String[]::new)); - } - } - - private final class PListObject { - - @Nullable - final PListObject parent; - final Object values; - - PListObject(@Nullable final PListObject parent, final Object values) { - this.parent = parent; - this.values = values; - } - - @SuppressWarnings("unchecked") - void addValue(final Object value) { - if (values instanceof final PropertySettable propertySettable) { - propertySettable.setProperty(path.last(), value); - } else if (values instanceof final List list) { - list.add(value); - } - } - } - - @Nullable - private PListObject currObject; - - @Nullable - private T result; - - private final PropertySettable.Factory objectFactory; - private final PListPathImpl path = new PListPathImpl(); - - /** captures the text content of an XML node */ - private final StringBuilder text = new StringBuilder(); - - PListContentHandler(final PropertySettable.Factory objectFactory) { - this.objectFactory = objectFactory; - } - - @SuppressWarnings("unchecked") - @Override - public void startElement(@Nullable final String uri, @Nullable final String localName, @Nullable final String qName, - @Nullable final Attributes attributes) throws SAXException { - assert localName != null; - - path.depth++; - - switch (localName) { - case "dict": - if (result == null) { - result = (T) objectFactory.create(path); - currObject = new PListObject(currObject, result); - } else { - currObject = new PListObject(currObject, objectFactory.create(path)); - } - break; - case "array": - if (result == null) { - result = (T) new ArrayList<>(); - currObject = new PListObject(currObject, result); - } else { - currObject = new PListObject(currObject, new ArrayList<>()); - } - break; - } - - text.setLength(0); - } - - @Override - public void endElement(@Nullable final String uri, @Nullable final String localName, @Nullable final String qName) throws SAXException { - assert localName != null; - - final var currObject = this.currObject; - if (currObject == null) { - throw new SAXException("Root or element not found!"); - } - - path.trim(); - path.depth--; - - switch (localName) { - case "key": - if (!(currObject.values instanceof PropertySettable)) { - LOGGER.log(WARNING, " tag can only be used inside an open element"); - break; - } - path.add(text.toString()); - break; - case "array", "dict": - final var parent = currObject.parent; - if (parent != null) { - parent.addValue(currObject.values); - this.currObject = parent; - } - break; - case "data", "string": - currObject.addValue(text.toString()); - break; - case "date": // e.g. 2007-10-25T12:36:35Z - try { - currObject.addValue(ZonedDateTime.parse(text.toString())); - } catch (final DateTimeParseException ex) { - LOGGER.log(WARNING, "Failed to parse date '" + text + "'. " + ex); - } - break; - case "integer": - try { - currObject.addValue(Integer.parseInt(text.toString())); - } catch (final NumberFormatException ex) { - LOGGER.log(WARNING, "Failed to parse integer '" + text + "'. " + ex); - } - break; - case "real": - try { - currObject.addValue(Float.parseFloat(text.toString())); - } catch (final NumberFormatException ex) { - LOGGER.log(WARNING, "Failed to parse real as float '" + text + "'. " + ex); - } - break; - case "true": - currObject.addValue(Boolean.TRUE); - break; - case "false": - currObject.addValue(Boolean.FALSE); - break; - case "plist": - break; - default: - LOGGER.log(WARNING, "Invalid tag name: " + localName); - } - } - - @Override - public void characters(final char @Nullable [] ch, final int start, final int length) throws SAXException { - text.append(ch, start, length); - } - - void characters(final String chars) { - text.append(chars); - } - - public T getResult() { - assert result != null; - return result; - } -} diff --git a/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/parser/PListParser.java b/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/parser/PListParser.java deleted file mode 100644 index 686672dc8..000000000 --- a/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/parser/PListParser.java +++ /dev/null @@ -1,19 +0,0 @@ -/** - * Copyright (c) 2022 Sebastian Thomschke and others. - * - * This program and the accompanying materials are made - * available under the terms of the Eclipse Public License 2.0 - * which is available at https://www.eclipse.org/legal/epl-2.0/ - * - * SPDX-License-Identifier: EPL-2.0 - * - * Contributors: - * Sebastian Thomschke - initial implementation - */ -package org.eclipse.tm4e.core.internal.parser; - -import java.io.Reader; - -public interface PListParser { - T parse(Reader contents) throws Exception; -} diff --git a/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/parser/PListParserJSON.java b/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/parser/PListParserJSON.java deleted file mode 100644 index 2a150d310..000000000 --- a/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/parser/PListParserJSON.java +++ /dev/null @@ -1,85 +0,0 @@ -/** - * Copyright (c) 2015-2018 Angelo ZERR. - * This program and the accompanying materials are made - * available under the terms of the Eclipse Public License 2.0 - * which is available at https://www.eclipse.org/legal/epl-2.0/ - * - * SPDX-License-Identifier: EPL-2.0 - * - * Contributors: - * Angelo Zerr - initial API and implementation - */ -package org.eclipse.tm4e.core.internal.parser; - -import java.io.IOException; -import java.io.Reader; - -import org.xml.sax.SAXException; - -import com.google.gson.stream.JsonReader; - -public final class PListParserJSON implements PListParser { - - private final PropertySettable.Factory objectFactory; - - public PListParserJSON(final PropertySettable.Factory objectFactory) { - this.objectFactory = objectFactory; - } - - @Override - public T parse(final Reader contents) throws IOException, SAXException { - final var pList = new PListContentHandler(objectFactory); - try (var reader = new JsonReader(contents)) { - // reader.setLenient(true); - boolean parsing = true; - pList.startElement(null, "plist", null, null); - while (parsing) { - final var nextToken = reader.peek(); - switch (nextToken) { - case BEGIN_ARRAY: - pList.startElement(null, "array", null, null); - reader.beginArray(); - break; - case END_ARRAY: - pList.endElement(null, "array", null); - reader.endArray(); - break; - case BEGIN_OBJECT: - pList.startElement(null, "dict", null, null); - reader.beginObject(); - break; - case END_OBJECT: - pList.endElement(null, "dict", null); - reader.endObject(); - break; - case NAME: - pList.startElement(null, "key", null, null); - pList.characters(reader.nextName()); - pList.endElement(null, "key", null); - break; - case NULL: - reader.nextNull(); - break; - case BOOLEAN: - reader.nextBoolean(); - break; - case NUMBER: - reader.nextLong(); - break; - case STRING: - pList.startElement(null, "string", null, null); - pList.characters(reader.nextString()); - pList.endElement(null, "string", null); - break; - case END_DOCUMENT: - parsing = false; - break; - default: - break; - } - } - pList.endElement(null, "plist", null); - } - return pList.getResult(); - } -} diff --git a/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/parser/PListParserXML.java b/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/parser/PListParserXML.java deleted file mode 100644 index b4587e5cd..000000000 --- a/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/parser/PListParserXML.java +++ /dev/null @@ -1,57 +0,0 @@ -/** - * Copyright (c) 2015-2017 Angelo ZERR. - * This program and the accompanying materials are made - * available under the terms of the Eclipse Public License 2.0 - * which is available at https://www.eclipse.org/legal/epl-2.0/ - * - * SPDX-License-Identifier: EPL-2.0 - * - * Contributors: - * Angelo Zerr - initial API and implementation - */ -package org.eclipse.tm4e.core.internal.parser; - -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.Reader; - -import javax.xml.XMLConstants; -import javax.xml.parsers.ParserConfigurationException; -import javax.xml.parsers.SAXParserFactory; - -import org.xml.sax.InputSource; -import org.xml.sax.SAXException; -import org.xml.sax.XMLReader; - -public final class PListParserXML implements PListParser { - - private final PropertySettable.Factory objectFactory; - - public PListParserXML(final PropertySettable.Factory objectFactory) { - this.objectFactory = objectFactory; - } - - @Override - public T parse(final Reader contents) throws IOException, ParserConfigurationException, SAXException { - final var spf = SAXParserFactory.newInstance(); - spf.setNamespaceAware(true); - - // make parser invulnerable to XXE attacks, see https://rules.sonarsource.com/java/RSPEC-2755 - spf.setFeature("http://xml.org/sax/features/external-general-entities", false); - spf.setFeature("http://xml.org/sax/features/external-parameter-entities", false); - - final var saxParser = spf.newSAXParser(); - - // make parser invulnerable to XXE attacks, see https://rules.sonarsource.com/java/RSPEC-2755 - saxParser.setProperty(XMLConstants.ACCESS_EXTERNAL_DTD, ""); - saxParser.setProperty(XMLConstants.ACCESS_EXTERNAL_SCHEMA, ""); - - final XMLReader xmlReader = saxParser.getXMLReader(); - xmlReader.setEntityResolver((publicId, systemId) -> new InputSource( - new ByteArrayInputStream("".getBytes()))); - final var result = new PListContentHandler(objectFactory); - xmlReader.setContentHandler(result); - xmlReader.parse(new InputSource(contents)); - return result.getResult(); - } -} diff --git a/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/parser/PListParserYAML.java b/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/parser/PListParserYAML.java deleted file mode 100644 index 2c115b973..000000000 --- a/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/parser/PListParserYAML.java +++ /dev/null @@ -1,90 +0,0 @@ -/** - * Copyright (c) 2022 Sebastian Thomschke and others. - * - * This program and the accompanying materials are made - * available under the terms of the Eclipse Public License 2.0 - * which is available at https://www.eclipse.org/legal/epl-2.0/ - * - * SPDX-License-Identifier: EPL-2.0 - * - * Contributors: - * Sebastian Thomschke - initial implementation - */ -package org.eclipse.tm4e.core.internal.parser; - -import static org.eclipse.tm4e.core.internal.utils.NullSafetyHelper.castNonNull; - -import java.io.Reader; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; - -import org.snakeyaml.engine.v2.api.Load; -import org.snakeyaml.engine.v2.api.LoadSettings; -import org.xml.sax.SAXException; - -/** - * Parses TextMate Grammar file in YAML format. - */ -public final class PListParserYAML implements PListParser { - - private final PropertySettable.Factory objectFactory; - - public PListParserYAML(final PropertySettable.Factory objectFactory) { - this.objectFactory = objectFactory; - } - - @SuppressWarnings("unchecked") - private void addListToPList(final PListContentHandler pList, final List list) throws SAXException { - pList.startElement(null, "array", null, null); - - for (final Object item : list) { - if (item instanceof final List items) { - addListToPList(pList, items); - } else if (item instanceof final Map map) { - addMapToPList(pList, map); - } else { - addStringToPList(pList, item.toString()); - } - } - - pList.endElement(null, "array", null); - } - - @SuppressWarnings("unchecked") - private void addMapToPList(final PListContentHandler pList, final Map map) throws SAXException { - pList.startElement(null, "dict", null, null); - - for (final Entry entry : map.entrySet()) { - pList.startElement(null, "key", null, null); - pList.characters(entry.getKey()); - pList.endElement(null, "key", null); - if (entry.getValue() instanceof final List list) { - addListToPList(pList, list); - } else if (entry.getValue() instanceof final Map valueMap) { - addMapToPList(pList, valueMap); - } else { - addStringToPList(pList, castNonNull(entry.getValue()).toString()); - } - } - - pList.endElement(null, "dict", null); - } - - private void addStringToPList(final PListContentHandler pList, final String value) throws SAXException { - pList.startElement(null, "string", null, null); - pList.characters(value); - pList.endElement(null, "string", null); - } - - @SuppressWarnings("unchecked") - @Override - public T parse(final Reader contents) throws SAXException { - final var yamlLoader = new Load(LoadSettings.builder().build()); - final var pList = new PListContentHandler(objectFactory); - pList.startElement(null, "plist", null, null); - addMapToPList(pList, (Map) yamlLoader.loadFromReader(contents)); - pList.endElement(null, "plist", null); - return pList.getResult(); - } -} diff --git a/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/parser/PListPath.java b/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/parser/PListPath.java deleted file mode 100644 index 9e82fb59a..000000000 --- a/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/parser/PListPath.java +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Copyright (c) 2022 Sebastian Thomschke and others. - * - * This program and the accompanying materials are made - * available under the terms of the Eclipse Public License 2.0 - * which is available at https://www.eclipse.org/legal/epl-2.0/ - * - * SPDX-License-Identifier: EPL-2.0 - * - * Contributors: - * Sebastian Thomschke - initial implementation - */ -package org.eclipse.tm4e.core.internal.parser; - -/** - * Represents the hierarchical path of an PList value calculated based on the <key> tags. E.g. - *
  • fileTypes - *
  • scopeName - *
  • repository/constants/patterns/patterns - *
  • repository/statements/patterns/include - *
  • repository/var-single-variable/beginCaptures - */ -public interface PListPath extends Iterable { - - String first(); - - String get(int index); - - String last(); - - int size(); -} diff --git a/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/parser/PropertySettable.java b/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/parser/PropertySettable.java index af6b07bd1..217a02774 100644 --- a/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/parser/PropertySettable.java +++ b/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/parser/PropertySettable.java @@ -14,8 +14,28 @@ public interface PropertySettable { - interface Factory { - PropertySettable create(I args); + public class ArrayList extends java.util.ArrayList implements PropertySettable { + + private static final long serialVersionUID = 1L; + + @Override + public void setProperty(final String name, final T value) { + final var idx = Integer.parseInt(name); + if (idx == size()) + add(value); + else + set(idx, value); + } + } + + public class HashMap extends java.util.HashMap implements PropertySettable { + + private static final long serialVersionUID = 1L; + + @Override + public void setProperty(final String name, final T value) { + put(name, value); + } } void setProperty(String name, V value); diff --git a/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/parser/TMParser.java b/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/parser/TMParser.java new file mode 100644 index 000000000..78a4fec56 --- /dev/null +++ b/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/parser/TMParser.java @@ -0,0 +1,70 @@ +/** + * Copyright (c) 2023 Vegard IT GmbH and others. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Sebastian Thomschke (Vegard IT) - initial implementation + */ +package org.eclipse.tm4e.core.internal.parser; + +import java.io.Reader; +import java.util.List; +import java.util.Map; + +public interface TMParser { + + interface ObjectFactory> { + + T createRoot(); + + /** + * @param sourceType {@link Map} | {@link List} + */ + PropertySettable createChild(PropertyPath path, final Class sourceType); + } + + /** + * Represents the hierarchical path of a property, e.g. + *
  • /fileTypes + *
  • /fileTypes/0 + *
  • /scopeName + *
  • /patterns/0/captures/0/name + *
  • /repository/constants/patterns/0/name + *
  • /repository/statements/patterns/3/include + *
  • /repository/variable/patterns/1/captures/1/name + */ + interface PropertyPath extends Iterable { + + /** + * @return {@link String} | {@link Integer} + * + * @throw NoSuchElementException + */ + Object first(); + + /** + * @param index 0-based + * + * @return {@link String} | {@link Integer} + * + * @throws IndexOutOfBoundsException + */ + Object get(int index); + + /** + * @return {@link String} | {@link Integer} + * + * @throw NoSuchElementException + */ + Object last(); + + int depth(); + } + + > T parse(Reader source, ObjectFactory factory) throws Exception; +} diff --git a/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/parser/TMParserJSON.java b/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/parser/TMParserJSON.java new file mode 100644 index 000000000..b8a2e1242 --- /dev/null +++ b/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/parser/TMParserJSON.java @@ -0,0 +1,80 @@ +/** + * Copyright (c) 2023 Vegard IT GmbH and others. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Sebastian Thomschke (Vegard IT) - initial implementation + */ +package org.eclipse.tm4e.core.internal.parser; + +import java.io.Reader; +import java.util.List; +import java.util.Map; + +import org.eclipse.jdt.annotation.NonNull; + +import com.google.gson.Gson; + +public class TMParserJSON implements TMParser { + + public final static TMParserJSON INSTANCE = new TMParserJSON(); + + private static final Gson LOADER = new Gson(); + + protected Map loadRaw(final Reader source) { + return LOADER.fromJson(source, Map.class); + } + + @Override + public final > T parse(final Reader source, final ObjectFactory factory) throws Exception { + final Map rawRoot = loadRaw(source); + return transform(rawRoot, factory); + } + + private > T transform(final Map rawRoot, final ObjectFactory factory) { + final var root = factory.createRoot(); + final var path = new TMParserPropertyPath(); + + for (final var e : rawRoot.entrySet()) { + addChild(factory, path, root, e.getKey(), e.getValue()); + } + return root; + } + + /** + * @param propertyId String | Integer + */ + private > void addChild(final ObjectFactory handler, final TMParserPropertyPath path, + final PropertySettable parent, final Object propertyId, final Object unmappedChild) { + path.add(propertyId); + if (unmappedChild instanceof final Map map) { + final var mappedChild = handler.createChild(path, unmappedChild.getClass()); + for (final Map.Entry<@NonNull ?, @NonNull ?> e : map.entrySet()) { + addChild(handler, path, mappedChild, e.getKey(), e.getValue()); + } + setProperty(parent, propertyId, mappedChild); + } else if (unmappedChild instanceof final List list) { + final var mappedChild = handler.createChild(path, unmappedChild.getClass()); + for (int i = 0, l = list.size(); i < l; i++) { + addChild(handler, path, mappedChild, i, list.get(i)); + } + setProperty(parent, propertyId, mappedChild); + } else { + setProperty(parent, propertyId, unmappedChild); + } + path.removeLast(); + } + + /** + * @param propertyId String | Integer + */ + @SuppressWarnings("unchecked") + private void setProperty(final PropertySettable settable, final Object propertyId, final Object value) { + ((PropertySettable) settable).setProperty(propertyId.toString(), value); + } +} diff --git a/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/parser/TMParserPList.java b/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/parser/TMParserPList.java new file mode 100644 index 000000000..5ae2b705a --- /dev/null +++ b/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/parser/TMParserPList.java @@ -0,0 +1,223 @@ +/** + * Copyright (c) 2015-2018 Angelo ZERR. + * Copyright (c) 2023 Vegard IT GmbH and others. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * - Angelo Zerr - initial API and implementation, see + * https://github.com/eclipse/tm4e/blob/95c17ade86677f3e2fd32e76222f71adfce18371/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/parser/PList.java + * - Sebastian Thomschke (Vegard IT) - major rewrite to support more configuration variants + */ +package org.eclipse.tm4e.core.internal.parser; + +import static java.lang.System.Logger.Level.ERROR; +import static org.eclipse.tm4e.core.internal.utils.NullSafetyHelper.castNonNull; + +import java.io.ByteArrayInputStream; +import java.io.Reader; +import java.lang.System.Logger; +import java.time.ZonedDateTime; +import java.time.format.DateTimeParseException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import javax.xml.XMLConstants; +import javax.xml.parsers.SAXParserFactory; + +import org.eclipse.jdt.annotation.NonNullByDefault; +import org.eclipse.jdt.annotation.Nullable; +import org.xml.sax.Attributes; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; +import org.xml.sax.XMLReader; +import org.xml.sax.helpers.DefaultHandler; + +public final class TMParserPList implements TMParser { + + private static final Logger LOGGER = System.getLogger(TMParserPList.class.getName()); + + private static final String PLIST_ARRAY = "array"; + private static final String PLIST_DICT = "dict"; + + public final static TMParserPList INSTANCE = new TMParserPList(); + + private static final class ParentRef { + final String sourceKind; + final PropertySettable parent; + @Nullable + Object nextPropertyToSet; + + ParentRef(final String sourceKind, final PropertySettable parent) { + this.sourceKind = sourceKind; + this.parent = parent; + } + } + + @Override + public > T parse(final Reader source, final ObjectFactory factory) + throws Exception { + final var spf = SAXParserFactory.newInstance(); + spf.setNamespaceAware(true); + + // make parser invulnerable to XXE attacks, see https://rules.sonarsource.com/java/RSPEC-2755 + spf.setFeature("http://xml.org/sax/features/external-general-entities", false); + spf.setFeature("http://xml.org/sax/features/external-parameter-entities", false); + + final var saxParser = spf.newSAXParser(); + + // make parser invulnerable to XXE attacks, see https://rules.sonarsource.com/java/RSPEC-2755 + saxParser.setProperty(XMLConstants.ACCESS_EXTERNAL_DTD, ""); + saxParser.setProperty(XMLConstants.ACCESS_EXTERNAL_SCHEMA, ""); + + final XMLReader xmlReader = saxParser.getXMLReader(); + xmlReader.setEntityResolver((publicId, systemId) -> new InputSource( + new ByteArrayInputStream("".getBytes()))); + + final T root = factory.createRoot(); + + xmlReader.setContentHandler(new DefaultHandler() { + + final List parents = new ArrayList<>(); + final TMParserPropertyPath path = new TMParserPropertyPath(); + + /** captures the text content of an XML node */ + final StringBuilder text = new StringBuilder(); + + @Override + public void characters(final char @Nullable [] chars, final int start, final int count) throws SAXException { + text.append(chars, start, count); + } + + @Override + @NonNullByDefault({}) + public void startElement(final String uri, final String localName, final String qName, final Attributes attributes) + throws SAXException { + text.setLength(0); + switch (localName) { + case PLIST_DICT: { + if (parents.isEmpty()) { + parents.add(new ParentRef(localName, root)); + return; + } + parents.add(new ParentRef(localName, factory.createChild(path, Map.class))); + break; + } + + case PLIST_ARRAY: { + final var newParentRef = new ParentRef(localName, factory.createChild(path, List.class)); + parents.add(newParentRef); + + newParentRef.nextPropertyToSet = 0; + path.add(newParentRef.nextPropertyToSet); + break; + } + } + } + + @Override + @NonNullByDefault({}) + public void endElement(final String uri, final String localName, final String qName) throws SAXException { + switch (localName) { + case PLIST_ARRAY: { + final var parentRef = parents.remove(parents.size() - 1); + + path.removeLast(); // removes the remaining array index from the path + setCurrentProperty(parentRef.parent); // register the constructed object with it's parent + break; + } + + case PLIST_DICT: { + final var parentRef = parents.remove(parents.size() - 1); + + if (!parents.isEmpty()) + setCurrentProperty(parentRef.parent); // register the constructed object with it's parent + break; + } + + case "key": { + final var parentRef = parents.get(parents.size() - 1); + + if (!PLIST_DICT.equals(parentRef.sourceKind)) { + LOGGER.log(ERROR, " tag can only be used inside an open element"); + break; + } + + final String key = text.toString(); + parentRef.nextPropertyToSet = key; + path.add(key); + break; + } + + case "data", "string": + setCurrentProperty(text.toString()); + break; + + case "date": // e.g. 2007-10-25T12:36:35Z + try { + setCurrentProperty(ZonedDateTime.parse(text.toString())); + } catch (final DateTimeParseException ex) { + LOGGER.log(ERROR, "Failed to parse date '" + text + "'. " + ex); + } + break; + + case "integer": + try { + setCurrentProperty(Integer.parseInt(text.toString())); + } catch (final NumberFormatException ex) { + LOGGER.log(ERROR, "Failed to parse integer '" + text + "'. " + ex); + } + break; + + case "real": + try { + setCurrentProperty(Float.parseFloat(text.toString())); + } catch (final NumberFormatException ex) { + LOGGER.log(ERROR, "Failed to parse real as float '" + text + "'. " + ex); + } + break; + + case "true": + setCurrentProperty(Boolean.TRUE); + break; + + case "false": + setCurrentProperty(Boolean.FALSE); + break; + + case "plist": + // ignore + break; + + default: + LOGGER.log(ERROR, "Invalid tag name: " + localName); + } + } + + @SuppressWarnings("unchecked") + protected void setCurrentProperty(final Object value) { + path.removeLast(); + final var obj = parents.get(parents.size() - 1); + switch (obj.sourceKind) { + case PLIST_ARRAY: + final var idx = castNonNull((Integer) obj.nextPropertyToSet); + ((PropertySettable) obj.parent).setProperty(idx.toString(), value); + obj.nextPropertyToSet = idx + 1; + path.add(obj.nextPropertyToSet); + break; + case PLIST_DICT: + ((PropertySettable) obj.parent).setProperty(castNonNull(obj.nextPropertyToSet).toString(), value); + break; + } + } + }); + + xmlReader.parse(new InputSource(source)); + + return root; + } +} diff --git a/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/parser/TMParserPropertyPath.java b/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/parser/TMParserPropertyPath.java new file mode 100644 index 000000000..ab93429b5 --- /dev/null +++ b/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/parser/TMParserPropertyPath.java @@ -0,0 +1,75 @@ +/** + * Copyright (c) 2023 Vegard IT GmbH and others. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Sebastian Thomschke (Vegard IT) - initial implementation + */ +package org.eclipse.tm4e.core.internal.parser; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.NoSuchElementException; +import java.util.stream.Collectors; + +final class TMParserPropertyPath extends ArrayList implements TMParser.PropertyPath { + + private static final long serialVersionUID = 1L; + + @Override + public Iterator iterator() { + final var it = super.iterator(); + return new Iterator<>() { + + @Override + public Object next() { + return it.next(); + } + + @Override + public boolean hasNext() { + return it.hasNext(); + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + }; + } + + @Override + public int depth() { + return size(); + } + + @Override + public Object first() { + if (isEmpty()) + throw new NoSuchElementException(); + return get(0); + } + + @Override + public Object last() { + if (isEmpty()) + throw new NoSuchElementException(); + return get(size() - 1); + } + + Object removeLast() { + return remove(size() - 1); + } + + @Override + public String toString() { + return "/" + stream() + .map(Object::toString) + .collect(Collectors.joining("/")); + } +} diff --git a/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/parser/TMParserYAML.java b/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/parser/TMParserYAML.java new file mode 100644 index 000000000..35ec6193b --- /dev/null +++ b/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/parser/TMParserYAML.java @@ -0,0 +1,39 @@ +/** + * Copyright (c) 2023 Vegard IT GmbH and others. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Sebastian Thomschke (Vegard IT) - initial implementation + */ +package org.eclipse.tm4e.core.internal.parser; + +import java.io.Reader; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; + +import org.snakeyaml.engine.v2.api.Load; +import org.snakeyaml.engine.v2.api.LoadSettings; + +public final class TMParserYAML extends TMParserJSON { + + public final static TMParserYAML INSTANCE = new TMParserYAML(); + + private static final LoadSettings LOAD_SETTINGS = LoadSettings.builder() + .setDefaultMap(HashMap::new) // + .setDefaultList(ArrayList::new) + .setDefaultSet(HashSet::new) + .build(); + + @Override + @SuppressWarnings({ "null", "unchecked" }) + protected Map loadRaw(final Reader source) { + return (Map) new Load(LOAD_SETTINGS).loadFromReader(source); + } +} diff --git a/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/theme/raw/RawTheme.java b/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/theme/raw/RawTheme.java index ae80fd346..a011927b5 100644 --- a/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/theme/raw/RawTheme.java +++ b/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/theme/raw/RawTheme.java @@ -12,14 +12,13 @@ package org.eclipse.tm4e.core.internal.theme.raw; import java.util.Collection; -import java.util.HashMap; import org.eclipse.jdt.annotation.Nullable; import org.eclipse.tm4e.core.internal.parser.PropertySettable; import org.eclipse.tm4e.core.internal.theme.IThemeSetting; -public final class RawTheme extends HashMap - implements IRawTheme, IRawThemeSetting, IThemeSetting, PropertySettable { +public final class RawTheme extends PropertySettable.HashMap<@Nullable Object> + implements IRawTheme, IRawThemeSetting, IThemeSetting { private static final long serialVersionUID = 1L; @@ -65,9 +64,4 @@ public String getBackground() { public String getForeground() { return (String) super.get("foreground"); } - - @Override - public void setProperty(final String name, final Object value) { - put(name, value); - } } diff --git a/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/theme/raw/RawThemeReader.java b/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/theme/raw/RawThemeReader.java index 63cc5c33d..70fa61c74 100644 --- a/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/theme/raw/RawThemeReader.java +++ b/org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/theme/raw/RawThemeReader.java @@ -16,12 +16,14 @@ */ package org.eclipse.tm4e.core.internal.theme.raw; -import org.eclipse.tm4e.core.internal.parser.PListParser; -import org.eclipse.tm4e.core.internal.parser.PListParserJSON; -import org.eclipse.tm4e.core.internal.parser.PListParserXML; -import org.eclipse.tm4e.core.internal.parser.PListParserYAML; -import org.eclipse.tm4e.core.internal.parser.PListPath; +import java.util.List; + import org.eclipse.tm4e.core.internal.parser.PropertySettable; +import org.eclipse.tm4e.core.internal.parser.TMParser; +import org.eclipse.tm4e.core.internal.parser.TMParser.ObjectFactory; +import org.eclipse.tm4e.core.internal.parser.TMParserJSON; +import org.eclipse.tm4e.core.internal.parser.TMParserPList; +import org.eclipse.tm4e.core.internal.parser.TMParserYAML; import org.eclipse.tm4e.core.registry.IThemeSource; /** @@ -29,23 +31,27 @@ */ public final class RawThemeReader { - private static final PropertySettable.Factory OBJECT_FACTORY = path -> new RawTheme(); + public static final ObjectFactory OBJECT_FACTORY = new ObjectFactory<>() { + @Override + public RawTheme createRoot() { + return new RawTheme(); + } - private static final PListParser JSON_PARSER = new PListParserJSON<>(OBJECT_FACTORY); - private static final PListParser XML_PARSER = new PListParserXML<>(OBJECT_FACTORY); - private static final PListParser YAML_PARSER = new PListParserYAML<>(OBJECT_FACTORY); + @Override + public PropertySettable createChild(final TMParser.PropertyPath path, final Class sourceType) { + return List.class.isAssignableFrom(sourceType) + ? new PropertySettable.ArrayList<>() + : new RawTheme(); + } + }; public static IRawTheme readTheme(final IThemeSource source) throws Exception { try (var reader = source.getReader()) { - switch (source.getContentType()) { - case JSON: - return JSON_PARSER.parse(reader); - case YAML: - return YAML_PARSER.parse(reader); - case XML: - default: - return XML_PARSER.parse(reader); - } + return switch (source.getContentType()) { + case JSON -> TMParserJSON.INSTANCE.parse(reader, OBJECT_FACTORY); + case YAML -> TMParserYAML.INSTANCE.parse(reader, OBJECT_FACTORY); + default -> TMParserPList.INSTANCE.parse(reader, OBJECT_FACTORY); + }; } } diff --git a/org.eclipse.tm4e.core/src/test/java/org/eclipse/tm4e/core/internal/parser/PListParserTest.java b/org.eclipse.tm4e.core/src/test/java/org/eclipse/tm4e/core/internal/parser/PListParserTest.java deleted file mode 100644 index f1f735f35..000000000 --- a/org.eclipse.tm4e.core/src/test/java/org/eclipse/tm4e/core/internal/parser/PListParserTest.java +++ /dev/null @@ -1,76 +0,0 @@ -/** - * Copyright (c) 2015-2017 Angelo ZERR. - * This program and the accompanying materials are made - * available under the terms of the Eclipse Public License 2.0 - * which is available at https://www.eclipse.org/legal/epl-2.0/ - * - * SPDX-License-Identifier: EPL-2.0 - * - * Contributors: - * Angelo Zerr - initial API and implementation - */ -package org.eclipse.tm4e.core.internal.parser; - -import static org.junit.jupiter.api.Assertions.*; - -import java.io.InputStreamReader; -import java.util.List; -import java.util.Set; - -import org.eclipse.tm4e.core.Data; -import org.eclipse.tm4e.core.internal.grammar.raw.RawGrammarReader; -import org.eclipse.tm4e.core.internal.grammar.raw.RawGrammar; -import org.junit.jupiter.api.MethodOrderer; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestMethodOrder; - -@TestMethodOrder(MethodOrderer.MethodName.class) -class PListParserTest { - - @Test - void testParseJSONPList() throws Exception { - final var parser = new PListParserJSON(RawGrammarReader.OBJECT_FACTORY); - try (var is = Data.class.getResourceAsStream("csharp.json")) { - final var grammar = parser.parse(new InputStreamReader(is)); - assertNotNull(grammar); - assertNotNull(grammar.getRepository()); - assertFalse(grammar.getFileTypes().isEmpty()); - assertEquals(List.of("cs"), grammar.getFileTypes()); - assertEquals("C#", grammar.getName()); - assertEquals("source.cs", grammar.getScopeName()); - assertEquals(List.of("cs"), grammar.getFileTypes()); - assertEquals(Set.of("fileTypes", "foldingStartMarker", "foldingStopMarker", "name", "patterns", "repository", "scopeName"), - grammar.keySet()); - } - } - - @Test - void testParseYAMLPlist() throws Exception { - final var parser = new PListParserYAML(RawGrammarReader.OBJECT_FACTORY); - try (var is = Data.class.getResourceAsStream("JavaScript.tmLanguage.yaml")) { - final var grammar = parser.parse(new InputStreamReader(is)); - assertNotNull(grammar); - assertNotNull(grammar.getRepository()); - assertFalse(grammar.getFileTypes().isEmpty()); - assertEquals(List.of("js", "jsx"), grammar.getFileTypes()); - assertEquals("JavaScript (with React support)", grammar.getName()); - assertEquals("source.js", grammar.getScopeName()); - assertEquals(Set.of("fileTypes", "name", "patterns", "repository", "scopeName", "uuid"), grammar.keySet()); - } - } - - @Test - void testParseXMLPlist() throws Exception { - final var parser = new PListParserXML(RawGrammarReader.OBJECT_FACTORY); - try (var is = Data.class.getResourceAsStream("JavaScript.tmLanguage")) { - final var grammar = parser.parse(new InputStreamReader(is)); - assertNotNull(grammar); - assertNotNull(grammar.getRepository()); - assertFalse(grammar.getFileTypes().isEmpty()); - assertEquals(List.of("js", "jsx"), grammar.getFileTypes()); - assertEquals("JavaScript (with React support)", grammar.getName()); - assertEquals("source.js", grammar.getScopeName()); - assertEquals(Set.of("fileTypes", "name", "patterns", "repository", "scopeName", "uuid"), grammar.keySet()); - } - } -} diff --git a/org.eclipse.tm4e.core/src/test/java/org/eclipse/tm4e/core/internal/parser/TMParserTest.java b/org.eclipse.tm4e.core/src/test/java/org/eclipse/tm4e/core/internal/parser/TMParserTest.java new file mode 100644 index 000000000..0dd7a800f --- /dev/null +++ b/org.eclipse.tm4e.core/src/test/java/org/eclipse/tm4e/core/internal/parser/TMParserTest.java @@ -0,0 +1,228 @@ +/** + * Copyright (c) 2023 Vegard IT GmbH and others. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Sebastian Thomschke (Vegard IT) - initial implementation + */ +package org.eclipse.tm4e.core.internal.parser; + +import static org.junit.jupiter.api.Assertions.*; + +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.StringReader; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.List; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; + +import org.eclipse.jdt.annotation.Nullable; +import org.eclipse.tm4e.core.Data; +import org.eclipse.tm4e.core.internal.grammar.raw.RawGrammar; +import org.eclipse.tm4e.core.internal.grammar.raw.RawGrammarReader; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestMethodOrder; + +@TestMethodOrder(MethodOrderer.MethodName.class) +class TMParserTest { + + @SuppressWarnings("null") + private void validateCaptures(final RawGrammar grammar) { + assertNotNull(grammar.getPatterns()); + assertEquals(grammar.getPatterns().size(), 1); + final var pattern = grammar.getPatterns().iterator().next(); + assertEquals("THE_PATTERN", pattern.getName()); + assertEquals("BEGIN_PATTERN", pattern.getBegin()); + assertEquals("END_PATTERN", pattern.getEnd()); + final var capures = pattern.getCaptures(); + assertEquals("THE_CAPTURE", capures.getCapture("0").getName()); + } + + @Test + void testParseCapturesJSON() throws Exception { + // test capture defined as JSON map + validateCaptures(TMParserJSON.INSTANCE.parse(new StringReader(""" + {"patterns": [{ + "name": "THE_PATTERN", + "captures": { + "0": { "name": "THE_CAPTURE" } + }, + "begin": "BEGIN_PATTERN", + "end": "END_PATTERN" + }]}"""), RawGrammarReader.OBJECT_FACTORY)); + + // test capture defined as JSON array + validateCaptures(TMParserJSON.INSTANCE.parse(new StringReader(""" + {"patterns": [{ + "name": "THE_PATTERN", + "captures": [ + { "name": "THE_CAPTURE" } + ], + "begin": "BEGIN_PATTERN", + "end": "END_PATTERN" + }]}"""), RawGrammarReader.OBJECT_FACTORY)); + } + + @Test + void testParseCapturesPList() throws Exception { + // test capture defined as PList dict + validateCaptures(TMParserPList.INSTANCE.parse(new StringReader(""" + + + patterns + + + name + THE_PATTERN + captures + + 0 + + name + THE_CAPTURE + + + begin + BEGIN_PATTERN + end + END_PATTERN + + + + """), RawGrammarReader.OBJECT_FACTORY)); + + // test capture defined as PList array + validateCaptures(TMParserPList.INSTANCE.parse(new StringReader(""" + + + patterns + + + name + THE_PATTERN + captures + + + name + THE_CAPTURE + + + begin + BEGIN_PATTERN + end + END_PATTERN + + + + """), RawGrammarReader.OBJECT_FACTORY)); + } + + @Test + void testParseCapturesYAML() throws Exception { + // test capture defined as YAML map + validateCaptures(TMParserYAML.INSTANCE.parse(new StringReader(""" + --- + patterns: + - name: THE_PATTERN + captures: + 0: + name: THE_CAPTURE + begin: "BEGIN_PATTERN" + end: "END_PATTERN" + """), RawGrammarReader.OBJECT_FACTORY)); + + // test capture defined as YAML list + validateCaptures(TMParserYAML.INSTANCE.parse(new StringReader(""" + --- + patterns: + - name: THE_PATTERN + captures: + - name: THE_CAPTURE + begin: "BEGIN_PATTERN" + end: "END_PATTERN" + """), RawGrammarReader.OBJECT_FACTORY)); + } + + @Test + void testParseJSON() throws Exception { + try (var is = Data.class.getResourceAsStream("csharp.json")) { + final var grammar = TMParserJSON.INSTANCE.parse(new InputStreamReader(is), RawGrammarReader.OBJECT_FACTORY); + assertNotNull(grammar.getRepository()); + assertFalse(grammar.getFileTypes().isEmpty()); + assertEquals(List.of("cs"), grammar.getFileTypes()); + assertEquals("C#", grammar.getName()); + assertEquals("source.cs", grammar.getScopeName()); + assertEquals(List.of("cs"), grammar.getFileTypes()); + assertEquals(Set.of("fileTypes", "foldingStartMarker", "foldingStopMarker", "name", "patterns", "repository", "scopeName"), + grammar.keySet()); + } + } + + @Test + void testParsePlist() throws Exception { + try (var is = Data.class.getResourceAsStream("JavaScript.tmLanguage")) { + final var grammar = TMParserPList.INSTANCE.parse(new InputStreamReader(is), RawGrammarReader.OBJECT_FACTORY); + assertNotNull(grammar); + assertNotNull(grammar.getRepository()); + assertFalse(grammar.getFileTypes().isEmpty()); + assertEquals(List.of("js", "jsx"), grammar.getFileTypes()); + assertEquals("JavaScript (with React support)", grammar.getName()); + assertEquals("source.js", grammar.getScopeName()); + assertEquals(Set.of("fileTypes", "name", "patterns", "repository", "scopeName", "uuid"), grammar.keySet()); + } + } + + @Test + void testParseYAML() throws Exception { + try (var is = Data.class.getResourceAsStream("JavaScript.tmLanguage.yaml")) { + final var grammar = TMParserYAML.INSTANCE.parse(new InputStreamReader(is), RawGrammarReader.OBJECT_FACTORY); + assertNotNull(grammar.getRepository()); + assertFalse(grammar.getFileTypes().isEmpty()); + assertEquals(List.of("js", "jsx"), grammar.getFileTypes()); + assertEquals("JavaScript (with React support)", grammar.getName()); + assertEquals("source.js", grammar.getScopeName()); + assertEquals(Set.of("fileTypes", "name", "patterns", "repository", "scopeName", "uuid"), grammar.keySet()); + } + } + + @Test + void testLanguagePackGrammars() throws IOException { + final var count = new AtomicInteger(); + Files.walkFileTree(Paths.get("../org.eclipse.tm4e.language_pack"), new SimpleFileVisitor() { + @Override + @SuppressWarnings("null") + public FileVisitResult visitFile(final Path file, final @Nullable BasicFileAttributes attrs) throws IOException { + if (file.getFileName().toString().endsWith("tmLanguage.json")) { + try (var input = Files.newBufferedReader(file)) { + System.out.println("Parsing [" + file + "]..."); + try { + final var grammar = TMParserJSON.INSTANCE.parse(input, RawGrammarReader.OBJECT_FACTORY); + count.incrementAndGet(); + assertFalse(grammar.getScopeName().isBlank()); + assertFalse(grammar.getPatterns().isEmpty()); + assertNotNull(grammar.getFileTypes()); + assertNotNull(grammar.getRepository()); + } catch (final Exception ex) { + throw new RuntimeException(ex); + } + } + } + return FileVisitResult.CONTINUE; + } + }); + System.out.println("Successfully parsed " + count.intValue() + " grammars."); + assertTrue(count.intValue() > 10, "Only " + count.intValue() + " grammars found, expected more than 10!"); + } +}