Skip to content

Commit

Permalink
Improve test cases
Browse files Browse the repository at this point in the history
  • Loading branch information
@sebthom
sebthom committed May 17, 2022
1 parent ae0bcaf commit 0854b6b
Show file tree
Hide file tree
Showing 5 changed files with 4,088 additions and 102 deletions.
116 changes: 113 additions & 3 deletions org.eclipse.tm4e.core/src/test/java/org/eclipse/tm4e/core/grammar/GrammarTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,16 +12,27 @@
package org.eclipse.tm4e.core.grammar;

import static org.eclipse.tm4e.core.internal.utils.NullSafetyHelper.*;
import static org.junit.jupiter.api.Assertions.*;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Reader;
import java.io.StringReader;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.List;
import java.util.stream.Collectors;

import org.eclipse.tm4e.core.Data;
import org.eclipse.tm4e.core.registry.IGrammarSource;
import org.eclipse.tm4e.core.registry.Registry;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;

/**
* Test for grammar tokenizer.
*
*/
public class GrammarTest {

Expand Down Expand Up @@ -61,7 +72,7 @@ public class GrammarTest {
"Token from 14 to 15 with scopes [source.js, meta.function.js, meta.decl.block.js, meta.brace.curly.js]" };

@Test
public void tokenizeLine() throws Exception {
public void tokenizeSingleLineExpression() throws Exception {
final var registry = new Registry();
final IGrammar grammar = registry.addGrammar(IGrammarSource.fromResource(Data.class, "JavaScript.tmLanguage"));
final ITokenizeLineResult lineTokens = castNonNull(grammar).tokenizeLine("function add(a,b) { return a+b; }");
Expand All @@ -74,7 +85,7 @@ public void tokenizeLine() throws Exception {
}

@Test
public void tokenizeLines() throws Exception {
public void tokenizeMultilineExpression() throws Exception {
final var registry = new Registry();
final IGrammar grammar = registry.addGrammar(IGrammarSource.fromResource(Data.class, "JavaScript.tmLanguage"));

Expand All @@ -93,6 +104,105 @@ public void tokenizeLines() throws Exception {
}
j = i;
}
}

@Test
public void tokenizeMultilineYaml() throws Exception {
final var registry = new Registry();
final var grammar = registry.addGrammar(IGrammarSource.fromResource(Data.class, "yaml.tmLanguage.json"));
final var lines = ">\n should.be.string.unquoted.block.yaml\n should.also.be.string.unquoted.block.yaml";
final var result = TokenizationUtils.tokenizeText(lines, grammar).iterator();
assertTrue(Arrays.stream(result.next().getTokens()).anyMatch(t -> t.getScopes().contains(
"keyword.control.flow.block-scalar.folded.yaml")));
assertTrue(Arrays.stream(result.next().getTokens())
.anyMatch(t -> t.getScopes().contains("string.unquoted.block.yaml")));
assertTrue(Arrays.stream(result.next().getTokens())
.anyMatch(t -> t.getScopes().contains("string.unquoted.block.yaml")));
}

@Test
void tokenizeTypeScriptFile() throws Exception {

final var grammar = new Registry()
.addGrammar(IGrammarSource.fromResource(Data.class, "TypeScript.tmLanguage.json"));

final List<String> expectedTokens;
try (var resource = Data.class.getResourceAsStream("raytracer_tokens.txt")) {
expectedTokens = new BufferedReader(new InputStreamReader(resource, StandardCharsets.UTF_8))
.lines()
.collect(Collectors.toList());
}

IStateStack stateStack = null;
int tokenIndex = -1;
try (var reader = new BufferedReader(new InputStreamReader(Data.class.getResourceAsStream("raytracer.ts")))) {
while (reader.ready()) {
final var lineTokens = grammar.tokenizeLine(reader.readLine(), stateStack);
stateStack = lineTokens.getRuleStack();
for (int i = 0; i < lineTokens.getTokens().length; i++) {
tokenIndex++;
final var token = lineTokens.getTokens()[i];
Assertions.assertEquals(
expectedTokens.get(tokenIndex),
"Token from " + token.getStartIndex() + " to " + token.getEndIndex() + " with scopes "
+ token.getScopes());
}
}
}
}

// TODO see https://github.com/microsoft/vscode-textmate/issues/173
@Disabled
@Test
void testShadowedRulesAreResolvedCorrectly() {
final var registry = new Registry();
final var grammar = registry.addGrammar(new IGrammarSource() {

@Override
public Reader getReader() throws IOException {
return new StringReader("""
{
"scopeName": "source.test",
"repository": {
"foo": {
"include": "#bar"
},
"bar": {
"match": "bar1",
"name": "outer"
}
},
"patterns": [{
"patterns": [{
"include": "#foo"
}],
"repository": {
"bar": {
"match": "bar1",
"name": "inner"
}
}
},
{
"begin": "begin",
"patterns": [{
"include": "#foo"
}],
"end": "end"
}
]
}
""");
}

@Override
public String getFilePath() {
return "test-grammar.json";
}
});

final var result = grammar.tokenizeLine("bar1");
assertEquals("[{startIndex: 0, endIndex: 4, scopes: [source.test, outer]}]",
Arrays.toString(result.getTokens()));
}
}
83 changes: 0 additions & 83 deletions org.eclipse.tm4e.core/src/test/java/org/eclipse/tm4e/core/grammar/GrammarTest2.java
View file

This file was deleted.

68 changes: 55 additions & 13 deletions ...st/java/org/eclipse/tm4e/core/internal/grammar/tokenattrs/EncodedTokenAttributesTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,78 +14,120 @@
import static org.eclipse.tm4e.core.internal.theme.FontStyle.*;

import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.MethodOrderer;
import org.junit.jupiter.api.Order;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestMethodOrder;

/**
* {@link EncodedTokenAttributes} tests same than vscode-textmate.
*
* @see <a href="https://github.com/Microsoft/vscode-textmate/blob/master/src/tests/grammar.test.ts">
* github.com/Microsoft/vscode-textmate/blob/master/src/tests/grammar.test.ts</a>
*/
@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
public class EncodedTokenAttributesTest {

@Test
@Order(1)
@DisplayName("StackElementMetadata works")
public void testWorks() {
final int value = EncodedTokenAttributes.set(0, 1, OptionalStandardTokenType.RegEx, null, Underline | Bold, 101,
final int value = EncodedTokenAttributes.set(0, 1, OptionalStandardTokenType.RegEx, false, Underline | Bold,
101,
102);
assertEquals(value, 1, StandardTokenType.RegEx, false, Underline | Bold, 101, 102);
}

@Test
@Order(2)
@DisplayName("StackElementMetadata can overwrite languageId")
public void testCanOverwriteLanguageId() {
int value = EncodedTokenAttributes.set(0, 1, OptionalStandardTokenType.RegEx, null, Underline | Bold, 101, 102);
int value = EncodedTokenAttributes.set(0, 1, OptionalStandardTokenType.RegEx, false, Underline | Bold, 101,
102);
assertEquals(value, 1, StandardTokenType.RegEx, false, Underline | Bold, 101, 102);

value = EncodedTokenAttributes.set(value, 2, OptionalStandardTokenType.NotSet, null, NotSet, 0, 0);
value = EncodedTokenAttributes.set(value, 2, OptionalStandardTokenType.NotSet, false, NotSet, 0, 0);
assertEquals(value, 2, StandardTokenType.RegEx, false, Underline | Bold, 101, 102);
}

@Test
@Order(3)
@DisplayName("StackElementMetadata can overwrite tokenType")
public void testCanOverwriteTokenType() {
int value = EncodedTokenAttributes.set(0, 1, OptionalStandardTokenType.RegEx, null, Underline | Bold, 101, 102);
int value = EncodedTokenAttributes.set(0, 1, OptionalStandardTokenType.RegEx, false, Underline | Bold, 101,
102);
assertEquals(value, 1, StandardTokenType.RegEx, false, Underline | Bold, 101, 102);

value = EncodedTokenAttributes.set(value, 0, OptionalStandardTokenType.Comment, null, NotSet, 0, 0);
value = EncodedTokenAttributes.set(value, 0, OptionalStandardTokenType.Comment, false, NotSet, 0, 0);
assertEquals(value, 1, StandardTokenType.Comment, false, Underline | Bold, 101, 102);
}

@Test
@Order(4)
@DisplayName("StackElementMetadata can overwrite font style")
public void testCanOverwriteFontStyle() {
int value = EncodedTokenAttributes.set(0, 1, OptionalStandardTokenType.RegEx, null, Underline | Bold, 101, 102);
int value = EncodedTokenAttributes.set(0, 1, OptionalStandardTokenType.RegEx, false, Underline | Bold, 101,
102);
assertEquals(value, 1, StandardTokenType.RegEx, false, Underline | Bold, 101, 102);

value = EncodedTokenAttributes.set(value, 0, OptionalStandardTokenType.NotSet, null, None, 0, 0);
value = EncodedTokenAttributes.set(value, 0, OptionalStandardTokenType.NotSet, false, None, 0, 0);
assertEquals(value, 1, StandardTokenType.RegEx, false, None, 101, 102);
}

@Test
@Order(5)
@DisplayName("StackElementMetadata can overwrite font style with strikethrough")
public void testCanOverwriteFontStyleWithStrikethrough() {
int value = EncodedTokenAttributes.set(0, 1, OptionalStandardTokenType.RegEx, null, Strikethrough, 101, 102);
int value = EncodedTokenAttributes.set(0, 1, OptionalStandardTokenType.RegEx, false, Strikethrough, 101, 102);
assertEquals(value, 1, StandardTokenType.RegEx, false, Strikethrough, 101, 102);

value = EncodedTokenAttributes.set(value, 0, OptionalStandardTokenType.NotSet, null, None, 0, 0);
value = EncodedTokenAttributes.set(value, 0, OptionalStandardTokenType.NotSet, false, None, 0, 0);
assertEquals(value, 1, StandardTokenType.RegEx, false, None, 101, 102);
}

@Test
@Order(6)
@DisplayName("StackElementMetadata can overwrite foreground")
public void testCanOverwriteForeground() {
int value = EncodedTokenAttributes.set(0, 1, OptionalStandardTokenType.RegEx, null, Underline | Bold, 101, 102);
int value = EncodedTokenAttributes.set(0, 1, OptionalStandardTokenType.RegEx, false, Underline | Bold, 101,
102);
assertEquals(value, 1, StandardTokenType.RegEx, false, Underline | Bold, 101, 102);

value = EncodedTokenAttributes.set(value, 0, OptionalStandardTokenType.NotSet, null, NotSet, 5, 0);
value = EncodedTokenAttributes.set(value, 0, OptionalStandardTokenType.NotSet, false, NotSet, 5, 0);
assertEquals(value, 1, StandardTokenType.RegEx, false, Underline | Bold, 5, 102);
}

@Test
@Order(7)
@DisplayName("StackElementMetadata can overwrite background")
public void testCanOverwriteBackground() {
int value = EncodedTokenAttributes.set(0, 1, OptionalStandardTokenType.RegEx, null, Underline | Bold, 101, 102);
int value = EncodedTokenAttributes.set(0, 1, OptionalStandardTokenType.RegEx, false, Underline | Bold, 101,
102);
assertEquals(value, 1, StandardTokenType.RegEx, false, Underline | Bold, 101, 102);

value = EncodedTokenAttributes.set(value, 0, OptionalStandardTokenType.NotSet, null, NotSet, 0, 7);
value = EncodedTokenAttributes.set(value, 0, OptionalStandardTokenType.NotSet, false, NotSet, 0, 7);
assertEquals(value, 1, StandardTokenType.RegEx, false, Underline | Bold, 101, 7);
}

@Test
@Order(8)
@DisplayName("StackElementMetadata can overwrite balanced bracket bit")
public void testCanOverwriteBalancedBracketBit() {
int value = EncodedTokenAttributes.set(0, 1, OptionalStandardTokenType.RegEx, false, Underline | Bold, 101,
102);
assertEquals(value, 1, StandardTokenType.RegEx, false, Underline | Bold, 101, 102);

value = EncodedTokenAttributes.set(value, 0, OptionalStandardTokenType.NotSet, true, NotSet, 0, 0);
assertEquals(value, 1, StandardTokenType.RegEx, true, Underline | Bold, 101, 102);

value = EncodedTokenAttributes.set(value, 0, OptionalStandardTokenType.NotSet, false, NotSet, 0, 0);
assertEquals(value, 1, StandardTokenType.RegEx, false, Underline | Bold, 101, 102);
}

@Test
@Order(9)
@DisplayName("StackElementMetadata can work at max values")
public void testCanWorkAtMaxValues() {
final int maxLangId = 255;
final int maxTokenType = StandardTokenType.Comment | StandardTokenType.Other | StandardTokenType.RegEx
Expand Down
Loading

0 comments on commit 0854b6b

Please sign in to comment.