From 9beaab43c53be3c38ee7b7a4247b268a3bb3a6e2 Mon Sep 17 00:00:00 2001 From: Holger Stenger Date: Tue, 12 Nov 2013 21:40:06 +0100 Subject: [PATCH 1/8] Replaced lexer with a new version based on a depth-first left-to-right traversal of the AST. The new lexer allows styling nested Markdown based on the innermost AST node. --- .../markdown/highlighter/MarkdownLexer.java | 89 ++-- .../highlighter/MarkdownLexerVisitor.java | 381 ++++++++++++++++++ .../markdown/highlighter/MarkdownToken.java | 21 +- .../highlighter/MarkdownTokenListBuilder.java | 70 ++++ .../highlighter/MarkdownTokenMap.java | 23 -- .../markdown/highlighter/MarkdownVisitor.java | 246 ----------- 6 files changed, 512 insertions(+), 318 deletions(-) create mode 100644 src/flow/netbeans/markdown/highlighter/MarkdownLexerVisitor.java create mode 100644 src/flow/netbeans/markdown/highlighter/MarkdownTokenListBuilder.java delete mode 100644 src/flow/netbeans/markdown/highlighter/MarkdownTokenMap.java delete mode 100644 src/flow/netbeans/markdown/highlighter/MarkdownVisitor.java diff --git a/src/flow/netbeans/markdown/highlighter/MarkdownLexer.java b/src/flow/netbeans/markdown/highlighter/MarkdownLexer.java index 52240de..bd811e6 100644 --- a/src/flow/netbeans/markdown/highlighter/MarkdownLexer.java +++ b/src/flow/netbeans/markdown/highlighter/MarkdownLexer.java @@ -1,37 +1,37 @@ package flow.netbeans.markdown.highlighter; import flow.netbeans.markdown.options.MarkdownGlobalOptions; +import java.util.Collections; import java.util.Iterator; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; import org.netbeans.api.lexer.Token; import org.netbeans.spi.lexer.Lexer; import org.netbeans.spi.lexer.LexerInput; import org.netbeans.spi.lexer.LexerRestartInfo; +import org.pegdown.ParsingTimeoutException; import org.pegdown.PegDownProcessor; import org.pegdown.ast.RootNode; class MarkdownLexer implements Lexer { + private static final Logger LOG = Logger.getLogger(MarkdownLexer.class.getName()); static Lexer create(LexerRestartInfo info) { return new MarkdownLexer(info); } - private MarkdownVisitor markdownVisitor = new MarkdownVisitor(); - private LexerRestartInfo info; - private final LexerInput input; + private final LexerRestartInfo info; boolean inited = false; private Iterator tokenIterator = null; - private MarkdownTokenMap tokenMap; - RootNode rootNode; MarkdownLexer(LexerRestartInfo info) { this.info = info; - this.input = info.input(); } @Override public Token nextToken() { - // Tokenize the input on the first nextToken() call if (!inited) { synchronized (this) { @@ -43,25 +43,23 @@ public Token nextToken() { } } + final int readLength = info.input().readLength(); + // Nothing here - if(input.readLength() <= 0) { + if(readLength <= 0) { return null; } - // There is something - but not tokenozed - if(rootNode.getChildren().isEmpty()) { - return info.tokenFactory().createToken(MarkdownTokenId.PLAIN); - } - - // Iterate over the Token Map and spit out netbeans tokens - while(tokenIterator.hasNext()) { + // Retrieve the next token from the iterator. + if(tokenIterator.hasNext()) { MarkdownToken token = tokenIterator.next(); return info.tokenFactory().createToken(token.getId(), token.getLength()); } // Legacy safety net. @todo remove this - if(input.readLength() > 0) { - return info.tokenFactory().createToken(MarkdownTokenId.PLAIN, input.readLength()); + if(readLength > 0) { + LOG.log(Level.WARNING, "Caught by legacy safety net"); + return info.tokenFactory().createToken(MarkdownTokenId.PLAIN, readLength); } return null; @@ -74,40 +72,39 @@ private void tokenizeInput() { MarkdownGlobalOptions markdownOptions = MarkdownGlobalOptions.getInstance(); PegDownProcessor markdownProcessor = new PegDownProcessor(markdownOptions.getExtensionsValue()); - LexerInput lexerInput = info.input(); - StringBuilder data = new StringBuilder(); - + // Read the complete input and feed the PegDown Parser // Selected extensions over plain markdown are taken in account - int i; - while ((i = lexerInput.read()) != LexerInput.EOF) { - data.append((char) i); + char[] markdownSource = readAll(info.input()); + + int inputLength = markdownSource.length; + + List tokens; + try { + RootNode rootNode = markdownProcessor.parseMarkdown(markdownSource); + + MarkdownTokenListBuilder builder = new MarkdownTokenListBuilder(inputLength); + MarkdownLexerVisitor visitor = new MarkdownLexerVisitor(builder); + rootNode.accept(visitor); + tokens = builder.build(); } - int inputLength = data.length(); - rootNode = markdownProcessor.parser.parse(data.toString().toCharArray()); - rootNode.accept(markdownVisitor); - MarkdownTokenMap tempMap = markdownVisitor.getTokenMap(); - - // The MarkdownVisitor does leave "gaps" in the tokenized input - so we fill theses - // gaps with MarkdownTokenId.PLAIN tokens - tokenMap = new MarkdownTokenMap(); - int j = 0; - while (j <= inputLength) { - if(tempMap.containsKey(j)) { - MarkdownToken currentToken = tempMap.get(j); - tokenMap.put(j, currentToken); - j += currentToken.getLength(); - continue; - } else { - tokenMap.put(j, new MarkdownToken(MarkdownTokenId.PLAIN, 1)); - } - j += 1; + catch (ParsingTimeoutException ex) { + LOG.log(Level.WARNING, "Time out while parsing Markdown source, falling back to no highlighting"); + tokens = Collections.singletonList(new MarkdownToken(MarkdownTokenId.PLAIN, 0, inputLength)); } - - // Shorthand iterator - tokenIterator = tokenMap.values().iterator(); + + tokenIterator = tokens.iterator(); } + private char[] readAll(LexerInput lexerInput) { + StringBuilder sb = new StringBuilder(); + int i; + while ((i = lexerInput.read()) != LexerInput.EOF) { + sb.append((char) i); + } + char[] markdownSource = sb.toString().toCharArray(); + return markdownSource; + } @Override public Object state() { @@ -117,4 +114,4 @@ public Object state() { @Override public void release() { } -} \ No newline at end of file +} diff --git a/src/flow/netbeans/markdown/highlighter/MarkdownLexerVisitor.java b/src/flow/netbeans/markdown/highlighter/MarkdownLexerVisitor.java new file mode 100644 index 0000000..2973123 --- /dev/null +++ b/src/flow/netbeans/markdown/highlighter/MarkdownLexerVisitor.java @@ -0,0 +1,381 @@ +package flow.netbeans.markdown.highlighter; + +import org.pegdown.ast.AbbreviationNode; +import org.pegdown.ast.AutoLinkNode; +import org.pegdown.ast.BlockQuoteNode; +import org.pegdown.ast.BulletListNode; +import org.pegdown.ast.CodeNode; +import org.pegdown.ast.DefinitionListNode; +import org.pegdown.ast.DefinitionNode; +import org.pegdown.ast.DefinitionTermNode; +import org.pegdown.ast.ExpImageNode; +import org.pegdown.ast.ExpLinkNode; +import org.pegdown.ast.HeaderNode; +import org.pegdown.ast.HtmlBlockNode; +import org.pegdown.ast.InlineHtmlNode; +import org.pegdown.ast.ListItemNode; +import org.pegdown.ast.MailLinkNode; +import org.pegdown.ast.Node; +import org.pegdown.ast.OrderedListNode; +import org.pegdown.ast.ParaNode; +import org.pegdown.ast.QuotedNode; +import org.pegdown.ast.RefImageNode; +import org.pegdown.ast.RefLinkNode; +import org.pegdown.ast.ReferenceNode; +import org.pegdown.ast.RootNode; +import org.pegdown.ast.SimpleNode; +import org.pegdown.ast.SpecialTextNode; +import org.pegdown.ast.StrongEmphSuperNode; +import org.pegdown.ast.SuperNode; +import org.pegdown.ast.TableBodyNode; +import org.pegdown.ast.TableCaptionNode; +import org.pegdown.ast.TableCellNode; +import org.pegdown.ast.TableColumnNode; +import org.pegdown.ast.TableHeaderNode; +import org.pegdown.ast.TableNode; +import org.pegdown.ast.TableRowNode; +import org.pegdown.ast.TextNode; +import org.pegdown.ast.VerbatimNode; +import org.pegdown.ast.Visitor; +import org.pegdown.ast.WikiLinkNode; + +/** + * + * @author Holger + */ +public class MarkdownLexerVisitor implements Visitor { + private final MarkdownTokenListBuilder builder; + + public MarkdownLexerVisitor(MarkdownTokenListBuilder builder) { + this.builder = builder; + } + + private void addLeafTreeToken(MarkdownTokenId id, Node node) { + builder.addLeafTreeToken(id, node); + } + + private void beginTreeToken(MarkdownTokenId id, Node node) { + builder.beginTreeToken(id, node); + } + + private void endTreeToken() { + builder.endTreeToken(); + } + + private void visitChildren(SuperNode node) { + for (Node child : node.getChildren()) { + child.accept(this); + } + } + + private void visitNode(Node node) { + if (node != null) { + node.accept(this); + } + } + + @Override + public void visit(RootNode node) { + beginTreeToken(MarkdownTokenId.PLAIN, node); + visitChildren(node); + endTreeToken(); + // Also pop the guard tree token. + endTreeToken(); + } + + @Override + public void visit(AbbreviationNode node) { + beginTreeToken(MarkdownTokenId.ABBREVIATION, node); + visitChildren(node); + visitNode(node.getExpansion()); + endTreeToken(); + } + + @Override + public void visit(AutoLinkNode node) { + addLeafTreeToken(MarkdownTokenId.AUTOLINK, node); + } + + @Override + public void visit(BlockQuoteNode node) { + beginTreeToken(MarkdownTokenId.BLOCKQUOTE, node); + visitChildren(node); + endTreeToken(); + } + + @Override + public void visit(BulletListNode node) { + beginTreeToken(MarkdownTokenId.BULLETLIST, node); + visitChildren(node); + endTreeToken(); + } + + @Override + public void visit(CodeNode node) { + addLeafTreeToken(MarkdownTokenId.CODE, node); + } + + @Override + public void visit(DefinitionListNode node) { + beginTreeToken(MarkdownTokenId.DEFINITION_LIST, node); + visitChildren(node); + endTreeToken(); + } + + @Override + public void visit(DefinitionNode node) { + beginTreeToken(MarkdownTokenId.DEFINITION, node); + visitChildren(node); + endTreeToken(); + } + + @Override + public void visit(DefinitionTermNode node) { + beginTreeToken(MarkdownTokenId.DEFINITION_TERM, node); + visitChildren(node); + endTreeToken(); + } + + @Override + public void visit(ExpImageNode node) { + beginTreeToken(MarkdownTokenId.EXPIMAGE, node); + visitChildren(node); + endTreeToken(); + } + + @Override + public void visit(ExpLinkNode node) { + beginTreeToken(MarkdownTokenId.EXPLINK, node); + visitChildren(node); + endTreeToken(); + } + + @Override + public void visit(HeaderNode node) { + switch (node.getLevel()) { + case 1: + beginTreeToken(MarkdownTokenId.HEADER1, node); + break; + case 2: + beginTreeToken(MarkdownTokenId.HEADER2, node); + break; + case 3: + beginTreeToken(MarkdownTokenId.HEADER3, node); + break; + case 4: + beginTreeToken(MarkdownTokenId.HEADER4, node); + break; + case 5: + beginTreeToken(MarkdownTokenId.HEADER5, node); + break; + case 6: + beginTreeToken(MarkdownTokenId.HEADER6, node); + break; + default: + beginTreeToken(MarkdownTokenId.PLAIN, node); + } + visitChildren(node); + endTreeToken(); + } + + @Override + public void visit(HtmlBlockNode node) { + addLeafTreeToken(MarkdownTokenId.HTMLBLOCK, node); + } + + @Override + public void visit(InlineHtmlNode node) { + addLeafTreeToken(MarkdownTokenId.INLINEHTML, node); + } + + @Override + public void visit(ListItemNode node) { + addLeafTreeToken(MarkdownTokenId.LISTITEM, node); + visitChildren(node); + endTreeToken(); + } + + @Override + public void visit(MailLinkNode node) { + addLeafTreeToken(MarkdownTokenId.MAILLINK, node); + } + + @Override + public void visit(OrderedListNode node) { + beginTreeToken(MarkdownTokenId.ORDEREDLIST, node); + visitChildren(node); + endTreeToken(); + } + + @Override + public void visit(ParaNode node) { + visitChildren(node); + } + + @Override + public void visit(QuotedNode node) { + beginTreeToken(MarkdownTokenId.QUOTED, node); + visitChildren(node); + endTreeToken(); + } + + @Override + public void visit(ReferenceNode node) { + beginTreeToken(MarkdownTokenId.REFERENCE, node); + visitChildren(node); + endTreeToken(); + } + + @Override + public void visit(RefImageNode node) { + beginTreeToken(MarkdownTokenId.REF_IMAGE, node); + visitChildren(node); + endTreeToken(); + } + + @Override + public void visit(RefLinkNode node) { + beginTreeToken(MarkdownTokenId.REF_LINK, node); + visitChildren(node); + endTreeToken(); + } + + @Override + public void visit(SimpleNode node) { + switch (node.getType()) { + case HRule: + addLeafTreeToken(MarkdownTokenId.HORIZONTALRULE, node); + break; + case Apostrophe: + case Ellipsis: + case Emdash: + case Endash: + case Linebreak: + case Nbsp: + break; + } + } + + @Override + public void visit(SpecialTextNode node) { + addLeafTreeToken(MarkdownTokenId.TEXT, node); + } + + @Override + public void visit(StrongEmphSuperNode node) { + beginTreeToken(node.isStrong() ? MarkdownTokenId.STRONG : MarkdownTokenId.EMPH, node); + visitChildren(node); + endTreeToken(); + } + + @Override + public void visit(TableBodyNode node) { + visitChildren(node); + } + + @Override + public void visit(TableCaptionNode node) { + visitChildren(node); + } + + @Override + public void visit(TableCellNode node) { + visitChildren(node); + } + + @Override + public void visit(TableColumnNode node) { + visitChildren(node); + } + + @Override + public void visit(TableHeaderNode node) { + visitChildren(node); + } + + @Override + public void visit(TableNode node) { + beginTreeToken(MarkdownTokenId.TABLE, node); + visitChildren(node); + endTreeToken(); + } + + @Override + public void visit(TableRowNode node) { + visitChildren(node); + } + + @Override + public void visit(VerbatimNode node) { + addLeafTreeToken(MarkdownTokenId.VERBATIM, node); + } + + @Override + public void visit(WikiLinkNode node) { + addLeafTreeToken(MarkdownTokenId.WIKILINK, node); + } + + @Override + public void visit(TextNode node) { +// addLeafTreeToken(MarkdownTokenId.TEXT, node); + } + + @Override + public void visit(SuperNode node) { + visitChildren(node); + } + + @Override + public void visit(Node node) { + addLeafTreeToken(MarkdownTokenId.PLAIN, node); + } + + protected static class MarkdownTreeToken { + private final MarkdownTokenId id; + + private final int startIndex; + + private final int endIndex; + + private final int depth; + + private int remainderIndex; + + public MarkdownTreeToken(MarkdownTokenId id, int depth, int startIndex, int endIndex) { + this.id = id; + this.startIndex = startIndex; + this.endIndex = endIndex; + this.depth = depth; + this.remainderIndex = startIndex; + } + + public MarkdownTokenId getId() { + return id; + } + + public int getStartIndex() { + return startIndex; + } + + public int getEndIndex() { + return endIndex; + } + + public int getDepth() { + return depth; + } + + public int getRemainderIndex() { + return remainderIndex; + } + + public void setRemainderIndex(int remainderIndex) { + this.remainderIndex = remainderIndex; + } + + @Override + public String toString() { + return id + "[" + startIndex + "-" + endIndex + "@" + remainderIndex + "]"; + } + } +} diff --git a/src/flow/netbeans/markdown/highlighter/MarkdownToken.java b/src/flow/netbeans/markdown/highlighter/MarkdownToken.java index 69696fa..4a7dc4c 100644 --- a/src/flow/netbeans/markdown/highlighter/MarkdownToken.java +++ b/src/flow/netbeans/markdown/highlighter/MarkdownToken.java @@ -3,18 +3,33 @@ public class MarkdownToken { private final MarkdownTokenId id; - private final int length; + private final int startIndex; + private final int endIndex; + @Deprecated public MarkdownToken(MarkdownTokenId id, int length) { + this(id, 0, length); + } + + public MarkdownToken(MarkdownTokenId id, int startIndex, int endIndex) { this.id = id; - this.length = length; + this.startIndex = startIndex; + this.endIndex = endIndex; } public MarkdownTokenId getId() { return id; } + public int getStartIndex() { + return startIndex; + } + + public int getEndIndex() { + return endIndex; + } + public int getLength() { - return length; + return endIndex - startIndex; } } diff --git a/src/flow/netbeans/markdown/highlighter/MarkdownTokenListBuilder.java b/src/flow/netbeans/markdown/highlighter/MarkdownTokenListBuilder.java new file mode 100644 index 0000000..ad6225f --- /dev/null +++ b/src/flow/netbeans/markdown/highlighter/MarkdownTokenListBuilder.java @@ -0,0 +1,70 @@ +package flow.netbeans.markdown.highlighter; + +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Deque; +import java.util.List; +import org.pegdown.ast.Node; + +/** + * + * @author Holger + */ +public class MarkdownTokenListBuilder { + private final Deque stack; + + private final List tokens; + + private int depth; + + public MarkdownTokenListBuilder(int inputLength) { + this.stack = new ArrayDeque(); + this.tokens = new ArrayList(); + this.depth = 0; + // Push guard tree token. + this.stack.push(new MarkdownLexerVisitor.MarkdownTreeToken(MarkdownTokenId.WHITESPACE, this.depth, 0, + inputLength)); + } + + public void addLeafTreeToken(MarkdownTokenId id, Node node) { + beginTreeToken(id, node); + endTreeToken(); + } + + public void beginTreeToken(MarkdownTokenId id, Node node) { + depth++; + MarkdownLexerVisitor.MarkdownTreeToken topTreeToken = stack.peek(); + final int startIndex = Math.max(node.getStartIndex(), topTreeToken.getStartIndex()); + final int endIndex = Math.min(node.getEndIndex(), topTreeToken.getEndIndex()); + addToken(topTreeToken.getId(), topTreeToken.getRemainderIndex(), startIndex); + topTreeToken.setRemainderIndex(endIndex); + MarkdownLexerVisitor.MarkdownTreeToken treeToken = new MarkdownLexerVisitor.MarkdownTreeToken(id, depth, + startIndex, endIndex); +// LOG.log(Level.INFO, "Added tree token to stack: {0}", treeToken.toString()); + stack.push(treeToken); + } + + public void endTreeToken() { + MarkdownLexerVisitor.MarkdownTreeToken treeToken = stack.pop(); +// LOG.log(Level.INFO, "Removed tree token from stack: {0}", treeToken.toString()); + addToken(treeToken.getId(), treeToken.getRemainderIndex(), treeToken.getEndIndex()); + depth--; + } + + private void addToken(MarkdownTokenId id, int startIndex, int endIndex) { + if (startIndex < endIndex) { + MarkdownToken token = new MarkdownToken(id, startIndex, endIndex); +// LOG.log(Level.INFO, "Added token to list: {0}", token); + tokens.add(token); + } + } + + public List build() { + if (stack.size() == 1) { + // Pop guard tree token. + endTreeToken(); + } + return Collections.unmodifiableList(tokens); + } +} diff --git a/src/flow/netbeans/markdown/highlighter/MarkdownTokenMap.java b/src/flow/netbeans/markdown/highlighter/MarkdownTokenMap.java deleted file mode 100644 index 4facce8..0000000 --- a/src/flow/netbeans/markdown/highlighter/MarkdownTokenMap.java +++ /dev/null @@ -1,23 +0,0 @@ - -package flow.netbeans.markdown.highlighter; - -import java.util.TreeMap; - -public class MarkdownTokenMap extends TreeMap { - - private int totalTokenLength = 0; - - public MarkdownTokenMap() { - this.totalTokenLength = 0; - } - - @Override - public MarkdownToken put(Number key, MarkdownToken token) { - this.totalTokenLength += token.getLength(); - return super.put(key, token); - } - - public int getTotalTokenLength() { - return totalTokenLength; - } -} diff --git a/src/flow/netbeans/markdown/highlighter/MarkdownVisitor.java b/src/flow/netbeans/markdown/highlighter/MarkdownVisitor.java deleted file mode 100644 index 20ba425..0000000 --- a/src/flow/netbeans/markdown/highlighter/MarkdownVisitor.java +++ /dev/null @@ -1,246 +0,0 @@ -package flow.netbeans.markdown.highlighter; - -import org.pegdown.ast.*; -import static org.pegdown.ast.SimpleNode.Type.HRule; - -public class MarkdownVisitor implements Visitor { - - private MarkdownTokenId currentToken = MarkdownTokenId.PLAIN; - - private final MarkdownTokenMap tokenMap = new MarkdownTokenMap(); - - @Override - public void visit(AbbreviationNode node) { - addToken(MarkdownTokenId.ABBREVIATION, node); - } - - @Override - public void visit(AutoLinkNode node) { - addToken(MarkdownTokenId.AUTOLINK, node); - } - - @Override - public void visit(BlockQuoteNode node) { - addToken(MarkdownTokenId.BLOCKQUOTE, node); - } - - @Override - public void visit(CodeNode node) { - addToken(MarkdownTokenId.CODE, node); - } - - @Override - public void visit(DefinitionListNode node) { - addToken(MarkdownTokenId.DEFINITION_LIST, node); - } - - @Override - public void visit(DefinitionNode node) { - addToken(MarkdownTokenId.DEFINITION, node); - } - - @Override - public void visit(DefinitionTermNode node) { - addToken(MarkdownTokenId.DEFINITION_TERM, node); - } - - @Override - public void visit(ExpImageNode node) { - addToken(MarkdownTokenId.EXPIMAGE, node); - } - - @Override - public void visit(ExpLinkNode node) { - addToken(MarkdownTokenId.EXPLINK, node); - } - - @Override - public void visit(HeaderNode node) { - final MarkdownTokenId tokenId; - switch (node.getLevel()) { - case 1: tokenId = MarkdownTokenId.HEADER1; break; - case 2: tokenId = MarkdownTokenId.HEADER2; break; - case 3: tokenId = MarkdownTokenId.HEADER3; break; - case 4: tokenId = MarkdownTokenId.HEADER4; break; - case 5: tokenId = MarkdownTokenId.HEADER5; break; - case 6: default: tokenId = MarkdownTokenId.HEADER6; break; - } - addToken(tokenId, node); - } - - @Override - public void visit(HtmlBlockNode node) { - addToken(MarkdownTokenId.HTMLBLOCK, node); - } - - @Override - public void visit(InlineHtmlNode node) { - addToken(MarkdownTokenId.INLINEHTML, node); - } - - @Override - public void visit(ListItemNode node) { - addToken(MarkdownTokenId.LISTITEM, node); - //visitChildren(node); - } - - @Override - public void visit(OrderedListNode node) { - addToken(MarkdownTokenId.ORDEREDLIST, node); - //visitChildren(node); - } - - @Override - public void visit(BulletListNode node) { - addToken(MarkdownTokenId.BULLETLIST, node); - //visitChildren(node); - } - - @Override - public void visit(MailLinkNode node) { - addToken(MarkdownTokenId.MAILLINK, node); - } - - @Override - public void visit(ParaNode node) { - visitChildren(node); - } - - @Override - public void visit(QuotedNode node) { - addToken(MarkdownTokenId.QUOTED, node); - } - - @Override - public void visit(ReferenceNode node) { - addToken(MarkdownTokenId.REFERENCE, node); - } - - @Override - public void visit(RefImageNode node) { - addToken(MarkdownTokenId.REF_IMAGE, node); - } - - @Override - public void visit(RefLinkNode node) { - addToken(MarkdownTokenId.REF_LINK, node); - } - - @Override - public void visit(SimpleNode node) { - switch (node.getType()) { - case HRule: - addToken(MarkdownTokenId.HORIZONTALRULE, node); - break; - } - } - - @Override - public void visit(SpecialTextNode node) { - - } - - @Override - public void visit(TableBodyNode node) { - - } - - @Override - public void visit(TableCellNode node) { - - } - - @Override - public void visit(TableColumnNode node) { - - } - - @Override - public void visit(TableHeaderNode node) { - - } - - @Override - public void visit(TableNode node) { - visitChildren(node); - } - - @Override - public void visit(TableRowNode node) { - - } - - @Override - public void visit(TableCaptionNode node) { - - } - - @Override - public void visit(VerbatimNode node) { - addToken(MarkdownTokenId.VERBATIM, node); - } - - @Override - public void visit(WikiLinkNode node) { - currentToken = MarkdownTokenId.WIKILINK; - } - - @Override - public void visit(TextNode node) { - - } - - @Override - public void visit(Node node) { - addToken(MarkdownTokenId.PLAIN, node); - } - - - @Override - public void visit(RootNode node) { - for (AbbreviationNode abbreviationNode : node.getAbbreviations()) { - abbreviationNode.accept(this); - } - for (ReferenceNode referenceNode : node.getReferences()) { - referenceNode.accept(this); - } - visitChildren(node); - } - - @Override - public void visit(SuperNode node) { - visitChildren(node); - } - - protected void visitChildren(Node node) { - for (Node child : node.getChildren()) { - child.accept(this); - } - } - - public MarkdownTokenId getCurrentToken() - { - MarkdownTokenId tk = currentToken; - currentToken = MarkdownTokenId.PLAIN; - return tk; - } - - @Override - public void visit(StrongEmphSuperNode node) { - addToken((node.isStrong() ? MarkdownTokenId.STRONG : MarkdownTokenId.EMPH), node); - visitChildren(node); - } - - private void addToken(MarkdownTokenId tokenId, Node node) { - MarkdownToken tkn = new MarkdownToken(tokenId, - node.getEndIndex()-node.getStartIndex() - ); - tokenMap.put(node.getStartIndex(), tkn); - currentToken = tokenId; - } - - public MarkdownTokenMap getTokenMap() { - return tokenMap; - } - -} \ No newline at end of file From 61915db92b55a346ba8bbeafefd097153701c0db Mon Sep 17 00:00:00 2001 From: Holger Stenger Date: Tue, 12 Nov 2013 23:39:06 +0100 Subject: [PATCH 2/8] Added unit tests for MarkdownTokenListBuilder. Refactored MarkdownTokenListBuilder to simplify testing. --- .../highlighter/MarkdownLexerVisitor.java | 4 +- .../markdown/highlighter/MarkdownToken.java | 5 + .../highlighter/MarkdownTokenListBuilder.java | 80 +++++++++---- .../MarkdownTokenListBuilderTest.java | 106 ++++++++++++++++++ .../matchers/ContainsTokensWithId.java | 45 ++++++++ .../matchers/ContainsTokensWithLength.java | 45 ++++++++ .../netbeans/markdown/matchers/Empty.java | 31 +++++ .../netbeans/markdown/matchers/EveryItem.java | 39 +++++++ .../netbeans/markdown/matchers/Matchers.java | 38 +++++++ .../markdown/matchers/NonZeroLength.java | 28 +++++ 10 files changed, 398 insertions(+), 23 deletions(-) create mode 100644 test/unit/src/flow/netbeans/markdown/highlighter/MarkdownTokenListBuilderTest.java create mode 100644 test/unit/src/flow/netbeans/markdown/matchers/ContainsTokensWithId.java create mode 100644 test/unit/src/flow/netbeans/markdown/matchers/ContainsTokensWithLength.java create mode 100644 test/unit/src/flow/netbeans/markdown/matchers/Empty.java create mode 100644 test/unit/src/flow/netbeans/markdown/matchers/EveryItem.java create mode 100644 test/unit/src/flow/netbeans/markdown/matchers/Matchers.java create mode 100644 test/unit/src/flow/netbeans/markdown/matchers/NonZeroLength.java diff --git a/src/flow/netbeans/markdown/highlighter/MarkdownLexerVisitor.java b/src/flow/netbeans/markdown/highlighter/MarkdownLexerVisitor.java index 2973123..d349f40 100644 --- a/src/flow/netbeans/markdown/highlighter/MarkdownLexerVisitor.java +++ b/src/flow/netbeans/markdown/highlighter/MarkdownLexerVisitor.java @@ -51,11 +51,11 @@ public MarkdownLexerVisitor(MarkdownTokenListBuilder builder) { } private void addLeafTreeToken(MarkdownTokenId id, Node node) { - builder.addLeafTreeToken(id, node); + builder.addLeafTreeToken(id, node.getStartIndex(), node.getEndIndex()); } private void beginTreeToken(MarkdownTokenId id, Node node) { - builder.beginTreeToken(id, node); + builder.beginTreeToken(id, node.getStartIndex(), node.getEndIndex()); } private void endTreeToken() { diff --git a/src/flow/netbeans/markdown/highlighter/MarkdownToken.java b/src/flow/netbeans/markdown/highlighter/MarkdownToken.java index 4a7dc4c..1668c93 100644 --- a/src/flow/netbeans/markdown/highlighter/MarkdownToken.java +++ b/src/flow/netbeans/markdown/highlighter/MarkdownToken.java @@ -32,4 +32,9 @@ public int getEndIndex() { public int getLength() { return endIndex - startIndex; } + + @Override + public String toString() { + return id + "(" + getLength() + ")[" + startIndex + "-" + endIndex + "]"; + } } diff --git a/src/flow/netbeans/markdown/highlighter/MarkdownTokenListBuilder.java b/src/flow/netbeans/markdown/highlighter/MarkdownTokenListBuilder.java index ad6225f..73623af 100644 --- a/src/flow/netbeans/markdown/highlighter/MarkdownTokenListBuilder.java +++ b/src/flow/netbeans/markdown/highlighter/MarkdownTokenListBuilder.java @@ -5,7 +5,6 @@ import java.util.Collections; import java.util.Deque; import java.util.List; -import org.pegdown.ast.Node; /** * @@ -16,40 +15,72 @@ public class MarkdownTokenListBuilder { private final List tokens; - private int depth; - - public MarkdownTokenListBuilder(int inputLength) { - this.stack = new ArrayDeque(); - this.tokens = new ArrayList(); - this.depth = 0; + /** + * Creates a {@link MarkdownTokenListBuilder}. The total length of the tokens created by the builder will be equal + * to the given total length. If the first added token does not start at position {@code 0}, the builder will insert + * a synthetic token to fill the gap. Likewise, if the last added token does not end at position + * {@code totalLength}, the builder will insert a synthetic token to fill the gap. + * @param totalLength The total length of the created tokens. + */ + public MarkdownTokenListBuilder(int totalLength) { + stack = new ArrayDeque(); + tokens = new ArrayList(); // Push guard tree token. - this.stack.push(new MarkdownLexerVisitor.MarkdownTreeToken(MarkdownTokenId.WHITESPACE, this.depth, 0, - inputLength)); + stack.push(new MarkdownLexerVisitor.MarkdownTreeToken(MarkdownTokenId.WHITESPACE, stack.size(), 0, + totalLength)); } - public void addLeafTreeToken(MarkdownTokenId id, Node node) { - beginTreeToken(id, node); + /** + * Add a token which contains no nested tokens. This method is equivalent to a call to + * {@link #beginTreeToken(flow.netbeans.markdown.highlighter.MarkdownTokenId, int, int)} directly followed by a call + * to {@link #endTreeToken()}. + * @param id The token id. + * @param startIndex The start index of the token (inclusive). + * @param endIndex The end index of the token (exclusive). + */ + public void addLeafTreeToken(MarkdownTokenId id, int startIndex, int endIndex) { + beginTreeToken(id, startIndex, endIndex); endTreeToken(); } - public void beginTreeToken(MarkdownTokenId id, Node node) { - depth++; + /** + * Begins a token which can contain nested tokens. The call to this method has to be matched with a call to + * {@link #endTreeToken()}. + * @param id The token id. + * @param startIndex The start index of the token (inclusive). + * @param endIndex The end index of the token (exclusive). + */ + public void beginTreeToken(final MarkdownTokenId id, final int startIndex, final int endIndex) { + if (stack.isEmpty()) { + throw new IllegalStateException("No tokens can be added after calling build"); + } MarkdownLexerVisitor.MarkdownTreeToken topTreeToken = stack.peek(); - final int startIndex = Math.max(node.getStartIndex(), topTreeToken.getStartIndex()); - final int endIndex = Math.min(node.getEndIndex(), topTreeToken.getEndIndex()); - addToken(topTreeToken.getId(), topTreeToken.getRemainderIndex(), startIndex); - topTreeToken.setRemainderIndex(endIndex); - MarkdownLexerVisitor.MarkdownTreeToken treeToken = new MarkdownLexerVisitor.MarkdownTreeToken(id, depth, - startIndex, endIndex); + final int effectiveStartIndex = Math.max(startIndex, topTreeToken.getStartIndex()); + final int effectiveEndIndex = Math.min(endIndex, topTreeToken.getEndIndex()); + addToken(topTreeToken.getId(), topTreeToken.getRemainderIndex(), effectiveStartIndex); + topTreeToken.setRemainderIndex(effectiveEndIndex); + MarkdownLexerVisitor.MarkdownTreeToken treeToken = new MarkdownLexerVisitor.MarkdownTreeToken(id, stack.size(), + effectiveStartIndex, effectiveEndIndex); // LOG.log(Level.INFO, "Added tree token to stack: {0}", treeToken.toString()); stack.push(treeToken); } + /** + * Ends a token which has been begun by a call to + * {@link #beginTreeToken(flow.netbeans.markdown.highlighter.MarkdownTokenId, int, int)}. + */ public void endTreeToken() { + if (stack.size() < 2) { + // Protect guard tree token. + throw new IllegalStateException("No token on stack"); + } + endTreeTokenUnchecked(); + } + + private void endTreeTokenUnchecked() { MarkdownLexerVisitor.MarkdownTreeToken treeToken = stack.pop(); // LOG.log(Level.INFO, "Removed tree token from stack: {0}", treeToken.toString()); addToken(treeToken.getId(), treeToken.getRemainderIndex(), treeToken.getEndIndex()); - depth--; } private void addToken(MarkdownTokenId id, int startIndex, int endIndex) { @@ -60,10 +91,17 @@ private void addToken(MarkdownTokenId id, int startIndex, int endIndex) { } } + /** + * Finishes the token list construction. + * @return The list of created tokens. + */ public List build() { + if (stack.size() > 1) { + throw new IllegalStateException("Still tokens on stack"); + } if (stack.size() == 1) { // Pop guard tree token. - endTreeToken(); + endTreeTokenUnchecked(); } return Collections.unmodifiableList(tokens); } diff --git a/test/unit/src/flow/netbeans/markdown/highlighter/MarkdownTokenListBuilderTest.java b/test/unit/src/flow/netbeans/markdown/highlighter/MarkdownTokenListBuilderTest.java new file mode 100644 index 0000000..ce87954 --- /dev/null +++ b/test/unit/src/flow/netbeans/markdown/highlighter/MarkdownTokenListBuilderTest.java @@ -0,0 +1,106 @@ + +package flow.netbeans.markdown.highlighter; + +import java.util.List; +import org.junit.Test; +import static org.junit.Assert.*; +import static org.hamcrest.CoreMatchers.*; +import static flow.netbeans.markdown.matchers.Matchers.*; +import static flow.netbeans.markdown.highlighter.MarkdownTokenId.*; + +import org.junit.Before; + +/** + * Unit tests for {@link MarkdownTokenListBuilder}. + */ +public class MarkdownTokenListBuilderTest { + private static final int DEFAULT_INPUT_LENGTH = 10; + + private MarkdownTokenListBuilder sut; + + @Before + public void createBuilder() { + sut = new MarkdownTokenListBuilder(DEFAULT_INPUT_LENGTH); + } + + @Test + public void testNoToken() { + final List tokens = sut.build(); + + assertThat(tokens, everyItem(nonZeroLength())); + assertThat(totalLengthOf(tokens), equalTo(DEFAULT_INPUT_LENGTH)); + + assertThat(tokens.size(), equalTo(1)); + } + + @Test + public void testSingleTokenWithFullCoverage() { + sut.addLeafTreeToken(CODE, 0, DEFAULT_INPUT_LENGTH); + + final List tokens = sut.build(); + + assertThat(tokens, everyItem(nonZeroLength())); + assertThat(totalLengthOf(tokens), equalTo(DEFAULT_INPUT_LENGTH)); + + assertThat(tokens.size(), equalTo(1)); + assertThat(tokens, containsTokensWithId(CODE)); + } + + @Test + public void testSingleTokenWithoutFullCoverage() { + sut.addLeafTreeToken(CODE, 1, DEFAULT_INPUT_LENGTH - 1); + + final List tokens = sut.build(); + + assertThat(tokens, everyItem(nonZeroLength())); + assertThat(totalLengthOf(tokens), equalTo(DEFAULT_INPUT_LENGTH)); + + assertThat(tokens.size(), equalTo(3)); + assertThat(tokens, containsTokensWithId(WHITESPACE, CODE, WHITESPACE)); + assertThat(tokens, containsTokensWithLength(1, DEFAULT_INPUT_LENGTH - 2, 1)); + } + + @Test + public void testNestedTokenWithLargerRangeThanOuterToken() { + sut.beginTreeToken(CODE, 1, DEFAULT_INPUT_LENGTH - 1); + sut.addLeafTreeToken(EMPH, 0, DEFAULT_INPUT_LENGTH); + sut.endTreeToken(); + + final List tokens = sut.build(); + + assertThat(tokens, everyItem(nonZeroLength())); + assertThat(totalLengthOf(tokens), equalTo(DEFAULT_INPUT_LENGTH)); + + assertThat(tokens.size(), equalTo(3)); + assertThat(tokens, containsTokensWithId(WHITESPACE, EMPH, WHITESPACE)); + assertThat(tokens, containsTokensWithLength(1, DEFAULT_INPUT_LENGTH - 2, 1)); + } + + @Test + public void testEmptyInput() { + sut = new MarkdownTokenListBuilder(0); + + List tokens = sut.build(); + + assertThat(tokens.size(), equalTo(0)); + } + + @Test(expected = IllegalStateException.class) + public void testUnmatchedEndTokenCall() { + sut.endTreeToken(); + } + + @Test(expected = IllegalStateException.class) + public void testUnmatchedBeginTokenCall() { + sut.beginTreeToken(CODE, 0, DEFAULT_INPUT_LENGTH); + + sut.build(); + } + + @Test(expected = IllegalStateException.class) + public void testBeginTokenCallAfterBuild() { + sut.build(); + + sut.beginTreeToken(CODE, 0, DEFAULT_INPUT_LENGTH); + } +} diff --git a/test/unit/src/flow/netbeans/markdown/matchers/ContainsTokensWithId.java b/test/unit/src/flow/netbeans/markdown/matchers/ContainsTokensWithId.java new file mode 100644 index 0000000..78d40d0 --- /dev/null +++ b/test/unit/src/flow/netbeans/markdown/matchers/ContainsTokensWithId.java @@ -0,0 +1,45 @@ + +package flow.netbeans.markdown.matchers; + +import flow.netbeans.markdown.highlighter.MarkdownToken; +import flow.netbeans.markdown.highlighter.MarkdownTokenId; +import java.util.Iterator; +import org.hamcrest.BaseMatcher; +import org.hamcrest.Description; + +/** + * + * @author Holger + */ +class ContainsTokensWithId extends BaseMatcher> { + private final MarkdownTokenId[] ids; + + public ContainsTokensWithId(MarkdownTokenId... ids) { + this.ids = ids.clone(); + } + + @Override + public boolean matches(Object item) { + Iterable tokens = (Iterable) item; + Iterator iterator = tokens.iterator(); + boolean result = true; + int index; + for (index = 0; index < ids.length && iterator.hasNext() && result; ++index) { + MarkdownToken token = iterator.next(); + if (!token.getId().equals(ids[index])) { + return false; + } + } + if (index < ids.length || iterator.hasNext()) { + return false; + } + return true; + } + + @Override + public void describeTo(Description description) { + description.appendText("token ids not equal to "); + description.appendValueList("", ", ", "", ids); + } + +} diff --git a/test/unit/src/flow/netbeans/markdown/matchers/ContainsTokensWithLength.java b/test/unit/src/flow/netbeans/markdown/matchers/ContainsTokensWithLength.java new file mode 100644 index 0000000..4e4fbfe --- /dev/null +++ b/test/unit/src/flow/netbeans/markdown/matchers/ContainsTokensWithLength.java @@ -0,0 +1,45 @@ + +package flow.netbeans.markdown.matchers; + +import flow.netbeans.markdown.highlighter.MarkdownToken; +import java.util.Arrays; +import java.util.Iterator; +import org.hamcrest.BaseMatcher; +import org.hamcrest.Description; + +/** + * + * @author Holger + */ +class ContainsTokensWithLength extends BaseMatcher> { + private final int[] lengths; + + public ContainsTokensWithLength(int... lengths) { + this.lengths = lengths.clone(); + } + + @Override + public boolean matches(Object item) { + Iterable tokens = (Iterable) item; + Iterator iterator = tokens.iterator(); + boolean result = true; + int index; + for (index = 0; index < lengths.length && iterator.hasNext() && result; ++index) { + MarkdownToken token = iterator.next(); + if (token.getLength() != lengths[index]) { + return false; + } + } + if (index < lengths.length || iterator.hasNext()) { + return false; + } + return true; + } + + @Override + public void describeTo(Description description) { + description.appendText("token lengths not equal to "); + description.appendValueList("", ", ", "", Arrays.asList(lengths)); + } + +} diff --git a/test/unit/src/flow/netbeans/markdown/matchers/Empty.java b/test/unit/src/flow/netbeans/markdown/matchers/Empty.java new file mode 100644 index 0000000..5cafd60 --- /dev/null +++ b/test/unit/src/flow/netbeans/markdown/matchers/Empty.java @@ -0,0 +1,31 @@ + +package flow.netbeans.markdown.matchers; + +import java.util.Collection; +import org.hamcrest.BaseMatcher; +import org.hamcrest.Description; + +/** + * + * @author Holger + */ +class Empty extends BaseMatcher> { + + public Empty() { + } + + @Override + public boolean matches(Object item) { + if (item instanceof Collection) { + final Collection collection = (Collection) item; + return collection.isEmpty(); + } + return false; + } + + @Override + public void describeTo(Description description) { + description.appendText("collection is not empty"); + } + +} diff --git a/test/unit/src/flow/netbeans/markdown/matchers/EveryItem.java b/test/unit/src/flow/netbeans/markdown/matchers/EveryItem.java new file mode 100644 index 0000000..dacfeda --- /dev/null +++ b/test/unit/src/flow/netbeans/markdown/matchers/EveryItem.java @@ -0,0 +1,39 @@ +package flow.netbeans.markdown.matchers; + +import java.util.Iterator; +import org.hamcrest.BaseMatcher; +import org.hamcrest.Description; +import org.hamcrest.Matcher; + +/** + * + * @author Holger + */ +public class EveryItem extends BaseMatcher> { + private final Matcher itemMatcher; + + public EveryItem(Matcher itemMatcher) { + this.itemMatcher = itemMatcher; + } + + @Override + public boolean matches(Object item) { + Iterable iterable = (Iterable) item; + boolean result = true; + Iterator iterator = iterable.iterator(); + while (result && iterator.hasNext()) { + final T element = iterator.next(); + if (!itemMatcher.matches(element)) { + result = false; + } + } + return result; + } + + @Override + public void describeTo(Description description) { + description.appendText("contains an element with "); + description.appendDescriptionOf(itemMatcher); + } + +} diff --git a/test/unit/src/flow/netbeans/markdown/matchers/Matchers.java b/test/unit/src/flow/netbeans/markdown/matchers/Matchers.java new file mode 100644 index 0000000..7379039 --- /dev/null +++ b/test/unit/src/flow/netbeans/markdown/matchers/Matchers.java @@ -0,0 +1,38 @@ + +package flow.netbeans.markdown.matchers; + +import flow.netbeans.markdown.highlighter.MarkdownToken; +import flow.netbeans.markdown.highlighter.MarkdownTokenId; +import org.hamcrest.Matcher; + +/** + * + * @author Holger + */ +public class Matchers { + private Matchers() {} + + public static Matcher> everyItem(Matcher itemMatcher) { + return new EveryItem(itemMatcher); + } + + public static Matcher nonZeroLength() { + return new NonZeroLength(); + } + + public static int totalLengthOf(Iterable iterable) { + int totalLength = 0; + for (MarkdownToken token : iterable) { + totalLength += token.getLength(); + } + return totalLength; + } + + public static Matcher> containsTokensWithId(final MarkdownTokenId... ids) { + return new ContainsTokensWithId(ids); + } + + public static Matcher> containsTokensWithLength(final int... lengths) { + return new ContainsTokensWithLength(lengths); + } +} diff --git a/test/unit/src/flow/netbeans/markdown/matchers/NonZeroLength.java b/test/unit/src/flow/netbeans/markdown/matchers/NonZeroLength.java new file mode 100644 index 0000000..7aac40b --- /dev/null +++ b/test/unit/src/flow/netbeans/markdown/matchers/NonZeroLength.java @@ -0,0 +1,28 @@ + +package flow.netbeans.markdown.matchers; + +import flow.netbeans.markdown.highlighter.MarkdownToken; +import org.hamcrest.BaseMatcher; +import org.hamcrest.Description; + +/** + * + * @author Holger + */ +class NonZeroLength extends BaseMatcher { + + public NonZeroLength() { + } + + @Override + public boolean matches(Object item) { + MarkdownToken token = (MarkdownToken) item; + return token.getLength() > 0; + } + + @Override + public void describeTo(Description description) { + description.appendText("length zero"); + } + +} From 7232deaca2855f84afcf9564243ec6f44ad662e0 Mon Sep 17 00:00:00 2001 From: Holger Stenger Date: Wed, 13 Nov 2013 18:54:40 +0100 Subject: [PATCH 3/8] Switched matcher base class to TypeSafeMatcher. --- .../MarkdownTokenListBuilderTest.java | 46 +++++++++++-------- .../matchers/ContainsTokensWithId.java | 9 ++-- .../matchers/ContainsTokensWithLength.java | 9 ++-- .../netbeans/markdown/matchers/Empty.java | 14 ++---- .../netbeans/markdown/matchers/EveryItem.java | 9 ++-- .../markdown/matchers/NonZeroLength.java | 9 ++-- 6 files changed, 48 insertions(+), 48 deletions(-) diff --git a/test/unit/src/flow/netbeans/markdown/highlighter/MarkdownTokenListBuilderTest.java b/test/unit/src/flow/netbeans/markdown/highlighter/MarkdownTokenListBuilderTest.java index ce87954..c2c5c96 100644 --- a/test/unit/src/flow/netbeans/markdown/highlighter/MarkdownTokenListBuilderTest.java +++ b/test/unit/src/flow/netbeans/markdown/highlighter/MarkdownTokenListBuilderTest.java @@ -16,16 +16,16 @@ public class MarkdownTokenListBuilderTest { private static final int DEFAULT_INPUT_LENGTH = 10; - private MarkdownTokenListBuilder sut; + private MarkdownTokenListBuilder builder; @Before public void createBuilder() { - sut = new MarkdownTokenListBuilder(DEFAULT_INPUT_LENGTH); + builder = new MarkdownTokenListBuilder(DEFAULT_INPUT_LENGTH); } @Test public void testNoToken() { - final List tokens = sut.build(); + final List tokens = builder.build(); assertThat(tokens, everyItem(nonZeroLength())); assertThat(totalLengthOf(tokens), equalTo(DEFAULT_INPUT_LENGTH)); @@ -35,9 +35,9 @@ public void testNoToken() { @Test public void testSingleTokenWithFullCoverage() { - sut.addLeafTreeToken(CODE, 0, DEFAULT_INPUT_LENGTH); + builder.addLeafTreeToken(CODE, 0, DEFAULT_INPUT_LENGTH); - final List tokens = sut.build(); + final List tokens = builder.build(); assertThat(tokens, everyItem(nonZeroLength())); assertThat(totalLengthOf(tokens), equalTo(DEFAULT_INPUT_LENGTH)); @@ -48,9 +48,9 @@ public void testSingleTokenWithFullCoverage() { @Test public void testSingleTokenWithoutFullCoverage() { - sut.addLeafTreeToken(CODE, 1, DEFAULT_INPUT_LENGTH - 1); + builder.addLeafTreeToken(CODE, 1, DEFAULT_INPUT_LENGTH - 1); - final List tokens = sut.build(); + final List tokens = builder.build(); assertThat(tokens, everyItem(nonZeroLength())); assertThat(totalLengthOf(tokens), equalTo(DEFAULT_INPUT_LENGTH)); @@ -62,11 +62,11 @@ public void testSingleTokenWithoutFullCoverage() { @Test public void testNestedTokenWithLargerRangeThanOuterToken() { - sut.beginTreeToken(CODE, 1, DEFAULT_INPUT_LENGTH - 1); - sut.addLeafTreeToken(EMPH, 0, DEFAULT_INPUT_LENGTH); - sut.endTreeToken(); + builder.beginTreeToken(CODE, 1, DEFAULT_INPUT_LENGTH - 1); + builder.addLeafTreeToken(EMPH, 0, DEFAULT_INPUT_LENGTH); + builder.endTreeToken(); - final List tokens = sut.build(); + final List tokens = builder.build(); assertThat(tokens, everyItem(nonZeroLength())); assertThat(totalLengthOf(tokens), equalTo(DEFAULT_INPUT_LENGTH)); @@ -78,29 +78,39 @@ public void testNestedTokenWithLargerRangeThanOuterToken() { @Test public void testEmptyInput() { - sut = new MarkdownTokenListBuilder(0); + builder = new MarkdownTokenListBuilder(0); - List tokens = sut.build(); + List tokens = builder.build(); assertThat(tokens.size(), equalTo(0)); } @Test(expected = IllegalStateException.class) public void testUnmatchedEndTokenCall() { - sut.endTreeToken(); + builder.endTreeToken(); } @Test(expected = IllegalStateException.class) public void testUnmatchedBeginTokenCall() { - sut.beginTreeToken(CODE, 0, DEFAULT_INPUT_LENGTH); + builder.beginTreeToken(CODE, 0, DEFAULT_INPUT_LENGTH); - sut.build(); + builder.build(); } @Test(expected = IllegalStateException.class) public void testBeginTokenCallAfterBuild() { - sut.build(); + builder.build(); - sut.beginTreeToken(CODE, 0, DEFAULT_INPUT_LENGTH); + builder.beginTreeToken(CODE, 0, DEFAULT_INPUT_LENGTH); + } + + @Test + public void testBuildTwice() { + builder.build(); + try { + builder.build(); + } catch (IllegalStateException ex) { + fail("Failed to build twice"); + } } } diff --git a/test/unit/src/flow/netbeans/markdown/matchers/ContainsTokensWithId.java b/test/unit/src/flow/netbeans/markdown/matchers/ContainsTokensWithId.java index 78d40d0..8651a8b 100644 --- a/test/unit/src/flow/netbeans/markdown/matchers/ContainsTokensWithId.java +++ b/test/unit/src/flow/netbeans/markdown/matchers/ContainsTokensWithId.java @@ -4,14 +4,14 @@ import flow.netbeans.markdown.highlighter.MarkdownToken; import flow.netbeans.markdown.highlighter.MarkdownTokenId; import java.util.Iterator; -import org.hamcrest.BaseMatcher; import org.hamcrest.Description; +import org.junit.internal.matchers.TypeSafeMatcher; /** * * @author Holger */ -class ContainsTokensWithId extends BaseMatcher> { +public class ContainsTokensWithId extends TypeSafeMatcher> { private final MarkdownTokenId[] ids; public ContainsTokensWithId(MarkdownTokenId... ids) { @@ -19,9 +19,8 @@ public ContainsTokensWithId(MarkdownTokenId... ids) { } @Override - public boolean matches(Object item) { - Iterable tokens = (Iterable) item; - Iterator iterator = tokens.iterator(); + public boolean matchesSafely(Iterable item) { + Iterator iterator = item.iterator(); boolean result = true; int index; for (index = 0; index < ids.length && iterator.hasNext() && result; ++index) { diff --git a/test/unit/src/flow/netbeans/markdown/matchers/ContainsTokensWithLength.java b/test/unit/src/flow/netbeans/markdown/matchers/ContainsTokensWithLength.java index 4e4fbfe..7b2b862 100644 --- a/test/unit/src/flow/netbeans/markdown/matchers/ContainsTokensWithLength.java +++ b/test/unit/src/flow/netbeans/markdown/matchers/ContainsTokensWithLength.java @@ -4,14 +4,14 @@ import flow.netbeans.markdown.highlighter.MarkdownToken; import java.util.Arrays; import java.util.Iterator; -import org.hamcrest.BaseMatcher; import org.hamcrest.Description; +import org.junit.internal.matchers.TypeSafeMatcher; /** * * @author Holger */ -class ContainsTokensWithLength extends BaseMatcher> { +class ContainsTokensWithLength extends TypeSafeMatcher> { private final int[] lengths; public ContainsTokensWithLength(int... lengths) { @@ -19,9 +19,8 @@ public ContainsTokensWithLength(int... lengths) { } @Override - public boolean matches(Object item) { - Iterable tokens = (Iterable) item; - Iterator iterator = tokens.iterator(); + public boolean matchesSafely(Iterable item) { + Iterator iterator = item.iterator(); boolean result = true; int index; for (index = 0; index < lengths.length && iterator.hasNext() && result; ++index) { diff --git a/test/unit/src/flow/netbeans/markdown/matchers/Empty.java b/test/unit/src/flow/netbeans/markdown/matchers/Empty.java index 5cafd60..852c6df 100644 --- a/test/unit/src/flow/netbeans/markdown/matchers/Empty.java +++ b/test/unit/src/flow/netbeans/markdown/matchers/Empty.java @@ -1,26 +1,20 @@ - package flow.netbeans.markdown.matchers; import java.util.Collection; -import org.hamcrest.BaseMatcher; import org.hamcrest.Description; +import org.junit.internal.matchers.TypeSafeMatcher; /** * * @author Holger */ -class Empty extends BaseMatcher> { - +class Empty extends TypeSafeMatcher> { public Empty() { } @Override - public boolean matches(Object item) { - if (item instanceof Collection) { - final Collection collection = (Collection) item; - return collection.isEmpty(); - } - return false; + public boolean matchesSafely(Collection item) { + return item.isEmpty(); } @Override diff --git a/test/unit/src/flow/netbeans/markdown/matchers/EveryItem.java b/test/unit/src/flow/netbeans/markdown/matchers/EveryItem.java index dacfeda..f379eed 100644 --- a/test/unit/src/flow/netbeans/markdown/matchers/EveryItem.java +++ b/test/unit/src/flow/netbeans/markdown/matchers/EveryItem.java @@ -1,15 +1,15 @@ package flow.netbeans.markdown.matchers; import java.util.Iterator; -import org.hamcrest.BaseMatcher; import org.hamcrest.Description; import org.hamcrest.Matcher; +import org.junit.internal.matchers.TypeSafeMatcher; /** * * @author Holger */ -public class EveryItem extends BaseMatcher> { +public class EveryItem extends TypeSafeMatcher> { private final Matcher itemMatcher; public EveryItem(Matcher itemMatcher) { @@ -17,10 +17,9 @@ public EveryItem(Matcher itemMatcher) { } @Override - public boolean matches(Object item) { - Iterable iterable = (Iterable) item; + public boolean matchesSafely(Iterable item) { boolean result = true; - Iterator iterator = iterable.iterator(); + Iterator iterator = item.iterator(); while (result && iterator.hasNext()) { final T element = iterator.next(); if (!itemMatcher.matches(element)) { diff --git a/test/unit/src/flow/netbeans/markdown/matchers/NonZeroLength.java b/test/unit/src/flow/netbeans/markdown/matchers/NonZeroLength.java index 7aac40b..46ddc72 100644 --- a/test/unit/src/flow/netbeans/markdown/matchers/NonZeroLength.java +++ b/test/unit/src/flow/netbeans/markdown/matchers/NonZeroLength.java @@ -2,22 +2,21 @@ package flow.netbeans.markdown.matchers; import flow.netbeans.markdown.highlighter.MarkdownToken; -import org.hamcrest.BaseMatcher; import org.hamcrest.Description; +import org.junit.internal.matchers.TypeSafeMatcher; /** * * @author Holger */ -class NonZeroLength extends BaseMatcher { +public class NonZeroLength extends TypeSafeMatcher { public NonZeroLength() { } @Override - public boolean matches(Object item) { - MarkdownToken token = (MarkdownToken) item; - return token.getLength() > 0; + public boolean matchesSafely(MarkdownToken item) { + return item.getLength() > 0; } @Override From c51ac65df2a361a3b8e89f8250160c2ca3a05888 Mon Sep 17 00:00:00 2001 From: Holger Stenger Date: Wed, 13 Nov 2013 20:15:24 +0100 Subject: [PATCH 4/8] Added high-level unit tests for the lexer. --- .../markdown/highlighter/MarkdownLexerVisitor.java | 4 +--- test/unit/data/goldenfiles/headings.pass | 2 +- test/unit/data/goldenfiles/nested.pass | 5 +++++ test/unit/data/goldenfiles/plain.pass | 1 + test/unit/data/testfiles/headings.md | 2 +- test/unit/data/testfiles/nested.md | 1 + test/unit/data/testfiles/paragraphs.md | 2 +- test/unit/data/testfiles/plain.md | 1 + .../markdown/highlighter/MarkdownLexerTest.java | 11 ++++++++++- .../markdown/highlighter/MarkdownLexerTestBase.java | 8 ++++---- 10 files changed, 26 insertions(+), 11 deletions(-) create mode 100644 test/unit/data/goldenfiles/nested.pass create mode 100644 test/unit/data/goldenfiles/plain.pass create mode 100644 test/unit/data/testfiles/nested.md create mode 100644 test/unit/data/testfiles/plain.md diff --git a/src/flow/netbeans/markdown/highlighter/MarkdownLexerVisitor.java b/src/flow/netbeans/markdown/highlighter/MarkdownLexerVisitor.java index d349f40..745b2c9 100644 --- a/src/flow/netbeans/markdown/highlighter/MarkdownLexerVisitor.java +++ b/src/flow/netbeans/markdown/highlighter/MarkdownLexerVisitor.java @@ -79,8 +79,6 @@ public void visit(RootNode node) { beginTreeToken(MarkdownTokenId.PLAIN, node); visitChildren(node); endTreeToken(); - // Also pop the guard tree token. - endTreeToken(); } @Override @@ -258,7 +256,7 @@ public void visit(SimpleNode node) { @Override public void visit(SpecialTextNode node) { - addLeafTreeToken(MarkdownTokenId.TEXT, node); + //addLeafTreeToken(MarkdownTokenId.TEXT, node); } @Override diff --git a/test/unit/data/goldenfiles/headings.pass b/test/unit/data/goldenfiles/headings.pass index e683b42..3affb76 100644 --- a/test/unit/data/goldenfiles/headings.pass +++ b/test/unit/data/goldenfiles/headings.pass @@ -1 +1 @@ -token #0 HEADER [# H1\n] \ No newline at end of file +token #0 HEADER1 [# H1\n] \ No newline at end of file diff --git a/test/unit/data/goldenfiles/nested.pass b/test/unit/data/goldenfiles/nested.pass new file mode 100644 index 0000000..e99ebe6 --- /dev/null +++ b/test/unit/data/goldenfiles/nested.pass @@ -0,0 +1,5 @@ +token #0 PLAIN [a ] +token #1 EMPH [*b ] +token #2 STRONG [__c__] +token #3 EMPH [ d*] +token #4 PLAIN [ e\n] diff --git a/test/unit/data/goldenfiles/plain.pass b/test/unit/data/goldenfiles/plain.pass new file mode 100644 index 0000000..14ebb04 --- /dev/null +++ b/test/unit/data/goldenfiles/plain.pass @@ -0,0 +1 @@ +token #0 PLAIN [a*b__c__d*e\n] diff --git a/test/unit/data/testfiles/headings.md b/test/unit/data/testfiles/headings.md index 4b86865..baf209b 100644 --- a/test/unit/data/testfiles/headings.md +++ b/test/unit/data/testfiles/headings.md @@ -1 +1 @@ -# H1 \ No newline at end of file +# H1 diff --git a/test/unit/data/testfiles/nested.md b/test/unit/data/testfiles/nested.md new file mode 100644 index 0000000..61864a0 --- /dev/null +++ b/test/unit/data/testfiles/nested.md @@ -0,0 +1 @@ +a *b __c__ d* e \ No newline at end of file diff --git a/test/unit/data/testfiles/paragraphs.md b/test/unit/data/testfiles/paragraphs.md index 3763824..87fea10 100644 --- a/test/unit/data/testfiles/paragraphs.md +++ b/test/unit/data/testfiles/paragraphs.md @@ -1 +1 @@ -Die Goldstumpfnase (Rhinopithecus roxellana) ist eine Primatenart aus der Gruppe der Schlankaffen (Presbytini). \ No newline at end of file +Die Goldstumpfnase (Rhinopithecus roxellana) ist eine Primatenart aus der Gruppe der Schlankaffen (Presbytini). diff --git a/test/unit/data/testfiles/plain.md b/test/unit/data/testfiles/plain.md new file mode 100644 index 0000000..0e827c1 --- /dev/null +++ b/test/unit/data/testfiles/plain.md @@ -0,0 +1 @@ +a*b__c__d*e \ No newline at end of file diff --git a/test/unit/src/flow/netbeans/markdown/highlighter/MarkdownLexerTest.java b/test/unit/src/flow/netbeans/markdown/highlighter/MarkdownLexerTest.java index f27dcd6..bb4ee87 100644 --- a/test/unit/src/flow/netbeans/markdown/highlighter/MarkdownLexerTest.java +++ b/test/unit/src/flow/netbeans/markdown/highlighter/MarkdownLexerTest.java @@ -12,7 +12,6 @@ public MarkdownLexerTest() { @Test public void testCreate() { - System.out.println("create"); String content = "\n"; Language language = MarkdownTokenId.language(); TokenHierarchy hierarchy = TokenHierarchy.create(content, language); @@ -28,4 +27,14 @@ public void testParagraphs() throws Exception { public void testHeadings() throws Exception { assertEquals(getGoldenFileContent("headings.pass"), getTestResult("headings.md")); } + + @Test + public void testNested() throws Exception { + assertEquals(getGoldenFileContent("nested.pass"), getTestResult("nested.md")); + } + + @Test + public void testPlain() throws Exception { + assertEquals(getGoldenFileContent("plain.pass"), getTestResult("plain.md")); + } } \ No newline at end of file diff --git a/test/unit/src/flow/netbeans/markdown/highlighter/MarkdownLexerTestBase.java b/test/unit/src/flow/netbeans/markdown/highlighter/MarkdownLexerTestBase.java index dcc17ca..ab04d38 100644 --- a/test/unit/src/flow/netbeans/markdown/highlighter/MarkdownLexerTestBase.java +++ b/test/unit/src/flow/netbeans/markdown/highlighter/MarkdownLexerTestBase.java @@ -58,15 +58,15 @@ protected File getTestDataDir() { } protected File getGoldenFilesDir() { - return new File(getTestDataDir(), "/goldenfiles"); + return new File(getTestDataDir(), "goldenfiles"); } protected File getTestFilesDir() { - return new File(getTestDataDir(), "/testfiles"); + return new File(getTestDataDir(), "testfiles"); } protected File getGoldenFile(String name) { - return new File(getGoldenFilesDir(), "/" + name); + return new File(getGoldenFilesDir(), name); } protected String getGoldenFileContent(String name) throws IOException { @@ -78,7 +78,7 @@ protected String getTestFileContent(String name) throws IOException { } protected File getTestFile(String name) { - return new File(getTestFilesDir(), "/" + name); + return new File(getTestFilesDir(), name); } /** From be2cbc2f213a48a474ae1bf0785e3bc4141332e3 Mon Sep 17 00:00:00 2001 From: Holger Stenger Date: Wed, 13 Nov 2013 20:44:10 +0100 Subject: [PATCH 5/8] Removed unused token type TEXT. --- src/flow/netbeans/markdown/highlighter/MarkdownTokenId.java | 1 - src/flow/netbeans/markdown/resources/Bundle.properties | 1 - src/flow/netbeans/markdown/resources/FontAndColors.xml | 1 - 3 files changed, 3 deletions(-) diff --git a/src/flow/netbeans/markdown/highlighter/MarkdownTokenId.java b/src/flow/netbeans/markdown/highlighter/MarkdownTokenId.java index e069ff3..6584f00 100644 --- a/src/flow/netbeans/markdown/highlighter/MarkdownTokenId.java +++ b/src/flow/netbeans/markdown/highlighter/MarkdownTokenId.java @@ -34,7 +34,6 @@ public enum MarkdownTokenId implements TokenId { PLAIN("plain"), STRONG("strong"), TABLE("table"), - TEXT("text"), VERBATIM("verbatim"), WIKILINK("wikilink"), DEFINITION("definition"), diff --git a/src/flow/netbeans/markdown/resources/Bundle.properties b/src/flow/netbeans/markdown/resources/Bundle.properties index 0e1c6d6..c3cc819 100644 --- a/src/flow/netbeans/markdown/resources/Bundle.properties +++ b/src/flow/netbeans/markdown/resources/Bundle.properties @@ -43,7 +43,6 @@ ref_link=Reference Link reference=Reference strong=Strong table=Table -text=Text verbatim=Verbatim whitespace=Whitespace wikilink=Wiki Link diff --git a/src/flow/netbeans/markdown/resources/FontAndColors.xml b/src/flow/netbeans/markdown/resources/FontAndColors.xml index 3a4e8e3..3298001 100644 --- a/src/flow/netbeans/markdown/resources/FontAndColors.xml +++ b/src/flow/netbeans/markdown/resources/FontAndColors.xml @@ -32,7 +32,6 @@ - From e23d3ff6dedf2460f4d3eab2c772fca58b3cc338 Mon Sep 17 00:00:00 2001 From: Holger Stenger Date: Thu, 14 Nov 2013 00:00:12 +0100 Subject: [PATCH 6/8] Added unit tests for the MarkdownLexerVisitor when used with the PegDown extensions over standard Markdown. --- .../MarkdownLexerVisitorExtensionsTest.java | 126 ++++++++++++++++++ .../MarkdownLexerVisitorTestSupport.java | 82 ++++++++++++ .../testresources/extensions/abbreviations.md | 4 + .../testresources/extensions/autolinks.md | 2 + .../testresources/extensions/definitions.md | 6 + .../extensions/fenced_code_blocks.md | 7 + .../testresources/extensions/hardwraps.md | 5 + .../testresources/extensions/quotes.md | 1 + .../testresources/extensions/smarts.md | 4 + .../extensions/suppress_html_blocks.md | 7 + .../extensions/suppress_inline_html.md | 1 + .../testresources/extensions/tables.md | 6 + .../testresources/extensions/wikilinks.md | 2 + 13 files changed, 253 insertions(+) create mode 100644 test/unit/src/flow/netbeans/markdown/highlighter/MarkdownLexerVisitorExtensionsTest.java create mode 100644 test/unit/src/flow/netbeans/markdown/highlighter/MarkdownLexerVisitorTestSupport.java create mode 100644 test/unit/src/flow/netbeans/markdown/testresources/extensions/abbreviations.md create mode 100644 test/unit/src/flow/netbeans/markdown/testresources/extensions/autolinks.md create mode 100644 test/unit/src/flow/netbeans/markdown/testresources/extensions/definitions.md create mode 100644 test/unit/src/flow/netbeans/markdown/testresources/extensions/fenced_code_blocks.md create mode 100644 test/unit/src/flow/netbeans/markdown/testresources/extensions/hardwraps.md create mode 100644 test/unit/src/flow/netbeans/markdown/testresources/extensions/quotes.md create mode 100644 test/unit/src/flow/netbeans/markdown/testresources/extensions/smarts.md create mode 100644 test/unit/src/flow/netbeans/markdown/testresources/extensions/suppress_html_blocks.md create mode 100644 test/unit/src/flow/netbeans/markdown/testresources/extensions/suppress_inline_html.md create mode 100644 test/unit/src/flow/netbeans/markdown/testresources/extensions/tables.md create mode 100644 test/unit/src/flow/netbeans/markdown/testresources/extensions/wikilinks.md diff --git a/test/unit/src/flow/netbeans/markdown/highlighter/MarkdownLexerVisitorExtensionsTest.java b/test/unit/src/flow/netbeans/markdown/highlighter/MarkdownLexerVisitorExtensionsTest.java new file mode 100644 index 0000000..88a2faa --- /dev/null +++ b/test/unit/src/flow/netbeans/markdown/highlighter/MarkdownLexerVisitorExtensionsTest.java @@ -0,0 +1,126 @@ + +package flow.netbeans.markdown.highlighter; + +import java.io.IOException; +import org.junit.Test; +import org.pegdown.Extensions; + +/** + * Unit tests for {@link MarkdownLexerVisitor} when used with the PegDown extensions to standard Markdown. + * At the moment only basic properties of the generated token sequence are checked: + *
    + *
  • All tokens have non-zero length.
    • + *
  • The total length of all tokens is equal to the length of the input.
    • + *
+ */ +public class MarkdownLexerVisitorExtensionsTest extends MarkdownLexerVisitorTestSupport { + @Test + public void testAbbreviationsEnabled() throws IOException { + runTestWithExtensions("extensions/abbreviations", Extensions.ABBREVIATIONS); + } + + @Test + public void testAbbreviationsDisabled() throws IOException { + runTestWithExtensions("extensions/abbreviations", 0); + } + + @Test + public void testAutolinksEnabled() throws IOException { + runTestWithExtensions("extensions/autolinks", Extensions.AUTOLINKS); + } + + @Test + public void testAutolinksDisabled() throws IOException { + runTestWithExtensions("extensions/autolinks", 0); + } + + @Test + public void testDefinitionsEnabled() throws IOException { + runTestWithExtensions("extensions/definitions", Extensions.DEFINITIONS); + } + + @Test + public void testDefinitionsDisabled() throws IOException { + runTestWithExtensions("extensions/definitions", 0); + } + + @Test + public void testFencedCodeBlocksEnabled() throws IOException { + runTestWithExtensions("extensions/fenced_code_blocks", Extensions.FENCED_CODE_BLOCKS); + } + + @Test + public void testFencedCodeBlocksDisabled() throws IOException { + runTestWithExtensions("extensions/fenced_code_blocks", 0); + } + + @Test + public void testHardwrapsEnabled() throws IOException { + runTestWithExtensions("extensions/hardwraps", Extensions.HARDWRAPS); + } + + @Test + public void testHardwrapsDisabled() throws IOException { + runTestWithExtensions("extensions/hardwraps", 0); + } + + @Test + public void testQuotesEnabled() throws IOException { + runTestWithExtensions("extensions/quotes", Extensions.QUOTES); + } + + @Test + public void testQuotesDisabled() throws IOException { + runTestWithExtensions("extensions/quotes", 0); + } + + @Test + public void testSmartsEnabled() throws IOException { + runTestWithExtensions("extensions/smarts", Extensions.SMARTS); + } + + @Test + public void testSmartsDisabled() throws IOException { + runTestWithExtensions("extensions/smarts", 0); + } + + @Test + public void testTablesEnabled() throws IOException { + runTestWithExtensions("extensions/tables", Extensions.TABLES); + } + + @Test + public void testTablesDisabled() throws IOException { + runTestWithExtensions("extensions/tables", 0); + } + + @Test + public void testWikilinksEnabled() throws IOException { + runTestWithExtensions("extensions/wikilinks", Extensions.WIKILINKS); + } + + @Test + public void testWikilinksDisabled() throws IOException { + runTestWithExtensions("extensions/wikilinks", 0); + } + + @Test + public void testSuppressHtmlBlocksEnabled() throws IOException { + runTestWithExtensions("extensions/suppress_html_blocks", Extensions.SUPPRESS_HTML_BLOCKS); + } + + @Test + public void testSuppressHtmlBlocksDisabled() throws IOException { + runTestWithExtensions("extensions/suppress_html_blocks", 0); + } + + @Test + public void testSuppressInlineHtmlEnabled() throws IOException { + runTestWithExtensions("extensions/suppress_inline_html", Extensions.SUPPRESS_INLINE_HTML); + } + + @Test + public void testSuppressInlineHtmlDisabled() throws IOException { + runTestWithExtensions("extensions/suppress_inline_html", 0); + } +} diff --git a/test/unit/src/flow/netbeans/markdown/highlighter/MarkdownLexerVisitorTestSupport.java b/test/unit/src/flow/netbeans/markdown/highlighter/MarkdownLexerVisitorTestSupport.java new file mode 100644 index 0000000..171e389 --- /dev/null +++ b/test/unit/src/flow/netbeans/markdown/highlighter/MarkdownLexerVisitorTestSupport.java @@ -0,0 +1,82 @@ + +package flow.netbeans.markdown.highlighter; + +import static org.junit.Assert.*; +import static org.hamcrest.CoreMatchers.*; +import static flow.netbeans.markdown.matchers.Matchers.*; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.List; +import org.pegdown.PegDownProcessor; +import org.pegdown.ast.RootNode; + +/** + * + * @author Holger + */ +public class MarkdownLexerVisitorTestSupport { + public static final String TEST_RESOURCES = "/flow/netbeans/markdown/testresources/"; + + public MarkdownLexerVisitorTestSupport() { + } + + public void runTestWithExtensions(String baseName, int extensions) throws IOException { + String source = getTestResourceAsString(baseName + ".md"); + PegDownProcessor processor = new PegDownProcessor(extensions); + RootNode rootNode = processor.parseMarkdown(source.toCharArray()); + + assertThat("rootNode", rootNode, notNullValue()); + + final int totalLength = source.length(); + MarkdownTokenListBuilder builder = new MarkdownTokenListBuilder(totalLength); + MarkdownLexerVisitor visitor = new MarkdownLexerVisitor(builder); + rootNode.accept(visitor); + List tokens = builder.build(); + + assertThat(tokens, everyItem(nonZeroLength())); + assertThat(totalLengthOf(tokens), equalTo(totalLength)); + } + + public String getTestResourceAsString(String name) throws IOException { + final String fullName = TEST_RESOURCES + name; + InputStream stream = getClass().getResourceAsStream(fullName); + if (stream == null) { + throw new FileNotFoundException("Test resource not found: " + fullName); + } + try { + InputStreamReader reader = new InputStreamReader(stream, "UTF-8"); + StringBuilder sb = new StringBuilder(); + try { + char[] buffer = new char[4096]; + boolean done = false; + while (!done) { + int count = reader.read(buffer); + if (count < 0) { + done = true; + } else { + sb.append(buffer, 0, count); + } + } + return buffer.toString(); + } + finally { + try { + reader.close(); + } + catch (IOException ex) { + // Ignore. + } + } + } + finally { + try { + stream.close(); + } catch (IOException ex) { + // Ignore. + } + } + } +} diff --git a/test/unit/src/flow/netbeans/markdown/testresources/extensions/abbreviations.md b/test/unit/src/flow/netbeans/markdown/testresources/extensions/abbreviations.md new file mode 100644 index 0000000..2cf622f --- /dev/null +++ b/test/unit/src/flow/netbeans/markdown/testresources/extensions/abbreviations.md @@ -0,0 +1,4 @@ +The HTML specification is maintained by the W3C. + +*[HTML]: Hyper Text Markup Language +*[W3C]: World Wide Web Consortium diff --git a/test/unit/src/flow/netbeans/markdown/testresources/extensions/autolinks.md b/test/unit/src/flow/netbeans/markdown/testresources/extensions/autolinks.md new file mode 100644 index 0000000..8ff1049 --- /dev/null +++ b/test/unit/src/flow/netbeans/markdown/testresources/extensions/autolinks.md @@ -0,0 +1,2 @@ +Undelimited http://example.com link +Undelimited user@example.com link \ No newline at end of file diff --git a/test/unit/src/flow/netbeans/markdown/testresources/extensions/definitions.md b/test/unit/src/flow/netbeans/markdown/testresources/extensions/definitions.md new file mode 100644 index 0000000..19239f4 --- /dev/null +++ b/test/unit/src/flow/netbeans/markdown/testresources/extensions/definitions.md @@ -0,0 +1,6 @@ +Apple +: Pomaceous fruit of plants of the genus Malus in + the family Rosaceae. + +Orange +: The fruit of an evergreen tree of the genus Citrus. diff --git a/test/unit/src/flow/netbeans/markdown/testresources/extensions/fenced_code_blocks.md b/test/unit/src/flow/netbeans/markdown/testresources/extensions/fenced_code_blocks.md new file mode 100644 index 0000000..39b49bb --- /dev/null +++ b/test/unit/src/flow/netbeans/markdown/testresources/extensions/fenced_code_blocks.md @@ -0,0 +1,7 @@ +``` +Fenced code block (GitHub Flavored Markdown) +``` + +~~~~ +Fenced code block (PHP Markdown Extras) +~~~~ diff --git a/test/unit/src/flow/netbeans/markdown/testresources/extensions/hardwraps.md b/test/unit/src/flow/netbeans/markdown/testresources/extensions/hardwraps.md new file mode 100644 index 0000000..e23cf84 --- /dev/null +++ b/test/unit/src/flow/netbeans/markdown/testresources/extensions/hardwraps.md @@ -0,0 +1,5 @@ +Roses are red +Violets are blue + +Roses are red +Violets are blue diff --git a/test/unit/src/flow/netbeans/markdown/testresources/extensions/quotes.md b/test/unit/src/flow/netbeans/markdown/testresources/extensions/quotes.md new file mode 100644 index 0000000..acca8ea --- /dev/null +++ b/test/unit/src/flow/netbeans/markdown/testresources/extensions/quotes.md @@ -0,0 +1 @@ +We have 'single' quotes, "double" quotes and <> quotes. \ No newline at end of file diff --git a/test/unit/src/flow/netbeans/markdown/testresources/extensions/smarts.md b/test/unit/src/flow/netbeans/markdown/testresources/extensions/smarts.md new file mode 100644 index 0000000..c5d92a6 --- /dev/null +++ b/test/unit/src/flow/netbeans/markdown/testresources/extensions/smarts.md @@ -0,0 +1,4 @@ +Ellipses ... and . . . +En dash -- +Em dash --- +Apostrophe ' \ No newline at end of file diff --git a/test/unit/src/flow/netbeans/markdown/testresources/extensions/suppress_html_blocks.md b/test/unit/src/flow/netbeans/markdown/testresources/extensions/suppress_html_blocks.md new file mode 100644 index 0000000..a8af75e --- /dev/null +++ b/test/unit/src/flow/netbeans/markdown/testresources/extensions/suppress_html_blocks.md @@ -0,0 +1,7 @@ +Paragraph before HTML block. + +

+This is an HTML block. +

+ +Paragraph after HTML block. diff --git a/test/unit/src/flow/netbeans/markdown/testresources/extensions/suppress_inline_html.md b/test/unit/src/flow/netbeans/markdown/testresources/extensions/suppress_inline_html.md new file mode 100644 index 0000000..d6b2afd --- /dev/null +++ b/test/unit/src/flow/netbeans/markdown/testresources/extensions/suppress_inline_html.md @@ -0,0 +1 @@ +This is inline HTML. \ No newline at end of file diff --git a/test/unit/src/flow/netbeans/markdown/testresources/extensions/tables.md b/test/unit/src/flow/netbeans/markdown/testresources/extensions/tables.md new file mode 100644 index 0000000..4720a13 --- /dev/null +++ b/test/unit/src/flow/netbeans/markdown/testresources/extensions/tables.md @@ -0,0 +1,6 @@ +| | Grouping || +| First Header | Second Header | Third Header | +| ------------ | :-----------: | -----------: | +| Content | *Long Cell* || +| Content | **Cell** | Cell | +| New section | More | Data | diff --git a/test/unit/src/flow/netbeans/markdown/testresources/extensions/wikilinks.md b/test/unit/src/flow/netbeans/markdown/testresources/extensions/wikilinks.md new file mode 100644 index 0000000..68f8b2f --- /dev/null +++ b/test/unit/src/flow/netbeans/markdown/testresources/extensions/wikilinks.md @@ -0,0 +1,2 @@ +Wikilink without title [[http://en.wikipedia.org/wiki/]] +Wikilink with title [[http://en.wikipedia.org/wiki/ "Wikipedia Main Page"]] \ No newline at end of file From d70eaa23fa21064a3a31800d4e6cfcf4d70fc34a Mon Sep 17 00:00:00 2001 From: Holger Stenger Date: Sun, 17 Nov 2013 21:15:45 +0100 Subject: [PATCH 7/8] Changed default token type to PLAIN. --- .../netbeans/markdown/highlighter/MarkdownTokenListBuilder.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/flow/netbeans/markdown/highlighter/MarkdownTokenListBuilder.java b/src/flow/netbeans/markdown/highlighter/MarkdownTokenListBuilder.java index 73623af..b5223b6 100644 --- a/src/flow/netbeans/markdown/highlighter/MarkdownTokenListBuilder.java +++ b/src/flow/netbeans/markdown/highlighter/MarkdownTokenListBuilder.java @@ -26,7 +26,7 @@ public MarkdownTokenListBuilder(int totalLength) { stack = new ArrayDeque(); tokens = new ArrayList(); // Push guard tree token. - stack.push(new MarkdownLexerVisitor.MarkdownTreeToken(MarkdownTokenId.WHITESPACE, stack.size(), 0, + stack.push(new MarkdownLexerVisitor.MarkdownTreeToken(MarkdownTokenId.PLAIN, stack.size(), 0, totalLength)); } From 552674ed36c75649314d05dab6b914af0d83db94 Mon Sep 17 00:00:00 2001 From: Holger Stenger Date: Sun, 17 Nov 2013 21:19:34 +0100 Subject: [PATCH 8/8] Added some Javadoc. --- .../markdown/highlighter/MarkdownTokenListBuilder.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/flow/netbeans/markdown/highlighter/MarkdownTokenListBuilder.java b/src/flow/netbeans/markdown/highlighter/MarkdownTokenListBuilder.java index b5223b6..0fdd1f0 100644 --- a/src/flow/netbeans/markdown/highlighter/MarkdownTokenListBuilder.java +++ b/src/flow/netbeans/markdown/highlighter/MarkdownTokenListBuilder.java @@ -7,8 +7,9 @@ import java.util.List; /** - * - * @author Holger + * The {@link MarkdownTokenListBuilder} is used to build a list of tokens during a depth-first left-to-right traversal + * of a PegDown AST. This functionality was extracted from the {@link MarkdownLexerVisitor} to simplify unit testing. + * @author Holger Stenger */ public class MarkdownTokenListBuilder { private final Deque stack;