fix link text vs link ref priority to match commonmark ref impl

VERSION-TODO.md

@@ -7,6 +7,7 @@
 - [Release 0.60.0](#release-0600)
   - [API Refactoring](#api-refactoring)
 - [Next 0.61.xx](#next-061xx)
+- [0.61.24](#06124)
 - [0.61.22](#06122)
 - [0.61.20](#06120)
 - [0.61.18](#06118)
@@ -207,7 +208,7 @@ Please give feedback on the upcoming changes if you have concerns about breaking
       extension add order
 * [ ] Fix: Change spec example to variable number of sections
 * [ ] Add: yaml front matter configurator for modules. See: [Yaml Front Matter
-                              Configuration](../../wiki/Yaml-Front-Matter-Configuration)
+                                  Configuration](../../wiki/Yaml-Front-Matter-Configuration)
 * [ ] Fix: table formatting caret position tracking affects alignment even when not inserting,
       just formatting. Need to keep track of whether format after typing or just format. Then
       caret position should only track but not force spaces behind it to be preserved. See
@@ -223,6 +224,26 @@ Please give feedback on the upcoming changes if you have concerns about breaking
 * [ ] Fix: Html converter to not add spaces between end of inline marker and next punctuation:
       `.,:;`
 
+## 0.61.24
+
+* Fix: link refs in link text should be collapsed.
+  * Add: `Parser.LINK_TEXT_PRIORITY_OVER_LINK_REF`, default `false`, commonmark spec does not
+    specify how to parse:
+
+        [[moon]](/uri)
+
+        [moon]: moon.jpg
+
+    with default implementation priority given to contained ref, renders as:
+
+        <p>[<a href="moon.jpg">moon</a>](/uri)</p>
+
+    when set to `true`, priority given to outer link and will render as:
+
+        <p><a href="/uri">[][moon]</a></p>
+
+    :information_source: Undefined reference links are always treated as text when in a link text element.
+
 ## 0.61.22
 
 * Fix: [#407, Link text inline content fails to parse image references], link and image refs in
@@ -2446,6 +2467,7 @@ Please give feedback on the upcoming changes if you have concerns about breaking
 [#396, DocumentParser stops reading too early resulting in the document being cut off]: https://github.com/vsch/flexmark-java/issues/396
 [#397, PR: Add base64 image support with docx rendering]: https://github.com/vsch/flexmark-java/pull/397
 [#398, Autolinks get cut off if they contain \`&amp;\` (escaped query params)]: https://github.com/vsch/flexmark-java/issues/398
+[#407, Link text inline content fails to parse image references]: https://github.com/vsch/flexmark-java/issues/407
 [@Xaelis]: https://github.com/Xaelis
 [Awesome Console]: https://plugins.jetbrains.com/plugin/7677-awesome-console "Awesome Console"
 [HtmlToMarkdownCustomizedSample.java]: https://github.com/vsch/flexmark-java/blob/master/flexmark-java-samples/src/com/vladsch/flexmark/java/samples/HtmlToMarkdownCustomizedSample.java
@@ -2458,6 +2480,4 @@ Please give feedback on the upcoming changes if you have concerns about breaking
 [migrate flexmark-java 0_40_x to 0_42_0]: https://github.com/vsch/flexmark-java/blob/master/assets/migrations/migrate%20flexmark-java%200_40_x%20to%200_42_0.xml
 <!-- @IGNORE PREVIOUS: link -->
 [migrate flexmark-java 0_42_x to 0_50_0.xml]: https://github.com/vsch/flexmark-java/blob/master/assets/migrations/migrate%20flexmark-java%200_42_x%20to%200_50_0.xml
-[#407, Link text inline content fails to parse image references]: https://github.com/vsch/flexmark-java/issues/407
-
 

VERSION.md

@@ -6,6 +6,7 @@
 
 - [Release 0.60.0](#release-0600)
   - [API Refactoring](#api-refactoring)
+- [0.61.24](#06124)
 - [0.61.22](#06122)
 - [0.61.20](#06120)
 - [0.61.18](#06118)
@@ -167,6 +168,26 @@ Please give feedback on the upcoming changes if you have concerns about breaking
     * `com.vladsch.flexmark.util.ast.NodeAdaptingVisitHandler`
     * `com.vladsch.flexmark.util.ast.NodeAdaptingVisitor`
 
+## 0.61.24
+
+* Fix: link refs in link text should be collapsed.
+  * Add: `Parser.LINK_TEXT_PRIORITY_OVER_LINK_REF`, default `false`, commonmark spec does not
+    specify how to parse:
+
+        [[moon]](/uri)
+
+        [moon]: moon.jpg
+
+    with default implementation priority given to contained ref, renders as:
+
+        <p>[<a href="moon.jpg">moon</a>](/uri)</p>
+
+    when set to `true`, priority given to outer link and will render as:
+
+        <p><a href="/uri">[][moon]</a></p>
+
+    :information_source: Undefined reference links are always treated as text when in a link text element.
+
 ## 0.61.22
 
 * Fix: [#407, Link text inline content fails to parse image references], link and image refs in

flexmark-core-test/flexmark-core-test.iml

@@ -23,7 +23,7 @@
       <sourceFolder url="file://$MODULE_DIR$/src/main/java" isTestSource="false" />
       <sourceFolder url="file://$MODULE_DIR$/src/test/java" isTestSource="true" />
       <sourceFolder url="file://$MODULE_DIR$/src/test/resources" type="java-test-resource" />
-      <sourceFolder url="file://$MODULE_DIR$" isTestSource="false" />
+      <sourceFolder url="file://$MODULE_DIR$/target/generated-test-sources/test-annotations" isTestSource="true" generated="true" />
       <excludeFolder url="file://$MODULE_DIR$/target" />
     </content>
     <orderEntry type="inheritedJdk" />
@@ -50,4 +50,4 @@
     <orderEntry type="library" name="Maven: org.apache.commons:commons-math3:3.2" level="project" />
     <orderEntry type="library" name="Maven: junit:junit:4.12" level="project" />
   </component>
-</module>
+</module>

flexmark-core-test/src/test/java/com/vladsch/flexmark/core/test/util/renderer/ComboIssuesSpecTest.java

@@ -34,6 +34,7 @@ final public class ComboIssuesSpecTest extends CoreRendererSpecTest {
         optionsMap.put("allow-javascript", new MutableDataSet().set(HtmlRenderer.SUPPRESSED_LINKS, ""));
         optionsMap.put("pass-through", new MutableDataSet().set(HtmlRenderer.FORMAT_FLAGS, LineAppendable.F_PASS_THROUGH));
         optionsMap.put("strip-indent", new MutableDataSet().set(TestUtils.SOURCE_INDENT, "> > "));
+        optionsMap.put("link-over-linkref", new MutableDataSet().set(Parser.LINK_TEXT_PRIORITY_OVER_LINK_REF, true));
         optionsMap.put("no-html-blocks", new MutableDataSet().set(Parser.HTML_BLOCK_PARSER, false));
         optionsMap.put("sub-parse", new MutableDataSet()
                 .set(TestUtils.SOURCE_PREFIX, "" +

flexmark-core-test/src/test/resources/core_issues_ast_spec.md

@@ -1327,7 +1327,7 @@ Document[0, 474]
 
 Issue #487
 
-Wrong syntax
+Image refs
 
 ```````````````````````````````` example Core Issues Tests - 407: 1
 [![][moon]](/uri)
@@ -1376,18 +1376,140 @@ Document[0, 35]
 ````````````````````````````````
 
 
+Link refs should be treated as text
+
 ```````````````````````````````` example Core Issues Tests - 407: 4
-![moon]
+[[][moon]](/uri)
 
 [moon]: moon.jpg
 .
-<p><img src="moon.jpg" alt="moon" /></p>
+<p>[<a href="moon.jpg"></a>](/uri)</p>
 .
-Document[0, 25]
-  Paragraph[0, 8] isTrailingBlankLine
-    ImageRef[0, 7] referenceOpen:[0, 2, "!["] reference:[2, 6, "moon"] referenceClose:[6, 7, "]"]
+Document[0, 34]
+  Paragraph[0, 17] isTrailingBlankLine
+    Text[0, 1] chars:[0, 1, "["]
+    LinkRef[1, 9] textOpen:[1, 2, "["] text:[2, 2] textClose:[2, 3, "]"] referenceOpen:[3, 4, "["] reference:[4, 8, "moon"] referenceClose:[8, 9, "]"]
+    Text[9, 16] chars:[9, 16, "](/uri)"]
+  Reference[18, 34] refOpen:[18, 19, "["] ref:[19, 23, "moon"] refClose:[23, 25, "]:"] url:[26, 34, "moon.jpg"]
+````````````````````````````````
+
+
+```````````````````````````````` example Core Issues Tests - 407: 5
+[[moon]](/uri)
+
+[moon]: moon.jpg
+.
+<p>[<a href="moon.jpg">moon</a>](/uri)</p>
+.
+Document[0, 32]
+  Paragraph[0, 15] isTrailingBlankLine
+    Text[0, 1] chars:[0, 1, "["]
+    LinkRef[1, 7] referenceOpen:[1, 2, "["] reference:[2, 6, "moon"] referenceClose:[6, 7, "]"]
       Text[2, 6] chars:[2, 6, "moon"]
-  Reference[9, 25] refOpen:[9, 10, "["] ref:[10, 14, "moon"] refClose:[14, 16, "]:"] url:[17, 25, "moon.jpg"]
+    Text[7, 14] chars:[7, 14, "](/uri)"]
+  Reference[16, 32] refOpen:[16, 17, "["] ref:[17, 21, "moon"] refClose:[21, 23, "]:"] url:[24, 32, "moon.jpg"]
+````````````````````````````````
+
+
+```````````````````````````````` example Core Issues Tests - 407: 6
+[[moon][]](/uri)
+
+[moon]: moon.jpg
+.
+<p>[<a href="moon.jpg">moon</a>](/uri)</p>
+.
+Document[0, 34]
+  Paragraph[0, 17] isTrailingBlankLine
+    Text[0, 1] chars:[0, 1, "["]
+    LinkRef[1, 9] referenceOpen:[1, 2, "["] reference:[2, 6, "moon"] referenceClose:[6, 7, "]"] textOpen:[7, 8, "["] textClose:[8, 9, "]"]
+      Text[2, 6] chars:[2, 6, "moon"]
+    Text[9, 16] chars:[9, 16, "](/uri)"]
+  Reference[18, 34] refOpen:[18, 19, "["] ref:[19, 23, "moon"] refClose:[23, 25, "]:"] url:[26, 34, "moon.jpg"]
+````````````````````````````````
+
+
+undefined should render the same as if link text has priority
+
+```````````````````````````````` example Core Issues Tests - 407: 7
+[[][moon]](/uri)
+.
+<p><a href="/uri">[][moon]</a></p>
+.
+Document[0, 16]
+  Paragraph[0, 16]
+    Link[0, 16] textOpen:[0, 1, "["] text:[1, 9, "[][moon]"] textClose:[9, 10, "]"] linkOpen:[10, 11, "("] url:[11, 15, "/uri"] pageRef:[11, 15, "/uri"] linkClose:[15, 16, ")"]
+      Text[1, 9] chars:[1, 9, "[][moon]"]
+````````````````````````````````
+
+
+```````````````````````````````` example Core Issues Tests - 407: 8
+[[moon]](/uri)
+.
+<p><a href="/uri">[moon]</a></p>
+.
+Document[0, 14]
+  Paragraph[0, 14]
+    Link[0, 14] textOpen:[0, 1, "["] text:[1, 7, "[moon]"] textClose:[7, 8, "]"] linkOpen:[8, 9, "("] url:[9, 13, "/uri"] pageRef:[9, 13, "/uri"] linkClose:[13, 14, ")"]
+      Text[1, 7] chars:[1, 7, "[moon]"]
+````````````````````````````````
+
+
+```````````````````````````````` example Core Issues Tests - 407: 9
+[[moon][]](/uri)
+.
+<p><a href="/uri">[moon][]</a></p>
+.
+Document[0, 16]
+  Paragraph[0, 16]
+    Link[0, 16] textOpen:[0, 1, "["] text:[1, 9, "[moon][]"] textClose:[9, 10, "]"] linkOpen:[10, 11, "("] url:[11, 15, "/uri"] pageRef:[11, 15, "/uri"] linkClose:[15, 16, ")"]
+      Text[1, 9] chars:[1, 9, "[moon][]"]
+````````````````````````````````
+
+
+Link refs should be treated as text
+
+```````````````````````````````` example(Core Issues Tests - 407: 10) options(link-over-linkref)
+[[][moon]](/uri)
+
+[moon]: moon.jpg
+.
+<p><a href="/uri">[][moon]</a></p>
+.
+Document[0, 34]
+  Paragraph[0, 17] isTrailingBlankLine
+    Link[0, 16] textOpen:[0, 1, "["] text:[1, 9, "[][moon]"] textClose:[9, 10, "]"] linkOpen:[10, 11, "("] url:[11, 15, "/uri"] pageRef:[11, 15, "/uri"] linkClose:[15, 16, ")"]
+      Text[1, 9] chars:[1, 9, "[][moon]"]
+  Reference[18, 34] refOpen:[18, 19, "["] ref:[19, 23, "moon"] refClose:[23, 25, "]:"] url:[26, 34, "moon.jpg"]
+````````````````````````````````
+
+
+```````````````````````````````` example(Core Issues Tests - 407: 11) options(link-over-linkref)
+[[moon]](/uri)
+
+[moon]: moon.jpg
+.
+<p><a href="/uri">[moon]</a></p>
+.
+Document[0, 32]
+  Paragraph[0, 15] isTrailingBlankLine
+    Link[0, 14] textOpen:[0, 1, "["] text:[1, 7, "[moon]"] textClose:[7, 8, "]"] linkOpen:[8, 9, "("] url:[9, 13, "/uri"] pageRef:[9, 13, "/uri"] linkClose:[13, 14, ")"]
+      Text[1, 7] chars:[1, 7, "[moon]"]
+  Reference[16, 32] refOpen:[16, 17, "["] ref:[17, 21, "moon"] refClose:[21, 23, "]:"] url:[24, 32, "moon.jpg"]
+````````````````````````````````
+
+
+```````````````````````````````` example(Core Issues Tests - 407: 12) options(link-over-linkref)
+[[moon][]](/uri)
+
+[moon]: moon.jpg
+.
+<p><a href="/uri">[moon][]</a></p>
+.
+Document[0, 34]
+  Paragraph[0, 17] isTrailingBlankLine
+    Link[0, 16] textOpen:[0, 1, "["] text:[1, 9, "[moon][]"] textClose:[9, 10, "]"] linkOpen:[10, 11, "("] url:[11, 15, "/uri"] pageRef:[11, 15, "/uri"] linkClose:[15, 16, ")"]
+      Text[1, 9] chars:[1, 9, "[moon][]"]
+  Reference[18, 34] refOpen:[18, 19, "["] ref:[19, 23, "moon"] refClose:[23, 25, "]:"] url:[26, 34, "moon.jpg"]
 ````````````````````````````````
 
 

flexmark-ext-gitlab/flexmark-ext-gitlab.iml

@@ -19,6 +19,7 @@
       <sourceFolder url="file://$MODULE_DIR$/src/main/java" isTestSource="false" />
       <sourceFolder url="file://$MODULE_DIR$/src/test/java" isTestSource="true" />
       <sourceFolder url="file://$MODULE_DIR$/src/test/resources" type="java-test-resource" />
+      <excludeFolder url="file://$MODULE_DIR$/target" />
     </content>
     <orderEntry type="inheritedJdk" />
     <orderEntry type="sourceFolder" forTests="false" />
@@ -39,4 +40,4 @@
     <orderEntry type="module" module-name="flexmark-core-test" scope="TEST" />
     <orderEntry type="library" name="org.jetbrains:annotations" level="project" />
   </component>
-</module>
+</module>

flexmark-ext-macros/flexmark-ext-macros.iml

@@ -21,6 +21,7 @@
       <sourceFolder url="file://$MODULE_DIR$/src/main/java" isTestSource="false" />
       <sourceFolder url="file://$MODULE_DIR$/src/test/java" isTestSource="true" />
       <sourceFolder url="file://$MODULE_DIR$/src/test/resources" type="java-test-resource" />
+      <excludeFolder url="file://$MODULE_DIR$/target" />
     </content>
     <orderEntry type="inheritedJdk" />
     <orderEntry type="sourceFolder" forTests="false" />
@@ -43,4 +44,4 @@
     <orderEntry type="module" module-name="flexmark-ext-tables" scope="TEST" />
     <orderEntry type="library" name="org.jetbrains:annotations" level="project" />
   </component>
-</module>
+</module>

flexmark-integration-test/flexmark-integration-test.iml

@@ -7,10 +7,6 @@
       <sourceFolder url="file://$MODULE_DIR$/target/generated-test-sources/test-annotations" isTestSource="true" generated="true" />
       <sourceFolder url="file://$MODULE_DIR$/src/test/resources" type="java-test-resource" />
       <excludeFolder url="file://$MODULE_DIR$/target" />
-      <excludeFolder url="file://$MODULE_DIR$/target/javadoc-bundle-options" />
-      <excludeFolder url="file://$MODULE_DIR$/target/maven-archiver" />
-      <excludeFolder url="file://$MODULE_DIR$/target/maven-status" />
-      <excludeFolder url="file://$MODULE_DIR$/target/surefire-reports" />
     </content>
     <orderEntry type="inheritedJdk" />
     <orderEntry type="sourceFolder" forTests="false" />
@@ -55,4 +51,4 @@
     <orderEntry type="module" module-name="flexmark-core-test" scope="TEST" />
     <orderEntry type="module" module-name="flexmark-test-specs" scope="TEST" />
   </component>
-</module>
+</module>

flexmark-pdf-converter/src/main/java/com/vladsch/flexmark/pdf/converter/PdfConverterExtension.java

@@ -24,12 +24,12 @@
  * Extension for converting Markdown to PDF
  * <p>
  * After document is rendered pass the HTML result to
- * {@link #exportToPdf(OutputStream, String, String, DataHolder)}
- * or {@link #exportToPdf(String, String, String, DataHolder)}
- * or {@link #exportToPdf(String, String, String, PdfRendererBuilder.TextDirection)}
- * or {@link #exportToPdf(String, String, String, PdfRendererBuilder.TextDirection, ProtectionPolicy protectionPolicy)}
- * or {@link #exportToPdf(OutputStream, String, String, PdfRendererBuilder.TextDirection)}
- * or {@link #exportToPdf(OutputStream, String, String, PdfRendererBuilder.TextDirection, ProtectionPolicy protectionPolicy)}
+ * {@link PdfConverterExtension#exportToPdf(OutputStream, String, String, DataHolder)}
+ * or {@link PdfConverterExtension#exportToPdf(String, String, String, DataHolder)}
+ * or {@link PdfConverterExtension#exportToPdf(String, String, String, PdfRendererBuilder.TextDirection)}
+ * or {@link PdfConverterExtension#exportToPdf(String, String, String, PdfRendererBuilder.TextDirection, ProtectionPolicy protectionPolicy)}
+ * or {@link PdfConverterExtension#exportToPdf(OutputStream, String, String, PdfRendererBuilder.TextDirection)}
+ * or {@link PdfConverterExtension#exportToPdf(OutputStream, String, String, PdfRendererBuilder.TextDirection, ProtectionPolicy protectionPolicy)}
  * <p>
  * The parsed Markdown text is rendered to HTML then converted to PDF
  * </p>

flexmark-tree-iteration/flexmark-tree-iteration.iml

@@ -23,6 +23,7 @@
       <sourceFolder url="file://$MODULE_DIR$/src/main/java" isTestSource="false" />
       <sourceFolder url="file://$MODULE_DIR$/src/test/java" isTestSource="true" />
       <sourceFolder url="file://$MODULE_DIR$/src/test/resources" type="java-test-resource" />
+      <excludeFolder url="file://$MODULE_DIR$/target" />
     </content>
     <orderEntry type="inheritedJdk" />
     <orderEntry type="sourceFolder" forTests="false" />
@@ -35,4 +36,4 @@
     <orderEntry type="library" name="org.jetbrains:annotations" level="project" />
     <orderEntry type="library" name="slf4j-api-1.7.26" level="project" />
   </component>
-</module>
+</module>

flexmark/flexmark.iml

@@ -9,11 +9,6 @@
       <sourceFolder url="file://$MODULE_DIR$/target/generated-test-sources/test-annotations" isTestSource="true" generated="true" />
       <sourceFolder url="file://$MODULE_DIR$/src/test/resources" type="java-test-resource" />
       <excludeFolder url="file://$MODULE_DIR$/target" />
-      <excludeFolder url="file://$MODULE_DIR$/target/apidocs" />
-      <excludeFolder url="file://$MODULE_DIR$/target/javadoc-bundle-options" />
-      <excludeFolder url="file://$MODULE_DIR$/target/maven-archiver" />
-      <excludeFolder url="file://$MODULE_DIR$/target/maven-status" />
-      <excludeFolder url="file://$MODULE_DIR$/target/surefire-reports" />
     </content>
     <orderEntry type="inheritedJdk" />
     <orderEntry type="sourceFolder" forTests="false" />
@@ -37,4 +32,4 @@
     <orderEntry type="library" name="log4j:log4j:1.2.17" level="project" />
     <orderEntry type="library" name="org.jetbrains:annotations" level="project" />
   </component>
-</module>
+</module>

flexmark/src/main/java/com/vladsch/flexmark/parser/InlineParserOptions.java

@@ -15,6 +15,7 @@ public class InlineParserOptions {
     final public boolean intellijDummyIdentifier;
     final public boolean parseJekyllMacrosInUrls;
     final public boolean useHardcodedLinkAddressParser;
+    final public boolean linkTextPriorityOverLinkRef;
 
     public InlineParserOptions(DataHolder options) {
         matchLookaheadFirst = Parser.MATCH_NESTED_LINK_REFS_FIRST.get(options);
@@ -29,5 +30,6 @@ public InlineParserOptions(DataHolder options) {
         codeSoftLineBreaks = Parser.CODE_SOFT_LINE_BREAKS.get(options);
         inlineDelimiterDirectionalPunctuations = Parser.INLINE_DELIMITER_DIRECTIONAL_PUNCTUATIONS.get(options);
         linksAllowMatchedParentheses = Parser.LINKS_ALLOW_MATCHED_PARENTHESES.get(options);
+        linkTextPriorityOverLinkRef = Parser.LINK_TEXT_PRIORITY_OVER_LINK_REF.get(options);
     }
 }

flexmark/src/main/java/com/vladsch/flexmark/parser/Parser.java

@@ -85,6 +85,7 @@ public class Parser implements IParse {
     final public static DataKey<Boolean> SPACE_IN_LINK_URLS = new DataKey<>("SPACE_IN_LINK_URLS", false);
     final public static DataKey<Boolean> SPACE_IN_LINK_ELEMENTS = new DataKey<>("SPACE_IN_LINK_ELEMENTS", false);
     final public static DataKey<Boolean> WWW_AUTO_LINK_ELEMENT = new DataKey<>("WWW_AUTO_LINK_ELEMENT", false);
+    final public static DataKey<Boolean> LINK_TEXT_PRIORITY_OVER_LINK_REF = new DataKey<>("LINK_TEXT_PRIORITY_OVER_LINK_REF", false);   // if true then link text containing link ref is treated as link text with link ref as text, else link is ignored and link ref is used
 
     final public static DataKey<Boolean> REFERENCE_PARAGRAPH_PRE_PROCESSOR = new DataKey<>("REFERENCE_BLOCK_PRE_PROCESSOR", true);
     final public static DataKey<Boolean> THEMATIC_BREAK_PARSER = new DataKey<>("THEMATIC_BREAK_PARSER", true);