Merge pull request #366 from digitalmoksha/bw-add-math

Add support for dollar math and code math
kivikakk · Mar 29, 2024 · 05a2532 · 05a2532
2 parents 01c56d9 + 971bf34
commit 05a2532
Show file tree

Hide file tree

Showing 22 changed files with 826 additions and 69 deletions.
diff --git a/README.md b/README.md
@@ -100,7 +100,7 @@ Options:
           Multiple extensions can be delimited with ",", e.g. --extension strikethrough,table
           
           [possible values: strikethrough, tagfilter, table, autolink, tasklist, superscript,
-          footnotes, description-lists, multiline-block-quotes]
+          footnotes, description-lists, multiline-block-quotes, math-dollars, math-code]
 
   -t, --to <FORMAT>
           Specify output format
@@ -256,6 +256,8 @@ Comrak additionally supports its own extensions, which are yet to be specced out
 - Description lists
 - Front matter
 - Shortcodes
+- Math
+- Multiline Blockquotes
 
 By default none are enabled; they are individually enabled with each parse by setting the appropriate values in the
 [`ComrakExtensionOptions` struct](https://docs.rs/comrak/newest/comrak/type.ComrakExtensionOptions.html).

diff --git a/examples/s-expr.rs b/examples/s-expr.rs
@@ -84,6 +84,8 @@ fn dump(source: &str) -> io::Result<()> {
         .footnotes(true)
         .description_lists(true)
         .multiline_block_quotes(true)
+        .math_dollars(true)
+        .math_code(true)
         .build()
         .unwrap();
 

diff --git a/fuzz/fuzz_targets/all_options.rs b/fuzz/fuzz_targets/all_options.rs
@@ -19,6 +19,8 @@ fuzz_target!(|s: &str| {
     extension.footnotes = true;
     extension.description_lists = true;
     extension.multiline_block_quotes = true;
+    extension.math_dollars = true;
+    extension.math_code = true;
     extension.front_matter_delimiter = Some("---".to_string());
     extension.shortcodes = true;
 

diff --git a/fuzz/fuzz_targets/quadratic.rs b/fuzz/fuzz_targets/quadratic.rs
@@ -193,6 +193,8 @@ struct FuzzExtensionOptions {
     footnotes: bool,
     description_lists: bool,
     multiline_block_quotes: bool,
+    math_dollars: bool,
+    math_code: bool,
     shortcodes: bool,
 }
 
@@ -208,6 +210,8 @@ impl FuzzExtensionOptions {
         extension.footnotes = self.footnotes;
         extension.description_lists = self.description_lists;
         extension.multiline_block_quotes = self.multiline_block_quotes;
+        extension.math_dollars = self.math_dollars;
+        extension.math_code = self.math_code;
         extension.shortcodes = self.shortcodes;
         extension.front_matter_delimiter = None;
         extension.header_ids = None;

diff --git a/script/cibuild b/script/cibuild
@@ -34,8 +34,12 @@ if [ x"$SPEC" = "xtrue" ]; then
 	# python3 roundtrip_tests.py --spec extensions-table-prefer-style-attributes.txt "$PROGRAM_ARG --table-prefer-style-attributes" --extensions "table strikethrough autolink tagfilter footnotes tasklist" || failed=1
 	python3 roundtrip_tests.py --spec extensions-full-info-string.txt "$PROGRAM_ARG --full-info-string" \
 		|| failed=1
-	python3 spec_tests.py --no-normalize --spec ../../../src/tests/fixtures/multiline_blockquote.txt "$PROGRAM_ARG -e multiline-block-quotes" \
+	python3 spec_tests.py --no-normalize --spec ../../../src/tests/fixtures/multiline_blockquote.md "$PROGRAM_ARG -e multiline-block-quotes" \
 		|| failed=1
+  python3 spec_tests.py --no-normalize --spec ../../../src/tests/fixtures/math_dollars.md "$PROGRAM_ARG -e math-dollars" \
+    || failed=1
+  python3 spec_tests.py --no-normalize --spec ../../../src/tests/fixtures/math_code.md "$PROGRAM_ARG -e math-code" \
+    || failed=1
 
 	python3 spec_tests.py --no-normalize --spec regression.txt "$PROGRAM_ARG" \
 		|| failed=1

diff --git a/src/cm.rs b/src/cm.rs
@@ -2,7 +2,7 @@ use crate::ctype::{isalpha, isdigit, ispunct, isspace};
 use crate::nodes::TableAlignment;
 use crate::nodes::{
     AstNode, ListDelimType, ListType, NodeCodeBlock, NodeHeading, NodeHtmlBlock, NodeLink,
-    NodeTable, NodeValue,
+    NodeMath, NodeTable, NodeValue,
 };
 #[cfg(feature = "shortcodes")]
 use crate::parser::shortcodes::NodeShortCode;
@@ -381,6 +381,7 @@ impl<'a, 'o> CommonMarkFormatter<'a, 'o> {
             NodeValue::Escaped => {
                 // noop - automatic escaping is already being done
             }
+            NodeValue::Math(ref math) => self.format_math(math, allow_wrap, entering),
         };
         true
     }
@@ -777,6 +778,31 @@ impl<'a, 'o> CommonMarkFormatter<'a, 'o> {
             self.write_all(b"]").unwrap();
         }
     }
+
+    fn format_math(&mut self, math: &NodeMath, allow_wrap: bool, entering: bool) {
+        if entering {
+            let literal = math.literal.as_bytes();
+            let start_fence = if math.dollar_math {
+                if math.display_math {
+                    "$$"
+                } else {
+                    "$"
+                }
+            } else {
+                "$`"
+            };
+
+            let end_fence = if start_fence == "$`" {
+                "`$"
+            } else {
+                start_fence
+            };
+
+            self.output(start_fence.as_bytes(), false, Escaping::Literal);
+            self.output(literal, allow_wrap, Escaping::Literal);
+            self.output(end_fence.as_bytes(), false, Escaping::Literal);
+        }
+    }
 }
 
 fn longest_char_sequence(literal: &[u8], ch: u8) -> usize {

diff --git a/src/html.rs b/src/html.rs
@@ -1,7 +1,8 @@
 //! The HTML renderer for the CommonMark AST, as well as helper functions.
 use crate::ctype::isspace;
 use crate::nodes::{
-    AstNode, ListType, NodeCode, NodeFootnoteDefinition, NodeTable, NodeValue, TableAlignment,
+    AstNode, ListType, NodeCode, NodeFootnoteDefinition, NodeMath, NodeTable, NodeValue,
+    TableAlignment,
 };
 use crate::parser::{Options, Plugins};
 use crate::scanners;
@@ -417,6 +418,9 @@ impl<'o> HtmlFormatter<'o> {
                             NodeValue::LineBreak | NodeValue::SoftBreak => {
                                 self.output.write_all(b" ")?;
                             }
+                            NodeValue::Math(NodeMath { ref literal, .. }) => {
+                                self.escape(literal.as_bytes())?;
+                            }
                             _ => (),
                         }
                         plain
@@ -445,6 +449,9 @@ impl<'o> HtmlFormatter<'o> {
                 output.extend_from_slice(literal.as_bytes())
             }
             NodeValue::LineBreak | NodeValue::SoftBreak => output.push(b' '),
+            NodeValue::Math(NodeMath { ref literal, .. }) => {
+                output.extend_from_slice(literal.as_bytes())
+            }
             _ => {
                 for n in node.children() {
                     Self::collect_text(n, output);
@@ -582,71 +589,77 @@ impl<'o> HtmlFormatter<'o> {
             },
             NodeValue::CodeBlock(ref ncb) => {
                 if entering {
-                    self.cr()?;
+                    if ncb.info.eq("math") {
+                        self.render_math_code_block(node, &ncb.literal)?;
+                    } else {
+                        self.cr()?;
 
-                    let mut first_tag = 0;
-                    let mut pre_attributes: HashMap<String, String> = HashMap::new();
-                    let mut code_attributes: HashMap<String, String> = HashMap::new();
-                    let code_attr: String;
+                        let mut first_tag = 0;
+                        let mut pre_attributes: HashMap<String, String> = HashMap::new();
+                        let mut code_attributes: HashMap<String, String> = HashMap::new();
+                        let code_attr: String;
 
-                    let literal = &ncb.literal.as_bytes();
-                    let info = &ncb.info.as_bytes();
+                        let literal = &ncb.literal.as_bytes();
+                        let info = &ncb.info.as_bytes();
 
-                    if !info.is_empty() {
-                        while first_tag < info.len() && !isspace(info[first_tag]) {
-                            first_tag += 1;
-                        }
+                        if !info.is_empty() {
+                            while first_tag < info.len() && !isspace(info[first_tag]) {
+                                first_tag += 1;
+                            }
 
-                        let lang_str = str::from_utf8(&info[..first_tag]).unwrap();
-                        let info_str = str::from_utf8(&info[first_tag..]).unwrap().trim();
+                            let lang_str = str::from_utf8(&info[..first_tag]).unwrap();
+                            let info_str = str::from_utf8(&info[first_tag..]).unwrap().trim();
 
-                        if self.options.render.github_pre_lang {
-                            pre_attributes.insert(String::from("lang"), lang_str.to_string());
+                            if self.options.render.github_pre_lang {
+                                pre_attributes.insert(String::from("lang"), lang_str.to_string());
 
-                            if self.options.render.full_info_string && !info_str.is_empty() {
-                                pre_attributes
-                                    .insert(String::from("data-meta"), info_str.trim().to_string());
-                            }
-                        } else {
-                            code_attr = format!("language-{}", lang_str);
-                            code_attributes.insert(String::from("class"), code_attr);
+                                if self.options.render.full_info_string && !info_str.is_empty() {
+                                    pre_attributes.insert(
+                                        String::from("data-meta"),
+                                        info_str.trim().to_string(),
+                                    );
+                                }
+                            } else {
+                                code_attr = format!("language-{}", lang_str);
+                                code_attributes.insert(String::from("class"), code_attr);
 
-                            if self.options.render.full_info_string && !info_str.is_empty() {
-                                code_attributes
-                                    .insert(String::from("data-meta"), info_str.to_string());
+                                if self.options.render.full_info_string && !info_str.is_empty() {
+                                    code_attributes
+                                        .insert(String::from("data-meta"), info_str.to_string());
+                                }
                             }
                         }
-                    }
 
-                    if self.options.render.sourcepos {
-                        let ast = node.data.borrow();
-                        pre_attributes
-                            .insert("data-sourcepos".to_string(), ast.sourcepos.to_string());
-                    }
+                        if self.options.render.sourcepos {
+                            let ast = node.data.borrow();
+                            pre_attributes
+                                .insert("data-sourcepos".to_string(), ast.sourcepos.to_string());
+                        }
 
-                    match self.plugins.render.codefence_syntax_highlighter {
-                        None => {
-                            write_opening_tag(self.output, "pre", pre_attributes)?;
-                            write_opening_tag(self.output, "code", code_attributes)?;
+                        match self.plugins.render.codefence_syntax_highlighter {
+                            None => {
+                                write_opening_tag(self.output, "pre", pre_attributes)?;
+                                write_opening_tag(self.output, "code", code_attributes)?;
 
-                            self.escape(literal)?;
+                                self.escape(literal)?;
 
-                            self.output.write_all(b"</code></pre>\n")?
-                        }
-                        Some(highlighter) => {
-                            highlighter.write_pre_tag(self.output, pre_attributes)?;
-                            highlighter.write_code_tag(self.output, code_attributes)?;
-
-                            highlighter.write_highlighted(
-                                self.output,
-                                match str::from_utf8(&info[..first_tag]) {
-                                    Ok(lang) => Some(lang),
-                                    Err(_) => None,
-                                },
-                                &ncb.literal,
-                            )?;
-
-                            self.output.write_all(b"</code></pre>\n")?
+                                self.output.write_all(b"</code></pre>\n")?
+                            }
+                            Some(highlighter) => {
+                                highlighter.write_pre_tag(self.output, pre_attributes)?;
+                                highlighter.write_code_tag(self.output, code_attributes)?;
+
+                                highlighter.write_highlighted(
+                                    self.output,
+                                    match str::from_utf8(&info[..first_tag]) {
+                                        Ok(lang) => Some(lang),
+                                        Err(_) => None,
+                                    },
+                                    &ncb.literal,
+                                )?;
+
+                                self.output.write_all(b"</code></pre>\n")?
+                            }
                         }
                     }
                 }
@@ -1015,6 +1028,16 @@ impl<'o> HtmlFormatter<'o> {
                     }
                 }
             }
+            NodeValue::Math(NodeMath {
+                ref literal,
+                display_math,
+                dollar_math,
+                ..
+            }) => {
+                if entering {
+                    self.render_math_inline(node, literal, display_math, dollar_math)?;
+                }
+            }
         }
         Ok(false)
     }
@@ -1056,4 +1079,68 @@ impl<'o> HtmlFormatter<'o> {
         }
         Ok(true)
     }
+
+    // Renders a math dollar inline, `$...$` and `$$...$$` using `<span>` to be similar
+    // to other renderers.
+    fn render_math_inline<'a>(
+        &mut self,
+        node: &'a AstNode<'a>,
+        literal: &String,
+        display_math: bool,
+        dollar_math: bool,
+    ) -> io::Result<()> {
+        let mut tag_attributes: Vec<(String, String)> = Vec::new();
+        let style_attr = if display_math { "display" } else { "inline" };
+        let tag: &str = if dollar_math { "span" } else { "code" };
+
+        tag_attributes.push((String::from("data-math-style"), String::from(style_attr)));
+
+        if self.options.render.sourcepos {
+            let ast = node.data.borrow();
+            tag_attributes.push(("data-sourcepos".to_string(), ast.sourcepos.to_string()));
+        }
+
+        write_opening_tag(self.output, tag, tag_attributes)?;
+        self.escape(literal.as_bytes())?;
+        write!(self.output, "</{}>", tag)?;
+
+        Ok(())
+    }
+
+    // Renders a math code block, ```` ```math ```` using `<pre><code>`
+    fn render_math_code_block<'a>(
+        &mut self,
+        node: &'a AstNode<'a>,
+        literal: &String,
+    ) -> io::Result<()> {
+        self.cr()?;
+
+        // use vectors to ensure attributes always written in the same order,
+        // for testing stability
+        let mut pre_attributes: Vec<(String, String)> = Vec::new();
+        let mut code_attributes: Vec<(String, String)> = Vec::new();
+        let lang_str = "math";
+
+        if self.options.render.github_pre_lang {
+            pre_attributes.push((String::from("lang"), lang_str.to_string()));
+            pre_attributes.push((String::from("data-math-style"), String::from("display")));
+        } else {
+            let code_attr = format!("language-{}", lang_str);
+            code_attributes.push((String::from("class"), code_attr));
+            code_attributes.push((String::from("data-math-style"), String::from("display")));
+        }
+
+        if self.options.render.sourcepos {
+            let ast = node.data.borrow();
+            pre_attributes.push(("data-sourcepos".to_string(), ast.sourcepos.to_string()));
+        }
+
+        write_opening_tag(self.output, "pre", pre_attributes)?;
+        write_opening_tag(self.output, "code", code_attributes)?;
+
+        self.escape(literal.as_bytes())?;
+        self.output.write_all(b"</code></pre>\n")?;
+
+        Ok(())
+    }
 }
diff --git a/src/main.rs b/src/main.rs
@@ -151,6 +151,8 @@ enum Extension {
     Footnotes,
     DescriptionLists,
     MultilineBlockQuotes,
+    MathDollars,
+    MathCode,
 }
 
 #[derive(Clone, Copy, Debug, ValueEnum)]
@@ -216,6 +218,8 @@ fn main() -> Result<(), Box<dyn Error>> {
         .footnotes(exts.contains(&Extension::Footnotes))
         .description_lists(exts.contains(&Extension::DescriptionLists))
         .multiline_block_quotes(exts.contains(&Extension::MultilineBlockQuotes))
+        .math_dollars(exts.contains(&Extension::MathDollars))
+        .math_code(exts.contains(&Extension::MathCode))
         .front_matter_delimiter(cli.front_matter_delimiter);
 
     #[cfg(feature = "shortcodes")]