Footnotes (#64)

* Add baseline test

* Some preliminary work.

* cont'd

* Add footnote reference

* Start postprocessing

* MVP: tests pass

* commonmark footnote out

* Factor out reference/footnote maps

* fix a memory leak & some asserts

* We don't assert/check snprintf elsewhere

* Remove bad linear search, extend test case

* cleanup

* man page update

* add footnotes as option

* bugfix (found in comrak first!)

* Shift static var into renderer struct

man/man3/cmark-gfm.3

@@ -1,4 +1,4 @@
-.TH cmark-gfm 3 "April 03, 2017" "LOCAL" "Library Functions Manual"
+.TH cmark-gfm 3 "November 09, 2017" "LOCAL" "Library Functions Manual"
 .SH
 NAME
 .PP
@@ -40,6 +40,7 @@ typedef enum {
   CMARK_NODE_PARAGRAPH      = CMARK_NODE_TYPE_BLOCK | 0x0008,
   CMARK_NODE_HEADING        = CMARK_NODE_TYPE_BLOCK | 0x0009,
   CMARK_NODE_THEMATIC_BREAK = CMARK_NODE_TYPE_BLOCK | 0x000a,
+  CMARK_NODE_FOOTNOTE_DEFINITION = CMARK_NODE_TYPE_BLOCK | 0x000b,
 
   /* Inline */
   CMARK_NODE_TEXT          = CMARK_NODE_TYPE_INLINE | 0x0001,
@@ -52,6 +53,7 @@ typedef enum {
   CMARK_NODE_STRONG        = CMARK_NODE_TYPE_INLINE | 0x0008,
   CMARK_NODE_LINK          = CMARK_NODE_TYPE_INLINE | 0x0009,
   CMARK_NODE_IMAGE         = CMARK_NODE_TYPE_INLINE | 0x000a,
+  CMARK_NODE_FOOTNOTE_REFERENCE = CMARK_NODE_TYPE_INLINE | 0x000b,
 } cmark_node_type;
 .RE
 \f[]
@@ -780,6 +782,20 @@ responsibility to free the returned buffer.
 As for \f[I]cmark_render_commonmark\f[], but specifying the allocator to
 use for the resulting string.
 
+.PP
+\fIchar *\f[] \fBcmark_render_plaintext\f[](\fIcmark_node *root\f[], \fIint options\f[], \fIint width\f[])
+
+.PP
+Render a \f[I]node\f[] tree as a plain text document. It is the caller's
+responsibility to free the returned buffer.
+
+.PP
+\fIchar *\f[] \fBcmark_render_plaintext_with_mem\f[](\fIcmark_node *root\f[], \fIint options\f[], \fIint width\f[], \fIcmark_mem *mem\f[])
+
+.PP
+As for \f[I]cmark_render_plaintext\f[], but specifying the allocator to
+use for the resulting string.
+
 .PP
 \fIchar *\f[] \fBcmark_render_latex\f[](\fIcmark_node *root\f[], \fIint options\f[], \fIint width\f[])
 
@@ -917,6 +933,30 @@ dashes.
 .PP
 Use GitHub\-style  tags for code blocks instead of .
 
+.PP
+.nf
+\fC
+.RS 0n
+#define CMARK_OPT_LIBERAL_HTML_TAG (1 << 12)
+.RE
+\f[]
+.fi
+
+.PP
+Be liberal in interpreting inline HTML tags.
+
+.PP
+.nf
+\fC
+.RS 0n
+#define CMARK_OPT_FOOTNOTES (1 << 13)
+.RE
+\f[]
+.fi
+
+.PP
+Parse footnotes.
+
 .SS
 Version information
 

src/CMakeLists.txt

@@ -15,6 +15,8 @@ set(HEADERS
   iterator.h
   chunk.h
   references.h
+  footnotes.h
+  map.h
   utf8.h
   scanners.h
   inlines.h
@@ -36,6 +38,8 @@ set(LIBRARY_SOURCES
   utf8.c
   buffer.c
   references.c
+  footnotes.c
+  map.c
   render.c
   man.c
   xml.c

src/blocks.c

@@ -21,6 +21,7 @@
 #include "inlines.h"
 #include "houdini.h"
 #include "buffer.h"
+#include "footnotes.h"
 
 #define CODE_INDENT 4
 #define TAB_STOP 4
@@ -97,7 +98,7 @@ static void cmark_parser_dispose(cmark_parser *parser) {
     cmark_node_free(parser->root);
 
   if (parser->refmap)
-    cmark_reference_map_free(parser->refmap);
+    cmark_map_free(parser->refmap);
 }
 
 static void cmark_parser_reset(cmark_parser *parser) {
@@ -408,7 +409,7 @@ void cmark_manage_extensions_special_characters(cmark_parser *parser, int add) {
 // Walk through node and all children, recursively, parsing
 // string content into inline content where appropriate.
 static void process_inlines(cmark_parser *parser,
-                            cmark_reference_map *refmap, int options) {
+                            cmark_map *refmap, int options) {
   cmark_iter *iter = cmark_iter_new(parser->root);
   cmark_node *cur;
   cmark_event_type ev_type;
@@ -429,6 +430,84 @@ static void process_inlines(cmark_parser *parser,
   cmark_iter_free(iter);
 }
 
+static int sort_footnote_by_ix(const void *_a, const void *_b) {
+  cmark_footnote *a = *(cmark_footnote **)_a;
+  cmark_footnote *b = *(cmark_footnote **)_b;
+  return (int)a->ix - (int)b->ix;
+}
+
+static void process_footnotes(cmark_parser *parser) {
+  // * Collect definitions in a map.
+  // * Iterate the references in the document in order, assigning indices to
+  //   definitions in the order they're seen.
+  // * Write out the footnotes at the bottom of the document in index order.
+
+  cmark_map *map = cmark_footnote_map_new(parser->mem);
+
+  cmark_iter *iter = cmark_iter_new(parser->root);
+  cmark_node *cur;
+  cmark_event_type ev_type;
+
+  while ((ev_type = cmark_iter_next(iter)) != CMARK_EVENT_DONE) {
+    cur = cmark_iter_get_node(iter);
+    if (ev_type == CMARK_EVENT_EXIT && cur->type == CMARK_NODE_FOOTNOTE_DEFINITION) {
+      cmark_node_unlink(cur);
+      cmark_footnote_create(map, cur);
+    }
+  }
+
+  cmark_iter_free(iter);
+  iter = cmark_iter_new(parser->root);
+  unsigned int ix = 0;
+
+  while ((ev_type = cmark_iter_next(iter)) != CMARK_EVENT_DONE) {
+    cur = cmark_iter_get_node(iter);
+    if (ev_type == CMARK_EVENT_EXIT && cur->type == CMARK_NODE_FOOTNOTE_REFERENCE) {
+      cmark_footnote *footnote = (cmark_footnote *)cmark_map_lookup(map, &cur->as.literal);
+      if (footnote) {
+        if (!footnote->ix)
+          footnote->ix = ++ix;
+
+        char n[32];
+        snprintf(n, sizeof(n), "%d", footnote->ix);
+        cmark_chunk_free(parser->mem, &cur->as.literal);
+        cmark_strbuf buf = CMARK_BUF_INIT(parser->mem);
+        cmark_strbuf_puts(&buf, n);
+
+        cur->as.literal = cmark_chunk_buf_detach(&buf);
+      } else {
+        cmark_node *text = (cmark_node *)parser->mem->calloc(1, sizeof(*text));
+        cmark_strbuf_init(parser->mem, &text->content, 0);
+        text->type = (uint16_t) CMARK_NODE_TEXT;
+
+        cmark_strbuf buf = CMARK_BUF_INIT(parser->mem);
+        cmark_strbuf_puts(&buf, "[^");
+        cmark_strbuf_put(&buf, cur->as.literal.data, cur->as.literal.len);
+        cmark_strbuf_putc(&buf, ']');
+
+        text->as.literal = cmark_chunk_buf_detach(&buf);
+        cmark_node_insert_after(cur, text);
+        cmark_node_free(cur);
+      }
+    }
+  }
+
+  cmark_iter_free(iter);
+
+  if (map->sorted) {
+    qsort(map->sorted, map->size, sizeof(cmark_map_entry *), sort_footnote_by_ix);
+    for (unsigned int i = 0; i < map->size; ++i) {
+      cmark_footnote *footnote = (cmark_footnote *)map->sorted[i];
+      if (!footnote->ix)
+        continue;
+      cmark_node_append_child(parser->root, footnote->node);
+      footnote->node = NULL;
+    }
+  }
+
+  cmark_map_free(map);
+}
+
 // Attempts to parse a list item marker (bullet or enumerated).
 // On success, returns length of the marker, and populates
 // data with the details.  On failure, returns 0.
@@ -533,6 +612,8 @@ static cmark_node *finalize_document(cmark_parser *parser) {
 
   finalize(parser, parser->root);
   process_inlines(parser, parser->refmap, parser->options);
+  if (parser->options & CMARK_OPT_FOOTNOTES)
+    process_footnotes(parser);
 
   return parser->root;
 }
@@ -759,6 +840,18 @@ static bool parse_block_quote_prefix(cmark_parser *parser, cmark_chunk *input) {
   return res;
 }
 
+static bool parse_footnote_definition_block_prefix(cmark_parser *parser, cmark_chunk *input,
+                                                   cmark_node *container) {
+  if (parser->indent >= 4) {
+    S_advance_offset(parser, input, 4, true);
+    return true;
+  } else if (input->len > 0 && (input->data[0] == '\n' || (input->data[0] == '\r' && input->data[1] == '\n'))) {
+    return true;
+  }
+
+  return false;
+}
+
 static bool parse_node_item_prefix(cmark_parser *parser, cmark_chunk *input,
                                    cmark_node *container) {
   bool res = false;
@@ -913,6 +1006,10 @@ static cmark_node *check_open_blocks(cmark_parser *parser, cmark_chunk *input,
       if (parser->blank)
         goto done;
       break;
+		case CMARK_NODE_FOOTNOTE_DEFINITION:
+			if (!parse_footnote_definition_block_prefix(parser, input, container))
+				goto done;
+			break;
     default:
       break;
     }
@@ -1024,6 +1121,21 @@ static void open_new_blocks(cmark_parser *parser, cmark_node **container,
       *container = add_child(parser, *container, CMARK_NODE_THEMATIC_BREAK,
                              parser->first_nonspace + 1);
       S_advance_offset(parser, input, input->len - 1 - parser->offset, false);
+    } else if (!indented &&
+               parser->options & CMARK_OPT_FOOTNOTES &&
+               (matched = scan_footnote_definition(input, parser->first_nonspace))) {
+      cmark_chunk c = cmark_chunk_dup(input, parser->first_nonspace + 2, matched - 2);
+      cmark_chunk_to_cstr(parser->mem, &c);
+
+      while (c.data[c.len - 1] != ']')
+        --c.len;
+      --c.len;
+
+      S_advance_offset(parser, input, parser->first_nonspace + matched - parser->offset, false);
+      *container = add_child(parser, *container, CMARK_NODE_FOOTNOTE_DEFINITION, parser->first_nonspace + matched + 1);
+      (*container)->as.literal = c;
+
+      (*container)->internal_offset = matched;
     } else if ((!indented || cont_type == CMARK_NODE_LIST) &&
                (matched = parse_list_marker(
                     parser->mem, input, parser->first_nonspace,

src/cmark.c

@@ -7,8 +7,8 @@
 #include "cmark.h"
 #include "buffer.h"
 
-cmark_node_type CMARK_NODE_LAST_BLOCK = CMARK_NODE_THEMATIC_BREAK;
-cmark_node_type CMARK_NODE_LAST_INLINE = CMARK_NODE_IMAGE;
+cmark_node_type CMARK_NODE_LAST_BLOCK = CMARK_NODE_FOOTNOTE_DEFINITION;
+cmark_node_type CMARK_NODE_LAST_INLINE = CMARK_NODE_FOOTNOTE_REFERENCE;
 
 int cmark_version() { return CMARK_VERSION; }
 

src/cmark.h

@@ -52,6 +52,7 @@ typedef enum {
   CMARK_NODE_PARAGRAPH      = CMARK_NODE_TYPE_BLOCK | 0x0008,
   CMARK_NODE_HEADING        = CMARK_NODE_TYPE_BLOCK | 0x0009,
   CMARK_NODE_THEMATIC_BREAK = CMARK_NODE_TYPE_BLOCK | 0x000a,
+  CMARK_NODE_FOOTNOTE_DEFINITION = CMARK_NODE_TYPE_BLOCK | 0x000b,
 
   /* Inline */
   CMARK_NODE_TEXT          = CMARK_NODE_TYPE_INLINE | 0x0001,
@@ -64,6 +65,7 @@ typedef enum {
   CMARK_NODE_STRONG        = CMARK_NODE_TYPE_INLINE | 0x0008,
   CMARK_NODE_LINK          = CMARK_NODE_TYPE_INLINE | 0x0009,
   CMARK_NODE_IMAGE         = CMARK_NODE_TYPE_INLINE | 0x000a,
+  CMARK_NODE_FOOTNOTE_REFERENCE = CMARK_NODE_TYPE_INLINE | 0x000b,
 } cmark_node_type;
 
 extern cmark_node_type CMARK_NODE_LAST_BLOCK;
@@ -718,6 +720,10 @@ char *cmark_render_latex_with_mem(cmark_node *root, int options, int width, cmar
  */
 #define CMARK_OPT_LIBERAL_HTML_TAG (1 << 12)
 
+/** Parse footnotes.
+ */
+#define CMARK_OPT_FOOTNOTES (1 << 13)
+
 /**
  * ## Version information
  */

src/commonmark.c

@@ -463,6 +463,29 @@ static int S_render_node(cmark_renderer *renderer, cmark_node *node,
     }
     break;
 
+  case CMARK_NODE_FOOTNOTE_REFERENCE:
+    if (entering) {
+      LIT("[^");
+      OUT(cmark_chunk_to_cstr(renderer->mem, &node->as.literal), false, LITERAL);
+      LIT("]");
+    }
+    break;
+
+  case CMARK_NODE_FOOTNOTE_DEFINITION:
+    if (entering) {
+      renderer->footnote_ix += 1;
+      LIT("[^");
+      char n[32];
+      snprintf(n, sizeof(n), "%d", renderer->footnote_ix);
+      OUT(n, false, LITERAL);
+      LIT("]:\n");
+
+      cmark_strbuf_puts(renderer->prefix, "    ");
+    } else {
+      cmark_strbuf_truncate(renderer->prefix, renderer->prefix->size - 4);
+    }
+    break;
+
   default:
     assert(false);
     break;

src/footnotes.c

@@ -0,0 +1,40 @@
+#include "cmark.h"
+#include "parser.h"
+#include "footnotes.h"
+#include "inlines.h"
+#include "chunk.h"
+
+static void footnote_free(cmark_map *map, cmark_map_entry *_ref) {
+  cmark_footnote *ref = (cmark_footnote *)_ref;
+  cmark_mem *mem = map->mem;
+  if (ref != NULL) {
+    mem->free(ref->entry.label);
+    if (ref->node)
+      cmark_node_free(ref->node);
+    mem->free(ref);
+  }
+}
+
+void cmark_footnote_create(cmark_map *map, cmark_node *node) {
+  cmark_footnote *ref;
+  unsigned char *reflabel = normalize_map_label(map->mem, &node->as.literal);
+
+  /* empty footnote name, or composed from only whitespace */
+  if (reflabel == NULL)
+    return;
+
+  assert(map->sorted == NULL);
+
+  ref = (cmark_footnote *)map->mem->calloc(1, sizeof(*ref));
+  ref->entry.label = reflabel;
+  ref->node = node;
+  ref->entry.age = map->size;
+  ref->entry.next = map->refs;
+
+  map->refs = (cmark_map_entry *)ref;
+  map->size++;
+}
+
+cmark_map *cmark_footnote_map_new(cmark_mem *mem) {
+  return cmark_map_new(mem, footnote_free);
+}

src/footnotes.h

@@ -0,0 +1,25 @@
+#ifndef CMARK_FOOTNOTES_H
+#define CMARK_FOOTNOTES_H
+
+#include "map.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct cmark_footnote {
+  cmark_map_entry entry;
+  cmark_node *node;
+  unsigned int ix;
+};
+
+typedef struct cmark_footnote cmark_footnote;
+
+void cmark_footnote_create(cmark_map *map, cmark_node *node);
+cmark_map *cmark_footnote_map_new(cmark_mem *mem);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif