generate: demote headings, and group link reference definitions (#620)

Before this commit, the implementation of `configlet generate` was
noticeably incomplete - it could produce files with incorrect heading
levels.

Given a Concept Exercise with an `introduction.md.tpl` file like:

    # Introduction

    ## Date and Time

    %{concept:date-time}

`configlet generate` would write an `introduction.md` file, inserting
the introduction for the `date-time` Concept, but without demoting its
headings:

    # Introduction

    ## Date and Time

    Blah blah.

    ## A heading from the date-time `introduction.md` file

    Blah blah.

With this commit, configlet will demote the level of an inserted heading
when appropriate:

    # Introduction

    ## Date and Time

    Blah blah.

    ### A heading from the date-time `introduction.md` file

    Blah blah.

`configlet generate` now also writes the same when given the template
structure of:

    # Introduction

    %{concept:date-time}

That is, it adds a level-2 heading when the placeholder is not already
under a level-2 heading - we concluded that this looks better on the
website. The heading's contents are taken from the concept's `name` in
the track-level `config.json` file, so you may prefer to omit such
level-2 headings in the template file (unless you want the heading
contents to be different).

This commit also fixes configlet's handling of link reference
definitions. Consider this `introduction.md.tpl` file:

    # Introduction

    ## Foo

    %{concept:foo}

    ## Bar

    %{concept:bar}

Before this commit, when more than one placeholder had a link reference
definition, `configlet generate` would produce something like:

    # Introduction

    ## Foo

    Here is a line with a link to [Foo][foo].

    [foo]: http://www.example.com

    ## Bar

    Here is another line with a link to [Bar][bar].

    [bar]: http://www.example.org

That is, it would not combine link reference definitions at the bottom.
With this commit, it does so - ordering by first usage, and
deduplicating them:

    # Introduction

    ## Foo

    Here is a line with a link to [Foo][foo].

    ## Bar

    Here is another line with a link to [Bar][bar].

    [foo]: http://www.example.com
    [bar]: http://www.example.org

The Markdown handling in this commit is deliberately rudimentary, and
has some limitations. We're doing it this way because there is no
pure-Nim Markdown parser and renderer that does what we want. The
plan is to use a wrapper for libcmark [1], the reference implementation
of the CommonMark Spec [2].

However, cmark is not designed foremost to be a Markdown formatter - for
example, it inlines a link destination from a matching link reference
definition. But we want to generate `introduction.md` files without that
inlining, so we need patch or workaround cmark's rendering. There's some
complexity there that's best left for another commit.

[1] https://github.com/commonmark/cmark
[2] https://spec.commonmark.org/0.30

Closes: #328
Closes: #626

src/generate/generate.nim

@@ -1,5 +1,54 @@
-import std/[strbasics, strformat, strscans, terminal]
-import ".."/[cli, helpers]
+import std/[parseutils, strbasics, strformat, strscans, strutils, sugar, tables,
+            terminal]
+import ".."/[cli, helpers, types_track_config]
+
+proc getConceptSlugLookup(trackDir: Path): Table[Slug, string] =
+  ## Returns a `Table` that maps each concept's `slug` to its `name`.
+  let concepts = TrackConfig.init(readFile(trackDir / "config.json")).concepts
+  collect:
+    for con in concepts:
+      {con.slug.Slug: con.name}
+
+func alterHeadings(s: string, linkDefs: var seq[string], h2 = ""): string =
+  result = newStringOfCap(s.len)
+  var i = 0
+  i += s.skipWhitespace()
+  # Skip the top-level heading (if any).
+  # The CommonMark Spec requires that an ATX heading has a a space, tab, or
+  # newline after the opening sequence of '#' characters.
+  # For now, support only spaces.
+  if s.continuesWith("# ", i):
+    i += s.skipUntil('\n', i)
+  if h2.len > 0:
+    result.add &"## {h2}"
+  # Demote other headings.
+  var inFencedCodeBlock = false
+  var inFencedCodeBlockTildes = false
+  var inCommentBlock = false
+  while i < s.len:
+    result.add s[i]
+    if s[i] == '\n':
+      # Add a '#' to a line that begins with '#', unless inside a code or HTML block.
+      if s.continuesWith("#", i+1) and not (inFencedCodeBlock or
+                                            inFencedCodeBlockTildes or inCommentBlock):
+        result.add '#'
+      elif s.continuesWith("[", i+1):
+        let j = s.find("]:", i+2)
+        if j > i+2 and j < s.find('\n', i+2):
+          var line = ""
+          i += s.parseUntil(line, '\n', i+1)
+          if line notin linkDefs:
+            linkDefs.add line
+      elif s.continuesWith("```", i+1):
+        inFencedCodeBlock = not inFencedCodeBlock
+      elif s.continuesWith("~~~", i+1):
+        inFencedCodeBlockTildes = not inFencedCodeBlockTildes
+      elif s.continuesWith("<!--", i+1):
+        inCommentBlock = true
+    elif inCommentBlock and s.continuesWith("-->", i):
+      inCommentBlock = false
+    inc i
+  strip result
 
 proc writeError(description: string, path: Path) =
   let descriptionPrefix = description & ":"
@@ -10,21 +59,21 @@ proc writeError(description: string, path: Path) =
   stderr.writeLine(path)
   stderr.write "\n"
 
-proc conceptIntroduction(trackDir: Path, slug: string,
-                         templatePath: Path): string =
-  ## Returns the contents of the `introduction.md` file for a `slug`, but
-  ## without any top-level heading, and without any leading/trailing whitespace.
-  let conceptDir = trackDir / "concepts" / slug
+proc conceptIntroduction(trackDir: Path, slug: Slug, templatePath: Path,
+                         linkDefs: var seq[string], h2 = ""): string =
+  ## Returns the contents of the `introduction.md` file for a `slug`, but:
+  ## - Without a first top-level heading.
+  ## - Adding a starting a second-level heading containing `h2`.
+  ## - Demoting the level of any other heading.
+  ## - Without any leading/trailing whitespace.
+  ## - Without any link reference definitions.
+  ##
+  ## Appends link reference definitions to `linkDefs`.
+  let conceptDir = trackDir / "concepts" / slug.string
   if dirExists(conceptDir):
     let path = conceptDir / "introduction.md"
     if fileExists(path):
-      result = readFile(path)
-      var i = 0
-      # Strip the top-level heading (if any)
-      if scanp(result, i, *{' ', '\t', '\v', '\c', '\n', '\f'}, "#", +' ',
-               +(~'\n')):
-        result.setSlice(i..result.high)
-      strip result
+      result = path.readFile().alterHeadings(linkDefs, h2)
     else:
       writeError(&"File {path} not found for concept '{slug}'", templatePath)
       quit(1)
@@ -33,25 +82,44 @@ proc conceptIntroduction(trackDir: Path, slug: string,
                templatePath)
     quit(1)
 
-proc generateIntroduction(trackDir: Path, templatePath: Path): string =
+proc generateIntroduction(trackDir: Path, templatePath: Path,
+                          slugLookup: Table[Slug, string]): string =
   ## Reads the file at `templatePath` and returns the content of the
   ## corresponding `introduction.md` file.
   let content = readFile(templatePath)
   result = newStringOfCap(1024)
 
   var i = 0
+  var headingLevel = 1
+  var linkDefs = newSeq[string]()
   while i < content.len:
-    var conceptSlug = ""
+    var conceptSlug = Slug ""
     # Here, we implement the syntax for a placeholder as %{concept:some-slug}
     # where we allow spaces after the opening brace, around the colon,
     # and before the closing brace. The slug must be in kebab-case.
     if scanp(content, i,
              "%{", *{' '}, "concept", *{' '}, ':', *{' '},
              +{'a'..'z', '-'} -> conceptSlug.add($_), *{' '}, '}'):
-      result.add conceptIntroduction(trackDir, conceptSlug, templatePath)
+      if conceptSlug in slugLookup:
+        let h2 = if headingLevel == 2: "" else: slugLookup[conceptSlug]
+        result.add conceptIntroduction(trackDir, conceptSlug, templatePath,
+                                       linkDefs, h2)
+      else:
+        writeError(&"Concept '{conceptSlug}' does not exist in track config.json",
+                   templatePath)
+        quit(1)
     else:
+      if content.continuesWith("\n#", i):
+        headingLevel = content.skipWhile({'#'}, i+1)
       result.add content[i]
       inc i
+  result.strip()
+  result.add '\n'
+  if linkDefs.len > 0:
+    result.add '\n'
+    for linkDef in linkDefs:
+      result.add linkDef
+      result.add '\n'
 
 proc generate*(conf: Conf) =
   ## For every Concept Exercise in `conf.trackDir` with an `introduction.md.tpl`
@@ -60,9 +128,11 @@ proc generate*(conf: Conf) =
 
   let conceptExercisesDir = trackDir / "exercises" / "concept"
   if dirExists(conceptExercisesDir):
+    let slugLookup = getConceptSlugLookup(trackDir)
     for conceptExerciseDir in getSortedSubdirs(conceptExercisesDir):
       let introductionTemplatePath = conceptExerciseDir / ".docs" / "introduction.md.tpl"
       if fileExists(introductionTemplatePath):
-        let introduction = generateIntroduction(trackDir, introductionTemplatePath)
+        let introduction = generateIntroduction(trackDir, introductionTemplatePath,
+                                                slugLookup)
         let introductionPath = introductionTemplatePath.string[0..^5] # Removes `.tpl`
         writeFile(introductionPath, introduction)

src/sync/sync_common.nim

@@ -25,18 +25,13 @@ proc postHook*(e: ConceptExercise | PracticeExercise) =
     stderr.writeLine msg
     quit 1
 
-func `==`*(x, y: Slug): bool {.borrow.}
-func `<`*(x, y: Slug): bool {.borrow.}
-
 func getSlugs*(e: seq[ConceptExercise] | seq[PracticeExercise]): seq[Slug] =
   ## Returns a seq of the slugs in `e`, in alphabetical order.
   result = newSeq[Slug](e.len)
   for i, item in e:
     result[i] = item.slug
   sort result
 
-func len*(slug: Slug): int {.borrow.}
-
 func truncateAndAdd*(s: var string, truncateLen: int, slug: Slug) =
   ## Truncates `s` to `truncateLen`, then appends `slug`.
   ##

src/types_track_config.nim

@@ -1,4 +1,4 @@
-import std/sets
+import std/[hashes, sets]
 import pkg/jsony
 import "."/[cli, helpers]
 
@@ -58,6 +58,11 @@ type
     ekPractice = "practice"
 
 func `$`*(slug: Slug): string {.borrow.}
+func `==`*(x, y: Slug): bool {.borrow.}
+func `<`*(x, y: Slug): bool {.borrow.}
+func len*(slug: Slug): int {.borrow.}
+func hash*(slug: Slug): Hash {.borrow.}
+func add*(s: var Slug, c: char) {.borrow.}
 
 proc init*(T: typedesc[TrackConfig]; trackConfigContents: string): T =
   ## Deserializes `trackConfigContents` using `jsony` to a `TrackConfig` object.

tests/all_tests.nim

@@ -1,6 +1,7 @@
 import "."/[
   test_binary,
   test_fmt,
+  test_generate,
   test_json,
   test_lint,
   test_probspecs,

tests/test_binary.nim

@@ -867,17 +867,16 @@ proc testsForSync(binaryPath: static string) =
                          &"failed to merge '{mainBranchName}' in " &
                          &"problem-specifications directory: '{probSpecsDir}'")
 
-proc prepareIntroductionFiles(trackDir, header, placeholder: string;
-                              removeIntro: bool) =
+proc prepareIntroductionFiles(trackDir, placeholder: string; removeIntro: bool) =
   # Writes an `introduction.md.tpl` file for the `bird-count` Concept Exercise,
-  # containing the given `header` and `placeholder`. Also removes the
-  # `introduction.md` file if `removeIntro` is `true`.
+  # containing the given `placeholder`. Also removes the `introduction.md` file
+  # if `removeIntro` is `true`.
   let
     docsPath = trackDir / "exercises" / "concept" / "bird-count" / ".docs"
     introPath = docsPath / "introduction.md"
     templatePath = introPath & ".tpl"
     templateContents = fmt"""
-      # {header}
+      # Introduction
 
       {placeholder}
     """.unindent()
@@ -892,7 +891,7 @@ proc testsForGenerate(binaryPath: string) =
 
     # Setup: clone a track repo, and checkout a known state
     setupExercismRepo("elixir", trackDir,
-                      "f3974abf6e0d4a434dfe3494d58581d399c18edb") # 2021-05-09
+                      "91ccf91940f32aff3726c772695b2de167d8192a") # 2022-06-12
 
     test "`configlet generate` exits with 0 when there are no `.md.tpl` files":
       execAndCheck(0, generateCmd, "")
@@ -901,8 +900,7 @@ proc testsForGenerate(binaryPath: string) =
       checkNoDiff(trackDir)
 
     # Valid placeholder syntax without spaces, and invalid slug
-    prepareIntroductionFiles(trackDir, "Recursion",
-                             "%{concept:not-a-real-concept-slug}",
+    prepareIntroductionFiles(trackDir, "%{concept:not-a-real-concept-slug}",
                              removeIntro = false)
 
     test "`configlet generate` exits with 1 for an invalid placeholder usage":
@@ -912,7 +910,7 @@ proc testsForGenerate(binaryPath: string) =
       checkNoDiff(trackDir)
 
     # Valid placeholder syntax without spaces, and valid slug
-    prepareIntroductionFiles(trackDir, "Recursion", "%{concept:recursion}",
+    prepareIntroductionFiles(trackDir, "%{concept:recursion}",
                              removeIntro = true)
 
     test "`configlet generate` exits with 0 for a valid `.md.tpl` file":
@@ -922,7 +920,7 @@ proc testsForGenerate(binaryPath: string) =
       checkNoDiff(trackDir)
 
     # Valid placeholder syntax with spaces, and valid slug
-    prepareIntroductionFiles(trackDir, "Recursion", "%{ concept : recursion }",
+    prepareIntroductionFiles(trackDir, "%{ concept : recursion }",
                              removeIntro = true)
 
     test "`configlet generate` exits with 0 for valid placeholder usage with spaces":

tests/test_generate.nim

@@ -0,0 +1,81 @@
+import std/[strutils, unittest]
+from "."/generate/generate {.all.} import alterHeadings
+
+proc testGenerate =
+  suite "generate":
+    test "alterHeadings":
+      const s = """
+        # Heading 1
+
+        The quick brown fox jumps over a lazy dog.
+
+        The five boxing wizards jump quickly.
+
+        ## Heading 2
+
+        The quick brown fox jumps over a lazy dog.
+
+        <!--
+        # This line is not a heading
+        This line is in an HTML comment block -->
+
+        ### Heading 3
+
+        The quick brown fox jumps over a lazy dog.
+
+        ```nim
+        # This line is not a heading
+        echo "hi"
+        ```
+
+        ## Heading 4
+
+        The quick brown fox jumps over a lazy dog.
+
+        ~~~nim
+        # This line is not a heading
+        echo "hi"
+        ~~~
+      """.unindent()
+
+      const expected = """
+        The quick brown fox jumps over a lazy dog.
+
+        The five boxing wizards jump quickly.
+
+        ### Heading 2
+
+        The quick brown fox jumps over a lazy dog.
+
+        <!--
+        # This line is not a heading
+        This line is in an HTML comment block -->
+
+        #### Heading 3
+
+        The quick brown fox jumps over a lazy dog.
+
+        ```nim
+        # This line is not a heading
+        echo "hi"
+        ```
+
+        ### Heading 4
+
+        The quick brown fox jumps over a lazy dog.
+
+        ~~~nim
+        # This line is not a heading
+        echo "hi"
+        ~~~""".unindent() # No final newline
+
+      var linkDefs = newSeq[string]()
+      check alterHeadings(s, linkDefs) == expected
+      check alterHeadings(s, linkDefs, "Maps") == "## Maps\n\n" & expected
+      check linkDefs.len == 0
+
+proc main =
+  testGenerate()
+
+main()
+{.used.}