From ef65ef9bcf3ab4c8218e1356d4d9ae83ca235b26 Mon Sep 17 00:00:00 2001 From: ee7 <45465154+ee7@users.noreply.github.com> Date: Sat, 2 Jul 2022 00:51:36 +0200 Subject: [PATCH] generate: demote headings, and group link reference definitions (#620) Before this commit, the implementation of `configlet generate` was noticeably incomplete - it could produce files with incorrect heading levels. Given a Concept Exercise with an `introduction.md.tpl` file like: # Introduction ## Date and Time %{concept:date-time} `configlet generate` would write an `introduction.md` file, inserting the introduction for the `date-time` Concept, but without demoting its headings: # Introduction ## Date and Time Blah blah. ## A heading from the date-time `introduction.md` file Blah blah. With this commit, configlet will demote the level of an inserted heading when appropriate: # Introduction ## Date and Time Blah blah. ### A heading from the date-time `introduction.md` file Blah blah. `configlet generate` now also writes the same when given the template structure of: # Introduction %{concept:date-time} That is, it adds a level-2 heading when the placeholder is not already under a level-2 heading - we concluded that this looks better on the website. The heading's contents are taken from the concept's `name` in the track-level `config.json` file, so you may prefer to omit such level-2 headings in the template file (unless you want the heading contents to be different). This commit also fixes configlet's handling of link reference definitions. Consider this `introduction.md.tpl` file: # Introduction ## Foo %{concept:foo} ## Bar %{concept:bar} Before this commit, when more than one placeholder had a link reference definition, `configlet generate` would produce something like: # Introduction ## Foo Here is a line with a link to [Foo][foo]. [foo]: http://www.example.com ## Bar Here is another line with a link to [Bar][bar]. [bar]: http://www.example.org That is, it would not combine link reference definitions at the bottom. With this commit, it does so - ordering by first usage, and deduplicating them: # Introduction ## Foo Here is a line with a link to [Foo][foo]. ## Bar Here is another line with a link to [Bar][bar]. [foo]: http://www.example.com [bar]: http://www.example.org The Markdown handling in this commit is deliberately rudimentary, and has some limitations. We're doing it this way because there is no pure-Nim Markdown parser and renderer that does what we want. The plan is to use a wrapper for libcmark [1], the reference implementation of the CommonMark Spec [2]. However, cmark is not designed foremost to be a Markdown formatter - for example, it inlines a link destination from a matching link reference definition. But we want to generate `introduction.md` files without that inlining, so we need patch or workaround cmark's rendering. There's some complexity there that's best left for another commit. [1] https://github.com/commonmark/cmark [2] https://spec.commonmark.org/0.30 Closes: #328 Closes: #626 --- src/generate/generate.nim | 106 ++++++++++++++++++++++++++++++------- src/sync/sync_common.nim | 5 -- src/types_track_config.nim | 7 ++- tests/all_tests.nim | 1 + tests/test_binary.nim | 18 +++---- tests/test_generate.nim | 81 ++++++++++++++++++++++++++++ 6 files changed, 184 insertions(+), 34 deletions(-) create mode 100644 tests/test_generate.nim diff --git a/src/generate/generate.nim b/src/generate/generate.nim index 1e5c2c0b..0f720c7c 100644 --- a/src/generate/generate.nim +++ b/src/generate/generate.nim @@ -1,5 +1,54 @@ -import std/[strbasics, strformat, strscans, terminal] -import ".."/[cli, helpers] +import std/[parseutils, strbasics, strformat, strscans, strutils, sugar, tables, + terminal] +import ".."/[cli, helpers, types_track_config] + +proc getConceptSlugLookup(trackDir: Path): Table[Slug, string] = + ## Returns a `Table` that maps each concept's `slug` to its `name`. + let concepts = TrackConfig.init(readFile(trackDir / "config.json")).concepts + collect: + for con in concepts: + {con.slug.Slug: con.name} + +func alterHeadings(s: string, linkDefs: var seq[string], h2 = ""): string = + result = newStringOfCap(s.len) + var i = 0 + i += s.skipWhitespace() + # Skip the top-level heading (if any). + # The CommonMark Spec requires that an ATX heading has a a space, tab, or + # newline after the opening sequence of '#' characters. + # For now, support only spaces. + if s.continuesWith("# ", i): + i += s.skipUntil('\n', i) + if h2.len > 0: + result.add &"## {h2}" + # Demote other headings. + var inFencedCodeBlock = false + var inFencedCodeBlockTildes = false + var inCommentBlock = false + while i < s.len: + result.add s[i] + if s[i] == '\n': + # Add a '#' to a line that begins with '#', unless inside a code or HTML block. + if s.continuesWith("#", i+1) and not (inFencedCodeBlock or + inFencedCodeBlockTildes or inCommentBlock): + result.add '#' + elif s.continuesWith("[", i+1): + let j = s.find("]:", i+2) + if j > i+2 and j < s.find('\n', i+2): + var line = "" + i += s.parseUntil(line, '\n', i+1) + if line notin linkDefs: + linkDefs.add line + elif s.continuesWith("```", i+1): + inFencedCodeBlock = not inFencedCodeBlock + elif s.continuesWith("~~~", i+1): + inFencedCodeBlockTildes = not inFencedCodeBlockTildes + elif s.continuesWith("", i): + inCommentBlock = false + inc i + strip result proc writeError(description: string, path: Path) = let descriptionPrefix = description & ":" @@ -10,21 +59,21 @@ proc writeError(description: string, path: Path) = stderr.writeLine(path) stderr.write "\n" -proc conceptIntroduction(trackDir: Path, slug: string, - templatePath: Path): string = - ## Returns the contents of the `introduction.md` file for a `slug`, but - ## without any top-level heading, and without any leading/trailing whitespace. - let conceptDir = trackDir / "concepts" / slug +proc conceptIntroduction(trackDir: Path, slug: Slug, templatePath: Path, + linkDefs: var seq[string], h2 = ""): string = + ## Returns the contents of the `introduction.md` file for a `slug`, but: + ## - Without a first top-level heading. + ## - Adding a starting a second-level heading containing `h2`. + ## - Demoting the level of any other heading. + ## - Without any leading/trailing whitespace. + ## - Without any link reference definitions. + ## + ## Appends link reference definitions to `linkDefs`. + let conceptDir = trackDir / "concepts" / slug.string if dirExists(conceptDir): let path = conceptDir / "introduction.md" if fileExists(path): - result = readFile(path) - var i = 0 - # Strip the top-level heading (if any) - if scanp(result, i, *{' ', '\t', '\v', '\c', '\n', '\f'}, "#", +' ', - +(~'\n')): - result.setSlice(i..result.high) - strip result + result = path.readFile().alterHeadings(linkDefs, h2) else: writeError(&"File {path} not found for concept '{slug}'", templatePath) quit(1) @@ -33,25 +82,44 @@ proc conceptIntroduction(trackDir: Path, slug: string, templatePath) quit(1) -proc generateIntroduction(trackDir: Path, templatePath: Path): string = +proc generateIntroduction(trackDir: Path, templatePath: Path, + slugLookup: Table[Slug, string]): string = ## Reads the file at `templatePath` and returns the content of the ## corresponding `introduction.md` file. let content = readFile(templatePath) result = newStringOfCap(1024) var i = 0 + var headingLevel = 1 + var linkDefs = newSeq[string]() while i < content.len: - var conceptSlug = "" + var conceptSlug = Slug "" # Here, we implement the syntax for a placeholder as %{concept:some-slug} # where we allow spaces after the opening brace, around the colon, # and before the closing brace. The slug must be in kebab-case. if scanp(content, i, "%{", *{' '}, "concept", *{' '}, ':', *{' '}, +{'a'..'z', '-'} -> conceptSlug.add($_), *{' '}, '}'): - result.add conceptIntroduction(trackDir, conceptSlug, templatePath) + if conceptSlug in slugLookup: + let h2 = if headingLevel == 2: "" else: slugLookup[conceptSlug] + result.add conceptIntroduction(trackDir, conceptSlug, templatePath, + linkDefs, h2) + else: + writeError(&"Concept '{conceptSlug}' does not exist in track config.json", + templatePath) + quit(1) else: + if content.continuesWith("\n#", i): + headingLevel = content.skipWhile({'#'}, i+1) result.add content[i] inc i + result.strip() + result.add '\n' + if linkDefs.len > 0: + result.add '\n' + for linkDef in linkDefs: + result.add linkDef + result.add '\n' proc generate*(conf: Conf) = ## For every Concept Exercise in `conf.trackDir` with an `introduction.md.tpl` @@ -60,9 +128,11 @@ proc generate*(conf: Conf) = let conceptExercisesDir = trackDir / "exercises" / "concept" if dirExists(conceptExercisesDir): + let slugLookup = getConceptSlugLookup(trackDir) for conceptExerciseDir in getSortedSubdirs(conceptExercisesDir): let introductionTemplatePath = conceptExerciseDir / ".docs" / "introduction.md.tpl" if fileExists(introductionTemplatePath): - let introduction = generateIntroduction(trackDir, introductionTemplatePath) + let introduction = generateIntroduction(trackDir, introductionTemplatePath, + slugLookup) let introductionPath = introductionTemplatePath.string[0..^5] # Removes `.tpl` writeFile(introductionPath, introduction) diff --git a/src/sync/sync_common.nim b/src/sync/sync_common.nim index d1a4361f..77ede4f5 100644 --- a/src/sync/sync_common.nim +++ b/src/sync/sync_common.nim @@ -25,9 +25,6 @@ proc postHook*(e: ConceptExercise | PracticeExercise) = stderr.writeLine msg quit 1 -func `==`*(x, y: Slug): bool {.borrow.} -func `<`*(x, y: Slug): bool {.borrow.} - func getSlugs*(e: seq[ConceptExercise] | seq[PracticeExercise]): seq[Slug] = ## Returns a seq of the slugs in `e`, in alphabetical order. result = newSeq[Slug](e.len) @@ -35,8 +32,6 @@ func getSlugs*(e: seq[ConceptExercise] | seq[PracticeExercise]): seq[Slug] = result[i] = item.slug sort result -func len*(slug: Slug): int {.borrow.} - func truncateAndAdd*(s: var string, truncateLen: int, slug: Slug) = ## Truncates `s` to `truncateLen`, then appends `slug`. ## diff --git a/src/types_track_config.nim b/src/types_track_config.nim index cf64a06c..62fb8503 100644 --- a/src/types_track_config.nim +++ b/src/types_track_config.nim @@ -1,4 +1,4 @@ -import std/sets +import std/[hashes, sets] import pkg/jsony import "."/[cli, helpers] @@ -58,6 +58,11 @@ type ekPractice = "practice" func `$`*(slug: Slug): string {.borrow.} +func `==`*(x, y: Slug): bool {.borrow.} +func `<`*(x, y: Slug): bool {.borrow.} +func len*(slug: Slug): int {.borrow.} +func hash*(slug: Slug): Hash {.borrow.} +func add*(s: var Slug, c: char) {.borrow.} proc init*(T: typedesc[TrackConfig]; trackConfigContents: string): T = ## Deserializes `trackConfigContents` using `jsony` to a `TrackConfig` object. diff --git a/tests/all_tests.nim b/tests/all_tests.nim index 7adc11ef..cb2bdcfd 100644 --- a/tests/all_tests.nim +++ b/tests/all_tests.nim @@ -1,6 +1,7 @@ import "."/[ test_binary, test_fmt, + test_generate, test_json, test_lint, test_probspecs, diff --git a/tests/test_binary.nim b/tests/test_binary.nim index 41a32a86..50617db2 100644 --- a/tests/test_binary.nim +++ b/tests/test_binary.nim @@ -867,17 +867,16 @@ proc testsForSync(binaryPath: static string) = &"failed to merge '{mainBranchName}' in " & &"problem-specifications directory: '{probSpecsDir}'") -proc prepareIntroductionFiles(trackDir, header, placeholder: string; - removeIntro: bool) = +proc prepareIntroductionFiles(trackDir, placeholder: string; removeIntro: bool) = # Writes an `introduction.md.tpl` file for the `bird-count` Concept Exercise, - # containing the given `header` and `placeholder`. Also removes the - # `introduction.md` file if `removeIntro` is `true`. + # containing the given `placeholder`. Also removes the `introduction.md` file + # if `removeIntro` is `true`. let docsPath = trackDir / "exercises" / "concept" / "bird-count" / ".docs" introPath = docsPath / "introduction.md" templatePath = introPath & ".tpl" templateContents = fmt""" - # {header} + # Introduction {placeholder} """.unindent() @@ -892,7 +891,7 @@ proc testsForGenerate(binaryPath: string) = # Setup: clone a track repo, and checkout a known state setupExercismRepo("elixir", trackDir, - "f3974abf6e0d4a434dfe3494d58581d399c18edb") # 2021-05-09 + "91ccf91940f32aff3726c772695b2de167d8192a") # 2022-06-12 test "`configlet generate` exits with 0 when there are no `.md.tpl` files": execAndCheck(0, generateCmd, "") @@ -901,8 +900,7 @@ proc testsForGenerate(binaryPath: string) = checkNoDiff(trackDir) # Valid placeholder syntax without spaces, and invalid slug - prepareIntroductionFiles(trackDir, "Recursion", - "%{concept:not-a-real-concept-slug}", + prepareIntroductionFiles(trackDir, "%{concept:not-a-real-concept-slug}", removeIntro = false) test "`configlet generate` exits with 1 for an invalid placeholder usage": @@ -912,7 +910,7 @@ proc testsForGenerate(binaryPath: string) = checkNoDiff(trackDir) # Valid placeholder syntax without spaces, and valid slug - prepareIntroductionFiles(trackDir, "Recursion", "%{concept:recursion}", + prepareIntroductionFiles(trackDir, "%{concept:recursion}", removeIntro = true) test "`configlet generate` exits with 0 for a valid `.md.tpl` file": @@ -922,7 +920,7 @@ proc testsForGenerate(binaryPath: string) = checkNoDiff(trackDir) # Valid placeholder syntax with spaces, and valid slug - prepareIntroductionFiles(trackDir, "Recursion", "%{ concept : recursion }", + prepareIntroductionFiles(trackDir, "%{ concept : recursion }", removeIntro = true) test "`configlet generate` exits with 0 for valid placeholder usage with spaces": diff --git a/tests/test_generate.nim b/tests/test_generate.nim new file mode 100644 index 00000000..129319e1 --- /dev/null +++ b/tests/test_generate.nim @@ -0,0 +1,81 @@ +import std/[strutils, unittest] +from "."/generate/generate {.all.} import alterHeadings + +proc testGenerate = + suite "generate": + test "alterHeadings": + const s = """ + # Heading 1 + + The quick brown fox jumps over a lazy dog. + + The five boxing wizards jump quickly. + + ## Heading 2 + + The quick brown fox jumps over a lazy dog. + + + + ### Heading 3 + + The quick brown fox jumps over a lazy dog. + + ```nim + # This line is not a heading + echo "hi" + ``` + + ## Heading 4 + + The quick brown fox jumps over a lazy dog. + + ~~~nim + # This line is not a heading + echo "hi" + ~~~ + """.unindent() + + const expected = """ + The quick brown fox jumps over a lazy dog. + + The five boxing wizards jump quickly. + + ### Heading 2 + + The quick brown fox jumps over a lazy dog. + + + + #### Heading 3 + + The quick brown fox jumps over a lazy dog. + + ```nim + # This line is not a heading + echo "hi" + ``` + + ### Heading 4 + + The quick brown fox jumps over a lazy dog. + + ~~~nim + # This line is not a heading + echo "hi" + ~~~""".unindent() # No final newline + + var linkDefs = newSeq[string]() + check alterHeadings(s, linkDefs) == expected + check alterHeadings(s, linkDefs, "Maps") == "## Maps\n\n" & expected + check linkDefs.len == 0 + +proc main = + testGenerate() + +main() +{.used.}