From a098a9c1f93aa6fe68c54bc3894fb365e2ef9419 Mon Sep 17 00:00:00 2001 From: ee7 <45465154+ee7@users.noreply.github.com> Date: Fri, 17 Jun 2022 17:37:01 +0200 Subject: [PATCH 01/26] generate: demote level of non-first headers To-do: update the integration tests accordingly. Closes: 328 --- src/generate/generate.nim | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/src/generate/generate.nim b/src/generate/generate.nim index 1e5c2c0b..45c99d08 100644 --- a/src/generate/generate.nim +++ b/src/generate/generate.nim @@ -10,20 +10,36 @@ proc writeError(description: string, path: Path) = stderr.writeLine(path) stderr.write "\n" +func demoteHeaders(s: string): string = + ## Demotes the level of any Markdown header in `s` by one. Supports only + ## headers that begin with a `#` character. + # Markdown implementations differ on whether a space is required after the + # final '#' character that begins the header. + result = newStringOfCap(s.len) + var i = 0 + while i < s.len: + result.add s[i] + if s[i] == '\n' and i+1 < s.len and s[i+1] == '#': + result.add '#' + inc i + proc conceptIntroduction(trackDir: Path, slug: string, templatePath: Path): string = - ## Returns the contents of the `introduction.md` file for a `slug`, but - ## without any top-level heading, and without any leading/trailing whitespace. + ## Returns the contents of the `introduction.md` file for a `slug`, but: + ## - Without a first top-level header. + ## - Demoting the level of any other header. + ## - Without any leading/trailing whitespace. let conceptDir = trackDir / "concepts" / slug if dirExists(conceptDir): let path = conceptDir / "introduction.md" if fileExists(path): result = readFile(path) var i = 0 - # Strip the top-level heading (if any) + # Strip the top-level header (if any) if scanp(result, i, *{' ', '\t', '\v', '\c', '\n', '\f'}, "#", +' ', +(~'\n')): result.setSlice(i..result.high) + result = demoteHeaders(result) strip result else: writeError(&"File {path} not found for concept '{slug}'", templatePath) From 8e4bc8aa914e306fe290832534f43fc65709163e Mon Sep 17 00:00:00 2001 From: ee7 <45465154+ee7@users.noreply.github.com> Date: Fri, 17 Jun 2022 17:37:02 +0200 Subject: [PATCH 02/26] generate: refactor top-level header removal --- src/generate/generate.nim | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/src/generate/generate.nim b/src/generate/generate.nim index 45c99d08..3256bb22 100644 --- a/src/generate/generate.nim +++ b/src/generate/generate.nim @@ -1,4 +1,4 @@ -import std/[strbasics, strformat, strscans, terminal] +import std/[parseutils, strbasics, strformat, strscans, terminal] import ".."/[cli, helpers] proc writeError(description: string, path: Path) = @@ -10,18 +10,22 @@ proc writeError(description: string, path: Path) = stderr.writeLine(path) stderr.write "\n" -func demoteHeaders(s: string): string = - ## Demotes the level of any Markdown header in `s` by one. Supports only - ## headers that begin with a `#` character. +func alterHeaders(s: string): string = # Markdown implementations differ on whether a space is required after the # final '#' character that begins the header. result = newStringOfCap(s.len) var i = 0 + i += s.skipWhitespace() + # Skip the top-level header (if any) + if i < s.len and s[i] == '#' and i+1 < s.len and s[i+1] == ' ': + i += s.skipUntil('\n', i) + # Demote other headers while i < s.len: result.add s[i] if s[i] == '\n' and i+1 < s.len and s[i+1] == '#': result.add '#' inc i + strip result proc conceptIntroduction(trackDir: Path, slug: string, templatePath: Path): string = @@ -33,14 +37,7 @@ proc conceptIntroduction(trackDir: Path, slug: string, if dirExists(conceptDir): let path = conceptDir / "introduction.md" if fileExists(path): - result = readFile(path) - var i = 0 - # Strip the top-level header (if any) - if scanp(result, i, *{' ', '\t', '\v', '\c', '\n', '\f'}, "#", +' ', - +(~'\n')): - result.setSlice(i..result.high) - result = demoteHeaders(result) - strip result + result = path.readFile().alterHeaders() else: writeError(&"File {path} not found for concept '{slug}'", templatePath) quit(1) From b62775e31be49309eb370a1a9e3b11f3630444e0 Mon Sep 17 00:00:00 2001 From: ee7 <45465154+ee7@users.noreply.github.com> Date: Fri, 17 Jun 2022 17:37:03 +0200 Subject: [PATCH 03/26] tests: generate: add test for header processing --- tests/all_tests.nim | 1 + tests/test_generate.nim | 62 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+) create mode 100644 tests/test_generate.nim diff --git a/tests/all_tests.nim b/tests/all_tests.nim index 7adc11ef..cb2bdcfd 100644 --- a/tests/all_tests.nim +++ b/tests/all_tests.nim @@ -1,6 +1,7 @@ import "."/[ test_binary, test_fmt, + test_generate, test_json, test_lint, test_probspecs, diff --git a/tests/test_generate.nim b/tests/test_generate.nim new file mode 100644 index 00000000..45d990fb --- /dev/null +++ b/tests/test_generate.nim @@ -0,0 +1,62 @@ +import std/[strutils, unittest] +from "."/generate/generate {.all.} import alterHeaders + +proc testGenerate = + suite "generate": + test "alterHeaders": + const s = """ + # Header 1 + + The quick brown fox jumps over a lazy dog. + + The five boxing wizards jump quickly. + + ## Header 2 + + The quick brown fox jumps over a lazy dog. + + The five boxing wizards jump quickly. + + ### Header 3 + + The quick brown fox jumps over a lazy dog. + + The five boxing wizards jump quickly. + + ## Header 4 + + The quick brown fox jumps over a lazy dog. + + The five boxing wizards jump quickly. + """.unindent() + + const expected = """ + The quick brown fox jumps over a lazy dog. + + The five boxing wizards jump quickly. + + ### Header 2 + + The quick brown fox jumps over a lazy dog. + + The five boxing wizards jump quickly. + + #### Header 3 + + The quick brown fox jumps over a lazy dog. + + The five boxing wizards jump quickly. + + ### Header 4 + + The quick brown fox jumps over a lazy dog. + + The five boxing wizards jump quickly.""".unindent() # No final newline + + check alterHeaders(s) == expected + +proc main = + testGenerate() + +main() +{.used.} From 528461562bc17c4579938cfe87e1cac42b173368 Mon Sep 17 00:00:00 2001 From: ee7 <45465154+ee7@users.noreply.github.com> Date: Sun, 19 Jun 2022 12:12:01 +0200 Subject: [PATCH 04/26] generate: don't alter `#` line inside fenced code block Previously, `configlet generate` assumed that every line beginning with a '#' character inside a concept `introduction.md` is a header. For our purposes, the most common exception to that idea is probably a comment inside a fenced code block. With this commit, we no longer add a '#' to such a line. The markdown handling here is rudimentary, but it may be good enough to avoid a full markdown parser for `configlet generate`. But this doesn't support other blocks, like HTML blocks. --- src/generate/generate.nim | 11 +++++++++-- tests/test_generate.nim | 10 ++++++++-- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/generate/generate.nim b/src/generate/generate.nim index 3256bb22..3aee85a3 100644 --- a/src/generate/generate.nim +++ b/src/generate/generate.nim @@ -20,10 +20,17 @@ func alterHeaders(s: string): string = if i < s.len and s[i] == '#' and i+1 < s.len and s[i+1] == ' ': i += s.skipUntil('\n', i) # Demote other headers + var inFencedCodeBlock = false while i < s.len: result.add s[i] - if s[i] == '\n' and i+1 < s.len and s[i+1] == '#': - result.add '#' + if s[i] == '\n': + # When inside a fenced code block, don't alter a line that begins with '#' + if i+1 < s.len and s[i+1] == '#' and not inFencedCodeBlock: + result.add '#' + elif i+3 < s.len and s[i+1] == '`' and s[i+2] == '`' and s[i+3] == '`': + inFencedCodeBlock = not inFencedCodeBlock + result.add "```" + i += 3 inc i strip result diff --git a/tests/test_generate.nim b/tests/test_generate.nim index 45d990fb..1ecae3f0 100644 --- a/tests/test_generate.nim +++ b/tests/test_generate.nim @@ -21,7 +21,10 @@ proc testGenerate = The quick brown fox jumps over a lazy dog. - The five boxing wizards jump quickly. + ```nim + # This line is not a header + echo "hi" + ``` ## Header 4 @@ -45,7 +48,10 @@ proc testGenerate = The quick brown fox jumps over a lazy dog. - The five boxing wizards jump quickly. + ```nim + # This line is not a header + echo "hi" + ``` ### Header 4 From cb4788687d771e895f1c5a668a68eaf08e38b350 Mon Sep 17 00:00:00 2001 From: ee7 <45465154+ee7@users.noreply.github.com> Date: Sun, 19 Jun 2022 15:15:01 +0200 Subject: [PATCH 05/26] generate: add starting level-2 heading with concept's name Also update the tests to assert that `configlet generate` makes no changes to the current state of the elixir track repo. In a later PR we'll update those tests to assert the same for `configlet generate -uy`. --- src/generate/generate.nim | 28 ++++++++++++++++++++-------- tests/test_binary.nim | 18 ++++++++---------- tests/test_generate.nim | 4 +++- 3 files changed, 31 insertions(+), 19 deletions(-) diff --git a/src/generate/generate.nim b/src/generate/generate.nim index 3aee85a3..322cc063 100644 --- a/src/generate/generate.nim +++ b/src/generate/generate.nim @@ -1,5 +1,11 @@ -import std/[parseutils, strbasics, strformat, strscans, terminal] -import ".."/[cli, helpers] +import std/[parseutils, strbasics, strformat, strscans, tables, terminal] +import ".."/[cli, helpers, types_track_config] + +proc getSlugLookup(trackDir: Path): Table[string, string] = + let concepts = TrackConfig.init(readFile(trackDir / "config.json")).concepts + result = initTable[string, string](concepts.len) + for `concept` in concepts: + result[`concept`.slug] = `concept`.name proc writeError(description: string, path: Path) = let descriptionPrefix = description & ":" @@ -10,7 +16,7 @@ proc writeError(description: string, path: Path) = stderr.writeLine(path) stderr.write "\n" -func alterHeaders(s: string): string = +func alterHeaders(s: string, title: string): string = # Markdown implementations differ on whether a space is required after the # final '#' character that begins the header. result = newStringOfCap(s.len) @@ -19,6 +25,7 @@ func alterHeaders(s: string): string = # Skip the top-level header (if any) if i < s.len and s[i] == '#' and i+1 < s.len and s[i+1] == ' ': i += s.skipUntil('\n', i) + result.add &"## {title}" # Demote other headers var inFencedCodeBlock = false while i < s.len: @@ -34,17 +41,18 @@ func alterHeaders(s: string): string = inc i strip result -proc conceptIntroduction(trackDir: Path, slug: string, +proc conceptIntroduction(trackDir: Path, slug: string, title: string, templatePath: Path): string = ## Returns the contents of the `introduction.md` file for a `slug`, but: ## - Without a first top-level header. + ## - Adding a starting a second-level header containing `title`. ## - Demoting the level of any other header. ## - Without any leading/trailing whitespace. let conceptDir = trackDir / "concepts" / slug if dirExists(conceptDir): let path = conceptDir / "introduction.md" if fileExists(path): - result = path.readFile().alterHeaders() + result = path.readFile().alterHeaders(title) else: writeError(&"File {path} not found for concept '{slug}'", templatePath) quit(1) @@ -53,7 +61,8 @@ proc conceptIntroduction(trackDir: Path, slug: string, templatePath) quit(1) -proc generateIntroduction(trackDir: Path, templatePath: Path): string = +proc generateIntroduction(trackDir: Path, templatePath: Path, + slugLookup: Table[string, string]): string = ## Reads the file at `templatePath` and returns the content of the ## corresponding `introduction.md` file. let content = readFile(templatePath) @@ -68,7 +77,8 @@ proc generateIntroduction(trackDir: Path, templatePath: Path): string = if scanp(content, i, "%{", *{' '}, "concept", *{' '}, ':', *{' '}, +{'a'..'z', '-'} -> conceptSlug.add($_), *{' '}, '}'): - result.add conceptIntroduction(trackDir, conceptSlug, templatePath) + let title = slugLookup[conceptSlug] + result.add conceptIntroduction(trackDir, conceptSlug, title, templatePath) else: result.add content[i] inc i @@ -80,9 +90,11 @@ proc generate*(conf: Conf) = let conceptExercisesDir = trackDir / "exercises" / "concept" if dirExists(conceptExercisesDir): + let slugLookup = getSlugLookup(trackDir) for conceptExerciseDir in getSortedSubdirs(conceptExercisesDir): let introductionTemplatePath = conceptExerciseDir / ".docs" / "introduction.md.tpl" if fileExists(introductionTemplatePath): - let introduction = generateIntroduction(trackDir, introductionTemplatePath) + let introduction = generateIntroduction(trackDir, introductionTemplatePath, + slugLookup) let introductionPath = introductionTemplatePath.string[0..^5] # Removes `.tpl` writeFile(introductionPath, introduction) diff --git a/tests/test_binary.nim b/tests/test_binary.nim index 926bb612..3012b605 100644 --- a/tests/test_binary.nim +++ b/tests/test_binary.nim @@ -872,17 +872,16 @@ proc testsForSync(binaryPath: static string) = &"failed to merge '{mainBranchName}' in " & &"problem-specifications directory: '{probSpecsDir}'") -proc prepareIntroductionFiles(trackDir, header, placeholder: string; - removeIntro: bool) = +proc prepareIntroductionFiles(trackDir, placeholder: string; removeIntro: bool) = # Writes an `introduction.md.tpl` file for the `bird-count` Concept Exercise, - # containing the given `header` and `placeholder`. Also removes the - # `introduction.md` file if `removeIntro` is `true`. + # containing the given `placeholder`. Also removes the `introduction.md` file + # if `removeIntro` is `true`. let docsPath = trackDir / "exercises" / "concept" / "bird-count" / ".docs" introPath = docsPath / "introduction.md" templatePath = introPath & ".tpl" templateContents = fmt""" - # {header} + # Introduction {placeholder} """.unindent() @@ -897,7 +896,7 @@ proc testsForGenerate(binaryPath: string) = # Setup: clone a track repo, and checkout a known state setupExercismRepo("elixir", trackDir, - "f3974abf6e0d4a434dfe3494d58581d399c18edb") # 2021-05-09 + "91ccf91940f32aff3726c772695b2de167d8192a") # 2022-06-12 test "`configlet generate` exits with 0 when there are no `.md.tpl` files": execAndCheck(0, generateCmd, "") @@ -906,8 +905,7 @@ proc testsForGenerate(binaryPath: string) = checkNoDiff(trackDir) # Valid placeholder syntax without spaces, and invalid slug - prepareIntroductionFiles(trackDir, "Recursion", - "%{concept:not-a-real-concept-slug}", + prepareIntroductionFiles(trackDir, "%{concept:not-a-real-concept-slug}", removeIntro = false) test "`configlet generate` exits with 1 for an invalid placeholder usage": @@ -917,7 +915,7 @@ proc testsForGenerate(binaryPath: string) = checkNoDiff(trackDir) # Valid placeholder syntax without spaces, and valid slug - prepareIntroductionFiles(trackDir, "Recursion", "%{concept:recursion}", + prepareIntroductionFiles(trackDir, "%{concept:recursion}", removeIntro = true) test "`configlet generate` exits with 0 for a valid `.md.tpl` file": @@ -927,7 +925,7 @@ proc testsForGenerate(binaryPath: string) = checkNoDiff(trackDir) # Valid placeholder syntax with spaces, and valid slug - prepareIntroductionFiles(trackDir, "Recursion", "%{ concept : recursion }", + prepareIntroductionFiles(trackDir, "%{ concept : recursion }", removeIntro = true) test "`configlet generate` exits with 0 for valid placeholder usage with spaces": diff --git a/tests/test_generate.nim b/tests/test_generate.nim index 1ecae3f0..719f2f72 100644 --- a/tests/test_generate.nim +++ b/tests/test_generate.nim @@ -34,6 +34,8 @@ proc testGenerate = """.unindent() const expected = """ + ## Operator Overloading + The quick brown fox jumps over a lazy dog. The five boxing wizards jump quickly. @@ -59,7 +61,7 @@ proc testGenerate = The five boxing wizards jump quickly.""".unindent() # No final newline - check alterHeaders(s) == expected + check alterHeaders(s, "Operator Overloading") == expected proc main = testGenerate() From 2f6506013518bab6b6a15b85b9a31dda60318c7f Mon Sep 17 00:00:00 2001 From: ee7 <45465154+ee7@users.noreply.github.com> Date: Wed, 22 Jun 2022 20:11:01 +0200 Subject: [PATCH 06/26] generate: simplify This is arguable. But I think this way is more readable overall by the time we add handling of HTML comment blocks. --- src/generate/generate.nim | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/generate/generate.nim b/src/generate/generate.nim index 322cc063..a02a9b6d 100644 --- a/src/generate/generate.nim +++ b/src/generate/generate.nim @@ -36,8 +36,6 @@ func alterHeaders(s: string, title: string): string = result.add '#' elif i+3 < s.len and s[i+1] == '`' and s[i+2] == '`' and s[i+3] == '`': inFencedCodeBlock = not inFencedCodeBlock - result.add "```" - i += 3 inc i strip result From 118fd2727d6c46bf51797c93cbf2e213dd730c5b Mon Sep 17 00:00:00 2001 From: ee7 <45465154+ee7@users.noreply.github.com> Date: Wed, 22 Jun 2022 20:11:02 +0200 Subject: [PATCH 07/26] generate: refactor via `continuesWith` More readable, and easier to extend to support HTML comment blocks, but probably marginally slower. --- src/generate/generate.nim | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/generate/generate.nim b/src/generate/generate.nim index a02a9b6d..5619b31f 100644 --- a/src/generate/generate.nim +++ b/src/generate/generate.nim @@ -1,4 +1,4 @@ -import std/[parseutils, strbasics, strformat, strscans, tables, terminal] +import std/[parseutils, strbasics, strformat, strscans, strutils, tables, terminal] import ".."/[cli, helpers, types_track_config] proc getSlugLookup(trackDir: Path): Table[string, string] = @@ -23,7 +23,7 @@ func alterHeaders(s: string, title: string): string = var i = 0 i += s.skipWhitespace() # Skip the top-level header (if any) - if i < s.len and s[i] == '#' and i+1 < s.len and s[i+1] == ' ': + if s.continuesWith("# ", i): i += s.skipUntil('\n', i) result.add &"## {title}" # Demote other headers @@ -32,9 +32,9 @@ func alterHeaders(s: string, title: string): string = result.add s[i] if s[i] == '\n': # When inside a fenced code block, don't alter a line that begins with '#' - if i+1 < s.len and s[i+1] == '#' and not inFencedCodeBlock: + if s.continuesWith("#", i+1) and not inFencedCodeBlock: result.add '#' - elif i+3 < s.len and s[i+1] == '`' and s[i+2] == '`' and s[i+3] == '`': + elif s.continuesWith("```", i+1): inFencedCodeBlock = not inFencedCodeBlock inc i strip result From da1d6951e54cf2c4a8b1400ec68f5ff4e6bc5ad2 Mon Sep 17 00:00:00 2001 From: ee7 <45465154+ee7@users.noreply.github.com> Date: Wed, 22 Jun 2022 20:11:03 +0200 Subject: [PATCH 08/26] generate: support HTML comment block This is only one of many possible HTML blocks [1], but probably the most important one. [1] https://spec.commonmark.org/0.30/#html-blocks --- src/generate/generate.nim | 9 +++++++-- tests/test_generate.nim | 8 ++++++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/generate/generate.nim b/src/generate/generate.nim index 5619b31f..5b537cb5 100644 --- a/src/generate/generate.nim +++ b/src/generate/generate.nim @@ -28,14 +28,19 @@ func alterHeaders(s: string, title: string): string = result.add &"## {title}" # Demote other headers var inFencedCodeBlock = false + var inCommentBlock = false while i < s.len: result.add s[i] if s[i] == '\n': - # When inside a fenced code block, don't alter a line that begins with '#' - if s.continuesWith("#", i+1) and not inFencedCodeBlock: + # Add a '#' to a line that begins with '#', unless inside a code or HTML block. + if s.continuesWith("#", i+1) and not (inFencedCodeBlock or inCommentBlock): result.add '#' elif s.continuesWith("```", i+1): inFencedCodeBlock = not inFencedCodeBlock + elif s.continuesWith("", i): + inCommentBlock = false inc i strip result diff --git a/tests/test_generate.nim b/tests/test_generate.nim index 719f2f72..f3e5eee9 100644 --- a/tests/test_generate.nim +++ b/tests/test_generate.nim @@ -15,7 +15,9 @@ proc testGenerate = The quick brown fox jumps over a lazy dog. - The five boxing wizards jump quickly. + ### Header 3 @@ -44,7 +46,9 @@ proc testGenerate = The quick brown fox jumps over a lazy dog. - The five boxing wizards jump quickly. + #### Header 3 From 3a514c27f947f3bc5396200853e259f9ef3fdc00 Mon Sep 17 00:00:00 2001 From: ee7 <45465154+ee7@users.noreply.github.com> Date: Wed, 22 Jun 2022 20:11:04 +0200 Subject: [PATCH 09/26] generate: don't insert h2 when under h2; demote dynamically Insert a h2 using the concept's name in the track `config.json`, and demote the headers in the `introduction.md` for the `foo` concept by 1: # Introduction %{concept:foo} Don't insert a h2, and demote by 1: # Introduction ## Foo %{concept:foo} Don't insert a h2, and demote by 2: # Introduction ## Some header Blah blah. ### Foo %{concept:foo} --- src/generate/generate.nim | 19 +++++++++++++------ tests/test_generate.nim | 8 +++++--- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/src/generate/generate.nim b/src/generate/generate.nim index 5b537cb5..43945b3e 100644 --- a/src/generate/generate.nim +++ b/src/generate/generate.nim @@ -16,7 +16,7 @@ proc writeError(description: string, path: Path) = stderr.writeLine(path) stderr.write "\n" -func alterHeaders(s: string, title: string): string = +func alterHeaders(s: string, title: string, headerLevel: int): string = # Markdown implementations differ on whether a space is required after the # final '#' character that begins the header. result = newStringOfCap(s.len) @@ -25,7 +25,8 @@ func alterHeaders(s: string, title: string): string = # Skip the top-level header (if any) if s.continuesWith("# ", i): i += s.skipUntil('\n', i) - result.add &"## {title}" + if headerLevel == 1: + result.add &"## {title}" # Demote other headers var inFencedCodeBlock = false var inCommentBlock = false @@ -34,7 +35,9 @@ func alterHeaders(s: string, title: string): string = if s[i] == '\n': # Add a '#' to a line that begins with '#', unless inside a code or HTML block. if s.continuesWith("#", i+1) and not (inFencedCodeBlock or inCommentBlock): - result.add '#' + let demotionAmount = if headerLevel in [1, 2]: 1 else: headerLevel - 1 + for _ in 1..demotionAmount: + result.add '#' elif s.continuesWith("```", i+1): inFencedCodeBlock = not inFencedCodeBlock elif s.continuesWith("", i): diff --git a/tests/test_generate.nim b/tests/test_generate.nim index b117559e..ed10371b 100644 --- a/tests/test_generate.nim +++ b/tests/test_generate.nim @@ -32,7 +32,10 @@ proc testGenerate = The quick brown fox jumps over a lazy dog. - The five boxing wizards jump quickly. + ~~~nim + # This line is not a header + echo "hi" + ~~~ """.unindent() const expected = """ @@ -61,7 +64,10 @@ proc testGenerate = The quick brown fox jumps over a lazy dog. - The five boxing wizards jump quickly.""".unindent() # No final newline + ~~~nim + # This line is not a header + echo "hi" + ~~~""".unindent() # No final newline check alterHeaders(s, "Maps", 1) == "## Maps\n\n" & expected check alterHeaders(s, "Maps", 2) == expected From 42a8d1481dff865e9736d16d00de9df9311c24d6 Mon Sep 17 00:00:00 2001 From: ee7 <45465154+ee7@users.noreply.github.com> Date: Fri, 24 Jun 2022 15:33:03 +0200 Subject: [PATCH 13/26] generate: put all reference links at the bottom Consider this `.md.tpl` file: # Introduction ## Foo %{concept:foo} ## Bar %{concept:bar} Before this commit, when more than one placeholder had a reference link, `configlet generate` would produce something like # Introduction ## Foo Here is a line with a link [Foo][foo]. [foo]: http://www.example.com ## Bar Here is another line with a link [Bar][bar]. [bar]: http://www.example.com With this commit, we place the reference links at the bottom, ordering by first usage, and deduplicating them: # Introduction ## Foo Here is a line with a link [Foo][foo]. ## Bar Here is another line with a link [Bar][bar]. [foo]: http://www.example.com [bar]: http://www.example.com But it's tricky to do this robustly without using a proper markdown parser. From the CommonMark Spec [1], the rules for reference links have some complexity: A link reference definition consists of a link label, optionally preceded by up to three spaces of indentation, followed by a colon (:), optional spaces or tabs (including up to one line ending), a link destination, optional spaces or tabs (including up to one line ending), and an optional link title, which if it is present must be separated from the link destination by spaces or tabs. No further character may occur. --- A link label begins with a left bracket ([) and ends with the first right bracket (]) that is not backslash-escaped. Between these brackets there must be at least one character that is not a space, tab, or line ending. Unescaped square bracket characters are not allowed inside the opening and closing square brackets of link labels. A link label can have at most 999 characters inside the square brackets. [1] https://spec.commonmark.org/0.30/#link-reference-definitions --- src/generate/generate.nim | 26 ++++++++++++++++++++++---- tests/test_generate.nim | 10 ++++++---- 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/src/generate/generate.nim b/src/generate/generate.nim index 6c90f708..67a829df 100644 --- a/src/generate/generate.nim +++ b/src/generate/generate.nim @@ -16,7 +16,8 @@ proc writeError(description: string, path: Path) = stderr.writeLine(path) stderr.write "\n" -func alterHeaders(s: string, title: string, headerLevel: int): string = +func alterHeaders(s: string, title: string, headerLevel: int, + links: var seq[string]): string = # Markdown implementations differ on whether a space is required after the # final '#' character that begins the header. result = newStringOfCap(s.len) @@ -40,6 +41,13 @@ func alterHeaders(s: string, title: string, headerLevel: int): string = let demotionAmount = if headerLevel in [1, 2]: 1 else: headerLevel - 1 for _ in 1..demotionAmount: result.add '#' + elif s.continuesWith("[", i+1): + let j = s.find("]:", i+2) + if j > i+2 and j < s.find('\n', i+2): + var line = "" + i += s.parseUntil(line, '\n', i+1) + if line notin links: + links.add line elif s.continuesWith("```", i+1): inFencedCodeBlock = not inFencedCodeBlock elif s.continuesWith("~~~", i+1): @@ -52,17 +60,21 @@ func alterHeaders(s: string, title: string, headerLevel: int): string = strip result proc conceptIntroduction(trackDir: Path, slug: string, title: string, - templatePath: Path, headerLevel: int): string = + templatePath: Path, headerLevel: int, + links: var seq[string]): string = ## Returns the contents of the `introduction.md` file for a `slug`, but: ## - Without a first top-level header. ## - Adding a starting a second-level header containing `title`. ## - Demoting the level of any other header. ## - Without any leading/trailing whitespace. + ## - Without any reference links. + ## + ## Appends reference links to `links`. let conceptDir = trackDir / "concepts" / slug if dirExists(conceptDir): let path = conceptDir / "introduction.md" if fileExists(path): - result = path.readFile().alterHeaders(title, headerLevel) + result = path.readFile().alterHeaders(title, headerLevel, links) else: writeError(&"File {path} not found for concept '{slug}'", templatePath) quit(1) @@ -80,6 +92,7 @@ proc generateIntroduction(trackDir: Path, templatePath: Path, var i = 0 var headerLevel = 1 + var links = newSeq[string]() while i < content.len: var conceptSlug = "" # Here, we implement the syntax for a placeholder as %{concept:some-slug} @@ -91,7 +104,7 @@ proc generateIntroduction(trackDir: Path, templatePath: Path, if conceptSlug in slugLookup: let title = slugLookup[conceptSlug] result.add conceptIntroduction(trackDir, conceptSlug, title, - templatePath, headerLevel) + templatePath, headerLevel, links) else: writeError(&"Concept '{conceptSlug}' does not exist in track config.json", templatePath) @@ -101,6 +114,11 @@ proc generateIntroduction(trackDir: Path, templatePath: Path, headerLevel = content.skipWhile({'#'}, i+1) result.add content[i] inc i + if links.len > 0: + result.add '\n' + for link in links: + result.add link + result.add '\n' proc generate*(conf: Conf) = ## For every Concept Exercise in `conf.trackDir` with an `introduction.md.tpl` diff --git a/tests/test_generate.nim b/tests/test_generate.nim index ed10371b..8c18a604 100644 --- a/tests/test_generate.nim +++ b/tests/test_generate.nim @@ -69,10 +69,12 @@ proc testGenerate = echo "hi" ~~~""".unindent() # No final newline - check alterHeaders(s, "Maps", 1) == "## Maps\n\n" & expected - check alterHeaders(s, "Maps", 2) == expected - check alterHeaders(s, "Maps", 3) == expected.replace("### ", "#### ") - check alterHeaders(s, "Maps", 4) == expected.replace("### ", "##### ") + var links = newSeq[string]() + check alterHeaders(s, "Maps", 1, links) == "## Maps\n\n" & expected + check alterHeaders(s, "Maps", 2, links) == expected + check alterHeaders(s, "Maps", 3, links) == expected.replace("### ", "#### ") + check alterHeaders(s, "Maps", 4, links) == expected.replace("### ", "##### ") + check links.len == 0 proc main = testGenerate() From d1a1ab1b7859ed4b726f3c6c052dd2e10f93eb84 Mon Sep 17 00:00:00 2001 From: ee7 <45465154+ee7@users.noreply.github.com> Date: Tue, 28 Jun 2022 13:25:01 +0200 Subject: [PATCH 14/26] generate: rename `getSlugLookup` proc Clarify the kind of slug. In the future we could even consider: type ConceptSlug* = distinct string ExerciseSlug* = distinct string --- src/generate/generate.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/generate/generate.nim b/src/generate/generate.nim index 67a829df..60e9a514 100644 --- a/src/generate/generate.nim +++ b/src/generate/generate.nim @@ -1,7 +1,7 @@ import std/[parseutils, strbasics, strformat, strscans, strutils, tables, terminal] import ".."/[cli, helpers, types_track_config] -proc getSlugLookup(trackDir: Path): Table[string, string] = +proc getConceptSlugLookup(trackDir: Path): Table[string, string] = let concepts = TrackConfig.init(readFile(trackDir / "config.json")).concepts result = initTable[string, string](concepts.len) for `concept` in concepts: @@ -127,7 +127,7 @@ proc generate*(conf: Conf) = let conceptExercisesDir = trackDir / "exercises" / "concept" if dirExists(conceptExercisesDir): - let slugLookup = getSlugLookup(trackDir) + let slugLookup = getConceptSlugLookup(trackDir) for conceptExerciseDir in getSortedSubdirs(conceptExercisesDir): let introductionTemplatePath = conceptExerciseDir / ".docs" / "introduction.md.tpl" if fileExists(introductionTemplatePath): From 394221c3326366bbc3e1bac262d92aaba5df6e42 Mon Sep 17 00:00:00 2001 From: ee7 <45465154+ee7@users.noreply.github.com> Date: Tue, 28 Jun 2022 13:25:02 +0200 Subject: [PATCH 15/26] generate: rename `header` to `heading` The CommonMark spec [1] calls them headings, not headers. [1] https://spec.commonmark.org/0.30/ --- src/generate/generate.nim | 30 +++++++++++++++--------------- tests/test_generate.nim | 38 +++++++++++++++++++------------------- 2 files changed, 34 insertions(+), 34 deletions(-) diff --git a/src/generate/generate.nim b/src/generate/generate.nim index 60e9a514..1b86a639 100644 --- a/src/generate/generate.nim +++ b/src/generate/generate.nim @@ -16,19 +16,19 @@ proc writeError(description: string, path: Path) = stderr.writeLine(path) stderr.write "\n" -func alterHeaders(s: string, title: string, headerLevel: int, - links: var seq[string]): string = +func alterHeadings(s: string, title: string, headingLevel: int, + links: var seq[string]): string = # Markdown implementations differ on whether a space is required after the - # final '#' character that begins the header. + # final '#' character that begins the heading. result = newStringOfCap(s.len) var i = 0 i += s.skipWhitespace() - # Skip the top-level header (if any) + # Skip the top-level heading (if any) if s.continuesWith("# ", i): i += s.skipUntil('\n', i) - if headerLevel == 1: + if headingLevel == 1: result.add &"## {title}" - # Demote other headers + # Demote other headings var inFencedCodeBlock = false var inFencedCodeBlockTildes = false var inCommentBlock = false @@ -38,7 +38,7 @@ func alterHeaders(s: string, title: string, headerLevel: int, # Add a '#' to a line that begins with '#', unless inside a code or HTML block. if s.continuesWith("#", i+1) and not (inFencedCodeBlock or inFencedCodeBlockTildes or inCommentBlock): - let demotionAmount = if headerLevel in [1, 2]: 1 else: headerLevel - 1 + let demotionAmount = if headingLevel in [1, 2]: 1 else: headingLevel - 1 for _ in 1..demotionAmount: result.add '#' elif s.continuesWith("[", i+1): @@ -60,12 +60,12 @@ func alterHeaders(s: string, title: string, headerLevel: int, strip result proc conceptIntroduction(trackDir: Path, slug: string, title: string, - templatePath: Path, headerLevel: int, + templatePath: Path, headingLevel: int, links: var seq[string]): string = ## Returns the contents of the `introduction.md` file for a `slug`, but: - ## - Without a first top-level header. - ## - Adding a starting a second-level header containing `title`. - ## - Demoting the level of any other header. + ## - Without a first top-level heading. + ## - Adding a starting a second-level heading containing `title`. + ## - Demoting the level of any other heading. ## - Without any leading/trailing whitespace. ## - Without any reference links. ## @@ -74,7 +74,7 @@ proc conceptIntroduction(trackDir: Path, slug: string, title: string, if dirExists(conceptDir): let path = conceptDir / "introduction.md" if fileExists(path): - result = path.readFile().alterHeaders(title, headerLevel, links) + result = path.readFile().alterHeadings(title, headingLevel, links) else: writeError(&"File {path} not found for concept '{slug}'", templatePath) quit(1) @@ -91,7 +91,7 @@ proc generateIntroduction(trackDir: Path, templatePath: Path, result = newStringOfCap(1024) var i = 0 - var headerLevel = 1 + var headingLevel = 1 var links = newSeq[string]() while i < content.len: var conceptSlug = "" @@ -104,14 +104,14 @@ proc generateIntroduction(trackDir: Path, templatePath: Path, if conceptSlug in slugLookup: let title = slugLookup[conceptSlug] result.add conceptIntroduction(trackDir, conceptSlug, title, - templatePath, headerLevel, links) + templatePath, headingLevel, links) else: writeError(&"Concept '{conceptSlug}' does not exist in track config.json", templatePath) quit(1) else: if content.continuesWith("\n#", i): - headerLevel = content.skipWhile({'#'}, i+1) + headingLevel = content.skipWhile({'#'}, i+1) result.add content[i] inc i if links.len > 0: diff --git a/tests/test_generate.nim b/tests/test_generate.nim index 8c18a604..ccb1620b 100644 --- a/tests/test_generate.nim +++ b/tests/test_generate.nim @@ -1,39 +1,39 @@ import std/[strutils, unittest] -from "."/generate/generate {.all.} import alterHeaders +from "."/generate/generate {.all.} import alterHeadings proc testGenerate = suite "generate": - test "alterHeaders": + test "alterHeadings": const s = """ - # Header 1 + # Heading 1 The quick brown fox jumps over a lazy dog. The five boxing wizards jump quickly. - ## Header 2 + ## Heading 2 The quick brown fox jumps over a lazy dog. - ### Header 3 + ### Heading 3 The quick brown fox jumps over a lazy dog. ```nim - # This line is not a header + # This line is not a heading echo "hi" ``` - ## Header 4 + ## Heading 4 The quick brown fox jumps over a lazy dog. ~~~nim - # This line is not a header + # This line is not a heading echo "hi" ~~~ """.unindent() @@ -43,37 +43,37 @@ proc testGenerate = The five boxing wizards jump quickly. - ### Header 2 + ### Heading 2 The quick brown fox jumps over a lazy dog. - #### Header 3 + #### Heading 3 The quick brown fox jumps over a lazy dog. ```nim - # This line is not a header + # This line is not a heading echo "hi" ``` - ### Header 4 + ### Heading 4 The quick brown fox jumps over a lazy dog. ~~~nim - # This line is not a header + # This line is not a heading echo "hi" ~~~""".unindent() # No final newline var links = newSeq[string]() - check alterHeaders(s, "Maps", 1, links) == "## Maps\n\n" & expected - check alterHeaders(s, "Maps", 2, links) == expected - check alterHeaders(s, "Maps", 3, links) == expected.replace("### ", "#### ") - check alterHeaders(s, "Maps", 4, links) == expected.replace("### ", "##### ") + check alterHeadings(s, "Maps", 1, links) == "## Maps\n\n" & expected + check alterHeadings(s, "Maps", 2, links) == expected + check alterHeadings(s, "Maps", 3, links) == expected.replace("### ", "#### ") + check alterHeadings(s, "Maps", 4, links) == expected.replace("### ", "##### ") check links.len == 0 proc main = From 3b0afc211dd206be0bdcfe469725c13a1fd62370 Mon Sep 17 00:00:00 2001 From: ee7 <45465154+ee7@users.noreply.github.com> Date: Tue, 28 Jun 2022 13:25:03 +0200 Subject: [PATCH 16/26] generate: move `writeError` proc down Make declaration order match usage order. --- src/generate/generate.nim | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/generate/generate.nim b/src/generate/generate.nim index 1b86a639..c5b9db31 100644 --- a/src/generate/generate.nim +++ b/src/generate/generate.nim @@ -7,15 +7,6 @@ proc getConceptSlugLookup(trackDir: Path): Table[string, string] = for `concept` in concepts: result[`concept`.slug] = `concept`.name -proc writeError(description: string, path: Path) = - let descriptionPrefix = description & ":" - if colorStderr: - stderr.styledWriteLine(fgRed, descriptionPrefix) - else: - stderr.writeLine(descriptionPrefix) - stderr.writeLine(path) - stderr.write "\n" - func alterHeadings(s: string, title: string, headingLevel: int, links: var seq[string]): string = # Markdown implementations differ on whether a space is required after the @@ -59,6 +50,15 @@ func alterHeadings(s: string, title: string, headingLevel: int, inc i strip result +proc writeError(description: string, path: Path) = + let descriptionPrefix = description & ":" + if colorStderr: + stderr.styledWriteLine(fgRed, descriptionPrefix) + else: + stderr.writeLine(descriptionPrefix) + stderr.writeLine(path) + stderr.write "\n" + proc conceptIntroduction(trackDir: Path, slug: string, title: string, templatePath: Path, headingLevel: int, links: var seq[string]): string = From 0d8845278e7e908a758df7a6bb540fca144e14a5 Mon Sep 17 00:00:00 2001 From: ee7 <45465154+ee7@users.noreply.github.com> Date: Tue, 28 Jun 2022 13:25:04 +0200 Subject: [PATCH 17/26] generate: add doc comment --- src/generate/generate.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/src/generate/generate.nim b/src/generate/generate.nim index c5b9db31..fe4ac83b 100644 --- a/src/generate/generate.nim +++ b/src/generate/generate.nim @@ -2,6 +2,7 @@ import std/[parseutils, strbasics, strformat, strscans, strutils, tables, termin import ".."/[cli, helpers, types_track_config] proc getConceptSlugLookup(trackDir: Path): Table[string, string] = + ## Returns a `Table` that maps each concept's `slug` to its `name`. let concepts = TrackConfig.init(readFile(trackDir / "config.json")).concepts result = initTable[string, string](concepts.len) for `concept` in concepts: From 94878f014a27288a6cc4cb8e52deae04934d8b89 Mon Sep 17 00:00:00 2001 From: ee7 <45465154+ee7@users.noreply.github.com> Date: Tue, 28 Jun 2022 13:25:05 +0200 Subject: [PATCH 18/26] generate: rename `links` to `linkDefs` --- src/generate/generate.nim | 24 ++++++++++++------------ tests/test_generate.nim | 12 ++++++------ 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/generate/generate.nim b/src/generate/generate.nim index fe4ac83b..00f044ed 100644 --- a/src/generate/generate.nim +++ b/src/generate/generate.nim @@ -9,7 +9,7 @@ proc getConceptSlugLookup(trackDir: Path): Table[string, string] = result[`concept`.slug] = `concept`.name func alterHeadings(s: string, title: string, headingLevel: int, - links: var seq[string]): string = + linkDefs: var seq[string]): string = # Markdown implementations differ on whether a space is required after the # final '#' character that begins the heading. result = newStringOfCap(s.len) @@ -38,8 +38,8 @@ func alterHeadings(s: string, title: string, headingLevel: int, if j > i+2 and j < s.find('\n', i+2): var line = "" i += s.parseUntil(line, '\n', i+1) - if line notin links: - links.add line + if line notin linkDefs: + linkDefs.add line elif s.continuesWith("```", i+1): inFencedCodeBlock = not inFencedCodeBlock elif s.continuesWith("~~~", i+1): @@ -62,20 +62,20 @@ proc writeError(description: string, path: Path) = proc conceptIntroduction(trackDir: Path, slug: string, title: string, templatePath: Path, headingLevel: int, - links: var seq[string]): string = + linkDefs: var seq[string]): string = ## Returns the contents of the `introduction.md` file for a `slug`, but: ## - Without a first top-level heading. ## - Adding a starting a second-level heading containing `title`. ## - Demoting the level of any other heading. ## - Without any leading/trailing whitespace. - ## - Without any reference links. + ## - Without any reference link definitions. ## - ## Appends reference links to `links`. + ## Appends reference link definitions to `linkDefs`. let conceptDir = trackDir / "concepts" / slug if dirExists(conceptDir): let path = conceptDir / "introduction.md" if fileExists(path): - result = path.readFile().alterHeadings(title, headingLevel, links) + result = path.readFile().alterHeadings(title, headingLevel, linkDefs) else: writeError(&"File {path} not found for concept '{slug}'", templatePath) quit(1) @@ -93,7 +93,7 @@ proc generateIntroduction(trackDir: Path, templatePath: Path, var i = 0 var headingLevel = 1 - var links = newSeq[string]() + var linkDefs = newSeq[string]() while i < content.len: var conceptSlug = "" # Here, we implement the syntax for a placeholder as %{concept:some-slug} @@ -105,7 +105,7 @@ proc generateIntroduction(trackDir: Path, templatePath: Path, if conceptSlug in slugLookup: let title = slugLookup[conceptSlug] result.add conceptIntroduction(trackDir, conceptSlug, title, - templatePath, headingLevel, links) + templatePath, headingLevel, linkDefs) else: writeError(&"Concept '{conceptSlug}' does not exist in track config.json", templatePath) @@ -115,10 +115,10 @@ proc generateIntroduction(trackDir: Path, templatePath: Path, headingLevel = content.skipWhile({'#'}, i+1) result.add content[i] inc i - if links.len > 0: + if linkDefs.len > 0: result.add '\n' - for link in links: - result.add link + for linkDef in linkDefs: + result.add linkDef result.add '\n' proc generate*(conf: Conf) = diff --git a/tests/test_generate.nim b/tests/test_generate.nim index ccb1620b..04775d53 100644 --- a/tests/test_generate.nim +++ b/tests/test_generate.nim @@ -69,12 +69,12 @@ proc testGenerate = echo "hi" ~~~""".unindent() # No final newline - var links = newSeq[string]() - check alterHeadings(s, "Maps", 1, links) == "## Maps\n\n" & expected - check alterHeadings(s, "Maps", 2, links) == expected - check alterHeadings(s, "Maps", 3, links) == expected.replace("### ", "#### ") - check alterHeadings(s, "Maps", 4, links) == expected.replace("### ", "##### ") - check links.len == 0 + var linkDefs = newSeq[string]() + check alterHeadings(s, "Maps", 1, linkDefs) == "## Maps\n\n" & expected + check alterHeadings(s, "Maps", 2, linkDefs) == expected + check alterHeadings(s, "Maps", 3, linkDefs) == expected.replace("### ", "#### ") + check alterHeadings(s, "Maps", 4, linkDefs) == expected.replace("### ", "##### ") + check linkDefs.len == 0 proc main = testGenerate() From 42b14608449b4118ea77ec100564b44118f04c02 Mon Sep 17 00:00:00 2001 From: ee7 <45465154+ee7@users.noreply.github.com> Date: Tue, 28 Jun 2022 13:25:06 +0200 Subject: [PATCH 19/26] generate: improve comments about headings --- src/generate/generate.nim | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/generate/generate.nim b/src/generate/generate.nim index 00f044ed..d4e99a3d 100644 --- a/src/generate/generate.nim +++ b/src/generate/generate.nim @@ -10,17 +10,18 @@ proc getConceptSlugLookup(trackDir: Path): Table[string, string] = func alterHeadings(s: string, title: string, headingLevel: int, linkDefs: var seq[string]): string = - # Markdown implementations differ on whether a space is required after the - # final '#' character that begins the heading. result = newStringOfCap(s.len) var i = 0 i += s.skipWhitespace() - # Skip the top-level heading (if any) + # Skip the top-level heading (if any). + # The CommonMark Spec requires that an ATX heading has a a space, tab, or + # newline after the opening sequence of '#' characters. + # For now, support only spaces. if s.continuesWith("# ", i): i += s.skipUntil('\n', i) if headingLevel == 1: result.add &"## {title}" - # Demote other headings + # Demote other headings. var inFencedCodeBlock = false var inFencedCodeBlockTildes = false var inCommentBlock = false From 5d333b83a6f0acabf75ab4d27d1d96903b9efaee Mon Sep 17 00:00:00 2001 From: ee7 <45465154+ee7@users.noreply.github.com> Date: Tue, 28 Jun 2022 13:25:07 +0200 Subject: [PATCH 20/26] generate: rename `title` to `h2` --- src/generate/generate.nim | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/generate/generate.nim b/src/generate/generate.nim index d4e99a3d..7508590f 100644 --- a/src/generate/generate.nim +++ b/src/generate/generate.nim @@ -8,7 +8,7 @@ proc getConceptSlugLookup(trackDir: Path): Table[string, string] = for `concept` in concepts: result[`concept`.slug] = `concept`.name -func alterHeadings(s: string, title: string, headingLevel: int, +func alterHeadings(s: string, h2: string, headingLevel: int, linkDefs: var seq[string]): string = result = newStringOfCap(s.len) var i = 0 @@ -20,7 +20,7 @@ func alterHeadings(s: string, title: string, headingLevel: int, if s.continuesWith("# ", i): i += s.skipUntil('\n', i) if headingLevel == 1: - result.add &"## {title}" + result.add &"## {h2}" # Demote other headings. var inFencedCodeBlock = false var inFencedCodeBlockTildes = false @@ -61,12 +61,12 @@ proc writeError(description: string, path: Path) = stderr.writeLine(path) stderr.write "\n" -proc conceptIntroduction(trackDir: Path, slug: string, title: string, +proc conceptIntroduction(trackDir: Path, slug: string, h2: string, templatePath: Path, headingLevel: int, linkDefs: var seq[string]): string = ## Returns the contents of the `introduction.md` file for a `slug`, but: ## - Without a first top-level heading. - ## - Adding a starting a second-level heading containing `title`. + ## - Adding a starting a second-level heading containing `h2`. ## - Demoting the level of any other heading. ## - Without any leading/trailing whitespace. ## - Without any reference link definitions. @@ -76,7 +76,7 @@ proc conceptIntroduction(trackDir: Path, slug: string, title: string, if dirExists(conceptDir): let path = conceptDir / "introduction.md" if fileExists(path): - result = path.readFile().alterHeadings(title, headingLevel, linkDefs) + result = path.readFile().alterHeadings(h2, headingLevel, linkDefs) else: writeError(&"File {path} not found for concept '{slug}'", templatePath) quit(1) @@ -104,8 +104,8 @@ proc generateIntroduction(trackDir: Path, templatePath: Path, "%{", *{' '}, "concept", *{' '}, ':', *{' '}, +{'a'..'z', '-'} -> conceptSlug.add($_), *{' '}, '}'): if conceptSlug in slugLookup: - let title = slugLookup[conceptSlug] - result.add conceptIntroduction(trackDir, conceptSlug, title, + let h2 = slugLookup[conceptSlug] + result.add conceptIntroduction(trackDir, conceptSlug, h2, templatePath, headingLevel, linkDefs) else: writeError(&"Concept '{conceptSlug}' does not exist in track config.json", From 09f4fca7c19736ef9b32add1a8aa8a84fb989cd4 Mon Sep 17 00:00:00 2001 From: ee7 <45465154+ee7@users.noreply.github.com> Date: Tue, 28 Jun 2022 13:25:08 +0200 Subject: [PATCH 21/26] generate: simplify demotion --- src/generate/generate.nim | 22 +++++++++------------- tests/test_generate.nim | 6 ++---- 2 files changed, 11 insertions(+), 17 deletions(-) diff --git a/src/generate/generate.nim b/src/generate/generate.nim index 7508590f..c9c51dfa 100644 --- a/src/generate/generate.nim +++ b/src/generate/generate.nim @@ -8,8 +8,7 @@ proc getConceptSlugLookup(trackDir: Path): Table[string, string] = for `concept` in concepts: result[`concept`.slug] = `concept`.name -func alterHeadings(s: string, h2: string, headingLevel: int, - linkDefs: var seq[string]): string = +func alterHeadings(s: string, linkDefs: var seq[string], h2 = ""): string = result = newStringOfCap(s.len) var i = 0 i += s.skipWhitespace() @@ -19,7 +18,7 @@ func alterHeadings(s: string, h2: string, headingLevel: int, # For now, support only spaces. if s.continuesWith("# ", i): i += s.skipUntil('\n', i) - if headingLevel == 1: + if h2.len > 0: result.add &"## {h2}" # Demote other headings. var inFencedCodeBlock = false @@ -31,9 +30,7 @@ func alterHeadings(s: string, h2: string, headingLevel: int, # Add a '#' to a line that begins with '#', unless inside a code or HTML block. if s.continuesWith("#", i+1) and not (inFencedCodeBlock or inFencedCodeBlockTildes or inCommentBlock): - let demotionAmount = if headingLevel in [1, 2]: 1 else: headingLevel - 1 - for _ in 1..demotionAmount: - result.add '#' + result.add '#' elif s.continuesWith("[", i+1): let j = s.find("]:", i+2) if j > i+2 and j < s.find('\n', i+2): @@ -61,9 +58,8 @@ proc writeError(description: string, path: Path) = stderr.writeLine(path) stderr.write "\n" -proc conceptIntroduction(trackDir: Path, slug: string, h2: string, - templatePath: Path, headingLevel: int, - linkDefs: var seq[string]): string = +proc conceptIntroduction(trackDir: Path, slug: string, templatePath: Path, + linkDefs: var seq[string], h2 = ""): string = ## Returns the contents of the `introduction.md` file for a `slug`, but: ## - Without a first top-level heading. ## - Adding a starting a second-level heading containing `h2`. @@ -76,7 +72,7 @@ proc conceptIntroduction(trackDir: Path, slug: string, h2: string, if dirExists(conceptDir): let path = conceptDir / "introduction.md" if fileExists(path): - result = path.readFile().alterHeadings(h2, headingLevel, linkDefs) + result = path.readFile().alterHeadings(linkDefs, h2) else: writeError(&"File {path} not found for concept '{slug}'", templatePath) quit(1) @@ -104,9 +100,9 @@ proc generateIntroduction(trackDir: Path, templatePath: Path, "%{", *{' '}, "concept", *{' '}, ':', *{' '}, +{'a'..'z', '-'} -> conceptSlug.add($_), *{' '}, '}'): if conceptSlug in slugLookup: - let h2 = slugLookup[conceptSlug] - result.add conceptIntroduction(trackDir, conceptSlug, h2, - templatePath, headingLevel, linkDefs) + let h2 = if headingLevel == 2: "" else: slugLookup[conceptSlug] + result.add conceptIntroduction(trackDir, conceptSlug, templatePath, + linkDefs, h2) else: writeError(&"Concept '{conceptSlug}' does not exist in track config.json", templatePath) diff --git a/tests/test_generate.nim b/tests/test_generate.nim index 04775d53..129319e1 100644 --- a/tests/test_generate.nim +++ b/tests/test_generate.nim @@ -70,10 +70,8 @@ proc testGenerate = ~~~""".unindent() # No final newline var linkDefs = newSeq[string]() - check alterHeadings(s, "Maps", 1, linkDefs) == "## Maps\n\n" & expected - check alterHeadings(s, "Maps", 2, linkDefs) == expected - check alterHeadings(s, "Maps", 3, linkDefs) == expected.replace("### ", "#### ") - check alterHeadings(s, "Maps", 4, linkDefs) == expected.replace("### ", "##### ") + check alterHeadings(s, linkDefs) == expected + check alterHeadings(s, linkDefs, "Maps") == "## Maps\n\n" & expected check linkDefs.len == 0 proc main = From c244d1a7cea778bdcb520fcbefe73e1c03ed6707 Mon Sep 17 00:00:00 2001 From: ee7 <45465154+ee7@users.noreply.github.com> Date: Tue, 28 Jun 2022 13:25:09 +0200 Subject: [PATCH 22/26] generate: avoid backticks The backticks look a bit strange, even if you know that `concept` is a keyword. Sidestep the issue. We named it `con` elsewhere: $ git grep --heading --break 'for .* in concepts:' src/info/info.nim 32: for con in concepts: src/lint/track_config.nim 303: for con in concepts: --- src/generate/generate.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/generate/generate.nim b/src/generate/generate.nim index c9c51dfa..a4f1fde1 100644 --- a/src/generate/generate.nim +++ b/src/generate/generate.nim @@ -5,8 +5,8 @@ proc getConceptSlugLookup(trackDir: Path): Table[string, string] = ## Returns a `Table` that maps each concept's `slug` to its `name`. let concepts = TrackConfig.init(readFile(trackDir / "config.json")).concepts result = initTable[string, string](concepts.len) - for `concept` in concepts: - result[`concept`.slug] = `concept`.name + for con in concepts: + result[con.slug] = con.name func alterHeadings(s: string, linkDefs: var seq[string], h2 = ""): string = result = newStringOfCap(s.len) From 1ecc512429995372482a5e9951aa13cd4d320355 Mon Sep 17 00:00:00 2001 From: ee7 <45465154+ee7@users.noreply.github.com> Date: Tue, 28 Jun 2022 13:25:10 +0200 Subject: [PATCH 23/26] generate: use distinct Slug --- src/generate/generate.nim | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/src/generate/generate.nim b/src/generate/generate.nim index a4f1fde1..0cd61fa9 100644 --- a/src/generate/generate.nim +++ b/src/generate/generate.nim @@ -1,12 +1,17 @@ -import std/[parseutils, strbasics, strformat, strscans, strutils, tables, terminal] +import std/[hashes, parseutils, strbasics, strformat, strscans, strutils, sugar, + tables, terminal] import ".."/[cli, helpers, types_track_config] -proc getConceptSlugLookup(trackDir: Path): Table[string, string] = +proc hash(slug: Slug): Hash {.borrow.} +proc `==`(x, y: Slug): bool {.borrow.} +proc add(s: var Slug, c: char) {.borrow.} + +proc getConceptSlugLookup(trackDir: Path): Table[Slug, string] = ## Returns a `Table` that maps each concept's `slug` to its `name`. let concepts = TrackConfig.init(readFile(trackDir / "config.json")).concepts - result = initTable[string, string](concepts.len) - for con in concepts: - result[con.slug] = con.name + collect: + for con in concepts: + {con.slug.Slug: con.name} func alterHeadings(s: string, linkDefs: var seq[string], h2 = ""): string = result = newStringOfCap(s.len) @@ -58,7 +63,7 @@ proc writeError(description: string, path: Path) = stderr.writeLine(path) stderr.write "\n" -proc conceptIntroduction(trackDir: Path, slug: string, templatePath: Path, +proc conceptIntroduction(trackDir: Path, slug: Slug, templatePath: Path, linkDefs: var seq[string], h2 = ""): string = ## Returns the contents of the `introduction.md` file for a `slug`, but: ## - Without a first top-level heading. @@ -68,7 +73,7 @@ proc conceptIntroduction(trackDir: Path, slug: string, templatePath: Path, ## - Without any reference link definitions. ## ## Appends reference link definitions to `linkDefs`. - let conceptDir = trackDir / "concepts" / slug + let conceptDir = trackDir / "concepts" / slug.string if dirExists(conceptDir): let path = conceptDir / "introduction.md" if fileExists(path): @@ -82,7 +87,7 @@ proc conceptIntroduction(trackDir: Path, slug: string, templatePath: Path, quit(1) proc generateIntroduction(trackDir: Path, templatePath: Path, - slugLookup: Table[string, string]): string = + slugLookup: Table[Slug, string]): string = ## Reads the file at `templatePath` and returns the content of the ## corresponding `introduction.md` file. let content = readFile(templatePath) @@ -92,7 +97,7 @@ proc generateIntroduction(trackDir: Path, templatePath: Path, var headingLevel = 1 var linkDefs = newSeq[string]() while i < content.len: - var conceptSlug = "" + var conceptSlug = Slug "" # Here, we implement the syntax for a placeholder as %{concept:some-slug} # where we allow spaces after the opening brace, around the colon, # and before the closing brace. The slug must be in kebab-case. From 607f369b84761d308af66c6b74d87ee618f61cbf Mon Sep 17 00:00:00 2001 From: ee7 <45465154+ee7@users.noreply.github.com> Date: Tue, 28 Jun 2022 13:25:11 +0200 Subject: [PATCH 24/26] generate, sync, types: move borrowed procs for Slug --- src/generate/generate.nim | 8 ++------ src/sync/sync_common.nim | 5 ----- src/types_track_config.nim | 7 ++++++- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/src/generate/generate.nim b/src/generate/generate.nim index 0cd61fa9..a5ada327 100644 --- a/src/generate/generate.nim +++ b/src/generate/generate.nim @@ -1,11 +1,7 @@ -import std/[hashes, parseutils, strbasics, strformat, strscans, strutils, sugar, - tables, terminal] +import std/[parseutils, strbasics, strformat, strscans, strutils, sugar, tables, + terminal] import ".."/[cli, helpers, types_track_config] -proc hash(slug: Slug): Hash {.borrow.} -proc `==`(x, y: Slug): bool {.borrow.} -proc add(s: var Slug, c: char) {.borrow.} - proc getConceptSlugLookup(trackDir: Path): Table[Slug, string] = ## Returns a `Table` that maps each concept's `slug` to its `name`. let concepts = TrackConfig.init(readFile(trackDir / "config.json")).concepts diff --git a/src/sync/sync_common.nim b/src/sync/sync_common.nim index d1a4361f..77ede4f5 100644 --- a/src/sync/sync_common.nim +++ b/src/sync/sync_common.nim @@ -25,9 +25,6 @@ proc postHook*(e: ConceptExercise | PracticeExercise) = stderr.writeLine msg quit 1 -func `==`*(x, y: Slug): bool {.borrow.} -func `<`*(x, y: Slug): bool {.borrow.} - func getSlugs*(e: seq[ConceptExercise] | seq[PracticeExercise]): seq[Slug] = ## Returns a seq of the slugs in `e`, in alphabetical order. result = newSeq[Slug](e.len) @@ -35,8 +32,6 @@ func getSlugs*(e: seq[ConceptExercise] | seq[PracticeExercise]): seq[Slug] = result[i] = item.slug sort result -func len*(slug: Slug): int {.borrow.} - func truncateAndAdd*(s: var string, truncateLen: int, slug: Slug) = ## Truncates `s` to `truncateLen`, then appends `slug`. ## diff --git a/src/types_track_config.nim b/src/types_track_config.nim index cf64a06c..62fb8503 100644 --- a/src/types_track_config.nim +++ b/src/types_track_config.nim @@ -1,4 +1,4 @@ -import std/sets +import std/[hashes, sets] import pkg/jsony import "."/[cli, helpers] @@ -58,6 +58,11 @@ type ekPractice = "practice" func `$`*(slug: Slug): string {.borrow.} +func `==`*(x, y: Slug): bool {.borrow.} +func `<`*(x, y: Slug): bool {.borrow.} +func len*(slug: Slug): int {.borrow.} +func hash*(slug: Slug): Hash {.borrow.} +func add*(s: var Slug, c: char) {.borrow.} proc init*(T: typedesc[TrackConfig]; trackConfigContents: string): T = ## Deserializes `trackConfigContents` using `jsony` to a `TrackConfig` object. From 4ee45efffb7dcc9f0b522929b4add13babaf705b Mon Sep 17 00:00:00 2001 From: ee7 <45465154+ee7@users.noreply.github.com> Date: Tue, 28 Jun 2022 15:03:01 +0200 Subject: [PATCH 25/26] generate: refer to "link reference definitions" That's what the CommonMark Spec [1] calls them. [1] https://spec.commonmark.org/0.30/#link-reference-definitions --- src/generate/generate.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/generate/generate.nim b/src/generate/generate.nim index a5ada327..73e0d2d1 100644 --- a/src/generate/generate.nim +++ b/src/generate/generate.nim @@ -66,9 +66,9 @@ proc conceptIntroduction(trackDir: Path, slug: Slug, templatePath: Path, ## - Adding a starting a second-level heading containing `h2`. ## - Demoting the level of any other heading. ## - Without any leading/trailing whitespace. - ## - Without any reference link definitions. + ## - Without any link reference definitions. ## - ## Appends reference link definitions to `linkDefs`. + ## Appends link reference definitions to `linkDefs`. let conceptDir = trackDir / "concepts" / slug.string if dirExists(conceptDir): let path = conceptDir / "introduction.md" From 7727f7364e6cd7372e3b3572d09f58f105ea2b09 Mon Sep 17 00:00:00 2001 From: ee7 <45465154+ee7@users.noreply.github.com> Date: Thu, 30 Jun 2022 12:36:01 +0200 Subject: [PATCH 26/26] generate: ensure exactly 1 newline at EOF --- src/generate/generate.nim | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/generate/generate.nim b/src/generate/generate.nim index 73e0d2d1..0f720c7c 100644 --- a/src/generate/generate.nim +++ b/src/generate/generate.nim @@ -113,6 +113,8 @@ proc generateIntroduction(trackDir: Path, templatePath: Path, headingLevel = content.skipWhile({'#'}, i+1) result.add content[i] inc i + result.strip() + result.add '\n' if linkDefs.len > 0: result.add '\n' for linkDef in linkDefs: