sync: handle docs, metadata, and filepaths (#366)

This commit adds important features to `configlet sync`.

Before this commit, the `sync` command operated only on tests. For each
Practice Exercise on the track that existed in the
`exercism/problem-specifications` repo, `sync` would check or update the
exercise's `.meta/tests.toml` file using the corresponding upstream
`canonical-data.json` file.

With this commit, the `sync` command still has that functionality
(which can now be toggled with the `--tests` option). But for such
Practice Exercises that come from `problem-specifications`, `sync` also
gains these new options:

- `--docs` to check or update each exercise's `.docs/instructions.md`
  (and possibly `.docs/introduction.md`) file, using the corresponding
  docs in `problem-specifications`.

- `--metadata` to check or update the `blurb`, `source`, and
  `source_url` values in each exercise's `.meta/config.json` file,
  using those in the corresponding `metadata.toml` file in
  `problem-specifications`.

And for both Concept Exercises and Practice Exercises, `sync` gains
these new options:

- `--filepaths` for populating empty or missing `files` values in an
  exercise `.meta/config.json` from the track `config.json` file.

- `-y/--yes` to auto-confirm every prompt for updating docs, filepaths,
  and metadata.

Note that the `-y/--yes` option does not affect prompts for updating
tests. Before this commit, `sync` had a `--mode` option that allowed the
user to specify whether they want to be prompted (the default) to
include/exclude/skip each unseen test individually, or non-interactively
include/exclude all of them. This commit replaces the `--mode` option:
the user should now pass the mode value to the `--tests` option instead
(for example, `--tests include`). The default is still `choose`.

A plain `configlet sync` operates on the full syncing scope, and so
does the same as `configlet sync --docs --filepaths --metadata --tests`.
The `sync` command still operates on every exercise by default, and
still does not create/alter the track's files unless the `-u/--update`
option is used.

When writing a `.meta/config.json` file, `sync` tries to maintain the
key order that it saw, aiming to minimize noise in diffs and PRs. An
exception is when the file is missing or lacks required keys, in which
case configlet will create those files and keys.

Therefore when adding a Practice Exercise `foo` that exists in
`problem-specifications` to a track, to create the docs and a starter
`.meta/config.json` file we can run:

    $ configlet sync -uy -e foo --docs --filepaths --metadata

And to interactively create the `.meta/tests.toml` file:

    $ configlet sync -u -e foo --tests

When updating exercise `.meta/config.json` files, configlet will
preserve a top-level key named `custom` and its value of any valid JSON
object. You can use this for track-specific feature flagging.

`sync` still performs a `git clone` of `problem-specifications` by
default. In the future, we intend for configlet to cache that repo - but
to skip cloning in the meantime, please continue to use the
`-o/--offline` and `-p/--prob-specs-dir` options. For example:

    $ configlet sync -o -p /path/to/local/problem-specifications

For more details about the new `sync` functionality and more usage
examples, please see the `README.md` file.

We will soon add the `configlet fmt` command, for rewriting JSON files
on a track in a canonical form without syncing. You may recall that the
configlet for Exercism v2 had a `fmt` command with similar
functionality.

This commit also fixes a bad error message: previously
`configlet sync -e foo` would say that the `foo` exercise is up to date,
even if `foo` didn't exist on the track.

Closes: #298
Closes: #179
Fixes: #31

.gitignore

@@ -21,6 +21,6 @@
 
 # Ignore temporary repos
 .problem-specifications/
-tests/.test_binary_elixir_track_repo/
-tests/.test_binary_nim_track_repo/
-tests/.test_binary_problem_specifications/
+tests/.test_elixir_track_repo/
+tests/.test_nim_track_repo/
+tests/.test_problem_specifications/

README.md

@@ -14,11 +14,17 @@ Commands:
   lint, sync, uuid, generate, info
 
 Options for sync:
-  -e, --exercise <slug>        Only sync this exercise
-  -m, --mode <mode>            What to do with missing test cases. Allowed values: c[hoose], i[nclude], e[xclude]
-  -p, --prob-specs-dir <dir>   Use this `problem-specifications` directory, rather than cloning temporarily
-  -o, --offline                Do not check that the directory specified by `-p, --prob-specs-dir` is up-to-date
-  -u, --update                 Prompt the user to include, exclude, or skip any missing tests
+  -e, --exercise <slug>        Only operate on this exercise
+  -p, --prob-specs-dir <dir>   Use this 'problem-specifications' directory, rather than cloning temporarily
+  -o, --offline                Do not check that the directory specified by --prob-specs-dir is up to date
+  -u, --update                 Prompt to update the seen data that are unsynced
+  -y, --yes                    Auto-confirm prompts from --update for updating docs, filepaths, and metadata
+      --docs                   Sync Practice Exercise '.docs/introduction.md' and '.docs/instructions.md' files
+      --filepaths              Populate empty 'files' values in Concept/Practice exercise '.meta/config.json' files
+      --metadata               Sync Practice Exercise '.meta/config.json' metadata values
+      --tests [mode]           Sync Practice Exercise '.meta/tests.toml' files.
+                               The mode value specifies how missing tests are handled when using --update.
+                               Allowed values: c[hoose], i[nclude], e[xclude] (default: choose)
 
 Options for uuid:
   -n, --num <int>              Number of UUIDs to generate
@@ -27,7 +33,8 @@ Global options:
   -h, --help                   Show this help message and exit
       --version                Show this tool's version information and exit
   -t, --track-dir <dir>        Specify a track directory to use instead of the current directory
-  -v, --verbosity <verbosity>  The verbosity of output. Allowed values: q[uiet], n[ormal], d[etailed]
+  -v, --verbosity <verbosity>  The verbosity of output.
+                               Allowed values: q[uiet], n[ormal], d[etailed] (default: normal)
 ```
 
 ## `configlet lint`
@@ -38,6 +45,89 @@ The `configlet lint` command is still under development. The list of currently i
 
 ## `configlet sync`
 
+A Practice Exercise on an Exercism track is often implemented from a specification in the [`exercism/problem-specifications`](https://github.com/exercism/problem-specifications) repo.
+
+Exercism deliberately requires that every exercise has its own copy of certain files (like `.docs/instructions.md`), even when that exercise exists in `problem-specifications`.
+Therefore configlet has a `sync` command, which can check that such Practice Exercises on a track are in sync with that upstream source, and can update them when updates are available.
+
+There are three kinds of data that can be updated from `problem-specifications`: documentation, metadata, and tests.
+There is also one kind of data that can be populated from the track-level `config.json` file: filepaths in exercise config files.
+
+We describe the checking and updating of these data kinds in individual sections below, but as a quick summary:
+- `configlet sync` only operates on exercises that exist in the track-level `config.json` file. Therefore if you are implementing a new exercise on a track and want to add the initial files with `configlet sync`, please add the exercise to the track-level `config.json` file first. If the exercise is not yet ready to be user-facing, please set its `status` value to `wip`.
+- A plain `configlet sync` makes no changes to the track, and checks every data kind for every exercise.
+- To operate on a subset of data kinds, use some combination of the `--docs`, `--filepaths`, `--metadata`, and `--tests` options.
+- To interactively update data on the track, use the `--update` option.
+- To non-interactively update docs, filepaths, and metadata on the track, use `--update --yes`.
+- To non-interactively include every unseen test for a given exercise, use e.g. `--update --tests include --exercise prime-factors`.
+- To skip downloading the `problem-specifications` repo, add `--offline --prob-specs-dir /path/to/local/problem-specifications`
+- Note that `configlet sync` tries to maintain the key order in exercise `.meta/config.json` files when updating. To write these files in a canonical form without syncing, please use the upcoming `configlet fmt` command. However, `configlet sync` _does_ add (possibly empty) required keys (`authors`, `files`, `blurb`) when they are missing. This is less "sync-like", but more ergonomic: when implementing a new exercise, you can use `sync` to create a starter `.meta/config.json` file.
+- `configlet sync` removes keys that are not in the spec. Custom key/value pairs are still supported: they must be written inside a JSON object named `custom`.
+- The exit code is 0 when all the seen data are synced when configlet exits, and 1 otherwise.
+
+Note that in `configlet` releases `4.0.0-alpha.34` and earlier, the `sync` command operated only on tests.
+
+### Docs
+
+A Practice Exercise that is derived from the `problem-specifications` repo must have a `.docs/instructions.md` file (and possibly a `.docs/introduction.md` file too) containing the exercise documentation from `problem-specifications`.
+
+To check every Practice Exercise on the track for available documentation updates (exiting with a non-zero exit code if at least one update is available):
+
+```
+$ configlet sync --docs
+```
+
+To interactively update the docs for every Practice Exercise, add the `--update` option (or `-u` for short):
+
+```
+$ configlet sync --docs --update
+```
+
+To non-interactively update the docs for every Practice Exercise, add the `--yes` option (or `-y` for short):
+
+```
+$ configlet sync --docs --update --yes
+```
+
+To operate on a single Practice Exercise, use the `--exercise` option (or `-e` for short).
+For example, to non-interactively update the docs for the `prime-factors` exercise:
+
+```
+$ configlet sync --docs -uy -e prime-factors
+```
+
+### Metadata
+
+Every exercise on a track must have a `.meta/config.json` file.
+For a Practice Exercise that is derived from the `problem-specifications` repo, this file should contain the `blurb`, `source` and `source_url` key/value pairs that exist in the corresponding upstream `metadata.toml` file.
+
+To check every Practice Exercise for available metadata updates (exiting with a non-zero exit code if at least one update is available):
+
+```
+$ configlet sync --metadata
+```
+
+To interactively update the metadata for every Practice Exercise, add the `--update` option (or `-u` for short):
+
+```
+$ configlet sync --metadata --update
+```
+
+To non-interactively update the metadata for every Practice Exercise, add the `--yes` option (or `-y` for short):
+
+```
+$ configlet sync --metadata --update --yes
+```
+
+To operate on a single Practice Exercise, use the `--exercise` option (or `-e` for short).
+For example, to non-interactively update the metadata for the `prime-factors` exercise:
+
+```
+$ configlet sync --metadata -uy -e prime-factors
+```
+
+### Tests
+
 If a track implements an exercise for which test data exists in the [problem-specifications repo](https://github.com/exercism/problem-specifications), the exercise _must_ contain a `.meta/tests.toml` file. The goal of the `tests.toml` file is to keep track of which tests are implemented by the exercise. Tests in this file are identified by their UUID and each test has a boolean value that indicates if it is implemented by that exercise.
 
 A `tests.toml` file has this format:
@@ -68,11 +158,66 @@ comment = "excluded because we don't want to add error handling to the exercise"
 
 In this case, the track has chosen to implement two of the three available tests. If a track uses a _test generator_ to generate an exercise's test suite, it _must_ use the contents of the `tests.toml` file to determine which tests to include in the generated test suite.
 
-The `sync` command allows tracks to keep `tests.toml` files up to date. A plain `configlet sync` performs no changes, and just compares the tests specified in the `tests.toml` files against the tests that are defined in the exercise's canonical data - if there are tests defined only in the latter, it prints a summary and exits with a non-zero exit code.
+To check every Practice Exercise `tests.toml` file for available tests updates (exiting with a non-zero exit code if there is at least one test case that appears in the exercise's canonical data, but not in the `tests.toml`):
+
+```
+$ configlet sync --tests
+```
+
+To interactively update the `tests.toml` file for every Practice Exercise, add the `--update` option:
+
+```
+$ configlet sync --tests --update
+```
+
+For each missing test, this prompts the user to choose whether to include/exclude/skip it, and updates the corresponding `tests.toml` file accordingly.
+Configlet writes an exercise's `tests.toml` file when the user has finished making choices for that exercise.
+This means that you can terminate configlet at a prompt (for example, by pressing Ctrl-C in the terminal) and only lose the syncing decisions for at most one exercise.
+
+To non-interactively include every unseen test case, use `--tests include`. For example, to do so for an exercise named `prime-factors`:
+
+```
+$ configlet sync --tests include -u -e prime-factors
+```
+
+Remember to actually implement these tests on the track!
+
+### Filepaths
+
+Finally, the `sync` command also handles "syncing" from a source that isn't `problem-specifications` - the track-level `config.json` file.
+Every Concept Exercise and Practice Exercise must have a `.meta/config.json` file with a `files` object that specifies the (relative) locations of the files that the exercise uses.
+Such filepaths usually follow a simple pattern, and so configlet can populate the exercise-level values from patterns in the `files` key of the track-level `config.json` file.
+
+To check that every Concept Exercise and Practice Exercise on the track has a fully populated `files` key (or at least one that cannot be populated from the track-level `files` key):
+
+```
+$ configlet sync --filepaths
+```
+
+(Note that `configlet lint` will also produce an error when an exercise has a missing/empty `files` key.)
+
+To populate empty/missing values of the exercise-level `files` key for every Concept Exercise and Practice Exercise from the patterns in the track-level `files` key:
+
+```
+$ configlet sync --filepaths --update
+```
+
+To do this non-interactively and for a single exercise named `prime-factors`:
+
+```
+$ configlet sync --filepaths -uy -e prime-factors
+```
 
-To interactively update the `tests.toml` files, use `configlet sync --update`. For each missing test, this prompts the user to choose whether to include/exclude/skip it, and updates the corresponding `tests.toml` file accordingly.
+### Using `sync` when adding a new exercise to a track
 
-The `configlet sync` command replaces the functionality of the older `canonical_data_syncer` application.
+The `sync` command is useful when adding a new exercise to a track. If you are adding a Practice Exercise named `foo` that exists in `problem-specifications`, one possible workflow is:
+1. Manually add an entry to the track-level `config.json` file for the exercise `foo`. This makes the exercise visible to `configlet sync`.
+1. Run `configlet sync --docs --filepaths --metadata -uy -e foo` to create the exercise's documentation, and a starter `.meta/config.json` file with populated `files`, `blurb`, and perhaps `source` and `source_url` values.
+1. Edit the exercise `.meta/config.json` file as desired. For example, add yourself to the `authors` array.
+1. Run `configlet sync --tests include -u -e foo` to create a `.meta/tests.toml` file with every test included.
+1. View that `.meta/tests.toml` file, and add `include = false` to any test case that the exercise will not implement.
+1. Implement the tests for the exercise to match those included in `.meta/tests.toml`.
+1. Add the other required files.
 
 ## `configlet uuid`
 

configlet.nimble

@@ -1,3 +1,5 @@
+import std/[hashes, os, strutils]
+
 # Package
 version       = "4.0.0"
 author        = "ee7"
@@ -23,3 +25,35 @@ requires "isaac#45a5cbbd54ff59ba3ed94242620c818b9aad1b5b"     # 0.1.3  (2017-11-
 
 task test, "Runs the test suite":
   exec "nim r ./tests/all_tests.nim"
+
+# Patch `cligen/parseopt3` so that "--foo --bar" is parsed as two long options,
+# even when `longNoVal` is both non-empty and lacks `foo`.
+before build:
+  # We want to support running `nimble build` before `cligen` is installed, so
+  # we can't `import cligen/parseopt3` and check the parsing directly.
+  # Instead, let's just hash the file and run `git apply` conditionally.
+  # First, get the path to `parseopt3.nim` in the `cligen` package.
+  let (output, exitCode) = gorgeEx("nimble path cligen")
+  if exitCode == 0:
+    let parseopt3Path = joinPath(output.strip(), "cligen", "parseopt3.nim")
+    if fileExists(parseopt3Path):
+      # Hash the file using `std/hashes`.
+      # Note that we can't import `std/md5` or `std/sha1` in a .nimble file.
+      let actualHash = parseopt3Path.readFile().hash()
+      const patchedHash = 1647921161 # Update when bumping `cligen` changes `parseopt3`.
+      if actualHash != patchedHash:
+        echo "Trying to patch parseopt3..."
+        echo "Found " & parseopt3Path
+        let patchPath = thisDir() / "parseopt3_allow_long_option_optional_value.patch"
+        let parseopt3Dir = parseopt3Path.parentDir()
+        # Apply the patch.
+        let cmd = "git -C " & parseopt3Dir & " apply --verbose " & patchPath
+        let (outp, exitCode) = gorgeEx(cmd)
+        echo outp
+        if exitCode != 0:
+          raise newException(AssertionDefect, "failed to apply patch")
+    else:
+      raise newException(AssertionDefect, "file does not exist: " & parseopt3Path)
+  else:
+    echo output
+    raise newException(AssertionDefect, "failed to get cligen path")

parseopt3_allow_long_option_optional_value.patch

@@ -0,0 +1,24 @@
+diff --git a/parseopt3.nim b/parseopt3.nim
+index a865952..850d016 100644
+--- a/parseopt3.nim
++++ b/parseopt3.nim
+@@ -267,8 +267,17 @@ proc doLong(p: var OptParser) =
+     p.kind = cmdError
+     return
+   if p.pos < p.cmd.len:                 # Take opt arg from next param
+-    p.val = p.cmd[p.pos]
+-    p.pos += 1
++    # If the next parameter begins with `-`, parse it as an option, even when
++    # `longNoVal` is both non-empty and lacks the given long option.
++    # This allows a long option `foo` to have an optional value, supporting both
++    # of the below forms:
++    #   --foo val1 --bar val2
++    #   --foo --bar val2
++    # Without the below line, `--bar` is parsed as the value of `--foo` in the
++    # latter case.
++    if not p.cmd[p.pos].startsWith("-"):
++      p.val = p.cmd[p.pos]
++      p.pos += 1
+   elif p.longNoVal.len != 0:
+     p.val = ""
+     p.pos += 1