Merge branch 'develop'

.github/dependencies.yml

@@ -14,3 +14,16 @@ dev:
     dependency for [yq](https://github.com/kislyuk/yq), which is used to
     generate the README; the README generator needs to use `tomlq` (which is a
     part of `yq`) to query `pyproject.toml`.
+  unzip: scripts (pyenv).
+  curl: scripts (pyenv).
+  git-core: scripts (pyenv).
+  gcc: scripts (pyenv).
+  make: scripts (pyenv).
+  zlib1g-dev: scripts (pyenv).
+  libbz2-dev: scripts (pyenv).
+  libreadline-dev: scripts (pyenv).
+  libsqlite3-dev: scripts (pyenv).
+  libssl-dev: scripts (pyenv).
+  libffi-dev:
+    bdist_wheel (otherwise `pip install .` fails). If installing pyenv, this
+    must be installed _first_.

.github/workflows/build-and-test.yml

@@ -37,9 +37,6 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version.gh }}
-      - name: Install nvm
-        run: |
-          curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.sh | bash
       - name: Set up .python-version
         run: |
           echo "${{ matrix.python-version.file }}"
@@ -55,6 +52,9 @@ jobs:
         run: |
           # See project/.github/dependencies.yml for a list of dependencies.
           sudo apt-get update && sudo apt-get install -y ${APT_DEPS}
+      - name: Install nvm
+        run: |
+          curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.sh | bash
       - name: Run everything
         run: |
           bash scripts/pre.sh

Dockerfile

@@ -6,8 +6,7 @@ WORKDIR /snipinator
 # apt-get -y --no-install-recommends install git=1:2.39.2-1.1 &&
 # apt-get -y upgrade &&
 
-COPY . /snipinator
-RUN apt-get -y update && apt-get -y --no-install-recommends install bash=5.2.15-2+b2 && \
+RUN apt-get -y update && apt-get -y --no-install-recommends install bash=5.2.15-2+b7 && \
   apt-get -y clean && \
   apt-get -y autoremove && \
   rm -rf /var/lib/apt/lists/* && \
@@ -17,13 +16,11 @@ RUN apt-get -y update && apt-get -y --no-install-recommends install bash=5.2.15-
   chown -R user:user /snipinator /home/user/.local && \
   chmod -R a+wrX /snipinator
 
-
+COPY --chown=user:user . /snipinator
 USER user
 WORKDIR /snipinator
 ENV PATH=/home/user/.local/bin:$PATH
 ENV PYTHONPATH=/home/user/.local/lib/python3.12/site-packages
-ENV PATH=/home/user/.local/bin:$PATH
-ENV PYTHONPATH=/home/user/.local/lib/python3.12/site-packages
 RUN pip install --no-cache-dir --prefix=/home/user/.local .
 
 # This is where the user will mount their data to.

README.md

@@ -12,6 +12,7 @@ SOURCE: `README.md.jinja2`.
 
 
 
+
 -->
 
 # <div align="center">[![Snipinator][1]][2]</div>
@@ -154,7 +155,7 @@ Note that `code.py` has a test:
 pip install snipinator
 
 # Install from git (https://github.com/realazthat/snipinator)
-pip install git+https://github.com/realazthat/snipinator.git@v2.1.0
+pip install git+https://github.com/realazthat/snipinator.git@v2.2.0
 ```
 
 ## 🚜 Usage
@@ -214,16 +215,7 @@ Note that `code.py` has a test:
 Fuller example:
 
 <!---->
-```bash
-
-python -m snipinator.cli \
-  -t "snipinator/examples/EXAMPLE.md.jinja2" \
-  --rm \
-  --force \
-  --create \
-  -o "snipinator/examples/EXAMPLE.generated.md" \
-  --chmod-ro
-```
+<img src="README.example.generated.svg" alt="Output of `bash ./snipinator/examples/example_example.sh`" />
 <!---->
 
 ## 💻 Command Line Options
@@ -440,6 +432,7 @@ def snippet(path: str,
             indented: Union[str, int, None] = None,
             backtickify: Union[bool, str] = False,
             decomentify: Union[bool, Literal['nl']] = False,
+            regex: Union[bool, str] = False,
             _ctx: _Context) -> Union[str, markupsafe.Markup]:
   """Returns a _delimited_ snippet from a file.
 
@@ -463,6 +456,10 @@ def snippet(path: str,
         the Jinja2 call unmolested by markdown formatters, because they will be
         inside of a comment section. "nl" adds additional newlines after the
         newline delimiters. Defaults to False.
+      regex (Union[bool, str], optional): If True, `start` and `end` will be
+        treated as regular expressions. Optionally, can pass in python regex
+        flags separated by `|` characters, e.g "IGNORECASE|MULTILINE". Defaults
+        to False.
       _ctx (_Context): This is used by the system and is not available as an
         argument.
 
@@ -498,6 +495,7 @@ def shell(args: str,
           include_args: bool = True,
           start: Optional[str] = None,
           end: Optional[str] = None,
+          regex: Union[bool, str] = False,
           _ctx: _Context) -> Union[str, markupsafe.Markup]:
   """Run a shell command and return the output.
 
@@ -567,6 +565,10 @@ def shell(args: str,
         delimiter. Defaults to None.
       end (str, optional): If specified, will return only the text before this
         delimiter. Defaults to None.
+      regex (Union[bool, str], optional): If True, `start` and `end` will be
+        treated as regular expressions. Optionally, can pass in python regex
+        flags separated by `|` characters, e.g "IGNORECASE|MULTILINE". Defaults
+        to False.
       _ctx (_Context): This is used by the system and is not available as an
         argument.
 
@@ -652,17 +654,25 @@ tag.
 <!---->
 ```bash
 
+# View the template file.
+cat "snipinator/examples/EXAMPLE.md.jinja2"
+
 # Use the published images at ghcr.io/realazthat/snipinator.
 # /data in the docker image is the working directory, so paths are simpler.
 docker run --rm --tty \
+  -u "$(id -u):$(id -g)" \
   -v "${PWD}:/data" \
-  ghcr.io/realazthat/snipinator:v2.1.0 \
+  ghcr.io/realazthat/snipinator:v2.2.0 \
   -t "snipinator/examples/EXAMPLE.md.jinja2" \
   --rm \
   --force \
   --create \
   -o "snipinator/examples/EXAMPLE.generated.md" \
   --chmod-ro
+
+# View the generated file.
+cat "snipinator/examples/EXAMPLE.generated.md"
+
 ```
 <!---->
 
@@ -674,8 +684,12 @@ repository.
 
 docker build -t my-snipinator-image .
 
+# View the template file.
+cat "snipinator/examples/EXAMPLE.md.jinja2"
+
 # /data in the docker image is the working directory, so paths are simpler.
 docker run --rm --tty \
+  -u "$(id -u):$(id -g)" \
   -v "${PWD}:/data" \
   my-snipinator-image \
   -t "snipinator/examples/EXAMPLE.md.jinja2" \
@@ -684,6 +698,10 @@ docker run --rm --tty \
   --create \
   -o "snipinator/examples/EXAMPLE.generated.md" \
   --chmod-ro
+
+# View the generated file.
+cat "snipinator/examples/EXAMPLE.generated.md"
+
 ```
 <!---->
 
@@ -831,9 +849,24 @@ Not complete, and not necessarily up to date. Make a PR
     jq: dependency for [yq](https://github.com/kislyuk/yq), which is used to generate
       the README; the README generator needs to use `tomlq` (which is a part of `yq`)
       to query `pyproject.toml`.
+    unzip: scripts (pyenv).
+    curl: scripts (pyenv).
+    git-core: scripts (pyenv).
+    gcc: scripts (pyenv).
+    make: scripts (pyenv).
+    zlib1g-dev: scripts (pyenv).
+    libbz2-dev: scripts (pyenv).
+    libreadline-dev: scripts (pyenv).
+    libsqlite3-dev: scripts (pyenv).
+    libssl-dev: scripts (pyenv).
+    libffi-dev: bdist_wheel (otherwise `pip install .` fails). If installing pyenv, this
+      must be installed _first_.
 
     ```
 
+    - On Ubuntu: `sudo apt-get update` and then
+      `sudo apt-get install -y bash findutils grep xxd git xxhash rsync expect jq unzip curl git-core gcc make zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev libssl-dev libffi-dev`.
+
   - Requires `pyenv`, or an exact matching version of python as in
     [./.python-version](./.python-version) (which is currently
     `3.8.0`).
@@ -905,23 +938,23 @@ These instructions are for maintainers of the project.
 [13]:
   https://github.com/realazthat/snipinator/actions/workflows/build-and-test.yml
 [14]:
-  https://img.shields.io/github/commits-since/realazthat/snipinator/v2.1.0/master?style=plastic
+  https://img.shields.io/github/commits-since/realazthat/snipinator/v2.2.0/master?style=plastic
 [15]:
-  https://github.com/realazthat/snipinator/compare/v2.1.0...master
+  https://github.com/realazthat/snipinator/compare/v2.2.0...master
 [16]:
   https://img.shields.io/github/last-commit/realazthat/snipinator/master?style=plastic
 [17]: https://github.com/realazthat/snipinator/commits/master
 [18]: https://github.com/realazthat/snipinator/tree/develop
 [19]:
   https://img.shields.io/github/actions/workflow/status/realazthat/snipinator/build-and-test.yml?branch=develop&style=plastic
 [20]:
-  https://img.shields.io/github/commits-since/realazthat/snipinator/v2.1.0/develop?style=plastic
+  https://img.shields.io/github/commits-since/realazthat/snipinator/v2.2.0/develop?style=plastic
 [21]:
-  https://github.com/realazthat/snipinator/compare/v2.1.0...develop
+  https://github.com/realazthat/snipinator/compare/v2.2.0...develop
 [22]:
-  https://img.shields.io/github/commits-since/realazthat/snipinator/v2.1.0/develop?style=plastic
+  https://img.shields.io/github/commits-since/realazthat/snipinator/v2.2.0/develop?style=plastic
 [23]:
-  https://github.com/realazthat/snipinator/compare/v2.1.0...develop
+  https://github.com/realazthat/snipinator/compare/v2.2.0...develop
 [24]:
   https://img.shields.io/github/last-commit/realazthat/snipinator/develop?style=plastic
 [25]: https://github.com/realazthat/snipinator/commits/develop

README.md.jinja2

@@ -7,6 +7,8 @@
                                  include_args=False) | trim %}
 {% set tested_python_versions=shell('python -m yq -r -c \'[.jobs["build-and-test"]["strategy"]["matrix"]["python-version"][].gh] | join(", ")\' .github/workflows/build-and-test.yml',
                                  include_args=False) | trim %}
+{% set dev_dependencies=shell('python -m yq -r -c \'.dev | keys_unsorted | join(" ")\' .github/dependencies.yml',
+                              include_args=False)|trim %}
 {% set badge_color='0A1E1E' %}
 
 -->
@@ -125,10 +127,16 @@ Generating the README:
 
 Fuller example:
 
-<!--{{ snippet('./snipinator/examples/example_example.sh',
-               start='# SNIPPET_START',
-               end='\n# SNIPPET_END',
-               backtickify='bash',
+<!--{{ shell('cat ./.deleteme/example_example.output',
+               start=': ECHO_SNIPPET_START',
+               end='^.*: ECHO_SNIPPET_END',
+               regex='MULTILINE',
+               rich='README.example.generated.svg',
+               rich_alt='Output of `bash ./snipinator/examples/example_example.sh`',
+               rich_bg_color='black',
+               rich_term='xterm-256color',
+               rich_cols=160,
+               include_args=False,
                decomentify='nl') }}-->
 
 ## 💻 Command Line Options
@@ -414,6 +422,9 @@ Not complete, and not necessarily up to date. Make a PR
     indented=4
     )}}
 
+    - On Ubuntu: `sudo apt-get update` and then
+      `sudo apt-get install -y {{dev_dependencies}}`.
+
   - Requires `pyenv`, or an exact matching version of python as in
     {{path('./.python-version', link='md')}} (which is currently
     `{{ rawsnippet('./.python-version') | trim }}`).

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "snipinator"
-version = "2.1.0"
+version = "2.2.0"
 description = "Python code snippets for markdown files, e.g READMEs, from actual (testable) code."
 authors = [{name = "AYF", email = "realazthat@gmail.com"}]
 license = {file = "LICENSE.md"}
@@ -28,10 +28,13 @@ dependencies = [
   "mdurl <1",
   "pexpect >4,<5",
   "ptyprocess >=0.7,<0.8",
+  "pyyaml >=6,<7",
   "Pygments >=2,<3",
   "rich >=13,<14",
   "rich-argparse >=1,<2",
-  "types-colorama >=0.4,<1"
+  "types-colorama >=0.4,<1",
+  "typing_extensions >=4,<5",
+  "types-pyyaml >=6,<7"
 ]
 keywords = [
   "github",
@@ -66,9 +69,11 @@ prod = [
   "pexpect==4.9.0",
   "ptyprocess==0.7.0",
   "pygments==2.17.2",
+  "pyyaml==6.0.1",
   "rich==13.7.1",
   "rich-argparse==1.4.0",
   "types-colorama==0.4.15.20240311",
+  "types-pyyaml==6.0.12.20240311",
   "typing-extensions==4.10.0"
 ]
 # Set of dev dependencies, all pinned, so that they are known to work. To add a
@@ -172,7 +177,7 @@ Repository = "https://github.com/realazthat/snipinator"
 packages = ["snipinator"]
 
 [tool.snipinator-project-metadata]
-last_release = "2.1.0"
-last_stable_release = "2.1.0"
+last_release = "2.2.0"
+last_stable_release = "2.2.0"
 
 [tool.tomlsort]

scripts/generate-readme.sh

@@ -13,9 +13,19 @@ TOML=${PROJ_PATH}/pyproject.toml EXTRA=dev \
   TARGET_VENV_PATH="${PWD}/.cache/scripts/.venv" \
   bash "${PROJ_PATH}/scripts/utilities/ensure-reqs.sh"
 
-bash scripts/format.sh
+# Runs in generate.sh.
+# bash scripts/format.sh
+
+# Runs in generate.sh.
+# bash scripts/run-all-examples.sh
+
+
+mkdir -p .deleteme
+# Try to make terminal output as consistent as possible.
+TERM=xterm-256color COLUMNS=160 LINES=40 \
+PS4="${GREEN}$ ${NC}" unbuffer bash -x ./snipinator/examples/example_example.sh \
+  > .deleteme/example_example.output 2>&1
 
-bash scripts/run-all-examples.sh
 
 python -m snipinator.cli \
   -t "${PROJ_PATH}/README.md.jinja2" \