Continuous Integration Optimization

.github/scripts/make-github-release.py

@@ -0,0 +1,90 @@
+import os
+import re
+import subprocess
+import sys
+from pathlib import Path
+
+from pypandoc import convert_text, download_pandoc
+
+
+CWD = Path().cwd()
+
+
+date_pattern = r"\(\d\d\d\d-\d\d-\d\d\)"
+version_pattern = r"\d+\.\d+(\.\d+)?(rc\d+)?"
+
+matches_version_header = re.compile(rf"^{version_pattern} {date_pattern}$").match
+
+
+def extract_relevant_contents(expected_version: str) -> str:
+    # This isn't solved with a pandoc filter as pandoc only recognizes the level 3
+    # headers as such.
+
+    line_feed = iter((CWD / "CHANGES.rst").read_text().splitlines())
+
+    while True:
+        line = next(line_feed)
+        if matches_version_header(line) and line.startswith(expected_version):
+            break
+
+    assert next(line_feed)[0] == "-"
+    assert next(line_feed) == ""
+
+    lines: list[str] = []
+    while not matches_version_header(line := next(line_feed)):
+        lines.append(line)
+
+    while lines[-1] == "":
+        lines.pop()
+
+    return "\n".join(lines)
+
+
+def make_github_release(notes: str, version: str):
+    options = []
+    options.extend(["--notes-file", "-"])
+    options.extend(["--title", f"delb {version}"])
+    options.append("--verify-tag")
+    if "rc" in version:
+        options.append("--prerelease")
+
+    result = subprocess.run(
+        [
+            "gh",
+            "release",
+            "create",
+        ]
+        + options
+        + [version],
+        capture_output=True,
+        encoding="utf-8",
+        input=notes,
+    )
+    print(result.stdout)
+    if result.returncode != 0:
+        print(result.stdout)
+    result.check_returncode()
+
+
+def make_release_notes(version: str) -> str:
+    os.environ["DELB_DOCS_BASE_URL"] = f"https://delb.readthedocs.io/en/{version}/"
+    return (
+        convert_text(
+            extract_relevant_contents(version),
+            format="rst",
+            to="markdown_strict",
+            extra_args=["--shift-heading-level-by=1", "--wrap=none"],
+            filters=[".github/scripts/release-notes-pandoc-filter.py"],
+        )
+        + "\n\n----\n\nThe package distributions are available at the "
+        + f"[Python Package Index](https://pypi.org/project/delb/)."
+    )
+
+
+def main(version: str):
+    download_pandoc()
+    make_github_release(notes=make_release_notes(version), version=version)
+
+
+if __name__ == "__main__":
+    main(sys.argv[1])

.github/scripts/release-notes-pandoc-filter.py

@@ -0,0 +1,59 @@
+import os
+import posixpath
+import sys
+from pathlib import Path
+
+from panflute import run_filter, Code, Link, Str
+from sphinx.util.inventory import InventoryFile
+
+
+BASE_URL = os.environ["DELB_DOCS_BASE_URL"]
+CWD = Path.cwd()
+ROLES_MAPPING = {
+    "attr": "py:attribute",
+    "class": "py:class",
+    "doc": "std:doc",
+    # ?"exception"?: "py:exception",
+    "func": "py:function",
+    "meth": "py:method",
+    "mod": "py:module",
+    # ???: "py:property",
+    "term": "std:term",
+}
+
+
+def bend_links(elem, doc):
+    if not (isinstance(elem, Code) and "interpreted-text" in elem.classes):
+        return elem
+
+    inventory_section = ROLES_MAPPING[elem.attributes["role"]]
+    target = doc.inventory[inventory_section].get(elem.text)
+
+    if target is None:
+        # this is okay for non-existing entries such as :meth:`NodeBase.…` and
+        # objects in other inventories like Python's docs
+        print(f"WARNING: No inventory object for '{elem.text}' found.", file=sys.stderr)
+        return Code(elem.text)
+
+    if (label_text := target[3]) == "-":
+        label_text = elem.text
+
+    if inventory_section.startswith("py:"):
+        label = Code(label_text)
+    else:
+        label = Str(f"„{label_text}”")
+
+    return Link(label, url=target[2])
+
+
+def prepare(doc):
+    with (CWD / "docs" / "build" / "html" / "objects.inv").open("rb") as f:
+        doc.inventory = InventoryFile.load(f, BASE_URL, posixpath.join)
+
+
+def main(doc=None):
+    return run_filter(bend_links, prepare=prepare)
+
+
+if __name__ == "__main__":
+    main()

.github/workflows/linkcheck.yml

@@ -11,6 +11,6 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
         with:
-            python-version: "3.12"
-      - run: pip install hatch
+            python-version: "3.x"
+      - run: pipx install hatch
       - run: hatch run docs:linkcheck

.github/workflows/publish.yml

@@ -0,0 +1,44 @@
+---
+
+name: Publish delb
+on:  # yamllint disable-line
+  push:
+    tags: ["*"]
+
+
+jobs:
+  build-and-test:
+    uses: ./.github/workflows/quality-checks.yml
+    with:
+      ref: ${{ env.GITHUB_REF_NAME }}
+
+  upload:
+    name: Publish to the cheeseshop
+    if: startsWith(github.ref, 'refs/tags/')
+    needs: ["build-and-test"]
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/delb
+    permissions:
+      id-token: write
+      contents: write
+
+    steps:
+      - name: Download package
+        uses: actions/download-artifact@v4
+        with:
+          name: Packages
+          path: dist
+      - name: Upload package to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+      - name: Install dependencies for creating GH release
+        run: pip install panflute pypandoc sphinx
+      # TODO build docs, so that objects.inv is available
+      - name: Create GH release
+        run: >-
+          python .github/scripts/make-github-release.py ${{ github.ref_name }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+...

.github/workflows/quality-checks.yml

@@ -22,26 +22,59 @@ on:
 
 jobs:
 
+  build:
+    runs-on: ubuntu-latest
+    outputs:
+      python-versions: ${{ steps.baipp.outputs.supported_python_classifiers_json_array }}
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.ref }}
+      - uses: hynek/build-and-inspect-python-package@v2
+        id: baipp
+
   unit-tests:
+    needs: ["build"]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: extractions/setup-just@v2
+      - run: pipx install hatch
+      - uses: actions/download-artifact@v4
+        with:
+          name: Packages
+          path: dist
+      - run: tar xf dist/*.tar.gz --strip-components=1
+      - uses: actions/setup-python@v5
+        with:
+          cache: pip
+          python-version: 3.x
+      - run: just pytest
+
+  compatibility-tests:
+    needs: ["build", "unit-tests"]
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version:
-          - 3.8  # 2024-10
-          - 3.9  # 2025-10
-          - "3.10"  # 2026-10
-          - "3.11"  # 2027-10
-          - "3.12"  # 2028-10
+        python-version: ${{ fromJson(needs.build.outputs.python-versions) }}
     steps:
       - uses: actions/checkout@v4
         with:
           ref: ${{ inputs.ref || github.ref }}
       - uses: actions/setup-python@v5
         with:
+          cache: pip
           python-version: ${{ matrix.python-version }}
       - uses: extractions/setup-just@v2
-      - run: pip install hatch
-      - run: just pytest
+      - run: pipx install hatch
+      - uses: actions/download-artifact@v4
+        with:
+          name: Packages
+          path: dist
+      - run: |
+          rm -r _delb delb
+          test "$(find dist -name 'delb-*.whl' | wc -l)" -eq 1
+          just test-wheel "$(find dist -name 'delb-*.whl')"
 
   other-quality-checks:
     runs-on: ubuntu-latest
@@ -57,9 +90,10 @@ jobs:
           ref: ${{ inputs.ref || github.ref }}
       - uses: actions/setup-python@v5
         with:
-          python-version: "3.12"
+          cache: pip
+          python-version: "3.x"
       - uses: extractions/setup-just@v2
-      - run: pip install hatch
+      - run: pipx install hatch
       - run: just ${{ matrix.target }}
 
 ...

Justfile

@@ -3,6 +3,10 @@ default: tests
 
 version := `hatch version`
 
+_assert_no_dev_version:
+  #!/usr/bin/env python3
+  if "dev" in "{{version}}":
+    raise SystemExit(1)
 
 # run benchmarks
 benchmarks:
@@ -38,18 +42,14 @@ mypy:
 pytest:
     hatch run unit-tests:check
 
-# release the current version on github & the PyPI
-release: tests
-    test "{{trim_end_match(version, '-dev')}}" = "{{version}}" || false
+# release the current version on github & (transitively) the PyPI
+release: _assert_no_dev_version tests
     {{just_executable()}} -f {{justfile()}} update-citation-file
     git add CITATION.cff
     git commit -m "Updates CITATION.cff"
     git tag {{version}}
     git push origin main
     git push origin {{version}}
-    hatch clean
-    hatch build
-    hatch publish
 
 # watch, build and serve HTML documentation at 0.0.0.0:8000
 serve-docs:
@@ -62,6 +62,10 @@ show-docs: docs
 # run all tests on normalized code
 tests: black code-lint mypy pytest doctest
 
+# run the testsuite against a wheel (installed from $WHEEL_PATH); intended to run on a CI platform
+test-wheel $WHEEL_PATH:
+    hatch run test-wheel:check
+
 # Generates and validates CITATION.cff
 update-citation-file:
     hatch run citation-file:cff-from-621

pyproject.toml

@@ -205,3 +205,12 @@ coverage-report = """
         --cov=_delb --cov=delb \
         tests
 """
+
+[tool.hatch.envs.test-wheel]
+template = "unit-tests"
+skip-install = true
+extra-dependencies = [
+    "delb @ {root:uri}/{env:WHEEL_PATH}",
+]
+[tool.hatch.envs.test-wheel.scripts]
+check = "python -m pytest tests"