Maintenance of testing and coverage

.github/workflows/tox.yml

@@ -44,11 +44,20 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0 # needed by setuptools-scm
+          submodules: true
 
+      - name: Set pre-commit cache
+        uses: actions/cache@v3
+        if: ${{ matrix.passed_name == 'lint' }}
+        with:
+          path: |
+            ~/.cache/pre-commit
+          key: pre-commit-${{ matrix.name || matrix.passed_name }}-${{ hashFiles('.pre-commit-config.yaml') }}
       - name: Set up Python ${{ matrix.python_version }}
         uses: actions/setup-python@v5
         with:
-          python-version: ${{ matrix.python_version }}
+          cache: pip
+          python-version: ${{ matrix.python_version || '3.10' }}
 
       - name: Install tox
         run: python3 -m pip install --upgrade "tox>=4.0.2"
@@ -63,15 +72,29 @@ jobs:
         continue-on-error: ${{ matrix.devel || false }}
         run: python3 -m tox -e ${{ matrix.passed_name }}
 
-      # - name: Upload coverage data
-      #   if: ${{ startsWith(matrix.passed_name, 'py') }}
-      #   uses: codecov/codecov-action@v3
-      #   with:
-      #     name: ${{ matrix.passed_name }}
-      #     fail_ci_if_error: false # see https://github.com/codecov/codecov-action/issues/598
-      #     token: ${{ secrets.CODECOV_TOKEN }}
-      #     verbose: true # optional (default = false)
+      - name: Upload coverage data
+        if: ${{ startsWith(matrix.passed_name, 'py') }}
+        uses: codecov/codecov-action@v3
+        with:
+          name: ${{ matrix.passed_name }}
+          token: ${{ secrets.CODECOV_TOKEN }}
+          verbose: true # optional (default = false)
 
+      - name: Archive logs
+        uses: actions/upload-artifact@v3
+        with:
+          name: logs.zip
+          path: .tox/**/log/
+
+      - name: Report failure if git reports dirty status
+        run: |
+          if [[ -n $(git status -s) ]]; then
+            # shellcheck disable=SC2016
+            echo -n '::error file=git-status::'
+            printf '### Failed as git reported modified and/or untracked files\n```\n%s\n```\n' "$(git status -s)" | tee -a "$GITHUB_STEP_SUMMARY"
+            exit 99
+          fi
+        # https://github.com/actions/toolkit/issues/193
   tox_passed:
     needs: tox
     runs-on: ubuntu-latest

.pre-commit-config.yaml

@@ -61,7 +61,6 @@ repos:
     rev: "1.3.1"
     hooks:
       - id: tox-ini-fmt
-
   - repo: https://github.com/astral-sh/ruff-pre-commit
     rev: "v0.1.11"
     hooks:

pyproject.toml

@@ -1,9 +1,9 @@
 [build-system]
-build-backend = "setuptools.build_meta"
 requires = [
-  "setuptools>=63", # required by pyproject+setuptools_scm integration
-  "setuptools_scm[toml]>=7.0.5" # required for "no-local-version" scheme
+  "setuptools >= 65.3.0", # required by pyproject+setuptools_scm integration and editable installs
+  "setuptools_scm[toml] >= 7.0.5" # required for "no-local-version" scheme
 ]
+build-backend = "setuptools.build_meta"
 
 [project]
 name = "tox-ansible"
@@ -41,6 +41,25 @@ tox-ansible = "tox_ansible.plugin"
 [tool.black]
 line-length = 100
 
+# Keep this default because xml/report do not know to use load it from config file:
+# data_file = ".coverage"
+[tool.coverage.paths]
+source = ["src", ".tox/*/site-packages"]
+
+[tool.coverage.report]
+exclude_lines = ["pragma: no cover", "if TYPE_CHECKING:"]
+omit = ["tests/*"]
+# Increase it just so it would pass on any single-python run
+fail_under = 21
+# skip_covered = true
+# skip_empty = true
+# During development we might remove code (files) with coverage data, and we dont want to fail:
+ignore_errors = true
+show_missing = true
+
+[tool.coverage.run]
+source = ["src"]
+
 [tool.pydoclint]
 allow-init-docstring = true
 arg-type-hints-in-docstring = false
@@ -74,7 +93,7 @@ enable = [
 ]
 
 [tool.pytest.ini_options]
-addopts = "-n=auto --dist=loadfile --maxfail=10 --durations=30 --showlocals"
+addopts = "--maxfail=10 --durations=30 --showlocals"
 
 [tool.ruff]
 fix = true

tests/conftest.py

@@ -94,5 +94,5 @@ def pytest_generate_tests(metafunc: Metafunc) -> None:
         metafunc.parametrize(
             "basic_environment",
             environment_configs,
-            ids=configs.sections(),
+            ids=[x.replace(":", "-") for x in configs.sections()],
         )

tests/integration/test_basic.py

@@ -9,6 +9,8 @@
 
 import pytest
 
+from tox_ansible.plugin import conf_passenv
+
 
 if TYPE_CHECKING:
     from pathlib import Path
@@ -92,3 +94,9 @@ def test_environment_config(
 
     assert "https://github.com/ansible/ansible/archive" in config["deps"]
     assert "XDG_CACHE_HOME" in config["set_env"]
+
+
+def test_import() -> None:
+    """Verify that module can be imported (used for coverage)."""
+    x = conf_passenv()
+    assert "GITHUB_TOKEN" in x

tox.ini

@@ -1,6 +1,7 @@
 [tox]
 requires =
-    tox>=4.2
+    setuptools>=65.3
+    tox>=4.6.3
 env_list =
     py
     lint
@@ -11,38 +12,44 @@ skip_missing_interpreters = true
 work_dir = {env:TOX_WORK_DIR:.tox}
 
 [testenv]
-description = Run pytest under {basepython} ({envpython})
+description = Run pytest under {basepython}
 package = editable
 extras =
     test
 pass_env =
     CI
     CONTAINER_*
+    COVERAGE_*
     DOCKER_*
     GITHUB_*
     HOME
+    LANG
+    LC_*
     PYTEST_*
     SSH_AUTH_SOCK
     TERM
     USER
 set_env =
-    COVERAGE_FILE = {env:COVERAGE_FILE:{toxworkdir}/.coverage.{envname}}
     COVERAGE_PROCESS_START = {toxinidir}/pyproject.toml
     FORCE_COLOR = 1
     PIP_CONSTRAINT = {toxinidir}/requirements.txt
     PRE_COMMIT_COLOR = always
     TERM = xterm-256color
-    py38, py39: PIP_CONSTRAINT = /dev/null
+    lint, py38, py39: PIP_CONSTRAINT = /dev/null
+commands_pre =
+    sh -c "rm -f .tox/.coverage* **/*.pyc 2>/dev/null || true"
 commands =
+    pip freeze
     coverage run -m pytest {posargs}
-    sh -c "coverage combine -q .tox/.coverage.* && coverage xml || true && coverage report"
+    coverage report
+    coverage xml
 allowlist_externals =
-    git
     rm
     sh
+editable = true
 
 [testenv:lint]
-description = Enforce quality standards under {basepython} ({envpython})
+description = Enforce quality standards under {basepython}
 skip_install = true
 deps =
     pre-commit