Refactor codecov to perform a single upload and use OIDC authenticati…

…on (#4182)

- Archive coverage report from each job
- Combine coverage reports inside check job
- Make a single upload to codecov.io
- Replace use of CODECOV token with OIDC authentication
- Remove 'test' GHA environment as we no longer need it

AAP-24264

.github/workflows/tox.yml

@@ -1,17 +1,16 @@
 ---
 name: tox
 
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
 on:
   push: # only publishes pushes to the main branch to TestPyPI
     branches: # any integration branch but not tag
       - "main"
   pull_request:
   workflow_call:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: true
 jobs:
   pre:
     name: pre
@@ -48,7 +47,6 @@ jobs:
 
     env:
       PYTEST_REQPASS: 453
-    environment: test
     steps:
       - uses: actions/checkout@v4
         with:
@@ -101,25 +99,14 @@ jobs:
       - name: tox -e ${{ matrix.passed_name }}
         run: python3 -m tox -e ${{ matrix.passed_name }}
 
-      - name: Combine coverage data
-        if: ${{ startsWith(matrix.passed_name, 'py') }}
-        # produce a single .coverage file at repo root
-        run: tox -e coverage
-
       - name: Archive logs
         uses: actions/upload-artifact@v4
         with:
           name: logs-${{ matrix.name }}.zip
-          path: .tox/**/log/
-
-      - name: Upload coverage data
-        if: ${{ startsWith(matrix.passed_name, 'py') }}
-        uses: codecov/codecov-action@v4
-        with:
-          name: ${{ matrix.name }}
-          token: ${{ secrets.CODECOV_TOKEN }}
-          verbose: true
-          fail_ci_if_error: true
+          path: |
+            .tox/**/log/
+            .tox/**/.coverage*
+            .tox/**/coverage.xml
 
       - name: Report failure if git reports dirty status
         run: |
@@ -130,20 +117,67 @@ jobs:
             exit 99
           fi
         # https://github.com/actions/toolkit/issues/193
+
   check:
     if: always()
+    permissions:
+      id-token: write
+      checks: read
 
     needs:
       - build
     runs-on: ubuntu-latest
+
     steps:
+      # checkout needed for codecov action which needs codecov.yml file
+      - uses: actions/checkout@v4
+
+      - name: Set up Python # likely needed for coverage
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - run: pip3 install 'coverage>=7.5.1'
+
       - name: Merge logs into a single archive
         uses: actions/upload-artifact/merge@v4
         with:
           name: logs.zip
           pattern: logs-*.zip
-          delete-merged: true
+          # artifacts like py312.zip and py312-macos do have overlapping files
+          separate-directories: true
+
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: logs.zip
+          path: .
+
+      - name: Check for expected number of coverage.xml reports
+        run: |
+          JOBS_PRODUCING_COVERAGE=4
+          if [ "$(find . -name coverage.xml | wc -l | bc)" -ne "${JOBS_PRODUCING_COVERAGE}" ]; then
+            echo "::error::Number of coverage.xml files was not the expected one (${JOBS_PRODUCING_COVERAGE}): $(find . -name coverage.xml |xargs echo)"
+            exit 1
+          fi
+
+      - name: Upload coverage data
+        uses: codecov/codecov-action@v4
+        with:
+          name: ${{ matrix.passed_name }}
+          fail_ci_if_error: true
+          use_oidc: true # cspell:ignore oidc
+
+      - name: Check codecov.io status
+        if: github.event_name == 'pull_request'
+        uses: coactions/codecov-status@main
+
       - name: Decide whether the needed jobs succeeded or failed
         uses: re-actors/alls-green@release/v1
         with:
           jobs: ${{ toJSON(needs) }}
+
+      - name: Delete Merged Artifacts
+        uses: actions/upload-artifact/merge@v4
+        with:
+          delete-merged: true

tox.ini

@@ -17,7 +17,17 @@ isolated_build = True
 usedevelop = true
 skip_install = false
 
-# do not put * in passenv as it may break builds do to reduced isolation
+commands_pre =
+  sh -c "rm -f {envdir}/.coverage.* 2>/dev/null || true"
+commands =
+  ansibledevel: ansible-galaxy install git+https://github.com/ansible-collections/community.general.git
+  # failsafe as pip may install incompatible dependencies
+  pip check
+  # failsafe for preventing changes that may break pytest collection
+  sh -c "PYTEST_ADDOPTS= python3 -m pytest -p no:cov --collect-only >>/dev/null"
+  # html report is used by Zuul CI to display reports
+  coverage run -m pytest {env:_EXTRAS} {env:PYTEST_ADDOPTS:} {posargs}
+  sh -c "coverage combine -a -q --data-file={envdir}/.coverage {toxworkdir}/*/.coverage.* && coverage xml --data-file={envdir}/.coverage -o {envdir}/coverage.xml --fail-under=0"
 passenv =
     CI
     CONTAINER_*
@@ -32,35 +42,26 @@ passenv =
     SSH_AUTH_SOCK
     TERM
 setenv =
-    ANSIBLE_CONFIG={toxinidir}/.ansible.cfg
-    ANSIBLE_DISPLAY_FAILED_STDERR=1
-    ANSIBLE_NOCOWS=1
-    ANSIBLE_VERBOSITY=1
-    COVERAGE_FILE={env:COVERAGE_FILE:{toxworkdir}/.coverage.{envname}}
-    COVERAGE_PROCESS_START={toxinidir}/pyproject.toml
-    MOLECULE_NO_LOG=0
-    PIP_DISABLE_PIP_VERSION_CHECK=1
-    PYTHONDONTWRITEBYTECODE=1
-    PYTHONUNBUFFERED=1
-    # Temporare remove "-n auto" as it seems to break coverage on this project.
-    _EXTRAS=-l
+  ANSIBLE_CONFIG={toxinidir}/.ansible.cfg
+  ANSIBLE_DISPLAY_FAILED_STDERR=1
+  ANSIBLE_NOCOWS=1
+  ANSIBLE_VERBOSITY=1
+  COVERAGE_FILE = {env:COVERAGE_FILE:{envdir}/.coverage.{envname}}
+  COVERAGE_PROCESS_START={toxinidir}/pyproject.toml
+  MOLECULE_NO_LOG=0
+  PIP_DISABLE_PIP_VERSION_CHECK=1
+  PYTHONDONTWRITEBYTECODE=1
+  PYTHONUNBUFFERED=1
+  # Temporare remove "-n auto" as it seems to break coverage on this project.
+  _EXTRAS=-l
 deps =
     devel: git+https://github.com/ansible/ansible#egg=ansible-core
     # pytest-molecule not used but we want to check that it does not conflict
     devel: git+https://github.com/ansible-community/pytest-molecule#egg=pytest-molecule
 extras =
     test
-commands =
-    ansibledevel: ansible-galaxy install git+https://github.com/ansible-collections/community.general.git
-    # failsafe as pip may install incompatible dependencies
-    pip check
-    # failsafe for preventing changes that may break pytest collection
-    sh -c "PYTEST_ADDOPTS= python3 -m pytest -p no:cov --collect-only >>/dev/null"
-    sh -c "rm -f .tox/.coverage.*"
-    # html report is used by Zuul CI to display reports
-    coverage run -m pytest {env:_EXTRAS} {env:PYTEST_ADDOPTS:} {posargs}
-
 allowlist_externals =
+    bash
     find
     rm
     sh
@@ -128,6 +129,7 @@ deps =
     molecule-plugins[azure,containers,docker,ec2,gce,podman,vagrant]>=23.0.0
     tox-ansible >= 1.5.1
     pipdeptree >= 2.0.0
+commands_pre =
 commands =
     pip check
     # disabled fails safe because we now install ansible-core on purpose
@@ -139,8 +141,6 @@ commands =
     molecule --version
     molecule drivers
     bash ./tools/smoketest.sh
-allowlist_externals =
-    bash
 
 [testenv:coverage]
 description = Combines and displays coverage results