From bb7863c66d0b11093914a26687b46467468912f2 Mon Sep 17 00:00:00 2001 From: Soren Ptak Date: Thu, 31 Aug 2023 20:05:49 -0400 Subject: [PATCH] Coverage-Cop Action Changes (#71) * Changes to the coverage-cop to upload the HTML report * Use -e in the echo groups, update the test for coverage-cop --- .github/workflows/test.yml | 110 ++++++++++++++++++++++++++++++----- coverage-cop/action.yml | 115 +++++++++++++++++++++++++++++++------ 2 files changed, 195 insertions(+), 30 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 476af6fc..aa2c2855 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,6 +7,13 @@ on: branches: [main] workflow_dispatch: +env: + # The bash escape character is \033 + bashPass: \033[32;1mPASSED - + bashInfo: \033[33;1mINFO - + bashFail: \033[31;1mFAILED - + bashEnd: \033[0m + jobs: test-format-check: runs-on: ubuntu-20.04 @@ -281,41 +288,118 @@ jobs: uses: ./spellings with: path: coreMQTT + test-coverage-cop: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 + - uses: actions/checkout@v3 with: - repository: FreeRTOS/coreMQTT + repository: skptak/coreMQTT ref: main path: coreMQTT - - name: Build + + - env: + stepName: Build Core MQTT + name: ${{ env.stepName }} run: | + # ${{ env.stepName }} + exitStatus=0 + + echo -e "::group::Install Lcov" sudo apt-get install -y lcov + echo -e "::endgroup::" + + echo -e "::group::${{ env.stepName }}" cmake -S ./coreMQTT/test -B build/ \ -G "Unix Makefiles" \ -DCMAKE_BUILD_TYPE=Debug \ -DBUILD_CLONE_SUBMODULES=ON \ -DCMAKE_C_FLAGS='--coverage -Wall -Wextra -Werror -DNDEBUG -DLIBRARY_LOG_LEVEL=LOG_DEBUG' make -C build/ all - - name: Test - run: | - cd build/ - ctest -E system --output-on-failure - cd .. - - name: Run Coverage + exitStatus=$? + echo -e "::endgroup::" + + # Only get to here if we passed the build + echo -e "${{ env.bashPass }} ${{ env.stepName }} ${{ env.bashEnd }}" + + + - name: Run CTest for CoreMQTT + run: ctest --test-dir build -E system --output-on-failure + + - env: + stepName: Passing Coverage Test + shell: bash run: | + # ${{ env.stepName }} + echo -e "::group::Build Coverage for coreMQTT" make -C build/ coverage + echo -e "::endgroup::" + + echo -e "::group::${{ env.stepName }}" declare -a EXCLUDE=("\*test/\*" "\*CMakeCCompilerId\*" "\*mocks\*") - echo ${EXCLUDE[@]} | xargs lcov --rc lcov_branch_coverage=1 -r build/coverage.info -o build/coverage.info + echo ${EXCLUDE[@]} | xargs lcov --rc lcov_branch_coverage=1 --remove build/coverage.info -o build/coverage.info + echo -e "::endgroup::" + lcov --rc lcov_branch_coverage=1 --list build/coverage.info - - name: Test coverage cop action + echo -e "${{ env.bashPass }} ${{env.stepName }} ${{ env.bashEnd }}" + + - name: Test Coverage Cop Action + id: success-coverage-cop uses: ./coverage-cop with: - path: ./build/coverage.info + coverage-file: ./build/coverage.info branch-coverage-min: 70 line-coverage-min: 100 + + - env: + stepName: Create Lower Branch and Line Coverage Report + id: failure-coverage-action + continue-on-error: true + shell: bash + run: | + # ${{ env.stepName }} + echo -e "::group::Build Coverage for coreMQTT" + make -C build/ clean + make -C build/ coverage + echo -e "::endgroup::" + + echo -e "::group::${{ env.stepName }}" + declare -a EXCLUDE=("\*test/\*") + set +e + lcov --rc lcov_branch_coverage=1 --remove build/coverage.info -o build/coverage.info + echo -e "::endgroup::" + + lcov --rc lcov_branch_coverage=1 --list build/coverage.info + set -e + echo -e "${{ env.bashPass }} ${{env.stepName }} ${{ env.bashEnd }}" + + - name: Functional | Failure | Coverage Cop Failure Test Case + id: failure-coverage-cop-action + continue-on-error: true + uses: ./coverage-cop + with: + coverage-file: ./build/coverage.info + branch-coverage-min: 100 + line-coverage-min: 100 + + - env: + stepName: Check Failure Test Case + name: ${{ env.stepName }} + if: success() || failure() + id: check-failure-test-cases + shell: bash + run: | + # ${{ env.stepName }} + exitStatus=0 + if [ "${{ steps.failure-coverage-cop-action.outcome }}" = "failure" ]; then + echo -e "${{ env.bashPass }} Functional | Failure | Coverage Cop Failure Test Case | Had Expected "failure" ${{ env.bashEnd }}" + else + echo -e "${{ env.bashFail }} Functional | Failure | Coverage Cop Failure Test Case | Had Unexpected "success" ${{ env.bashEnd }}" + exitStatus=1 + fi + exit $exitStatus + test-memory-statistics: runs-on: ubuntu-latest steps: diff --git a/coverage-cop/action.yml b/coverage-cop/action.yml index 1cdb584a..4b21596b 100644 --- a/coverage-cop/action.yml +++ b/coverage-cop/action.yml @@ -1,42 +1,123 @@ name: 'coverage-cop' description: 'CI Check Coverage results of unit tests (using lcov)' inputs: - path: + coverage-file: description: 'Path to lcov output file containing coverage data.' required: true branch-coverage-min: description: 'The minumum required branch coverage (in %) for success' required: false - default: 100 + default: 95 line-coverage-min: description: 'The minumum required line coverage (in %) for success' required: false - default: 100 + default: 95 runs: using: "composite" - steps: - - name: Print coverage data - run: lcov --list --rc lcov_branch_coverage=1 ${{ inputs.path }} + steps: + - env: + stepName: Install Dependencies + bashPass: \033[32;1mPASSED - + bashInfo: \033[33;1mINFO - + bashFail: \033[31;1mFAILED - + bashEnd: \033[0 shell: bash - - name: Install lcov (if not present) - run: sudo apt-get install lcov + name: ${{ env.stepName }} + run: | + # ${{ env.stepName }} + echo -e "::group::${{ env.stepName }}" + sudo apt-get install lcov fd-find + echo -e "::endgroup::" + echo -e "${{ env.bashPass }} ${{env.stepName }} ${{ env.bashEnd }}" + + - env: + stepName: Check Line and Branch Coverage + bashPass: \033[32;1mPASSED - + bashInfo: \033[33;1mINFO - + bashFail: \033[31;1mFAILED - + bashEnd: \033[0 + name: ${{ env.stepName }} + id: action-check-line-and-branch-coverage shell: bash - - name: Check coverage run: | - LINE_COVERAGE=$(lcov --list ${{ inputs.path }} | tail -n 1 | cut -d '|' -f 2 | sed -n "s/\([^%]*\)%.*/\1/p") - BRANCH_COVERAGE=$(lcov --rc lcov_branch_coverage=1 --list ${{ inputs.path }} | tail -n 1 | cut -d '|' -f 4 | sed -n "s/\([^%]*\)%.*/\1/p") + # ${{ env.stepName }} + + # Print the received code cov report. + # TODO: The way it grabs the line/branch coverage is a little complicated + # I'd like to see if this can be done simpler. + echo -e " ${{ env.bashInfo }} Received LCov Report: ${{ inputs.line-coverage-min }} ${{ env.bashEnd}}" + lcov --list --rc lcov_branch_coverage=1 ${{ inputs.coverage-file }} + LINE_COVERAGE=$(lcov --list ${{ inputs.coverage-file }} | tail -n 1 | cut -d '|' -f 2 | sed -n "s/\([^%]*\)%.*/\1/p") + BRANCH_COVERAGE=$(lcov --rc lcov_branch_coverage=1 --list ${{ inputs.coverage-file }} | tail -n 1 | cut -d '|' -f 4 | sed -n "s/\([^%]*\)%.*/\1/p") RESULT=0 - echo "Required line coverage: ${{ inputs.line-coverage-min }}" - echo "Line coverage: $LINE_COVERAGE" + + # Check Line Coverage + echo -e " ${{ env.bashInfo }} Required Line Coverage: ${{ inputs.line-coverage-min }} ${{ env.bashEnd}}" + echo -e " ${{ env.bashInfo }} Received Line Coverage: $LINE_COVERAGE ${{ env.bashEnd}}" if [[ $(echo "$LINE_COVERAGE < ${{ inputs.line-coverage-min }}" | bc) -ne 0 ]]; then - echo "Line Coverage is too low." + echo -e "${{ env.bashFail }} Line Coverage is too low. ${{ env.bashEnd }}" RESULT=1 fi - echo "Required branch coverage: ${{ inputs.branch-coverage-min }}" - echo "Branch coverage: $BRANCH_COVERAGE" + + echo -e " ${{ env.bashInfo }} Required Branch Coverage: ${{ inputs.branch-coverage-min }} ${{ env.bashEnd}}" + echo -e " ${{ env.bashInfo }} Received Branch Coverage: $BRANCH_COVERAGE ${{ env.bashEnd}}" if [[ $(echo "$BRANCH_COVERAGE < ${{ inputs.branch-coverage-min }}" | bc) -ne 0 ]]; then - echo "Branch Coverage is too low." + echo -e "${{ env.bashFail }} Branch Coverage is too low. ${{ env.bashEnd }}" RESULT=1 fi + + if [ $RESULT -eq 0 ]; then + echo -e "${{ env.bashPass }} ${{ env.stepName }} ${{ env.bashEnd }}" + else + echo -e "::group::Create Failed Codecov HTML Report" + genhtml --rc lcov_branch_coverage=1 --ignore-errors source ${{ inputs.coverage-file }} --legend --title "$(basename `git rev-parse --show-toplevel`) $(git rev-parse HEAD)" --output-directory=CodecovHTMLReport + zip -r CodecovHTMLReport.zip CodecovHTMLReport + echo -e "::endgroup::" + echo -e "${{ env.bashFail }} ${{ env.stepName }} ${{ env.bashEnd }}" + fi exit $RESULT + + - name: Upload Failed Codecov HTML Report + if: failure() && ( steps.action-check-line-and-branch-coverage.outcome == 'failure' ) + id: upload-codecov-report + uses: actions/upload-artifact@v3 + with: + name: CodecovHTMLReport + path: CodecovHTMLReport.zip + retention-days: 5 + + - env: + stepName: Codecov Report Info + bashPass: \033[32;1m + bashInfo: \033[33;1m + bashFail: \033[31;1m + bashEnd: \033[0 + if: failure() && ( steps.upload-codecov-report.outcome == 'success' ) shell: bash + run: | + # ${{ env.stepName }} + echo -e "${{ env.bashInfo }} A zip file of the failed Codecov report has been attached to this workflow ${{ env.bashEnd }}" + echo -e "${{ env.bashInfo }} This can be accessed by returning to the bottom of the summary page of the workflow run ${{ env.bashEnd }}" + echo -e "${{ env.bashInfo }} At the bottom of the page will be a CodecovHTMLReport.zip file that you can download ${{ env.bashEnd }}" + echo -e "${{ env.bashInfo }} Unzip the file and then open the index.html file in your browser for more info missing branch and line coverage ${{ env.bashEnd }}" + exit 1 + + + +# We should use this - it creates a link on their website that you can access +# And it displays the results there, which is great. +# But it means we need to set up a Codecov dashboard for each repo and then +# Add that to a secret to use by default when using it. +# CorePKCS11 evidently has this, but I don't know who set that up or what approvals it took +# So For now I'm going to create the html report and add it to the run. +# More info here: https://about.Codecov.io/blog/how-to-set-up-Codecov-with-c-and-github-actions/ +# - env: +# stepName: Upload Line and Branch Report +# name: ${{ env.stepName }} +# if: failure() +# uses: Codecov/Codecov-action@v3 +# with: +# files: ${{ inputs.coverage-file }} +# flags: unit_tests +# fail_ci_if_error: false +# verbose: false