From d329702b6935894381496294a873115e06973524 Mon Sep 17 00:00:00 2001 From: hanbings Date: Sun, 23 Jun 2024 13:02:12 -0400 Subject: [PATCH] Provide GNU test comparison comments for PRs in Github Actions. (#400) --- .github/workflows/ci.yml | 4 +- .github/workflows/comment.yml | 70 +++++++++ .github/workflows/compat.yml | 270 +++++++++++++++++----------------- 3 files changed, 207 insertions(+), 137 deletions(-) create mode 100644 .github/workflows/comment.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eccb9ff8..c6231caf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -117,8 +117,8 @@ jobs: cargo test --all --no-fail-fast ${{ matrix.cargo_flags }} env: CARGO_INCREMENTAL: "0" - RUSTFLAGS: '-Zprofile -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests -Cpanic=abort -Cdebug-assertions=off' - RUSTDOCFLAGS: '-Zprofile -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests -Cpanic=abort -Cdebug-assertions=off' + RUSTFLAGS: "-Zprofile -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests -Cpanic=abort -Cdebug-assertions=off" + RUSTDOCFLAGS: "-Zprofile -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests -Cpanic=abort -Cdebug-assertions=off" - name: Generate coverage data id: grcov diff --git a/.github/workflows/comment.yml b/.github/workflows/comment.yml new file mode 100644 index 00000000..de319e41 --- /dev/null +++ b/.github/workflows/comment.yml @@ -0,0 +1,70 @@ +on: + workflow_run: + workflows: [External testsuites] + types: [completed] + +name: Comment Test results on the PR + +jobs: + upload-pr-comment: + name: Upload PR comment + runs-on: ubuntu-latest + if: github.event_name == 'pull_request' + permissions: + actions: read + pull-requests: write + + steps: + - name: List Annotations + uses: actions/github-script@v7 + with: + script: | + let runs = await github.rest.checks.listForRef({ + owner: context.repo.owner, + repo: context.repo.repo, + ref: '${{ github.event.pull_request.head.sha }}' + }); + + let names = ['Run GNU findutils tests', 'Run BFS tests']; + let results = []; + runs.data.check_runs.filter(check => names.includes(check.name)).forEach(run => results.push(run)); + + let annotations = { data: []}; + for (let result of results) { + let run = await github.rest.checks.listAnnotations({ + owner: context.repo.owner, + repo: context.repo.repo, + check_run_id: result.id + }); + + run.data.forEach(data => { + annotations.data.push({ + run: result.name, + annotation: data + }); + }); + } + + let fs = require('fs'); + fs.writeFileSync('${{ github.workspace }}/annotations.json', JSON.stringify(annotations)); + - name: Comment on PR + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + let fs = require('fs'); + let annotations = JSON.parse(fs.readFileSync('${{ github.workspace }}/annotations.json', 'utf8')); + + let annotationContent = annotations + .data + .map(annotation => `${annotation.run}: ${annotation.annotation.message}`) + .join('\n'); + + console.log(annotationContent); + + github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: ${{ github.event.number }}, + body: 'GNU testsuite comparison:\n```\n' + annotationContent + '```' + }); \ No newline at end of file diff --git a/.github/workflows/compat.yml b/.github/workflows/compat.yml index 4b62b486..cddef701 100644 --- a/.github/workflows/compat.yml +++ b/.github/workflows/compat.yml @@ -7,143 +7,143 @@ jobs: name: Run GNU findutils tests runs-on: ubuntu-latest steps: - - name: Checkout findutils - uses: actions/checkout@v4 - with: - path: findutils - - name: Checkout GNU findutils - uses: actions/checkout@v4 - with: - repository: gnu-mirror-unofficial/findutils - path: findutils.gnu - ref: 5768a03ddfb5e18b1682e339d6cdd24ff721c510 - submodules: true - - name: Install `rust` toolchain - run: | - ## Install `rust` toolchain - rustup toolchain install stable --no-self-update -c rustfmt --profile minimal - rustup default stable - - name: Install dependencies - shell: bash - run: | - # Enable sources & install dependencies - sudo find /etc/apt/sources.list* -type f -exec sed -i 'p; s/^deb /deb-src /' '{}' + - sudo apt-get update - sudo apt-get build-dep findutils - - name: Run GNU tests - shell: bash - run: | - cd findutils - bash util/build-gnu.sh ||: - - name: Extract testing info - shell: bash - run: | - - name: Upload gnu-test-report - uses: actions/upload-artifact@v4 - with: - name: gnu-test-report - path: | - findutils.gnu/find/testsuite/*.log - findutils.gnu/xargs/testsuite/*.log - findutils.gnu/tests/**/*.log - - name: Upload gnu-result - uses: actions/upload-artifact@v4 - with: - name: gnu-result - path: gnu-result.json - - name: Download the result - uses: dawidd6/action-download-artifact@v6 - with: - workflow: compat.yml - workflow_conclusion: completed - name: gnu-result - repo: uutils/findutils - branch: main - path: dl - - name: Download the log - uses: dawidd6/action-download-artifact@v6 - with: - workflow: compat.yml - workflow_conclusion: completed - name: gnu-test-report - repo: uutils/findutils - branch: main - path: dl - - name: Compare failing tests against master - shell: bash - run: | - ./findutils/util/diff-gnu.sh ./dl ./findutils.gnu - - name: Compare against main results - shell: bash - run: | - mv dl/gnu-result.json latest-gnu-result.json - python findutils/util/compare_gnu_result.py + - name: Checkout findutils + uses: actions/checkout@v4 + with: + path: findutils + - name: Checkout GNU findutils + uses: actions/checkout@v4 + with: + repository: gnu-mirror-unofficial/findutils + path: findutils.gnu + ref: 5768a03ddfb5e18b1682e339d6cdd24ff721c510 + submodules: true + - name: Install `rust` toolchain + run: | + ## Install `rust` toolchain + rustup toolchain install stable --no-self-update -c rustfmt --profile minimal + rustup default stable + - name: Install dependencies + shell: bash + run: | + # Enable sources & install dependencies + sudo find /etc/apt/sources.list* -type f -exec sed -i 'p; s/^deb /deb-src /' '{}' + + sudo apt-get update + sudo apt-get build-dep findutils + - name: Run GNU tests + shell: bash + run: | + cd findutils + bash util/build-gnu.sh ||: + - name: Extract testing info + shell: bash + run: | + - name: Upload gnu-test-report + uses: actions/upload-artifact@v4 + with: + name: gnu-test-report + path: | + findutils.gnu/find/testsuite/*.log + findutils.gnu/xargs/testsuite/*.log + findutils.gnu/tests/**/*.log + - name: Upload gnu-result + uses: actions/upload-artifact@v4 + with: + name: gnu-result + path: gnu-result.json + - name: Download the result + uses: dawidd6/action-download-artifact@v6 + with: + workflow: compat.yml + workflow_conclusion: completed + name: gnu-result + repo: uutils/findutils + branch: main + path: dl + - name: Download the log + uses: dawidd6/action-download-artifact@v6 + with: + workflow: compat.yml + workflow_conclusion: completed + name: gnu-test-report + repo: uutils/findutils + branch: main + path: dl + - name: Compare failing tests against master + shell: bash + run: | + ./findutils/util/diff-gnu.sh ./dl ./findutils.gnu + - name: Compare against main results + shell: bash + run: | + mv dl/gnu-result.json latest-gnu-result.json + python findutils/util/compare_gnu_result.py bfs-tests: name: Run BFS tests runs-on: ubuntu-latest steps: - - name: Checkout findutils - uses: actions/checkout@v4 - with: - path: findutils - - name: Checkout BFS - uses: actions/checkout@v4 - with: - repository: tavianator/bfs - path: bfs - ref: '3.1.3' - - name: Install `rust` toolchain - run: | - ## Install `rust` toolchain - rustup toolchain install stable --no-self-update -c rustfmt --profile minimal - rustup default stable - - name: Install dependencies - shell: bash - run: | - # Enable sources & install dependencies - sudo find /etc/apt/sources.list* -type f -exec sed -i 'p; s/^deb /deb-src /' '{}' + - sudo apt-get update - sudo apt-get build-dep bfs - - name: Run BFS tests - shell: bash - run: | - cd findutils - bash util/build-bfs.sh ||: - - name: Upload bfs-test-report - uses: actions/upload-artifact@v4 - with: - name: bfs-test-report - path: bfs/tests.log - - name: Upload bfs-result - uses: actions/upload-artifact@v4 - with: - name: bfs-result - path: bfs-result.json - - name: Download the result - uses: dawidd6/action-download-artifact@v6 - with: - workflow: compat.yml - workflow_conclusion: completed - name: bfs-result - repo: uutils/findutils - branch: main - path: dl - - name: Download the log - uses: dawidd6/action-download-artifact@v6 - with: - workflow: compat.yml - workflow_conclusion: completed - name: bfs-test-report - repo: uutils/findutils - branch: main - path: dl - - name: Compare failing tests against main - shell: bash - run: | - ./findutils/util/diff-bfs.sh dl/tests.log bfs/tests.log - - name: Compare against main results - shell: bash - run: | - mv dl/bfs-result.json latest-bfs-result.json - python findutils/util/compare_bfs_result.py + - name: Checkout findutils + uses: actions/checkout@v4 + with: + path: findutils + - name: Checkout BFS + uses: actions/checkout@v4 + with: + repository: tavianator/bfs + path: bfs + ref: "3.1.3" + - name: Install `rust` toolchain + run: | + ## Install `rust` toolchain + rustup toolchain install stable --no-self-update -c rustfmt --profile minimal + rustup default stable + - name: Install dependencies + shell: bash + run: | + # Enable sources & install dependencies + sudo find /etc/apt/sources.list* -type f -exec sed -i 'p; s/^deb /deb-src /' '{}' + + sudo apt-get update + sudo apt-get build-dep bfs + - name: Run BFS tests + shell: bash + run: | + cd findutils + bash util/build-bfs.sh ||: + - name: Upload bfs-test-report + uses: actions/upload-artifact@v4 + with: + name: bfs-test-report + path: bfs/tests.log + - name: Upload bfs-result + uses: actions/upload-artifact@v4 + with: + name: bfs-result + path: bfs-result.json + - name: Download the result + uses: dawidd6/action-download-artifact@v6 + with: + workflow: compat.yml + workflow_conclusion: completed + name: bfs-result + repo: uutils/findutils + branch: main + path: dl + - name: Download the log + uses: dawidd6/action-download-artifact@v6 + with: + workflow: compat.yml + workflow_conclusion: completed + name: bfs-test-report + repo: uutils/findutils + branch: main + path: dl + - name: Compare failing tests against main + shell: bash + run: | + ./findutils/util/diff-bfs.sh dl/tests.log bfs/tests.log + - name: Compare against main results + shell: bash + run: | + mv dl/bfs-result.json latest-bfs-result.json + python findutils/util/compare_bfs_result.py \ No newline at end of file