Skip to content

Commit

Permalink
Provide GNU test comparison comments for PRs in Github Actions. (uuti…
Browse files Browse the repository at this point in the history
  • Loading branch information
hanbings committed Jun 23, 2024
1 parent 84e4be8 commit d329702
Show file tree
Hide file tree
Showing 3 changed files with 207 additions and 137 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -117,8 +117,8 @@ jobs:
cargo test --all --no-fail-fast ${{ matrix.cargo_flags }}
env:
CARGO_INCREMENTAL: "0"
RUSTFLAGS: '-Zprofile -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests -Cpanic=abort -Cdebug-assertions=off'
RUSTDOCFLAGS: '-Zprofile -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests -Cpanic=abort -Cdebug-assertions=off'
RUSTFLAGS: "-Zprofile -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests -Cpanic=abort -Cdebug-assertions=off"
RUSTDOCFLAGS: "-Zprofile -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests -Cpanic=abort -Cdebug-assertions=off"

- name: Generate coverage data
id: grcov
Expand Down
70 changes: 70 additions & 0 deletions .github/workflows/comment.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
on:
workflow_run:
workflows: [External testsuites]
types: [completed]

name: Comment Test results on the PR

jobs:
upload-pr-comment:
name: Upload PR comment
runs-on: ubuntu-latest
if: github.event_name == 'pull_request'
permissions:
actions: read
pull-requests: write

steps:
- name: List Annotations
uses: actions/github-script@v7
with:
script: |
let runs = await github.rest.checks.listForRef({
owner: context.repo.owner,
repo: context.repo.repo,
ref: '${{ github.event.pull_request.head.sha }}'
});
let names = ['Run GNU findutils tests', 'Run BFS tests'];
let results = [];
runs.data.check_runs.filter(check => names.includes(check.name)).forEach(run => results.push(run));
let annotations = { data: []};
for (let result of results) {
let run = await github.rest.checks.listAnnotations({
owner: context.repo.owner,
repo: context.repo.repo,
check_run_id: result.id
});
run.data.forEach(data => {
annotations.data.push({
run: result.name,
annotation: data
});
});
}
let fs = require('fs');
fs.writeFileSync('${{ github.workspace }}/annotations.json', JSON.stringify(annotations));
- name: Comment on PR
uses: actions/github-script@v7
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
let fs = require('fs');
let annotations = JSON.parse(fs.readFileSync('${{ github.workspace }}/annotations.json', 'utf8'));
let annotationContent = annotations
.data
.map(annotation => `${annotation.run}: ${annotation.annotation.message}`)
.join('\n');
console.log(annotationContent);
github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: ${{ github.event.number }},
body: 'GNU testsuite comparison:\n```\n' + annotationContent + '```'
});
270 changes: 135 additions & 135 deletions .github/workflows/compat.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,143 +7,143 @@ jobs:
name: Run GNU findutils tests
runs-on: ubuntu-latest
steps:
- name: Checkout findutils
uses: actions/checkout@v4
with:
path: findutils
- name: Checkout GNU findutils
uses: actions/checkout@v4
with:
repository: gnu-mirror-unofficial/findutils
path: findutils.gnu
ref: 5768a03ddfb5e18b1682e339d6cdd24ff721c510
submodules: true
- name: Install `rust` toolchain
run: |
## Install `rust` toolchain
rustup toolchain install stable --no-self-update -c rustfmt --profile minimal
rustup default stable
- name: Install dependencies
shell: bash
run: |
# Enable sources & install dependencies
sudo find /etc/apt/sources.list* -type f -exec sed -i 'p; s/^deb /deb-src /' '{}' +
sudo apt-get update
sudo apt-get build-dep findutils
- name: Run GNU tests
shell: bash
run: |
cd findutils
bash util/build-gnu.sh ||:
- name: Extract testing info
shell: bash
run: |
- name: Upload gnu-test-report
uses: actions/upload-artifact@v4
with:
name: gnu-test-report
path: |
findutils.gnu/find/testsuite/*.log
findutils.gnu/xargs/testsuite/*.log
findutils.gnu/tests/**/*.log
- name: Upload gnu-result
uses: actions/upload-artifact@v4
with:
name: gnu-result
path: gnu-result.json
- name: Download the result
uses: dawidd6/action-download-artifact@v6
with:
workflow: compat.yml
workflow_conclusion: completed
name: gnu-result
repo: uutils/findutils
branch: main
path: dl
- name: Download the log
uses: dawidd6/action-download-artifact@v6
with:
workflow: compat.yml
workflow_conclusion: completed
name: gnu-test-report
repo: uutils/findutils
branch: main
path: dl
- name: Compare failing tests against master
shell: bash
run: |
./findutils/util/diff-gnu.sh ./dl ./findutils.gnu
- name: Compare against main results
shell: bash
run: |
mv dl/gnu-result.json latest-gnu-result.json
python findutils/util/compare_gnu_result.py
- name: Checkout findutils
uses: actions/checkout@v4
with:
path: findutils
- name: Checkout GNU findutils
uses: actions/checkout@v4
with:
repository: gnu-mirror-unofficial/findutils
path: findutils.gnu
ref: 5768a03ddfb5e18b1682e339d6cdd24ff721c510
submodules: true
- name: Install `rust` toolchain
run: |
## Install `rust` toolchain
rustup toolchain install stable --no-self-update -c rustfmt --profile minimal
rustup default stable
- name: Install dependencies
shell: bash
run: |
# Enable sources & install dependencies
sudo find /etc/apt/sources.list* -type f -exec sed -i 'p; s/^deb /deb-src /' '{}' +
sudo apt-get update
sudo apt-get build-dep findutils
- name: Run GNU tests
shell: bash
run: |
cd findutils
bash util/build-gnu.sh ||:
- name: Extract testing info
shell: bash
run: |
- name: Upload gnu-test-report
uses: actions/upload-artifact@v4
with:
name: gnu-test-report
path: |
findutils.gnu/find/testsuite/*.log
findutils.gnu/xargs/testsuite/*.log
findutils.gnu/tests/**/*.log
- name: Upload gnu-result
uses: actions/upload-artifact@v4
with:
name: gnu-result
path: gnu-result.json
- name: Download the result
uses: dawidd6/action-download-artifact@v6
with:
workflow: compat.yml
workflow_conclusion: completed
name: gnu-result
repo: uutils/findutils
branch: main
path: dl
- name: Download the log
uses: dawidd6/action-download-artifact@v6
with:
workflow: compat.yml
workflow_conclusion: completed
name: gnu-test-report
repo: uutils/findutils
branch: main
path: dl
- name: Compare failing tests against master
shell: bash
run: |
./findutils/util/diff-gnu.sh ./dl ./findutils.gnu
- name: Compare against main results
shell: bash
run: |
mv dl/gnu-result.json latest-gnu-result.json
python findutils/util/compare_gnu_result.py
bfs-tests:
name: Run BFS tests
runs-on: ubuntu-latest
steps:
- name: Checkout findutils
uses: actions/checkout@v4
with:
path: findutils
- name: Checkout BFS
uses: actions/checkout@v4
with:
repository: tavianator/bfs
path: bfs
ref: '3.1.3'
- name: Install `rust` toolchain
run: |
## Install `rust` toolchain
rustup toolchain install stable --no-self-update -c rustfmt --profile minimal
rustup default stable
- name: Install dependencies
shell: bash
run: |
# Enable sources & install dependencies
sudo find /etc/apt/sources.list* -type f -exec sed -i 'p; s/^deb /deb-src /' '{}' +
sudo apt-get update
sudo apt-get build-dep bfs
- name: Run BFS tests
shell: bash
run: |
cd findutils
bash util/build-bfs.sh ||:
- name: Upload bfs-test-report
uses: actions/upload-artifact@v4
with:
name: bfs-test-report
path: bfs/tests.log
- name: Upload bfs-result
uses: actions/upload-artifact@v4
with:
name: bfs-result
path: bfs-result.json
- name: Download the result
uses: dawidd6/action-download-artifact@v6
with:
workflow: compat.yml
workflow_conclusion: completed
name: bfs-result
repo: uutils/findutils
branch: main
path: dl
- name: Download the log
uses: dawidd6/action-download-artifact@v6
with:
workflow: compat.yml
workflow_conclusion: completed
name: bfs-test-report
repo: uutils/findutils
branch: main
path: dl
- name: Compare failing tests against main
shell: bash
run: |
./findutils/util/diff-bfs.sh dl/tests.log bfs/tests.log
- name: Compare against main results
shell: bash
run: |
mv dl/bfs-result.json latest-bfs-result.json
python findutils/util/compare_bfs_result.py
- name: Checkout findutils
uses: actions/checkout@v4
with:
path: findutils
- name: Checkout BFS
uses: actions/checkout@v4
with:
repository: tavianator/bfs
path: bfs
ref: "3.1.3"
- name: Install `rust` toolchain
run: |
## Install `rust` toolchain
rustup toolchain install stable --no-self-update -c rustfmt --profile minimal
rustup default stable
- name: Install dependencies
shell: bash
run: |
# Enable sources & install dependencies
sudo find /etc/apt/sources.list* -type f -exec sed -i 'p; s/^deb /deb-src /' '{}' +
sudo apt-get update
sudo apt-get build-dep bfs
- name: Run BFS tests
shell: bash
run: |
cd findutils
bash util/build-bfs.sh ||:
- name: Upload bfs-test-report
uses: actions/upload-artifact@v4
with:
name: bfs-test-report
path: bfs/tests.log
- name: Upload bfs-result
uses: actions/upload-artifact@v4
with:
name: bfs-result
path: bfs-result.json
- name: Download the result
uses: dawidd6/action-download-artifact@v6
with:
workflow: compat.yml
workflow_conclusion: completed
name: bfs-result
repo: uutils/findutils
branch: main
path: dl
- name: Download the log
uses: dawidd6/action-download-artifact@v6
with:
workflow: compat.yml
workflow_conclusion: completed
name: bfs-test-report
repo: uutils/findutils
branch: main
path: dl
- name: Compare failing tests against main
shell: bash
run: |
./findutils/util/diff-bfs.sh dl/tests.log bfs/tests.log
- name: Compare against main results
shell: bash
run: |
mv dl/bfs-result.json latest-bfs-result.json
python findutils/util/compare_bfs_result.py

0 comments on commit d329702

Please sign in to comment.