Binding Perf Tests #17
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # This workflow is used to run benchmarks to compare a pull request (specified by the input 'pull_request') | |
| # and the current master. This workflow assumes the pull requests (mmtk-core PRs and the binding PRs are based on | |
| # their corresponding master). Otherwise the workflow may fail or return unexpected results. | |
| name: Binding Perf Tests | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| pull_request: | |
| description: The pull request number to run the perf tests. The workflow compares the specified pull request with the trunk. | |
| required: true | |
| concurrency: | |
| # Cancels pending runs when a PR gets updated. | |
| group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} | |
| cancel-in-progress: true | |
| jobs: | |
| # Figure out binding PRs. | |
| binding-refs: | |
| uses: ./.github/workflows/pr-binding-refs.yml | |
| with: | |
| pull_request: ${{ github.event.inputs.pull_request }} | |
| # Figure out the core PR. We need to get the specific commit and repo from a pull request number. | |
| mmtk-refs: | |
| runs-on: ubuntu-22.04 | |
| outputs: | |
| mmtk_repo: ${{ steps.print.outputs.mmtk_repo }} | |
| mmtk-ref: ${{ steps.print.outputs.mmtk_ref }} | |
| steps: | |
| - name: Get mmtk-core repo from pull_request | |
| uses: actions/github-script@v6 | |
| id: core-repo | |
| with: | |
| result-encoding: string | |
| script: | | |
| var res = (await github.rest.pulls.get({ | |
| owner: "mmtk", | |
| repo: "mmtk-core", | |
| pull_number: ${{ github.event.inputs.pull_request }} | |
| })) | |
| return res.data.head.repo.full_name | |
| - name: Get mmtk-core refs from pull_request | |
| uses: actions/github-script@v6 | |
| id: core-ref | |
| with: | |
| result-encoding: string | |
| script: | | |
| var res = (await github.rest.pulls.get({ | |
| owner: "mmtk", | |
| repo: "mmtk-core", | |
| pull_number: ${{ github.event.inputs.pull_request }} | |
| })) | |
| return res.data.head.sha | |
| - id: print | |
| run: | | |
| echo "::set-output name=mmtk_repo::${{ steps.core-repo.outputs.result }}" | |
| echo "::set-output name=mmtk_ref::${{ steps.core-ref.outputs.result }}" | |
| # Run perf compare for JikesRVM | |
| jikesrvm-perf-compare: | |
| runs-on: [self-hosted, Linux, freq-scaling-off] | |
| needs: [binding-refs, mmtk-refs] | |
| steps: | |
| # Trunk - we always use master from the mmtk org | |
| # - binding | |
| - name: Checkout JikesRVM Binding Trunk | |
| uses: actions/checkout@v2 | |
| with: | |
| repository: mmtk/mmtk-jikesrvm | |
| ref: master | |
| path: mmtk-jikesrvm-trunk | |
| - name: Checkout JikesRVM for Trunk | |
| working-directory: mmtk-jikesrvm-trunk | |
| run: ./.github/scripts/ci-checkout.sh | |
| # - core | |
| - name: Checkout MMTk Core | |
| uses: actions/checkout@v2 | |
| with: | |
| repository: mmtk/mmtk-core | |
| ref: master | |
| path: mmtk-core-trunk | |
| # Branch | |
| # - binding | |
| - name: Checkout JikesRVM Binding Branch | |
| uses: actions/checkout@v2 | |
| with: | |
| repository: ${{ needs.binding-refs.outputs.jikesrvm_binding_repo }} | |
| ref: ${{ needs.binding-refs.outputs.jikesrvm_binding_ref }} | |
| path: mmtk-jikesrvm-branch | |
| # - core | |
| - name: Checkout MMTk Core | |
| uses: actions/checkout@v2 | |
| with: | |
| repository: ${{ needs.mmtk-refs.outputs.mmtk_repo }} | |
| ref: ${{ needs.mmtk-refs.outputs.mmtk_ref }} | |
| path: mmtk-core-branch | |
| - name: Checkout JikesRVM for Branch | |
| working-directory: mmtk-jikesrvm-branch | |
| run: ./.github/scripts/ci-checkout.sh | |
| # Checkout perf-kit | |
| - name: Checkout Perf Kit | |
| uses: actions/checkout@v2 | |
| with: | |
| repository: mmtk/ci-perf-kit | |
| token: ${{ secrets.CI_ACCESS_TOKEN }} | |
| ref: "0.6.8" | |
| path: ci-perf-kit | |
| submodules: true | |
| # setup | |
| # Use rust-toolchain in the trunk (it doesnt matter much - if the toolchains defined in the trunk and the branch are different, we cant run anyway) | |
| - name: Setup Rust Toolchain | |
| run: echo "RUSTUP_TOOLCHAIN=`cat mmtk-core-trunk/rust-toolchain`" >> $GITHUB_ENV | |
| - name: Setup | |
| run: | | |
| mkdir -p ci-perf-kit/running/benchmarks/dacapo | |
| cp /usr/share/benchmarks/dacapo/dacapo-2006-10-MR2.jar ci-perf-kit/running/benchmarks/dacapo/ | |
| # run compare | |
| - uses: hasura/comment-progress@v2.1.0 | |
| with: | |
| github-token: ${{ secrets.CI_ACCESS_TOKEN }} | |
| repository: 'mmtk/mmtk-core' | |
| number: ${{ github.event.inputs.pull_request }} | |
| id: jikesrvm-perf-compare-comment | |
| message: 'Running benchmarks for JikesRVM...' | |
| recreate: true | |
| - name: Compare Performance | |
| id: run | |
| run: | | |
| JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-amd64 ./ci-perf-kit/scripts/jikesrvm-compare.sh mmtk-jikesrvm-trunk/ mmtk-core-trunk/ mmtk-jikesrvm-branch/ mmtk-core-branch/ jikesrvm-compare-report.md | |
| # set report.md to output | |
| - uses: pCYSl5EDgo/cat@master | |
| id: cat | |
| with: | |
| path: jikesrvm-compare-report.md | |
| # upload run results | |
| - uses: actions/upload-artifact@v2 | |
| with: | |
| name: jikesrvm-log | |
| path: ci-perf-kit/running/results/log | |
| - uses: actions/upload-artifact@v2 | |
| with: | |
| name: jikesrvm-compare-report.md | |
| path: jikesrvm-compare-report.md | |
| # report | |
| - uses: hasura/comment-progress@v2.1.0 | |
| if: always() | |
| with: | |
| github-token: ${{ secrets.CI_ACCESS_TOKEN }} | |
| repository: 'mmtk/mmtk-core' | |
| number: ${{ github.event.inputs.pull_request }} | |
| id: jikesrvm-perf-compare-comment | |
| message: ${{ steps.cat.outputs.text }} | |
| append: true | |
| # Cleanup logs (this is necessary for self-hosted runners) | |
| # [`always()`](https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#always) ensures we clean up the logs even if some steps fail | |
| - name: Clean up logs and reports | |
| if: always() | |
| run: | | |
| rm -rf ci-perf-kit/running/results/log/* | |
| rm jikesrvm-compare-report.md | |
| openjdk-perf-compare: | |
| runs-on: [self-hosted, Linux, freq-scaling-off] | |
| needs: [binding-refs, mmtk-refs] | |
| steps: | |
| # Trunk - we always use master from the mmtk org | |
| # - binding | |
| - name: Checkout OpenJDK Binding Trunk | |
| uses: actions/checkout@v2 | |
| with: | |
| repository: mmtk/mmtk-openjdk | |
| ref: master | |
| path: mmtk-openjdk-trunk | |
| - name: Checkout OpenJDK for Trunk | |
| working-directory: mmtk-openjdk-trunk | |
| run: ./.github/scripts/ci-checkout.sh | |
| # -core | |
| - name: Checkout MMTk Core | |
| uses: actions/checkout@v2 | |
| with: | |
| repository: mmtk/mmtk-core | |
| ref: master | |
| path: mmtk-core-trunk | |
| # Branch | |
| # - binding | |
| - name: Checkout OpenJDK Binding Branch | |
| uses: actions/checkout@v2 | |
| with: | |
| repository: ${{ needs.binding-refs.outputs.openjdk_binding_repo }} | |
| ref: ${{ needs.binding-refs.outputs.openjdk_binding_ref }} | |
| path: mmtk-openjdk-branch | |
| - name: Checkout OpenJDK for Branch | |
| working-directory: mmtk-openjdk-branch | |
| run: ./.github/scripts/ci-checkout.sh | |
| # - core | |
| - name: Checkout MMTk Core | |
| uses: actions/checkout@v2 | |
| with: | |
| repository: ${{ needs.mmtk-refs.outputs.mmtk_repo }} | |
| ref: ${{ needs.mmtk-refs.outputs.mmtk_ref }} | |
| path: mmtk-core-branch | |
| # checkout perf-kit | |
| - name: Checkout Perf Kit | |
| uses: actions/checkout@v2 | |
| with: | |
| repository: mmtk/ci-perf-kit | |
| token: ${{ secrets.CI_ACCESS_TOKEN }} | |
| ref: "0.6.8" | |
| path: ci-perf-kit | |
| submodules: true | |
| # setup | |
| # Use rust-toolchain in the trunk (it doesnt matter much - if the toolchains defined in the trunk and the branch are different, we cant run anyway) | |
| - name: Setup Rust Toolchain | |
| run: echo "RUSTUP_TOOLCHAIN=`cat mmtk-core-trunk/rust-toolchain`" >> $GITHUB_ENV | |
| - name: Setup | |
| run: | | |
| mkdir -p ci-perf-kit/running/benchmarks/dacapo | |
| cp /usr/share/benchmarks/dacapo/dacapo-2006-10-MR2.jar ci-perf-kit/running/benchmarks/dacapo/ | |
| # run compare | |
| - uses: hasura/comment-progress@v2.1.0 | |
| with: | |
| github-token: ${{ secrets.CI_ACCESS_TOKEN }} | |
| repository: 'mmtk/mmtk-core' | |
| number: ${{ github.event.inputs.pull_request }} | |
| id: openjdk-perf-compare-comment | |
| message: 'Running benchmarks for OpenJDK...' | |
| recreate: true | |
| - name: Compare Performance | |
| id: run | |
| run: | | |
| ./ci-perf-kit/scripts/openjdk-compare.sh mmtk-openjdk-trunk/ mmtk-core-trunk/ mmtk-openjdk-branch/ mmtk-core-branch/ openjdk-compare-report.md | |
| # set report.md to output | |
| - uses: pCYSl5EDgo/cat@master | |
| id: cat | |
| with: | |
| path: openjdk-compare-report.md | |
| # upload run results | |
| - uses: actions/upload-artifact@v2 | |
| with: | |
| name: openjdk-log | |
| path: ci-perf-kit/running/results/log | |
| - uses: actions/upload-artifact@v2 | |
| with: | |
| name: openjdk-compare-report.md | |
| path: openjdk-compare-report.md | |
| # report | |
| - uses: hasura/comment-progress@v2.1.0 | |
| if: always() | |
| with: | |
| github-token: ${{ secrets.CI_ACCESS_TOKEN }} | |
| repository: 'mmtk/mmtk-core' | |
| number: ${{ github.event.inputs.pull_request }} | |
| id: openjdk-perf-compare-comment | |
| message: ${{ steps.cat.outputs.text }} | |
| append: true | |
| # Cleanup logs (this is necessary for self-hosted runners) | |
| - name: Clean up logs and reports | |
| if: always() | |
| run: | | |
| rm -rf ci-perf-kit/running/results/log/* | |
| rm openjdk-compare-report.md |