⏱️ Benchmark execution time comparison with the main branch #4464
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: ⏱️ Benchmark execution time comparison with the main branch | |
| on: | |
| issue_comment: | |
| types: [created, edited, deleted] | |
| jobs: | |
| building-pr-branch: | |
| if: (github.event.issue.pull_request != null) && github.event.comment.body == '!github easy-benchmark' | |
| runs-on: ubuntu-20.04 | |
| timeout-minutes: 15 | |
| steps: | |
| - name: Checkout to PR branch | |
| uses: actions/checkout@v3 | |
| - name: Install requirements | |
| run: sudo ./.github/scripts/dependency.sh | |
| - name: Setup Rust toolchain and cache | |
| uses: actions-rust-lang/setup-rust-toolchain@v1.3.7 | |
| - name: Install Just | |
| uses: taiki-e/install-action@just | |
| - name: Building PR branch | |
| run: just youki-release | |
| - name: Uploading PR build to artifact | |
| uses: actions/upload-artifact@v2 | |
| with: | |
| name: pr-youki | |
| path: ./youki | |
| building-main-branch: | |
| if: (github.event.issue.pull_request != null) && github.event.comment.body == '!github easy-benchmark' | |
| runs-on: ubuntu-20.04 | |
| timeout-minutes: 15 | |
| steps: | |
| - name: Checkout to main branch | |
| uses: actions/checkout@v3 | |
| with: | |
| ref: main | |
| - name: Install requirements | |
| run: sudo ./.github/scripts/dependency.sh | |
| - name: Setup Rust toolchain and cache | |
| uses: actions-rust-lang/setup-rust-toolchain@v1.3.7 | |
| - name: Install just | |
| uses: taiki-e/install-action@just | |
| - name: Building main branch | |
| run: just youki-release | |
| - name: Uploading main build to artifact | |
| uses: actions/upload-artifact@v2 | |
| with: | |
| name: main-youki | |
| path: ./youki | |
| benchmark-exec: | |
| if: (github.event.issue.pull_request != null) && github.event.comment.body == '!github easy-benchmark' | |
| needs: | |
| - building-pr-branch | |
| - building-main-branch | |
| runs-on: ubuntu-20.04 | |
| timeout-minutes: 15 | |
| steps: | |
| - name: Setup Linux env | |
| run: | | |
| sudo apt -y update | |
| sudo apt install jq podman | |
| - name: Checkout to PR branch | |
| uses: actions/checkout@v3 | |
| - name: Downloading PR build from artifact | |
| uses: actions/download-artifact@v2 | |
| with: | |
| name: pr-youki | |
| path: ./pr_youki | |
| - name: Downloading main build from artifact | |
| uses: actions/download-artifact@v2 | |
| with: | |
| name: main-youki | |
| path: ./main_youki | |
| - name: Preparing binaries | |
| run: | | |
| mv ./main_youki/youki ./youki_main | |
| mv ./pr_youki/youki ./youki_pr | |
| chmod +x ./youki_main | |
| chmod +x ./youki_pr | |
| - name: Preparing benchmark bundle | |
| run: | | |
| mkdir -p ./benchmark_exec_bundle/rootfs | |
| cd benchmark_exec_bundle | |
| sudo bash -c 'podman export $(podman create busybox) | tar -C rootfs -xvf -' | |
| ../youki_main spec | |
| sed -i 's/"sh"/"sh", "-c", "echo Hi, my PID is $$; echo Bye Bye"/' config.json | |
| sed -i 's/"terminal": true/"terminal": false/' config.json | |
| - name: Installing hyperfine | |
| run: | | |
| curl -s https://api.github.com/repos/sharkdp/hyperfine/releases/latest \ | |
| | jq -r '.assets[] | select(.name? | match("hyperfine-musl_.*_amd64.deb")) | .browser_download_url' \ | |
| | wget -qi - -O hyperfine.deb | |
| sudo dpkg -i hyperfine.deb | |
| - name: Running benchmark | |
| run: | | |
| hyperfine --prepare 'sudo sync; echo 3 | sudo tee /proc/sys/vm/drop_caches' \ | |
| --export-markdown bench_exec_result.md \ | |
| --warmup 10 \ | |
| --min-runs 100 \ | |
| --command-name 'main youki build' \ | |
| 'sudo ./youki_main run -b benchmark_exec_bundle main_container && sudo ./youki_main delete -f main_container' \ | |
| --command-name 'pr youki build' \ | |
| 'sudo ./youki_pr run -b benchmark_exec_bundle pr_container && sudo ./youki_pr delete -f pr_container' | |
| - name: Adding info into result | |
| run: | | |
| sed -i '1i > commit (${{ github.event.pull_request.head.sha }})\n' bench_exec_result.md | |
| # since the GITHUB_TOKEN is needed to let the bot commit messages in the PR | |
| # but right now it is controlled by the organization. | |
| # TODO: change back to use this when the permission granted | |
| # - name: Writing report to PR comment | |
| # uses: marocchino/sticky-pull-request-comment@v2 | |
| # with: | |
| # append: true | |
| # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| # path: ./bench_exec_result.md | |
| # temp solution, print the benchmark result in the terminal. | |
| # glow is the prettiest markdown rendering tool I know. | |
| # TODO: remove this step when GITHUB_TOKEN permission granted | |
| - name: Printing report in the terminal | |
| run: | | |
| curl -s https://api.github.com/repos/charmbracelet/glow/releases/latest \ | |
| | jq -r '.assets[] | select(.name? | match("glow_.*_linux_amd64.deb")) | .browser_download_url' \ | |
| | wget -qi - -O glow.deb | |
| sudo dpkg -i glow.deb | |
| glow bench_exec_result.md |