Skip to content

Upload Benchmark Results #28343

Upload Benchmark Results

Upload Benchmark Results #28343

# Copyright (c) Facebook, Inc. and its affiliates.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
name: Upload Benchmark Results
on:
workflow_dispatch:
inputs:
run_id:
description: 'workflow run id to use the artifacts from'
required: true
workflow_run:
workflows: ["Ubuntu Benchmark"]
types:
- completed
permissions:
contents: read
actions: read
statuses: write
jobs:
upload:
runs-on: ubuntu-latest
if: ${{ (github.event.workflow_run.conclusion == 'success' ||
github.event_name == 'workflow_dispatch') &&
github.repository == 'facebookincubator/velox' }}
steps:
- name: 'Download artifacts'
id: 'download'
uses: actions/github-script@v6
with:
script: |
const run_id = "${{ github.event.workflow_run.id || inputs.run_id }}";
let benchmark_run = await github.rest.actions.getWorkflowRun({
owner: context.repo.owner,
repo: context.repo.repo,
run_id: run_id,
});
let artifacts = await github.rest.actions.listWorkflowRunArtifacts({
owner: context.repo.owner,
repo: context.repo.repo,
run_id: run_id,
});
let result_artifact = artifacts.data.artifacts.filter((artifact) => {
return artifact.name == "benchmark-results"
})[0];
let pr_artifact = artifacts.data.artifacts.filter((artifact) => {
return artifact.name == "pr_number"
})[0];
let result_download = await github.rest.actions.downloadArtifact({
owner: context.repo.owner,
repo: context.repo.repo,
artifact_id: result_artifact.id,
archive_format: 'zip',
});
let pr_download = await github.rest.actions.downloadArtifact({
owner: context.repo.owner,
repo: context.repo.repo,
artifact_id: pr_artifact.id,
archive_format: 'zip',
});
var fs = require('fs');
fs.writeFileSync('${{github.workspace}}/benchmark-results.zip', Buffer.from(result_download.data));
fs.writeFileSync('${{github.workspace}}/pr_number.zip', Buffer.from(pr_download.data));
core.setOutput('contender_sha', benchmark_run.data.head_sha);
if (benchmark_run.data.event == 'push') {
core.setOutput('merge_commit_message', benchmark_run.data.head_commit.message);
} else {
core.setOutput('merge_commit_message', '');
}
- name: Extract artifact
id: extract
run: |
unzip benchmark-results.zip -d benchmark-results
unzip pr_number.zip
pr_number=$(grep -ox '[[:digit:]]*' pr_number.txt | head -1)
if [ "$pr_number" -ge 0 ]; then
echo "Found PR number: $pr_number"
else
echo '::error :: Malformed input, aborting!'
exit 1
fi
echo "pr_number=$pr_number" >> $GITHUB_OUTPUT
- uses: actions/checkout@v3
with:
path: velox
- uses: actions/setup-python@v4
with:
python-version: '3.8'
cache: 'pip'
cache-dependency-path: "velox/scripts/*"
- name: "Install dependencies"
run: python -m pip install -r velox/scripts/benchmark-requirements.txt
- name: "Upload results"
env:
CONBENCH_URL: "https://velox-conbench.voltrondata.run/"
CONBENCH_MACHINE_INFO_NAME: "GitHub-runner-8-core"
CONBENCH_EMAIL: "${{ secrets.CONBENCH_EMAIL }}"
CONBENCH_PASSWORD: "${{ secrets.CONBENCH_PASSWORD }}"
CONBENCH_PROJECT_REPOSITORY: "${{ github.repository }}"
CONBENCH_PROJECT_COMMIT: "${{ steps.download.outputs.contender_sha }}"
run: |
if [ "${{ steps.extract.outputs.pr_number }}" -gt 0]; then
export CONBENCH_PROJECT_PR_NUMBER="${{ steps.extract.outputs.pr_number }}"
fi
./velox/scripts/benchmark-runner.py upload \
--run_id "GHA-${{ github.run_id }}-${{ github.run_attempt }}" \
--pr_number "${{ steps.extract.outputs.pr_number }}" \
--sha "${{ steps.download.outputs.contender_sha }}" \
--output_dir "${{ github.workspace }}/benchmark-results/contender/"
- name: "Check the status of the upload"
# Status functions like failure() only work in `if:`
if: failure()
id: status
run: echo "failed=true" >> $GITHUB_OUTPUT
- name: "Create a GitHub Status on the contender commit (whether the upload was successful)"
uses: actions/github-script@v6
if: ${{ !cancelled() && steps.extract.conclusion != 'failure' }}
with:
script: |
let url = 'https://github.com/${{github.repository}}/actions/runs/${{ github.run_id }}'
let state = 'success'
let description = 'Result upload succeeded!'
if(${{ steps.status.outputs.failed || false }}) {
state = 'failure'
description = 'Result upload failed!'
}
github.rest.repos.createCommitStatus({
owner: context.repo.owner,
repo: context.repo.repo,
sha: '${{ steps.download.outputs.contender_sha }}',
state: state,
target_url: url,
description: description,
context: 'Benchmark Result Upload'
})
- name: Create a GitHub Check benchmark report on the contender comment, and if merge-commit, a comment on the merged PR
env:
CONBENCH_URL: "https://velox-conbench.voltrondata.run/"
GITHUB_APP_ID: "${{ secrets.GH_APP_ID }}"
GITHUB_APP_PRIVATE_KEY: "${{ secrets.GH_APP_PRIVATE_KEY }}"
run: |
./velox/scripts/benchmark-alert.py \
--contender-sha "${{ steps.download.outputs.contender_sha }}" \
--merge-commit-message "${{ steps.download.outputs.merge_commit_message }}" \
--z-score-threshold 50