diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a42280f7f..59711b570 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -164,7 +164,17 @@ jobs: - name: Benchmark tests run: | - make test-benchmark-compare + BENCHSTAT_OUTPUT_FILE=result.txt make test-benchmark-compare + - run: | + echo "\`\`\`" >> "$GITHUB_STEP_SUMMARY" + cat result.txt >> "$GITHUB_STEP_SUMMARY" + echo "\`\`\`" >> "$GITHUB_STEP_SUMMARY" + cat <> "$GITHUB_STEP_SUMMARY" +
+ The table shows the median and 90% confidence interval (CI) summaries for each benchmark comparing the HEAD and the BASE, and an A/B comparison under "vs base". The last column shows the statistical p-value with ten runs (n=10). + The last row has the Geometric Mean (geomean) for the given rows in the table. + Refer to [benchstat's documentation](https://pkg.go.dev/golang.org/x/perf/cmd/benchstat) for more help. + EOL ci-build-kube-state-metrics: name: ci-build-kube-state-metrics diff --git a/tests/compare_benchmarks.sh b/tests/compare_benchmarks.sh index b501e77d0..fcf5f0280 100755 --- a/tests/compare_benchmarks.sh +++ b/tests/compare_benchmarks.sh @@ -9,7 +9,7 @@ set -u [[ "$#" -le 2 ]] || echo "At least one argument required, $# provided." REF_CURRENT="$(git rev-parse --abbrev-ref HEAD)" -REF_TO_COMPARE=$1 +REF_TO_COMPARE="${1}" COUNT=${2:-"1"} @@ -42,4 +42,8 @@ echo "" echo "### Result" echo "old=${REF_TO_COMPARE} new=${REF_CURRENT}" -benchstat "${RESULT_TO_COMPARE}" "${RESULT_CURRENT}" +if [[ -z "${BENCHSTAT_OUTPUT_FILE}" ]]; then + benchstat BASE="${RESULT_TO_COMPARE}" HEAD="${RESULT_CURRENT}" | tee "${BENCHSTAT_OUTPUT_FILE}" +else + benchstat BASE="${RESULT_TO_COMPARE}" HEAD="${RESULT_CURRENT}" +fi