From 85fb5fdc3e5a460b16737cb768e745d15126e876 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Manuel=20R=C3=BCger?= <manuel@rueg.eu>
Date: Thu, 8 Feb 2024 23:54:15 +0100
Subject: [PATCH] Add benchmark tooling
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This adds a way to compare benchmarks that we use in kube-state-metrics
already, hopefully allowing for better comparisons when applying
changes.

Signed-off-by: Manuel Rüger <manuel@rueg.eu>
---
 .github/workflows/compare-benchmarks.yaml | 23 ++++++++++++
 Makefile                                  |  9 +++++
 tests/compare_benchmarks.sh               | 43 +++++++++++++++++++++++
 3 files changed, 75 insertions(+)
 create mode 100644 .github/workflows/compare-benchmarks.yaml
 create mode 100755 tests/compare_benchmarks.sh

diff --git a/.github/workflows/compare-benchmarks.yaml b/.github/workflows/compare-benchmarks.yaml
new file mode 100644
index 000000000..16ea8b112
--- /dev/null
+++ b/.github/workflows/compare-benchmarks.yaml
@@ -0,0 +1,23 @@
+---
+name: Benchmarks on AMD64
+permissions: read-all
+on: [push, pull_request]
+jobs:
+  ci-benchmark-tests:
+    name: ci-benchmark-tests
+    runs-on: ubuntu-latest
+    steps:
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v4
+    - id: goversion
+      run: echo "goversion=$(cat .go-version)" >> "$GITHUB_OUTPUT"
+
+    - name: Set up Go 1.x
+      uses: actions/setup-go@v5
+      with:
+        go-version: ${{ steps.goversion.outputs.goversion }}
+      id: go
+
+    - name: Benchmark tests
+      run: |
+        make test-benchmark-compare
diff --git a/Makefile b/Makefile
index b0d019802..ce844ce65 100644
--- a/Makefile
+++ b/Makefile
@@ -1,5 +1,6 @@
 BRANCH=`git rev-parse --abbrev-ref HEAD`
 COMMIT=`git rev-parse --short HEAD`
+LATEST_RELEASE_BRANCH := release-$(shell grep -ohE "[0-9]+.[0-9]+" version/version.go)
 GOLDFLAGS="-X main.branch $(BRANCH) -X main.commit $(COMMIT)"
 GOFILES = $(shell find . -name \*.go)
 
@@ -94,3 +95,11 @@ test-failpoint:
 test-robustness: gofail-enable build
 	sudo env PATH=$$PATH go test -v ${TESTFLAGS} ./tests/dmflakey -test.root
 	sudo env PATH=$(PWD)/bin:$$PATH go test -v ${TESTFLAGS} ${ROBUSTNESS_TESTFLAGS} ./tests/robustness -test.root
+
+.PHONY: test-benchmark-compare
+# Runs benchmark tests on the current git ref and the last release and compares
+# the two.
+test-benchmark-compare:
+	@git fetch
+	./tests/compare_benchmarks.sh main
+	./tests/compare_benchmarks.sh ${LATEST_RELEASE_BRANCH}
diff --git a/tests/compare_benchmarks.sh b/tests/compare_benchmarks.sh
new file mode 100755
index 000000000..e2b343024
--- /dev/null
+++ b/tests/compare_benchmarks.sh
@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+# https://github.com/kubernetes/kube-state-metrics/blob/main/tests/compare_benchmarks.sh (originally written by mxinden)
+
+# exit immediately when a command fails
+set -e
+# only exit with zero if all commands of the pipeline exit successfully
+set -o pipefail
+# error on unset variables
+set -u
+
+[[ "$#" -eq 1 ]] || echo "One argument required, $# provided."
+
+REF_CURRENT="$(git rev-parse --abbrev-ref HEAD)"
+REF_TO_COMPARE=$1
+
+RESULT_CURRENT="$(mktemp)-${REF_CURRENT}"
+RESULT_TO_COMPARE="$(mktemp)-${REF_TO_COMPARE}"
+
+echo ""
+echo "### Testing ${REF_CURRENT}"
+
+go test -benchmem -run=NONE -bench=. ./... | tee "${RESULT_CURRENT}"
+
+echo ""
+echo "### Done testing ${REF_CURRENT}"
+
+echo ""
+echo "### Testing ${REF_TO_COMPARE}"
+
+git checkout "${REF_TO_COMPARE}"
+
+go test -benchmem -run=NONE -bench=. ./... | tee "${RESULT_TO_COMPARE}"
+
+echo ""
+echo "### Done testing ${REF_TO_COMPARE}"
+
+git checkout -
+
+echo ""
+echo "### Result"
+echo "old=${REF_TO_COMPARE} new=${REF_CURRENT}"
+
+benchstat "${RESULT_TO_COMPARE}" "${RESULT_CURRENT}"