test: test runtime benchmarks

.github/workflows/check-code.yml

@@ -328,3 +328,43 @@ jobs:
             --checks=all \
           live \
             --uri=wss://${{ matrix.runtime }}.kilt.io
+
+  test-runtime-benchmarks:
+    name: Test runtime benchmarks
+    runs-on: ubuntu-latest
+    container:
+      image: paritytech/ci-unified:bullseye-1.74.0
+    env:
+      # Configured by the Docker image. We can't change this unless the image does it.
+      CARGO_HOME: /usr/local/cargo
+    needs: cargo-clippy
+    if: ${{ github.event_name == 'push'}}
+
+    strategy:
+      matrix:
+        runtime:
+          - peregrine
+          - spiritnet
+      fail-fast: false
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Free Disk Space
+        uses: jlumbroso/free-disk-space@main
+        with:
+          tool-cache: true
+
+      - name: Set up Cargo cache
+        uses: actions/cache@v4
+        with:
+          path: |
+            ${{ env.CARGO_HOME }}/bin/
+            ${{ env.CARGO_HOME }}/registry/index/
+            ${{ env.CARGO_HOME }}/registry/cache/
+            ${{ env.CARGO_HOME }}/git/db/
+          key: ${{ github.job }}-${{ github.ref }}-${{ matrix.runtime }}-${{ hashFiles('**/Cargo.lock') }}
+
+      - name: Run runtime benchmarks
+        run: bash -x scripts/run_benches_for_runtime.sh ${{ matrix.runtime }} dev

scripts/run_benches_for_pallets.sh

@@ -38,11 +38,7 @@ for pallet in "${pallets[@]}"; do
 	./target/release/kilt-parachain benchmark pallet \
 		--template=".maintain/weight-template.hbs" \
 		--header="HEADER-GPL" \
-		--execution=wasm \
-		--wasm-execution=compiled \
 		--heap-pages=4096 \
-		--steps=50 \
-		--repeat=20 \
 		--chain="${chain}" \
 		--pallet="$pallet" \
 		--extrinsic="*" \

scripts/run_benches_for_runtime.sh

@@ -5,8 +5,11 @@
 # current Substrate reference machine: https://github.com/paritytech/substrate/pull/5848
 
 runtime=${1-"peregrine"}
+profile=${2-"release"}
+
 chain=$([ "$1" == "spiritnet" ] && echo "spiritnet-dev" || echo "dev")
-standard_args="--release --locked --features=runtime-benchmarks --bin=kilt-parachain"
+# Dev profile is the debug target
+standard_args="--profile $2 --locked --features=runtime-benchmarks --bin=kilt-parachain"
 
 pallets=(
 	pallet-migration
@@ -44,32 +47,41 @@ pallets=(
 	pallet-asset-switch
 )
 
-
-
 # Add Peregrine-only pallets here!
 if [ "$runtime" = "peregrine" ]; then
   pallets+=(
 	pallet-sudo
   )
 fi
 
-echo "[+] Running all runtime benchmarks for $runtime --chain=$chain"
+echo "[+] Running all runtime benchmarks for \"$runtime\", \"--chain=$chain\" and profile \"$profile\""
 
 cargo build $standard_args
 
+if [ $profile == "dev" ]; then
+    target_folder="debug"
+	# We care about benchmark correctness, not accuracy.
+	additional_args="--steps=2 --repeat=1 --default-pov-mode=ignored --no-verify"
+else
+    target_folder=$profile
+	additional_args="--header=\"HEADER-GPL\" --template=\".maintain/runtime-weight-template.hbs\" --output=\"./runtimes/${runtime}/src/weights/\""
+fi
+
 for pallet in "${pallets[@]}"; do
 	echo "Runtime: $runtime. Pallet: $pallet"
 	# shellcheck disable=SC2086
-	./target/release/kilt-parachain benchmark pallet \
-		--template=".maintain/runtime-weight-template.hbs" \
-		--header="HEADER-GPL" \
-		--wasm-execution=compiled \
+	./target/$target_folder/kilt-parachain benchmark pallet \
 		--heap-pages=4096 \
-		--steps=50 \
-		--repeat=20 \
 		--chain="${chain}" \
 		--pallet="$pallet" \
 		--extrinsic="*" \
-		--output="./runtimes/${runtime}/src/weights/"
+		$additional_args
+
+	bench_status=$?
+
+	# Exit with error as soon as one benchmark fails
+	if [ $bench_status -ne 0 ]; then
+    	exit $bench_status
+  	fi
 
 done