From 6bd03b7b8da50c358954ba6bf4adf4dcdf8254cc Mon Sep 17 00:00:00 2001 From: SCC Date: Mon, 12 Dec 2022 13:53:39 +0800 Subject: [PATCH 1/5] add local benchmark testing shell --- scripts/local-pallets-benchmark.sh | 106 +++++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100755 scripts/local-pallets-benchmark.sh diff --git a/scripts/local-pallets-benchmark.sh b/scripts/local-pallets-benchmark.sh new file mode 100755 index 0000000000..67222919ce --- /dev/null +++ b/scripts/local-pallets-benchmark.sh @@ -0,0 +1,106 @@ +#!/usr/bin/env bash + +# This script can be used for running moonbeam's benchmarks. +# +# The moonbeam binary is required to be compiled with --features=runtime-benchmarks +# in release mode. + +set -e + +BINARY="./target/release/litentry-collator" + + +if [[ ! -f "${BINARY}" ]]; then + echo "binary '${BINARY}' does not exist." + echo "ensure that the moonbeam binary is compiled with 'make build-node-benchmarks ' nd in release mode." + exit 1 +fi + +function help { + echo "USAGE:" + echo " ${0} litentry|litmuts|rococo pallet_name benchmark_method [--check]" + echo "" + echo "EXAMPLES:" + echo " ${0} " "list all benchmarks and provide a selection to choose from" + echo " ${0} --check " "list all benchmarks and provide a selection to choose from, runs in 'check' mode (reduced steps and repetitions)" + echo " ${0} foo bar " "run a benchmark for pallet 'foo' and benchmark 'bar'" + echo " ${0} foo bar --check " "run a benchmark for pallet 'foo' and benchmark 'bar' in 'check' mode (reduced steps and repetitions)" + echo " ${0} foo bar --all " "run a benchmark for all pallets" + echo " ${0} foo bar --all --check " "run a benchmark for all pallets in 'check' mode (reduced steps and repetitions)" +} + +#echo "make output dir" +#mkdir weights +CHAIN_TYPE="--chain=$1-dev" + +function choose_and_bench { + readarray -t options < <(${BINARY} benchmark pallet --list $CHAIN_TYPE | sed 1d) + options+=('EXIT') + + select opt in "${options[@]}"; do + IFS=', ' read -ra parts <<< "${opt}" + [[ "${opt}" == 'EXIT' ]] && exit 0 + + echo "inputs args =====> ${parts[0]}" "${parts[1]}" "${2}" + bench "${parts[0]}" "${parts[1]}" "${1}" + break + done +} + +function bench { + echo "pallet ${1}-weights.rs" + OUTPUT="--output=$(pwd)/weights/${1}-weights.rs" + + echo "benchmarking '${2}::${3}' --check=${4}, writing results to '${OUTPUT}'" + + if [[ $PALLET == *"parachain_staking"* ]]; then + echo "will run ${1} benchmark code" + STEPS=25 + REPEAT=20 + else + echo "will run other pallet (${1}) benchmark code" + STEPS=20 + REPEAT=50 + fi + + echo "chain_type <====> $CHAIN_TYPE " + + WASMTIME_BACKTRACE_DETAILS=1 ${BINARY} benchmark pallet \ + $CHAIN_TYPE \ + --execution=wasm \ + --db-cache=20 \ + --wasm-execution=compiled \ + --pallet="${2}" \ + --extrinsic="${3}" \ + --heap-pages=4096 \ + --steps="$STEPS" \ + --repeat="$REPEAT" \ + --header=./LICENSE_HEADER \ + $TEMPLATE \ + $OUTPUT + +} + +if [[ "${@}" =~ "--help" ]]; then + help +else + CHECK=0 + if [[ "${@}" =~ "--check" ]]; then + CHECK=1 + set -o noglob && set -- ${@/'--check'} && set +o noglob + fi + + ALL=0 + if [[ "${@}" =~ "--all" ]]; then + ALL=1 + fi + + if [[ "${ALL}" -eq 1 ]]; then + mkdir -p weights/ + bench '*' '*' "${CHECK}" "weights/" + elif [[ $# -ne 2 ]]; then + choose_and_bench "${CHECK}" + else + bench "${2}" "${3}" "${CHECK}" + fi +fi From 1e1b9f05784e9814a967211d482bae163aba4882 Mon Sep 17 00:00:00 2001 From: Scc Date: Tue, 13 Dec 2022 10:18:01 +0800 Subject: [PATCH 2/5] fix benmarking scripts --- scripts/local-pallets-benchmark.sh | 44 +++++++++++++++--------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/scripts/local-pallets-benchmark.sh b/scripts/local-pallets-benchmark.sh index 67222919ce..f29fcc951b 100755 --- a/scripts/local-pallets-benchmark.sh +++ b/scripts/local-pallets-benchmark.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash -# This script can be used for running moonbeam's benchmarks. +# This script can be used for running litentry's benchmarks. # -# The moonbeam binary is required to be compiled with --features=runtime-benchmarks +# The litentry binary is required to be compiled with make build-node-benchmarks # in release mode. set -e @@ -12,7 +12,7 @@ BINARY="./target/release/litentry-collator" if [[ ! -f "${BINARY}" ]]; then echo "binary '${BINARY}' does not exist." - echo "ensure that the moonbeam binary is compiled with 'make build-node-benchmarks ' nd in release mode." + echo "ensure that the litentry binary is compiled with 'make build-node-benchmarks ' nd in release mode." exit 1 fi @@ -21,17 +21,21 @@ function help { echo " ${0} litentry|litmuts|rococo pallet_name benchmark_method [--check]" echo "" echo "EXAMPLES:" - echo " ${0} " "list all benchmarks and provide a selection to choose from" - echo " ${0} --check " "list all benchmarks and provide a selection to choose from, runs in 'check' mode (reduced steps and repetitions)" - echo " ${0} foo bar " "run a benchmark for pallet 'foo' and benchmark 'bar'" - echo " ${0} foo bar --check " "run a benchmark for pallet 'foo' and benchmark 'bar' in 'check' mode (reduced steps and repetitions)" - echo " ${0} foo bar --all " "run a benchmark for all pallets" - echo " ${0} foo bar --all --check " "run a benchmark for all pallets in 'check' mode (reduced steps and repetitions)" + echo " ${0} " "list all benchmarks and provide a selection to choose from" + echo " ${0} --check " "list all benchmarks and provide a selection to choose from, runs in 'check' mode (reduced steps and repetitions)" + echo " ${0} foo bar " "run a benchmark for pallet 'foo' and benchmark 'bar'" + echo " ${0} foo bar --all " "run a benchmark for all pallets" + echo " ${0} foo bar --all --check " "run a benchmark for all pallets in 'check' mode (reduced steps and repetitions)" } -#echo "make output dir" -#mkdir weights -CHAIN_TYPE="--chain=$1-dev" +WEIGHTS_PATH="$(pwd)/weights" +if [ ! -d $WEIGHTS_PATH ];then + mkdir -p weights/ + +fi + +CHAIN_TYPE="--chain=${1}-dev" +PALLET=${2//-/_} function choose_and_bench { readarray -t options < <(${BINARY} benchmark pallet --list $CHAIN_TYPE | sed 1d) @@ -41,17 +45,16 @@ function choose_and_bench { IFS=', ' read -ra parts <<< "${opt}" [[ "${opt}" == 'EXIT' ]] && exit 0 - echo "inputs args =====> ${parts[0]}" "${parts[1]}" "${2}" - bench "${parts[0]}" "${parts[1]}" "${1}" + bench "${parts[0]}" "${parts[1]}" "${2}" break done } function bench { echo "pallet ${1}-weights.rs" - OUTPUT="--output=$(pwd)/weights/${1}-weights.rs" + OUTPUT="--output=$(pwd)/weights/${1}_weights.rs" - echo "benchmarking '${2}::${3}' --check=${4}, writing results to '${OUTPUT}'" + echo "benchmarking '${1}::${2}' --check=${3}, writing results to '${OUTPUT}'" if [[ $PALLET == *"parachain_staking"* ]]; then echo "will run ${1} benchmark code" @@ -59,18 +62,16 @@ function bench { REPEAT=20 else echo "will run other pallet (${1}) benchmark code" - STEPS=20 - REPEAT=50 + STEPS=2 + REPEAT=5 fi - echo "chain_type <====> $CHAIN_TYPE " - WASMTIME_BACKTRACE_DETAILS=1 ${BINARY} benchmark pallet \ $CHAIN_TYPE \ --execution=wasm \ --db-cache=20 \ --wasm-execution=compiled \ - --pallet="${2}" \ + --pallet="$PALLET" \ --extrinsic="${3}" \ --heap-pages=4096 \ --steps="$STEPS" \ @@ -96,7 +97,6 @@ else fi if [[ "${ALL}" -eq 1 ]]; then - mkdir -p weights/ bench '*' '*' "${CHECK}" "weights/" elif [[ $# -ne 2 ]]; then choose_and_bench "${CHECK}" From f9c8079f3f1ab089d1b9e7fba6fdc633c331c815 Mon Sep 17 00:00:00 2001 From: Scc Date: Tue, 13 Dec 2022 10:19:23 +0800 Subject: [PATCH 3/5] fix some syntax --- scripts/local-pallets-benchmark.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/local-pallets-benchmark.sh b/scripts/local-pallets-benchmark.sh index f29fcc951b..7b25df073e 100755 --- a/scripts/local-pallets-benchmark.sh +++ b/scripts/local-pallets-benchmark.sh @@ -62,8 +62,8 @@ function bench { REPEAT=20 else echo "will run other pallet (${1}) benchmark code" - STEPS=2 - REPEAT=5 + STEPS=20 + REPEAT=50 fi WASMTIME_BACKTRACE_DETAILS=1 ${BINARY} benchmark pallet \ From a2c787d18104031bb36a194d9b98518dfbbeb188 Mon Sep 17 00:00:00 2001 From: Scc Date: Tue, 13 Dec 2022 13:34:37 +0800 Subject: [PATCH 4/5] fix some syntax --- scripts/local-pallets-benchmark.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/local-pallets-benchmark.sh b/scripts/local-pallets-benchmark.sh index 7b25df073e..d5889fc2bd 100755 --- a/scripts/local-pallets-benchmark.sh +++ b/scripts/local-pallets-benchmark.sh @@ -12,7 +12,7 @@ BINARY="./target/release/litentry-collator" if [[ ! -f "${BINARY}" ]]; then echo "binary '${BINARY}' does not exist." - echo "ensure that the litentry binary is compiled with 'make build-node-benchmarks ' nd in release mode." + echo "ensure that the litentry binary is compiled with 'make build-node-benchmarks' and in release mode." exit 1 fi @@ -36,6 +36,7 @@ fi CHAIN_TYPE="--chain=${1}-dev" PALLET=${2//-/_} +EXTRINSIC=${3//-/_} function choose_and_bench { readarray -t options < <(${BINARY} benchmark pallet --list $CHAIN_TYPE | sed 1d) @@ -72,7 +73,7 @@ function bench { --db-cache=20 \ --wasm-execution=compiled \ --pallet="$PALLET" \ - --extrinsic="${3}" \ + --extrinsic="$EXTRINSIC" \ --heap-pages=4096 \ --steps="$STEPS" \ --repeat="$REPEAT" \ From 5e5d791d234f3e2a3523dfc416c011602ad9edcc Mon Sep 17 00:00:00 2001 From: scc Date: Tue, 13 Dec 2022 13:51:46 +0800 Subject: [PATCH 5/5] fix some syntax --- scripts/local-pallets-benchmark.sh | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/scripts/local-pallets-benchmark.sh b/scripts/local-pallets-benchmark.sh index d5889fc2bd..87c7ee17d9 100755 --- a/scripts/local-pallets-benchmark.sh +++ b/scripts/local-pallets-benchmark.sh @@ -24,8 +24,7 @@ function help { echo " ${0} " "list all benchmarks and provide a selection to choose from" echo " ${0} --check " "list all benchmarks and provide a selection to choose from, runs in 'check' mode (reduced steps and repetitions)" echo " ${0} foo bar " "run a benchmark for pallet 'foo' and benchmark 'bar'" - echo " ${0} foo bar --all " "run a benchmark for all pallets" - echo " ${0} foo bar --all --check " "run a benchmark for all pallets in 'check' mode (reduced steps and repetitions)" + echo " ${0} foo bar all " "run the pallet all benchmark method" } WEIGHTS_PATH="$(pwd)/weights" @@ -36,7 +35,14 @@ fi CHAIN_TYPE="--chain=${1}-dev" PALLET=${2//-/_} -EXTRINSIC=${3//-/_} + +EXTRINSIC= + +if [[ ${3} == 'all' ]];then + EXTRINSIC=* +else + EXTRINSIC=${3//-/_} +fi function choose_and_bench { readarray -t options < <(${BINARY} benchmark pallet --list $CHAIN_TYPE | sed 1d)