diff --git a/.github/workflows/check-prdoc.yml b/.github/workflows/check-prdoc.yml index 5503b61d6817..91701ca036e0 100644 --- a/.github/workflows/check-prdoc.yml +++ b/.github/workflows/check-prdoc.yml @@ -56,4 +56,4 @@ jobs: run: | echo "Checking for PR#${GITHUB_PR}" echo "You can find more information about PRDoc at $PRDOC_DOC" - $ENGINE run --rm -v $PWD:/repo $IMAGE check -n ${GITHUB_PR} + $ENGINE run --rm -v $PWD:/repo -e RUST_LOG=info $IMAGE check -n ${GITHUB_PR} diff --git a/.github/workflows/release-99_notif-published.yml b/.github/workflows/release-99_notif-published.yml index 397795fafa4e..732db15d9c0c 100644 --- a/.github/workflows/release-99_notif-published.yml +++ b/.github/workflows/release-99_notif-published.yml @@ -32,6 +32,12 @@ jobs: - name: '#polkadotvalidatorlounge:web3.foundation' room: '!NZrbtteFeqYKCUGQtr:matrix.parity.io' pre-releases: false + - name: '#polkadot-announcements:parity.io' + room: '!UqHPWiCBGZWxrmYBkF:matrix.parity.io' + pre-releases: false + - name: '#kusama-announce:parity.io' + room: '!FMwxpQnYhRCNDRsYGI:matrix.parity.io' + pre-releases: false steps: - name: Matrix notification to ${{ matrix.channel.name }} diff --git a/.gitlab/pipeline/check.yml b/.gitlab/pipeline/check.yml index cdb5d1b05d09..4d71a473372d 100644 --- a/.gitlab/pipeline/check.yml +++ b/.gitlab/pipeline/check.yml @@ -108,8 +108,10 @@ check-toml-format: export RUST_LOG=remote-ext=debug,runtime=debug echo "---------- Downloading try-runtime CLI ----------" - curl -sL https://github.com/paritytech/try-runtime-cli/releases/download/v0.5.0/try-runtime-x86_64-unknown-linux-musl -o try-runtime + curl -sL https://github.com/paritytech/try-runtime-cli/releases/download/v0.5.4/try-runtime-x86_64-unknown-linux-musl -o try-runtime chmod +x ./try-runtime + echo "Using try-runtime-cli version:" + ./try-runtime --version echo "---------- Building ${PACKAGE} runtime ----------" time cargo build --release --locked -p "$PACKAGE" --features try-runtime diff --git a/.gitlab/pipeline/zombienet/polkadot.yml b/.gitlab/pipeline/zombienet/polkadot.yml index 54eb6db48cae..97572f029d00 100644 --- a/.gitlab/pipeline/zombienet/polkadot.yml +++ b/.gitlab/pipeline/zombienet/polkadot.yml @@ -158,6 +158,14 @@ zombienet-polkadot-functional-0011-async-backing-6-seconds-rate: --local-dir="${LOCAL_DIR}/functional" --test="0011-async-backing-6-seconds-rate.zndsl" +zombienet-polkadot-functional-0012-elastic-scaling-mvp: + extends: + - .zombienet-polkadot-common + script: + - /home/nonroot/zombie-net/scripts/ci/run-test-local-env-manager.sh + --local-dir="${LOCAL_DIR}/functional" + --test="0012-elastic-scaling-mvp.zndsl" + zombienet-polkadot-smoke-0001-parachains-smoke-test: extends: - .zombienet-polkadot-common diff --git a/Cargo.lock b/Cargo.lock index 93ee9cc99c44..a23be53b34fd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4047,6 +4047,25 @@ dependencies = [ "sp-trie", ] +[[package]] +name = "cumulus-primitives-storage-weight-reclaim" +version = "1.0.0" +dependencies = [ + "cumulus-primitives-core", + "cumulus-primitives-proof-size-hostfunction", + "cumulus-test-runtime", + "docify", + "frame-support", + "frame-system", + "log", + "parity-scale-codec", + "scale-info", + "sp-io", + "sp-runtime", + "sp-std 14.0.0", + "sp-trie", +] + [[package]] name = "cumulus-primitives-timestamp" version = "0.7.0" @@ -4209,6 +4228,7 @@ dependencies = [ "cumulus-primitives-core", "cumulus-primitives-parachain-inherent", "cumulus-primitives-proof-size-hostfunction", + "cumulus-primitives-storage-weight-reclaim", "cumulus-test-relay-sproof-builder", "cumulus-test-runtime", "cumulus-test-service", @@ -4253,6 +4273,7 @@ version = "0.1.0" dependencies = [ "cumulus-pallet-parachain-system", "cumulus-primitives-core", + "cumulus-primitives-storage-weight-reclaim", "frame-executive", "frame-support", "frame-system", @@ -4295,6 +4316,7 @@ dependencies = [ "cumulus-client-service", "cumulus-pallet-parachain-system", "cumulus-primitives-core", + "cumulus-primitives-storage-weight-reclaim", "cumulus-relay-chain-inprocess-interface", "cumulus-relay-chain-interface", "cumulus-relay-chain-minimal-node", @@ -11340,6 +11362,7 @@ dependencies = [ "cumulus-pallet-xcm", "cumulus-pallet-xcmp-queue", "cumulus-primitives-core", + "cumulus-primitives-storage-weight-reclaim", "cumulus-primitives-utility", "frame-benchmarking", "frame-executive", @@ -12489,7 +12512,9 @@ dependencies = [ "polkadot-primitives", "polkadot-primitives-test-helpers", "polkadot-statement-table", + "rstest", "sc-keystore", + "schnellru", "sp-application-crypto", "sp-core", "sp-keyring", @@ -13159,6 +13184,7 @@ version = "7.0.0" dependencies = [ "bitvec", "hex-literal", + "log", "parity-scale-codec", "polkadot-core-primitives", "polkadot-parachain-primitives", @@ -13325,6 +13351,7 @@ dependencies = [ "polkadot-runtime-metrics", "rand", "rand_chacha 0.3.1", + "rstest", "rustc-hex", "sc-keystore", "scale-info", @@ -13556,6 +13583,7 @@ dependencies = [ "parity-scale-codec", "polkadot-primitives", "sp-core", + "tracing-gum", ] [[package]] @@ -14764,6 +14792,12 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" +[[package]] +name = "relative-path" +version = "1.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e898588f33fdd5b9420719948f9f2a32c922a246964576f71ba7f24f80610fbc" + [[package]] name = "remote-ext-tests-bags-list" version = "1.0.0" @@ -15146,6 +15180,35 @@ dependencies = [ "winapi", ] +[[package]] +name = "rstest" +version = "0.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97eeab2f3c0a199bc4be135c36c924b6590b88c377d416494288c14f2db30199" +dependencies = [ + "futures", + "futures-timer", + "rstest_macros", + "rustc_version 0.4.0", +] + +[[package]] +name = "rstest_macros" +version = "0.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d428f8247852f894ee1be110b375111b586d4fa431f6c46e64ba5a0dcccbe605" +dependencies = [ + "cfg-if", + "glob", + "proc-macro2", + "quote", + "regex", + "relative-path", + "rustc_version 0.4.0", + "syn 2.0.50", + "unicode-ident", +] + [[package]] name = "rtnetlink" version = "0.10.1" @@ -22320,10 +22383,12 @@ dependencies = [ "pallet-transaction-payment", "pallet-xcm", "parity-scale-codec", + "polkadot-service", "polkadot-test-client", "polkadot-test-runtime", "polkadot-test-service", "sp-consensus", + "sp-core", "sp-keyring", "sp-runtime", "sp-state-machine", diff --git a/Cargo.toml b/Cargo.toml index 774ce1b52a39..1d27cfe95392 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -127,6 +127,7 @@ members = [ "cumulus/primitives/core", "cumulus/primitives/parachain-inherent", "cumulus/primitives/proof-size-hostfunction", + "cumulus/primitives/storage-weight-reclaim", "cumulus/primitives/timestamp", "cumulus/primitives/utility", "cumulus/test/client", diff --git a/bridges/testing/environments/rococo-westend/bridges_rococo_westend.sh b/bridges/testing/environments/rococo-westend/bridges_rococo_westend.sh index 84764cdaca38..de5be2e0af88 100755 --- a/bridges/testing/environments/rococo-westend/bridges_rococo_westend.sh +++ b/bridges/testing/environments/rococo-westend/bridges_rococo_westend.sh @@ -1,7 +1,7 @@ #!/bin/bash # import common functions -source "${BASH_SOURCE%/*}/../../utils/bridges.sh" +source "$FRAMEWORK_PATH/utils/bridges.sh" # Expected sovereign accounts. # diff --git a/bridges/testing/environments/rococo-westend/helper.sh b/bridges/testing/environments/rococo-westend/helper.sh index 211a5b53b3d9..0a13ded213f5 100755 --- a/bridges/testing/environments/rococo-westend/helper.sh +++ b/bridges/testing/environments/rococo-westend/helper.sh @@ -1,3 +1,3 @@ #!/bin/bash -$POLKADOT_SDK_PATH/bridges/testing/environments/rococo-westend/bridges_rococo_westend.sh "$@" +$ENV_PATH/bridges_rococo_westend.sh "$@" diff --git a/bridges/testing/environments/rococo-westend/rococo-init.zndsl b/bridges/testing/environments/rococo-westend/rococo-init.zndsl index 145f2df73a6e..c913e4db31f4 100644 --- a/bridges/testing/environments/rococo-westend/rococo-init.zndsl +++ b/bridges/testing/environments/rococo-westend/rococo-init.zndsl @@ -1,8 +1,8 @@ -Description: User is able to transfer WND from Westend Asset Hub to Rococo Asset Hub and back +Description: Check if the HRMP channel between Rococo BH and Rococo AH was opened successfully Network: ./bridge_hub_rococo_local_network.toml Creds: config # ensure that initialization has completed -asset-hub-rococo-collator1: js-script ../../js-helpers/wait-hrmp-channel-opened.js with "1013" within 300 seconds +asset-hub-rococo-collator1: js-script {{FRAMEWORK_PATH}}/js-helpers/wait-hrmp-channel-opened.js with "1013" within 300 seconds diff --git a/bridges/testing/environments/rococo-westend/rococo.zndsl b/bridges/testing/environments/rococo-westend/rococo.zndsl index bd8681af2196..5b49c7c632fa 100644 --- a/bridges/testing/environments/rococo-westend/rococo.zndsl +++ b/bridges/testing/environments/rococo-westend/rococo.zndsl @@ -1,7 +1,7 @@ -Description: User is able to transfer WND from Westend Asset Hub to Rococo Asset Hub and back +Description: Check if the with-Westend GRANPDA pallet was initialized at Rococo BH Network: ./bridge_hub_rococo_local_network.toml Creds: config # relay is already started - let's wait until with-Westend GRANPDA pallet is initialized at Rococo -bridge-hub-rococo-collator1: js-script ../../js-helpers/best-finalized-header-at-bridged-chain.js with "Westend,0" within 400 seconds +bridge-hub-rococo-collator1: js-script {{FRAMEWORK_PATH}}/js-helpers/best-finalized-header-at-bridged-chain.js with "Westend,0" within 400 seconds diff --git a/bridges/testing/environments/rococo-westend/spawn.sh b/bridges/testing/environments/rococo-westend/spawn.sh index 5a0d65ce65db..cbd0b1bc623a 100755 --- a/bridges/testing/environments/rococo-westend/spawn.sh +++ b/bridges/testing/environments/rococo-westend/spawn.sh @@ -4,7 +4,7 @@ set -e trap "trap - SIGTERM && kill -9 -$$" SIGINT SIGTERM EXIT -source "${BASH_SOURCE%/*}/../../utils/zombienet.sh" +source "$FRAMEWORK_PATH/utils/zombienet.sh" # whether to init the chains (open HRMP channels, set XCM version, create reserve assets, etc) init=0 diff --git a/bridges/testing/environments/rococo-westend/start_relayer.sh b/bridges/testing/environments/rococo-westend/start_relayer.sh index c57d4f1a4374..7ddd312d395a 100755 --- a/bridges/testing/environments/rococo-westend/start_relayer.sh +++ b/bridges/testing/environments/rococo-westend/start_relayer.sh @@ -2,8 +2,8 @@ set -e -source "${BASH_SOURCE%/*}/../../utils/common.sh" -source "${BASH_SOURCE%/*}/../../utils/zombienet.sh" +source "$FRAMEWORK_PATH/utils/common.sh" +source "$FRAMEWORK_PATH/utils/zombienet.sh" rococo_dir=$1 westend_dir=$2 diff --git a/bridges/testing/environments/rococo-westend/westend-init.zndsl b/bridges/testing/environments/rococo-westend/westend-init.zndsl index 2f8e665d592d..0f5428eed3b0 100644 --- a/bridges/testing/environments/rococo-westend/westend-init.zndsl +++ b/bridges/testing/environments/rococo-westend/westend-init.zndsl @@ -1,7 +1,7 @@ -Description: User is able to transfer ROC from Rococo Asset Hub to Westend Asset Hub and back +Description: Check if the HRMP channel between Westend BH and Westend AH was opened successfully Network: ./bridge_hub_westend_local_network.toml Creds: config # ensure that initialization has completed -asset-hub-westend-collator1: js-script ../../js-helpers/wait-hrmp-channel-opened.js with "1002" within 600 seconds +asset-hub-westend-collator1: js-script {{FRAMEWORK_PATH}}/js-helpers/wait-hrmp-channel-opened.js with "1002" within 600 seconds diff --git a/bridges/testing/environments/rococo-westend/westend.zndsl b/bridges/testing/environments/rococo-westend/westend.zndsl index c75ae579d27a..07968838852f 100644 --- a/bridges/testing/environments/rococo-westend/westend.zndsl +++ b/bridges/testing/environments/rococo-westend/westend.zndsl @@ -1,6 +1,6 @@ -Description: User is able to transfer ROC from Rococo Asset Hub to Westend Asset Hub and back +Description: Check if the with-Rococo GRANPDA pallet was initialized at Westend BH Network: ./bridge_hub_westend_local_network.toml Creds: config # relay is already started - let's wait until with-Rococo GRANPDA pallet is initialized at Westend -bridge-hub-westend-collator1: js-script ../../js-helpers/best-finalized-header-at-bridged-chain.js with "Rococo,0" within 400 seconds +bridge-hub-westend-collator1: js-script {{FRAMEWORK_PATH}}/js-helpers/best-finalized-header-at-bridged-chain.js with "Rococo,0" within 400 seconds diff --git a/bridges/testing/js-helpers/best-finalized-header-at-bridged-chain.js b/bridges/testing/framework/js-helpers/best-finalized-header-at-bridged-chain.js similarity index 100% rename from bridges/testing/js-helpers/best-finalized-header-at-bridged-chain.js rename to bridges/testing/framework/js-helpers/best-finalized-header-at-bridged-chain.js diff --git a/bridges/testing/js-helpers/chains/rococo-at-westend.js b/bridges/testing/framework/js-helpers/chains/rococo-at-westend.js similarity index 100% rename from bridges/testing/js-helpers/chains/rococo-at-westend.js rename to bridges/testing/framework/js-helpers/chains/rococo-at-westend.js diff --git a/bridges/testing/js-helpers/chains/westend-at-rococo.js b/bridges/testing/framework/js-helpers/chains/westend-at-rococo.js similarity index 100% rename from bridges/testing/js-helpers/chains/westend-at-rococo.js rename to bridges/testing/framework/js-helpers/chains/westend-at-rococo.js diff --git a/bridges/testing/js-helpers/native-assets-balance-increased.js b/bridges/testing/framework/js-helpers/native-assets-balance-increased.js similarity index 100% rename from bridges/testing/js-helpers/native-assets-balance-increased.js rename to bridges/testing/framework/js-helpers/native-assets-balance-increased.js diff --git a/bridges/testing/js-helpers/only-mandatory-headers-synced-when-idle.js b/bridges/testing/framework/js-helpers/only-mandatory-headers-synced-when-idle.js similarity index 100% rename from bridges/testing/js-helpers/only-mandatory-headers-synced-when-idle.js rename to bridges/testing/framework/js-helpers/only-mandatory-headers-synced-when-idle.js diff --git a/bridges/testing/js-helpers/only-required-headers-synced-when-idle.js b/bridges/testing/framework/js-helpers/only-required-headers-synced-when-idle.js similarity index 100% rename from bridges/testing/js-helpers/only-required-headers-synced-when-idle.js rename to bridges/testing/framework/js-helpers/only-required-headers-synced-when-idle.js diff --git a/bridges/testing/js-helpers/relayer-rewards.js b/bridges/testing/framework/js-helpers/relayer-rewards.js similarity index 100% rename from bridges/testing/js-helpers/relayer-rewards.js rename to bridges/testing/framework/js-helpers/relayer-rewards.js diff --git a/bridges/testing/js-helpers/utils.js b/bridges/testing/framework/js-helpers/utils.js similarity index 100% rename from bridges/testing/js-helpers/utils.js rename to bridges/testing/framework/js-helpers/utils.js diff --git a/bridges/testing/js-helpers/wait-hrmp-channel-opened.js b/bridges/testing/framework/js-helpers/wait-hrmp-channel-opened.js similarity index 100% rename from bridges/testing/js-helpers/wait-hrmp-channel-opened.js rename to bridges/testing/framework/js-helpers/wait-hrmp-channel-opened.js diff --git a/bridges/testing/js-helpers/wrapped-assets-balance.js b/bridges/testing/framework/js-helpers/wrapped-assets-balance.js similarity index 100% rename from bridges/testing/js-helpers/wrapped-assets-balance.js rename to bridges/testing/framework/js-helpers/wrapped-assets-balance.js diff --git a/bridges/testing/utils/bridges.sh b/bridges/testing/framework/utils/bridges.sh similarity index 98% rename from bridges/testing/utils/bridges.sh rename to bridges/testing/framework/utils/bridges.sh index cfde5dfd26b7..7c8399461584 100755 --- a/bridges/testing/utils/bridges.sh +++ b/bridges/testing/framework/utils/bridges.sh @@ -41,8 +41,8 @@ function ensure_polkadot_js_api() { echo "" echo "" echo "-------------------" - echo "Installing (nodejs) sub module: $(dirname "$0")/generate_hex_encoded_call" - pushd $(dirname "$0")/generate_hex_encoded_call + echo "Installing (nodejs) sub module: ${BASH_SOURCE%/*}/generate_hex_encoded_call" + pushd ${BASH_SOURCE%/*}/generate_hex_encoded_call npm install popd fi diff --git a/bridges/testing/utils/common.sh b/bridges/testing/framework/utils/common.sh similarity index 100% rename from bridges/testing/utils/common.sh rename to bridges/testing/framework/utils/common.sh diff --git a/bridges/testing/utils/generate_hex_encoded_call/index.js b/bridges/testing/framework/utils/generate_hex_encoded_call/index.js similarity index 100% rename from bridges/testing/utils/generate_hex_encoded_call/index.js rename to bridges/testing/framework/utils/generate_hex_encoded_call/index.js diff --git a/bridges/testing/utils/generate_hex_encoded_call/package-lock.json b/bridges/testing/framework/utils/generate_hex_encoded_call/package-lock.json similarity index 100% rename from bridges/testing/utils/generate_hex_encoded_call/package-lock.json rename to bridges/testing/framework/utils/generate_hex_encoded_call/package-lock.json diff --git a/bridges/testing/utils/generate_hex_encoded_call/package.json b/bridges/testing/framework/utils/generate_hex_encoded_call/package.json similarity index 100% rename from bridges/testing/utils/generate_hex_encoded_call/package.json rename to bridges/testing/framework/utils/generate_hex_encoded_call/package.json diff --git a/bridges/testing/utils/zombienet.sh b/bridges/testing/framework/utils/zombienet.sh similarity index 100% rename from bridges/testing/utils/zombienet.sh rename to bridges/testing/framework/utils/zombienet.sh diff --git a/bridges/testing/run-new-test.sh b/bridges/testing/run-new-test.sh index 2ed2a412b8a7..7c84a69aa47d 100755 --- a/bridges/testing/run-new-test.sh +++ b/bridges/testing/run-new-test.sh @@ -21,6 +21,7 @@ do done export POLKADOT_SDK_PATH=`realpath ${BASH_SOURCE%/*}/../..` +export FRAMEWORK_PATH=`realpath ${BASH_SOURCE%/*}/framework` # set path to binaries if [ "$ZOMBIENET_DOCKER_PATHS" -eq 1 ]; then diff --git a/bridges/testing/tests/0001-asset-transfer/roc-reaches-westend.zndsl b/bridges/testing/tests/0001-asset-transfer/roc-reaches-westend.zndsl index 203c95b73eb2..4725362ae826 100644 --- a/bridges/testing/tests/0001-asset-transfer/roc-reaches-westend.zndsl +++ b/bridges/testing/tests/0001-asset-transfer/roc-reaches-westend.zndsl @@ -1,12 +1,12 @@ Description: User is able to transfer ROC from Rococo Asset Hub to Westend Asset Hub and back -Network: ../../environments/rococo-westend/bridge_hub_westend_local_network.toml +Network: {{ENV_PATH}}/bridge_hub_westend_local_network.toml Creds: config # send ROC to //Alice from Rococo AH to Westend AH -asset-hub-westend-collator1: run ../../environments/rococo-westend/helper.sh with "reserve-transfer-assets-from-asset-hub-rococo-local" within 120 seconds +asset-hub-westend-collator1: run {{ENV_PATH}}/helper.sh with "reserve-transfer-assets-from-asset-hub-rococo-local" within 120 seconds # check that //Alice received the ROC on Westend AH -asset-hub-westend-collator1: js-script ../../js-helpers/wrapped-assets-balance.js with "5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY,0,Rococo" within 300 seconds +asset-hub-westend-collator1: js-script {{FRAMEWORK_PATH}}/js-helpers/wrapped-assets-balance.js with "5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY,0,Rococo" within 300 seconds # check that the relayer //Charlie is rewarded by Westend AH -bridge-hub-westend-collator1: js-script ../../js-helpers/relayer-rewards.js with "5FLSigC9HGRKVhB9FiEo4Y3koPsNmBmLJbpXg2mp1hXcS59Y,0x00000002,0x6268726F,ThisChain,0" within 30 seconds +bridge-hub-westend-collator1: js-script {{FRAMEWORK_PATH}}/js-helpers/relayer-rewards.js with "5FLSigC9HGRKVhB9FiEo4Y3koPsNmBmLJbpXg2mp1hXcS59Y,0x00000002,0x6268726F,ThisChain,0" within 30 seconds diff --git a/bridges/testing/tests/0001-asset-transfer/run.sh b/bridges/testing/tests/0001-asset-transfer/run.sh index 8a053ee72097..a7bb122919b4 100755 --- a/bridges/testing/tests/0001-asset-transfer/run.sh +++ b/bridges/testing/tests/0001-asset-transfer/run.sh @@ -2,17 +2,19 @@ set -e -source "${BASH_SOURCE%/*}/../../utils/common.sh" -source "${BASH_SOURCE%/*}/../../utils/zombienet.sh" +source "${BASH_SOURCE%/*}/../../framework/utils/common.sh" +source "${BASH_SOURCE%/*}/../../framework/utils/zombienet.sh" -${BASH_SOURCE%/*}/../../environments/rococo-westend/spawn.sh --init --start-relayer & +export ENV_PATH=`realpath ${BASH_SOURCE%/*}/../../environments/rococo-westend` + +$ENV_PATH/spawn.sh --init --start-relayer & env_pid=$! -ensure_process_file $env_pid $TEST_DIR/rococo.env 400 +ensure_process_file $env_pid $TEST_DIR/rococo.env 600 rococo_dir=`cat $TEST_DIR/rococo.env` echo -ensure_process_file $env_pid $TEST_DIR/westend.env 180 +ensure_process_file $env_pid $TEST_DIR/westend.env 300 westend_dir=`cat $TEST_DIR/westend.env` echo diff --git a/bridges/testing/tests/0001-asset-transfer/wnd-reaches-rococo.zndsl b/bridges/testing/tests/0001-asset-transfer/wnd-reaches-rococo.zndsl index bbd95db9cfda..77267239c3b1 100644 --- a/bridges/testing/tests/0001-asset-transfer/wnd-reaches-rococo.zndsl +++ b/bridges/testing/tests/0001-asset-transfer/wnd-reaches-rococo.zndsl @@ -1,12 +1,12 @@ Description: User is able to transfer WND from Westend Asset Hub to Rococo Asset Hub and back -Network: ../../environments/rococo-westend/bridge_hub_rococo_local_network.toml +Network: {{ENV_PATH}}/bridge_hub_rococo_local_network.toml Creds: config # send WND to //Alice from Westend AH to Rococo AH -asset-hub-rococo-collator1: run ../../environments/rococo-westend/helper.sh with "reserve-transfer-assets-from-asset-hub-westend-local" within 120 seconds +asset-hub-rococo-collator1: run {{ENV_PATH}}/helper.sh with "reserve-transfer-assets-from-asset-hub-westend-local" within 120 seconds # check that //Alice received the WND on Rococo AH -asset-hub-rococo-collator1: js-script ../../js-helpers/wrapped-assets-balance.js with "5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY,0,Westend" within 300 seconds +asset-hub-rococo-collator1: js-script {{FRAMEWORK_PATH}}/js-helpers/wrapped-assets-balance.js with "5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY,0,Westend" within 300 seconds # check that the relayer //Charlie is rewarded by Rococo AH -bridge-hub-rococo-collator1: js-script ../../js-helpers/relayer-rewards.js with "5FLSigC9HGRKVhB9FiEo4Y3koPsNmBmLJbpXg2mp1hXcS59Y,0x00000002,0x62687764,ThisChain,0" within 30 seconds +bridge-hub-rococo-collator1: js-script {{FRAMEWORK_PATH}}/js-helpers/relayer-rewards.js with "5FLSigC9HGRKVhB9FiEo4Y3koPsNmBmLJbpXg2mp1hXcS59Y,0x00000002,0x62687764,ThisChain,0" within 30 seconds diff --git a/bridges/testing/tests/0001-asset-transfer/wroc-reaches-rococo.zndsl b/bridges/testing/tests/0001-asset-transfer/wroc-reaches-rococo.zndsl index 4c0a4675234e..f72b76e6026b 100644 --- a/bridges/testing/tests/0001-asset-transfer/wroc-reaches-rococo.zndsl +++ b/bridges/testing/tests/0001-asset-transfer/wroc-reaches-rococo.zndsl @@ -1,10 +1,10 @@ Description: User is able to transfer ROC from Rococo Asset Hub to Westend Asset Hub and back -Network: ../../environments/rococo-westend/bridge_hub_westend_local_network.toml +Network: {{ENV_PATH}}/bridge_hub_westend_local_network.toml Creds: config # send wROC back to Alice from Westend AH to Rococo AH -asset-hub-rococo-collator1: run ../../environments/rococo-westend/helper.sh with "withdraw-reserve-assets-from-asset-hub-westend-local" within 120 seconds +asset-hub-rococo-collator1: run {{ENV_PATH}}/helper.sh with "withdraw-reserve-assets-from-asset-hub-westend-local" within 120 seconds # check that //Alice received the wROC on Rococo AH # (we wait until //Alice account increases here - there are no other transactions that may increase it) -asset-hub-rococo-collator1: js-script ../../js-helpers/native-assets-balance-increased.js with "5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY" within 300 seconds +asset-hub-rococo-collator1: js-script {{FRAMEWORK_PATH}}/js-helpers/native-assets-balance-increased.js with "5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY" within 300 seconds diff --git a/bridges/testing/tests/0001-asset-transfer/wwnd-reaches-westend.zndsl b/bridges/testing/tests/0001-asset-transfer/wwnd-reaches-westend.zndsl index 3acded97d5cc..9d893c8d90a8 100644 --- a/bridges/testing/tests/0001-asset-transfer/wwnd-reaches-westend.zndsl +++ b/bridges/testing/tests/0001-asset-transfer/wwnd-reaches-westend.zndsl @@ -1,10 +1,10 @@ Description: User is able to transfer ROC from Rococo Asset Hub to Westend Asset Hub and back -Network: ../../environments/rococo-westend/bridge_hub_westend_local_network.toml +Network: {{ENV_PATH}}/bridge_hub_westend_local_network.toml Creds: config # send wWND back to Alice from Rococo AH to Westend AH -asset-hub-westend-collator1: run ../../environments/rococo-westend/helper.sh with "withdraw-reserve-assets-from-asset-hub-rococo-local" within 120 seconds +asset-hub-westend-collator1: run {{ENV_PATH}}/helper.sh with "withdraw-reserve-assets-from-asset-hub-rococo-local" within 120 seconds # check that //Alice received the wWND on Westend AH # (we wait until //Alice account increases here - there are no other transactions that may increase it) -asset-hub-westend-collator1: js-script ../../js-helpers/native-assets-balance-increased.js with "5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY" within 300 seconds +asset-hub-westend-collator1: js-script {{FRAMEWORK_PATH}}/js-helpers/native-assets-balance-increased.js with "5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY" within 300 seconds diff --git a/bridges/testing/tests/0002-mandatory-headers-synced-while-idle/rococo-to-westend.zndsl b/bridges/testing/tests/0002-mandatory-headers-synced-while-idle/rococo-to-westend.zndsl index 82a1a103b14a..6e381f537732 100644 --- a/bridges/testing/tests/0002-mandatory-headers-synced-while-idle/rococo-to-westend.zndsl +++ b/bridges/testing/tests/0002-mandatory-headers-synced-while-idle/rococo-to-westend.zndsl @@ -1,8 +1,8 @@ Description: While relayer is idle, we only sync mandatory Rococo (and a single Rococo BH) headers to Westend BH. -Network: ../../environments/rococo-westend/bridge_hub_westend_local_network.toml +Network: {{ENV_PATH}}/bridge_hub_westend_local_network.toml Creds: config # ensure that relayer is only syncing mandatory headers while idle. This includes both headers that were # generated while relay was offline and those in the next 100 seconds while script is active. -bridge-hub-westend-collator1: js-script ../../js-helpers/only-mandatory-headers-synced-when-idle.js with "300,rococo-at-westend" within 600 seconds +bridge-hub-westend-collator1: js-script {{FRAMEWORK_PATH}}/js-helpers/only-mandatory-headers-synced-when-idle.js with "300,rococo-at-westend" within 600 seconds diff --git a/bridges/testing/tests/0002-mandatory-headers-synced-while-idle/run.sh b/bridges/testing/tests/0002-mandatory-headers-synced-while-idle/run.sh index 423f4a1bcc0f..7d5b8d927366 100755 --- a/bridges/testing/tests/0002-mandatory-headers-synced-while-idle/run.sh +++ b/bridges/testing/tests/0002-mandatory-headers-synced-while-idle/run.sh @@ -2,19 +2,19 @@ set -e -source "${BASH_SOURCE%/*}/../../utils/common.sh" -source "${BASH_SOURCE%/*}/../../utils/zombienet.sh" +source "${BASH_SOURCE%/*}/../../framework/utils/common.sh" +source "${BASH_SOURCE%/*}/../../framework/utils/zombienet.sh" -# We use `--relayer-delay` in order to sleep some time before starting relayer. -# We want to sleep for at least 1 session, which is expected to be 60 seconds for test environment. -${BASH_SOURCE%/*}/../../environments/rococo-westend/spawn.sh & +export ENV_PATH=`realpath ${BASH_SOURCE%/*}/../../environments/rococo-westend` + +$ENV_PATH/spawn.sh & env_pid=$! -ensure_process_file $env_pid $TEST_DIR/rococo.env 400 +ensure_process_file $env_pid $TEST_DIR/rococo.env 600 rococo_dir=`cat $TEST_DIR/rococo.env` echo -ensure_process_file $env_pid $TEST_DIR/westend.env 180 +ensure_process_file $env_pid $TEST_DIR/westend.env 300 westend_dir=`cat $TEST_DIR/westend.env` echo diff --git a/bridges/testing/tests/0002-mandatory-headers-synced-while-idle/westend-to-rococo.zndsl b/bridges/testing/tests/0002-mandatory-headers-synced-while-idle/westend-to-rococo.zndsl index 865813246252..b4b3e4367916 100644 --- a/bridges/testing/tests/0002-mandatory-headers-synced-while-idle/westend-to-rococo.zndsl +++ b/bridges/testing/tests/0002-mandatory-headers-synced-while-idle/westend-to-rococo.zndsl @@ -1,7 +1,7 @@ Description: While relayer is idle, we only sync mandatory Westend (and a single Westend BH) headers to Rococo BH. -Network: ../../environments/rococo-westend/bridge_hub_rococo_local_network.toml +Network: {{ENV_PATH}}/bridge_hub_rococo_local_network.toml Creds: config # ensure that relayer is only syncing mandatory headers while idle. This includes both headers that were # generated while relay was offline and those in the next 100 seconds while script is active. -bridge-hub-rococo-collator1: js-script ../../js-helpers/only-mandatory-headers-synced-when-idle.js with "300,westend-at-rococo" within 600 seconds +bridge-hub-rococo-collator1: js-script {{FRAMEWORK_PATH}}/js-helpers/only-mandatory-headers-synced-when-idle.js with "300,westend-at-rococo" within 600 seconds diff --git a/cumulus/client/consensus/aura/src/collators/basic.rs b/cumulus/client/consensus/aura/src/collators/basic.rs index 8740b06005d6..52b83254951f 100644 --- a/cumulus/client/consensus/aura/src/collators/basic.rs +++ b/cumulus/client/consensus/aura/src/collators/basic.rs @@ -33,12 +33,12 @@ use cumulus_relay_chain_interface::RelayChainInterface; use polkadot_node_primitives::CollationResult; use polkadot_overseer::Handle as OverseerHandle; -use polkadot_primitives::{CollatorPair, Id as ParaId}; +use polkadot_primitives::{CollatorPair, Id as ParaId, ValidationCode}; use futures::{channel::mpsc::Receiver, prelude::*}; use sc_client_api::{backend::AuxStore, BlockBackend, BlockOf}; use sc_consensus::BlockImport; -use sp_api::ProvideRuntimeApi; +use sp_api::{CallApiAt, ProvideRuntimeApi}; use sp_application_crypto::AppPublic; use sp_blockchain::HeaderBackend; use sp_consensus::SyncOracle; @@ -47,6 +47,7 @@ use sp_core::crypto::Pair; use sp_inherents::CreateInherentDataProviders; use sp_keystore::KeystorePtr; use sp_runtime::traits::{Block as BlockT, Header as HeaderT, Member}; +use sp_state_machine::Backend as _; use std::{convert::TryFrom, sync::Arc, time::Duration}; use crate::collator as collator_util; @@ -100,6 +101,7 @@ where + AuxStore + HeaderBackend + BlockBackend + + CallApiAt + Send + Sync + 'static, @@ -172,6 +174,22 @@ where continue } + let Ok(Some(code)) = + params.para_client.state_at(parent_hash).map_err(drop).and_then(|s| { + s.storage(&sp_core::storage::well_known_keys::CODE).map_err(drop) + }) + else { + continue; + }; + + super::check_validation_code_or_log( + &ValidationCode::from(code).hash(), + params.para_id, + ¶ms.relay_client, + *request.relay_parent(), + ) + .await; + let relay_parent_header = match params.relay_client.header(RBlockId::hash(*request.relay_parent())).await { Err(e) => reject_with_error!(e), diff --git a/cumulus/client/consensus/aura/src/collators/lookahead.rs b/cumulus/client/consensus/aura/src/collators/lookahead.rs index a9f33173d832..161f10d55a19 100644 --- a/cumulus/client/consensus/aura/src/collators/lookahead.rs +++ b/cumulus/client/consensus/aura/src/collators/lookahead.rs @@ -290,10 +290,7 @@ where // If the longest chain has space, build upon that. Otherwise, don't // build at all. potential_parents.sort_by_key(|a| a.depth); - let initial_parent = match potential_parents.pop() { - None => continue, - Some(p) => p, - }; + let Some(initial_parent) = potential_parents.pop() else { continue }; // Build in a loop until not allowed. Note that the authorities can change // at any block, so we need to re-claim our slot every time. @@ -301,6 +298,10 @@ where let mut parent_header = initial_parent.header; let overseer_handle = &mut params.overseer_handle; + // We mainly call this to inform users at genesis if there is a mismatch with the + // on-chain data. + collator.collator_service().check_block_status(parent_hash, &parent_header); + // This needs to change to support elastic scaling, but for continuously // scheduled chains this ensures that the backlog will grow steadily. for n_built in 0..2 { @@ -353,6 +354,14 @@ where Some(v) => v, }; + super::check_validation_code_or_log( + &validation_code_hash, + params.para_id, + ¶ms.relay_client, + relay_parent, + ) + .await; + match collator .collate( &parent_header, diff --git a/cumulus/client/consensus/aura/src/collators/mod.rs b/cumulus/client/consensus/aura/src/collators/mod.rs index 4c7b759daf73..6e0067d0cedb 100644 --- a/cumulus/client/consensus/aura/src/collators/mod.rs +++ b/cumulus/client/consensus/aura/src/collators/mod.rs @@ -20,5 +20,60 @@ //! included parachain block, as well as the [`lookahead`] collator, which prospectively //! builds on parachain blocks which have not yet been included in the relay chain. +use cumulus_relay_chain_interface::RelayChainInterface; +use polkadot_primitives::{ + Hash as RHash, Id as ParaId, OccupiedCoreAssumption, ValidationCodeHash, +}; + pub mod basic; pub mod lookahead; + +/// Check the `local_validation_code_hash` against the validation code hash in the relay chain +/// state. +/// +/// If the code hashes do not match, it prints a warning. +async fn check_validation_code_or_log( + local_validation_code_hash: &ValidationCodeHash, + para_id: ParaId, + relay_client: &impl RelayChainInterface, + relay_parent: RHash, +) { + let state_validation_code_hash = match relay_client + .validation_code_hash(relay_parent, para_id, OccupiedCoreAssumption::Included) + .await + { + Ok(hash) => hash, + Err(error) => { + tracing::debug!( + target: super::LOG_TARGET, + %error, + ?relay_parent, + %para_id, + "Failed to fetch validation code hash", + ); + return + }, + }; + + match state_validation_code_hash { + Some(state) => + if state != *local_validation_code_hash { + tracing::warn!( + target: super::LOG_TARGET, + %para_id, + ?relay_parent, + ?local_validation_code_hash, + relay_validation_code_hash = ?state, + "Parachain code doesn't match validation code stored in the relay chain state", + ); + }, + None => { + tracing::warn!( + target: super::LOG_TARGET, + %para_id, + ?relay_parent, + "Could not find validation code for parachain in the relay chain state.", + ); + }, + } +} diff --git a/cumulus/client/consensus/common/src/tests.rs b/cumulus/client/consensus/common/src/tests.rs index 597d1ab2acc2..bfb95ae388ae 100644 --- a/cumulus/client/consensus/common/src/tests.rs +++ b/cumulus/client/consensus/common/src/tests.rs @@ -136,6 +136,15 @@ impl RelayChainInterface for Relaychain { Ok(Some(PersistedValidationData { parent_head, ..Default::default() })) } + async fn validation_code_hash( + &self, + _: PHash, + _: ParaId, + _: OccupiedCoreAssumption, + ) -> RelayChainResult> { + unimplemented!("Not needed for test") + } + async fn candidate_pending_availability( &self, _: PHash, diff --git a/cumulus/client/network/src/tests.rs b/cumulus/client/network/src/tests.rs index e03f470753bb..d986635f961c 100644 --- a/cumulus/client/network/src/tests.rs +++ b/cumulus/client/network/src/tests.rs @@ -117,6 +117,15 @@ impl RelayChainInterface for DummyRelayChainInterface { })) } + async fn validation_code_hash( + &self, + _: PHash, + _: ParaId, + _: OccupiedCoreAssumption, + ) -> RelayChainResult> { + unimplemented!("Not needed for test") + } + async fn candidate_pending_availability( &self, _: PHash, diff --git a/cumulus/client/relay-chain-inprocess-interface/src/lib.rs b/cumulus/client/relay-chain-inprocess-interface/src/lib.rs index 866214fe2c52..6ea02b2e7c1f 100644 --- a/cumulus/client/relay-chain-inprocess-interface/src/lib.rs +++ b/cumulus/client/relay-chain-inprocess-interface/src/lib.rs @@ -21,7 +21,7 @@ use cumulus_primitives_core::{ relay_chain::{ runtime_api::ParachainHost, Block as PBlock, BlockId, CommittedCandidateReceipt, Hash as PHash, Header as PHeader, InboundHrmpMessage, OccupiedCoreAssumption, SessionIndex, - ValidatorId, + ValidationCodeHash, ValidatorId, }, InboundDownwardMessage, ParaId, PersistedValidationData, }; @@ -115,6 +115,19 @@ impl RelayChainInterface for RelayChainInProcessInterface { )?) } + async fn validation_code_hash( + &self, + hash: PHash, + para_id: ParaId, + occupied_core_assumption: OccupiedCoreAssumption, + ) -> RelayChainResult> { + Ok(self.full_client.runtime_api().validation_code_hash( + hash, + para_id, + occupied_core_assumption, + )?) + } + async fn candidate_pending_availability( &self, hash: PHash, diff --git a/cumulus/client/relay-chain-interface/src/lib.rs b/cumulus/client/relay-chain-interface/src/lib.rs index de5e7891b30d..bb93e6a168c8 100644 --- a/cumulus/client/relay-chain-interface/src/lib.rs +++ b/cumulus/client/relay-chain-interface/src/lib.rs @@ -30,7 +30,7 @@ use cumulus_primitives_core::relay_chain::BlockId; pub use cumulus_primitives_core::{ relay_chain::{ CommittedCandidateReceipt, Hash as PHash, Header as PHeader, InboundHrmpMessage, - OccupiedCoreAssumption, SessionIndex, ValidatorId, + OccupiedCoreAssumption, SessionIndex, ValidationCodeHash, ValidatorId, }, InboundDownwardMessage, ParaId, PersistedValidationData, }; @@ -194,6 +194,15 @@ pub trait RelayChainInterface: Send + Sync { relay_parent: PHash, relevant_keys: &Vec>, ) -> RelayChainResult; + + /// Returns the validation code hash for the given `para_id` using the given + /// `occupied_core_assumption`. + async fn validation_code_hash( + &self, + relay_parent: PHash, + para_id: ParaId, + occupied_core_assumption: OccupiedCoreAssumption, + ) -> RelayChainResult>; } #[async_trait] @@ -301,4 +310,15 @@ where async fn header(&self, block_id: BlockId) -> RelayChainResult> { (**self).header(block_id).await } + + async fn validation_code_hash( + &self, + relay_parent: PHash, + para_id: ParaId, + occupied_core_assumption: OccupiedCoreAssumption, + ) -> RelayChainResult> { + (**self) + .validation_code_hash(relay_parent, para_id, occupied_core_assumption) + .await + } } diff --git a/cumulus/client/relay-chain-rpc-interface/src/lib.rs b/cumulus/client/relay-chain-rpc-interface/src/lib.rs index 96f8fc8b5563..3a4c186e301e 100644 --- a/cumulus/client/relay-chain-rpc-interface/src/lib.rs +++ b/cumulus/client/relay-chain-rpc-interface/src/lib.rs @@ -19,7 +19,7 @@ use core::time::Duration; use cumulus_primitives_core::{ relay_chain::{ CommittedCandidateReceipt, Hash as RelayHash, Header as RelayHeader, InboundHrmpMessage, - OccupiedCoreAssumption, SessionIndex, ValidatorId, + OccupiedCoreAssumption, SessionIndex, ValidationCodeHash, ValidatorId, }, InboundDownwardMessage, ParaId, PersistedValidationData, }; @@ -110,6 +110,17 @@ impl RelayChainInterface for RelayChainRpcInterface { .await } + async fn validation_code_hash( + &self, + hash: RelayHash, + para_id: ParaId, + occupied_core_assumption: OccupiedCoreAssumption, + ) -> RelayChainResult> { + self.rpc_client + .validation_code_hash(hash, para_id, occupied_core_assumption) + .await + } + async fn candidate_pending_availability( &self, hash: RelayHash, diff --git a/cumulus/client/relay-chain-rpc-interface/src/rpc_client.rs b/cumulus/client/relay-chain-rpc-interface/src/rpc_client.rs index a912997e947d..6578210a259c 100644 --- a/cumulus/client/relay-chain-rpc-interface/src/rpc_client.rs +++ b/cumulus/client/relay-chain-rpc-interface/src/rpc_client.rs @@ -647,6 +647,20 @@ impl RelayChainRpcClient { .await } + pub async fn validation_code_hash( + &self, + at: RelayHash, + para_id: ParaId, + occupied_core_assumption: OccupiedCoreAssumption, + ) -> Result, RelayChainError> { + self.call_remote_runtime_function( + "ParachainHost_validation_code_hash", + at, + Some((para_id, occupied_core_assumption)), + ) + .await + } + fn send_register_message_to_worker( &self, message: RpcDispatcherMessage, diff --git a/cumulus/pallets/aura-ext/src/lib.rs b/cumulus/pallets/aura-ext/src/lib.rs index 34a41557152d..31b571816a0c 100644 --- a/cumulus/pallets/aura-ext/src/lib.rs +++ b/cumulus/pallets/aura-ext/src/lib.rs @@ -117,12 +117,6 @@ pub mod pallet { impl BuildGenesisConfig for GenesisConfig { fn build(&self) { let authorities = Aura::::authorities(); - - assert!( - !authorities.is_empty(), - "AuRa authorities empty, maybe wrong order in `construct_runtime!`?", - ); - Authorities::::put(authorities); } } diff --git a/cumulus/pallets/parachain-system/src/validate_block/implementation.rs b/cumulus/pallets/parachain-system/src/validate_block/implementation.rs index ce3b724420f1..ecab7a9a0931 100644 --- a/cumulus/pallets/parachain-system/src/validate_block/implementation.rs +++ b/cumulus/pallets/parachain-system/src/validate_block/implementation.rs @@ -16,7 +16,7 @@ //! The actual implementation of the validate block functionality. -use super::{trie_cache, MemoryOptimizedValidationParams}; +use super::{trie_cache, trie_recorder, MemoryOptimizedValidationParams}; use cumulus_primitives_core::{ relay_chain::Hash as RHash, ParachainBlockData, PersistedValidationData, }; @@ -34,12 +34,14 @@ use sp_externalities::{set_and_run_with_externalities, Externalities}; use sp_io::KillStorageResult; use sp_runtime::traits::{Block as BlockT, Extrinsic, HashingFor, Header as HeaderT}; use sp_std::prelude::*; -use sp_trie::MemoryDB; +use sp_trie::{MemoryDB, ProofSizeProvider}; +use trie_recorder::SizeOnlyRecorderProvider; type TrieBackend = sp_state_machine::TrieBackend< MemoryDB>, HashingFor, trie_cache::CacheProvider>, + SizeOnlyRecorderProvider>, >; type Ext<'a, B> = sp_state_machine::Ext<'a, HashingFor, TrieBackend>; @@ -48,6 +50,9 @@ fn with_externalities R, R>(f: F) -> R { sp_externalities::with_externalities(f).expect("Environmental externalities not set.") } +// Recorder instance to be used during this validate_block call. +environmental::environmental!(recorder: trait ProofSizeProvider); + /// Validate the given parachain block. /// /// This function is doing roughly the following: @@ -120,6 +125,7 @@ where sp_std::mem::drop(storage_proof); + let mut recorder = SizeOnlyRecorderProvider::new(); let cache_provider = trie_cache::CacheProvider::new(); // We use the storage root of the `parent_head` to ensure that it is the correct root. // This is already being done above while creating the in-memory db, but let's be paranoid!! @@ -128,6 +134,7 @@ where *parent_header.state_root(), cache_provider, ) + .with_recorder(recorder.clone()) .build(); let _guard = ( @@ -167,9 +174,11 @@ where .replace_implementation(host_default_child_storage_next_key), sp_io::offchain_index::host_set.replace_implementation(host_offchain_index_set), sp_io::offchain_index::host_clear.replace_implementation(host_offchain_index_clear), + cumulus_primitives_proof_size_hostfunction::storage_proof_size::host_storage_proof_size + .replace_implementation(host_storage_proof_size), ); - run_with_externalities::(&backend, || { + run_with_externalities_and_recorder::(&backend, &mut recorder, || { let relay_chain_proof = crate::RelayChainStateProof::new( PSC::SelfParaId::get(), inherent_data.validation_data.relay_parent_storage_root, @@ -190,7 +199,7 @@ where } }); - run_with_externalities::(&backend, || { + run_with_externalities_and_recorder::(&backend, &mut recorder, || { let head_data = HeadData(block.header().encode()); E::execute_block(block); @@ -265,15 +274,17 @@ fn validate_validation_data( ); } -/// Run the given closure with the externalities set. -fn run_with_externalities R>( +/// Run the given closure with the externalities and recorder set. +fn run_with_externalities_and_recorder R>( backend: &TrieBackend, + recorder: &mut SizeOnlyRecorderProvider>, execute: F, ) -> R { let mut overlay = sp_state_machine::OverlayedChanges::default(); let mut ext = Ext::::new(&mut overlay, backend); + recorder.reset(); - set_and_run_with_externalities(&mut ext, || execute()) + recorder::using(recorder, || set_and_run_with_externalities(&mut ext, || execute())) } fn host_storage_read(key: &[u8], value_out: &mut [u8], value_offset: u32) -> Option { @@ -305,6 +316,10 @@ fn host_storage_clear(key: &[u8]) { with_externalities(|ext| ext.place_storage(key.to_vec(), None)) } +fn host_storage_proof_size() -> u64 { + recorder::with(|rec| rec.estimate_encoded_size()).expect("Recorder is always set; qed") as _ +} + fn host_storage_root(version: StateVersion) -> Vec { with_externalities(|ext| ext.storage_root(version)) } diff --git a/cumulus/pallets/parachain-system/src/validate_block/tests.rs b/cumulus/pallets/parachain-system/src/validate_block/tests.rs index f17ac6007a09..a9fb65e11089 100644 --- a/cumulus/pallets/parachain-system/src/validate_block/tests.rs +++ b/cumulus/pallets/parachain-system/src/validate_block/tests.rs @@ -59,7 +59,7 @@ fn call_validate_block( } fn create_test_client() -> (Client, Header) { - let client = TestClientBuilder::new().build(); + let client = TestClientBuilder::new().enable_import_proof_recording().build(); let genesis_header = client .header(client.chain_info().genesis_hash) diff --git a/cumulus/pallets/parachain-system/src/validate_block/trie_recorder.rs b/cumulus/pallets/parachain-system/src/validate_block/trie_recorder.rs index e73aef70aa49..48310670c074 100644 --- a/cumulus/pallets/parachain-system/src/validate_block/trie_recorder.rs +++ b/cumulus/pallets/parachain-system/src/validate_block/trie_recorder.rs @@ -97,6 +97,7 @@ pub(crate) struct SizeOnlyRecorderProvider { } impl SizeOnlyRecorderProvider { + /// Create a new instance of [`SizeOnlyRecorderProvider`] pub fn new() -> Self { Self { seen_nodes: Default::default(), @@ -104,6 +105,13 @@ impl SizeOnlyRecorderProvider { recorded_keys: Default::default(), } } + + /// Reset the internal state. + pub fn reset(&self) { + self.seen_nodes.borrow_mut().clear(); + *self.encoded_size.borrow_mut() = 0; + self.recorded_keys.borrow_mut().clear(); + } } impl sp_trie::TrieRecorderProvider for SizeOnlyRecorderProvider { @@ -281,6 +289,9 @@ mod tests { reference_recorder.estimate_encoded_size(), recorder_for_test.estimate_encoded_size() ); + + recorder_for_test.reset(); + assert_eq!(recorder_for_test.estimate_encoded_size(), 0) } } } diff --git a/cumulus/parachain-template/node/src/command.rs b/cumulus/parachain-template/node/src/command.rs index 72b3ab7bb4b9..82624ae0be59 100644 --- a/cumulus/parachain-template/node/src/command.rs +++ b/cumulus/parachain-template/node/src/command.rs @@ -1,5 +1,6 @@ use std::net::SocketAddr; +use cumulus_client_service::storage_proof_size::HostFunctions as ReclaimHostFunctions; use cumulus_primitives_core::ParaId; use frame_benchmarking_cli::{BenchmarkCmd, SUBSTRATE_REFERENCE_HARDWARE}; use log::info; @@ -183,7 +184,7 @@ pub fn run() -> Result<()> { match cmd { BenchmarkCmd::Pallet(cmd) => if cfg!(feature = "runtime-benchmarks") { - runner.sync_run(|config| cmd.run::, ()>(config)) + runner.sync_run(|config| cmd.run::, ReclaimHostFunctions>(config)) } else { Err("Benchmarking wasn't enabled when building the node. \ You can enable it with `--features runtime-benchmarks`." diff --git a/cumulus/parachain-template/node/src/service.rs b/cumulus/parachain-template/node/src/service.rs index 830b6e82f969..4dd24803e9b1 100644 --- a/cumulus/parachain-template/node/src/service.rs +++ b/cumulus/parachain-template/node/src/service.rs @@ -40,7 +40,10 @@ use substrate_prometheus_endpoint::Registry; pub struct ParachainNativeExecutor; impl sc_executor::NativeExecutionDispatch for ParachainNativeExecutor { - type ExtendHostFunctions = frame_benchmarking::benchmarking::HostFunctions; + type ExtendHostFunctions = ( + cumulus_client_service::storage_proof_size::HostFunctions, + frame_benchmarking::benchmarking::HostFunctions, + ); fn dispatch(method: &str, data: &[u8]) -> Option> { parachain_template_runtime::api::dispatch(method, data) @@ -100,10 +103,11 @@ pub fn new_partial(config: &Configuration) -> Result let executor = ParachainExecutor::new_with_wasm_executor(wasm); let (client, backend, keystore_container, task_manager) = - sc_service::new_full_parts::( + sc_service::new_full_parts_record_import::( config, telemetry.as_ref().map(|(_, telemetry)| telemetry.handle()), executor, + true, )?; let client = Arc::new(client); diff --git a/cumulus/parachain-template/pallets/template/src/lib.rs b/cumulus/parachain-template/pallets/template/src/lib.rs index 5f3252bfc3a7..24226d6cf406 100644 --- a/cumulus/parachain-template/pallets/template/src/lib.rs +++ b/cumulus/parachain-template/pallets/template/src/lib.rs @@ -32,7 +32,6 @@ pub mod pallet { // The pallet's runtime storage items. // https://docs.substrate.io/v3/runtime/storage #[pallet::storage] - #[pallet::getter(fn something)] // Learn more about declaring storage items: // https://docs.substrate.io/v3/runtime/storage#declaring-storage-items pub type Something = StorageValue<_, u32>; diff --git a/cumulus/parachain-template/pallets/template/src/tests.rs b/cumulus/parachain-template/pallets/template/src/tests.rs index 527aec8ed00c..9ad3076be2cc 100644 --- a/cumulus/parachain-template/pallets/template/src/tests.rs +++ b/cumulus/parachain-template/pallets/template/src/tests.rs @@ -1,4 +1,4 @@ -use crate::{mock::*, Error}; +use crate::{mock::*, Error, Something}; use frame_support::{assert_noop, assert_ok}; #[test] @@ -7,7 +7,7 @@ fn it_works_for_default_value() { // Dispatch a signed extrinsic. assert_ok!(TemplateModule::do_something(RuntimeOrigin::signed(1), 42)); // Read pallet storage and assert an expected result. - assert_eq!(TemplateModule::something(), Some(42)); + assert_eq!(Something::::get(), Some(42)); }); } diff --git a/cumulus/parachain-template/runtime/Cargo.toml b/cumulus/parachain-template/runtime/Cargo.toml index 44d96ffc4e62..1873bd0a23eb 100644 --- a/cumulus/parachain-template/runtime/Cargo.toml +++ b/cumulus/parachain-template/runtime/Cargo.toml @@ -73,6 +73,7 @@ cumulus-pallet-xcm = { path = "../../pallets/xcm", default-features = false } cumulus-pallet-xcmp-queue = { path = "../../pallets/xcmp-queue", default-features = false } cumulus-primitives-core = { path = "../../primitives/core", default-features = false } cumulus-primitives-utility = { path = "../../primitives/utility", default-features = false } +cumulus-primitives-storage-weight-reclaim = { path = "../../primitives/storage-weight-reclaim", default-features = false } pallet-collator-selection = { path = "../../pallets/collator-selection", default-features = false } parachains-common = { path = "../../parachains/common", default-features = false } parachain-info = { package = "staging-parachain-info", path = "../../parachains/pallets/parachain-info", default-features = false } @@ -87,6 +88,7 @@ std = [ "cumulus-pallet-xcm/std", "cumulus-pallet-xcmp-queue/std", "cumulus-primitives-core/std", + "cumulus-primitives-storage-weight-reclaim/std", "cumulus-primitives-utility/std", "frame-benchmarking?/std", "frame-executive/std", diff --git a/cumulus/parachain-template/runtime/src/lib.rs b/cumulus/parachain-template/runtime/src/lib.rs index d9bc111fcef7..cee9b33bf37c 100644 --- a/cumulus/parachain-template/runtime/src/lib.rs +++ b/cumulus/parachain-template/runtime/src/lib.rs @@ -107,6 +107,7 @@ pub type SignedExtra = ( frame_system::CheckNonce, frame_system::CheckWeight, pallet_transaction_payment::ChargeTransactionPayment, + cumulus_primitives_storage_weight_reclaim::StorageWeightReclaim, ); /// Unchecked extrinsic type as expected by this runtime. diff --git a/cumulus/parachains/integration-tests/emulated/chains/relays/rococo/src/genesis.rs b/cumulus/parachains/integration-tests/emulated/chains/relays/rococo/src/genesis.rs index 7db9679f1c3e..55437645b052 100644 --- a/cumulus/parachains/integration-tests/emulated/chains/relays/rococo/src/genesis.rs +++ b/cumulus/parachains/integration-tests/emulated/chains/relays/rococo/src/genesis.rs @@ -78,7 +78,7 @@ pub fn genesis() -> Storage { }, babe: rococo_runtime::BabeConfig { authorities: Default::default(), - epoch_config: Some(rococo_runtime::BABE_GENESIS_EPOCH_CONFIG), + epoch_config: rococo_runtime::BABE_GENESIS_EPOCH_CONFIG, ..Default::default() }, sudo: rococo_runtime::SudoConfig { diff --git a/cumulus/parachains/integration-tests/emulated/chains/relays/westend/src/genesis.rs b/cumulus/parachains/integration-tests/emulated/chains/relays/westend/src/genesis.rs index 578b307dd933..700b80e63f6c 100644 --- a/cumulus/parachains/integration-tests/emulated/chains/relays/westend/src/genesis.rs +++ b/cumulus/parachains/integration-tests/emulated/chains/relays/westend/src/genesis.rs @@ -94,7 +94,7 @@ pub fn genesis() -> Storage { }, babe: westend_runtime::BabeConfig { authorities: Default::default(), - epoch_config: Some(westend_runtime::BABE_GENESIS_EPOCH_CONFIG), + epoch_config: westend_runtime::BABE_GENESIS_EPOCH_CONFIG, ..Default::default() }, configuration: westend_runtime::ConfigurationConfig { config: get_host_config() }, diff --git a/cumulus/parachains/runtimes/contracts/contracts-rococo/src/contracts.rs b/cumulus/parachains/runtimes/contracts/contracts-rococo/src/contracts.rs index 7b89f2df8077..681b95ce6a53 100644 --- a/cumulus/parachains/runtimes/contracts/contracts-rococo/src/contracts.rs +++ b/cumulus/parachains/runtimes/contracts/contracts-rococo/src/contracts.rs @@ -72,5 +72,6 @@ impl Config for Runtime { type RuntimeHoldReason = RuntimeHoldReason; type Debug = (); type Environment = (); + type ApiVersion = (); type Xcm = pallet_xcm::Pallet; } diff --git a/cumulus/polkadot-parachain/src/chain_spec/bridge_hubs.rs b/cumulus/polkadot-parachain/src/chain_spec/bridge_hubs.rs index 1db826ea7daf..15e8a1bf11a0 100644 --- a/cumulus/polkadot-parachain/src/chain_spec/bridge_hubs.rs +++ b/cumulus/polkadot-parachain/src/chain_spec/bridge_hubs.rs @@ -25,7 +25,10 @@ use std::str::FromStr; #[derive(Debug, PartialEq)] pub enum BridgeHubRuntimeType { Kusama, + KusamaLocal, + Polkadot, + PolkadotLocal, Rococo, RococoLocal, @@ -44,7 +47,9 @@ impl FromStr for BridgeHubRuntimeType { fn from_str(value: &str) -> Result { match value { polkadot::BRIDGE_HUB_POLKADOT => Ok(BridgeHubRuntimeType::Polkadot), + polkadot::BRIDGE_HUB_POLKADOT_LOCAL => Ok(BridgeHubRuntimeType::PolkadotLocal), kusama::BRIDGE_HUB_KUSAMA => Ok(BridgeHubRuntimeType::Kusama), + kusama::BRIDGE_HUB_KUSAMA_LOCAL => Ok(BridgeHubRuntimeType::KusamaLocal), westend::BRIDGE_HUB_WESTEND => Ok(BridgeHubRuntimeType::Westend), westend::BRIDGE_HUB_WESTEND_LOCAL => Ok(BridgeHubRuntimeType::WestendLocal), westend::BRIDGE_HUB_WESTEND_DEVELOPMENT => Ok(BridgeHubRuntimeType::WestendDevelopment), @@ -103,6 +108,7 @@ impl BridgeHubRuntimeType { Some("Bob".to_string()), |_| (), ))), + other => Err(std::format!("No default config present for {:?}", other)), } } } @@ -242,6 +248,7 @@ pub mod rococo { /// Sub-module for Kusama setup pub mod kusama { pub(crate) const BRIDGE_HUB_KUSAMA: &str = "bridge-hub-kusama"; + pub(crate) const BRIDGE_HUB_KUSAMA_LOCAL: &str = "bridge-hub-kusama-local"; } /// Sub-module for Westend setup. @@ -358,4 +365,5 @@ pub mod westend { /// Sub-module for Polkadot setup pub mod polkadot { pub(crate) const BRIDGE_HUB_POLKADOT: &str = "bridge-hub-polkadot"; + pub(crate) const BRIDGE_HUB_POLKADOT_LOCAL: &str = "bridge-hub-polkadot-local"; } diff --git a/cumulus/polkadot-parachain/src/command.rs b/cumulus/polkadot-parachain/src/command.rs index acba32f048be..4d44879af515 100644 --- a/cumulus/polkadot-parachain/src/command.rs +++ b/cumulus/polkadot-parachain/src/command.rs @@ -23,6 +23,7 @@ use crate::{ }, service::{new_partial, Block}, }; +use cumulus_client_service::storage_proof_size::HostFunctions as ReclaimHostFunctions; use cumulus_primitives_core::ParaId; use frame_benchmarking_cli::{BenchmarkCmd, SUBSTRATE_REFERENCE_HARDWARE}; use log::info; @@ -584,7 +585,7 @@ pub fn run() -> Result<()> { match cmd { BenchmarkCmd::Pallet(cmd) => if cfg!(feature = "runtime-benchmarks") { - runner.sync_run(|config| cmd.run::, ()>(config)) + runner.sync_run(|config| cmd.run::, ReclaimHostFunctions>(config)) } else { Err("Benchmarking wasn't enabled when building the node. \ You can enable it with `--features runtime-benchmarks`." @@ -755,14 +756,16 @@ pub fn run() -> Result<()> { .map_err(Into::into), BridgeHub(bridge_hub_runtime_type) => match bridge_hub_runtime_type { - chain_spec::bridge_hubs::BridgeHubRuntimeType::Polkadot => + chain_spec::bridge_hubs::BridgeHubRuntimeType::Polkadot | + chain_spec::bridge_hubs::BridgeHubRuntimeType::PolkadotLocal => crate::service::start_generic_aura_node::< RuntimeApi, AuraId, >(config, polkadot_config, collator_options, id, hwbench) .await .map(|r| r.0), - chain_spec::bridge_hubs::BridgeHubRuntimeType::Kusama => + chain_spec::bridge_hubs::BridgeHubRuntimeType::Kusama | + chain_spec::bridge_hubs::BridgeHubRuntimeType::KusamaLocal => crate::service::start_generic_aura_node::< RuntimeApi, AuraId, diff --git a/cumulus/polkadot-parachain/src/service.rs b/cumulus/polkadot-parachain/src/service.rs index 553975b01a80..4e06cd38f1d7 100644 --- a/cumulus/polkadot-parachain/src/service.rs +++ b/cumulus/polkadot-parachain/src/service.rs @@ -68,11 +68,15 @@ use substrate_prometheus_endpoint::Registry; use polkadot_primitives::CollatorPair; #[cfg(not(feature = "runtime-benchmarks"))] -type HostFunctions = sp_io::SubstrateHostFunctions; +type HostFunctions = + (sp_io::SubstrateHostFunctions, cumulus_client_service::storage_proof_size::HostFunctions); #[cfg(feature = "runtime-benchmarks")] -type HostFunctions = - (sp_io::SubstrateHostFunctions, frame_benchmarking::benchmarking::HostFunctions); +type HostFunctions = ( + sp_io::SubstrateHostFunctions, + cumulus_client_service::storage_proof_size::HostFunctions, + frame_benchmarking::benchmarking::HostFunctions, +); type ParachainClient = TFullClient>; @@ -274,10 +278,11 @@ where .build(); let (client, backend, keystore_container, task_manager) = - sc_service::new_full_parts::( + sc_service::new_full_parts_record_import::( config, telemetry.as_ref().map(|(_, telemetry)| telemetry.handle()), executor, + true, )?; let client = Arc::new(client); diff --git a/cumulus/primitives/proof-size-hostfunction/src/lib.rs b/cumulus/primitives/proof-size-hostfunction/src/lib.rs index 6da6235e585a..8ebc58ea450d 100644 --- a/cumulus/primitives/proof-size-hostfunction/src/lib.rs +++ b/cumulus/primitives/proof-size-hostfunction/src/lib.rs @@ -18,6 +18,7 @@ #![cfg_attr(not(feature = "std"), no_std)] +#[cfg(feature = "std")] use sp_externalities::ExternalitiesExt; use sp_runtime_interface::runtime_interface; @@ -35,7 +36,8 @@ pub const PROOF_RECORDING_DISABLED: u64 = u64::MAX; pub trait StorageProofSize { /// Returns the current storage proof size. fn storage_proof_size(&mut self) -> u64 { - self.extension::().map_or(u64::MAX, |e| e.storage_proof_size()) + self.extension::() + .map_or(PROOF_RECORDING_DISABLED, |e| e.storage_proof_size()) } } diff --git a/cumulus/primitives/storage-weight-reclaim/Cargo.toml b/cumulus/primitives/storage-weight-reclaim/Cargo.toml new file mode 100644 index 000000000000..4835fb5192b8 --- /dev/null +++ b/cumulus/primitives/storage-weight-reclaim/Cargo.toml @@ -0,0 +1,46 @@ +[package] +name = "cumulus-primitives-storage-weight-reclaim" +version = "1.0.0" +authors.workspace = true +edition.workspace = true +description = "Utilities to reclaim storage weight." +license = "Apache-2.0" + +[lints] +workspace = true + +[dependencies] +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } +log = { workspace = true } +scale-info = { version = "2.10.0", default-features = false, features = ["derive"] } + +frame-support = { path = "../../../substrate/frame/support", default-features = false } +frame-system = { path = "../../../substrate/frame/system", default-features = false } + +sp-runtime = { path = "../../../substrate/primitives/runtime", default-features = false } +sp-std = { path = "../../../substrate/primitives/std", default-features = false } + +cumulus-primitives-core = { path = "../../primitives/core", default-features = false } +cumulus-primitives-proof-size-hostfunction = { path = "../../primitives/proof-size-hostfunction", default-features = false } +docify = "0.2.7" + +[dev-dependencies] +sp-trie = { path = "../../../substrate/primitives/trie", default-features = false } +sp-io = { path = "../../../substrate/primitives/io", default-features = false } +cumulus-test-runtime = { path = "../../test/runtime" } + +[features] +default = ["std"] +std = [ + "codec/std", + "cumulus-primitives-core/std", + "cumulus-primitives-proof-size-hostfunction/std", + "frame-support/std", + "frame-system/std", + "log/std", + "scale-info/std", + "sp-io/std", + "sp-runtime/std", + "sp-std/std", + "sp-trie/std", +] diff --git a/cumulus/primitives/storage-weight-reclaim/src/lib.rs b/cumulus/primitives/storage-weight-reclaim/src/lib.rs new file mode 100644 index 000000000000..5dddc92e3955 --- /dev/null +++ b/cumulus/primitives/storage-weight-reclaim/src/lib.rs @@ -0,0 +1,663 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Mechanism to reclaim PoV proof size weight after an extrinsic has been applied. + +#![cfg_attr(not(feature = "std"), no_std)] + +use codec::{Decode, Encode}; +use cumulus_primitives_core::Weight; +use cumulus_primitives_proof_size_hostfunction::{ + storage_proof_size::storage_proof_size, PROOF_RECORDING_DISABLED, +}; +use frame_support::{ + dispatch::{DispatchInfo, PostDispatchInfo}, + weights::WeightMeter, +}; +use frame_system::Config; +use scale_info::TypeInfo; +use sp_runtime::{ + traits::{DispatchInfoOf, Dispatchable, PostDispatchInfoOf, SignedExtension}, + transaction_validity::TransactionValidityError, + DispatchResult, +}; +use sp_std::marker::PhantomData; + +const LOG_TARGET: &'static str = "runtime::storage_reclaim"; + +/// `StorageWeightReclaimer` is a mechanism for manually reclaiming storage weight. +/// +/// It internally keeps track of the proof size and storage weight at initialization time. At +/// reclaim it computes the real consumed storage weight and refunds excess weight. +/// +/// # Example +#[doc = docify::embed!("src/lib.rs", simple_reclaimer_example)] +pub struct StorageWeightReclaimer { + previous_remaining_proof_size: u64, + previous_reported_proof_size: Option, +} + +impl StorageWeightReclaimer { + /// Creates a new `StorageWeightReclaimer` instance and initializes it with the storage + /// size provided by `weight_meter` and reported proof size from the node. + #[must_use = "Must call `reclaim_with_meter` to reclaim the weight"] + pub fn new(weight_meter: &WeightMeter) -> StorageWeightReclaimer { + let previous_remaining_proof_size = weight_meter.remaining().proof_size(); + let previous_reported_proof_size = get_proof_size(); + Self { previous_remaining_proof_size, previous_reported_proof_size } + } + + /// Check the consumed storage weight and calculate the consumed excess weight. + fn reclaim(&mut self, remaining_weight: Weight) -> Option { + let current_remaining_weight = remaining_weight.proof_size(); + let current_storage_proof_size = get_proof_size()?; + let previous_storage_proof_size = self.previous_reported_proof_size?; + let used_weight = + self.previous_remaining_proof_size.saturating_sub(current_remaining_weight); + let reported_used_size = + current_storage_proof_size.saturating_sub(previous_storage_proof_size); + let reclaimable = used_weight.saturating_sub(reported_used_size); + log::trace!( + target: LOG_TARGET, + "Found reclaimable storage weight. benchmarked: {used_weight}, consumed: {reported_used_size}" + ); + + self.previous_remaining_proof_size = current_remaining_weight.saturating_add(reclaimable); + self.previous_reported_proof_size = Some(current_storage_proof_size); + Some(Weight::from_parts(0, reclaimable)) + } + + /// Check the consumed storage weight and add the reclaimed + /// weight budget back to `weight_meter`. + pub fn reclaim_with_meter(&mut self, weight_meter: &mut WeightMeter) -> Option { + let reclaimed = self.reclaim(weight_meter.remaining())?; + weight_meter.reclaim_proof_size(reclaimed.proof_size()); + Some(reclaimed) + } +} + +/// Returns the current storage proof size from the host side. +/// +/// Returns `None` if proof recording is disabled on the host. +pub fn get_proof_size() -> Option { + let proof_size = storage_proof_size(); + (proof_size != PROOF_RECORDING_DISABLED).then_some(proof_size) +} + +/// Storage weight reclaim mechanism. +/// +/// This extension checks the size of the node-side storage proof +/// before and after executing a given extrinsic. The difference between +/// benchmarked and spent weight can be reclaimed. +#[derive(Encode, Decode, Clone, Eq, PartialEq, Default, TypeInfo)] +#[scale_info(skip_type_params(T))] +pub struct StorageWeightReclaim(PhantomData); + +impl StorageWeightReclaim { + /// Create a new `StorageWeightReclaim` instance. + pub fn new() -> Self { + Self(Default::default()) + } +} + +impl core::fmt::Debug for StorageWeightReclaim { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> Result<(), core::fmt::Error> { + let _ = write!(f, "StorageWeightReclaim"); + Ok(()) + } +} + +impl SignedExtension for StorageWeightReclaim +where + T::RuntimeCall: Dispatchable, +{ + const IDENTIFIER: &'static str = "StorageWeightReclaim"; + + type AccountId = T::AccountId; + type Call = T::RuntimeCall; + type AdditionalSigned = (); + type Pre = Option; + + fn additional_signed( + &self, + ) -> Result + { + Ok(()) + } + + fn pre_dispatch( + self, + _who: &Self::AccountId, + _call: &Self::Call, + _info: &sp_runtime::traits::DispatchInfoOf, + _len: usize, + ) -> Result { + Ok(get_proof_size()) + } + + fn post_dispatch( + pre: Option, + info: &DispatchInfoOf, + post_info: &PostDispatchInfoOf, + _len: usize, + _result: &DispatchResult, + ) -> Result<(), TransactionValidityError> { + let Some(Some(pre_dispatch_proof_size)) = pre else { + return Ok(()); + }; + + let Some(post_dispatch_proof_size) = get_proof_size() else { + log::debug!( + target: LOG_TARGET, + "Proof recording enabled during pre-dispatch, now disabled. This should not happen." + ); + return Ok(()) + }; + let benchmarked_weight = info.weight.proof_size(); + let consumed_weight = post_dispatch_proof_size.saturating_sub(pre_dispatch_proof_size); + + // Unspent weight according to the `actual_weight` from `PostDispatchInfo` + // This unspent weight will be refunded by the `CheckWeight` extension, so we need to + // account for that. + let unspent = post_info.calc_unspent(info).proof_size(); + let storage_size_diff = + benchmarked_weight.saturating_sub(unspent).abs_diff(consumed_weight as u64); + + // This value will be reclaimed by [`frame_system::CheckWeight`], so we need to calculate + // that in. + frame_system::BlockWeight::::mutate(|current| { + if consumed_weight > benchmarked_weight { + log::error!( + target: LOG_TARGET, + "Benchmarked storage weight smaller than consumed storage weight. benchmarked: {benchmarked_weight} consumed: {consumed_weight} unspent: {unspent}" + ); + current.accrue(Weight::from_parts(0, storage_size_diff), info.class) + } else { + log::trace!( + target: LOG_TARGET, + "Reclaiming storage weight. benchmarked: {benchmarked_weight}, consumed: {consumed_weight} unspent: {unspent}" + ); + current.reduce(Weight::from_parts(0, storage_size_diff), info.class) + } + }); + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use frame_support::{ + assert_ok, + dispatch::DispatchClass, + weights::{Weight, WeightMeter}, + }; + use frame_system::{BlockWeight, CheckWeight}; + use sp_runtime::{AccountId32, BuildStorage}; + use sp_std::marker::PhantomData; + use sp_trie::proof_size_extension::ProofSizeExt; + + type Test = cumulus_test_runtime::Runtime; + const CALL: &::RuntimeCall = + &cumulus_test_runtime::RuntimeCall::System(frame_system::Call::set_heap_pages { + pages: 0u64, + }); + const ALICE: AccountId32 = AccountId32::new([1u8; 32]); + const LEN: usize = 0; + + pub fn new_test_ext() -> sp_io::TestExternalities { + let ext: sp_io::TestExternalities = cumulus_test_runtime::RuntimeGenesisConfig::default() + .build_storage() + .unwrap() + .into(); + ext + } + + struct TestRecorder { + return_values: Box<[usize]>, + counter: std::sync::atomic::AtomicUsize, + } + + impl TestRecorder { + fn new(values: &[usize]) -> Self { + TestRecorder { return_values: values.into(), counter: Default::default() } + } + } + + impl sp_trie::ProofSizeProvider for TestRecorder { + fn estimate_encoded_size(&self) -> usize { + let counter = self.counter.fetch_add(1, core::sync::atomic::Ordering::Relaxed); + self.return_values[counter] + } + } + + fn setup_test_externalities(proof_values: &[usize]) -> sp_io::TestExternalities { + let mut test_ext = new_test_ext(); + let test_recorder = TestRecorder::new(proof_values); + test_ext.register_extension(ProofSizeExt::new(test_recorder)); + test_ext + } + + fn set_current_storage_weight(new_weight: u64) { + BlockWeight::::mutate(|current_weight| { + current_weight.set(Weight::from_parts(0, new_weight), DispatchClass::Normal); + }); + } + + #[test] + fn basic_refund() { + // The real cost will be 100 bytes of storage size + let mut test_ext = setup_test_externalities(&[0, 100]); + + test_ext.execute_with(|| { + set_current_storage_weight(1000); + + // Benchmarked storage weight: 500 + let info = DispatchInfo { weight: Weight::from_parts(0, 500), ..Default::default() }; + let post_info = PostDispatchInfo::default(); + + let pre = StorageWeightReclaim::(PhantomData) + .pre_dispatch(&ALICE, CALL, &info, LEN) + .unwrap(); + assert_eq!(pre, Some(0)); + + assert_ok!(CheckWeight::::post_dispatch(None, &info, &post_info, 0, &Ok(()))); + // We expect a refund of 400 + assert_ok!(StorageWeightReclaim::::post_dispatch( + Some(pre), + &info, + &post_info, + LEN, + &Ok(()) + )); + + assert_eq!(BlockWeight::::get().total().proof_size(), 600); + }) + } + + #[test] + fn does_nothing_without_extension() { + let mut test_ext = new_test_ext(); + + // Proof size extension not registered + test_ext.execute_with(|| { + set_current_storage_weight(1000); + + // Benchmarked storage weight: 500 + let info = DispatchInfo { weight: Weight::from_parts(0, 500), ..Default::default() }; + let post_info = PostDispatchInfo::default(); + + let pre = StorageWeightReclaim::(PhantomData) + .pre_dispatch(&ALICE, CALL, &info, LEN) + .unwrap(); + assert_eq!(pre, None); + + assert_ok!(CheckWeight::::post_dispatch(None, &info, &post_info, 0, &Ok(()))); + assert_ok!(StorageWeightReclaim::::post_dispatch( + Some(pre), + &info, + &post_info, + LEN, + &Ok(()) + )); + + assert_eq!(BlockWeight::::get().total().proof_size(), 1000); + }) + } + + #[test] + fn negative_refund_is_added_to_weight() { + let mut test_ext = setup_test_externalities(&[100, 300]); + + test_ext.execute_with(|| { + set_current_storage_weight(1000); + // Benchmarked storage weight: 100 + let info = DispatchInfo { weight: Weight::from_parts(0, 100), ..Default::default() }; + let post_info = PostDispatchInfo::default(); + + let pre = StorageWeightReclaim::(PhantomData) + .pre_dispatch(&ALICE, CALL, &info, LEN) + .unwrap(); + assert_eq!(pre, Some(100)); + + // We expect no refund + assert_ok!(CheckWeight::::post_dispatch(None, &info, &post_info, 0, &Ok(()))); + assert_ok!(StorageWeightReclaim::::post_dispatch( + Some(pre), + &info, + &post_info, + LEN, + &Ok(()) + )); + + assert_eq!(BlockWeight::::get().total().proof_size(), 1100); + }) + } + + #[test] + fn test_zero_proof_size() { + let mut test_ext = setup_test_externalities(&[0, 0]); + + test_ext.execute_with(|| { + let info = DispatchInfo { weight: Weight::from_parts(0, 500), ..Default::default() }; + let post_info = PostDispatchInfo::default(); + + let pre = StorageWeightReclaim::(PhantomData) + .pre_dispatch(&ALICE, CALL, &info, LEN) + .unwrap(); + assert_eq!(pre, Some(0)); + + assert_ok!(CheckWeight::::post_dispatch(None, &info, &post_info, 0, &Ok(()))); + assert_ok!(StorageWeightReclaim::::post_dispatch( + Some(pre), + &info, + &post_info, + LEN, + &Ok(()) + )); + + assert_eq!(BlockWeight::::get().total().proof_size(), 0); + }); + } + + #[test] + fn test_larger_pre_dispatch_proof_size() { + let mut test_ext = setup_test_externalities(&[300, 100]); + + test_ext.execute_with(|| { + set_current_storage_weight(1300); + + let info = DispatchInfo { weight: Weight::from_parts(0, 500), ..Default::default() }; + let post_info = PostDispatchInfo::default(); + + let pre = StorageWeightReclaim::(PhantomData) + .pre_dispatch(&ALICE, CALL, &info, LEN) + .unwrap(); + assert_eq!(pre, Some(300)); + + assert_ok!(CheckWeight::::post_dispatch(None, &info, &post_info, 0, &Ok(()))); + assert_ok!(StorageWeightReclaim::::post_dispatch( + Some(pre), + &info, + &post_info, + LEN, + &Ok(()) + )); + + assert_eq!(BlockWeight::::get().total().proof_size(), 800); + }); + } + + #[test] + fn test_incorporates_check_weight_unspent_weight() { + let mut test_ext = setup_test_externalities(&[100, 300]); + + test_ext.execute_with(|| { + set_current_storage_weight(1000); + + // Benchmarked storage weight: 300 + let info = DispatchInfo { weight: Weight::from_parts(100, 300), ..Default::default() }; + + // Actual weight is 50 + let post_info = PostDispatchInfo { + actual_weight: Some(Weight::from_parts(50, 250)), + pays_fee: Default::default(), + }; + + let pre = StorageWeightReclaim::(PhantomData) + .pre_dispatch(&ALICE, CALL, &info, LEN) + .unwrap(); + assert_eq!(pre, Some(100)); + + // The `CheckWeight` extension will refunt `actual_weight` from `PostDispatchInfo` + // we always need to call `post_dispatch` to verify that they interoperate correctly. + assert_ok!(CheckWeight::::post_dispatch(None, &info, &post_info, 0, &Ok(()))); + assert_ok!(StorageWeightReclaim::::post_dispatch( + Some(pre), + &info, + &post_info, + LEN, + &Ok(()) + )); + + assert_eq!(BlockWeight::::get().total().proof_size(), 900); + }) + } + + #[test] + fn test_incorporates_check_weight_unspent_weight_on_negative() { + let mut test_ext = setup_test_externalities(&[100, 300]); + + test_ext.execute_with(|| { + set_current_storage_weight(1000); + // Benchmarked storage weight: 50 + let info = DispatchInfo { weight: Weight::from_parts(100, 50), ..Default::default() }; + + // Actual weight is 25 + let post_info = PostDispatchInfo { + actual_weight: Some(Weight::from_parts(50, 25)), + pays_fee: Default::default(), + }; + + let pre = StorageWeightReclaim::(PhantomData) + .pre_dispatch(&ALICE, CALL, &info, LEN) + .unwrap(); + assert_eq!(pre, Some(100)); + + // The `CheckWeight` extension will refunt `actual_weight` from `PostDispatchInfo` + // we always need to call `post_dispatch` to verify that they interoperate correctly. + assert_ok!(CheckWeight::::post_dispatch(None, &info, &post_info, 0, &Ok(()))); + assert_ok!(StorageWeightReclaim::::post_dispatch( + Some(pre), + &info, + &post_info, + LEN, + &Ok(()) + )); + + assert_eq!(BlockWeight::::get().total().proof_size(), 1150); + }) + } + + #[test] + fn test_incorporates_check_weight_unspent_weight_reverse_order() { + let mut test_ext = setup_test_externalities(&[100, 300]); + + test_ext.execute_with(|| { + set_current_storage_weight(1000); + + // Benchmarked storage weight: 300 + let info = DispatchInfo { weight: Weight::from_parts(100, 300), ..Default::default() }; + + // Actual weight is 50 + let post_info = PostDispatchInfo { + actual_weight: Some(Weight::from_parts(50, 250)), + pays_fee: Default::default(), + }; + + let pre = StorageWeightReclaim::(PhantomData) + .pre_dispatch(&ALICE, CALL, &info, LEN) + .unwrap(); + assert_eq!(pre, Some(100)); + + assert_ok!(StorageWeightReclaim::::post_dispatch( + Some(pre), + &info, + &post_info, + LEN, + &Ok(()) + )); + // `CheckWeight` gets called after `StorageWeightReclaim` this time. + // The `CheckWeight` extension will refunt `actual_weight` from `PostDispatchInfo` + // we always need to call `post_dispatch` to verify that they interoperate correctly. + assert_ok!(CheckWeight::::post_dispatch(None, &info, &post_info, 0, &Ok(()))); + + assert_eq!(BlockWeight::::get().total().proof_size(), 900); + }) + } + + #[test] + fn test_incorporates_check_weight_unspent_weight_on_negative_reverse_order() { + let mut test_ext = setup_test_externalities(&[100, 300]); + + test_ext.execute_with(|| { + set_current_storage_weight(1000); + // Benchmarked storage weight: 50 + let info = DispatchInfo { weight: Weight::from_parts(100, 50), ..Default::default() }; + + // Actual weight is 25 + let post_info = PostDispatchInfo { + actual_weight: Some(Weight::from_parts(50, 25)), + pays_fee: Default::default(), + }; + + let pre = StorageWeightReclaim::(PhantomData) + .pre_dispatch(&ALICE, CALL, &info, LEN) + .unwrap(); + assert_eq!(pre, Some(100)); + + assert_ok!(StorageWeightReclaim::::post_dispatch( + Some(pre), + &info, + &post_info, + LEN, + &Ok(()) + )); + // `CheckWeight` gets called after `StorageWeightReclaim` this time. + // The `CheckWeight` extension will refunt `actual_weight` from `PostDispatchInfo` + // we always need to call `post_dispatch` to verify that they interoperate correctly. + assert_ok!(CheckWeight::::post_dispatch(None, &info, &post_info, 0, &Ok(()))); + + assert_eq!(BlockWeight::::get().total().proof_size(), 1150); + }) + } + + #[test] + fn storage_size_reported_correctly() { + let mut test_ext = setup_test_externalities(&[1000]); + test_ext.execute_with(|| { + assert_eq!(get_proof_size(), Some(1000)); + }); + + let mut test_ext = new_test_ext(); + + let test_recorder = TestRecorder::new(&[0]); + + test_ext.register_extension(ProofSizeExt::new(test_recorder)); + + test_ext.execute_with(|| { + assert_eq!(get_proof_size(), Some(0)); + }); + } + + #[test] + fn storage_size_disabled_reported_correctly() { + let mut test_ext = setup_test_externalities(&[PROOF_RECORDING_DISABLED as usize]); + + test_ext.execute_with(|| { + assert_eq!(get_proof_size(), None); + }); + } + + #[test] + fn test_reclaim_helper() { + let mut test_ext = setup_test_externalities(&[1000, 1300, 1800]); + + test_ext.execute_with(|| { + let mut remaining_weight_meter = WeightMeter::with_limit(Weight::from_parts(0, 2000)); + let mut reclaim_helper = StorageWeightReclaimer::new(&remaining_weight_meter); + remaining_weight_meter.consume(Weight::from_parts(0, 500)); + let reclaimed = reclaim_helper.reclaim_with_meter(&mut remaining_weight_meter); + + assert_eq!(reclaimed, Some(Weight::from_parts(0, 200))); + + remaining_weight_meter.consume(Weight::from_parts(0, 800)); + let reclaimed = reclaim_helper.reclaim_with_meter(&mut remaining_weight_meter); + assert_eq!(reclaimed, Some(Weight::from_parts(0, 300))); + assert_eq!(remaining_weight_meter.remaining(), Weight::from_parts(0, 1200)); + }); + } + + #[test] + fn test_reclaim_helper_does_not_reclaim_negative() { + // Benchmarked weight does not change at all + let mut test_ext = setup_test_externalities(&[1000, 1300]); + + test_ext.execute_with(|| { + let mut remaining_weight_meter = WeightMeter::with_limit(Weight::from_parts(0, 1000)); + let mut reclaim_helper = StorageWeightReclaimer::new(&remaining_weight_meter); + let reclaimed = reclaim_helper.reclaim_with_meter(&mut remaining_weight_meter); + + assert_eq!(reclaimed, Some(Weight::from_parts(0, 0))); + assert_eq!(remaining_weight_meter.remaining(), Weight::from_parts(0, 1000)); + }); + + // Benchmarked weight increases less than storage proof consumes + let mut test_ext = setup_test_externalities(&[1000, 1300]); + + test_ext.execute_with(|| { + let mut remaining_weight_meter = WeightMeter::with_limit(Weight::from_parts(0, 1000)); + let mut reclaim_helper = StorageWeightReclaimer::new(&remaining_weight_meter); + remaining_weight_meter.consume(Weight::from_parts(0, 0)); + let reclaimed = reclaim_helper.reclaim_with_meter(&mut remaining_weight_meter); + + assert_eq!(reclaimed, Some(Weight::from_parts(0, 0))); + }); + } + + /// Just here for doc purposes + fn get_benched_weight() -> Weight { + Weight::from_parts(0, 5) + } + + /// Just here for doc purposes + fn do_work() {} + + #[docify::export_content(simple_reclaimer_example)] + fn reclaim_with_weight_meter() { + let mut remaining_weight_meter = WeightMeter::with_limit(Weight::from_parts(10, 10)); + + let benched_weight = get_benched_weight(); + + // It is important to instantiate the `StorageWeightReclaimer` before we consume the weight + // for a piece of work from the weight meter. + let mut reclaim_helper = StorageWeightReclaimer::new(&remaining_weight_meter); + + if remaining_weight_meter.try_consume(benched_weight).is_ok() { + // Perform some work that takes has `benched_weight` storage weight. + do_work(); + + // Reclaimer will detect that we only consumed 2 bytes, so 3 bytes are reclaimed. + let reclaimed = reclaim_helper.reclaim_with_meter(&mut remaining_weight_meter); + + // We reclaimed 3 bytes of storage size! + assert_eq!(reclaimed, Some(Weight::from_parts(0, 3))); + assert_eq!(BlockWeight::::get().total().proof_size(), 10); + assert_eq!(remaining_weight_meter.remaining(), Weight::from_parts(10, 8)); + } + } + + #[test] + fn test_reclaim_helper_works_with_meter() { + // The node will report 12 - 10 = 2 consumed storage size between the calls. + let mut test_ext = setup_test_externalities(&[10, 12]); + + test_ext.execute_with(|| { + // Initial storage size is 10. + set_current_storage_weight(10); + reclaim_with_weight_meter(); + }); + } +} diff --git a/cumulus/test/client/Cargo.toml b/cumulus/test/client/Cargo.toml index 7190172101cb..028733ce2355 100644 --- a/cumulus/test/client/Cargo.toml +++ b/cumulus/test/client/Cargo.toml @@ -41,6 +41,7 @@ cumulus-test-relay-sproof-builder = { path = "../relay-sproof-builder" } cumulus-primitives-core = { path = "../../primitives/core" } cumulus-primitives-proof-size-hostfunction = { path = "../../primitives/proof-size-hostfunction" } cumulus-primitives-parachain-inherent = { path = "../../primitives/parachain-inherent" } +cumulus-primitives-storage-weight-reclaim = { path = "../../primitives/storage-weight-reclaim" } [features] runtime-benchmarks = [ diff --git a/cumulus/test/client/src/lib.rs b/cumulus/test/client/src/lib.rs index df63f683de6b..c46f4da7f678 100644 --- a/cumulus/test/client/src/lib.rs +++ b/cumulus/test/client/src/lib.rs @@ -151,6 +151,7 @@ pub fn generate_extrinsic_with_pair( frame_system::CheckNonce::::from(nonce), frame_system::CheckWeight::::new(), pallet_transaction_payment::ChargeTransactionPayment::::from(tip), + cumulus_primitives_storage_weight_reclaim::StorageWeightReclaim::::new(), ); let function = function.into(); @@ -158,7 +159,7 @@ pub fn generate_extrinsic_with_pair( let raw_payload = SignedPayload::from_raw( function.clone(), extra.clone(), - ((), VERSION.spec_version, genesis_block, current_block_hash, (), (), ()), + ((), VERSION.spec_version, genesis_block, current_block_hash, (), (), (), ()), ); let signature = raw_payload.using_encoded(|e| origin.sign(e)); @@ -203,13 +204,16 @@ pub fn validate_block( let mut ext_ext = ext.ext(); let heap_pages = HeapAllocStrategy::Static { extra_pages: 1024 }; - let executor = WasmExecutor::::builder() - .with_execution_method(WasmExecutionMethod::default()) - .with_max_runtime_instances(1) - .with_runtime_cache_size(2) - .with_onchain_heap_alloc_strategy(heap_pages) - .with_offchain_heap_alloc_strategy(heap_pages) - .build(); + let executor = WasmExecutor::<( + sp_io::SubstrateHostFunctions, + cumulus_primitives_proof_size_hostfunction::storage_proof_size::HostFunctions, + )>::builder() + .with_execution_method(WasmExecutionMethod::default()) + .with_max_runtime_instances(1) + .with_runtime_cache_size(2) + .with_onchain_heap_alloc_strategy(heap_pages) + .with_offchain_heap_alloc_strategy(heap_pages) + .build(); executor .uncached_call( diff --git a/cumulus/test/runtime/Cargo.toml b/cumulus/test/runtime/Cargo.toml index 5902a62512be..449a8b819bc0 100644 --- a/cumulus/test/runtime/Cargo.toml +++ b/cumulus/test/runtime/Cargo.toml @@ -39,6 +39,7 @@ sp-version = { path = "../../../substrate/primitives/version", default-features # Cumulus cumulus-pallet-parachain-system = { path = "../../pallets/parachain-system", default-features = false, features = ["parameterized-consensus-hook"] } cumulus-primitives-core = { path = "../../primitives/core", default-features = false } +cumulus-primitives-storage-weight-reclaim = { path = "../../primitives/storage-weight-reclaim", default-features = false } [build-dependencies] substrate-wasm-builder = { path = "../../../substrate/utils/wasm-builder", optional = true } @@ -49,6 +50,7 @@ std = [ "codec/std", "cumulus-pallet-parachain-system/std", "cumulus-primitives-core/std", + "cumulus-primitives-storage-weight-reclaim/std", "frame-executive/std", "frame-support/std", "frame-system-rpc-runtime-api/std", diff --git a/cumulus/test/runtime/src/lib.rs b/cumulus/test/runtime/src/lib.rs index 6068f895c83b..5fb314109844 100644 --- a/cumulus/test/runtime/src/lib.rs +++ b/cumulus/test/runtime/src/lib.rs @@ -331,6 +331,7 @@ pub type SignedExtra = ( frame_system::CheckNonce, frame_system::CheckWeight, pallet_transaction_payment::ChargeTransactionPayment, + cumulus_primitives_storage_weight_reclaim::StorageWeightReclaim, ); /// Unchecked extrinsic type as expected by this runtime. pub type UncheckedExtrinsic = diff --git a/cumulus/test/service/Cargo.toml b/cumulus/test/service/Cargo.toml index b26f0b9967cf..27273f4e0a8d 100644 --- a/cumulus/test/service/Cargo.toml +++ b/cumulus/test/service/Cargo.toml @@ -81,6 +81,7 @@ cumulus-relay-chain-minimal-node = { path = "../../client/relay-chain-minimal-no cumulus-client-pov-recovery = { path = "../../client/pov-recovery" } cumulus-test-relay-sproof-builder = { path = "../relay-sproof-builder" } cumulus-pallet-parachain-system = { path = "../../pallets/parachain-system", default-features = false, features = ["parameterized-consensus-hook"] } +cumulus-primitives-storage-weight-reclaim = { path = "../../primitives/storage-weight-reclaim" } pallet-timestamp = { path = "../../../substrate/frame/timestamp" } [dev-dependencies] diff --git a/cumulus/test/service/src/lib.rs b/cumulus/test/service/src/lib.rs index 1c2e1db97414..3554a383f219 100644 --- a/cumulus/test/service/src/lib.rs +++ b/cumulus/test/service/src/lib.rs @@ -112,7 +112,7 @@ pub type AnnounceBlockFn = Arc>) + Send + Sync>; pub struct RuntimeExecutor; impl sc_executor::NativeExecutionDispatch for RuntimeExecutor { - type ExtendHostFunctions = (); + type ExtendHostFunctions = cumulus_client_service::storage_proof_size::HostFunctions; fn dispatch(method: &str, data: &[u8]) -> Option> { cumulus_test_runtime::api::dispatch(method, data) @@ -894,11 +894,12 @@ pub fn construct_extrinsic( frame_system::CheckNonce::::from(nonce), frame_system::CheckWeight::::new(), pallet_transaction_payment::ChargeTransactionPayment::::from(tip), + cumulus_primitives_storage_weight_reclaim::StorageWeightReclaim::::new(), ); let raw_payload = runtime::SignedPayload::from_raw( function.clone(), extra.clone(), - ((), runtime::VERSION.spec_version, genesis_block, current_block_hash, (), (), ()), + ((), runtime::VERSION.spec_version, genesis_block, current_block_hash, (), (), (), ()), ); let signature = raw_payload.using_encoded(|e| caller.sign(e)); runtime::UncheckedExtrinsic::new_signed( diff --git a/docker/dockerfiles/bridges_zombienet_tests_injected.Dockerfile b/docker/dockerfiles/bridges_zombienet_tests_injected.Dockerfile index fde9cc6e7cf3..4bfb73acda05 100644 --- a/docker/dockerfiles/bridges_zombienet_tests_injected.Dockerfile +++ b/docker/dockerfiles/bridges_zombienet_tests_injected.Dockerfile @@ -45,7 +45,7 @@ RUN mkdir -p /home/nonroot/bridges-polkadot-sdk COPY ./artifacts/bridges-polkadot-sdk /home/nonroot/bridges-polkadot-sdk # also prepare `generate_hex_encoded_call` for running RUN set -eux; \ - cd /home/nonroot/bridges-polkadot-sdk/bridges/testing/utils/generate_hex_encoded_call; \ + cd /home/nonroot/bridges-polkadot-sdk/bridges/testing/framework/utils/generate_hex_encoded_call; \ npm install # check if executable works in this container diff --git a/polkadot/node/core/backing/Cargo.toml b/polkadot/node/core/backing/Cargo.toml index b0cf041e38da..d0c1f9aa4832 100644 --- a/polkadot/node/core/backing/Cargo.toml +++ b/polkadot/node/core/backing/Cargo.toml @@ -22,6 +22,7 @@ bitvec = { version = "1.0.0", default-features = false, features = ["alloc"] } gum = { package = "tracing-gum", path = "../../gum" } thiserror = { workspace = true } fatality = "0.0.6" +schnellru = "0.2.1" [dev-dependencies] sp-core = { path = "../../../../substrate/primitives/core" } @@ -31,5 +32,6 @@ sc-keystore = { path = "../../../../substrate/client/keystore" } sp-tracing = { path = "../../../../substrate/primitives/tracing" } futures = { version = "0.3.21", features = ["thread-pool"] } assert_matches = "1.4.0" +rstest = "0.18.2" polkadot-node-subsystem-test-helpers = { path = "../../subsystem-test-helpers" } test-helpers = { package = "polkadot-primitives-test-helpers", path = "../../../primitives/test-helpers" } diff --git a/polkadot/node/core/backing/src/error.rs b/polkadot/node/core/backing/src/error.rs index 1b00a62510b7..64955a393962 100644 --- a/polkadot/node/core/backing/src/error.rs +++ b/polkadot/node/core/backing/src/error.rs @@ -48,6 +48,9 @@ pub enum Error { #[error("Candidate is not found")] CandidateNotFound, + #[error("CoreIndex cannot be determined for a candidate")] + CoreIndexUnavailable, + #[error("Signature is invalid")] InvalidSignature, diff --git a/polkadot/node/core/backing/src/lib.rs b/polkadot/node/core/backing/src/lib.rs index 98bbd6232add..69bf2e956a0f 100644 --- a/polkadot/node/core/backing/src/lib.rs +++ b/polkadot/node/core/backing/src/lib.rs @@ -77,6 +77,7 @@ use futures::{ stream::FuturesOrdered, FutureExt, SinkExt, StreamExt, TryFutureExt, }; +use schnellru::{ByLength, LruMap}; use error::{Error, FatalResult}; use polkadot_node_primitives::{ @@ -104,10 +105,12 @@ use polkadot_node_subsystem_util::{ Validator, }; use polkadot_primitives::{ + vstaging::{node_features::FeatureIndex, NodeFeatures}, BackedCandidate, CandidateCommitments, CandidateHash, CandidateReceipt, - CommittedCandidateReceipt, CoreIndex, CoreState, ExecutorParams, Hash, Id as ParaId, - PersistedValidationData, PvfExecKind, SigningContext, ValidationCode, ValidatorId, - ValidatorIndex, ValidatorSignature, ValidityAttestation, + CommittedCandidateReceipt, CoreIndex, CoreState, ExecutorParams, GroupIndex, GroupRotationInfo, + Hash, Id as ParaId, IndexedVec, PersistedValidationData, PvfExecKind, SessionIndex, + SigningContext, ValidationCode, ValidatorId, ValidatorIndex, ValidatorSignature, + ValidityAttestation, }; use sp_keystore::KeystorePtr; use statement_table::{ @@ -118,7 +121,7 @@ use statement_table::{ }, Config as TableConfig, Context as TableContextTrait, Table, }; -use util::vstaging::get_disabled_validators_with_fallback; +use util::{runtime::request_node_features, vstaging::get_disabled_validators_with_fallback}; mod error; @@ -209,7 +212,9 @@ struct PerRelayParentState { /// The hash of the relay parent on top of which this job is doing it's work. parent: Hash, /// The `ParaId` assigned to the local validator at this relay parent. - assignment: Option, + assigned_para: Option, + /// The `CoreIndex` assigned to the local validator at this relay parent. + assigned_core: Option, /// The candidates that are backed by enough validators in their group, by hash. backed: HashSet, /// The table of candidates and statements under this relay-parent. @@ -224,6 +229,15 @@ struct PerRelayParentState { fallbacks: HashMap, /// The minimum backing votes threshold. minimum_backing_votes: u32, + /// If true, we're appending extra bits in the BackedCandidate validator indices bitfield, + /// which represent the assigned core index. True if ElasticScalingMVP is enabled. + inject_core_index: bool, + /// The core states for all cores. + cores: Vec, + /// The validator index -> group mapping at this relay parent. + validator_to_group: Arc>>, + /// The associated group rotation information. + group_rotation_info: GroupRotationInfo, } struct PerCandidateState { @@ -275,6 +289,9 @@ struct State { /// This is guaranteed to have an entry for each candidate with a relay parent in the implicit /// or explicit view for which a `Seconded` statement has been successfully imported. per_candidate: HashMap, + /// Cache the per-session Validator->Group mapping. + validator_to_group_cache: + LruMap>>>, /// A cloneable sender which is dispatched to background candidate validation tasks to inform /// the main task of the result. background_validation_tx: mpsc::Sender<(Hash, ValidatedCandidateCommand)>, @@ -292,6 +309,7 @@ impl State { per_leaf: HashMap::default(), per_relay_parent: HashMap::default(), per_candidate: HashMap::new(), + validator_to_group_cache: LruMap::new(ByLength::new(2)), background_validation_tx, keystore, } @@ -379,10 +397,10 @@ struct AttestingData { backing: Vec, } -#[derive(Default)] +#[derive(Default, Debug)] struct TableContext { validator: Option, - groups: HashMap>, + groups: HashMap>, validators: Vec, disabled_validators: Vec, } @@ -404,7 +422,7 @@ impl TableContext { impl TableContextTrait for TableContext { type AuthorityId = ValidatorIndex; type Digest = CandidateHash; - type GroupId = ParaId; + type GroupId = CoreIndex; type Signature = ValidatorSignature; type Candidate = CommittedCandidateReceipt; @@ -412,15 +430,11 @@ impl TableContextTrait for TableContext { candidate.hash() } - fn candidate_group(candidate: &CommittedCandidateReceipt) -> ParaId { - candidate.descriptor().para_id + fn is_member_of(&self, authority: &ValidatorIndex, core: &CoreIndex) -> bool { + self.groups.get(core).map_or(false, |g| g.iter().any(|a| a == authority)) } - fn is_member_of(&self, authority: &ValidatorIndex, group: &ParaId) -> bool { - self.groups.get(group).map_or(false, |g| g.iter().any(|a| a == authority)) - } - - fn get_group_size(&self, group: &ParaId) -> Option { + fn get_group_size(&self, group: &CoreIndex) -> Option { self.groups.get(group).map(|g| g.len()) } } @@ -442,19 +456,20 @@ fn primitive_statement_to_table(s: &SignedFullStatementWithPVD) -> TableSignedSt fn table_attested_to_backed( attested: TableAttestedCandidate< - ParaId, + CoreIndex, CommittedCandidateReceipt, ValidatorIndex, ValidatorSignature, >, table_context: &TableContext, + inject_core_index: bool, ) -> Option { - let TableAttestedCandidate { candidate, validity_votes, group_id: para_id } = attested; + let TableAttestedCandidate { candidate, validity_votes, group_id: core_index } = attested; let (ids, validity_votes): (Vec<_>, Vec) = validity_votes.into_iter().map(|(id, vote)| (id, vote.into())).unzip(); - let group = table_context.groups.get(¶_id)?; + let group = table_context.groups.get(&core_index)?; let mut validator_indices = BitVec::with_capacity(group.len()); @@ -479,14 +494,15 @@ fn table_attested_to_backed( } vote_positions.sort_by_key(|(_orig, pos_in_group)| *pos_in_group); - Some(BackedCandidate { + Some(BackedCandidate::new( candidate, - validity_votes: vote_positions + vote_positions .into_iter() .map(|(pos_in_votes, _pos_in_group)| validity_votes[pos_in_votes].clone()) .collect(), validator_indices, - }) + inject_core_index.then_some(core_index), + )) } async fn store_available_data( @@ -971,7 +987,14 @@ async fn handle_active_leaves_update( // construct a `PerRelayParent` from the runtime API // and insert it. - let per = construct_per_relay_parent_state(ctx, maybe_new, &state.keystore, mode).await?; + let per = construct_per_relay_parent_state( + ctx, + maybe_new, + &state.keystore, + &mut state.validator_to_group_cache, + mode, + ) + .await?; if let Some(per) = per { state.per_relay_parent.insert(maybe_new, per); @@ -981,31 +1004,112 @@ async fn handle_active_leaves_update( Ok(()) } +macro_rules! try_runtime_api { + ($x: expr) => { + match $x { + Ok(x) => x, + Err(err) => { + // Only bubble up fatal errors. + error::log_error(Err(Into::::into(err).into()))?; + + // We can't do candidate validation work if we don't have the + // requisite runtime API data. But these errors should not take + // down the node. + return Ok(None) + }, + } + }; +} + +fn core_index_from_statement( + validator_to_group: &IndexedVec>, + group_rotation_info: &GroupRotationInfo, + cores: &[CoreState], + statement: &SignedFullStatementWithPVD, +) -> Option { + let compact_statement = statement.as_unchecked(); + let candidate_hash = CandidateHash(*compact_statement.unchecked_payload().candidate_hash()); + + let n_cores = cores.len(); + + gum::trace!( + target:LOG_TARGET, + ?group_rotation_info, + ?statement, + ?validator_to_group, + n_cores = ?cores.len(), + ?candidate_hash, + "Extracting core index from statement" + ); + + let statement_validator_index = statement.validator_index(); + let Some(Some(group_index)) = validator_to_group.get(statement_validator_index) else { + gum::debug!( + target: LOG_TARGET, + ?group_rotation_info, + ?statement, + ?validator_to_group, + n_cores = ?cores.len() , + ?candidate_hash, + "Invalid validator index: {:?}", + statement_validator_index + ); + return None + }; + + // First check if the statement para id matches the core assignment. + let core_index = group_rotation_info.core_for_group(*group_index, n_cores); + + if core_index.0 as usize > n_cores { + gum::warn!(target: LOG_TARGET, ?candidate_hash, ?core_index, n_cores, "Invalid CoreIndex"); + return None + } + + if let StatementWithPVD::Seconded(candidate, _pvd) = statement.payload() { + let candidate_para_id = candidate.descriptor.para_id; + let assigned_para_id = match &cores[core_index.0 as usize] { + CoreState::Free => { + gum::debug!(target: LOG_TARGET, ?candidate_hash, "Invalid CoreIndex, core is not assigned to any para_id"); + return None + }, + CoreState::Occupied(occupied) => + if let Some(next) = &occupied.next_up_on_available { + next.para_id + } else { + return None + }, + CoreState::Scheduled(scheduled) => scheduled.para_id, + }; + + if assigned_para_id != candidate_para_id { + gum::debug!( + target: LOG_TARGET, + ?candidate_hash, + ?core_index, + ?assigned_para_id, + ?candidate_para_id, + "Invalid CoreIndex, core is assigned to a different para_id" + ); + return None + } + return Some(core_index) + } else { + return Some(core_index) + } +} + /// Load the data necessary to do backing work on top of a relay-parent. #[overseer::contextbounds(CandidateBacking, prefix = self::overseer)] async fn construct_per_relay_parent_state( ctx: &mut Context, relay_parent: Hash, keystore: &KeystorePtr, + validator_to_group_cache: &mut LruMap< + SessionIndex, + Arc>>, + >, mode: ProspectiveParachainsMode, ) -> Result, Error> { - macro_rules! try_runtime_api { - ($x: expr) => { - match $x { - Ok(x) => x, - Err(err) => { - // Only bubble up fatal errors. - error::log_error(Err(Into::::into(err).into()))?; - - // We can't do candidate validation work if we don't have the - // requisite runtime API data. But these errors should not take - // down the node. - return Ok(None) - }, - } - }; - } - let parent = relay_parent; let (session_index, validators, groups, cores) = futures::try_join!( @@ -1020,6 +1124,16 @@ async fn construct_per_relay_parent_state( .map_err(Error::JoinMultiple)?; let session_index = try_runtime_api!(session_index); + + let inject_core_index = request_node_features(parent, session_index, ctx.sender()) + .await? + .unwrap_or(NodeFeatures::EMPTY) + .get(FeatureIndex::ElasticScalingMVP as usize) + .map(|b| *b) + .unwrap_or(false); + + gum::debug!(target: LOG_TARGET, inject_core_index, ?parent, "New state"); + let validators: Vec<_> = try_runtime_api!(validators); let (validator_groups, group_rotation_info) = try_runtime_api!(groups); let cores = try_runtime_api!(cores); @@ -1055,18 +1169,24 @@ async fn construct_per_relay_parent_state( }, }; - let mut groups = HashMap::new(); let n_cores = cores.len(); - let mut assignment = None; - for (idx, core) in cores.into_iter().enumerate() { + let mut groups = HashMap::>::new(); + let mut assigned_core = None; + let mut assigned_para = None; + + for (idx, core) in cores.iter().enumerate() { let core_para_id = match core { CoreState::Scheduled(scheduled) => scheduled.para_id, CoreState::Occupied(occupied) => if mode.is_enabled() { // Async backing makes it legal to build on top of // occupied core. - occupied.candidate_descriptor.para_id + if let Some(next) = &occupied.next_up_on_available { + next.para_id + } else { + continue + } } else { continue }, @@ -1077,11 +1197,27 @@ async fn construct_per_relay_parent_state( let group_index = group_rotation_info.group_for_core(core_index, n_cores); if let Some(g) = validator_groups.get(group_index.0 as usize) { if validator.as_ref().map_or(false, |v| g.contains(&v.index())) { - assignment = Some(core_para_id); + assigned_para = Some(core_para_id); + assigned_core = Some(core_index); } - groups.insert(core_para_id, g.clone()); + groups.insert(core_index, g.clone()); } } + gum::debug!(target: LOG_TARGET, ?groups, "TableContext"); + + let validator_to_group = validator_to_group_cache + .get_or_insert(session_index, || { + let mut vector = vec![None; validators.len()]; + + for (group_idx, validator_group) in validator_groups.iter().enumerate() { + for validator in validator_group { + vector[validator.0 as usize] = Some(GroupIndex(group_idx as u32)); + } + } + + Arc::new(IndexedVec::<_, _>::from(vector)) + }) + .expect("Just inserted"); let table_context = TableContext { validator, groups, validators, disabled_validators }; let table_config = TableConfig { @@ -1094,7 +1230,8 @@ async fn construct_per_relay_parent_state( Ok(Some(PerRelayParentState { prospective_parachains_mode: mode, parent, - assignment, + assigned_core, + assigned_para, backed: HashSet::new(), table: Table::new(table_config), table_context, @@ -1102,6 +1239,10 @@ async fn construct_per_relay_parent_state( awaiting_validation: HashSet::new(), fallbacks: HashMap::new(), minimum_backing_votes, + inject_core_index, + cores, + validator_to_group: validator_to_group.clone(), + group_rotation_info, })) } @@ -1519,15 +1660,16 @@ async fn import_statement( per_candidate: &mut HashMap, statement: &SignedFullStatementWithPVD, ) -> Result, Error> { + let candidate_hash = statement.payload().candidate_hash(); + gum::debug!( target: LOG_TARGET, statement = ?statement.payload().to_compact(), validator_index = statement.validator_index().0, + ?candidate_hash, "Importing statement", ); - let candidate_hash = statement.payload().candidate_hash(); - // If this is a new candidate (statement is 'seconded' and candidate is unknown), // we need to create an entry in the `PerCandidateState` map. // @@ -1593,7 +1735,15 @@ async fn import_statement( let stmt = primitive_statement_to_table(statement); - Ok(rp_state.table.import_statement(&rp_state.table_context, stmt)) + let core = core_index_from_statement( + &rp_state.validator_to_group, + &rp_state.group_rotation_info, + &rp_state.cores, + statement, + ) + .ok_or(Error::CoreIndexUnavailable)?; + + Ok(rp_state.table.import_statement(&rp_state.table_context, core, stmt)) } /// Handles a summary received from [`import_statement`] and dispatches `Backed` notifications and @@ -1615,8 +1765,12 @@ async fn post_import_statement_actions( // `HashSet::insert` returns true if the thing wasn't in there already. if rp_state.backed.insert(candidate_hash) { - if let Some(backed) = table_attested_to_backed(attested, &rp_state.table_context) { - let para_id = backed.candidate.descriptor.para_id; + if let Some(backed) = table_attested_to_backed( + attested, + &rp_state.table_context, + rp_state.inject_core_index, + ) { + let para_id = backed.candidate().descriptor.para_id; gum::debug!( target: LOG_TARGET, candidate_hash = ?candidate_hash, @@ -1637,7 +1791,7 @@ async fn post_import_statement_actions( // notify collator protocol. ctx.send_message(CollatorProtocolMessage::Backed { para_id, - para_head: backed.candidate.descriptor.para_head, + para_head: backed.candidate().descriptor.para_head, }) .await; // Notify statement distribution of backed candidate. @@ -1654,8 +1808,14 @@ async fn post_import_statement_actions( ); ctx.send_unbounded_message(message); } + } else { + gum::debug!(target: LOG_TARGET, ?candidate_hash, "Cannot get BackedCandidate"); } + } else { + gum::debug!(target: LOG_TARGET, ?candidate_hash, "Candidate already known"); } + } else { + gum::debug!(target: LOG_TARGET, "No attested candidate"); } issue_new_misbehaviors(ctx, rp_state.parent, &mut rp_state.table); @@ -1722,18 +1882,20 @@ async fn background_validate_and_make_available( if rp_state.awaiting_validation.insert(candidate_hash) { // spawn background task. let bg = async move { - if let Err(e) = validate_and_make_available(params).await { - if let Error::BackgroundValidationMpsc(error) = e { + if let Err(error) = validate_and_make_available(params).await { + if let Error::BackgroundValidationMpsc(error) = error { gum::debug!( target: LOG_TARGET, + ?candidate_hash, ?error, "Mpsc background validation mpsc died during validation- leaf no longer active?" ); } else { gum::error!( target: LOG_TARGET, - "Failed to validate and make available: {:?}", - e + ?candidate_hash, + ?error, + "Failed to validate and make available", ); } } @@ -1859,9 +2021,10 @@ async fn maybe_validate_and_import( let candidate_hash = summary.candidate; - if Some(summary.group_id) != rp_state.assignment { + if Some(summary.group_id) != rp_state.assigned_core { return Ok(()) } + let attesting = match statement.payload() { StatementWithPVD::Seconded(receipt, _) => { let attesting = AttestingData { @@ -2004,10 +2167,11 @@ async fn handle_second_message( } // Sanity check that candidate is from our assignment. - if Some(candidate.descriptor().para_id) != rp_state.assignment { + if Some(candidate.descriptor().para_id) != rp_state.assigned_para { gum::debug!( target: LOG_TARGET, - our_assignment = ?rp_state.assignment, + our_assignment_core = ?rp_state.assigned_core, + our_assignment_para = ?rp_state.assigned_para, collation = ?candidate.descriptor().para_id, "Subsystem asked to second for para outside of our assignment", ); @@ -2015,6 +2179,14 @@ async fn handle_second_message( return Ok(()) } + gum::debug!( + target: LOG_TARGET, + our_assignment_core = ?rp_state.assigned_core, + our_assignment_para = ?rp_state.assigned_para, + collation = ?candidate.descriptor().para_id, + "Current assignments vs collation", + ); + // If the message is a `CandidateBackingMessage::Second`, sign and dispatch a // Seconded statement only if we have not signed a Valid statement for the requested candidate. // @@ -2087,7 +2259,13 @@ fn handle_get_backed_candidates_message( &rp_state.table_context, rp_state.minimum_backing_votes, ) - .and_then(|attested| table_attested_to_backed(attested, &rp_state.table_context)) + .and_then(|attested| { + table_attested_to_backed( + attested, + &rp_state.table_context, + rp_state.inject_core_index, + ) + }) }) .collect(); diff --git a/polkadot/node/core/backing/src/tests/mod.rs b/polkadot/node/core/backing/src/tests/mod.rs index 1957f4e19c54..e3cc5727435a 100644 --- a/polkadot/node/core/backing/src/tests/mod.rs +++ b/polkadot/node/core/backing/src/tests/mod.rs @@ -33,9 +33,10 @@ use polkadot_node_subsystem::{ }; use polkadot_node_subsystem_test_helpers as test_helpers; use polkadot_primitives::{ - CandidateDescriptor, GroupRotationInfo, HeadData, PersistedValidationData, PvfExecKind, - ScheduledCore, SessionIndex, LEGACY_MIN_BACKING_VOTES, + vstaging::node_features, CandidateDescriptor, GroupRotationInfo, HeadData, + PersistedValidationData, PvfExecKind, ScheduledCore, SessionIndex, LEGACY_MIN_BACKING_VOTES, }; +use rstest::rstest; use sp_application_crypto::AppCrypto; use sp_keyring::Sr25519Keyring; use sp_keystore::Keystore; @@ -65,19 +66,21 @@ fn dummy_pvd() -> PersistedValidationData { } } -struct TestState { +pub(crate) struct TestState { chain_ids: Vec, keystore: KeystorePtr, validators: Vec, validator_public: Vec, validation_data: PersistedValidationData, validator_groups: (Vec>, GroupRotationInfo), + validator_to_group: IndexedVec>, availability_cores: Vec, head_data: HashMap, signing_context: SigningContext, relay_parent: Hash, minimum_backing_votes: u32, disabled_validators: Vec, + node_features: NodeFeatures, } impl TestState { @@ -114,6 +117,11 @@ impl Default for TestState { .into_iter() .map(|g| g.into_iter().map(ValidatorIndex).collect()) .collect(); + let validator_to_group: IndexedVec<_, _> = + vec![Some(0), Some(1), Some(0), Some(0), None, Some(0)] + .into_iter() + .map(|x| x.map(|x| GroupIndex(x))) + .collect(); let group_rotation_info = GroupRotationInfo { session_start_block: 0, group_rotation_frequency: 100, now: 1 }; @@ -143,6 +151,7 @@ impl Default for TestState { validators, validator_public, validator_groups: (validator_groups, group_rotation_info), + validator_to_group, availability_cores, head_data, validation_data, @@ -150,6 +159,7 @@ impl Default for TestState { relay_parent, minimum_backing_votes: LEGACY_MIN_BACKING_VOTES, disabled_validators: Vec::new(), + node_features: Default::default(), } } } @@ -285,6 +295,16 @@ async fn test_startup(virtual_overseer: &mut VirtualOverseer, test_state: &TestS } ); + // Node features request from runtime: all features are disabled. + assert_matches!( + virtual_overseer.recv().await, + AllMessages::RuntimeApi( + RuntimeApiMessage::Request(_parent, RuntimeApiRequest::NodeFeatures(_session_index, tx)) + ) => { + tx.send(Ok(test_state.node_features.clone())).unwrap(); + } + ); + // Check if subsystem job issues a request for the minimum backing votes. assert_matches!( virtual_overseer.recv().await, @@ -477,9 +497,20 @@ fn backing_second_works() { } // Test that the candidate reaches quorum successfully. -#[test] -fn backing_works() { - let test_state = TestState::default(); +#[rstest] +#[case(true)] +#[case(false)] +fn backing_works(#[case] elastic_scaling_mvp: bool) { + let mut test_state = TestState::default(); + if elastic_scaling_mvp { + test_state + .node_features + .resize((node_features::FeatureIndex::ElasticScalingMVP as u8 + 1) as usize, false); + test_state + .node_features + .set(node_features::FeatureIndex::ElasticScalingMVP as u8 as usize, true); + } + test_harness(test_state.keystore.clone(), |mut virtual_overseer| async move { test_startup(&mut virtual_overseer, &test_state).await; @@ -630,6 +661,31 @@ fn backing_works() { virtual_overseer.send(FromOrchestra::Communication { msg: statement }).await; + let (tx, rx) = oneshot::channel(); + let msg = CandidateBackingMessage::GetBackedCandidates( + vec![(candidate_a_hash, test_state.relay_parent)], + tx, + ); + + virtual_overseer.send(FromOrchestra::Communication { msg }).await; + + let candidates = rx.await.unwrap(); + assert_eq!(1, candidates.len()); + assert_eq!(candidates[0].validity_votes().len(), 3); + + let (validator_indices, maybe_core_index) = + candidates[0].validator_indices_and_core_index(elastic_scaling_mvp); + if elastic_scaling_mvp { + assert_eq!(maybe_core_index.unwrap(), CoreIndex(0)); + } else { + assert!(maybe_core_index.is_none()); + } + + assert_eq!( + validator_indices, + bitvec::bitvec![u8, bitvec::order::Lsb0; 1, 1, 0, 1].as_bitslice() + ); + virtual_overseer .send(FromOrchestra::Signal(OverseerSignal::ActiveLeaves( ActiveLeavesUpdate::stop_work(test_state.relay_parent), @@ -639,6 +695,107 @@ fn backing_works() { }); } +#[test] +fn extract_core_index_from_statement_works() { + let test_state = TestState::default(); + + let pov_a = PoV { block_data: BlockData(vec![42, 43, 44]) }; + let pvd_a = dummy_pvd(); + let validation_code_a = ValidationCode(vec![1, 2, 3]); + + let pov_hash = pov_a.hash(); + + let mut candidate = TestCandidateBuilder { + para_id: test_state.chain_ids[0], + relay_parent: test_state.relay_parent, + pov_hash, + erasure_root: make_erasure_root(&test_state, pov_a.clone(), pvd_a.clone()), + persisted_validation_data_hash: pvd_a.hash(), + validation_code: validation_code_a.0.clone(), + ..Default::default() + } + .build(); + + let public2 = Keystore::sr25519_generate_new( + &*test_state.keystore, + ValidatorId::ID, + Some(&test_state.validators[2].to_seed()), + ) + .expect("Insert key into keystore"); + + let signed_statement_1 = SignedFullStatementWithPVD::sign( + &test_state.keystore, + StatementWithPVD::Seconded(candidate.clone(), pvd_a.clone()), + &test_state.signing_context, + ValidatorIndex(2), + &public2.into(), + ) + .ok() + .flatten() + .expect("should be signed"); + + let public1 = Keystore::sr25519_generate_new( + &*test_state.keystore, + ValidatorId::ID, + Some(&test_state.validators[1].to_seed()), + ) + .expect("Insert key into keystore"); + + let signed_statement_2 = SignedFullStatementWithPVD::sign( + &test_state.keystore, + StatementWithPVD::Seconded(candidate.clone(), pvd_a.clone()), + &test_state.signing_context, + ValidatorIndex(1), + &public1.into(), + ) + .ok() + .flatten() + .expect("should be signed"); + + candidate.descriptor.para_id = test_state.chain_ids[1]; + + let signed_statement_3 = SignedFullStatementWithPVD::sign( + &test_state.keystore, + StatementWithPVD::Seconded(candidate, pvd_a.clone()), + &test_state.signing_context, + ValidatorIndex(1), + &public1.into(), + ) + .ok() + .flatten() + .expect("should be signed"); + + let core_index_1 = core_index_from_statement( + &test_state.validator_to_group, + &test_state.validator_groups.1, + &test_state.availability_cores, + &signed_statement_1, + ) + .unwrap(); + + assert_eq!(core_index_1, CoreIndex(0)); + + let core_index_2 = core_index_from_statement( + &test_state.validator_to_group, + &test_state.validator_groups.1, + &test_state.availability_cores, + &signed_statement_2, + ); + + // Must be none, para_id in descriptor is different than para assigned to core + assert_eq!(core_index_2, None); + + let core_index_3 = core_index_from_statement( + &test_state.validator_to_group, + &test_state.validator_groups.1, + &test_state.availability_cores, + &signed_statement_3, + ) + .unwrap(); + + assert_eq!(core_index_3, CoreIndex(1)); +} + #[test] fn backing_works_while_validation_ongoing() { let test_state = TestState::default(); @@ -801,20 +958,20 @@ fn backing_works_while_validation_ongoing() { let candidates = rx.await.unwrap(); assert_eq!(1, candidates.len()); - assert_eq!(candidates[0].validity_votes.len(), 3); + assert_eq!(candidates[0].validity_votes().len(), 3); assert!(candidates[0] - .validity_votes + .validity_votes() .contains(&ValidityAttestation::Implicit(signed_a.signature().clone()))); assert!(candidates[0] - .validity_votes + .validity_votes() .contains(&ValidityAttestation::Explicit(signed_b.signature().clone()))); assert!(candidates[0] - .validity_votes + .validity_votes() .contains(&ValidityAttestation::Explicit(signed_c.signature().clone()))); assert_eq!( - candidates[0].validator_indices, - bitvec::bitvec![u8, bitvec::order::Lsb0; 1, 0, 1, 1], + candidates[0].validator_indices_and_core_index(false), + (bitvec::bitvec![u8, bitvec::order::Lsb0; 1, 0, 1, 1].as_bitslice(), None) ); virtual_overseer @@ -1422,7 +1579,7 @@ fn backing_works_after_failed_validation() { fn candidate_backing_reorders_votes() { use sp_core::Encode; - let para_id = ParaId::from(10); + let core_idx = CoreIndex(10); let validators = vec![ Sr25519Keyring::Alice, Sr25519Keyring::Bob, @@ -1436,7 +1593,7 @@ fn candidate_backing_reorders_votes() { let validator_groups = { let mut validator_groups = HashMap::new(); validator_groups - .insert(para_id, vec![0, 1, 2, 3, 4, 5].into_iter().map(ValidatorIndex).collect()); + .insert(core_idx, vec![0, 1, 2, 3, 4, 5].into_iter().map(ValidatorIndex).collect()); validator_groups }; @@ -1466,10 +1623,10 @@ fn candidate_backing_reorders_votes() { (ValidatorIndex(3), fake_attestation(3)), (ValidatorIndex(1), fake_attestation(1)), ], - group_id: para_id, + group_id: core_idx, }; - let backed = table_attested_to_backed(attested, &table_context).unwrap(); + let backed = table_attested_to_backed(attested, &table_context, false).unwrap(); let expected_bitvec = { let mut validator_indices = BitVec::::with_capacity(6); @@ -1486,8 +1643,11 @@ fn candidate_backing_reorders_votes() { let expected_attestations = vec![fake_attestation(1).into(), fake_attestation(3).into(), fake_attestation(5).into()]; - assert_eq!(backed.validator_indices, expected_bitvec); - assert_eq!(backed.validity_votes, expected_attestations); + assert_eq!( + backed.validator_indices_and_core_index(false), + (expected_bitvec.as_bitslice(), None) + ); + assert_eq!(backed.validity_votes(), expected_attestations); } // Test whether we retry on failed PoV fetching. diff --git a/polkadot/node/core/backing/src/tests/prospective_parachains.rs b/polkadot/node/core/backing/src/tests/prospective_parachains.rs index 578f21bef665..94310d2aa164 100644 --- a/polkadot/node/core/backing/src/tests/prospective_parachains.rs +++ b/polkadot/node/core/backing/src/tests/prospective_parachains.rs @@ -185,6 +185,16 @@ async fn activate_leaf( } ); + // Node features request from runtime: all features are disabled. + assert_matches!( + virtual_overseer.recv().await, + AllMessages::RuntimeApi( + RuntimeApiMessage::Request(parent, RuntimeApiRequest::NodeFeatures(_session_index, tx)) + ) if parent == hash => { + tx.send(Ok(Default::default())).unwrap(); + } + ); + // Check if subsystem job issues a request for the minimum backing votes. assert_matches!( virtual_overseer.recv().await, @@ -305,10 +315,11 @@ async fn assert_hypothetical_frontier_requests( ) => { let idx = match expected_requests.iter().position(|r| r.0 == request) { Some(idx) => idx, - None => panic!( + None => + panic!( "unexpected hypothetical frontier request, no match found for {:?}", request - ), + ), }; let resp = std::mem::take(&mut expected_requests[idx].1); tx.send(resp).unwrap(); @@ -1268,6 +1279,7 @@ fn concurrent_dependent_candidates() { let statement_b = CandidateBackingMessage::Statement(leaf_parent, signed_b.clone()); virtual_overseer.send(FromOrchestra::Communication { msg: statement_a }).await; + // At this point the subsystem waits for response, the previous message is received, // send a second one without blocking. let _ = virtual_overseer @@ -1388,7 +1400,19 @@ fn concurrent_dependent_candidates() { assert_eq!(sess_idx, 1); tx.send(Ok(Some(ExecutorParams::default()))).unwrap(); }, + AllMessages::RuntimeApi(RuntimeApiMessage::Request( + _parent, + RuntimeApiRequest::ValidatorGroups(tx), + )) => { + tx.send(Ok(test_state.validator_groups.clone())).unwrap(); + }, + AllMessages::RuntimeApi(RuntimeApiMessage::Request( + _parent, + RuntimeApiRequest::AvailabilityCores(tx), + )) => { + tx.send(Ok(test_state.availability_cores.clone())).unwrap(); + }, _ => panic!("unexpected message received from overseer: {:?}", msg), } } @@ -1419,7 +1443,6 @@ fn seconding_sanity_check_occupy_same_depth() { let leaf_parent = get_parent_hash(leaf_hash); let activated = new_leaf(leaf_hash, LEAF_BLOCK_NUMBER); - let min_block_number = LEAF_BLOCK_NUMBER - LEAF_ANCESTRY_LEN; let min_relay_parents = vec![(para_id_a, min_block_number), (para_id_b, min_block_number)]; let test_leaf_a = TestLeaf { activated, min_relay_parents }; @@ -1555,13 +1578,14 @@ fn occupied_core_assignment() { const LEAF_A_BLOCK_NUMBER: BlockNumber = 100; const LEAF_A_ANCESTRY_LEN: BlockNumber = 3; let para_id = test_state.chain_ids[0]; + let previous_para_id = test_state.chain_ids[1]; // Set the core state to occupied. let mut candidate_descriptor = ::test_helpers::dummy_candidate_descriptor(Hash::zero()); - candidate_descriptor.para_id = para_id; + candidate_descriptor.para_id = previous_para_id; test_state.availability_cores[0] = CoreState::Occupied(OccupiedCore { group_responsible: Default::default(), - next_up_on_available: None, + next_up_on_available: Some(ScheduledCore { para_id, collator: None }), occupied_since: 100_u32, time_out_at: 200_u32, next_up_on_time_out: None, diff --git a/polkadot/node/core/provisioner/src/lib.rs b/polkadot/node/core/provisioner/src/lib.rs index 51f768d782e0..a29cf72afb14 100644 --- a/polkadot/node/core/provisioner/src/lib.rs +++ b/polkadot/node/core/provisioner/src/lib.rs @@ -681,10 +681,17 @@ async fn request_backable_candidates( CoreState::Free => continue, }; + // We should be calling this once per para rather than per core. + // TODO: Will be fixed in https://github.com/paritytech/polkadot-sdk/pull/3233. + // For now, at least make sure we don't supply the same candidate multiple times in case a + // para has multiple cores scheduled. let response = get_backable_candidate(relay_parent, para_id, required_path, sender).await?; - match response { - Some((hash, relay_parent)) => selected_candidates.push((hash, relay_parent)), + Some((hash, relay_parent)) => { + if !selected_candidates.iter().any(|bc| &(hash, relay_parent) == bc) { + selected_candidates.push((hash, relay_parent)) + } + }, None => { gum::debug!( target: LOG_TARGET, @@ -726,6 +733,7 @@ async fn select_candidates( ) .await?, }; + gum::debug!(target: LOG_TARGET, ?selected_candidates, "Got backable candidates"); // now get the backed candidates corresponding to these candidate receipts let (tx, rx) = oneshot::channel(); @@ -758,7 +766,7 @@ async fn select_candidates( // keep only one candidate with validation code. let mut with_validation_code = false; candidates.retain(|c| { - if c.candidate.commitments.new_validation_code.is_some() { + if c.candidate().commitments.new_validation_code.is_some() { if with_validation_code { return false } diff --git a/polkadot/node/core/provisioner/src/tests.rs b/polkadot/node/core/provisioner/src/tests.rs index b26df8ddb910..87c0e7a65d35 100644 --- a/polkadot/node/core/provisioner/src/tests.rs +++ b/polkadot/node/core/provisioner/src/tests.rs @@ -460,13 +460,16 @@ mod select_candidates { let expected_backed = expected_candidates .iter() - .map(|c| BackedCandidate { - candidate: CommittedCandidateReceipt { - descriptor: c.descriptor.clone(), - commitments: Default::default(), - }, - validity_votes: Vec::new(), - validator_indices: default_bitvec(MOCK_GROUP_SIZE), + .map(|c| { + BackedCandidate::new( + CommittedCandidateReceipt { + descriptor: c.descriptor().clone(), + commitments: Default::default(), + }, + Vec::new(), + default_bitvec(MOCK_GROUP_SIZE), + None, + ) }) .collect(); @@ -486,7 +489,7 @@ mod select_candidates { result.into_iter().for_each(|c| { assert!( - expected_candidates.iter().any(|c2| c.candidate.corresponds_to(c2)), + expected_candidates.iter().any(|c2| c.candidate().corresponds_to(c2)), "Failed to find candidate: {:?}", c, ) @@ -532,10 +535,13 @@ mod select_candidates { // Build possible outputs from select_candidates let backed_candidates: Vec<_> = committed_receipts .iter() - .map(|committed_receipt| BackedCandidate { - candidate: committed_receipt.clone(), - validity_votes: Vec::new(), - validator_indices: default_bitvec(MOCK_GROUP_SIZE), + .map(|committed_receipt| { + BackedCandidate::new( + committed_receipt.clone(), + Vec::new(), + default_bitvec(MOCK_GROUP_SIZE), + None, + ) }) .collect(); @@ -566,7 +572,7 @@ mod select_candidates { result.into_iter().for_each(|c| { assert!( - expected_backed_filtered.iter().any(|c2| c.candidate.corresponds_to(c2)), + expected_backed_filtered.iter().any(|c2| c.candidate().corresponds_to(c2)), "Failed to find candidate: {:?}", c, ) @@ -605,13 +611,16 @@ mod select_candidates { let expected_backed = expected_candidates .iter() - .map(|c| BackedCandidate { - candidate: CommittedCandidateReceipt { - descriptor: c.descriptor.clone(), - commitments: Default::default(), - }, - validity_votes: Vec::new(), - validator_indices: default_bitvec(MOCK_GROUP_SIZE), + .map(|c| { + BackedCandidate::new( + CommittedCandidateReceipt { + descriptor: c.descriptor.clone(), + commitments: Default::default(), + }, + Vec::new(), + default_bitvec(MOCK_GROUP_SIZE), + None, + ) }) .collect(); @@ -631,7 +640,7 @@ mod select_candidates { result.into_iter().for_each(|c| { assert!( - expected_candidates.iter().any(|c2| c.candidate.corresponds_to(c2)), + expected_candidates.iter().any(|c2| c.candidate().corresponds_to(c2)), "Failed to find candidate: {:?}", c, ) @@ -671,13 +680,16 @@ mod select_candidates { let expected_backed = expected_candidates .iter() - .map(|c| BackedCandidate { - candidate: CommittedCandidateReceipt { - descriptor: c.descriptor.clone(), - commitments: Default::default(), - }, - validity_votes: Vec::new(), - validator_indices: default_bitvec(MOCK_GROUP_SIZE), + .map(|c| { + BackedCandidate::new( + CommittedCandidateReceipt { + descriptor: c.descriptor().clone(), + commitments: Default::default(), + }, + Vec::new(), + default_bitvec(MOCK_GROUP_SIZE), + None, + ) }) .collect(); @@ -697,7 +709,7 @@ mod select_candidates { result.into_iter().for_each(|c| { assert!( - expected_candidates.iter().any(|c2| c.candidate.corresponds_to(c2)), + expected_candidates.iter().any(|c2| c.candidate().corresponds_to(c2)), "Failed to find candidate: {:?}", c, ) diff --git a/polkadot/node/test/service/src/chain_spec.rs b/polkadot/node/test/service/src/chain_spec.rs index 0295090b9521..4cc387317c3f 100644 --- a/polkadot/node/test/service/src/chain_spec.rs +++ b/polkadot/node/test/service/src/chain_spec.rs @@ -169,6 +169,7 @@ fn polkadot_testnet_genesis( paras_availability_period: 4, no_show_slots: 10, minimum_validation_upgrade_delay: 5, + max_downward_message_size: 1024, ..Default::default() }, } diff --git a/polkadot/primitives/Cargo.toml b/polkadot/primitives/Cargo.toml index 58d3d1001610..e63fb621c788 100644 --- a/polkadot/primitives/Cargo.toml +++ b/polkadot/primitives/Cargo.toml @@ -14,6 +14,7 @@ bitvec = { version = "1.0.0", default-features = false, features = ["alloc", "se hex-literal = "0.4.1" parity-scale-codec = { version = "3.6.1", default-features = false, features = ["bit-vec", "derive"] } scale-info = { version = "2.10.0", default-features = false, features = ["bit-vec", "derive", "serde"] } +log = { workspace = true, default-features = false } serde = { features = ["alloc", "derive"], workspace = true } application-crypto = { package = "sp-application-crypto", path = "../../substrate/primitives/application-crypto", default-features = false, features = ["serde"] } @@ -38,6 +39,7 @@ std = [ "application-crypto/std", "bitvec/std", "inherents/std", + "log/std", "parity-scale-codec/std", "polkadot-core-primitives/std", "polkadot-parachain-primitives/std", diff --git a/polkadot/primitives/src/v6/mod.rs b/polkadot/primitives/src/v6/mod.rs index 4938d20d2d1b..89431f7801f7 100644 --- a/polkadot/primitives/src/v6/mod.rs +++ b/polkadot/primitives/src/v6/mod.rs @@ -16,7 +16,7 @@ //! `V6` Primitives. -use bitvec::vec::BitVec; +use bitvec::{field::BitField, slice::BitSlice, vec::BitVec}; use parity_scale_codec::{Decode, Encode}; use scale_info::TypeInfo; use sp_std::{ @@ -72,6 +72,7 @@ pub use metrics::{ /// The key type ID for a collator key. pub const COLLATOR_KEY_TYPE_ID: KeyTypeId = KeyTypeId(*b"coll"); +const LOG_TARGET: &str = "runtime::primitives"; mod collator_app { use application_crypto::{app_crypto, sr25519}; @@ -706,19 +707,50 @@ pub type UncheckedSignedAvailabilityBitfields = Vec { /// The candidate referred to. - pub candidate: CommittedCandidateReceipt, + candidate: CommittedCandidateReceipt, /// The validity votes themselves, expressed as signatures. - pub validity_votes: Vec, - /// The indices of the validators within the group, expressed as a bitfield. - pub validator_indices: BitVec, + validity_votes: Vec, + /// The indices of the validators within the group, expressed as a bitfield. May be extended + /// beyond the backing group size to contain the assigned core index, if ElasticScalingMVP is + /// enabled. + validator_indices: BitVec, } impl BackedCandidate { - /// Get a reference to the descriptor of the para. + /// Constructor + pub fn new( + candidate: CommittedCandidateReceipt, + validity_votes: Vec, + validator_indices: BitVec, + core_index: Option, + ) -> Self { + let mut instance = Self { candidate, validity_votes, validator_indices }; + if let Some(core_index) = core_index { + instance.inject_core_index(core_index); + } + instance + } + + /// Get a reference to the descriptor of the candidate. pub fn descriptor(&self) -> &CandidateDescriptor { &self.candidate.descriptor } + /// Get a reference to the committed candidate receipt of the candidate. + pub fn candidate(&self) -> &CommittedCandidateReceipt { + &self.candidate + } + + /// Get a reference to the validity votes of the candidate. + pub fn validity_votes(&self) -> &[ValidityAttestation] { + &self.validity_votes + } + + /// Get a mutable reference to validity votes of the para. + pub fn validity_votes_mut(&mut self) -> &mut Vec { + &mut self.validity_votes + } + /// Compute this candidate's hash. pub fn hash(&self) -> CandidateHash where @@ -734,6 +766,48 @@ impl BackedCandidate { { self.candidate.to_plain() } + + /// Get a copy of the validator indices and the assumed core index, if any. + pub fn validator_indices_and_core_index( + &self, + core_index_enabled: bool, + ) -> (&BitSlice, Option) { + // This flag tells us if the block producers must enable Elastic Scaling MVP hack. + // It extends `BackedCandidate::validity_indices` to store a 8 bit core index. + if core_index_enabled { + let core_idx_offset = self.validator_indices.len().saturating_sub(8); + if core_idx_offset > 0 { + let (validator_indices_slice, core_idx_slice) = + self.validator_indices.split_at(core_idx_offset); + return ( + validator_indices_slice, + Some(CoreIndex(core_idx_slice.load::() as u32)), + ); + } + } + + (&self.validator_indices, None) + } + + /// Inject a core index in the validator_indices bitvec. + fn inject_core_index(&mut self, core_index: CoreIndex) { + let core_index_to_inject: BitVec = + BitVec::from_vec(vec![core_index.0 as u8]); + self.validator_indices.extend(core_index_to_inject); + } + + /// Update the validator indices and core index in the candidate. + pub fn set_validator_indices_and_core_index( + &mut self, + new_indices: BitVec, + maybe_core_index: Option, + ) { + self.validator_indices = new_indices; + + if let Some(core_index) = maybe_core_index { + self.inject_core_index(core_index); + } + } } /// Verify the backing of the given candidate. @@ -746,43 +820,65 @@ impl BackedCandidate { /// /// Returns either an error, indicating that one of the signatures was invalid or that the index /// was out-of-bounds, or the number of signatures checked. -pub fn check_candidate_backing + Clone + Encode>( - backed: &BackedCandidate, +pub fn check_candidate_backing + Clone + Encode + core::fmt::Debug>( + candidate_hash: CandidateHash, + validity_votes: &[ValidityAttestation], + validator_indices: &BitSlice, signing_context: &SigningContext, group_len: usize, validator_lookup: impl Fn(usize) -> Option, ) -> Result { - if backed.validator_indices.len() != group_len { + if validator_indices.len() != group_len { + log::debug!( + target: LOG_TARGET, + "Check candidate backing: indices mismatch: group_len = {} , indices_len = {}", + group_len, + validator_indices.len(), + ); return Err(()) } - if backed.validity_votes.len() > group_len { + if validity_votes.len() > group_len { + log::debug!( + target: LOG_TARGET, + "Check candidate backing: Too many votes, expected: {}, found: {}", + group_len, + validity_votes.len(), + ); return Err(()) } - // this is known, even in runtime, to be blake2-256. - let hash = backed.candidate.hash(); - let mut signed = 0; - for ((val_in_group_idx, _), attestation) in backed - .validator_indices + for ((val_in_group_idx, _), attestation) in validator_indices .iter() .enumerate() .filter(|(_, signed)| **signed) - .zip(backed.validity_votes.iter()) + .zip(validity_votes.iter()) { let validator_id = validator_lookup(val_in_group_idx).ok_or(())?; - let payload = attestation.signed_payload(hash, signing_context); + let payload = attestation.signed_payload(candidate_hash, signing_context); let sig = attestation.signature(); if sig.verify(&payload[..], &validator_id) { signed += 1; } else { + log::debug!( + target: LOG_TARGET, + "Check candidate backing: Invalid signature. validator_id = {:?}, validator_index = {} ", + validator_id, + val_in_group_idx, + ); return Err(()) } } - if signed != backed.validity_votes.len() { + if signed != validity_votes.len() { + log::error!( + target: LOG_TARGET, + "Check candidate backing: Too many signatures, expected = {}, found = {}", + validity_votes.len(), + signed, + ); return Err(()) } @@ -1859,6 +1955,34 @@ pub enum PvfExecKind { #[cfg(test)] mod tests { use super::*; + use bitvec::bitvec; + use primitives::sr25519; + + pub fn dummy_committed_candidate_receipt() -> CommittedCandidateReceipt { + let zeros = Hash::zero(); + + CommittedCandidateReceipt { + descriptor: CandidateDescriptor { + para_id: 0.into(), + relay_parent: zeros, + collator: CollatorId::from(sr25519::Public::from_raw([0; 32])), + persisted_validation_data_hash: zeros, + pov_hash: zeros, + erasure_root: zeros, + signature: CollatorSignature::from(sr25519::Signature([0u8; 64])), + para_head: zeros, + validation_code_hash: ValidationCode(vec![1, 2, 3, 4, 5, 6, 7, 8, 9]).hash(), + }, + commitments: CandidateCommitments { + head_data: HeadData(vec![]), + upward_messages: vec![].try_into().expect("empty vec fits within bounds"), + new_validation_code: None, + horizontal_messages: vec![].try_into().expect("empty vec fits within bounds"), + processed_downward_messages: 0, + hrmp_watermark: 0_u32, + }, + } + } #[test] fn group_rotation_info_calculations() { @@ -1933,4 +2057,73 @@ mod tests { assert!(zero_b.leading_zeros() >= zero_u.leading_zeros()); } + + #[test] + fn test_backed_candidate_injected_core_index() { + let initial_validator_indices = bitvec![u8, bitvec::order::Lsb0; 0, 1, 0, 1]; + let mut candidate = BackedCandidate::new( + dummy_committed_candidate_receipt(), + vec![], + initial_validator_indices.clone(), + None, + ); + + // No core index supplied, ElasticScalingMVP is off. + let (validator_indices, core_index) = candidate.validator_indices_and_core_index(false); + assert_eq!(validator_indices, initial_validator_indices.as_bitslice()); + assert!(core_index.is_none()); + + // No core index supplied, ElasticScalingMVP is on. Still, decoding will be ok if backing + // group size is <= 8, to give a chance to parachains that don't have multiple cores + // assigned. + let (validator_indices, core_index) = candidate.validator_indices_and_core_index(true); + assert_eq!(validator_indices, initial_validator_indices.as_bitslice()); + assert!(core_index.is_none()); + + let encoded_validator_indices = candidate.validator_indices.clone(); + candidate.set_validator_indices_and_core_index(validator_indices.into(), core_index); + assert_eq!(candidate.validator_indices, encoded_validator_indices); + + // No core index supplied, ElasticScalingMVP is on. Decoding is corrupted if backing group + // size larger than 8. + let candidate = BackedCandidate::new( + dummy_committed_candidate_receipt(), + vec![], + bitvec![u8, bitvec::order::Lsb0; 0, 1, 0, 1, 0, 1, 0, 1, 0], + None, + ); + let (validator_indices, core_index) = candidate.validator_indices_and_core_index(true); + assert_eq!(validator_indices, bitvec![u8, bitvec::order::Lsb0; 0].as_bitslice()); + assert!(core_index.is_some()); + + // Core index supplied, ElasticScalingMVP is off. Core index will be treated as normal + // validator indices. Runtime will check against this. + let candidate = BackedCandidate::new( + dummy_committed_candidate_receipt(), + vec![], + bitvec![u8, bitvec::order::Lsb0; 0, 1, 0, 1], + Some(CoreIndex(10)), + ); + let (validator_indices, core_index) = candidate.validator_indices_and_core_index(false); + assert_eq!( + validator_indices, + bitvec![u8, bitvec::order::Lsb0; 0, 1, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0] + ); + assert!(core_index.is_none()); + + // Core index supplied, ElasticScalingMVP is on. + let mut candidate = BackedCandidate::new( + dummy_committed_candidate_receipt(), + vec![], + bitvec![u8, bitvec::order::Lsb0; 0, 1, 0, 1], + Some(CoreIndex(10)), + ); + let (validator_indices, core_index) = candidate.validator_indices_and_core_index(true); + assert_eq!(validator_indices, bitvec![u8, bitvec::order::Lsb0; 0, 1, 0, 1]); + assert_eq!(core_index, Some(CoreIndex(10))); + + let encoded_validator_indices = candidate.validator_indices.clone(); + candidate.set_validator_indices_and_core_index(validator_indices.into(), core_index); + assert_eq!(candidate.validator_indices, encoded_validator_indices); + } } diff --git a/polkadot/primitives/src/vstaging/mod.rs b/polkadot/primitives/src/vstaging/mod.rs index 630bcf8679ad..39d9dfc02c5b 100644 --- a/polkadot/primitives/src/vstaging/mod.rs +++ b/polkadot/primitives/src/vstaging/mod.rs @@ -64,9 +64,13 @@ pub mod node_features { /// Tells if tranch0 assignments could be sent in a single certificate. /// Reserved for: `` EnableAssignmentsV2 = 0, + /// This feature enables the extension of `BackedCandidate::validator_indices` by 8 bits. + /// The value stored there represents the assumed core index where the candidates + /// are backed. This is needed for the elastic scaling MVP. + ElasticScalingMVP = 1, /// First unassigned feature bit. /// Every time a new feature flag is assigned it should take this value. /// and this should be incremented. - FirstUnassigned = 1, + FirstUnassigned = 2, } } diff --git a/polkadot/runtime/parachains/Cargo.toml b/polkadot/runtime/parachains/Cargo.toml index 311a62b6c917..610401454763 100644 --- a/polkadot/runtime/parachains/Cargo.toml +++ b/polkadot/runtime/parachains/Cargo.toml @@ -69,6 +69,7 @@ sp-tracing = { path = "../../../substrate/primitives/tracing" } sp-crypto-hashing = { path = "../../../substrate/primitives/crypto/hashing" } thousands = "0.2.0" assert_matches = "1" +rstest = "0.18.2" serde_json = { workspace = true, default-features = true } [features] diff --git a/polkadot/runtime/parachains/src/builder.rs b/polkadot/runtime/parachains/src/builder.rs index 016b3fca589a..500bc70cfa75 100644 --- a/polkadot/runtime/parachains/src/builder.rs +++ b/polkadot/runtime/parachains/src/builder.rs @@ -587,11 +587,12 @@ impl BenchBuilder { }) .collect(); - BackedCandidate:: { + BackedCandidate::::new( candidate, validity_votes, - validator_indices: bitvec::bitvec![u8, bitvec::order::Lsb0; 1; group_validators.len()], - } + bitvec::bitvec![u8, bitvec::order::Lsb0; 1; group_validators.len()], + None, + ) }) .collect() } diff --git a/polkadot/runtime/parachains/src/inclusion/mod.rs b/polkadot/runtime/parachains/src/inclusion/mod.rs index 90af9cde00a8..16e2e93b5617 100644 --- a/polkadot/runtime/parachains/src/inclusion/mod.rs +++ b/polkadot/runtime/parachains/src/inclusion/mod.rs @@ -47,10 +47,7 @@ use scale_info::TypeInfo; use sp_runtime::{traits::One, DispatchError, SaturatedConversion, Saturating}; #[cfg(feature = "std")] use sp_std::fmt; -use sp_std::{ - collections::{btree_map::BTreeMap, btree_set::BTreeSet}, - prelude::*, -}; +use sp_std::{collections::btree_set::BTreeSet, prelude::*}; pub use pallet::*; @@ -601,18 +598,16 @@ impl Pallet { /// scheduled cores. If these conditions are not met, the execution of the function fails. pub(crate) fn process_candidates( allowed_relay_parents: &AllowedRelayParentsTracker>, - candidates: Vec>, - scheduled: &BTreeMap, + candidates: Vec<(BackedCandidate, CoreIndex)>, group_validators: GV, + core_index_enabled: bool, ) -> Result, DispatchError> where GV: Fn(GroupIndex) -> Option>, { let now = >::block_number(); - ensure!(candidates.len() <= scheduled.len(), Error::::UnscheduledCandidate); - - if scheduled.is_empty() { + if candidates.is_empty() { return Ok(ProcessedCandidates::default()) } @@ -648,7 +643,7 @@ impl Pallet { // // In the meantime, we do certain sanity checks on the candidates and on the scheduled // list. - for (candidate_idx, backed_candidate) in candidates.iter().enumerate() { + for (candidate_idx, (backed_candidate, core_index)) in candidates.iter().enumerate() { let relay_parent_hash = backed_candidate.descriptor().relay_parent; let para_id = backed_candidate.descriptor().para_id; @@ -663,7 +658,7 @@ impl Pallet { let relay_parent_number = match check_ctx.verify_backed_candidate( &allowed_relay_parents, candidate_idx, - backed_candidate, + backed_candidate.candidate(), )? { Err(FailedToCreatePVD) => { log::debug!( @@ -679,11 +674,22 @@ impl Pallet { Ok(rpn) => rpn, }; - let para_id = backed_candidate.descriptor().para_id; + let (validator_indices, _) = + backed_candidate.validator_indices_and_core_index(core_index_enabled); + + log::debug!( + target: LOG_TARGET, + "Candidate {:?} on {:?}, + core_index_enabled = {}", + backed_candidate.hash(), + core_index, + core_index_enabled + ); + + check_assignment_in_order(core_index)?; + let mut backers = bitvec::bitvec![u8, BitOrderLsb0; 0; validators.len()]; - let core_idx = *scheduled.get(¶_id).ok_or(Error::::UnscheduledCandidate)?; - check_assignment_in_order(core_idx)?; ensure!( >::get(¶_id).is_none() && >::get(¶_id).is_none(), @@ -694,7 +700,7 @@ impl Pallet { // assigned to core at block `N + 1`. Thus, `relay_parent_number + 1` // will always land in the current session. let group_idx = >::group_assigned_to_core( - core_idx, + *core_index, relay_parent_number + One::one(), ) .ok_or_else(|| { @@ -711,7 +717,9 @@ impl Pallet { // check the signatures in the backing and that it is a majority. { let maybe_amount_validated = primitives::check_candidate_backing( - &backed_candidate, + backed_candidate.candidate().hash(), + backed_candidate.validity_votes(), + validator_indices, &signing_context, group_vals.len(), |intra_group_vi| { @@ -738,16 +746,15 @@ impl Pallet { let mut backer_idx_and_attestation = Vec::<(ValidatorIndex, ValidityAttestation)>::with_capacity( - backed_candidate.validator_indices.count_ones(), + validator_indices.count_ones(), ); let candidate_receipt = backed_candidate.receipt(); - for ((bit_idx, _), attestation) in backed_candidate - .validator_indices + for ((bit_idx, _), attestation) in validator_indices .iter() .enumerate() .filter(|(_, signed)| **signed) - .zip(backed_candidate.validity_votes.iter().cloned()) + .zip(backed_candidate.validity_votes().iter().cloned()) { let val_idx = group_vals.get(bit_idx).expect("this query succeeded above; qed"); @@ -760,7 +767,7 @@ impl Pallet { } core_indices_and_backers.push(( - (core_idx, para_id), + (*core_index, para_id), backers, group_idx, relay_parent_number, @@ -772,7 +779,7 @@ impl Pallet { // one more sweep for actually writing to storage. let core_indices = core_indices_and_backers.iter().map(|(c, ..)| *c).collect(); - for (candidate, (core, backers, group, relay_parent_number)) in + for ((candidate, _), (core, backers, group, relay_parent_number)) in candidates.into_iter().zip(core_indices_and_backers) { let para_id = candidate.descriptor().para_id; @@ -782,16 +789,18 @@ impl Pallet { bitvec::bitvec![u8, BitOrderLsb0; 0; validators.len()]; Self::deposit_event(Event::::CandidateBacked( - candidate.candidate.to_plain(), - candidate.candidate.commitments.head_data.clone(), + candidate.candidate().to_plain(), + candidate.candidate().commitments.head_data.clone(), core.0, group, )); - let candidate_hash = candidate.candidate.hash(); + let candidate_hash = candidate.candidate().hash(); - let (descriptor, commitments) = - (candidate.candidate.descriptor, candidate.candidate.commitments); + let (descriptor, commitments) = ( + candidate.candidate().descriptor.clone(), + candidate.candidate().commitments.clone(), + ); >::insert( ¶_id, @@ -1195,10 +1204,10 @@ impl CandidateCheckContext { &self, allowed_relay_parents: &AllowedRelayParentsTracker>, candidate_idx: usize, - backed_candidate: &BackedCandidate<::Hash>, + backed_candidate_receipt: &CommittedCandidateReceipt<::Hash>, ) -> Result, FailedToCreatePVD>, Error> { - let para_id = backed_candidate.descriptor().para_id; - let relay_parent = backed_candidate.descriptor().relay_parent; + let para_id = backed_candidate_receipt.descriptor().para_id; + let relay_parent = backed_candidate_receipt.descriptor().relay_parent; // Check that the relay-parent is one of the allowed relay-parents. let (relay_parent_storage_root, relay_parent_number) = { @@ -1223,13 +1232,13 @@ impl CandidateCheckContext { let expected = persisted_validation_data.hash(); ensure!( - expected == backed_candidate.descriptor().persisted_validation_data_hash, + expected == backed_candidate_receipt.descriptor().persisted_validation_data_hash, Error::::ValidationDataHashMismatch, ); } ensure!( - backed_candidate.descriptor().check_collator_signature().is_ok(), + backed_candidate_receipt.descriptor().check_collator_signature().is_ok(), Error::::NotCollatorSigned, ); @@ -1237,25 +1246,25 @@ impl CandidateCheckContext { // A candidate for a parachain without current validation code is not scheduled. .ok_or_else(|| Error::::UnscheduledCandidate)?; ensure!( - backed_candidate.descriptor().validation_code_hash == validation_code_hash, + backed_candidate_receipt.descriptor().validation_code_hash == validation_code_hash, Error::::InvalidValidationCodeHash, ); ensure!( - backed_candidate.descriptor().para_head == - backed_candidate.candidate.commitments.head_data.hash(), + backed_candidate_receipt.descriptor().para_head == + backed_candidate_receipt.commitments.head_data.hash(), Error::::ParaHeadMismatch, ); if let Err(err) = self.check_validation_outputs( para_id, relay_parent_number, - &backed_candidate.candidate.commitments.head_data, - &backed_candidate.candidate.commitments.new_validation_code, - backed_candidate.candidate.commitments.processed_downward_messages, - &backed_candidate.candidate.commitments.upward_messages, - BlockNumberFor::::from(backed_candidate.candidate.commitments.hrmp_watermark), - &backed_candidate.candidate.commitments.horizontal_messages, + &backed_candidate_receipt.commitments.head_data, + &backed_candidate_receipt.commitments.new_validation_code, + backed_candidate_receipt.commitments.processed_downward_messages, + &backed_candidate_receipt.commitments.upward_messages, + BlockNumberFor::::from(backed_candidate_receipt.commitments.hrmp_watermark), + &backed_candidate_receipt.commitments.horizontal_messages, ) { log::debug!( target: LOG_TARGET, diff --git a/polkadot/runtime/parachains/src/inclusion/tests.rs b/polkadot/runtime/parachains/src/inclusion/tests.rs index 557b7b71a9e3..d2b5a67c3e45 100644 --- a/polkadot/runtime/parachains/src/inclusion/tests.rs +++ b/polkadot/runtime/parachains/src/inclusion/tests.rs @@ -120,6 +120,7 @@ pub(crate) fn back_candidate( keystore: &KeystorePtr, signing_context: &SigningContext, kind: BackingKind, + core_index: Option, ) -> BackedCandidate { let mut validator_indices = bitvec::bitvec![u8, BitOrderLsb0; 0; group.len()]; let threshold = effective_minimum_backing_votes( @@ -155,15 +156,20 @@ pub(crate) fn back_candidate( validity_votes.push(ValidityAttestation::Explicit(signature).into()); } - let backed = BackedCandidate { candidate, validity_votes, validator_indices }; + let backed = + BackedCandidate::new(candidate, validity_votes, validator_indices.clone(), core_index); - let successfully_backed = - primitives::check_candidate_backing(&backed, signing_context, group.len(), |i| { - Some(validators[group[i].0 as usize].public().into()) - }) - .ok() - .unwrap_or(0) >= - threshold; + let successfully_backed = primitives::check_candidate_backing( + backed.candidate().hash(), + backed.validity_votes(), + validator_indices.as_bitslice(), + signing_context, + group.len(), + |i| Some(validators[group[i].0 as usize].public().into()), + ) + .ok() + .unwrap_or(0) >= + threshold; match kind { BackingKind::Unanimous | BackingKind::Threshold => assert!(successfully_backed), @@ -919,38 +925,16 @@ fn candidate_checks() { let thread_a_assignment = (thread_a, CoreIndex::from(2)); let allowed_relay_parents = default_allowed_relay_parent_tracker(); - // unscheduled candidate. - { - let mut candidate = TestCandidateBuilder { - para_id: chain_a, - relay_parent: System::parent_hash(), - pov_hash: Hash::repeat_byte(1), - persisted_validation_data_hash: make_vdata_hash(chain_a).unwrap(), - hrmp_watermark: RELAY_PARENT_NUM, - ..Default::default() - } - .build(); - collator_sign_candidate(Sr25519Keyring::One, &mut candidate); - - let backed = back_candidate( - candidate, - &validators, - group_validators(GroupIndex::from(0)).unwrap().as_ref(), - &keystore, - &signing_context, - BackingKind::Threshold, - ); - - assert_noop!( - ParaInclusion::process_candidates( - &allowed_relay_parents, - vec![backed], - &[chain_b_assignment].into_iter().collect(), - &group_validators, - ), - Error::::UnscheduledCandidate - ); - } + // no candidates. + assert_eq!( + ParaInclusion::process_candidates( + &allowed_relay_parents, + vec![], + &group_validators, + false + ), + Ok(ProcessedCandidates::default()) + ); // candidates out of order. { @@ -984,6 +968,7 @@ fn candidate_checks() { &keystore, &signing_context, BackingKind::Threshold, + None, ); let backed_b = back_candidate( @@ -993,15 +978,16 @@ fn candidate_checks() { &keystore, &signing_context, BackingKind::Threshold, + None, ); // out-of-order manifests as unscheduled. assert_noop!( ParaInclusion::process_candidates( &allowed_relay_parents, - vec![backed_b, backed_a], - &[chain_a_assignment, chain_b_assignment].into_iter().collect(), + vec![(backed_b, chain_b_assignment.1), (backed_a, chain_a_assignment.1)], &group_validators, + false ), Error::::ScheduledOutOfOrder ); @@ -1027,14 +1013,15 @@ fn candidate_checks() { &keystore, &signing_context, BackingKind::Lacking, + None, ); assert_noop!( ParaInclusion::process_candidates( &allowed_relay_parents, - vec![backed], - &[chain_a_assignment].into_iter().collect(), + vec![(backed, chain_a_assignment.1)], &group_validators, + false ), Error::::InsufficientBacking ); @@ -1075,6 +1062,7 @@ fn candidate_checks() { &keystore, &signing_context, BackingKind::Threshold, + None, ); let backed_b = back_candidate( @@ -1084,14 +1072,15 @@ fn candidate_checks() { &keystore, &signing_context, BackingKind::Threshold, + None, ); assert_noop!( ParaInclusion::process_candidates( &allowed_relay_parents, - vec![backed_b, backed_a], - &[chain_a_assignment, chain_b_assignment].into_iter().collect(), + vec![(backed_b, chain_b_assignment.1), (backed_a, chain_a_assignment.1)], &group_validators, + false ), Error::::DisallowedRelayParent ); @@ -1122,14 +1111,15 @@ fn candidate_checks() { &keystore, &signing_context, BackingKind::Threshold, + None, ); assert_noop!( ParaInclusion::process_candidates( &allowed_relay_parents, - vec![backed], - &[thread_a_assignment].into_iter().collect(), + vec![(backed, thread_a_assignment.1)], &group_validators, + false ), Error::::NotCollatorSigned ); @@ -1156,6 +1146,7 @@ fn candidate_checks() { &keystore, &signing_context, BackingKind::Threshold, + None, ); let candidate = TestCandidateBuilder::default().build(); @@ -1177,9 +1168,9 @@ fn candidate_checks() { assert_noop!( ParaInclusion::process_candidates( &allowed_relay_parents, - vec![backed], - &[chain_a_assignment].into_iter().collect(), + vec![(backed, chain_a_assignment.1)], &group_validators, + false ), Error::::CandidateScheduledBeforeParaFree ); @@ -1212,14 +1203,15 @@ fn candidate_checks() { &keystore, &signing_context, BackingKind::Threshold, + None, ); assert_noop!( ParaInclusion::process_candidates( &allowed_relay_parents, - vec![backed], - &[chain_a_assignment].into_iter().collect(), + vec![(backed, chain_a_assignment.1)], &group_validators, + false ), Error::::CandidateScheduledBeforeParaFree ); @@ -1249,6 +1241,7 @@ fn candidate_checks() { &keystore, &signing_context, BackingKind::Threshold, + None, ); { @@ -1267,9 +1260,9 @@ fn candidate_checks() { assert_noop!( ParaInclusion::process_candidates( &allowed_relay_parents, - vec![backed], - &[chain_a_assignment].into_iter().collect(), + vec![(backed, chain_a_assignment.1)], &group_validators, + false ), Error::::PrematureCodeUpgrade ); @@ -1296,14 +1289,15 @@ fn candidate_checks() { &keystore, &signing_context, BackingKind::Threshold, + None, ); assert_eq!( ParaInclusion::process_candidates( &allowed_relay_parents, - vec![backed], - &[chain_a_assignment].into_iter().collect(), + vec![(backed, chain_a_assignment.1)], &group_validators, + false ), Err(Error::::ValidationDataHashMismatch.into()), ); @@ -1331,14 +1325,15 @@ fn candidate_checks() { &keystore, &signing_context, BackingKind::Threshold, + None, ); assert_noop!( ParaInclusion::process_candidates( &allowed_relay_parents, - vec![backed], - &[chain_a_assignment].into_iter().collect(), + vec![(backed, chain_a_assignment.1)], &group_validators, + false ), Error::::InvalidValidationCodeHash ); @@ -1366,14 +1361,15 @@ fn candidate_checks() { &keystore, &signing_context, BackingKind::Threshold, + None, ); assert_noop!( ParaInclusion::process_candidates( &allowed_relay_parents, - vec![backed], - &[chain_a_assignment].into_iter().collect(), + vec![(backed, chain_a_assignment.1)], &group_validators, + false ), Error::::ParaHeadMismatch ); @@ -1486,6 +1482,7 @@ fn backing_works() { &keystore, &signing_context, BackingKind::Threshold, + None, ); let backed_b = back_candidate( @@ -1495,6 +1492,7 @@ fn backing_works() { &keystore, &signing_context, BackingKind::Threshold, + None, ); let backed_c = back_candidate( @@ -1504,15 +1502,20 @@ fn backing_works() { &keystore, &signing_context, BackingKind::Threshold, + None, ); - let backed_candidates = vec![backed_a.clone(), backed_b.clone(), backed_c]; + let backed_candidates = vec![ + (backed_a.clone(), chain_a_assignment.1), + (backed_b.clone(), chain_b_assignment.1), + (backed_c, thread_a_assignment.1), + ]; let get_backing_group_idx = { // the order defines the group implicitly for this test case let backed_candidates_with_groups = backed_candidates .iter() .enumerate() - .map(|(idx, backed_candidate)| (backed_candidate.hash(), GroupIndex(idx as _))) + .map(|(idx, (backed_candidate, _))| (backed_candidate.hash(), GroupIndex(idx as _))) .collect::>(); move |candidate_hash_x: CandidateHash| -> Option { @@ -1532,10 +1535,8 @@ fn backing_works() { } = ParaInclusion::process_candidates( &allowed_relay_parents, backed_candidates.clone(), - &[chain_a_assignment, chain_b_assignment, thread_a_assignment] - .into_iter() - .collect(), &group_validators, + false, ) .expect("candidates scheduled, in order, and backed"); @@ -1554,22 +1555,22 @@ fn backing_works() { CandidateHash, (CandidateReceipt, Vec<(ValidatorIndex, ValidityAttestation)>), >::new(); - backed_candidates.into_iter().for_each(|backed_candidate| { + backed_candidates.into_iter().for_each(|(backed_candidate, _)| { let candidate_receipt_with_backers = intermediate .entry(backed_candidate.hash()) .or_insert_with(|| (backed_candidate.receipt(), Vec::new())); - - assert_eq!( - backed_candidate.validity_votes.len(), - backed_candidate.validator_indices.count_ones() - ); + let (validator_indices, None) = + backed_candidate.validator_indices_and_core_index(false) + else { + panic!("Expected no injected core index") + }; + assert_eq!(backed_candidate.validity_votes().len(), validator_indices.count_ones()); candidate_receipt_with_backers.1.extend( - backed_candidate - .validator_indices + validator_indices .iter() .enumerate() .filter(|(_, signed)| **signed) - .zip(backed_candidate.validity_votes.iter().cloned()) + .zip(backed_candidate.validity_votes().iter().cloned()) .filter_map(|((validator_index_within_group, _), attestation)| { let grp_idx = get_backing_group_idx(backed_candidate.hash()).unwrap(); group_validators(grp_idx).map(|validator_indices| { @@ -1666,6 +1667,257 @@ fn backing_works() { }); } +#[test] +fn backing_works_with_elastic_scaling_mvp() { + let chain_a = ParaId::from(1_u32); + let chain_b = ParaId::from(2_u32); + let thread_a = ParaId::from(3_u32); + + // The block number of the relay-parent for testing. + const RELAY_PARENT_NUM: BlockNumber = 4; + + let paras = vec![ + (chain_a, ParaKind::Parachain), + (chain_b, ParaKind::Parachain), + (thread_a, ParaKind::Parathread), + ]; + let validators = vec![ + Sr25519Keyring::Alice, + Sr25519Keyring::Bob, + Sr25519Keyring::Charlie, + Sr25519Keyring::Dave, + Sr25519Keyring::Ferdie, + ]; + let keystore: KeystorePtr = Arc::new(LocalKeystore::in_memory()); + for validator in validators.iter() { + Keystore::sr25519_generate_new( + &*keystore, + PARACHAIN_KEY_TYPE_ID, + Some(&validator.to_seed()), + ) + .unwrap(); + } + let validator_public = validator_pubkeys(&validators); + + new_test_ext(genesis_config(paras)).execute_with(|| { + shared::Pallet::::set_active_validators_ascending(validator_public.clone()); + shared::Pallet::::set_session_index(5); + + run_to_block(5, |_| None); + + let signing_context = + SigningContext { parent_hash: System::parent_hash(), session_index: 5 }; + + let group_validators = |group_index: GroupIndex| { + match group_index { + group_index if group_index == GroupIndex::from(0) => Some(vec![0, 1]), + group_index if group_index == GroupIndex::from(1) => Some(vec![2, 3]), + group_index if group_index == GroupIndex::from(2) => Some(vec![4]), + _ => panic!("Group index out of bounds for 2 parachains and 1 parathread core"), + } + .map(|vs| vs.into_iter().map(ValidatorIndex).collect::>()) + }; + + // When processing candidates, we compute the group index from scheduler. + let validator_groups = vec![ + vec![ValidatorIndex(0), ValidatorIndex(1)], + vec![ValidatorIndex(2), ValidatorIndex(3)], + vec![ValidatorIndex(4)], + ]; + Scheduler::set_validator_groups(validator_groups); + + let allowed_relay_parents = default_allowed_relay_parent_tracker(); + + let mut candidate_a = TestCandidateBuilder { + para_id: chain_a, + relay_parent: System::parent_hash(), + pov_hash: Hash::repeat_byte(1), + persisted_validation_data_hash: make_vdata_hash(chain_a).unwrap(), + hrmp_watermark: RELAY_PARENT_NUM, + ..Default::default() + } + .build(); + collator_sign_candidate(Sr25519Keyring::One, &mut candidate_a); + + let mut candidate_b_1 = TestCandidateBuilder { + para_id: chain_b, + relay_parent: System::parent_hash(), + pov_hash: Hash::repeat_byte(2), + persisted_validation_data_hash: make_vdata_hash(chain_b).unwrap(), + hrmp_watermark: RELAY_PARENT_NUM, + ..Default::default() + } + .build(); + collator_sign_candidate(Sr25519Keyring::One, &mut candidate_b_1); + + let mut candidate_b_2 = TestCandidateBuilder { + para_id: chain_b, + relay_parent: System::parent_hash(), + pov_hash: Hash::repeat_byte(3), + persisted_validation_data_hash: make_vdata_hash(chain_b).unwrap(), + hrmp_watermark: RELAY_PARENT_NUM, + ..Default::default() + } + .build(); + collator_sign_candidate(Sr25519Keyring::One, &mut candidate_b_2); + + let backed_a = back_candidate( + candidate_a.clone(), + &validators, + group_validators(GroupIndex::from(0)).unwrap().as_ref(), + &keystore, + &signing_context, + BackingKind::Threshold, + None, + ); + + let backed_b_1 = back_candidate( + candidate_b_1.clone(), + &validators, + group_validators(GroupIndex::from(1)).unwrap().as_ref(), + &keystore, + &signing_context, + BackingKind::Threshold, + Some(CoreIndex(1)), + ); + + let backed_b_2 = back_candidate( + candidate_b_2.clone(), + &validators, + group_validators(GroupIndex::from(2)).unwrap().as_ref(), + &keystore, + &signing_context, + BackingKind::Threshold, + Some(CoreIndex(2)), + ); + + let backed_candidates = vec![ + (backed_a.clone(), CoreIndex(0)), + (backed_b_1.clone(), CoreIndex(1)), + (backed_b_2.clone(), CoreIndex(2)), + ]; + let get_backing_group_idx = { + // the order defines the group implicitly for this test case + let backed_candidates_with_groups = backed_candidates + .iter() + .enumerate() + .map(|(idx, (backed_candidate, _))| (backed_candidate.hash(), GroupIndex(idx as _))) + .collect::>(); + + move |candidate_hash_x: CandidateHash| -> Option { + backed_candidates_with_groups.iter().find_map(|(candidate_hash, grp)| { + if *candidate_hash == candidate_hash_x { + Some(*grp) + } else { + None + } + }) + } + }; + + let ProcessedCandidates { + core_indices: occupied_cores, + candidate_receipt_with_backing_validator_indices, + } = ParaInclusion::process_candidates( + &allowed_relay_parents, + backed_candidates.clone(), + &group_validators, + true, + ) + .expect("candidates scheduled, in order, and backed"); + + // Both b candidates will be backed. However, only one will be recorded on-chain and proceed + // with being made available. + assert_eq!( + occupied_cores, + vec![ + (CoreIndex::from(0), chain_a), + (CoreIndex::from(1), chain_b), + (CoreIndex::from(2), chain_b), + ] + ); + + // Transform the votes into the setup we expect + let mut expected = std::collections::HashMap::< + CandidateHash, + (CandidateReceipt, Vec<(ValidatorIndex, ValidityAttestation)>), + >::new(); + backed_candidates.into_iter().for_each(|(backed_candidate, _)| { + let candidate_receipt_with_backers = expected + .entry(backed_candidate.hash()) + .or_insert_with(|| (backed_candidate.receipt(), Vec::new())); + let (validator_indices, _maybe_core_index) = + backed_candidate.validator_indices_and_core_index(true); + assert_eq!(backed_candidate.validity_votes().len(), validator_indices.count_ones()); + candidate_receipt_with_backers.1.extend( + validator_indices + .iter() + .enumerate() + .filter(|(_, signed)| **signed) + .zip(backed_candidate.validity_votes().iter().cloned()) + .filter_map(|((validator_index_within_group, _), attestation)| { + let grp_idx = get_backing_group_idx(backed_candidate.hash()).unwrap(); + group_validators(grp_idx).map(|validator_indices| { + (validator_indices[validator_index_within_group], attestation) + }) + }), + ); + }); + + assert_eq!( + expected, + candidate_receipt_with_backing_validator_indices + .into_iter() + .map(|c| (c.0.hash(), c)) + .collect() + ); + + let backers = { + let num_backers = effective_minimum_backing_votes( + group_validators(GroupIndex(0)).unwrap().len(), + configuration::Pallet::::config().minimum_backing_votes, + ); + backing_bitfield(&(0..num_backers).collect::>()) + }; + assert_eq!( + >::get(&chain_a), + Some(CandidatePendingAvailability { + core: CoreIndex::from(0), + hash: candidate_a.hash(), + descriptor: candidate_a.descriptor, + availability_votes: default_availability_votes(), + relay_parent_number: System::block_number() - 1, + backed_in_number: System::block_number(), + backers, + backing_group: GroupIndex::from(0), + }) + ); + assert_eq!( + >::get(&chain_a), + Some(candidate_a.commitments), + ); + + // Only one candidate for b will be recorded on chain. + assert_eq!( + >::get(&chain_b), + Some(CandidatePendingAvailability { + core: CoreIndex::from(2), + hash: candidate_b_2.hash(), + descriptor: candidate_b_2.descriptor, + availability_votes: default_availability_votes(), + relay_parent_number: System::block_number() - 1, + backed_in_number: System::block_number(), + backers: backing_bitfield(&[4]), + backing_group: GroupIndex::from(2), + }) + ); + assert_eq!( + >::get(&chain_b), + Some(candidate_b_2.commitments), + ); + }); +} + #[test] fn can_include_candidate_with_ok_code_upgrade() { let chain_a = ParaId::from(1_u32); @@ -1740,14 +1992,15 @@ fn can_include_candidate_with_ok_code_upgrade() { &keystore, &signing_context, BackingKind::Threshold, + None, ); let ProcessedCandidates { core_indices: occupied_cores, .. } = ParaInclusion::process_candidates( &allowed_relay_parents, - vec![backed_a], - &[chain_a_assignment].into_iter().collect(), + vec![(backed_a, chain_a_assignment.1)], &group_validators, + false, ) .expect("candidates scheduled, in order, and backed"); @@ -1932,6 +2185,7 @@ fn check_allowed_relay_parents() { &keystore, &signing_context_a, BackingKind::Threshold, + None, ); let backed_b = back_candidate( @@ -1941,6 +2195,7 @@ fn check_allowed_relay_parents() { &keystore, &signing_context_b, BackingKind::Threshold, + None, ); let backed_c = back_candidate( @@ -1950,17 +2205,20 @@ fn check_allowed_relay_parents() { &keystore, &signing_context_c, BackingKind::Threshold, + None, ); - let backed_candidates = vec![backed_a, backed_b, backed_c]; + let backed_candidates = vec![ + (backed_a, chain_a_assignment.1), + (backed_b, chain_b_assignment.1), + (backed_c, thread_a_assignment.1), + ]; ParaInclusion::process_candidates( &allowed_relay_parents, backed_candidates.clone(), - &[chain_a_assignment, chain_b_assignment, thread_a_assignment] - .into_iter() - .collect(), &group_validators, + false, ) .expect("candidates scheduled, in order, and backed"); }); @@ -2189,14 +2447,15 @@ fn para_upgrade_delay_scheduled_from_inclusion() { &keystore, &signing_context, BackingKind::Threshold, + None, ); let ProcessedCandidates { core_indices: occupied_cores, .. } = ParaInclusion::process_candidates( &allowed_relay_parents, - vec![backed_a], - &[chain_a_assignment].into_iter().collect(), + vec![(backed_a, chain_a_assignment.1)], &group_validators, + false, ) .expect("candidates scheduled, in order, and backed"); diff --git a/polkadot/runtime/parachains/src/paras_inherent/benchmarking.rs b/polkadot/runtime/parachains/src/paras_inherent/benchmarking.rs index 0f6b23ae1b39..ad3fa8e0dc71 100644 --- a/polkadot/runtime/parachains/src/paras_inherent/benchmarking.rs +++ b/polkadot/runtime/parachains/src/paras_inherent/benchmarking.rs @@ -120,7 +120,7 @@ benchmarks! { // with `v` validity votes. // let votes = v as usize; let votes = min(scheduler::Pallet::::group_validators(GroupIndex::from(0)).unwrap().len(), v as usize); - assert_eq!(benchmark.backed_candidates.get(0).unwrap().validity_votes.len(), votes); + assert_eq!(benchmark.backed_candidates.get(0).unwrap().validity_votes().len(), votes); benchmark.bitfields.clear(); benchmark.disputes.clear(); @@ -177,7 +177,7 @@ benchmarks! { // There is 1 backed assert_eq!(benchmark.backed_candidates.len(), 1); assert_eq!( - benchmark.backed_candidates.get(0).unwrap().validity_votes.len(), + benchmark.backed_candidates.get(0).unwrap().validity_votes().len(), votes, ); diff --git a/polkadot/runtime/parachains/src/paras_inherent/mod.rs b/polkadot/runtime/parachains/src/paras_inherent/mod.rs index 81e092f0a991..cebf858c24ab 100644 --- a/polkadot/runtime/parachains/src/paras_inherent/mod.rs +++ b/polkadot/runtime/parachains/src/paras_inherent/mod.rs @@ -43,15 +43,14 @@ use frame_support::{ use frame_system::pallet_prelude::*; use pallet_babe::{self, ParentBlockRandomness}; use primitives::{ - effective_minimum_backing_votes, BackedCandidate, CandidateHash, CandidateReceipt, - CheckedDisputeStatementSet, CheckedMultiDisputeStatementSet, CoreIndex, DisputeStatementSet, - InherentData as ParachainsInherentData, MultiDisputeStatementSet, ScrapedOnChainVotes, - SessionIndex, SignedAvailabilityBitfields, SigningContext, UncheckedSignedAvailabilityBitfield, - UncheckedSignedAvailabilityBitfields, ValidatorId, ValidatorIndex, ValidityAttestation, - PARACHAINS_INHERENT_IDENTIFIER, + effective_minimum_backing_votes, vstaging::node_features::FeatureIndex, BackedCandidate, + CandidateHash, CandidateReceipt, CheckedDisputeStatementSet, CheckedMultiDisputeStatementSet, + CoreIndex, DisputeStatementSet, InherentData as ParachainsInherentData, + MultiDisputeStatementSet, ScrapedOnChainVotes, SessionIndex, SignedAvailabilityBitfields, + SigningContext, UncheckedSignedAvailabilityBitfield, UncheckedSignedAvailabilityBitfields, + ValidatorId, ValidatorIndex, ValidityAttestation, PARACHAINS_INHERENT_IDENTIFIER, }; use rand::{seq::SliceRandom, SeedableRng}; - use scale_info::TypeInfo; use sp_runtime::traits::{Header as HeaderT, One}; use sp_std::{ @@ -145,6 +144,10 @@ pub mod pallet { DisputeInvalid, /// A candidate was backed by a disabled validator BackedByDisabled, + /// A candidate was backed even though the paraid was not scheduled. + BackedOnUnscheduledCore, + /// Too many candidates supplied. + UnscheduledCandidate, } /// Whether the paras inherent was included within this block. @@ -585,25 +588,39 @@ impl Pallet { let freed = collect_all_freed_cores::(freed_concluded.iter().cloned()); >::free_cores_and_fill_claimqueue(freed, now); - let scheduled = >::scheduled_paras() - .map(|(core_idx, para_id)| (para_id, core_idx)) - .collect(); METRICS.on_candidates_processed_total(backed_candidates.len() as u64); - let SanitizedBackedCandidates { backed_candidates, votes_from_disabled_were_dropped } = - sanitize_backed_candidates::( - backed_candidates, - &allowed_relay_parents, - |candidate_idx: usize, - backed_candidate: &BackedCandidate<::Hash>| - -> bool { - let para_id = backed_candidate.descriptor().para_id; - let prev_context = >::para_most_recent_context(para_id); - let check_ctx = CandidateCheckContext::::new(prev_context); - - // never include a concluded-invalid candidate - current_concluded_invalid_disputes.contains(&backed_candidate.hash()) || + let core_index_enabled = configuration::Pallet::::config() + .node_features + .get(FeatureIndex::ElasticScalingMVP as usize) + .map(|b| *b) + .unwrap_or(false); + + let mut scheduled: BTreeMap> = BTreeMap::new(); + let mut total_scheduled_cores = 0; + + for (core_idx, para_id) in >::scheduled_paras() { + total_scheduled_cores += 1; + scheduled.entry(para_id).or_default().insert(core_idx); + } + + let SanitizedBackedCandidates { + backed_candidates_with_core, + votes_from_disabled_were_dropped, + dropped_unscheduled_candidates, + } = sanitize_backed_candidates::( + backed_candidates, + &allowed_relay_parents, + |candidate_idx: usize, + backed_candidate: &BackedCandidate<::Hash>| + -> bool { + let para_id = backed_candidate.descriptor().para_id; + let prev_context = >::para_most_recent_context(para_id); + let check_ctx = CandidateCheckContext::::new(prev_context); + + // never include a concluded-invalid candidate + current_concluded_invalid_disputes.contains(&backed_candidate.hash()) || // Instead of checking the candidates with code upgrades twice // move the checking up here and skip it in the training wheels fallback. // That way we avoid possible duplicate checks while assuring all @@ -611,13 +628,19 @@ impl Pallet { // // NOTE: this is the only place where we check the relay-parent. check_ctx - .verify_backed_candidate(&allowed_relay_parents, candidate_idx, backed_candidate) + .verify_backed_candidate(&allowed_relay_parents, candidate_idx, backed_candidate.candidate()) .is_err() - }, - &scheduled, - ); + }, + scheduled, + core_index_enabled, + ); + + ensure!( + backed_candidates_with_core.len() <= total_scheduled_cores, + Error::::UnscheduledCandidate + ); - METRICS.on_candidates_sanitized(backed_candidates.len() as u64); + METRICS.on_candidates_sanitized(backed_candidates_with_core.len() as u64); // In `Enter` context (invoked during execution) there should be no backing votes from // disabled validators because they should have been filtered out during inherent data @@ -626,15 +649,22 @@ impl Pallet { ensure!(!votes_from_disabled_were_dropped, Error::::BackedByDisabled); } + // In `Enter` context (invoked during execution) we shouldn't have filtered any candidates + // due to a para not being scheduled. They have been filtered during inherent data + // preparation (`ProvideInherent` context). Abort in such cases. + if context == ProcessInherentDataContext::Enter { + ensure!(!dropped_unscheduled_candidates, Error::::BackedOnUnscheduledCore); + } + // Process backed candidates according to scheduled cores. let inclusion::ProcessedCandidates::< as HeaderT>::Hash> { core_indices: occupied, candidate_receipt_with_backing_validator_indices, } = >::process_candidates( &allowed_relay_parents, - backed_candidates.clone(), - &scheduled, + backed_candidates_with_core.clone(), >::group_validators, + core_index_enabled, )?; // Note which of the scheduled cores were actually occupied by a backed candidate. >::occupied(occupied.into_iter().map(|e| (e.0, e.1)).collect()); @@ -651,8 +681,15 @@ impl Pallet { let bitfields = bitfields.into_iter().map(|v| v.into_unchecked()).collect(); - let processed = - ParachainsInherentData { bitfields, backed_candidates, disputes, parent_header }; + let processed = ParachainsInherentData { + bitfields, + backed_candidates: backed_candidates_with_core + .into_iter() + .map(|(candidate, _)| candidate) + .collect(), + disputes, + parent_header, + }; Ok((processed, Some(all_weight_after).into())) } } @@ -774,7 +811,7 @@ fn apply_weight_limit( .iter() .enumerate() .filter_map(|(idx, candidate)| { - candidate.candidate.commitments.new_validation_code.as_ref().map(|_code| idx) + candidate.candidate().commitments.new_validation_code.as_ref().map(|_code| idx) }) .collect::>(); @@ -916,16 +953,22 @@ pub(crate) fn sanitize_bitfields( // Result from `sanitize_backed_candidates` #[derive(Debug, PartialEq)] struct SanitizedBackedCandidates { - // Sanitized backed candidates. The `Vec` is sorted according to the occupied core index. - backed_candidates: Vec>, + // Sanitized backed candidates along with the assigned core. The `Vec` is sorted according to + // the occupied core index. + backed_candidates_with_core: Vec<(BackedCandidate, CoreIndex)>, // Set to true if any votes from disabled validators were dropped from the input. votes_from_disabled_were_dropped: bool, + // Set to true if any candidates were dropped due to filtering done in + // `map_candidates_to_cores` + dropped_unscheduled_candidates: bool, } /// Filter out: /// 1. any candidates that have a concluded invalid dispute -/// 2. all backing votes from disabled validators -/// 3. any candidates that end up with less than `effective_minimum_backing_votes` backing votes +/// 2. any unscheduled candidates, as well as candidates whose paraid has multiple cores assigned +/// but have no injected core index. +/// 3. all backing votes from disabled validators +/// 4. any candidates that end up with less than `effective_minimum_backing_votes` backing votes /// /// `scheduled` follows the same naming scheme as provided in the /// guide: Currently `free` but might become `occupied`. @@ -944,7 +987,8 @@ fn sanitize_backed_candidates< mut backed_candidates: Vec>, allowed_relay_parents: &AllowedRelayParentsTracker>, mut candidate_has_concluded_invalid_dispute_or_is_invalid: F, - scheduled: &BTreeMap, + scheduled: BTreeMap>, + core_index_enabled: bool, ) -> SanitizedBackedCandidates { // Remove any candidates that were concluded invalid. // This does not assume sorting. @@ -952,22 +996,23 @@ fn sanitize_backed_candidates< !candidate_has_concluded_invalid_dispute_or_is_invalid(candidate_idx, backed_candidate) }); - // Assure the backed candidate's `ParaId`'s core is free. - // This holds under the assumption that `Scheduler::schedule` is called _before_. - // We don't check the relay-parent because this is done in the closure when - // constructing the inherent and during actual processing otherwise. - - backed_candidates.retain(|backed_candidate| { - let desc = backed_candidate.descriptor(); + let initial_candidate_count = backed_candidates.len(); + // Map candidates to scheduled cores. Filter out any unscheduled candidates. + let mut backed_candidates_with_core = map_candidates_to_cores::( + &allowed_relay_parents, + scheduled, + core_index_enabled, + backed_candidates, + ); - scheduled.get(&desc.para_id).is_some() - }); + let dropped_unscheduled_candidates = + initial_candidate_count != backed_candidates_with_core.len(); // Filter out backing statements from disabled validators - let dropped_disabled = filter_backed_statements_from_disabled_validators::( - &mut backed_candidates, + let votes_from_disabled_were_dropped = filter_backed_statements_from_disabled_validators::( + &mut backed_candidates_with_core, &allowed_relay_parents, - scheduled, + core_index_enabled, ); // Sort the `Vec` last, once there is a guarantee that these @@ -975,14 +1020,12 @@ fn sanitize_backed_candidates< // but more importantly are scheduled for a free core. // This both avoids extra work for obviously invalid candidates, // but also allows this to be done in place. - backed_candidates.sort_by(|x, y| { - // Never panics, since we filtered all panic arguments out in the previous `fn retain`. - scheduled[&x.descriptor().para_id].cmp(&scheduled[&y.descriptor().para_id]) - }); + backed_candidates_with_core.sort_by(|(_x, core_x), (_y, core_y)| core_x.cmp(&core_y)); SanitizedBackedCandidates { - backed_candidates, - votes_from_disabled_were_dropped: dropped_disabled, + dropped_unscheduled_candidates, + votes_from_disabled_were_dropped, + backed_candidates_with_core, } } @@ -1071,9 +1114,12 @@ fn limit_and_sanitize_disputes< // few more sanity checks. Returns `true` if at least one statement is removed and `false` // otherwise. fn filter_backed_statements_from_disabled_validators( - backed_candidates: &mut Vec::Hash>>, + backed_candidates_with_core: &mut Vec<( + BackedCandidate<::Hash>, + CoreIndex, + )>, allowed_relay_parents: &AllowedRelayParentsTracker>, - scheduled: &BTreeMap, + core_index_enabled: bool, ) -> bool { let disabled_validators = BTreeSet::<_>::from_iter(shared::Pallet::::disabled_validators().into_iter()); @@ -1083,7 +1129,7 @@ fn filter_backed_statements_from_disabled_validators *core_idx, - None => { - log::debug!(target: LOG_TARGET, "Can't get core idx of a backed candidate for para id {:?}. Dropping the candidate.", bc.descriptor().para_id); - return false - } - }; + backed_candidates_with_core.retain_mut(|(bc, core_idx)| { + let (validator_indices, maybe_core_index) = bc.validator_indices_and_core_index(core_index_enabled); + let mut validator_indices = BitVec::<_>::from(validator_indices); // Get relay parent block number of the candidate. We need this to get the group index assigned to this core at this block number let relay_parent_block_number = match allowed_relay_parents @@ -1116,7 +1156,7 @@ fn filter_backed_statements_from_disabled_validators>::group_assigned_to_core( - core_idx, + *core_idx, relay_parent_block_number + One::one(), ) { Some(group_idx) => group_idx, @@ -1138,12 +1178,15 @@ fn filter_backed_statements_from_disabled_validators::from_iter(validator_group.iter().map(|idx| disabled_validators.contains(idx))); // The indices of statements from disabled validators in `BackedCandidate`. We have to drop these. - let indices_to_drop = disabled_indices.clone() & &bc.validator_indices; + let indices_to_drop = disabled_indices.clone() & &validator_indices; // Apply the bitmask to drop the disabled validator from `validator_indices` - bc.validator_indices &= !disabled_indices; + validator_indices &= !disabled_indices; + // Update the backed candidate + bc.set_validator_indices_and_core_index(validator_indices, maybe_core_index); + // Remove the corresponding votes from `validity_votes` for idx in indices_to_drop.iter_ones().rev() { - bc.validity_votes.remove(idx); + bc.validity_votes_mut().remove(idx); } // If at least one statement was dropped we need to return `true` @@ -1154,10 +1197,9 @@ fn filter_backed_statements_from_disabled_validators( + allowed_relay_parents: &AllowedRelayParentsTracker>, + mut scheduled: BTreeMap>, + core_index_enabled: bool, + candidates: Vec>, +) -> Vec<(BackedCandidate, CoreIndex)> { + let mut backed_candidates_with_core = Vec::with_capacity(candidates.len()); + + // We keep a candidate if the parachain has only one core assigned or if + // a core index is provided by block author and it's indeed scheduled. + for backed_candidate in candidates { + let maybe_injected_core_index = get_injected_core_index::( + allowed_relay_parents, + &backed_candidate, + core_index_enabled, + ); + + let scheduled_cores = scheduled.get_mut(&backed_candidate.descriptor().para_id); + // Candidates without scheduled cores are silently filtered out. + if let Some(scheduled_cores) = scheduled_cores { + if let Some(core_idx) = maybe_injected_core_index { + if scheduled_cores.contains(&core_idx) { + scheduled_cores.remove(&core_idx); + backed_candidates_with_core.push((backed_candidate, core_idx)); + } + } else if scheduled_cores.len() == 1 { + backed_candidates_with_core + .push((backed_candidate, scheduled_cores.pop_first().expect("Length is 1"))); + } + } + } + + backed_candidates_with_core +} + +fn get_injected_core_index( + allowed_relay_parents: &AllowedRelayParentsTracker>, + candidate: &BackedCandidate, + core_index_enabled: bool, +) -> Option { + // After stripping the 8 bit extensions, the `validator_indices` field length is expected + // to be equal to backing group size. If these don't match, the `CoreIndex` is badly encoded, + // or not supported. + let (validator_indices, maybe_core_idx) = + candidate.validator_indices_and_core_index(core_index_enabled); + + let Some(core_idx) = maybe_core_idx else { return None }; + + let relay_parent_block_number = + match allowed_relay_parents.acquire_info(candidate.descriptor().relay_parent, None) { + Some((_, block_num)) => block_num, + None => { + log::debug!( + target: LOG_TARGET, + "Relay parent {:?} for candidate {:?} is not in the allowed relay parents. Dropping the candidate.", + candidate.descriptor().relay_parent, + candidate.candidate().hash(), + ); + return None + }, + }; + + // Get the backing group of the candidate backed at `core_idx`. + let group_idx = match >::group_assigned_to_core( + core_idx, + relay_parent_block_number + One::one(), + ) { + Some(group_idx) => group_idx, + None => { + log::debug!( + target: LOG_TARGET, + "Can't get the group index for core idx {:?}. Dropping the candidate {:?}.", + core_idx, + candidate.candidate().hash(), + ); + return None + }, + }; + + let group_validators = match >::group_validators(group_idx) { + Some(validators) => validators, + None => return None, + }; + + if group_validators.len() == validator_indices.len() { + Some(core_idx) + } else { + None + } } diff --git a/polkadot/runtime/parachains/src/paras_inherent/tests.rs b/polkadot/runtime/parachains/src/paras_inherent/tests.rs index 6f3eac35685a..defb2f4404f5 100644 --- a/polkadot/runtime/parachains/src/paras_inherent/tests.rs +++ b/polkadot/runtime/parachains/src/paras_inherent/tests.rs @@ -26,7 +26,10 @@ mod enter { use crate::{ builder::{Bench, BenchBuilder}, mock::{mock_assigner, new_test_ext, BlockLength, BlockWeights, MockGenesisConfig, Test}, - scheduler::common::Assignment, + scheduler::{ + common::{Assignment, AssignmentProvider, AssignmentProviderConfig}, + ParasEntry, + }, }; use assert_matches::assert_matches; use frame_support::assert_ok; @@ -697,6 +700,25 @@ mod enter { 2 ); + // One core was scheduled. We should put the assignment back, before calling enter(). + let now = >::block_number() + 1; + let used_cores = 5; + let cores = (0..used_cores) + .into_iter() + .map(|i| { + let AssignmentProviderConfig { ttl, .. } = + scheduler::Pallet::::assignment_provider_config(CoreIndex(i)); + // Load an assignment into provider so that one is present to pop + let assignment = + ::AssignmentProvider::get_mock_assignment( + CoreIndex(i), + ParaId::from(i), + ); + (CoreIndex(i), [ParasEntry::new(assignment, now + ttl)].into()) + }) + .collect(); + scheduler::ClaimQueue::::set(cores); + assert_ok!(Pallet::::enter( frame_system::RawOrigin::None.into(), limit_inherent_data, @@ -980,6 +1002,7 @@ mod sanitizers { AvailabilityBitfield, GroupIndex, Hash, Id as ParaId, SignedAvailabilityBitfield, ValidatorIndex, }; + use rstest::rstest; use sp_core::crypto::UncheckedFrom; use crate::mock::Test; @@ -1238,12 +1261,13 @@ mod sanitizers { // Backed candidates and scheduled parachains used for `sanitize_backed_candidates` testing struct TestData { backed_candidates: Vec, - scheduled_paras: BTreeMap, + all_backed_candidates_with_core: Vec<(BackedCandidate, CoreIndex)>, + scheduled_paras: BTreeMap>, } // Generate test data for the candidates and assert that the evnironment is set as expected // (check the comments for details) - fn get_test_data() -> TestData { + fn get_test_data(core_index_enabled: bool) -> TestData { const RELAY_PARENT_NUM: u32 = 3; // Add the relay parent to `shared` pallet. Otherwise some code (e.g. filtering backing @@ -1285,9 +1309,14 @@ mod sanitizers { shared::Pallet::::set_active_validators_ascending(validator_ids); // Two scheduled parachains - ParaId(1) on CoreIndex(0) and ParaId(2) on CoreIndex(1) - let scheduled = (0_usize..2) + let scheduled: BTreeMap> = (0_usize..2) .into_iter() - .map(|idx| (ParaId::from(1_u32 + idx as u32), CoreIndex::from(idx as u32))) + .map(|idx| { + ( + ParaId::from(1_u32 + idx as u32), + [CoreIndex::from(idx as u32)].into_iter().collect(), + ) + }) .collect::>(); // Set the validator groups in `scheduler` @@ -1301,7 +1330,7 @@ mod sanitizers { ( CoreIndex::from(0), VecDeque::from([ParasEntry::new( - Assignment::Pool { para_id: 1.into(), core_index: CoreIndex(1) }, + Assignment::Pool { para_id: 1.into(), core_index: CoreIndex(0) }, RELAY_PARENT_NUM, )]), ), @@ -1319,12 +1348,12 @@ mod sanitizers { match group_index { group_index if group_index == GroupIndex::from(0) => Some(vec![0, 1]), group_index if group_index == GroupIndex::from(1) => Some(vec![2, 3]), - _ => panic!("Group index out of bounds for 2 parachains and 1 parathread core"), + _ => panic!("Group index out of bounds"), } .map(|m| m.into_iter().map(ValidatorIndex).collect::>()) }; - // Two backed candidates from each parachain + // One backed candidate from each parachain let backed_candidates = (0_usize..2) .into_iter() .map(|idx0| { @@ -1348,6 +1377,7 @@ mod sanitizers { &keystore, &signing_context, BackingKind::Threshold, + core_index_enabled.then_some(CoreIndex(idx0 as u32)), ); backed }) @@ -1369,13 +1399,373 @@ mod sanitizers { ] ); - TestData { backed_candidates, scheduled_paras: scheduled } + let all_backed_candidates_with_core = backed_candidates + .iter() + .map(|candidate| { + // Only one entry for this test data. + ( + candidate.clone(), + scheduled + .get(&candidate.descriptor().para_id) + .unwrap() + .first() + .copied() + .unwrap(), + ) + }) + .collect(); + + TestData { + backed_candidates, + scheduled_paras: scheduled, + all_backed_candidates_with_core, + } } - #[test] - fn happy_path() { + // Generate test data for the candidates and assert that the evnironment is set as expected + // (check the comments for details) + // Para 1 scheduled on core 0 and core 1. Two candidates are supplied. + // Para 2 scheduled on cores 2 and 3. One candidate supplied. + // Para 3 scheduled on core 4. One candidate supplied. + // Para 4 scheduled on core 5. Two candidates supplied. + // Para 5 scheduled on core 6. No candidates supplied. + fn get_test_data_multiple_cores_per_para(core_index_enabled: bool) -> TestData { + const RELAY_PARENT_NUM: u32 = 3; + + // Add the relay parent to `shared` pallet. Otherwise some code (e.g. filtering backing + // votes) won't behave correctly + shared::Pallet::::add_allowed_relay_parent( + default_header().hash(), + Default::default(), + RELAY_PARENT_NUM, + 1, + ); + + let header = default_header(); + let relay_parent = header.hash(); + let session_index = SessionIndex::from(0_u32); + + let keystore = LocalKeystore::in_memory(); + let keystore = Arc::new(keystore) as KeystorePtr; + let signing_context = SigningContext { parent_hash: relay_parent, session_index }; + + let validators = vec![ + keyring::Sr25519Keyring::Alice, + keyring::Sr25519Keyring::Bob, + keyring::Sr25519Keyring::Charlie, + keyring::Sr25519Keyring::Dave, + keyring::Sr25519Keyring::Eve, + keyring::Sr25519Keyring::Ferdie, + keyring::Sr25519Keyring::One, + ]; + for validator in validators.iter() { + Keystore::sr25519_generate_new( + &*keystore, + PARACHAIN_KEY_TYPE_ID, + Some(&validator.to_seed()), + ) + .unwrap(); + } + + // Set active validators in `shared` pallet + let validator_ids = + validators.iter().map(|v| v.public().into()).collect::>(); + shared::Pallet::::set_active_validators_ascending(validator_ids); + + // Set the validator groups in `scheduler` + scheduler::Pallet::::set_validator_groups(vec![ + vec![ValidatorIndex(0)], + vec![ValidatorIndex(1)], + vec![ValidatorIndex(2)], + vec![ValidatorIndex(3)], + vec![ValidatorIndex(4)], + vec![ValidatorIndex(5)], + vec![ValidatorIndex(6)], + ]); + + // Update scheduler's claimqueue with the parachains + scheduler::Pallet::::set_claimqueue(BTreeMap::from([ + ( + CoreIndex::from(0), + VecDeque::from([ParasEntry::new( + Assignment::Pool { para_id: 1.into(), core_index: CoreIndex(0) }, + RELAY_PARENT_NUM, + )]), + ), + ( + CoreIndex::from(1), + VecDeque::from([ParasEntry::new( + Assignment::Pool { para_id: 1.into(), core_index: CoreIndex(1) }, + RELAY_PARENT_NUM, + )]), + ), + ( + CoreIndex::from(2), + VecDeque::from([ParasEntry::new( + Assignment::Pool { para_id: 2.into(), core_index: CoreIndex(2) }, + RELAY_PARENT_NUM, + )]), + ), + ( + CoreIndex::from(3), + VecDeque::from([ParasEntry::new( + Assignment::Pool { para_id: 2.into(), core_index: CoreIndex(3) }, + RELAY_PARENT_NUM, + )]), + ), + ( + CoreIndex::from(4), + VecDeque::from([ParasEntry::new( + Assignment::Pool { para_id: 3.into(), core_index: CoreIndex(4) }, + RELAY_PARENT_NUM, + )]), + ), + ( + CoreIndex::from(5), + VecDeque::from([ParasEntry::new( + Assignment::Pool { para_id: 4.into(), core_index: CoreIndex(5) }, + RELAY_PARENT_NUM, + )]), + ), + ( + CoreIndex::from(6), + VecDeque::from([ParasEntry::new( + Assignment::Pool { para_id: 5.into(), core_index: CoreIndex(6) }, + RELAY_PARENT_NUM, + )]), + ), + ])); + + // Callback used for backing candidates + let group_validators = |group_index: GroupIndex| { + match group_index { + group_index if group_index == GroupIndex::from(0) => Some(vec![0]), + group_index if group_index == GroupIndex::from(1) => Some(vec![1]), + group_index if group_index == GroupIndex::from(2) => Some(vec![2]), + group_index if group_index == GroupIndex::from(3) => Some(vec![3]), + group_index if group_index == GroupIndex::from(4) => Some(vec![4]), + group_index if group_index == GroupIndex::from(5) => Some(vec![5]), + group_index if group_index == GroupIndex::from(6) => Some(vec![6]), + + _ => panic!("Group index out of bounds"), + } + .map(|m| m.into_iter().map(ValidatorIndex).collect::>()) + }; + + let mut backed_candidates = vec![]; + let mut all_backed_candidates_with_core = vec![]; + + // Para 1 + { + let mut candidate = TestCandidateBuilder { + para_id: ParaId::from(1), + relay_parent, + pov_hash: Hash::repeat_byte(1 as u8), + persisted_validation_data_hash: [42u8; 32].into(), + hrmp_watermark: RELAY_PARENT_NUM, + ..Default::default() + } + .build(); + + collator_sign_candidate(Sr25519Keyring::One, &mut candidate); + + let backed: BackedCandidate = back_candidate( + candidate, + &validators, + group_validators(GroupIndex::from(0 as u32)).unwrap().as_ref(), + &keystore, + &signing_context, + BackingKind::Threshold, + core_index_enabled.then_some(CoreIndex(0 as u32)), + ); + backed_candidates.push(backed.clone()); + if core_index_enabled { + all_backed_candidates_with_core.push((backed, CoreIndex(0))); + } + + let mut candidate = TestCandidateBuilder { + para_id: ParaId::from(1), + relay_parent, + pov_hash: Hash::repeat_byte(2 as u8), + persisted_validation_data_hash: [42u8; 32].into(), + hrmp_watermark: RELAY_PARENT_NUM, + ..Default::default() + } + .build(); + + collator_sign_candidate(Sr25519Keyring::One, &mut candidate); + + let backed = back_candidate( + candidate, + &validators, + group_validators(GroupIndex::from(1 as u32)).unwrap().as_ref(), + &keystore, + &signing_context, + BackingKind::Threshold, + core_index_enabled.then_some(CoreIndex(1 as u32)), + ); + backed_candidates.push(backed.clone()); + if core_index_enabled { + all_backed_candidates_with_core.push((backed, CoreIndex(1))); + } + } + + // Para 2 + { + let mut candidate = TestCandidateBuilder { + para_id: ParaId::from(2), + relay_parent, + pov_hash: Hash::repeat_byte(3 as u8), + persisted_validation_data_hash: [42u8; 32].into(), + hrmp_watermark: RELAY_PARENT_NUM, + ..Default::default() + } + .build(); + + collator_sign_candidate(Sr25519Keyring::One, &mut candidate); + + let backed = back_candidate( + candidate, + &validators, + group_validators(GroupIndex::from(2 as u32)).unwrap().as_ref(), + &keystore, + &signing_context, + BackingKind::Threshold, + core_index_enabled.then_some(CoreIndex(2 as u32)), + ); + backed_candidates.push(backed.clone()); + if core_index_enabled { + all_backed_candidates_with_core.push((backed, CoreIndex(2))); + } + } + + // Para 3 + { + let mut candidate = TestCandidateBuilder { + para_id: ParaId::from(3), + relay_parent, + pov_hash: Hash::repeat_byte(4 as u8), + persisted_validation_data_hash: [42u8; 32].into(), + hrmp_watermark: RELAY_PARENT_NUM, + ..Default::default() + } + .build(); + + collator_sign_candidate(Sr25519Keyring::One, &mut candidate); + + let backed = back_candidate( + candidate, + &validators, + group_validators(GroupIndex::from(4 as u32)).unwrap().as_ref(), + &keystore, + &signing_context, + BackingKind::Threshold, + core_index_enabled.then_some(CoreIndex(4 as u32)), + ); + backed_candidates.push(backed.clone()); + all_backed_candidates_with_core.push((backed, CoreIndex(4))); + } + + // Para 4 + { + let mut candidate = TestCandidateBuilder { + para_id: ParaId::from(4), + relay_parent, + pov_hash: Hash::repeat_byte(5 as u8), + persisted_validation_data_hash: [42u8; 32].into(), + hrmp_watermark: RELAY_PARENT_NUM, + ..Default::default() + } + .build(); + + collator_sign_candidate(Sr25519Keyring::One, &mut candidate); + + let backed = back_candidate( + candidate, + &validators, + group_validators(GroupIndex::from(5 as u32)).unwrap().as_ref(), + &keystore, + &signing_context, + BackingKind::Threshold, + None, + ); + backed_candidates.push(backed.clone()); + all_backed_candidates_with_core.push((backed, CoreIndex(5))); + + let mut candidate = TestCandidateBuilder { + para_id: ParaId::from(4), + relay_parent, + pov_hash: Hash::repeat_byte(6 as u8), + persisted_validation_data_hash: [42u8; 32].into(), + hrmp_watermark: RELAY_PARENT_NUM, + ..Default::default() + } + .build(); + + collator_sign_candidate(Sr25519Keyring::One, &mut candidate); + + let backed = back_candidate( + candidate, + &validators, + group_validators(GroupIndex::from(5 as u32)).unwrap().as_ref(), + &keystore, + &signing_context, + BackingKind::Threshold, + core_index_enabled.then_some(CoreIndex(5 as u32)), + ); + backed_candidates.push(backed.clone()); + } + + // No candidate for para 5. + + // State sanity checks + assert_eq!( + >::scheduled_paras().collect::>(), + vec![ + (CoreIndex(0), ParaId::from(1)), + (CoreIndex(1), ParaId::from(1)), + (CoreIndex(2), ParaId::from(2)), + (CoreIndex(3), ParaId::from(2)), + (CoreIndex(4), ParaId::from(3)), + (CoreIndex(5), ParaId::from(4)), + (CoreIndex(6), ParaId::from(5)), + ] + ); + let mut scheduled: BTreeMap> = BTreeMap::new(); + for (core_idx, para_id) in >::scheduled_paras() { + scheduled.entry(para_id).or_default().insert(core_idx); + } + + assert_eq!( + shared::Pallet::::active_validator_indices(), + vec![ + ValidatorIndex(0), + ValidatorIndex(1), + ValidatorIndex(2), + ValidatorIndex(3), + ValidatorIndex(4), + ValidatorIndex(5), + ValidatorIndex(6), + ] + ); + + TestData { + backed_candidates, + scheduled_paras: scheduled, + all_backed_candidates_with_core, + } + } + + #[rstest] + #[case(false)] + #[case(true)] + fn happy_path(#[case] core_index_enabled: bool) { new_test_ext(MockGenesisConfig::default()).execute_with(|| { - let TestData { backed_candidates, scheduled_paras: scheduled } = get_test_data(); + let TestData { + backed_candidates, + all_backed_candidates_with_core, + scheduled_paras: scheduled, + } = get_test_data(core_index_enabled); let has_concluded_invalid = |_idx: usize, _backed_candidate: &BackedCandidate| -> bool { false }; @@ -1385,47 +1775,95 @@ mod sanitizers { backed_candidates.clone(), &>::allowed_relay_parents(), has_concluded_invalid, - &scheduled + scheduled, + core_index_enabled ), SanitizedBackedCandidates { - backed_candidates, - votes_from_disabled_were_dropped: false + backed_candidates_with_core: all_backed_candidates_with_core, + votes_from_disabled_were_dropped: false, + dropped_unscheduled_candidates: false } ); + }); + } + + #[rstest] + #[case(false)] + #[case(true)] + fn test_with_multiple_cores_per_para(#[case] core_index_enabled: bool) { + new_test_ext(MockGenesisConfig::default()).execute_with(|| { + let TestData { + backed_candidates, + all_backed_candidates_with_core: expected_all_backed_candidates_with_core, + scheduled_paras: scheduled, + } = get_test_data_multiple_cores_per_para(core_index_enabled); + + let has_concluded_invalid = + |_idx: usize, _backed_candidate: &BackedCandidate| -> bool { false }; - {} + assert_eq!( + sanitize_backed_candidates::( + backed_candidates.clone(), + &>::allowed_relay_parents(), + has_concluded_invalid, + scheduled, + core_index_enabled + ), + SanitizedBackedCandidates { + backed_candidates_with_core: expected_all_backed_candidates_with_core, + votes_from_disabled_were_dropped: false, + dropped_unscheduled_candidates: true + } + ); }); } // nothing is scheduled, so no paraids match, thus all backed candidates are skipped - #[test] - fn nothing_scheduled() { + #[rstest] + #[case(false, false)] + #[case(true, true)] + #[case(false, true)] + #[case(true, false)] + fn nothing_scheduled( + #[case] core_index_enabled: bool, + #[case] multiple_cores_per_para: bool, + ) { new_test_ext(MockGenesisConfig::default()).execute_with(|| { - let TestData { backed_candidates, scheduled_paras: _ } = get_test_data(); - let scheduled = &BTreeMap::new(); + let TestData { backed_candidates, .. } = if multiple_cores_per_para { + get_test_data_multiple_cores_per_para(core_index_enabled) + } else { + get_test_data(core_index_enabled) + }; + let scheduled = BTreeMap::new(); let has_concluded_invalid = |_idx: usize, _backed_candidate: &BackedCandidate| -> bool { false }; let SanitizedBackedCandidates { - backed_candidates: sanitized_backed_candidates, + backed_candidates_with_core: sanitized_backed_candidates, votes_from_disabled_were_dropped, + dropped_unscheduled_candidates, } = sanitize_backed_candidates::( backed_candidates.clone(), &>::allowed_relay_parents(), has_concluded_invalid, - &scheduled, + scheduled, + core_index_enabled, ); assert!(sanitized_backed_candidates.is_empty()); assert!(!votes_from_disabled_were_dropped); + assert!(dropped_unscheduled_candidates); }); } // candidates that have concluded as invalid are filtered out - #[test] - fn invalid_are_filtered_out() { + #[rstest] + #[case(false)] + #[case(true)] + fn invalid_are_filtered_out(#[case] core_index_enabled: bool) { new_test_ext(MockGenesisConfig::default()).execute_with(|| { - let TestData { backed_candidates, scheduled_paras: scheduled } = get_test_data(); + let TestData { backed_candidates, scheduled_paras: scheduled, .. } = + get_test_data(core_index_enabled); // mark every second one as concluded invalid let set = { @@ -1440,45 +1878,55 @@ mod sanitizers { let has_concluded_invalid = |_idx: usize, candidate: &BackedCandidate| set.contains(&candidate.hash()); let SanitizedBackedCandidates { - backed_candidates: sanitized_backed_candidates, + backed_candidates_with_core: sanitized_backed_candidates, votes_from_disabled_were_dropped, + dropped_unscheduled_candidates, } = sanitize_backed_candidates::( backed_candidates.clone(), &>::allowed_relay_parents(), has_concluded_invalid, - &scheduled, + scheduled, + core_index_enabled, ); assert_eq!(sanitized_backed_candidates.len(), backed_candidates.len() / 2); assert!(!votes_from_disabled_were_dropped); + assert!(!dropped_unscheduled_candidates); }); } - #[test] - fn disabled_non_signing_validator_doesnt_get_filtered() { + #[rstest] + #[case(false)] + #[case(true)] + fn disabled_non_signing_validator_doesnt_get_filtered(#[case] core_index_enabled: bool) { new_test_ext(MockGenesisConfig::default()).execute_with(|| { - let TestData { mut backed_candidates, scheduled_paras } = get_test_data(); + let TestData { mut all_backed_candidates_with_core, .. } = + get_test_data(core_index_enabled); // Disable Eve set_disabled_validators(vec![4]); - let before = backed_candidates.clone(); + let before = all_backed_candidates_with_core.clone(); // Eve is disabled but no backing statement is signed by it so nothing should be // filtered assert!(!filter_backed_statements_from_disabled_validators::( - &mut backed_candidates, + &mut all_backed_candidates_with_core, &>::allowed_relay_parents(), - &scheduled_paras + core_index_enabled )); - assert_eq!(backed_candidates, before); + assert_eq!(all_backed_candidates_with_core, before); }); } - - #[test] - fn drop_statements_from_disabled_without_dropping_candidate() { + #[rstest] + #[case(false)] + #[case(true)] + fn drop_statements_from_disabled_without_dropping_candidate( + #[case] core_index_enabled: bool, + ) { new_test_ext(MockGenesisConfig::default()).execute_with(|| { - let TestData { mut backed_candidates, scheduled_paras } = get_test_data(); + let TestData { mut all_backed_candidates_with_core, .. } = + get_test_data(core_index_enabled); // Disable Alice set_disabled_validators(vec![0]); @@ -1491,61 +1939,83 @@ mod sanitizers { configuration::Pallet::::force_set_active_config(hc); // Verify the initial state is as expected - assert_eq!(backed_candidates.get(0).unwrap().validity_votes.len(), 2); assert_eq!( - backed_candidates.get(0).unwrap().validator_indices.get(0).unwrap(), - true + all_backed_candidates_with_core.get(0).unwrap().0.validity_votes().len(), + 2 ); - assert_eq!( - backed_candidates.get(0).unwrap().validator_indices.get(1).unwrap(), - true - ); - let untouched = backed_candidates.get(1).unwrap().clone(); + let (validator_indices, maybe_core_index) = all_backed_candidates_with_core + .get(0) + .unwrap() + .0 + .validator_indices_and_core_index(core_index_enabled); + if core_index_enabled { + assert!(maybe_core_index.is_some()); + } else { + assert!(maybe_core_index.is_none()); + } + + assert_eq!(validator_indices.get(0).unwrap(), true); + assert_eq!(validator_indices.get(1).unwrap(), true); + let untouched = all_backed_candidates_with_core.get(1).unwrap().0.clone(); assert!(filter_backed_statements_from_disabled_validators::( - &mut backed_candidates, + &mut all_backed_candidates_with_core, &>::allowed_relay_parents(), - &scheduled_paras + core_index_enabled )); + let (validator_indices, maybe_core_index) = all_backed_candidates_with_core + .get(0) + .unwrap() + .0 + .validator_indices_and_core_index(core_index_enabled); + if core_index_enabled { + assert!(maybe_core_index.is_some()); + } else { + assert!(maybe_core_index.is_none()); + } + // there should still be two backed candidates - assert_eq!(backed_candidates.len(), 2); + assert_eq!(all_backed_candidates_with_core.len(), 2); // but the first one should have only one validity vote - assert_eq!(backed_candidates.get(0).unwrap().validity_votes.len(), 1); - // Validator 0 vote should be dropped, validator 1 - retained assert_eq!( - backed_candidates.get(0).unwrap().validator_indices.get(0).unwrap(), - false - ); - assert_eq!( - backed_candidates.get(0).unwrap().validator_indices.get(1).unwrap(), - true + all_backed_candidates_with_core.get(0).unwrap().0.validity_votes().len(), + 1 ); + // Validator 0 vote should be dropped, validator 1 - retained + assert_eq!(validator_indices.get(0).unwrap(), false); + assert_eq!(validator_indices.get(1).unwrap(), true); // the second candidate shouldn't be modified - assert_eq!(*backed_candidates.get(1).unwrap(), untouched); + assert_eq!(all_backed_candidates_with_core.get(1).unwrap().0, untouched); }); } - #[test] - fn drop_candidate_if_all_statements_are_from_disabled() { + #[rstest] + #[case(false)] + #[case(true)] + fn drop_candidate_if_all_statements_are_from_disabled(#[case] core_index_enabled: bool) { new_test_ext(MockGenesisConfig::default()).execute_with(|| { - let TestData { mut backed_candidates, scheduled_paras } = get_test_data(); + let TestData { mut all_backed_candidates_with_core, .. } = + get_test_data(core_index_enabled); // Disable Alice and Bob set_disabled_validators(vec![0, 1]); // Verify the initial state is as expected - assert_eq!(backed_candidates.get(0).unwrap().validity_votes.len(), 2); - let untouched = backed_candidates.get(1).unwrap().clone(); + assert_eq!( + all_backed_candidates_with_core.get(0).unwrap().0.validity_votes().len(), + 2 + ); + let untouched = all_backed_candidates_with_core.get(1).unwrap().0.clone(); assert!(filter_backed_statements_from_disabled_validators::( - &mut backed_candidates, + &mut all_backed_candidates_with_core, &>::allowed_relay_parents(), - &scheduled_paras + core_index_enabled )); - assert_eq!(backed_candidates.len(), 1); - assert_eq!(*backed_candidates.get(0).unwrap(), untouched); + assert_eq!(all_backed_candidates_with_core.len(), 1); + assert_eq!(all_backed_candidates_with_core.get(0).unwrap().0, untouched); }); } } diff --git a/polkadot/runtime/parachains/src/paras_inherent/weights.rs b/polkadot/runtime/parachains/src/paras_inherent/weights.rs index 05cc53fae046..0f4e5be572a6 100644 --- a/polkadot/runtime/parachains/src/paras_inherent/weights.rs +++ b/polkadot/runtime/parachains/src/paras_inherent/weights.rs @@ -149,11 +149,11 @@ pub fn backed_candidate_weight( candidate: &BackedCandidate, ) -> Weight { set_proof_size_to_tx_size( - if candidate.candidate.commitments.new_validation_code.is_some() { + if candidate.candidate().commitments.new_validation_code.is_some() { <::WeightInfo as WeightInfo>::enter_backed_candidate_code_upgrade() } else { <::WeightInfo as WeightInfo>::enter_backed_candidates_variable( - candidate.validity_votes.len() as u32, + candidate.validity_votes().len() as u32, ) }, candidate, diff --git a/polkadot/runtime/rococo/src/lib.rs b/polkadot/runtime/rococo/src/lib.rs index 03e96ab388b7..1566911fc228 100644 --- a/polkadot/runtime/rococo/src/lib.rs +++ b/polkadot/runtime/rococo/src/lib.rs @@ -46,9 +46,8 @@ use sp_std::{cmp::Ordering, collections::btree_map::BTreeMap, prelude::*}; use runtime_parachains::{ assigner_coretime as parachains_assigner_coretime, - assigner_on_demand as parachains_assigner_on_demand, - assigner_parachains as parachains_assigner_parachains, - configuration as parachains_configuration, coretime, disputes as parachains_disputes, + assigner_on_demand as parachains_assigner_on_demand, configuration as parachains_configuration, + coretime, disputes as parachains_disputes, disputes::slashing as parachains_slashing, dmp as parachains_dmp, hrmp as parachains_hrmp, inclusion as parachains_inclusion, inclusion::{AggregateMessageOrigin, UmpQueueId}, @@ -1038,8 +1037,6 @@ impl parachains_assigner_on_demand::Config for Runtime { type WeightInfo = weights::runtime_parachains_assigner_on_demand::WeightInfo; } -impl parachains_assigner_parachains::Config for Runtime {} - impl parachains_assigner_coretime::Config for Runtime {} impl parachains_initializer::Config for Runtime { @@ -1401,7 +1398,6 @@ construct_runtime! { ParasSlashing: parachains_slashing = 63, MessageQueue: pallet_message_queue = 64, OnDemandAssignmentProvider: parachains_assigner_on_demand = 66, - ParachainsAssignmentProvider: parachains_assigner_parachains = 67, CoretimeAssignmentProvider: parachains_assigner_coretime = 68, // Parachain Onboarding Pallets. Start indices at 70 to leave room. diff --git a/polkadot/runtime/test-runtime/src/xcm_config.rs b/polkadot/runtime/test-runtime/src/xcm_config.rs index a81d35f4d788..a48bca17e9ff 100644 --- a/polkadot/runtime/test-runtime/src/xcm_config.rs +++ b/polkadot/runtime/test-runtime/src/xcm_config.rs @@ -16,14 +16,16 @@ use frame_support::{ parameter_types, - traits::{Everything, Nothing}, + traits::{Everything, Get, Nothing}, weights::Weight, }; use frame_system::EnsureRoot; +use polkadot_runtime_parachains::FeeTracker; +use runtime_common::xcm_sender::{ChildParachainRouter, PriceForMessageDelivery}; use xcm::latest::prelude::*; use xcm_builder::{ AllowUnpaidExecutionFrom, EnsureXcmOrigin, FixedWeightBounds, FrameTransactionalProcessor, - SignedAccountId32AsNative, SignedToAccountId32, + SignedAccountId32AsNative, SignedToAccountId32, WithUniqueTopic, }; use xcm_executor::{ traits::{TransactAsset, WeightTrader}, @@ -36,6 +38,8 @@ parameter_types! { pub const MaxInstructions: u32 = 100; pub const MaxAssetsIntoHolding: u32 = 16; pub const UniversalLocation: xcm::latest::InteriorLocation = xcm::latest::Junctions::Here; + pub TokenLocation: Location = Here.into_location(); + pub FeeAssetId: AssetId = AssetId(TokenLocation::get()); } /// Type to convert an `Origin` type value into a `Location` value which represents an interior @@ -45,17 +49,43 @@ pub type LocalOriginToLocation = ( SignedToAccountId32, ); -pub struct DoNothingRouter; -impl SendXcm for DoNothingRouter { - type Ticket = (); - fn validate(_dest: &mut Option, _msg: &mut Option>) -> SendResult<()> { - Ok(((), Assets::new())) - } - fn deliver(_: ()) -> Result { - Ok([0; 32]) +/// Implementation of [`PriceForMessageDelivery`], returning a different price +/// based on whether a message contains a reanchored asset or not. +/// This implementation ensures that messages with non-reanchored assets return higher +/// prices than messages with reanchored assets. +/// Useful for `deposit_reserve_asset_works_for_any_xcm_sender` integration test. +pub struct TestDeliveryPrice(sp_std::marker::PhantomData<(A, F)>); +impl, F: FeeTracker> PriceForMessageDelivery for TestDeliveryPrice { + type Id = F::Id; + + fn price_for_delivery(_: Self::Id, msg: &Xcm<()>) -> Assets { + let base_fee: super::Balance = 1_000_000; + + let parents = msg.iter().find_map(|xcm| match xcm { + ReserveAssetDeposited(assets) => { + let AssetId(location) = &assets.inner().first().unwrap().id; + Some(location.parents) + }, + _ => None, + }); + + // If no asset is found, price defaults to `base_fee`. + let amount = base_fee + .saturating_add(base_fee.saturating_mul(parents.unwrap_or(0) as super::Balance)); + + (A::get(), amount).into() } } +pub type PriceForChildParachainDelivery = TestDeliveryPrice; + +/// The XCM router. When we want to send an XCM message, we use this type. It amalgamates all of our +/// individual routers. +pub type XcmRouter = WithUniqueTopic< + // Only one router so far - use DMP to communicate with child parachains. + ChildParachainRouter, +>; + pub type Barrier = AllowUnpaidExecutionFrom; pub struct DummyAssetTransactor; @@ -99,7 +129,7 @@ type OriginConverter = ( pub struct XcmConfig; impl xcm_executor::Config for XcmConfig { type RuntimeCall = super::RuntimeCall; - type XcmSender = DoNothingRouter; + type XcmSender = XcmRouter; type AssetTransactor = DummyAssetTransactor; type OriginConverter = OriginConverter; type IsReserve = (); @@ -133,7 +163,7 @@ impl pallet_xcm::Config for crate::Runtime { type UniversalLocation = UniversalLocation; type SendXcmOrigin = EnsureXcmOrigin; type Weigher = FixedWeightBounds; - type XcmRouter = DoNothingRouter; + type XcmRouter = XcmRouter; type XcmExecuteFilter = Everything; type XcmExecutor = xcm_executor::XcmExecutor; type XcmTeleportFilter = Everything; diff --git a/polkadot/runtime/westend/src/lib.rs b/polkadot/runtime/westend/src/lib.rs index bbb010f60bff..b0b70bff8de9 100644 --- a/polkadot/runtime/westend/src/lib.rs +++ b/polkadot/runtime/westend/src/lib.rs @@ -68,9 +68,8 @@ use runtime_common::{ }; use runtime_parachains::{ assigner_coretime as parachains_assigner_coretime, - assigner_on_demand as parachains_assigner_on_demand, - assigner_parachains as parachains_assigner_parachains, - configuration as parachains_configuration, coretime, disputes as parachains_disputes, + assigner_on_demand as parachains_assigner_on_demand, configuration as parachains_configuration, + coretime, disputes as parachains_disputes, disputes::slashing as parachains_slashing, dmp as parachains_dmp, hrmp as parachains_hrmp, inclusion as parachains_inclusion, inclusion::{AggregateMessageOrigin, UmpQueueId}, @@ -1243,8 +1242,6 @@ impl parachains_assigner_on_demand::Config for Runtime { type WeightInfo = weights::runtime_parachains_assigner_on_demand::WeightInfo; } -impl parachains_assigner_parachains::Config for Runtime {} - impl parachains_assigner_coretime::Config for Runtime {} impl parachains_initializer::Config for Runtime { @@ -1505,7 +1502,6 @@ construct_runtime! { ParaSessionInfo: parachains_session_info = 52, ParasDisputes: parachains_disputes = 53, ParasSlashing: parachains_slashing = 54, - ParachainsAssignmentProvider: parachains_assigner_parachains = 55, OnDemandAssignmentProvider: parachains_assigner_on_demand = 56, CoretimeAssignmentProvider: parachains_assigner_coretime = 57, diff --git a/polkadot/statement-table/Cargo.toml b/polkadot/statement-table/Cargo.toml index 6403b822ed9b..37b8a99d640a 100644 --- a/polkadot/statement-table/Cargo.toml +++ b/polkadot/statement-table/Cargo.toml @@ -13,3 +13,4 @@ workspace = true parity-scale-codec = { version = "3.6.1", default-features = false, features = ["derive"] } sp-core = { path = "../../substrate/primitives/core" } primitives = { package = "polkadot-primitives", path = "../primitives" } +gum = { package = "tracing-gum", path = "../node/gum" } diff --git a/polkadot/statement-table/src/generic.rs b/polkadot/statement-table/src/generic.rs index 22bffde5acc1..2ee6f6a4f781 100644 --- a/polkadot/statement-table/src/generic.rs +++ b/polkadot/statement-table/src/generic.rs @@ -36,6 +36,7 @@ use primitives::{ }; use parity_scale_codec::{Decode, Encode}; +const LOG_TARGET: &str = "parachain::statement-table"; /// Context for the statement table. pub trait Context { @@ -53,9 +54,6 @@ pub trait Context { /// get the digest of a candidate. fn candidate_digest(candidate: &Self::Candidate) -> Self::Digest; - /// get the group of a candidate. - fn candidate_group(candidate: &Self::Candidate) -> Self::GroupId; - /// Whether a authority is a member of a group. /// Members are meant to submit candidates and vote on validity. fn is_member_of(&self, authority: &Self::AuthorityId, group: &Self::GroupId) -> bool; @@ -342,13 +340,13 @@ impl Table { pub fn import_statement( &mut self, context: &Ctx, + group_id: Ctx::GroupId, statement: SignedStatement, ) -> Option> { let SignedStatement { statement, signature, sender: signer } = statement; - let res = match statement { Statement::Seconded(candidate) => - self.import_candidate(context, signer.clone(), candidate, signature), + self.import_candidate(context, signer.clone(), candidate, signature, group_id), Statement::Valid(digest) => self.validity_vote(context, signer.clone(), digest, ValidityVote::Valid(signature)), }; @@ -387,9 +385,10 @@ impl Table { authority: Ctx::AuthorityId, candidate: Ctx::Candidate, signature: Ctx::Signature, + group: Ctx::GroupId, ) -> ImportResult { - let group = Ctx::candidate_group(&candidate); if !context.is_member_of(&authority, &group) { + gum::debug!(target: LOG_TARGET, authority = ?authority, group = ?group, "New `Misbehavior::UnauthorizedStatement`, candidate backed by validator that doesn't belong to expected group" ); return Err(Misbehavior::UnauthorizedStatement(UnauthorizedStatement { statement: SignedStatement { signature, @@ -634,10 +633,6 @@ mod tests { Digest(candidate.1) } - fn candidate_group(candidate: &Candidate) -> GroupId { - GroupId(candidate.0) - } - fn is_member_of(&self, authority: &AuthorityId, group: &GroupId) -> bool { self.authorities.get(authority).map(|v| v == group).unwrap_or(false) } @@ -675,10 +670,10 @@ mod tests { sender: AuthorityId(1), }; - table.import_statement(&context, statement_a); + table.import_statement(&context, GroupId(2), statement_a); assert!(!table.detected_misbehavior.contains_key(&AuthorityId(1))); - table.import_statement(&context, statement_b); + table.import_statement(&context, GroupId(2), statement_b); assert_eq!( table.detected_misbehavior[&AuthorityId(1)][0], Misbehavior::MultipleCandidates(MultipleCandidates { @@ -711,10 +706,10 @@ mod tests { sender: AuthorityId(1), }; - table.import_statement(&context, statement_a); + table.import_statement(&context, GroupId(2), statement_a); assert!(!table.detected_misbehavior.contains_key(&AuthorityId(1))); - table.import_statement(&context, statement_b); + table.import_statement(&context, GroupId(2), statement_b); assert!(!table.detected_misbehavior.contains_key(&AuthorityId(1))); } @@ -735,7 +730,7 @@ mod tests { sender: AuthorityId(1), }; - table.import_statement(&context, statement); + table.import_statement(&context, GroupId(2), statement); assert_eq!( table.detected_misbehavior[&AuthorityId(1)][0], @@ -769,7 +764,7 @@ mod tests { }; let candidate_a_digest = Digest(100); - table.import_statement(&context, candidate_a); + table.import_statement(&context, GroupId(2), candidate_a); assert!(!table.detected_misbehavior.contains_key(&AuthorityId(1))); assert!(!table.detected_misbehavior.contains_key(&AuthorityId(2))); @@ -779,7 +774,7 @@ mod tests { signature: Signature(2), sender: AuthorityId(2), }; - table.import_statement(&context, bad_validity_vote); + table.import_statement(&context, GroupId(3), bad_validity_vote); assert_eq!( table.detected_misbehavior[&AuthorityId(2)][0], @@ -811,7 +806,7 @@ mod tests { sender: AuthorityId(1), }; - table.import_statement(&context, statement); + table.import_statement(&context, GroupId(2), statement); assert!(!table.detected_misbehavior.contains_key(&AuthorityId(1))); let invalid_statement = SignedStatement { @@ -820,7 +815,7 @@ mod tests { sender: AuthorityId(1), }; - table.import_statement(&context, invalid_statement); + table.import_statement(&context, GroupId(2), invalid_statement); assert!(table.detected_misbehavior.contains_key(&AuthorityId(1))); } @@ -842,7 +837,7 @@ mod tests { }; let candidate_digest = Digest(100); - table.import_statement(&context, statement); + table.import_statement(&context, GroupId(2), statement); assert!(!table.detected_misbehavior.contains_key(&AuthorityId(1))); let extra_vote = SignedStatement { @@ -851,7 +846,7 @@ mod tests { sender: AuthorityId(1), }; - table.import_statement(&context, extra_vote); + table.import_statement(&context, GroupId(2), extra_vote); assert_eq!( table.detected_misbehavior[&AuthorityId(1)][0], Misbehavior::ValidityDoubleVote(ValidityDoubleVote::IssuedAndValidity( @@ -910,7 +905,7 @@ mod tests { }; let candidate_digest = Digest(100); - table.import_statement(&context, statement); + table.import_statement(&context, GroupId(2), statement); assert!(!table.detected_misbehavior.contains_key(&AuthorityId(1))); assert!(table.attested_candidate(&candidate_digest, &context, 2).is_none()); @@ -921,7 +916,7 @@ mod tests { sender: AuthorityId(2), }; - table.import_statement(&context, vote); + table.import_statement(&context, GroupId(2), vote); assert!(!table.detected_misbehavior.contains_key(&AuthorityId(2))); assert!(table.attested_candidate(&candidate_digest, &context, 2).is_some()); } @@ -944,7 +939,7 @@ mod tests { }; let summary = table - .import_statement(&context, statement) + .import_statement(&context, GroupId(2), statement) .expect("candidate import to give summary"); assert_eq!(summary.candidate, Digest(100)); @@ -971,7 +966,7 @@ mod tests { }; let candidate_digest = Digest(100); - table.import_statement(&context, statement); + table.import_statement(&context, GroupId(2), statement); assert!(!table.detected_misbehavior.contains_key(&AuthorityId(1))); let vote = SignedStatement { @@ -980,8 +975,9 @@ mod tests { sender: AuthorityId(2), }; - let summary = - table.import_statement(&context, vote).expect("candidate vote to give summary"); + let summary = table + .import_statement(&context, GroupId(2), vote) + .expect("candidate vote to give summary"); assert!(!table.detected_misbehavior.contains_key(&AuthorityId(2))); diff --git a/polkadot/statement-table/src/lib.rs b/polkadot/statement-table/src/lib.rs index d4629330ac01..3740d15cc4f3 100644 --- a/polkadot/statement-table/src/lib.rs +++ b/polkadot/statement-table/src/lib.rs @@ -35,8 +35,8 @@ pub use generic::{Config, Context, Table}; pub mod v2 { use crate::generic; use primitives::{ - CandidateHash, CommittedCandidateReceipt, CompactStatement as PrimitiveStatement, Id, - ValidatorIndex, ValidatorSignature, + CandidateHash, CommittedCandidateReceipt, CompactStatement as PrimitiveStatement, + CoreIndex, ValidatorIndex, ValidatorSignature, }; /// Statements about candidates on the network. @@ -59,7 +59,7 @@ pub mod v2 { >; /// A summary of import of a statement. - pub type Summary = generic::Summary; + pub type Summary = generic::Summary; impl<'a> From<&'a Statement> for PrimitiveStatement { fn from(s: &'a Statement) -> PrimitiveStatement { diff --git a/polkadot/xcm/xcm-executor/integration-tests/Cargo.toml b/polkadot/xcm/xcm-executor/integration-tests/Cargo.toml index cafe12dc587f..1e572e6210a2 100644 --- a/polkadot/xcm/xcm-executor/integration-tests/Cargo.toml +++ b/polkadot/xcm/xcm-executor/integration-tests/Cargo.toml @@ -20,6 +20,7 @@ pallet-xcm = { path = "../../pallet-xcm" } polkadot-test-client = { path = "../../../node/test/client" } polkadot-test-runtime = { path = "../../../runtime/test-runtime" } polkadot-test-service = { path = "../../../node/test/service" } +polkadot-service = { path = "../../../node/service" } sp-consensus = { path = "../../../../substrate/primitives/consensus/common" } sp-keyring = { path = "../../../../substrate/primitives/keyring" } sp-runtime = { path = "../../../../substrate/primitives/runtime", default-features = false } @@ -27,6 +28,7 @@ sp-state-machine = { path = "../../../../substrate/primitives/state-machine" } xcm = { package = "staging-xcm", path = "../..", default-features = false } xcm-executor = { package = "staging-xcm-executor", path = ".." } sp-tracing = { path = "../../../../substrate/primitives/tracing" } +sp-core = { path = "../../../../substrate/primitives/core" } [features] default = ["std"] diff --git a/polkadot/xcm/xcm-executor/integration-tests/src/lib.rs b/polkadot/xcm/xcm-executor/integration-tests/src/lib.rs index 79d6cb1c411b..da7fc0d97825 100644 --- a/polkadot/xcm/xcm-executor/integration-tests/src/lib.rs +++ b/polkadot/xcm/xcm-executor/integration-tests/src/lib.rs @@ -19,12 +19,14 @@ use codec::Encode; use frame_support::{dispatch::GetDispatchInfo, weights::Weight}; +use polkadot_service::chain_spec::get_account_id_from_seed; use polkadot_test_client::{ BlockBuilderExt, ClientBlockImportExt, DefaultTestClientBuilderExt, InitPolkadotBlockBuilder, TestClientBuilder, TestClientBuilderExt, }; use polkadot_test_runtime::{pallet_test_notifier, xcm_config::XcmConfig}; use polkadot_test_service::construct_extrinsic; +use sp_core::sr25519; use sp_runtime::traits::Block; use sp_state_machine::InspectState; use xcm::{latest::prelude::*, VersionedResponse, VersionedXcm}; @@ -323,3 +325,84 @@ fn query_response_elicits_handler() { ))); }); } + +/// Simulates a cross-chain message from Parachain to Parachain through Relay Chain +/// that deposits assets into the reserve of the destination. +/// Regression test for `DepostiReserveAsset` changes in +/// +#[test] +fn deposit_reserve_asset_works_for_any_xcm_sender() { + sp_tracing::try_init_simple(); + let mut client = TestClientBuilder::new().build(); + + // Init values for the simulated origin Parachain + let amount_to_send: u128 = 1_000_000_000_000; + let assets: Assets = (Parent, amount_to_send).into(); + let fee_asset_item = 0; + let max_assets = assets.len() as u32; + let fees = assets.get(fee_asset_item as usize).unwrap().clone(); + let weight_limit = Unlimited; + let reserve = Location::parent(); + let dest = Location::new(1, [Parachain(2000)]); + let beneficiary_id = get_account_id_from_seed::("Alice"); + let beneficiary = Location::new(0, [AccountId32 { network: None, id: beneficiary_id.into() }]); + + // spends up to half of fees for execution on reserve and other half for execution on + // destination + let fee1 = amount_to_send.saturating_div(2); + let fee2 = amount_to_send.saturating_sub(fee1); + let fees_half_1 = Asset::from((fees.id.clone(), Fungible(fee1))); + let fees_half_2 = Asset::from((fees.id.clone(), Fungible(fee2))); + + let reserve_context = ::UniversalLocation::get(); + // identifies fee item as seen by `reserve` - to be used at reserve chain + let reserve_fees = fees_half_1.reanchored(&reserve, &reserve_context).unwrap(); + // identifies fee item as seen by `dest` - to be used at destination chain + let dest_fees = fees_half_2.reanchored(&dest, &reserve_context).unwrap(); + // identifies assets as seen by `reserve` - to be used at reserve chain + let assets_reanchored = assets.reanchored(&reserve, &reserve_context).unwrap(); + // identifies `dest` as seen by `reserve` + let dest = dest.reanchored(&reserve, &reserve_context).unwrap(); + // xcm to be executed at dest + let xcm_on_dest = Xcm(vec![ + BuyExecution { fees: dest_fees, weight_limit: weight_limit.clone() }, + DepositAsset { assets: Wild(AllCounted(max_assets)), beneficiary }, + ]); + // xcm to be executed at reserve + let msg = Xcm(vec![ + WithdrawAsset(assets_reanchored), + ClearOrigin, + BuyExecution { fees: reserve_fees, weight_limit }, + DepositReserveAsset { assets: Wild(AllCounted(max_assets)), dest, xcm: xcm_on_dest }, + ]); + + let mut block_builder = client.init_polkadot_block_builder(); + + // Simulate execution of an incoming XCM message at the reserve chain + let execute = construct_extrinsic( + &client, + polkadot_test_runtime::RuntimeCall::Xcm(pallet_xcm::Call::execute { + message: Box::new(VersionedXcm::from(msg)), + max_weight: Weight::from_parts(1_000_000_000, 1024 * 1024), + }), + sp_keyring::Sr25519Keyring::Alice, + 0, + ); + + block_builder.push_polkadot_extrinsic(execute).expect("pushes extrinsic"); + + let block = block_builder.build().expect("Finalizes the block").block; + let block_hash = block.hash(); + + futures::executor::block_on(client.import(sp_consensus::BlockOrigin::Own, block)) + .expect("imports the block"); + + client.state_at(block_hash).expect("state should exist").inspect_state(|| { + assert!(polkadot_test_runtime::System::events().iter().any(|r| matches!( + r.event, + polkadot_test_runtime::RuntimeEvent::Xcm(pallet_xcm::Event::Attempted { + outcome: Outcome::Complete { .. } + }), + ))); + }); +} diff --git a/polkadot/xcm/xcm-executor/src/lib.rs b/polkadot/xcm/xcm-executor/src/lib.rs index b26779f3ae9d..c61e1e1d15bc 100644 --- a/polkadot/xcm/xcm-executor/src/lib.rs +++ b/polkadot/xcm/xcm-executor/src/lib.rs @@ -826,9 +826,9 @@ impl XcmExecutor { // be weighed let to_weigh = self.holding.saturating_take(assets.clone()); self.holding.subsume_assets(to_weigh.clone()); - + let to_weigh_reanchored = Self::reanchored(to_weigh, &dest, None); let mut message_to_weigh = - vec![ReserveAssetDeposited(to_weigh.into()), ClearOrigin]; + vec![ReserveAssetDeposited(to_weigh_reanchored), ClearOrigin]; message_to_weigh.extend(xcm.0.clone().into_iter()); let (_, fee) = validate_send::(dest.clone(), Xcm(message_to_weigh))?; diff --git a/polkadot/zombienet_tests/functional/0012-elastic-scaling-mvp.toml b/polkadot/zombienet_tests/functional/0012-elastic-scaling-mvp.toml new file mode 100644 index 000000000000..0dfd814e10a5 --- /dev/null +++ b/polkadot/zombienet_tests/functional/0012-elastic-scaling-mvp.toml @@ -0,0 +1,38 @@ +[settings] +timeout = 1000 +bootnode = true + +[relaychain.genesis.runtimeGenesis.patch.configuration.config] + max_validators_per_core = 2 + needed_approvals = 4 + coretime_cores = 2 + +[relaychain] +default_image = "{{ZOMBIENET_INTEGRATION_TEST_IMAGE}}" +chain = "rococo-local" +default_command = "polkadot" + +[relaychain.default_resources] +limits = { memory = "4G", cpu = "2" } +requests = { memory = "2G", cpu = "1" } + + [[relaychain.nodes]] + name = "alice" + validator = "true" + + [[relaychain.node_groups]] + name = "validator" + count = 3 + args = [ "-lparachain=debug,runtime=debug"] + +[[parachains]] +id = 2000 +default_command = "polkadot-parachain" +add_to_genesis = false +register_para = true +onboard_as_parachain = false + + [parachains.collator] + name = "collator2000" + command = "polkadot-parachain" + args = [ "-lparachain=debug" ] diff --git a/polkadot/zombienet_tests/functional/0012-elastic-scaling-mvp.zndsl b/polkadot/zombienet_tests/functional/0012-elastic-scaling-mvp.zndsl new file mode 100644 index 000000000000..a7193c9282b9 --- /dev/null +++ b/polkadot/zombienet_tests/functional/0012-elastic-scaling-mvp.zndsl @@ -0,0 +1,28 @@ +Description: Test that a paraid acquiring multiple cores does not brick itself if ElasticScalingMVP feature is enabled +Network: ./0012-elastic-scaling-mvp.toml +Creds: config + +# Check authority status. +validator: reports node_roles is 4 + +validator: reports substrate_block_height{status="finalized"} is at least 10 within 100 seconds + +# Ensure parachain was able to make progress. +validator: parachain 2000 block height is at least 10 within 200 seconds + +# Register the second core assigned to this parachain. +alice: js-script ./0012-register-para.js return is 0 within 600 seconds + +validator: reports substrate_block_height{status="finalized"} is at least 35 within 100 seconds + +# Parachain will now be stalled +validator: parachain 2000 block height is lower than 20 within 300 seconds + +# Enable the ElasticScalingMVP node feature. +alice: js-script ./0012-enable-node-feature.js with "1" return is 0 within 600 seconds + +# Wait two sessions for the config to be updated. +sleep 120 seconds + +# Ensure parachain is now making progress. +validator: parachain 2000 block height is at least 30 within 200 seconds diff --git a/polkadot/zombienet_tests/functional/0012-enable-node-feature.js b/polkadot/zombienet_tests/functional/0012-enable-node-feature.js new file mode 100644 index 000000000000..4822e1f66447 --- /dev/null +++ b/polkadot/zombienet_tests/functional/0012-enable-node-feature.js @@ -0,0 +1,37 @@ +async function run(nodeName, networkInfo, index) { + const { wsUri, userDefinedTypes } = networkInfo.nodesByName[nodeName]; + const api = await zombie.connect(wsUri, userDefinedTypes); + + await zombie.util.cryptoWaitReady(); + + // account to submit tx + const keyring = new zombie.Keyring({ type: "sr25519" }); + const alice = keyring.addFromUri("//Alice"); + + await new Promise(async (resolve, reject) => { + const unsub = await api.tx.sudo + .sudo(api.tx.configuration.setNodeFeature(Number(index), true)) + .signAndSend(alice, ({ status, isError }) => { + if (status.isInBlock) { + console.log( + `Transaction included at blockhash ${status.asInBlock}`, + ); + } else if (status.isFinalized) { + console.log( + `Transaction finalized at blockHash ${status.asFinalized}`, + ); + unsub(); + return resolve(); + } else if (isError) { + console.log(`Transaction error`); + reject(`Transaction error`); + } + }); + }); + + + + return 0; +} + +module.exports = { run }; diff --git a/polkadot/zombienet_tests/functional/0012-register-para.js b/polkadot/zombienet_tests/functional/0012-register-para.js new file mode 100644 index 000000000000..25c7e4f5ffdd --- /dev/null +++ b/polkadot/zombienet_tests/functional/0012-register-para.js @@ -0,0 +1,37 @@ +async function run(nodeName, networkInfo, _jsArgs) { + const { wsUri, userDefinedTypes } = networkInfo.nodesByName[nodeName]; + const api = await zombie.connect(wsUri, userDefinedTypes); + + await zombie.util.cryptoWaitReady(); + + // account to submit tx + const keyring = new zombie.Keyring({ type: "sr25519" }); + const alice = keyring.addFromUri("//Alice"); + + await new Promise(async (resolve, reject) => { + const unsub = await api.tx.sudo + .sudo(api.tx.coretime.assignCore(0, 35, [[{ task: 2000 }, 57600]], null)) + .signAndSend(alice, ({ status, isError }) => { + if (status.isInBlock) { + console.log( + `Transaction included at blockhash ${status.asInBlock}`, + ); + } else if (status.isFinalized) { + console.log( + `Transaction finalized at blockHash ${status.asFinalized}`, + ); + unsub(); + return resolve(); + } else if (isError) { + console.log(`Transaction error`); + reject(`Transaction error`); + } + }); + }); + + + + return 0; +} + +module.exports = { run }; diff --git a/prdoc/pr_3002.prdoc b/prdoc/pr_3002.prdoc new file mode 100644 index 000000000000..511a07e39c47 --- /dev/null +++ b/prdoc/pr_3002.prdoc @@ -0,0 +1,29 @@ +# Schema: Polkadot SDK PRDoc Schema (prdoc) v1.0.0 +# See doc at https://raw.githubusercontent.com/paritytech/polkadot-sdk/master/prdoc/schema_user.json + +title: PoV Reclaim Runtime Side +author: skunert +topic: runtime +doc: + - audience: Runtime Dev + description: | + Adds a mechanism to reclaim proof size weight. + 1. Introduces a new `SignedExtension` that reclaims the difference + between benchmarked proof size weight and actual consumed proof size weight. + 2. Introduces a manual mechanism, `StorageWeightReclaimer`, to reclaim excess storage weight for situations + that require manual weight management. The most prominent case is the `on_idle` hook. + 3. Adds the `storage_proof_size` host function to the PVF. Parachain nodes should add it to ensure compatibility. + + To enable proof size reclaiming, add the host `storage_proof_size` host function to the parachain node. Add the + `StorageWeightReclaim` `SignedExtension` to your runtime and enable proof recording during block import. + + +crates: + - name: "cumulus-primitives-storage-weight-reclaim" +host_functions: + - name: "storage_proof_size" + description: | + This host function is used to pass the current size of the storage proof to the runtime. + It was introduced before but becomes relevant now. + Note: This host function is intended to be used through `cumulus_primitives_storage_weight_reclaim::get_proof_size`. + Direct usage is not recommended. diff --git a/prdoc/pr_3231.prdoc b/prdoc/pr_3231.prdoc new file mode 100644 index 000000000000..26e96d3635b1 --- /dev/null +++ b/prdoc/pr_3231.prdoc @@ -0,0 +1,11 @@ +title: Allow parachain which acquires multiple coretime cores to make progress + +doc: + - audience: Node Operator + description: | + Adds the needed changes so that parachains which acquire multiple coretime cores can still make progress. + Only one of the cores will be able to be occupied at a time. + Only works if the ElasticScalingMVP node feature is enabled in the runtime and the block author validator is + updated to include this change. + +crates: [ ] diff --git a/prdoc/pr_3324.prdoc b/prdoc/pr_3324.prdoc new file mode 100644 index 000000000000..0425fbf317c8 --- /dev/null +++ b/prdoc/pr_3324.prdoc @@ -0,0 +1,13 @@ +# Schema: Polkadot SDK PRDoc Schema (prdoc) v1.0.0 +# See doc at https://raw.githubusercontent.com/paritytech/polkadot-sdk/master/prdoc/schema_user.json + +title: "Fix weight calculation and event emission in pallet-membership" + +doc: + - audience: Runtime Dev + description: | + Bug fix for the membership pallet to use correct weights. Also no event will be emitted + anymore when `change_key` is called with identical accounts. + +crates: + - name: pallet-membership diff --git a/prdoc/pr_3371.prdoc b/prdoc/pr_3371.prdoc new file mode 100644 index 000000000000..605d540772f0 --- /dev/null +++ b/prdoc/pr_3371.prdoc @@ -0,0 +1,19 @@ +# Schema: Polkadot SDK PRDoc Schema (prdoc) v1.0.0 +# See doc at https://raw.githubusercontent.com/paritytech/polkadot-sdk/master/prdoc/schema_user.json + +title: removed `pallet::getter` from example pallets + +doc: + - audience: Runtime Dev + description: | + This PR removes all the `pallet::getter` usages from the template pallets found in the Substrate and Cumulus template nodes, and from the Substrate example pallets. + The purpose is to discourage developers to use this macro, that is currently being removed and soon will be deprecated. + +crates: + - name: pallet-template + - name: pallet-parachain-template + - name: pallet-example-basic + - name: pallet-example-kitchensink + - name: pallet-example-offchain-worker + - name: pallet-example-split + diff --git a/prdoc/pr_3412.prdoc b/prdoc/pr_3412.prdoc new file mode 100644 index 000000000000..d68f05e45dcf --- /dev/null +++ b/prdoc/pr_3412.prdoc @@ -0,0 +1,17 @@ +# Schema: Polkadot SDK PRDoc Schema (prdoc) v1.0.0 +# See doc at https://raw.githubusercontent.com/paritytech/polkadot-sdk/master/prdoc/schema_user.json + +title: "[FRAME] Add genesis test and remove some checks" + +doc: + - audience: Runtime Dev + description: | + The construct_runtime macro now generates a test to assert that all `GenesisConfig`s of all + pallets can be build within the runtime. This ensures that the `BuildGenesisConfig` runtime + API works. + Further, some checks from a few pallets were removed to make this pass. + +crates: + - name: pallet-babe + - name: pallet-aura-ext + - name: pallet-session diff --git a/prdoc/pr_3415.prdoc b/prdoc/pr_3415.prdoc new file mode 100644 index 000000000000..c56a5d3ffaa1 --- /dev/null +++ b/prdoc/pr_3415.prdoc @@ -0,0 +1,9 @@ +title: "[pallet-contracts] Add APIVersion to the config." + +doc: + - audience: Runtime Dev + description: | + Add `APIVersion` to the config to communicate the state of the Host functions exposed by the pallet. + +crates: + - name: pallet-contracts diff --git a/prdoc/pr_3435.prdoc b/prdoc/pr_3435.prdoc new file mode 100644 index 000000000000..7d7896bff7d4 --- /dev/null +++ b/prdoc/pr_3435.prdoc @@ -0,0 +1,16 @@ +# Schema: Polkadot SDK PRDoc Schema (prdoc) v1.0.0 +# See doc at https://raw.githubusercontent.com/paritytech/polkadot-sdk/master/prdoc/schema_user.json + +title: Fix BEEFY-related gossip messages error logs + +doc: + - audience: Node Operator + description: | + Added logic to pump the gossip engine while waiting for other things + to make sure gossiped messages get consumed (practically discarded + until worker is fully initialized). + This fixes an issue where node operators saw error logs, and fixes + potential RAM bloat when BEEFY initialization takes a long time + (i.e. during clean sync). +crates: + - name: sc-consensus-beefy diff --git a/prdoc/pr_3447.prdoc b/prdoc/pr_3447.prdoc new file mode 100644 index 000000000000..1d8d4f409f77 --- /dev/null +++ b/prdoc/pr_3447.prdoc @@ -0,0 +1,13 @@ +# Schema: Polkadot SDK PRDoc Schema (prdoc) v1.0.0 +# See doc at https://raw.githubusercontent.com/paritytech/polkadot-sdk/master/prdoc/schema_user.json + +title: Use generic hash for runtime wasm in resolve_state_version_from_wasm + +doc: + - audience: Node Dev + description: | + Changes the runtime hash algorithm used in resolve_state_version_from_wasm from DefaultHasher to a caller-provided + one (usually HashingFor). Fixes a bug where the runtime wasm was being compiled again when it was not + needed, because the hash did not match +crates: + - name: sc-chain-spec diff --git a/substrate/bin/node-template/pallets/template/src/lib.rs b/substrate/bin/node-template/pallets/template/src/lib.rs index 4a2e53baa774..90dfe3701458 100644 --- a/substrate/bin/node-template/pallets/template/src/lib.rs +++ b/substrate/bin/node-template/pallets/template/src/lib.rs @@ -90,9 +90,7 @@ pub mod pallet { /// /// In this template, we are declaring a storage item called `Something` that stores a single /// `u32` value. Learn more about runtime storage here: - /// The [`getter`] macro generates a function to conveniently retrieve the value from storage. #[pallet::storage] - #[pallet::getter(fn something)] pub type Something = StorageValue<_, u32>; /// Events that functions in this pallet can emit. @@ -187,7 +185,7 @@ pub mod pallet { let _who = ensure_signed(origin)?; // Read a value from storage. - match Pallet::::something() { + match Something::::get() { // Return an error if the value has not been set. None => Err(Error::::NoneValue.into()), Some(old) => { diff --git a/substrate/bin/node-template/pallets/template/src/tests.rs b/substrate/bin/node-template/pallets/template/src/tests.rs index 7c2b853ee4dc..83e4bea7377b 100644 --- a/substrate/bin/node-template/pallets/template/src/tests.rs +++ b/substrate/bin/node-template/pallets/template/src/tests.rs @@ -1,4 +1,4 @@ -use crate::{mock::*, Error, Event}; +use crate::{mock::*, Error, Event, Something}; use frame_support::{assert_noop, assert_ok}; #[test] @@ -9,7 +9,7 @@ fn it_works_for_default_value() { // Dispatch a signed extrinsic. assert_ok!(TemplateModule::do_something(RuntimeOrigin::signed(1), 42)); // Read pallet storage and assert an expected result. - assert_eq!(TemplateModule::something(), Some(42)); + assert_eq!(Something::::get(), Some(42)); // Assert that the correct event was deposited System::assert_last_event(Event::SomethingStored { something: 42, who: 1 }.into()); }); diff --git a/substrate/bin/node/cli/tests/res/default_genesis_config.json b/substrate/bin/node/cli/tests/res/default_genesis_config.json index 1465a6497cad..e21fbb47da8c 100644 --- a/substrate/bin/node/cli/tests/res/default_genesis_config.json +++ b/substrate/bin/node/cli/tests/res/default_genesis_config.json @@ -2,7 +2,13 @@ "system": {}, "babe": { "authorities": [], - "epochConfig": null + "epochConfig": { + "allowed_slots": "PrimaryAndSecondaryVRFSlots", + "c": [ + 1, + 4 + ] + } }, "indices": { "indices": [] diff --git a/substrate/bin/node/runtime/src/lib.rs b/substrate/bin/node/runtime/src/lib.rs index 24c8f6f48e96..b34d8c89e568 100644 --- a/substrate/bin/node/runtime/src/lib.rs +++ b/substrate/bin/node/runtime/src/lib.rs @@ -1367,6 +1367,7 @@ impl pallet_contracts::Config for Runtime { type CodeHashLockupDepositPercent = CodeHashLockupDepositPercent; type Debug = (); type Environment = (); + type ApiVersion = (); type Xcm = (); } diff --git a/substrate/bin/node/testing/src/genesis.rs b/substrate/bin/node/testing/src/genesis.rs index 6ec21fbe0934..c79612d68444 100644 --- a/substrate/bin/node/testing/src/genesis.rs +++ b/substrate/bin/node/testing/src/genesis.rs @@ -20,9 +20,8 @@ use crate::keyring::*; use kitchensink_runtime::{ - constants::currency::*, AccountId, AssetsConfig, BabeConfig, BalancesConfig, GluttonConfig, - GrandpaConfig, IndicesConfig, RuntimeGenesisConfig, SessionConfig, SocietyConfig, StakerStatus, - StakingConfig, BABE_GENESIS_EPOCH_CONFIG, + constants::currency::*, AccountId, AssetsConfig, BalancesConfig, IndicesConfig, + RuntimeGenesisConfig, SessionConfig, SocietyConfig, StakerStatus, StakingConfig, }; use sp_keyring::Ed25519Keyring; use sp_runtime::Perbill; @@ -47,7 +46,6 @@ pub fn config_endowed(extra_endowed: Vec) -> RuntimeGenesisConfig { endowed.extend(extra_endowed.into_iter().map(|endowed| (endowed, 100 * DOLLARS))); RuntimeGenesisConfig { - system: Default::default(), indices: IndicesConfig { indices: vec![] }, balances: BalancesConfig { balances: endowed }, session: SessionConfig { @@ -69,39 +67,8 @@ pub fn config_endowed(extra_endowed: Vec) -> RuntimeGenesisConfig { invulnerables: vec![alice(), bob(), charlie()], ..Default::default() }, - babe: BabeConfig { - authorities: vec![], - epoch_config: Some(BABE_GENESIS_EPOCH_CONFIG), - ..Default::default() - }, - grandpa: GrandpaConfig { authorities: vec![], _config: Default::default() }, - beefy: Default::default(), - im_online: Default::default(), - authority_discovery: Default::default(), - democracy: Default::default(), - council: Default::default(), - technical_committee: Default::default(), - technical_membership: Default::default(), - elections: Default::default(), - sudo: Default::default(), - treasury: Default::default(), society: SocietyConfig { pot: 0 }, - vesting: Default::default(), assets: AssetsConfig { assets: vec![(9, alice(), true, 1)], ..Default::default() }, - pool_assets: Default::default(), - transaction_storage: Default::default(), - transaction_payment: Default::default(), - alliance: Default::default(), - alliance_motion: Default::default(), - nomination_pools: Default::default(), - safe_mode: Default::default(), - tx_pause: Default::default(), - glutton: GluttonConfig { - compute: Default::default(), - storage: Default::default(), - trash_data_count: Default::default(), - ..Default::default() - }, - mixnet: Default::default(), + ..Default::default() } } diff --git a/substrate/client/basic-authorship/src/basic_authorship.rs b/substrate/client/basic-authorship/src/basic_authorship.rs index c07f3e639c3e..fdc3d4918de3 100644 --- a/substrate/client/basic-authorship/src/basic_authorship.rs +++ b/substrate/client/basic-authorship/src/basic_authorship.rs @@ -87,6 +87,22 @@ pub struct ProposerFactory { _phantom: PhantomData, } +impl Clone for ProposerFactory { + fn clone(&self) -> Self { + Self { + spawn_handle: self.spawn_handle.clone(), + client: self.client.clone(), + transaction_pool: self.transaction_pool.clone(), + metrics: self.metrics.clone(), + default_block_size_limit: self.default_block_size_limit, + soft_deadline_percent: self.soft_deadline_percent, + telemetry: self.telemetry.clone(), + include_proof_in_block_size_estimation: self.include_proof_in_block_size_estimation, + _phantom: self._phantom, + } + } +} + impl ProposerFactory { /// Create a new proposer factory. /// diff --git a/substrate/client/chain-spec/src/chain_spec.rs b/substrate/client/chain-spec/src/chain_spec.rs index fe8fcfda216e..78e81e10d2b6 100644 --- a/substrate/client/chain-spec/src/chain_spec.rs +++ b/substrate/client/chain-spec/src/chain_spec.rs @@ -1237,15 +1237,7 @@ mod tests { "TestName", "test", ChainType::Local, - move || substrate_test_runtime::RuntimeGenesisConfig { - babe: substrate_test_runtime::BabeConfig { - epoch_config: Some( - substrate_test_runtime::TEST_RUNTIME_BABE_EPOCH_CONFIGURATION, - ), - ..Default::default() - }, - ..Default::default() - }, + || Default::default(), Vec::new(), None, None, diff --git a/substrate/client/chain-spec/src/genesis_block.rs b/substrate/client/chain-spec/src/genesis_block.rs index 6aa156a620a7..3c7b9f64dcd6 100644 --- a/substrate/client/chain-spec/src/genesis_block.rs +++ b/substrate/client/chain-spec/src/genesis_block.rs @@ -18,23 +18,25 @@ //! Tool for creating the genesis block. -use std::{collections::hash_map::DefaultHasher, marker::PhantomData, sync::Arc}; +use std::{marker::PhantomData, sync::Arc}; +use codec::Encode; use sc_client_api::{backend::Backend, BlockImportOperation}; use sc_executor::RuntimeVersionOf; use sp_core::storage::{well_known_keys, StateVersion, Storage}; use sp_runtime::{ - traits::{Block as BlockT, Hash as HashT, Header as HeaderT, Zero}, + traits::{Block as BlockT, Hash as HashT, HashingFor, Header as HeaderT, Zero}, BuildStorage, }; /// Return the state version given the genesis storage and executor. -pub fn resolve_state_version_from_wasm( +pub fn resolve_state_version_from_wasm( storage: &Storage, executor: &E, ) -> sp_blockchain::Result where E: RuntimeVersionOf, + H: HashT, { if let Some(wasm) = storage.top.get(well_known_keys::CODE) { let mut ext = sp_state_machine::BasicExternalities::new_empty(); // just to read runtime version. @@ -43,12 +45,7 @@ where let runtime_code = sp_core::traits::RuntimeCode { code_fetcher: &code_fetcher, heap_pages: None, - hash: { - use std::hash::{Hash, Hasher}; - let mut state = DefaultHasher::new(); - wasm.hash(&mut state); - state.finish().to_le_bytes().to_vec() - }, + hash: ::hash(wasm).encode(), }; let runtime_version = RuntimeVersionOf::runtime_version(executor, &mut ext, &runtime_code) .map_err(|e| sp_blockchain::Error::VersionInvalid(e.to_string()))?; @@ -129,7 +126,8 @@ impl, E: RuntimeVersionOf> BuildGenesisBlock sp_blockchain::Result<(Block, Self::BlockImportOperation)> { let Self { genesis_storage, commit_genesis_state, backend, executor, _phantom } = self; - let genesis_state_version = resolve_state_version_from_wasm(&genesis_storage, &executor)?; + let genesis_state_version = + resolve_state_version_from_wasm::<_, HashingFor>(&genesis_storage, &executor)?; let mut op = backend.begin_operation()?; let state_root = op.set_genesis_state(genesis_storage, commit_genesis_state, genesis_state_version)?; diff --git a/substrate/client/chain-spec/src/genesis_config_builder.rs b/substrate/client/chain-spec/src/genesis_config_builder.rs index 8766dd5c5ad2..6b956316203c 100644 --- a/substrate/client/chain-spec/src/genesis_config_builder.rs +++ b/substrate/client/chain-spec/src/genesis_config_builder.rs @@ -149,7 +149,7 @@ mod tests { ::new(substrate_test_runtime::wasm_binary_unwrap()) .get_default_config() .unwrap(); - let expected = r#"{"system":{},"babe":{"authorities":[],"epochConfig":null},"substrateTest":{"authorities":[]},"balances":{"balances":[]}}"#; + let expected = r#"{"babe": {"authorities": [], "epochConfig": {"allowed_slots": "PrimaryAndSecondaryVRFSlots", "c": [1, 4]}}, "balances": {"balances": []}, "substrateTest": {"authorities": []}, "system": {}}"#; assert_eq!(from_str::(expected).unwrap(), config); } diff --git a/substrate/client/consensus/babe/rpc/src/lib.rs b/substrate/client/consensus/babe/rpc/src/lib.rs index 8b183e7dfdcd..a3e811baecff 100644 --- a/substrate/client/consensus/babe/rpc/src/lib.rs +++ b/substrate/client/consensus/babe/rpc/src/lib.rs @@ -258,7 +258,7 @@ mod tests { let request = r#"{"jsonrpc":"2.0","method":"babe_epochAuthorship","params": [],"id":1}"#; let (response, _) = api.raw_json_request(request, 1).await.unwrap(); - let expected = r#"{"jsonrpc":"2.0","result":{"5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY":{"primary":[0],"secondary":[1,2,4],"secondary_vrf":[]}},"id":1}"#; + let expected = r#"{"jsonrpc":"2.0","result":{"5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY":{"primary":[0],"secondary":[],"secondary_vrf":[1,2,4]}},"id":1}"#; assert_eq!(response, expected); } diff --git a/substrate/client/consensus/beefy/src/communication/gossip.rs b/substrate/client/consensus/beefy/src/communication/gossip.rs index 8a0b0a74308f..eb43c9173d75 100644 --- a/substrate/client/consensus/beefy/src/communication/gossip.rs +++ b/substrate/client/consensus/beefy/src/communication/gossip.rs @@ -56,6 +56,8 @@ pub(super) enum Action { Keep(H, ReputationChange), // discard, applying cost/benefit to originator. Discard(ReputationChange), + // ignore, no cost/benefit applied to originator. + DiscardNoReport, } /// An outcome of examining a message. @@ -68,7 +70,7 @@ enum Consider { /// Message is from the future. Reject. RejectFuture, /// Message cannot be evaluated. Reject. - RejectOutOfScope, + CannotEvaluate, } /// BEEFY gossip message type that gets encoded and sent on the network. @@ -168,18 +170,14 @@ impl Filter { .as_ref() .map(|f| // only from current set and only [filter.start, filter.end] - if set_id < f.validator_set.id() { + if set_id < f.validator_set.id() || round < f.start { Consider::RejectPast - } else if set_id > f.validator_set.id() { - Consider::RejectFuture - } else if round < f.start { - Consider::RejectPast - } else if round > f.end { + } else if set_id > f.validator_set.id() || round > f.end { Consider::RejectFuture } else { Consider::Accept }) - .unwrap_or(Consider::RejectOutOfScope) + .unwrap_or(Consider::CannotEvaluate) } /// Return true if `round` is >= than `max(session_start, best_beefy)`, @@ -199,7 +197,7 @@ impl Filter { Consider::Accept } ) - .unwrap_or(Consider::RejectOutOfScope) + .unwrap_or(Consider::CannotEvaluate) } /// Add new _known_ `round` to the set of seen valid justifications. @@ -244,7 +242,7 @@ where pub(crate) fn new( known_peers: Arc>>, ) -> (GossipValidator, TracingUnboundedReceiver) { - let (tx, rx) = tracing_unbounded("mpsc_beefy_gossip_validator", 10_000); + let (tx, rx) = tracing_unbounded("mpsc_beefy_gossip_validator", 100_000); let val = GossipValidator { votes_topic: votes_topic::(), justifs_topic: proofs_topic::(), @@ -289,7 +287,9 @@ where match filter.consider_vote(round, set_id) { Consider::RejectPast => return Action::Discard(cost::OUTDATED_MESSAGE), Consider::RejectFuture => return Action::Discard(cost::FUTURE_MESSAGE), - Consider::RejectOutOfScope => return Action::Discard(cost::OUT_OF_SCOPE_MESSAGE), + // When we can't evaluate, it's our fault (e.g. filter not initialized yet), we + // discard the vote without punishing or rewarding the sending peer. + Consider::CannotEvaluate => return Action::DiscardNoReport, Consider::Accept => {}, } @@ -330,7 +330,9 @@ where match guard.consider_finality_proof(round, set_id) { Consider::RejectPast => return Action::Discard(cost::OUTDATED_MESSAGE), Consider::RejectFuture => return Action::Discard(cost::FUTURE_MESSAGE), - Consider::RejectOutOfScope => return Action::Discard(cost::OUT_OF_SCOPE_MESSAGE), + // When we can't evaluate, it's our fault (e.g. filter not initialized yet), we + // discard the proof without punishing or rewarding the sending peer. + Consider::CannotEvaluate => return Action::DiscardNoReport, Consider::Accept => {}, } @@ -357,7 +359,9 @@ where Action::Keep(self.justifs_topic, benefit::VALIDATED_PROOF) } }) - .unwrap_or(Action::Discard(cost::OUT_OF_SCOPE_MESSAGE)) + // When we can't evaluate, it's our fault (e.g. filter not initialized yet), we + // discard the proof without punishing or rewarding the sending peer. + .unwrap_or(Action::DiscardNoReport) }; if matches!(action, Action::Keep(_, _)) { self.gossip_filter.write().mark_round_as_proven(round); @@ -404,6 +408,7 @@ where self.report(*sender, cb); ValidationResult::Discard }, + Action::DiscardNoReport => ValidationResult::Discard, } } @@ -579,8 +584,8 @@ pub(crate) mod tests { // filter not initialized let res = gv.validate(&mut context, &sender, &encoded); assert!(matches!(res, ValidationResult::Discard)); - expected_report.cost_benefit = cost::OUT_OF_SCOPE_MESSAGE; - assert_eq!(report_stream.try_recv().unwrap(), expected_report); + // nothing reported + assert!(report_stream.try_recv().is_err()); gv.update_filter(GossipFilterCfg { start: 0, end: 10, validator_set: &validator_set }); // nothing in cache first time diff --git a/substrate/client/consensus/beefy/src/communication/mod.rs b/substrate/client/consensus/beefy/src/communication/mod.rs index 3827559057dd..6fda63688e69 100644 --- a/substrate/client/consensus/beefy/src/communication/mod.rs +++ b/substrate/client/consensus/beefy/src/communication/mod.rs @@ -90,8 +90,6 @@ mod cost { pub(super) const BAD_SIGNATURE: Rep = Rep::new(-100, "BEEFY: Bad signature"); // Message received with vote from voter not in validator set. pub(super) const UNKNOWN_VOTER: Rep = Rep::new(-150, "BEEFY: Unknown voter"); - // A message received that cannot be evaluated relative to our current state. - pub(super) const OUT_OF_SCOPE_MESSAGE: Rep = Rep::new(-500, "BEEFY: Out-of-scope message"); // Message containing invalid proof. pub(super) const INVALID_PROOF: Rep = Rep::new(-5000, "BEEFY: Invalid commit"); // Reputation cost per signature checked for invalid proof. diff --git a/substrate/client/consensus/beefy/src/import.rs b/substrate/client/consensus/beefy/src/import.rs index fc19ecc30142..ed8ed68c4e8d 100644 --- a/substrate/client/consensus/beefy/src/import.rs +++ b/substrate/client/consensus/beefy/src/import.rs @@ -159,7 +159,7 @@ where // The proof is valid and the block is imported and final, we can import. debug!( target: LOG_TARGET, - "🥩 import justif {:?} for block number {:?}.", proof, number + "🥩 import justif {} for block number {:?}.", proof, number ); // Send the justification to the BEEFY voter for processing. self.justification_sender diff --git a/substrate/client/consensus/beefy/src/lib.rs b/substrate/client/consensus/beefy/src/lib.rs index 11a105561e47..323af1bc8305 100644 --- a/substrate/client/consensus/beefy/src/lib.rs +++ b/substrate/client/consensus/beefy/src/lib.rs @@ -31,7 +31,7 @@ use crate::{ import::BeefyBlockImport, metrics::register_metrics, }; -use futures::{stream::Fuse, StreamExt}; +use futures::{stream::Fuse, FutureExt, StreamExt}; use log::{debug, error, info, warn}; use parking_lot::Mutex; use prometheus::Registry; @@ -40,17 +40,21 @@ use sc_consensus::BlockImport; use sc_network::{NetworkRequest, NotificationService, ProtocolName}; use sc_network_gossip::{GossipEngine, Network as GossipNetwork, Syncing as GossipSyncing}; use sp_api::ProvideRuntimeApi; -use sp_blockchain::{ - Backend as BlockchainBackend, Error as ClientError, HeaderBackend, Result as ClientResult, -}; +use sp_blockchain::{Backend as BlockchainBackend, HeaderBackend}; use sp_consensus::{Error as ConsensusError, SyncOracle}; use sp_consensus_beefy::{ - ecdsa_crypto::AuthorityId, BeefyApi, MmrRootHash, PayloadProvider, ValidatorSet, + ecdsa_crypto::AuthorityId, BeefyApi, ConsensusLog, MmrRootHash, PayloadProvider, ValidatorSet, + BEEFY_ENGINE_ID, }; use sp_keystore::KeystorePtr; use sp_mmr_primitives::MmrApi; use sp_runtime::traits::{Block, Header as HeaderT, NumberFor, Zero}; -use std::{collections::BTreeMap, marker::PhantomData, sync::Arc, time::Duration}; +use std::{ + collections::{BTreeMap, VecDeque}, + marker::PhantomData, + sync::Arc, + time::Duration, +}; mod aux_schema; mod error; @@ -63,9 +67,19 @@ pub mod communication; pub mod import; pub mod justification; +use crate::{ + communication::{gossip::GossipValidator, peers::PeerReport}, + justification::BeefyVersionedFinalityProof, + keystore::BeefyKeystore, + metrics::VoterMetrics, + round::Rounds, + worker::{BeefyWorker, PersistedState}, +}; pub use communication::beefy_protocol_name::{ gossip_protocol_name, justifications_protocol_name as justifs_protocol_name, }; +use sc_utils::mpsc::TracingUnboundedReceiver; +use sp_runtime::generic::OpaqueDigestItemId; #[cfg(test)] mod tests; @@ -209,6 +223,247 @@ pub struct BeefyParams { /// Handler for incoming BEEFY justifications requests from a remote peer. pub on_demand_justifications_handler: BeefyJustifsRequestHandler, } +/// Helper object holding BEEFY worker communication/gossip components. +/// +/// These are created once, but will be reused if worker is restarted/reinitialized. +pub(crate) struct BeefyComms { + pub gossip_engine: GossipEngine, + pub gossip_validator: Arc>, + pub gossip_report_stream: TracingUnboundedReceiver, + pub on_demand_justifications: OnDemandJustificationsEngine, +} + +/// Helper builder object for building [worker::BeefyWorker]. +/// +/// It has to do it in two steps: initialization and build, because the first step can sleep waiting +/// for certain chain and backend conditions, and while sleeping we still need to pump the +/// GossipEngine. Once initialization is done, the GossipEngine (and other pieces) are added to get +/// the complete [worker::BeefyWorker] object. +pub(crate) struct BeefyWorkerBuilder { + // utilities + backend: Arc, + runtime: Arc, + key_store: BeefyKeystore, + // voter metrics + metrics: Option, + persisted_state: PersistedState, +} + +impl BeefyWorkerBuilder +where + B: Block + codec::Codec, + BE: Backend, + R: ProvideRuntimeApi, + R::Api: BeefyApi, +{ + /// This will wait for the chain to enable BEEFY (if not yet enabled) and also wait for the + /// backend to sync all headers required by the voter to build a contiguous chain of mandatory + /// justifications. Then it builds the initial voter state using a combination of previously + /// persisted state in AUX DB and latest chain information/progress. + /// + /// Returns a sane `BeefyWorkerBuilder` that can build the `BeefyWorker`. + pub async fn async_initialize( + backend: Arc, + runtime: Arc, + key_store: BeefyKeystore, + metrics: Option, + min_block_delta: u32, + gossip_validator: Arc>, + finality_notifications: &mut Fuse>, + ) -> Result { + // Wait for BEEFY pallet to be active before starting voter. + let (beefy_genesis, best_grandpa) = + wait_for_runtime_pallet(&*runtime, finality_notifications).await?; + + let persisted_state = Self::load_or_init_state( + beefy_genesis, + best_grandpa, + min_block_delta, + backend.clone(), + runtime.clone(), + &key_store, + &metrics, + ) + .await?; + // Update the gossip validator with the right starting round and set id. + persisted_state + .gossip_filter_config() + .map(|f| gossip_validator.update_filter(f))?; + + Ok(BeefyWorkerBuilder { backend, runtime, key_store, metrics, persisted_state }) + } + + /// Takes rest of missing pieces as params and builds the `BeefyWorker`. + pub fn build( + self, + payload_provider: P, + sync: Arc, + comms: BeefyComms, + links: BeefyVoterLinks, + pending_justifications: BTreeMap, BeefyVersionedFinalityProof>, + ) -> BeefyWorker { + BeefyWorker { + backend: self.backend, + runtime: self.runtime, + key_store: self.key_store, + metrics: self.metrics, + persisted_state: self.persisted_state, + payload_provider, + sync, + comms, + links, + pending_justifications, + } + } + + // If no persisted state present, walk back the chain from first GRANDPA notification to either: + // - latest BEEFY finalized block, or if none found on the way, + // - BEEFY pallet genesis; + // Enqueue any BEEFY mandatory blocks (session boundaries) found on the way, for voter to + // finalize. + async fn init_state( + beefy_genesis: NumberFor, + best_grandpa: ::Header, + min_block_delta: u32, + backend: Arc, + runtime: Arc, + ) -> Result, Error> { + let blockchain = backend.blockchain(); + + let beefy_genesis = runtime + .runtime_api() + .beefy_genesis(best_grandpa.hash()) + .ok() + .flatten() + .filter(|genesis| *genesis == beefy_genesis) + .ok_or_else(|| Error::Backend("BEEFY pallet expected to be active.".into()))?; + // Walk back the imported blocks and initialize voter either, at the last block with + // a BEEFY justification, or at pallet genesis block; voter will resume from there. + let mut sessions = VecDeque::new(); + let mut header = best_grandpa.clone(); + let state = loop { + if let Some(true) = blockchain + .justifications(header.hash()) + .ok() + .flatten() + .map(|justifs| justifs.get(BEEFY_ENGINE_ID).is_some()) + { + debug!( + target: LOG_TARGET, + "🥩 Initialize BEEFY voter at last BEEFY finalized block: {:?}.", + *header.number() + ); + let best_beefy = *header.number(); + // If no session boundaries detected so far, just initialize new rounds here. + if sessions.is_empty() { + let active_set = + expect_validator_set(runtime.as_ref(), backend.as_ref(), &header).await?; + let mut rounds = Rounds::new(best_beefy, active_set); + // Mark the round as already finalized. + rounds.conclude(best_beefy); + sessions.push_front(rounds); + } + let state = PersistedState::checked_new( + best_grandpa, + best_beefy, + sessions, + min_block_delta, + beefy_genesis, + ) + .ok_or_else(|| Error::Backend("Invalid BEEFY chain".into()))?; + break state + } + + if *header.number() == beefy_genesis { + // We've reached BEEFY genesis, initialize voter here. + let genesis_set = + expect_validator_set(runtime.as_ref(), backend.as_ref(), &header).await?; + info!( + target: LOG_TARGET, + "🥩 Loading BEEFY voter state from genesis on what appears to be first startup. \ + Starting voting rounds at block {:?}, genesis validator set {:?}.", + beefy_genesis, + genesis_set, + ); + + sessions.push_front(Rounds::new(beefy_genesis, genesis_set)); + break PersistedState::checked_new( + best_grandpa, + Zero::zero(), + sessions, + min_block_delta, + beefy_genesis, + ) + .ok_or_else(|| Error::Backend("Invalid BEEFY chain".into()))? + } + + if let Some(active) = find_authorities_change::(&header) { + debug!( + target: LOG_TARGET, + "🥩 Marking block {:?} as BEEFY Mandatory.", + *header.number() + ); + sessions.push_front(Rounds::new(*header.number(), active)); + } + + // Move up the chain. + header = wait_for_parent_header(blockchain, header, HEADER_SYNC_DELAY).await?; + }; + + aux_schema::write_current_version(backend.as_ref())?; + aux_schema::write_voter_state(backend.as_ref(), &state)?; + Ok(state) + } + + async fn load_or_init_state( + beefy_genesis: NumberFor, + best_grandpa: ::Header, + min_block_delta: u32, + backend: Arc, + runtime: Arc, + key_store: &BeefyKeystore, + metrics: &Option, + ) -> Result, Error> { + // Initialize voter state from AUX DB if compatible. + if let Some(mut state) = crate::aux_schema::load_persistent(backend.as_ref())? + // Verify state pallet genesis matches runtime. + .filter(|state| state.pallet_genesis() == beefy_genesis) + { + // Overwrite persisted state with current best GRANDPA block. + state.set_best_grandpa(best_grandpa.clone()); + // Overwrite persisted data with newly provided `min_block_delta`. + state.set_min_block_delta(min_block_delta); + debug!(target: LOG_TARGET, "🥩 Loading BEEFY voter state from db: {:?}.", state); + + // Make sure that all the headers that we need have been synced. + let mut new_sessions = vec![]; + let mut header = best_grandpa.clone(); + while *header.number() > state.best_beefy() { + if state.voting_oracle().can_add_session(*header.number()) { + if let Some(active) = find_authorities_change::(&header) { + new_sessions.push((active, *header.number())); + } + } + header = + wait_for_parent_header(backend.blockchain(), header, HEADER_SYNC_DELAY).await?; + } + + // Make sure we didn't miss any sessions during node restart. + for (validator_set, new_session_start) in new_sessions.drain(..).rev() { + debug!( + target: LOG_TARGET, + "🥩 Handling missed BEEFY session after node restart: {:?}.", + new_session_start + ); + state.init_session_at(new_session_start, validator_set, key_store, metrics); + } + return Ok(state) + } + + // No valid voter-state persisted, re-initialize from pallet genesis. + Self::init_state(beefy_genesis, best_grandpa, min_block_delta, backend, runtime).await + } +} /// Start the BEEFY gadget. /// @@ -277,7 +532,7 @@ pub async fn start_beefy_gadget( known_peers, prometheus_registry.clone(), ); - let mut beefy_comms = worker::BeefyComms { + let mut beefy_comms = BeefyComms { gossip_engine, gossip_validator, gossip_report_stream, @@ -287,57 +542,45 @@ pub async fn start_beefy_gadget( // We re-create and re-run the worker in this loop in order to quickly reinit and resume after // select recoverable errors. loop { - // Wait for BEEFY pallet to be active before starting voter. - let (beefy_genesis, best_grandpa) = match wait_for_runtime_pallet( - &*runtime, - &mut beefy_comms.gossip_engine, - &mut finality_notifications, - ) - .await - { - Ok(res) => res, - Err(e) => { - error!(target: LOG_TARGET, "Error: {:?}. Terminating.", e); - return - }, - }; - - let mut worker_base = worker::BeefyWorkerBase { - backend: backend.clone(), - runtime: runtime.clone(), - key_store: key_store.clone().into(), - metrics: metrics.clone(), - _phantom: Default::default(), - }; - - let persisted_state = match worker_base - .load_or_init_state(beefy_genesis, best_grandpa, min_block_delta) - .await - { - Ok(state) => state, - Err(e) => { - error!(target: LOG_TARGET, "Error: {:?}. Terminating.", e); - return - }, + // Make sure to pump gossip engine while waiting for initialization conditions. + let worker_builder = loop { + futures::select! { + builder_init_result = BeefyWorkerBuilder::async_initialize( + backend.clone(), + runtime.clone(), + key_store.clone().into(), + metrics.clone(), + min_block_delta, + beefy_comms.gossip_validator.clone(), + &mut finality_notifications, + ).fuse() => { + match builder_init_result { + Ok(builder) => break builder, + Err(e) => { + error!(target: LOG_TARGET, "🥩 Error: {:?}. Terminating.", e); + return + }, + } + }, + // Pump peer reports + _ = &mut beefy_comms.gossip_report_stream.next() => { + continue + }, + // Pump gossip engine. + _ = &mut beefy_comms.gossip_engine => { + error!(target: LOG_TARGET, "🥩 Gossip engine has unexpectedly terminated."); + return + } + } }; - // Update the gossip validator with the right starting round and set id. - if let Err(e) = persisted_state - .gossip_filter_config() - .map(|f| beefy_comms.gossip_validator.update_filter(f)) - { - error!(target: LOG_TARGET, "Error: {:?}. Terminating.", e); - return - } - let worker = worker::BeefyWorker { - base: worker_base, - payload_provider: payload_provider.clone(), - sync: sync.clone(), - comms: beefy_comms, - links: links.clone(), - pending_justifications: BTreeMap::new(), - persisted_state, - }; + let worker = worker_builder.build( + payload_provider.clone(), + sync.clone(), + beefy_comms, + links.clone(), + BTreeMap::new(), + ); match futures::future::select( Box::pin(worker.run(&mut block_import_justif, &mut finality_notifications)), @@ -404,9 +647,8 @@ where /// Should be called only once during worker initialization. async fn wait_for_runtime_pallet( runtime: &R, - mut gossip_engine: &mut GossipEngine, finality: &mut Fuse>, -) -> ClientResult<(NumberFor, ::Header)> +) -> Result<(NumberFor, ::Header), Error> where B: Block, R: ProvideRuntimeApi, @@ -414,33 +656,24 @@ where { info!(target: LOG_TARGET, "🥩 BEEFY gadget waiting for BEEFY pallet to become available..."); loop { - futures::select! { - notif = finality.next() => { - let notif = match notif { - Some(notif) => notif, - None => break - }; - let at = notif.header.hash(); - if let Some(start) = runtime.runtime_api().beefy_genesis(at).ok().flatten() { - if *notif.header.number() >= start { - // Beefy pallet available, return header for best grandpa at the time. - info!( - target: LOG_TARGET, - "🥩 BEEFY pallet available: block {:?} beefy genesis {:?}", - notif.header.number(), start - ); - return Ok((start, notif.header)) - } - } - }, - _ = gossip_engine => { - break + let notif = finality.next().await.ok_or_else(|| { + let err_msg = "🥩 Finality stream has unexpectedly terminated.".into(); + error!(target: LOG_TARGET, "{}", err_msg); + Error::Backend(err_msg) + })?; + let at = notif.header.hash(); + if let Some(start) = runtime.runtime_api().beefy_genesis(at).ok().flatten() { + if *notif.header.number() >= start { + // Beefy pallet available, return header for best grandpa at the time. + info!( + target: LOG_TARGET, + "🥩 BEEFY pallet available: block {:?} beefy genesis {:?}", + notif.header.number(), start + ); + return Ok((start, notif.header)) } } } - let err_msg = "🥩 Gossip engine has unexpectedly terminated.".into(); - error!(target: LOG_TARGET, "{}", err_msg); - Err(ClientError::Backend(err_msg)) } /// Provides validator set active `at_header`. It tries to get it from state, otherwise falls @@ -474,7 +707,7 @@ where if let Ok(Some(active)) = runtime.runtime_api().validator_set(header.hash()) { return Ok(active) } else { - match worker::find_authorities_change::(&header) { + match find_authorities_change::(&header) { Some(active) => return Ok(active), // Move up the chain. Ultimately we'll get it from chain genesis state, or error out // there. @@ -486,3 +719,18 @@ where } } } + +/// Scan the `header` digest log for a BEEFY validator set change. Return either the new +/// validator set or `None` in case no validator set change has been signaled. +pub(crate) fn find_authorities_change(header: &B::Header) -> Option> +where + B: Block, +{ + let id = OpaqueDigestItemId::Consensus(&BEEFY_ENGINE_ID); + + let filter = |log: ConsensusLog| match log { + ConsensusLog::AuthoritiesChange(validator_set) => Some(validator_set), + _ => None, + }; + header.digest().convert_first(|l| l.try_to(id).and_then(filter)) +} diff --git a/substrate/client/consensus/beefy/src/tests.rs b/substrate/client/consensus/beefy/src/tests.rs index 0e6ff210b158..d106c9dcd881 100644 --- a/substrate/client/consensus/beefy/src/tests.rs +++ b/substrate/client/consensus/beefy/src/tests.rs @@ -32,8 +32,8 @@ use crate::{ gossip_protocol_name, justification::*, wait_for_runtime_pallet, - worker::{BeefyWorkerBase, PersistedState}, - BeefyRPCLinks, BeefyVoterLinks, KnownPeers, + worker::PersistedState, + BeefyRPCLinks, BeefyVoterLinks, BeefyWorkerBuilder, KnownPeers, }; use futures::{future, stream::FuturesUnordered, Future, FutureExt, StreamExt}; use parking_lot::Mutex; @@ -368,27 +368,19 @@ async fn voter_init_setup( api: &TestApi, ) -> Result, Error> { let backend = net.peer(0).client().as_backend(); - let known_peers = Arc::new(Mutex::new(KnownPeers::new())); - let (gossip_validator, _) = GossipValidator::new(known_peers); - let gossip_validator = Arc::new(gossip_validator); - let mut gossip_engine = sc_network_gossip::GossipEngine::new( - net.peer(0).network_service().clone(), - net.peer(0).sync_service().clone(), - net.peer(0).take_notification_service(&beefy_gossip_proto_name()).unwrap(), - "/beefy/whatever", - gossip_validator, - None, - ); - let (beefy_genesis, best_grandpa) = - wait_for_runtime_pallet(api, &mut gossip_engine, finality).await.unwrap(); - let mut worker_base = BeefyWorkerBase { + let (beefy_genesis, best_grandpa) = wait_for_runtime_pallet(api, finality).await.unwrap(); + let key_store = None.into(); + let metrics = None; + BeefyWorkerBuilder::load_or_init_state( + beefy_genesis, + best_grandpa, + 1, backend, - runtime: Arc::new(api.clone()), - key_store: None.into(), - metrics: None, - _phantom: Default::default(), - }; - worker_base.load_or_init_state(beefy_genesis, best_grandpa, 1).await + Arc::new(api.clone()), + &key_store, + &metrics, + ) + .await } // Spawns beefy voters. Returns a future to spawn on the runtime. @@ -1065,32 +1057,7 @@ async fn should_initialize_voter_at_custom_genesis() { net.peer(0).client().as_client().finalize_block(hashes[8], None).unwrap(); // load persistent state - nothing in DB, should init at genesis - // - // NOTE: code from `voter_init_setup()` is moved here because the new network event system - // doesn't allow creating a new `GossipEngine` as the notification handle is consumed by the - // first `GossipEngine` - let known_peers = Arc::new(Mutex::new(KnownPeers::new())); - let (gossip_validator, _) = GossipValidator::new(known_peers); - let gossip_validator = Arc::new(gossip_validator); - let mut gossip_engine = sc_network_gossip::GossipEngine::new( - net.peer(0).network_service().clone(), - net.peer(0).sync_service().clone(), - net.peer(0).take_notification_service(&beefy_gossip_proto_name()).unwrap(), - "/beefy/whatever", - gossip_validator, - None, - ); - let (beefy_genesis, best_grandpa) = - wait_for_runtime_pallet(&api, &mut gossip_engine, &mut finality).await.unwrap(); - let mut worker_base = BeefyWorkerBase { - backend: backend.clone(), - runtime: Arc::new(api), - key_store: None.into(), - metrics: None, - _phantom: Default::default(), - }; - let persisted_state = - worker_base.load_or_init_state(beefy_genesis, best_grandpa, 1).await.unwrap(); + let persisted_state = voter_init_setup(&mut net, &mut finality, &api).await.unwrap(); // Test initialization at session boundary. // verify voter initialized with single session starting at block `custom_pallet_genesis` (7) @@ -1120,18 +1087,7 @@ async fn should_initialize_voter_at_custom_genesis() { net.peer(0).client().as_client().finalize_block(hashes[10], None).unwrap(); // load persistent state - state preset in DB, but with different pallet genesis - // the network state persists and uses the old `GossipEngine` initialized for `peer(0)` - let (beefy_genesis, best_grandpa) = - wait_for_runtime_pallet(&api, &mut gossip_engine, &mut finality).await.unwrap(); - let mut worker_base = BeefyWorkerBase { - backend: backend.clone(), - runtime: Arc::new(api), - key_store: None.into(), - metrics: None, - _phantom: Default::default(), - }; - let new_persisted_state = - worker_base.load_or_init_state(beefy_genesis, best_grandpa, 1).await.unwrap(); + let new_persisted_state = voter_init_setup(&mut net, &mut finality, &api).await.unwrap(); // verify voter initialized with single session starting at block `new_pallet_genesis` (10) let sessions = new_persisted_state.voting_oracle().sessions(); @@ -1322,32 +1278,7 @@ async fn should_catch_up_when_loading_saved_voter_state() { let api = TestApi::with_validator_set(&validator_set); // load persistent state - nothing in DB, should init at genesis - // - // NOTE: code from `voter_init_setup()` is moved here because the new network event system - // doesn't allow creating a new `GossipEngine` as the notification handle is consumed by the - // first `GossipEngine` - let known_peers = Arc::new(Mutex::new(KnownPeers::new())); - let (gossip_validator, _) = GossipValidator::new(known_peers); - let gossip_validator = Arc::new(gossip_validator); - let mut gossip_engine = sc_network_gossip::GossipEngine::new( - net.peer(0).network_service().clone(), - net.peer(0).sync_service().clone(), - net.peer(0).take_notification_service(&beefy_gossip_proto_name()).unwrap(), - "/beefy/whatever", - gossip_validator, - None, - ); - let (beefy_genesis, best_grandpa) = - wait_for_runtime_pallet(&api, &mut gossip_engine, &mut finality).await.unwrap(); - let mut worker_base = BeefyWorkerBase { - backend: backend.clone(), - runtime: Arc::new(api.clone()), - key_store: None.into(), - metrics: None, - _phantom: Default::default(), - }; - let persisted_state = - worker_base.load_or_init_state(beefy_genesis, best_grandpa, 1).await.unwrap(); + let persisted_state = voter_init_setup(&mut net, &mut finality, &api).await.unwrap(); // Test initialization at session boundary. // verify voter initialized with two sessions starting at blocks 1 and 10 @@ -1374,18 +1305,7 @@ async fn should_catch_up_when_loading_saved_voter_state() { // finalize 25 without justifications net.peer(0).client().as_client().finalize_block(hashes[25], None).unwrap(); // load persistent state - state preset in DB - // the network state persists and uses the old `GossipEngine` initialized for `peer(0)` - let (beefy_genesis, best_grandpa) = - wait_for_runtime_pallet(&api, &mut gossip_engine, &mut finality).await.unwrap(); - let mut worker_base = BeefyWorkerBase { - backend: backend.clone(), - runtime: Arc::new(api), - key_store: None.into(), - metrics: None, - _phantom: Default::default(), - }; - let persisted_state = - worker_base.load_or_init_state(beefy_genesis, best_grandpa, 1).await.unwrap(); + let persisted_state = voter_init_setup(&mut net, &mut finality, &api).await.unwrap(); // Verify voter initialized with old sessions plus a new one starting at block 20. // There shouldn't be any duplicates. diff --git a/substrate/client/consensus/beefy/src/worker.rs b/substrate/client/consensus/beefy/src/worker.rs index d7a229003cc4..c8eb19621ba5 100644 --- a/substrate/client/consensus/beefy/src/worker.rs +++ b/substrate/client/consensus/beefy/src/worker.rs @@ -17,46 +17,42 @@ // along with this program. If not, see . use crate::{ - aux_schema, communication::{ - gossip::{proofs_topic, votes_topic, GossipFilterCfg, GossipMessage, GossipValidator}, + gossip::{proofs_topic, votes_topic, GossipFilterCfg, GossipMessage}, peers::PeerReport, - request_response::outgoing_requests_engine::{OnDemandJustificationsEngine, ResponseInfo}, + request_response::outgoing_requests_engine::ResponseInfo, }, error::Error, - expect_validator_set, + find_authorities_change, justification::BeefyVersionedFinalityProof, keystore::BeefyKeystore, metric_inc, metric_set, metrics::VoterMetrics, round::{Rounds, VoteImportResult}, - wait_for_parent_header, BeefyVoterLinks, HEADER_SYNC_DELAY, LOG_TARGET, + BeefyComms, BeefyVoterLinks, LOG_TARGET, }; use codec::{Codec, Decode, DecodeAll, Encode}; use futures::{stream::Fuse, FutureExt, StreamExt}; use log::{debug, error, info, log_enabled, trace, warn}; use sc_client_api::{Backend, FinalityNotification, FinalityNotifications, HeaderBackend}; -use sc_network_gossip::GossipEngine; -use sc_utils::{mpsc::TracingUnboundedReceiver, notification::NotificationReceiver}; +use sc_utils::notification::NotificationReceiver; use sp_api::ProvideRuntimeApi; use sp_arithmetic::traits::{AtLeast32Bit, Saturating}; -use sp_blockchain::Backend as BlockchainBackend; use sp_consensus::SyncOracle; use sp_consensus_beefy::{ check_equivocation_proof, ecdsa_crypto::{AuthorityId, Signature}, - BeefyApi, BeefySignatureHasher, Commitment, ConsensusLog, EquivocationProof, PayloadProvider, - ValidatorSet, VersionedFinalityProof, VoteMessage, BEEFY_ENGINE_ID, + BeefyApi, BeefySignatureHasher, Commitment, EquivocationProof, PayloadProvider, ValidatorSet, + VersionedFinalityProof, VoteMessage, BEEFY_ENGINE_ID, }; use sp_runtime::{ - generic::{BlockId, OpaqueDigestItemId}, + generic::BlockId, traits::{Block, Header, NumberFor, Zero}, SaturatedConversion, }; use std::{ collections::{BTreeMap, BTreeSet, VecDeque}, fmt::Debug, - marker::PhantomData, sync::Arc, }; @@ -180,8 +176,8 @@ impl VoterOracle { } } - // Check if an observed session can be added to the Oracle. - fn can_add_session(&self, session_start: NumberFor) -> bool { + /// Check if an observed session can be added to the Oracle. + pub fn can_add_session(&self, session_start: NumberFor) -> bool { let latest_known_session_start = self.sessions.back().map(|session| session.session_start()); Some(session_start) > latest_known_session_start @@ -319,229 +315,28 @@ impl PersistedState { self.voting_oracle.best_grandpa_block_header = best_grandpa; } + pub fn voting_oracle(&self) -> &VoterOracle { + &self.voting_oracle + } + pub(crate) fn gossip_filter_config(&self) -> Result, Error> { let (start, end) = self.voting_oracle.accepted_interval()?; let validator_set = self.voting_oracle.current_validator_set()?; Ok(GossipFilterCfg { start, end, validator_set }) } -} - -/// Helper object holding BEEFY worker communication/gossip components. -/// -/// These are created once, but will be reused if worker is restarted/reinitialized. -pub(crate) struct BeefyComms { - pub gossip_engine: GossipEngine, - pub gossip_validator: Arc>, - pub gossip_report_stream: TracingUnboundedReceiver, - pub on_demand_justifications: OnDemandJustificationsEngine, -} - -pub(crate) struct BeefyWorkerBase { - // utilities - pub backend: Arc, - pub runtime: Arc, - pub key_store: BeefyKeystore, - - /// BEEFY client metrics. - pub metrics: Option, - - pub _phantom: PhantomData, -} - -impl BeefyWorkerBase -where - B: Block + Codec, - BE: Backend, - R: ProvideRuntimeApi, - R::Api: BeefyApi, -{ - // If no persisted state present, walk back the chain from first GRANDPA notification to either: - // - latest BEEFY finalized block, or if none found on the way, - // - BEEFY pallet genesis; - // Enqueue any BEEFY mandatory blocks (session boundaries) found on the way, for voter to - // finalize. - async fn init_state( - &self, - beefy_genesis: NumberFor, - best_grandpa: ::Header, - min_block_delta: u32, - ) -> Result, Error> { - let blockchain = self.backend.blockchain(); - - let beefy_genesis = self - .runtime - .runtime_api() - .beefy_genesis(best_grandpa.hash()) - .ok() - .flatten() - .filter(|genesis| *genesis == beefy_genesis) - .ok_or_else(|| Error::Backend("BEEFY pallet expected to be active.".into()))?; - // Walk back the imported blocks and initialize voter either, at the last block with - // a BEEFY justification, or at pallet genesis block; voter will resume from there. - let mut sessions = VecDeque::new(); - let mut header = best_grandpa.clone(); - let state = loop { - if let Some(true) = blockchain - .justifications(header.hash()) - .ok() - .flatten() - .map(|justifs| justifs.get(BEEFY_ENGINE_ID).is_some()) - { - debug!( - target: LOG_TARGET, - "🥩 Initialize BEEFY voter at last BEEFY finalized block: {:?}.", - *header.number() - ); - let best_beefy = *header.number(); - // If no session boundaries detected so far, just initialize new rounds here. - if sessions.is_empty() { - let active_set = - expect_validator_set(self.runtime.as_ref(), self.backend.as_ref(), &header) - .await?; - let mut rounds = Rounds::new(best_beefy, active_set); - // Mark the round as already finalized. - rounds.conclude(best_beefy); - sessions.push_front(rounds); - } - let state = PersistedState::checked_new( - best_grandpa, - best_beefy, - sessions, - min_block_delta, - beefy_genesis, - ) - .ok_or_else(|| Error::Backend("Invalid BEEFY chain".into()))?; - break state - } - - if *header.number() == beefy_genesis { - // We've reached BEEFY genesis, initialize voter here. - let genesis_set = - expect_validator_set(self.runtime.as_ref(), self.backend.as_ref(), &header) - .await?; - info!( - target: LOG_TARGET, - "🥩 Loading BEEFY voter state from genesis on what appears to be first startup. \ - Starting voting rounds at block {:?}, genesis validator set {:?}.", - beefy_genesis, - genesis_set, - ); - - sessions.push_front(Rounds::new(beefy_genesis, genesis_set)); - break PersistedState::checked_new( - best_grandpa, - Zero::zero(), - sessions, - min_block_delta, - beefy_genesis, - ) - .ok_or_else(|| Error::Backend("Invalid BEEFY chain".into()))? - } - - if let Some(active) = find_authorities_change::(&header) { - debug!( - target: LOG_TARGET, - "🥩 Marking block {:?} as BEEFY Mandatory.", - *header.number() - ); - sessions.push_front(Rounds::new(*header.number(), active)); - } - - // Move up the chain. - header = wait_for_parent_header(blockchain, header, HEADER_SYNC_DELAY).await?; - }; - - aux_schema::write_current_version(self.backend.as_ref())?; - aux_schema::write_voter_state(self.backend.as_ref(), &state)?; - Ok(state) - } - - pub async fn load_or_init_state( - &mut self, - beefy_genesis: NumberFor, - best_grandpa: ::Header, - min_block_delta: u32, - ) -> Result, Error> { - // Initialize voter state from AUX DB if compatible. - if let Some(mut state) = crate::aux_schema::load_persistent(self.backend.as_ref())? - // Verify state pallet genesis matches runtime. - .filter(|state| state.pallet_genesis() == beefy_genesis) - { - // Overwrite persisted state with current best GRANDPA block. - state.set_best_grandpa(best_grandpa.clone()); - // Overwrite persisted data with newly provided `min_block_delta`. - state.set_min_block_delta(min_block_delta); - debug!(target: LOG_TARGET, "🥩 Loading BEEFY voter state from db: {:?}.", state); - - // Make sure that all the headers that we need have been synced. - let mut new_sessions = vec![]; - let mut header = best_grandpa.clone(); - while *header.number() > state.best_beefy() { - if state.voting_oracle.can_add_session(*header.number()) { - if let Some(active) = find_authorities_change::(&header) { - new_sessions.push((active, *header.number())); - } - } - header = - wait_for_parent_header(self.backend.blockchain(), header, HEADER_SYNC_DELAY) - .await?; - } - - // Make sure we didn't miss any sessions during node restart. - for (validator_set, new_session_start) in new_sessions.drain(..).rev() { - debug!( - target: LOG_TARGET, - "🥩 Handling missed BEEFY session after node restart: {:?}.", - new_session_start - ); - self.init_session_at(&mut state, validator_set, new_session_start); - } - return Ok(state) - } - - // No valid voter-state persisted, re-initialize from pallet genesis. - self.init_state(beefy_genesis, best_grandpa, min_block_delta).await - } - - /// Verify `active` validator set for `block` against the key store - /// - /// We want to make sure that we have _at least one_ key in our keystore that - /// is part of the validator set, that's because if there are no local keys - /// then we can't perform our job as a validator. - /// - /// Note that for a non-authority node there will be no keystore, and we will - /// return an error and don't check. The error can usually be ignored. - fn verify_validator_set( - &self, - block: &NumberFor, - active: &ValidatorSet, - ) -> Result<(), Error> { - let active: BTreeSet<&AuthorityId> = active.validators().iter().collect(); - - let public_keys = self.key_store.public_keys()?; - let store: BTreeSet<&AuthorityId> = public_keys.iter().collect(); - - if store.intersection(&active).count() == 0 { - let msg = "no authority public key found in store".to_string(); - debug!(target: LOG_TARGET, "🥩 for block {:?} {}", block, msg); - metric_inc!(self.metrics, beefy_no_authority_found_in_store); - Err(Error::Keystore(msg)) - } else { - Ok(()) - } - } /// Handle session changes by starting new voting round for mandatory blocks. - fn init_session_at( + pub fn init_session_at( &mut self, - persisted_state: &mut PersistedState, - validator_set: ValidatorSet, new_session_start: NumberFor, + validator_set: ValidatorSet, + key_store: &BeefyKeystore, + metrics: &Option, ) { debug!(target: LOG_TARGET, "🥩 New active validator set: {:?}", validator_set); // BEEFY should finalize a mandatory block during each session. - if let Ok(active_session) = persisted_state.voting_oracle.active_rounds() { + if let Ok(active_session) = self.voting_oracle.active_rounds() { if !active_session.mandatory_done() { debug!( target: LOG_TARGET, @@ -549,20 +344,20 @@ where validator_set.id(), active_session.validator_set_id(), ); - metric_inc!(self.metrics, beefy_lagging_sessions); + metric_inc!(metrics, beefy_lagging_sessions); } } if log_enabled!(target: LOG_TARGET, log::Level::Debug) { // verify the new validator set - only do it if we're also logging the warning - let _ = self.verify_validator_set(&new_session_start, &validator_set); + if verify_validator_set::(&new_session_start, &validator_set, key_store).is_err() { + metric_inc!(metrics, beefy_no_authority_found_in_store); + } } let id = validator_set.id(); - persisted_state - .voting_oracle - .add_session(Rounds::new(new_session_start, validator_set)); - metric_set!(self.metrics, beefy_validator_set_id, id); + self.voting_oracle.add_session(Rounds::new(new_session_start, validator_set)); + metric_set!(metrics, beefy_validator_set_id, id); info!( target: LOG_TARGET, "🥩 New Rounds for validator set id: {:?} with session_start {:?}", @@ -574,9 +369,10 @@ where /// A BEEFY worker/voter that follows the BEEFY protocol pub(crate) struct BeefyWorker { - pub base: BeefyWorkerBase, - - // utils + // utilities + pub backend: Arc, + pub runtime: Arc, + pub key_store: BeefyKeystore, pub payload_provider: P, pub sync: Arc, @@ -592,6 +388,8 @@ pub(crate) struct BeefyWorker { pub pending_justifications: BTreeMap, BeefyVersionedFinalityProof>, /// Persisted voter state. pub persisted_state: PersistedState, + /// BEEFY voter metrics + pub metrics: Option, } impl BeefyWorker @@ -622,8 +420,12 @@ where validator_set: ValidatorSet, new_session_start: NumberFor, ) { - self.base - .init_session_at(&mut self.persisted_state, validator_set, new_session_start); + self.persisted_state.init_session_at( + new_session_start, + validator_set, + &self.key_store, + &self.metrics, + ); } fn handle_finality_notification( @@ -639,8 +441,7 @@ where notification.tree_route, ); - self.base - .runtime + self.runtime .runtime_api() .beefy_genesis(header.hash()) .ok() @@ -654,7 +455,7 @@ where self.persisted_state.set_best_grandpa(header.clone()); // Check all (newly) finalized blocks for new session(s). - let backend = self.base.backend.clone(); + let backend = self.backend.clone(); for header in notification .tree_route .iter() @@ -673,7 +474,7 @@ where } if new_session_added { - crate::aux_schema::write_voter_state(&*self.base.backend, &self.persisted_state) + crate::aux_schema::write_voter_state(&*self.backend, &self.persisted_state) .map_err(|e| Error::Backend(e.to_string()))?; } @@ -707,7 +508,7 @@ where true, ); }, - RoundAction::Drop => metric_inc!(self.base.metrics, beefy_stale_votes), + RoundAction::Drop => metric_inc!(self.metrics, beefy_stale_votes), RoundAction::Enqueue => error!(target: LOG_TARGET, "🥩 unexpected vote: {:?}.", vote), }; Ok(()) @@ -727,23 +528,23 @@ where match self.voting_oracle().triage_round(block_num)? { RoundAction::Process => { debug!(target: LOG_TARGET, "🥩 Process justification for round: {:?}.", block_num); - metric_inc!(self.base.metrics, beefy_imported_justifications); + metric_inc!(self.metrics, beefy_imported_justifications); self.finalize(justification)? }, RoundAction::Enqueue => { debug!(target: LOG_TARGET, "🥩 Buffer justification for round: {:?}.", block_num); if self.pending_justifications.len() < MAX_BUFFERED_JUSTIFICATIONS { self.pending_justifications.entry(block_num).or_insert(justification); - metric_inc!(self.base.metrics, beefy_buffered_justifications); + metric_inc!(self.metrics, beefy_buffered_justifications); } else { - metric_inc!(self.base.metrics, beefy_buffered_justifications_dropped); + metric_inc!(self.metrics, beefy_buffered_justifications_dropped); warn!( target: LOG_TARGET, "🥩 Buffer justification dropped for round: {:?}.", block_num ); } }, - RoundAction::Drop => metric_inc!(self.base.metrics, beefy_stale_justifications), + RoundAction::Drop => metric_inc!(self.metrics, beefy_stale_justifications), }; Ok(()) } @@ -765,7 +566,7 @@ where // We created the `finality_proof` and know to be valid. // New state is persisted after finalization. self.finalize(finality_proof.clone())?; - metric_inc!(self.base.metrics, beefy_good_votes_processed); + metric_inc!(self.metrics, beefy_good_votes_processed); return Ok(Some(finality_proof)) }, VoteImportResult::Ok => { @@ -776,20 +577,17 @@ where .map(|(mandatory_num, _)| mandatory_num == block_number) .unwrap_or(false) { - crate::aux_schema::write_voter_state( - &*self.base.backend, - &self.persisted_state, - ) - .map_err(|e| Error::Backend(e.to_string()))?; + crate::aux_schema::write_voter_state(&*self.backend, &self.persisted_state) + .map_err(|e| Error::Backend(e.to_string()))?; } - metric_inc!(self.base.metrics, beefy_good_votes_processed); + metric_inc!(self.metrics, beefy_good_votes_processed); }, VoteImportResult::Equivocation(proof) => { - metric_inc!(self.base.metrics, beefy_equivocation_votes); + metric_inc!(self.metrics, beefy_equivocation_votes); self.report_equivocation(proof)?; }, - VoteImportResult::Invalid => metric_inc!(self.base.metrics, beefy_invalid_votes), - VoteImportResult::Stale => metric_inc!(self.base.metrics, beefy_stale_votes), + VoteImportResult::Invalid => metric_inc!(self.metrics, beefy_invalid_votes), + VoteImportResult::Stale => metric_inc!(self.metrics, beefy_stale_votes), }; Ok(None) } @@ -816,15 +614,14 @@ where // Set new best BEEFY block number. self.persisted_state.set_best_beefy(block_num); - crate::aux_schema::write_voter_state(&*self.base.backend, &self.persisted_state) + crate::aux_schema::write_voter_state(&*self.backend, &self.persisted_state) .map_err(|e| Error::Backend(e.to_string()))?; - metric_set!(self.base.metrics, beefy_best_block, block_num); + metric_set!(self.metrics, beefy_best_block, block_num); self.comms.on_demand_justifications.cancel_requests_older_than(block_num); if let Err(e) = self - .base .backend .blockchain() .expect_block_hash_from_id(&BlockId::Number(block_num)) @@ -834,8 +631,7 @@ where .notify(|| Ok::<_, ()>(hash)) .expect("forwards closure result; the closure always returns Ok; qed."); - self.base - .backend + self.backend .append_justification(hash, (BEEFY_ENGINE_ID, finality_proof.encode())) }) { debug!( @@ -872,13 +668,13 @@ where for (num, justification) in justifs_to_process.into_iter() { debug!(target: LOG_TARGET, "🥩 Handle buffered justification for: {:?}.", num); - metric_inc!(self.base.metrics, beefy_imported_justifications); + metric_inc!(self.metrics, beefy_imported_justifications); if let Err(err) = self.finalize(justification) { error!(target: LOG_TARGET, "🥩 Error finalizing block: {}", err); } } metric_set!( - self.base.metrics, + self.metrics, beefy_buffered_justifications, self.pending_justifications.len() ); @@ -890,7 +686,7 @@ where fn try_to_vote(&mut self) -> Result<(), Error> { // Vote if there's now a new vote target. if let Some(target) = self.voting_oracle().voting_target() { - metric_set!(self.base.metrics, beefy_should_vote_on, target); + metric_set!(self.metrics, beefy_should_vote_on, target); if target > self.persisted_state.best_voted { self.do_vote(target)?; } @@ -910,7 +706,6 @@ where self.persisted_state.voting_oracle.best_grandpa_block_header.clone() } else { let hash = self - .base .backend .blockchain() .expect_block_hash_from_id(&BlockId::Number(target_number)) @@ -922,7 +717,7 @@ where Error::Backend(err_msg) })?; - self.base.backend.blockchain().expect_header(hash).map_err(|err| { + self.backend.blockchain().expect_header(hash).map_err(|err| { let err_msg = format!( "Couldn't get header for block #{:?} ({:?}) (error: {:?}), skipping vote..", target_number, hash, err @@ -942,7 +737,7 @@ where let rounds = self.persisted_state.voting_oracle.active_rounds_mut()?; let (validators, validator_set_id) = (rounds.validators(), rounds.validator_set_id()); - let authority_id = if let Some(id) = self.base.key_store.authority_id(validators) { + let authority_id = if let Some(id) = self.key_store.authority_id(validators) { debug!(target: LOG_TARGET, "🥩 Local authority id: {:?}", id); id } else { @@ -956,7 +751,7 @@ where let commitment = Commitment { payload, block_number: target_number, validator_set_id }; let encoded_commitment = commitment.encode(); - let signature = match self.base.key_store.sign(&authority_id, &encoded_commitment) { + let signature = match self.key_store.sign(&authority_id, &encoded_commitment) { Ok(sig) => sig, Err(err) => { warn!(target: LOG_TARGET, "🥩 Error signing commitment: {:?}", err); @@ -981,7 +776,7 @@ where .gossip_engine .gossip_message(proofs_topic::(), encoded_proof, true); } else { - metric_inc!(self.base.metrics, beefy_votes_sent); + metric_inc!(self.metrics, beefy_votes_sent); debug!(target: LOG_TARGET, "🥩 Sent vote message: {:?}", vote); let encoded_vote = GossipMessage::::Vote(vote).encode(); self.comms.gossip_engine.gossip_message(votes_topic::(), encoded_vote, false); @@ -989,8 +784,8 @@ where // Persist state after vote to avoid double voting in case of voter restarts. self.persisted_state.best_voted = target_number; - metric_set!(self.base.metrics, beefy_best_voted, target_number); - crate::aux_schema::write_voter_state(&*self.base.backend, &self.persisted_state) + metric_set!(self.metrics, beefy_best_voted, target_number); + crate::aux_schema::write_voter_state(&*self.backend, &self.persisted_state) .map_err(|e| Error::Backend(e.to_string())) } @@ -1164,7 +959,7 @@ where if !check_equivocation_proof::<_, _, BeefySignatureHasher>(&proof) { debug!(target: LOG_TARGET, "🥩 Skip report for bad equivocation {:?}", proof); return Ok(()) - } else if let Some(local_id) = self.base.key_store.authority_id(validators) { + } else if let Some(local_id) = self.key_store.authority_id(validators) { if offender_id == local_id { warn!(target: LOG_TARGET, "🥩 Skip equivocation report for own equivocation"); return Ok(()) @@ -1173,7 +968,6 @@ where let number = *proof.round_number(); let hash = self - .base .backend .blockchain() .expect_block_hash_from_id(&BlockId::Number(number)) @@ -1184,7 +978,7 @@ where ); Error::Backend(err_msg) })?; - let runtime_api = self.base.runtime.runtime_api(); + let runtime_api = self.runtime.runtime_api(); // generate key ownership proof at that block let key_owner_proof = match runtime_api .generate_key_ownership_proof(hash, validator_set_id, offender_id) @@ -1201,7 +995,7 @@ where }; // submit equivocation report at **best** block - let best_block_hash = self.base.backend.blockchain().info().best_hash; + let best_block_hash = self.backend.blockchain().info().best_hash; runtime_api .submit_report_equivocation_unsigned_extrinsic(best_block_hash, proof, key_owner_proof) .map_err(Error::RuntimeApi)?; @@ -1210,21 +1004,6 @@ where } } -/// Scan the `header` digest log for a BEEFY validator set change. Return either the new -/// validator set or `None` in case no validator set change has been signaled. -pub(crate) fn find_authorities_change(header: &B::Header) -> Option> -where - B: Block, -{ - let id = OpaqueDigestItemId::Consensus(&BEEFY_ENGINE_ID); - - let filter = |log: ConsensusLog| match log { - ConsensusLog::AuthoritiesChange(validator_set) => Some(validator_set), - _ => None, - }; - header.digest().convert_first(|l| l.try_to(id).and_then(filter)) -} - /// Calculate next block number to vote on. /// /// Return `None` if there is no votable target yet. @@ -1261,11 +1040,42 @@ where } } +/// Verify `active` validator set for `block` against the key store +/// +/// We want to make sure that we have _at least one_ key in our keystore that +/// is part of the validator set, that's because if there are no local keys +/// then we can't perform our job as a validator. +/// +/// Note that for a non-authority node there will be no keystore, and we will +/// return an error and don't check. The error can usually be ignored. +fn verify_validator_set( + block: &NumberFor, + active: &ValidatorSet, + key_store: &BeefyKeystore, +) -> Result<(), Error> { + let active: BTreeSet<&AuthorityId> = active.validators().iter().collect(); + + let public_keys = key_store.public_keys()?; + let store: BTreeSet<&AuthorityId> = public_keys.iter().collect(); + + if store.intersection(&active).count() == 0 { + let msg = "no authority public key found in store".to_string(); + debug!(target: LOG_TARGET, "🥩 for block {:?} {}", block, msg); + Err(Error::Keystore(msg)) + } else { + Ok(()) + } +} + #[cfg(test)] pub(crate) mod tests { use super::*; use crate::{ - communication::notification::{BeefyBestBlockStream, BeefyVersionedFinalityProofStream}, + communication::{ + gossip::GossipValidator, + notification::{BeefyBestBlockStream, BeefyVersionedFinalityProofStream}, + request_response::outgoing_requests_engine::OnDemandJustificationsEngine, + }, tests::{ create_beefy_keystore, get_beefy_streams, make_beefy_ids, BeefyPeer, BeefyTestNet, TestApi, @@ -1275,6 +1085,7 @@ pub(crate) mod tests { use futures::{future::poll_fn, task::Poll}; use parking_lot::Mutex; use sc_client_api::{Backend as BackendT, HeaderBackend}; + use sc_network_gossip::GossipEngine; use sc_network_sync::SyncingService; use sc_network_test::TestNetFactory; use sp_blockchain::Backend as BlockchainBackendT; @@ -1283,7 +1094,7 @@ pub(crate) mod tests { known_payloads::MMR_ROOT_ID, mmr::MmrRootProvider, test_utils::{generate_equivocation_proof, Keyring}, - Payload, SignedCommitment, + ConsensusLog, Payload, SignedCommitment, }; use sp_runtime::traits::{Header as HeaderT, One}; use substrate_test_runtime_client::{ @@ -1292,10 +1103,6 @@ pub(crate) mod tests { }; impl PersistedState { - pub fn voting_oracle(&self) -> &VoterOracle { - &self.voting_oracle - } - pub fn active_round(&self) -> Result<&Rounds, Error> { self.voting_oracle.active_rounds() } @@ -1391,13 +1198,10 @@ pub(crate) mod tests { on_demand_justifications, }; BeefyWorker { - base: BeefyWorkerBase { - backend, - runtime: api, - key_store: Some(keystore).into(), - metrics, - _phantom: Default::default(), - }, + backend, + runtime: api, + key_store: Some(keystore).into(), + metrics, payload_provider, sync: Arc::new(sync), links, @@ -1675,19 +1479,22 @@ pub(crate) mod tests { let mut worker = create_beefy_worker(net.peer(0), &keys[0], 1, validator_set.clone()); // keystore doesn't contain other keys than validators' - assert_eq!(worker.base.verify_validator_set(&1, &validator_set), Ok(())); + assert_eq!(verify_validator_set::(&1, &validator_set, &worker.key_store), Ok(())); // unknown `Bob` key let keys = &[Keyring::Bob]; let validator_set = ValidatorSet::new(make_beefy_ids(keys), 0).unwrap(); let err_msg = "no authority public key found in store".to_string(); let expected = Err(Error::Keystore(err_msg)); - assert_eq!(worker.base.verify_validator_set(&1, &validator_set), expected); + assert_eq!(verify_validator_set::(&1, &validator_set, &worker.key_store), expected); // worker has no keystore - worker.base.key_store = None.into(); + worker.key_store = None.into(); let expected_err = Err(Error::Keystore("no Keystore".into())); - assert_eq!(worker.base.verify_validator_set(&1, &validator_set), expected_err); + assert_eq!( + verify_validator_set::(&1, &validator_set, &worker.key_store), + expected_err + ); } #[tokio::test] @@ -1839,7 +1646,7 @@ pub(crate) mod tests { let mut net = BeefyTestNet::new(1); let mut worker = create_beefy_worker(net.peer(0), &keys[0], 1, validator_set.clone()); - worker.base.runtime = api_alice.clone(); + worker.runtime = api_alice.clone(); // let there be a block with num = 1: let _ = net.peer(0).push_blocks(1, false); diff --git a/substrate/client/rpc-spec-v2/src/transaction/event.rs b/substrate/client/rpc-spec-v2/src/transaction/event.rs index 8b80fcda17be..882ac8490b07 100644 --- a/substrate/client/rpc-spec-v2/src/transaction/event.rs +++ b/substrate/client/rpc-spec-v2/src/transaction/event.rs @@ -20,23 +20,6 @@ use serde::{Deserialize, Serialize}; -/// The transaction was broadcasted to a number of peers. -/// -/// # Note -/// -/// The RPC does not guarantee that the peers have received the -/// transaction. -/// -/// When the number of peers is zero, the event guarantees that -/// shutting down the local node will lead to the transaction -/// not being included in the chain. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct TransactionBroadcasted { - /// The number of peers the transaction was broadcasted to. - pub num_peers: usize, -} - /// The transaction was included in a block of the chain. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -59,9 +42,6 @@ pub struct TransactionError { #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct TransactionDropped { - /// True if the transaction was broadcasted to other peers and - /// may still be included in the block. - pub broadcasted: bool, /// Reason of the event. pub error: String, } @@ -70,20 +50,17 @@ pub struct TransactionDropped { /// /// The status events can be grouped based on their kinds as: /// -/// 1. Runtime validated the transaction: +/// 1. Runtime validated the transaction and it entered the pool: /// - `Validated` /// -/// 2. Inside the `Ready` queue: -/// - `Broadcast` -/// -/// 3. Leaving the pool: +/// 2. Leaving the pool: /// - `BestChainBlockIncluded` /// - `Invalid` /// -/// 4. Block finalized: +/// 3. Block finalized: /// - `Finalized` /// -/// 5. At any time: +/// 4. At any time: /// - `Dropped` /// - `Error` /// @@ -101,8 +78,6 @@ pub struct TransactionDropped { pub enum TransactionEvent { /// The transaction was validated by the runtime. Validated, - /// The transaction was broadcasted to a number of peers. - Broadcasted(TransactionBroadcasted), /// The transaction was included in a best block of the chain. /// /// # Note @@ -159,7 +134,6 @@ enum TransactionEventBlockIR { #[serde(tag = "event")] enum TransactionEventNonBlockIR { Validated, - Broadcasted(TransactionBroadcasted), Error(TransactionError), Invalid(TransactionError), Dropped(TransactionDropped), @@ -186,8 +160,6 @@ impl From> for TransactionEventIR { match value { TransactionEvent::Validated => TransactionEventIR::NonBlock(TransactionEventNonBlockIR::Validated), - TransactionEvent::Broadcasted(event) => - TransactionEventIR::NonBlock(TransactionEventNonBlockIR::Broadcasted(event)), TransactionEvent::BestChainBlockIncluded(event) => TransactionEventIR::Block(TransactionEventBlockIR::BestChainBlockIncluded(event)), TransactionEvent::Finalized(event) => @@ -207,8 +179,6 @@ impl From> for TransactionEvent { match value { TransactionEventIR::NonBlock(status) => match status { TransactionEventNonBlockIR::Validated => TransactionEvent::Validated, - TransactionEventNonBlockIR::Broadcasted(event) => - TransactionEvent::Broadcasted(event), TransactionEventNonBlockIR::Error(event) => TransactionEvent::Error(event), TransactionEventNonBlockIR::Invalid(event) => TransactionEvent::Invalid(event), TransactionEventNonBlockIR::Dropped(event) => TransactionEvent::Dropped(event), @@ -239,19 +209,6 @@ mod tests { assert_eq!(event_dec, event); } - #[test] - fn broadcasted_event() { - let event: TransactionEvent<()> = - TransactionEvent::Broadcasted(TransactionBroadcasted { num_peers: 2 }); - let ser = serde_json::to_string(&event).unwrap(); - - let exp = r#"{"event":"broadcasted","numPeers":2}"#; - assert_eq!(ser, exp); - - let event_dec: TransactionEvent<()> = serde_json::from_str(exp).unwrap(); - assert_eq!(event_dec, event); - } - #[test] fn best_chain_event() { let event: TransactionEvent<()> = TransactionEvent::BestChainBlockIncluded(None); @@ -320,13 +277,11 @@ mod tests { #[test] fn dropped_event() { - let event: TransactionEvent<()> = TransactionEvent::Dropped(TransactionDropped { - broadcasted: true, - error: "abc".to_string(), - }); + let event: TransactionEvent<()> = + TransactionEvent::Dropped(TransactionDropped { error: "abc".to_string() }); let ser = serde_json::to_string(&event).unwrap(); - let exp = r#"{"event":"dropped","broadcasted":true,"error":"abc"}"#; + let exp = r#"{"event":"dropped","error":"abc"}"#; assert_eq!(ser, exp); let event_dec: TransactionEvent<()> = serde_json::from_str(exp).unwrap(); diff --git a/substrate/client/rpc-spec-v2/src/transaction/mod.rs b/substrate/client/rpc-spec-v2/src/transaction/mod.rs index 74268a5372a3..514ccf047dc2 100644 --- a/substrate/client/rpc-spec-v2/src/transaction/mod.rs +++ b/substrate/client/rpc-spec-v2/src/transaction/mod.rs @@ -35,9 +35,6 @@ pub mod transaction; pub mod transaction_broadcast; pub use api::{TransactionApiServer, TransactionBroadcastApiServer}; -pub use event::{ - TransactionBlock, TransactionBroadcasted, TransactionDropped, TransactionError, - TransactionEvent, -}; +pub use event::{TransactionBlock, TransactionDropped, TransactionError, TransactionEvent}; pub use transaction::Transaction; pub use transaction_broadcast::TransactionBroadcast; diff --git a/substrate/client/rpc-spec-v2/src/transaction/transaction.rs b/substrate/client/rpc-spec-v2/src/transaction/transaction.rs index 17889b3bad2a..d44006392dca 100644 --- a/substrate/client/rpc-spec-v2/src/transaction/transaction.rs +++ b/substrate/client/rpc-spec-v2/src/transaction/transaction.rs @@ -22,10 +22,7 @@ use crate::{ transaction::{ api::TransactionApiServer, error::Error, - event::{ - TransactionBlock, TransactionBroadcasted, TransactionDropped, TransactionError, - TransactionEvent, - }, + event::{TransactionBlock, TransactionDropped, TransactionError, TransactionEvent}, }, SubscriptionTaskExecutor, }; @@ -113,9 +110,7 @@ where match submit.await { Ok(stream) => { - let mut state = TransactionState::new(); - let stream = - stream.filter_map(move |event| async move { state.handle_event(event) }); + let stream = stream.filter_map(move |event| async move { handle_event(event) }); pipe_from_stream(pending, stream.boxed()).await; }, Err(err) => { @@ -131,66 +126,34 @@ where } } -/// The transaction's state that needs to be preserved between -/// multiple events generated by the transaction-pool. -/// -/// # Note -/// -/// In the future, the RPC server can submit only the last event when multiple -/// identical events happen in a row. -#[derive(Clone, Copy)] -struct TransactionState { - /// True if the transaction was previously broadcasted. - broadcasted: bool, -} - -impl TransactionState { - /// Construct a new [`TransactionState`]. - pub fn new() -> Self { - TransactionState { broadcasted: false } - } - - /// Handle events generated by the transaction-pool and convert them - /// to the new API expected state. - #[inline] - pub fn handle_event( - &mut self, - event: TransactionStatus, - ) -> Option> { - match event { - TransactionStatus::Ready | TransactionStatus::Future => - Some(TransactionEvent::::Validated), - TransactionStatus::Broadcast(peers) => { - // Set the broadcasted flag once if we submitted the transaction to - // at least one peer. - self.broadcasted = self.broadcasted || !peers.is_empty(); - - Some(TransactionEvent::Broadcasted(TransactionBroadcasted { - num_peers: peers.len(), - })) - }, - TransactionStatus::InBlock((hash, index)) => - Some(TransactionEvent::BestChainBlockIncluded(Some(TransactionBlock { - hash, - index, - }))), - TransactionStatus::Retracted(_) => Some(TransactionEvent::BestChainBlockIncluded(None)), - TransactionStatus::FinalityTimeout(_) => - Some(TransactionEvent::Dropped(TransactionDropped { - broadcasted: self.broadcasted, - error: "Maximum number of finality watchers has been reached".into(), - })), - TransactionStatus::Finalized((hash, index)) => - Some(TransactionEvent::Finalized(TransactionBlock { hash, index })), - TransactionStatus::Usurped(_) => Some(TransactionEvent::Invalid(TransactionError { - error: "Extrinsic was rendered invalid by another extrinsic".into(), - })), - TransactionStatus::Dropped => Some(TransactionEvent::Invalid(TransactionError { - error: "Extrinsic dropped from the pool due to exceeding limits".into(), - })), - TransactionStatus::Invalid => Some(TransactionEvent::Invalid(TransactionError { - error: "Extrinsic marked as invalid".into(), +/// Handle events generated by the transaction-pool and convert them +/// to the new API expected state. +#[inline] +pub fn handle_event( + event: TransactionStatus, +) -> Option> { + match event { + TransactionStatus::Ready | TransactionStatus::Future => + Some(TransactionEvent::::Validated), + TransactionStatus::InBlock((hash, index)) => + Some(TransactionEvent::BestChainBlockIncluded(Some(TransactionBlock { hash, index }))), + TransactionStatus::Retracted(_) => Some(TransactionEvent::BestChainBlockIncluded(None)), + TransactionStatus::FinalityTimeout(_) => + Some(TransactionEvent::Dropped(TransactionDropped { + error: "Maximum number of finality watchers has been reached".into(), })), - } + TransactionStatus::Finalized((hash, index)) => + Some(TransactionEvent::Finalized(TransactionBlock { hash, index })), + TransactionStatus::Usurped(_) => Some(TransactionEvent::Invalid(TransactionError { + error: "Extrinsic was rendered invalid by another extrinsic".into(), + })), + TransactionStatus::Dropped => Some(TransactionEvent::Invalid(TransactionError { + error: "Extrinsic dropped from the pool due to exceeding limits".into(), + })), + TransactionStatus::Invalid => Some(TransactionEvent::Invalid(TransactionError { + error: "Extrinsic marked as invalid".into(), + })), + // These are the events that are not supported by the new API. + TransactionStatus::Broadcast(_) => None, } } diff --git a/substrate/client/service/src/client/client.rs b/substrate/client/service/src/client/client.rs index aa9c1b80a29a..108c258595a7 100644 --- a/substrate/client/service/src/client/client.rs +++ b/substrate/client/service/src/client/client.rs @@ -675,8 +675,10 @@ where // This is use by fast sync for runtime version to be resolvable from // changes. - let state_version = - resolve_state_version_from_wasm(&storage, &self.executor)?; + let state_version = resolve_state_version_from_wasm::<_, HashingFor>( + &storage, + &self.executor, + )?; let state_root = operation.op.reset_storage(storage, state_version)?; if state_root != *import_headers.post().state_root() { // State root mismatch when importing state. This should not happen in diff --git a/substrate/frame/babe/src/lib.rs b/substrate/frame/babe/src/lib.rs index a6e44390dbc5..5fb107dde3ba 100644 --- a/substrate/frame/babe/src/lib.rs +++ b/substrate/frame/babe/src/lib.rs @@ -323,7 +323,7 @@ pub mod pallet { #[pallet::genesis_config] pub struct GenesisConfig { pub authorities: Vec<(AuthorityId, BabeAuthorityWeight)>, - pub epoch_config: Option, + pub epoch_config: BabeEpochConfiguration, #[serde(skip)] pub _config: sp_std::marker::PhantomData, } @@ -333,9 +333,7 @@ pub mod pallet { fn build(&self) { SegmentIndex::::put(0); Pallet::::initialize_genesis_authorities(&self.authorities); - EpochConfig::::put( - self.epoch_config.clone().expect("epoch_config must not be None"), - ); + EpochConfig::::put(&self.epoch_config); } } diff --git a/substrate/frame/contracts/mock-network/src/parachain/contracts_config.rs b/substrate/frame/contracts/mock-network/src/parachain/contracts_config.rs index dadba394e264..3f26c6f372ef 100644 --- a/substrate/frame/contracts/mock-network/src/parachain/contracts_config.rs +++ b/substrate/frame/contracts/mock-network/src/parachain/contracts_config.rs @@ -94,5 +94,6 @@ impl pallet_contracts::Config for Runtime { type WeightPrice = Self; type Debug = (); type Environment = (); + type ApiVersion = (); type Xcm = pallet_xcm::Pallet; } diff --git a/substrate/frame/contracts/src/lib.rs b/substrate/frame/contracts/src/lib.rs index 943bf5f85d4e..d8939ee88baf 100644 --- a/substrate/frame/contracts/src/lib.rs +++ b/substrate/frame/contracts/src/lib.rs @@ -214,6 +214,18 @@ pub struct Environment { block_number: EnvironmentType>, } +/// Defines the current version of the HostFn APIs. +/// This is used to communicate the available APIs in pallet-contracts. +/// +/// The version is bumped any time a new HostFn is added or stabilized. +#[derive(Encode, Decode, TypeInfo)] +pub struct ApiVersion(u16); +impl Default for ApiVersion { + fn default() -> Self { + Self(1) + } +} + #[frame_support::pallet] pub mod pallet { use super::*; @@ -402,6 +414,12 @@ pub mod pallet { #[pallet::constant] type Environment: Get>; + /// The version of the HostFn APIs that are available in the runtime. + /// + /// Only valid value is `()`. + #[pallet::constant] + type ApiVersion: Get; + /// A type that exposes XCM APIs, allowing contracts to interact with other parachains, and /// execute XCM programs. type Xcm: xcm_builder::Controller< diff --git a/substrate/frame/contracts/src/tests.rs b/substrate/frame/contracts/src/tests.rs index b39fc79b62a0..2d1c5b315a34 100644 --- a/substrate/frame/contracts/src/tests.rs +++ b/substrate/frame/contracts/src/tests.rs @@ -465,6 +465,7 @@ impl Config for Test { type MaxDelegateDependencies = MaxDelegateDependencies; type Debug = TestDebug; type Environment = (); + type ApiVersion = (); type Xcm = (); } diff --git a/substrate/frame/examples/basic/src/benchmarking.rs b/substrate/frame/examples/basic/src/benchmarking.rs index 4b2ebb41fbda..65ca3089aba4 100644 --- a/substrate/frame/examples/basic/src/benchmarking.rs +++ b/substrate/frame/examples/basic/src/benchmarking.rs @@ -48,7 +48,7 @@ mod benchmarks { set_dummy(RawOrigin::Root, value); // The execution phase is just running `set_dummy` extrinsic call // This is the optional benchmark verification phase, asserting certain states. - assert_eq!(Pallet::::dummy(), Some(value)) + assert_eq!(Dummy::::get(), Some(value)) } // An example method that returns a Result that can be called within a benchmark diff --git a/substrate/frame/examples/basic/src/lib.rs b/substrate/frame/examples/basic/src/lib.rs index dad4d01978f9..12cadc969fd7 100644 --- a/substrate/frame/examples/basic/src/lib.rs +++ b/substrate/frame/examples/basic/src/lib.rs @@ -286,9 +286,7 @@ pub mod pallet { let _sender = ensure_signed(origin)?; // Read the value of dummy from storage. - // let dummy = Self::dummy(); - // Will also work using the `::get` on the storage item type itself: - // let dummy = >::get(); + // let dummy = Dummy::::get(); // Calculate the new value. // let new_dummy = dummy.map_or(increase_by, |dummy| dummy + increase_by); @@ -381,20 +379,14 @@ pub mod pallet { // - `Foo::put(1); Foo::get()` returns `1`; // - `Foo::kill(); Foo::get()` returns `0` (u32::default()). #[pallet::storage] - // The getter attribute generate a function on `Pallet` placeholder: - // `fn getter_name() -> Type` for basic value items or - // `fn getter_name(key: KeyType) -> ValueType` for map items. - #[pallet::getter(fn dummy)] pub(super) type Dummy = StorageValue<_, T::Balance>; // A map that has enumerable entries. #[pallet::storage] - #[pallet::getter(fn bar)] pub(super) type Bar = StorageMap<_, Blake2_128Concat, T::AccountId, T::Balance>; // this one uses the query kind: `ValueQuery`, we'll demonstrate the usage of 'mutate' API. #[pallet::storage] - #[pallet::getter(fn foo)] pub(super) type Foo = StorageValue<_, T::Balance, ValueQuery>; #[pallet::storage] @@ -433,10 +425,10 @@ impl Pallet { fn accumulate_foo(origin: T::RuntimeOrigin, increase_by: T::Balance) -> DispatchResult { let _sender = ensure_signed(origin)?; - let prev = >::get(); + let prev = Foo::::get(); // Because Foo has 'default', the type of 'foo' in closure is the raw type instead of an // Option<> type. - let result = >::mutate(|foo| { + let result = Foo::::mutate(|foo| { *foo = foo.saturating_add(increase_by); *foo }); diff --git a/substrate/frame/examples/basic/src/tests.rs b/substrate/frame/examples/basic/src/tests.rs index 9434ace35ffe..207e46e428dd 100644 --- a/substrate/frame/examples/basic/src/tests.rs +++ b/substrate/frame/examples/basic/src/tests.rs @@ -119,25 +119,25 @@ fn it_works_for_optional_value() { // Check that GenesisBuilder works properly. let val1 = 42; let val2 = 27; - assert_eq!(Example::dummy(), Some(val1)); + assert_eq!(Dummy::::get(), Some(val1)); // Check that accumulate works when we have Some value in Dummy already. assert_ok!(Example::accumulate_dummy(RuntimeOrigin::signed(1), val2)); - assert_eq!(Example::dummy(), Some(val1 + val2)); + assert_eq!(Dummy::::get(), Some(val1 + val2)); // Check that accumulate works when we Dummy has None in it. >::on_initialize(2); assert_ok!(Example::accumulate_dummy(RuntimeOrigin::signed(1), val1)); - assert_eq!(Example::dummy(), Some(val1 + val2 + val1)); + assert_eq!(Dummy::::get(), Some(val1 + val2 + val1)); }); } #[test] fn it_works_for_default_value() { new_test_ext().execute_with(|| { - assert_eq!(Example::foo(), 24); + assert_eq!(Foo::::get(), 24); assert_ok!(Example::accumulate_foo(RuntimeOrigin::signed(1), 1)); - assert_eq!(Example::foo(), 25); + assert_eq!(Foo::::get(), 25); }); } @@ -146,7 +146,7 @@ fn set_dummy_works() { new_test_ext().execute_with(|| { let test_val = 133; assert_ok!(Example::set_dummy(RuntimeOrigin::root(), test_val.into())); - assert_eq!(Example::dummy(), Some(test_val)); + assert_eq!(Dummy::::get(), Some(test_val)); }); } diff --git a/substrate/frame/examples/kitchensink/src/benchmarking.rs b/substrate/frame/examples/kitchensink/src/benchmarking.rs index 24da581fc967..5f1d378e06fe 100644 --- a/substrate/frame/examples/kitchensink/src/benchmarking.rs +++ b/substrate/frame/examples/kitchensink/src/benchmarking.rs @@ -51,7 +51,7 @@ mod benchmarks { set_foo(RawOrigin::Root, value, 10u128); // The execution phase is just running `set_foo` extrinsic call // This is the optional benchmark verification phase, asserting certain states. - assert_eq!(Pallet::::foo(), Some(value)) + assert_eq!(Foo::::get(), Some(value)) } // This line generates test cases for benchmarking, and could be run by: diff --git a/substrate/frame/examples/kitchensink/src/lib.rs b/substrate/frame/examples/kitchensink/src/lib.rs index 18429bc967d7..b7425b0c0846 100644 --- a/substrate/frame/examples/kitchensink/src/lib.rs +++ b/substrate/frame/examples/kitchensink/src/lib.rs @@ -125,7 +125,6 @@ pub mod pallet { #[pallet::storage] #[pallet::unbounded] // optional #[pallet::storage_prefix = "OtherFoo"] // optional - #[pallet::getter(fn foo)] // optional pub type Foo = StorageValue; #[pallet::type_value] diff --git a/substrate/frame/examples/offchain-worker/src/lib.rs b/substrate/frame/examples/offchain-worker/src/lib.rs index 6c1fa6ea8ec4..d21c8b4cfd24 100644 --- a/substrate/frame/examples/offchain-worker/src/lib.rs +++ b/substrate/frame/examples/offchain-worker/src/lib.rs @@ -332,7 +332,6 @@ pub mod pallet { /// /// This is used to calculate average price, should have bounded size. #[pallet::storage] - #[pallet::getter(fn prices)] pub(super) type Prices = StorageValue<_, BoundedVec, ValueQuery>; /// Defines the block when next unsigned transaction will be accepted. @@ -341,7 +340,6 @@ pub mod pallet { /// we only allow one transaction every `T::UnsignedInterval` blocks. /// This storage entry defines when new transaction is going to be accepted. #[pallet::storage] - #[pallet::getter(fn next_unsigned_at)] pub(super) type NextUnsignedAt = StorageValue<_, BlockNumberFor, ValueQuery>; } @@ -479,7 +477,7 @@ impl Pallet { ) -> Result<(), &'static str> { // Make sure we don't fetch the price if unsigned transaction is going to be rejected // anyway. - let next_unsigned_at = >::get(); + let next_unsigned_at = NextUnsignedAt::::get(); if next_unsigned_at > block_number { return Err("Too early to send unsigned transaction") } @@ -513,7 +511,7 @@ impl Pallet { ) -> Result<(), &'static str> { // Make sure we don't fetch the price if unsigned transaction is going to be rejected // anyway. - let next_unsigned_at = >::get(); + let next_unsigned_at = NextUnsignedAt::::get(); if next_unsigned_at > block_number { return Err("Too early to send unsigned transaction") } @@ -543,7 +541,7 @@ impl Pallet { ) -> Result<(), &'static str> { // Make sure we don't fetch the price if unsigned transaction is going to be rejected // anyway. - let next_unsigned_at = >::get(); + let next_unsigned_at = NextUnsignedAt::::get(); if next_unsigned_at > block_number { return Err("Too early to send unsigned transaction") } @@ -664,7 +662,7 @@ impl Pallet { /// Calculate current average price. fn average_price() -> Option { - let prices = >::get(); + let prices = Prices::::get(); if prices.is_empty() { None } else { @@ -677,7 +675,7 @@ impl Pallet { new_price: &u32, ) -> TransactionValidity { // Now let's check if the transaction has any chance to succeed. - let next_unsigned_at = >::get(); + let next_unsigned_at = NextUnsignedAt::::get(); if &next_unsigned_at > block_number { return InvalidTransaction::Stale.into() } diff --git a/substrate/frame/examples/split/src/lib.rs b/substrate/frame/examples/split/src/lib.rs index 74d2e0cc24b7..5245d90e390c 100644 --- a/substrate/frame/examples/split/src/lib.rs +++ b/substrate/frame/examples/split/src/lib.rs @@ -107,7 +107,7 @@ pub mod pallet { let _who = ensure_signed(origin)?; // Read a value from storage. - match >::get() { + match Something::::get() { // Return an error if the value has not been set. None => return Err(Error::::NoneValue.into()), Some(old) => { diff --git a/substrate/frame/membership/src/lib.rs b/substrate/frame/membership/src/lib.rs index 19b5e54d308c..f32263970038 100644 --- a/substrate/frame/membership/src/lib.rs +++ b/substrate/frame/membership/src/lib.rs @@ -27,7 +27,7 @@ use frame_support::{ traits::{ChangeMembers, Contains, Get, InitializeMembers, SortedMembers}, BoundedVec, }; -use sp_runtime::traits::StaticLookup; +use sp_runtime::traits::{StaticLookup, UniqueSaturatedInto}; use sp_std::prelude::*; pub mod migrations; @@ -163,12 +163,16 @@ pub mod pallet { /// /// May only be called from `T::AddOrigin`. #[pallet::call_index(0)] - #[pallet::weight({50_000_000})] - pub fn add_member(origin: OriginFor, who: AccountIdLookupOf) -> DispatchResult { + #[pallet::weight(T::WeightInfo::add_member(T::MaxMembers::get()))] + pub fn add_member( + origin: OriginFor, + who: AccountIdLookupOf, + ) -> DispatchResultWithPostInfo { T::AddOrigin::ensure_origin(origin)?; let who = T::Lookup::lookup(who)?; let mut members = >::get(); + let init_length = members.len(); let location = members.binary_search(&who).err().ok_or(Error::::AlreadyMember)?; members .try_insert(location, who.clone()) @@ -179,19 +183,24 @@ pub mod pallet { T::MembershipChanged::change_members_sorted(&[who], &[], &members[..]); Self::deposit_event(Event::MemberAdded); - Ok(()) + + Ok(Some(T::WeightInfo::add_member(init_length as u32)).into()) } /// Remove a member `who` from the set. /// /// May only be called from `T::RemoveOrigin`. #[pallet::call_index(1)] - #[pallet::weight({50_000_000})] - pub fn remove_member(origin: OriginFor, who: AccountIdLookupOf) -> DispatchResult { + #[pallet::weight(T::WeightInfo::remove_member(T::MaxMembers::get()))] + pub fn remove_member( + origin: OriginFor, + who: AccountIdLookupOf, + ) -> DispatchResultWithPostInfo { T::RemoveOrigin::ensure_origin(origin)?; let who = T::Lookup::lookup(who)?; let mut members = >::get(); + let init_length = members.len(); let location = members.binary_search(&who).ok().ok_or(Error::::NotMember)?; members.remove(location); @@ -201,7 +210,7 @@ pub mod pallet { Self::rejig_prime(&members); Self::deposit_event(Event::MemberRemoved); - Ok(()) + Ok(Some(T::WeightInfo::remove_member(init_length as u32)).into()) } /// Swap out one member `remove` for another `add`. @@ -210,18 +219,18 @@ pub mod pallet { /// /// Prime membership is *not* passed from `remove` to `add`, if extant. #[pallet::call_index(2)] - #[pallet::weight({50_000_000})] + #[pallet::weight(T::WeightInfo::swap_member(T::MaxMembers::get()))] pub fn swap_member( origin: OriginFor, remove: AccountIdLookupOf, add: AccountIdLookupOf, - ) -> DispatchResult { + ) -> DispatchResultWithPostInfo { T::SwapOrigin::ensure_origin(origin)?; let remove = T::Lookup::lookup(remove)?; let add = T::Lookup::lookup(add)?; if remove == add { - return Ok(()) + return Ok(().into()); } let mut members = >::get(); @@ -236,7 +245,7 @@ pub mod pallet { Self::rejig_prime(&members); Self::deposit_event(Event::MembersSwapped); - Ok(()) + Ok(Some(T::WeightInfo::swap_member(members.len() as u32)).into()) } /// Change the membership to a new set, disregarding the existing membership. Be nice and @@ -244,7 +253,7 @@ pub mod pallet { /// /// May only be called from `T::ResetOrigin`. #[pallet::call_index(3)] - #[pallet::weight({50_000_000})] + #[pallet::weight(T::WeightInfo::reset_members(members.len().unique_saturated_into()))] pub fn reset_members(origin: OriginFor, members: Vec) -> DispatchResult { T::ResetOrigin::ensure_origin(origin)?; @@ -267,56 +276,65 @@ pub mod pallet { /// /// Prime membership is passed from the origin account to `new`, if extant. #[pallet::call_index(4)] - #[pallet::weight({50_000_000})] - pub fn change_key(origin: OriginFor, new: AccountIdLookupOf) -> DispatchResult { + #[pallet::weight(T::WeightInfo::change_key(T::MaxMembers::get()))] + pub fn change_key( + origin: OriginFor, + new: AccountIdLookupOf, + ) -> DispatchResultWithPostInfo { let remove = ensure_signed(origin)?; let new = T::Lookup::lookup(new)?; - if remove != new { - let mut members = >::get(); - let location = - members.binary_search(&remove).ok().ok_or(Error::::NotMember)?; - let _ = members.binary_search(&new).err().ok_or(Error::::AlreadyMember)?; - members[location] = new.clone(); - members.sort(); - - >::put(&members); - - T::MembershipChanged::change_members_sorted( - &[new.clone()], - &[remove.clone()], - &members[..], - ); - - if Prime::::get() == Some(remove) { - Prime::::put(&new); - T::MembershipChanged::set_prime(Some(new)); - } + if remove == new { + return Ok(().into()); + } + + let mut members = >::get(); + let members_length = members.len() as u32; + let location = members.binary_search(&remove).ok().ok_or(Error::::NotMember)?; + let _ = members.binary_search(&new).err().ok_or(Error::::AlreadyMember)?; + members[location] = new.clone(); + members.sort(); + + >::put(&members); + + T::MembershipChanged::change_members_sorted( + &[new.clone()], + &[remove.clone()], + &members[..], + ); + + if Prime::::get() == Some(remove) { + Prime::::put(&new); + T::MembershipChanged::set_prime(Some(new)); } Self::deposit_event(Event::KeyChanged); - Ok(()) + Ok(Some(T::WeightInfo::change_key(members_length)).into()) } /// Set the prime member. Must be a current member. /// /// May only be called from `T::PrimeOrigin`. #[pallet::call_index(5)] - #[pallet::weight({50_000_000})] - pub fn set_prime(origin: OriginFor, who: AccountIdLookupOf) -> DispatchResult { + #[pallet::weight(T::WeightInfo::set_prime(T::MaxMembers::get()))] + pub fn set_prime( + origin: OriginFor, + who: AccountIdLookupOf, + ) -> DispatchResultWithPostInfo { T::PrimeOrigin::ensure_origin(origin)?; let who = T::Lookup::lookup(who)?; - Self::members().binary_search(&who).ok().ok_or(Error::::NotMember)?; + let members = Self::members(); + members.binary_search(&who).ok().ok_or(Error::::NotMember)?; Prime::::put(&who); T::MembershipChanged::set_prime(Some(who)); - Ok(()) + Ok(Some(T::WeightInfo::set_prime(members.len() as u32)).into()) } /// Remove the prime member if it exists. /// /// May only be called from `T::PrimeOrigin`. #[pallet::call_index(6)] - #[pallet::weight({50_000_000})] + #[pallet::weight(T::WeightInfo::clear_prime())] pub fn clear_prime(origin: OriginFor) -> DispatchResult { T::PrimeOrigin::ensure_origin(origin)?; Prime::::kill(); @@ -442,7 +460,7 @@ mod benchmark { } // er keep the prime common between incoming and outgoing to make sure it is rejigged. - reset_member { + reset_members { let m in 1 .. T::MaxMembers::get(); let members = (1..m+1).map(|i| account("member", i, SEED)).collect::>(); @@ -500,8 +518,7 @@ mod benchmark { } clear_prime { - let m in 1 .. T::MaxMembers::get(); - let members = (0..m).map(|i| account("member", i, SEED)).collect::>(); + let members = (0..T::MaxMembers::get()).map(|i| account("member", i, SEED)).collect::>(); let prime = members.last().cloned().unwrap(); set_members::(members, None); }: { @@ -526,7 +543,8 @@ mod tests { use sp_runtime::{bounded_vec, traits::BadOrigin, BuildStorage}; use frame_support::{ - assert_noop, assert_ok, derive_impl, ord_parameter_types, parameter_types, + assert_noop, assert_ok, assert_storage_noop, derive_impl, ord_parameter_types, + parameter_types, traits::{ConstU32, StorageVersion}, }; use frame_system::EnsureSignedBy; @@ -716,6 +734,17 @@ mod tests { }); } + #[test] + fn swap_member_with_identical_arguments_changes_nothing() { + new_test_ext().execute_with(|| { + assert_storage_noop!(assert_ok!(Membership::swap_member( + RuntimeOrigin::signed(3), + 10, + 10 + ))); + }); + } + #[test] fn change_key_works() { new_test_ext().execute_with(|| { @@ -745,6 +774,13 @@ mod tests { }); } + #[test] + fn change_key_with_same_caller_as_argument_changes_nothing() { + new_test_ext().execute_with(|| { + assert_storage_noop!(assert_ok!(Membership::change_key(RuntimeOrigin::signed(10), 10))); + }); + } + #[test] fn reset_members_works() { new_test_ext().execute_with(|| { diff --git a/substrate/frame/membership/src/weights.rs b/substrate/frame/membership/src/weights.rs index 18ea7fcb315a..2d18848b89ab 100644 --- a/substrate/frame/membership/src/weights.rs +++ b/substrate/frame/membership/src/weights.rs @@ -55,10 +55,10 @@ pub trait WeightInfo { fn add_member(m: u32, ) -> Weight; fn remove_member(m: u32, ) -> Weight; fn swap_member(m: u32, ) -> Weight; - fn reset_member(m: u32, ) -> Weight; + fn reset_members(m: u32, ) -> Weight; fn change_key(m: u32, ) -> Weight; fn set_prime(m: u32, ) -> Weight; - fn clear_prime(m: u32, ) -> Weight; + fn clear_prime() -> Weight; } /// Weights for pallet_membership using the Substrate node and recommended hardware. @@ -142,7 +142,7 @@ impl WeightInfo for SubstrateWeight { /// Storage: TechnicalCommittee Prime (r:0 w:1) /// Proof Skipped: TechnicalCommittee Prime (max_values: Some(1), max_size: None, mode: Measured) /// The range of component `m` is `[1, 100]`. - fn reset_member(m: u32, ) -> Weight { + fn reset_members(m: u32, ) -> Weight { // Proof Size summary in bytes: // Measured: `312 + m * (64 ±0)` // Estimated: `4687 + m * (64 ±0)` @@ -200,15 +200,12 @@ impl WeightInfo for SubstrateWeight { /// Proof: TechnicalMembership Prime (max_values: Some(1), max_size: Some(32), added: 527, mode: MaxEncodedLen) /// Storage: TechnicalCommittee Prime (r:0 w:1) /// Proof Skipped: TechnicalCommittee Prime (max_values: Some(1), max_size: None, mode: Measured) - /// The range of component `m` is `[1, 100]`. - fn clear_prime(m: u32, ) -> Weight { + fn clear_prime() -> Weight { // Proof Size summary in bytes: // Measured: `0` // Estimated: `0` // Minimum execution time: 3_373_000 picoseconds. Weight::from_parts(3_750_452, 0) - // Standard Error: 142 - .saturating_add(Weight::from_parts(505, 0).saturating_mul(m.into())) .saturating_add(T::DbWeight::get().writes(2_u64)) } } @@ -293,7 +290,7 @@ impl WeightInfo for () { /// Storage: TechnicalCommittee Prime (r:0 w:1) /// Proof Skipped: TechnicalCommittee Prime (max_values: Some(1), max_size: None, mode: Measured) /// The range of component `m` is `[1, 100]`. - fn reset_member(m: u32, ) -> Weight { + fn reset_members(m: u32, ) -> Weight { // Proof Size summary in bytes: // Measured: `312 + m * (64 ±0)` // Estimated: `4687 + m * (64 ±0)` @@ -351,15 +348,12 @@ impl WeightInfo for () { /// Proof: TechnicalMembership Prime (max_values: Some(1), max_size: Some(32), added: 527, mode: MaxEncodedLen) /// Storage: TechnicalCommittee Prime (r:0 w:1) /// Proof Skipped: TechnicalCommittee Prime (max_values: Some(1), max_size: None, mode: Measured) - /// The range of component `m` is `[1, 100]`. - fn clear_prime(m: u32, ) -> Weight { + fn clear_prime() -> Weight { // Proof Size summary in bytes: // Measured: `0` // Estimated: `0` // Minimum execution time: 3_373_000 picoseconds. Weight::from_parts(3_750_452, 0) - // Standard Error: 142 - .saturating_add(Weight::from_parts(505, 0).saturating_mul(m.into())) .saturating_add(RocksDbWeight::get().writes(2_u64)) } } diff --git a/substrate/frame/session/src/lib.rs b/substrate/frame/session/src/lib.rs index 178d43f596b2..fc35cd6ddd82 100644 --- a/substrate/frame/session/src/lib.rs +++ b/substrate/frame/session/src/lib.rs @@ -460,27 +460,13 @@ pub mod pallet { ); self.keys.iter().map(|x| x.1.clone()).collect() }); - assert!( - !initial_validators_0.is_empty(), - "Empty validator set for session 0 in genesis block!" - ); let initial_validators_1 = T::SessionManager::new_session_genesis(1) .unwrap_or_else(|| initial_validators_0.clone()); - assert!( - !initial_validators_1.is_empty(), - "Empty validator set for session 1 in genesis block!" - ); let queued_keys: Vec<_> = initial_validators_1 - .iter() - .cloned() - .map(|v| { - ( - v.clone(), - Pallet::::load_keys(&v).expect("Validator in session 1 missing keys!"), - ) - }) + .into_iter() + .filter_map(|v| Pallet::::load_keys(&v).map(|k| (v, k))) .collect(); // Tell everyone about the genesis session keys diff --git a/substrate/frame/support/procedural/src/construct_runtime/expand/config.rs b/substrate/frame/support/procedural/src/construct_runtime/expand/config.rs index ffe55bceb80e..5613047359a7 100644 --- a/substrate/frame/support/procedural/src/construct_runtime/expand/config.rs +++ b/substrate/frame/support/procedural/src/construct_runtime/expand/config.rs @@ -99,6 +99,17 @@ pub fn expand_outer_config( ::on_genesis(); } } + + /// Test the `Default` derive impl of the `RuntimeGenesisConfig`. + #[cfg(test)] + #[test] + fn test_genesis_config_builds() { + #scrate::__private::sp_io::TestExternalities::default().execute_with(|| { + ::build( + &RuntimeGenesisConfig::default() + ); + }); + } } } diff --git a/substrate/frame/support/src/lib.rs b/substrate/frame/support/src/lib.rs index 8935acf4e2bb..cd12da6de54e 100644 --- a/substrate/frame/support/src/lib.rs +++ b/substrate/frame/support/src/lib.rs @@ -780,7 +780,7 @@ macro_rules! assert_err_with_weight { $crate::assert_err!($call.map(|_| ()).map_err(|e| e.error), $err); assert_eq!(dispatch_err_with_post.post_info.actual_weight, $weight); } else { - panic!("expected Err(_), got Ok(_).") + ::core::panic!("expected Err(_), got Ok(_).") } }; } diff --git a/substrate/frame/system/src/lib.rs b/substrate/frame/system/src/lib.rs index 069217bcee46..1405c7303f87 100644 --- a/substrate/frame/system/src/lib.rs +++ b/substrate/frame/system/src/lib.rs @@ -148,6 +148,7 @@ use sp_io::TestExternalities; pub mod limits; #[cfg(test)] pub(crate) mod mock; + pub mod offchain; mod extensions; @@ -847,7 +848,7 @@ pub mod pallet { #[pallet::storage] #[pallet::whitelist_storage] #[pallet::getter(fn block_weight)] - pub(super) type BlockWeight = StorageValue<_, ConsumedWeight, ValueQuery>; + pub type BlockWeight = StorageValue<_, ConsumedWeight, ValueQuery>; /// Total length (in bytes) for all extrinsics put together, for the current block. #[pallet::storage] diff --git a/substrate/primitives/consensus/babe/src/lib.rs b/substrate/primitives/consensus/babe/src/lib.rs index d6b2cdd55e0d..ff0b4568226e 100644 --- a/substrate/primitives/consensus/babe/src/lib.rs +++ b/substrate/primitives/consensus/babe/src/lib.rs @@ -256,6 +256,12 @@ pub struct BabeEpochConfiguration { pub allowed_slots: AllowedSlots, } +impl Default for BabeEpochConfiguration { + fn default() -> Self { + Self { c: (1, 4), allowed_slots: AllowedSlots::PrimaryAndSecondaryVRFSlots } + } +} + /// Verifies the equivocation proof by making sure that: both headers have /// different hashes, are targetting the same slot, and have valid signatures by /// the same authority. diff --git a/substrate/primitives/consensus/beefy/src/commitment.rs b/substrate/primitives/consensus/beefy/src/commitment.rs index 1f0fb34ebf10..335c6b604f04 100644 --- a/substrate/primitives/consensus/beefy/src/commitment.rs +++ b/substrate/primitives/consensus/beefy/src/commitment.rs @@ -97,6 +97,19 @@ pub struct SignedCommitment { pub signatures: Vec>, } +impl sp_std::fmt::Display + for SignedCommitment +{ + fn fmt(&self, f: &mut sp_std::fmt::Formatter<'_>) -> sp_std::fmt::Result { + let signatures_count = self.signatures.iter().filter(|s| s.is_some()).count(); + write!( + f, + "SignedCommitment(commitment: {:?}, signatures_count: {})", + self.commitment, signatures_count + ) + } +} + impl SignedCommitment { /// Return the number of collected signatures. pub fn no_of_signatures(&self) -> usize { @@ -241,6 +254,14 @@ pub enum VersionedFinalityProof { V1(SignedCommitment), } +impl sp_std::fmt::Display for VersionedFinalityProof { + fn fmt(&self, f: &mut sp_std::fmt::Formatter<'_>) -> sp_std::fmt::Result { + match self { + VersionedFinalityProof::V1(sc) => write!(f, "VersionedFinalityProof::V1({})", sc), + } + } +} + impl From> for VersionedFinalityProof { fn from(commitment: SignedCommitment) -> Self { VersionedFinalityProof::V1(commitment) diff --git a/substrate/primitives/weights/src/weight_meter.rs b/substrate/primitives/weights/src/weight_meter.rs index 584d22304c3a..1738948e4c3c 100644 --- a/substrate/primitives/weights/src/weight_meter.rs +++ b/substrate/primitives/weights/src/weight_meter.rs @@ -149,6 +149,11 @@ impl WeightMeter { pub fn can_consume(&self, w: Weight) -> bool { self.consumed.checked_add(&w).map_or(false, |t| t.all_lte(self.limit)) } + + /// Reclaim the given weight. + pub fn reclaim_proof_size(&mut self, s: u64) { + self.consumed.saturating_reduce(Weight::from_parts(0, s)); + } } #[cfg(test)] @@ -277,6 +282,21 @@ mod tests { assert_eq!(meter.consumed(), Weight::from_parts(5, 10)); } + #[test] + #[cfg(debug_assertions)] + fn reclaim_works() { + let mut meter = WeightMeter::with_limit(Weight::from_parts(5, 10)); + + meter.consume(Weight::from_parts(5, 10)); + assert_eq!(meter.consumed(), Weight::from_parts(5, 10)); + + meter.reclaim_proof_size(3); + assert_eq!(meter.consumed(), Weight::from_parts(5, 7)); + + meter.reclaim_proof_size(10); + assert_eq!(meter.consumed(), Weight::from_parts(5, 0)); + } + #[test] #[cfg(debug_assertions)] #[should_panic(expected = "Weight counter overflow")] diff --git a/substrate/test-utils/client/src/lib.rs b/substrate/test-utils/client/src/lib.rs index e3f06e275635..d283b24f286a 100644 --- a/substrate/test-utils/client/src/lib.rs +++ b/substrate/test-utils/client/src/lib.rs @@ -72,6 +72,7 @@ pub struct TestClientBuilder, bad_blocks: BadBlocks, enable_offchain_indexing_api: bool, + enable_import_proof_recording: bool, no_genesis: bool, } @@ -120,6 +121,7 @@ impl bad_blocks: None, enable_offchain_indexing_api: false, no_genesis: false, + enable_import_proof_recording: false, } } @@ -165,6 +167,12 @@ impl self } + /// Enable proof recording on import. + pub fn enable_import_proof_recording(mut self) -> Self { + self.enable_import_proof_recording = true; + self + } + /// Disable writing genesis. pub fn set_no_genesis(mut self) -> Self { self.no_genesis = true; @@ -202,6 +210,7 @@ impl }; let client_config = ClientConfig { + enable_import_proof_recording: self.enable_import_proof_recording, offchain_indexing_api: self.enable_offchain_indexing_api, no_genesis: self.no_genesis, ..Default::default() diff --git a/substrate/test-utils/runtime/src/genesismap.rs b/substrate/test-utils/runtime/src/genesismap.rs index 5ed9c8a64588..9e972886b377 100644 --- a/substrate/test-utils/runtime/src/genesismap.rs +++ b/substrate/test-utils/runtime/src/genesismap.rs @@ -124,7 +124,6 @@ impl GenesisStorageBuilder { .into_iter() .map(|x| (x.into(), 1)) .collect(), - epoch_config: Some(crate::TEST_RUNTIME_BABE_EPOCH_CONFIGURATION), ..Default::default() }, substrate_test: substrate_test_pallet::GenesisConfig { diff --git a/substrate/test-utils/runtime/src/lib.rs b/substrate/test-utils/runtime/src/lib.rs index 8bc6f72a82ec..db9ff187b707 100644 --- a/substrate/test-utils/runtime/src/lib.rs +++ b/substrate/test-utils/runtime/src/lib.rs @@ -1291,7 +1291,7 @@ mod tests { let r = Vec::::decode(&mut &r[..]).unwrap(); let json = String::from_utf8(r.into()).expect("returned value is json. qed."); - let expected = r#"{"system":{},"babe":{"authorities":[],"epochConfig":null},"substrateTest":{"authorities":[]},"balances":{"balances":[]}}"#; + let expected = r#"{"system":{},"babe":{"authorities":[],"epochConfig":{"c":[1,4],"allowed_slots":"PrimaryAndSecondaryVRFSlots"}},"substrateTest":{"authorities":[]},"balances":{"balances":[]}}"#; assert_eq!(expected.to_string(), json); } diff --git a/substrate/utils/wasm-builder/src/wasm_project.rs b/substrate/utils/wasm-builder/src/wasm_project.rs index bc8c9b3b4b39..9ffb5c72fd95 100644 --- a/substrate/utils/wasm-builder/src/wasm_project.rs +++ b/substrate/utils/wasm-builder/src/wasm_project.rs @@ -855,7 +855,7 @@ fn build_bloaty_blob( println!("{} {}", colorize_info_message("Using rustc version:"), cargo_cmd.rustc_version()); // Use `process::exit(1)` to have a clean error output. - if build_cmd.status().map(|s| s.success()).is_err() { + if !matches!(build_cmd.status().map(|s| s.success()), Ok(true)) { process::exit(1); } @@ -877,7 +877,7 @@ fn build_bloaty_blob( if polkavm_path .metadata() .map(|polkavm_metadata| { - polkavm_metadata.modified().unwrap() >= elf_metadata.modified().unwrap() + polkavm_metadata.modified().unwrap() < elf_metadata.modified().unwrap() }) .unwrap_or(true) {