diff --git a/.gitlab/pipeline/zombienet.yml b/.gitlab/pipeline/zombienet.yml index 52948e1eb719..7be4ba1663e6 100644 --- a/.gitlab/pipeline/zombienet.yml +++ b/.gitlab/pipeline/zombienet.yml @@ -1,8 +1,9 @@ .zombienet-refs: extends: .build-refs variables: - ZOMBIENET_IMAGE: "docker.io/paritytech/zombienet:v1.3.99" + ZOMBIENET_IMAGE: "docker.io/paritytech/zombienet:v1.3.103" PUSHGATEWAY_URL: "http://zombienet-prometheus-pushgateway.managed-monitoring:9091/metrics/job/zombie-metrics" + DEBUG: "zombie,zombie::network-node,zombie::kube::client::logs" include: # substrate tests diff --git a/.gitlab/pipeline/zombienet/cumulus.yml b/.gitlab/pipeline/zombienet/cumulus.yml index c473f5c5fed7..a7f321505bac 100644 --- a/.gitlab/pipeline/zombienet/cumulus.yml +++ b/.gitlab/pipeline/zombienet/cumulus.yml @@ -15,7 +15,6 @@ - echo "${COL_IMAGE}" - echo "${GH_DIR}" - echo "${LOCAL_DIR}" - - export DEBUG=zombie - export RELAY_IMAGE=${POLKADOT_IMAGE} - export COL_IMAGE=${COL_IMAGE} diff --git a/.gitlab/pipeline/zombienet/polkadot.yml b/.gitlab/pipeline/zombienet/polkadot.yml index 38c5332f3097..a9f0eb930337 100644 --- a/.gitlab/pipeline/zombienet/polkadot.yml +++ b/.gitlab/pipeline/zombienet/polkadot.yml @@ -10,7 +10,6 @@ - if [[ $CI_COMMIT_REF_NAME == *"gh-readonly-queue"* ]]; then export DOCKER_IMAGES_VERSION="${CI_COMMIT_SHORT_SHA}"; fi - export PIPELINE_IMAGE_TAG=${DOCKER_IMAGES_VERSION} - export BUILD_RELEASE_VERSION="$(cat ./artifacts/BUILD_RELEASE_VERSION)" # from build-linux-stable job - - export DEBUG=zombie,zombie::network-node - export ZOMBIENET_INTEGRATION_TEST_IMAGE="${POLKADOT_IMAGE}":${PIPELINE_IMAGE_TAG} - export COL_IMAGE="${COLANDER_IMAGE}":${PIPELINE_IMAGE_TAG} - export CUMULUS_IMAGE="docker.io/paritypr/polkadot-parachain-debug:${DOCKER_IMAGES_VERSION}" diff --git a/.gitlab/pipeline/zombienet/substrate.yml b/.gitlab/pipeline/zombienet/substrate.yml index 8a627c454f9f..2013ffd571cf 100644 --- a/.gitlab/pipeline/zombienet/substrate.yml +++ b/.gitlab/pipeline/zombienet/substrate.yml @@ -13,7 +13,6 @@ - echo "${ZOMBIENET_IMAGE}" - echo "${GH_DIR}" - echo "${LOCAL_DIR}" - - export DEBUG=zombie,zombie::network-node - export ZOMBIENET_INTEGRATION_TEST_IMAGE="${SUBSTRATE_IMAGE}":${SUBSTRATE_IMAGE_TAG} - echo "${ZOMBIENET_INTEGRATION_TEST_IMAGE}" stage: zombienet diff --git a/Cargo.lock b/Cargo.lock index 7711d51d0df6..7ff2da4f5472 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5362,20 +5362,19 @@ dependencies = [ [[package]] name = "fatality" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61ecdc33d04db74fc23db9f54f6f314c61d29f810d58ba423d0c204888365458" +checksum = "ec6f82451ff7f0568c6181287189126d492b5654e30a788add08027b6363d019" dependencies = [ "fatality-proc-macro", - "syn 2.0.61", "thiserror", ] [[package]] name = "fatality-proc-macro" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac67b350787fd4a934752e30ddb45569da000e14bf3e499224302778b7a918ab" +checksum = "eb42427514b063d97ce21d5199f36c0c307d981434a6be32582bc79fe5bd2303" dependencies = [ "expander", "indexmap 2.2.3", @@ -5383,7 +5382,6 @@ dependencies = [ "proc-macro2 1.0.82", "quote 1.0.35", "syn 2.0.61", - "thiserror", ] [[package]] @@ -8876,12 +8874,13 @@ dependencies = [ [[package]] name = "nix" -version = "0.27.1" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053" +checksum = "ab2156c4fce2f8df6c499cc1c763e4394b7482525bf2a9701c9d79d215f519e4" dependencies = [ "bitflags 2.4.0", "cfg-if", + "cfg_aliases", "libc", ] @@ -12535,7 +12534,7 @@ version = "6.0.0" dependencies = [ "assert_cmd", "color-eyre", - "nix 0.27.1", + "nix 0.28.0", "polkadot-cli", "polkadot-core-primitives", "polkadot-node-core-pvf", @@ -13219,7 +13218,7 @@ dependencies = [ "futures", "landlock", "libc", - "nix 0.27.1", + "nix 0.28.0", "parity-scale-codec", "polkadot-parachain-primitives", "polkadot-primitives", @@ -13244,7 +13243,7 @@ dependencies = [ "cfg-if", "cpu-time", "libc", - "nix 0.27.1", + "nix 0.28.0", "parity-scale-codec", "polkadot-node-core-pvf-common", "polkadot-parachain-primitives", @@ -13260,7 +13259,7 @@ dependencies = [ "cfg-if", "criterion", "libc", - "nix 0.27.1", + "nix 0.28.0", "parity-scale-codec", "polkadot-node-core-pvf-common", "polkadot-primitives", @@ -13553,7 +13552,7 @@ dependencies = [ "hex-literal", "jsonrpsee", "log", - "nix 0.27.1", + "nix 0.28.0", "pallet-transaction-payment", "pallet-transaction-payment-rpc", "pallet-transaction-payment-rpc-runtime-api", @@ -20120,7 +20119,7 @@ dependencies = [ "kitchensink-runtime", "log", "mmr-gadget", - "nix 0.27.1", + "nix 0.28.0", "node-primitives", "node-rpc", "node-testing", @@ -20520,7 +20519,7 @@ version = "0.1.0" dependencies = [ "assert_cmd", "futures", - "nix 0.27.1", + "nix 0.28.0", "node-primitives", "regex", "sc-cli", diff --git a/cumulus/parachains/chain-specs/coretime-westend.json b/cumulus/parachains/chain-specs/coretime-westend.json index ab2e97fdbf41..74edd5b2cd96 100644 --- a/cumulus/parachains/chain-specs/coretime-westend.json +++ b/cumulus/parachains/chain-specs/coretime-westend.json @@ -5,6 +5,8 @@ "bootNodes": [ "/dns/westend-coretime-collator-node-0.parity-testnet.parity.io/tcp/30333/p2p/12D3KooWP93Dzk8T7GWxyWw9jhLcz8Pksokk3R9vL2eEH337bNkT", "/dns/westend-coretime-collator-node-1.parity-testnet.parity.io/tcp/30333/p2p/12D3KooWMh2imeAzsZKGQgm2cv6Uoep3GBYtwGfujt1bs5YfVzkH", + "/dns/coretime-westend-bootnode.radiumblock.com/tcp/30333/p2p/12D3KooWK7Zj1mCPg6h3eMp7v6akJ1o6AocRr59NLusDwBXQgrhw", + "/dns/coretime-westend-bootnode.radiumblock.com/tcp/30336/wss/p2p/12D3KooWK7Zj1mCPg6h3eMp7v6akJ1o6AocRr59NLusDwBXQgrhw", "/dns/westend-coretime-collator-node-0.parity-testnet.parity.io/tcp/30335/ws/p2p/12D3KooWP93Dzk8T7GWxyWw9jhLcz8Pksokk3R9vL2eEH337bNkT", "/dns/westend-coretime-collator-node-1.parity-testnet.parity.io/tcp/30335/ws/p2p/12D3KooWMh2imeAzsZKGQgm2cv6Uoep3GBYtwGfujt1bs5YfVzkH", "/dns/westend-coretime-collator-node-0.parity-testnet.parity.io/tcp/443/wss/p2p/12D3KooWP93Dzk8T7GWxyWw9jhLcz8Pksokk3R9vL2eEH337bNkT", diff --git a/cumulus/polkadot-parachain/Cargo.toml b/cumulus/polkadot-parachain/Cargo.toml index f21a5baf973b..3d0aa94e8dea 100644 --- a/cumulus/polkadot-parachain/Cargo.toml +++ b/cumulus/polkadot-parachain/Cargo.toml @@ -118,7 +118,7 @@ substrate-build-script-utils = { path = "../../substrate/utils/build-script-util [dev-dependencies] assert_cmd = "2.0" -nix = { version = "0.27.1", features = ["signal"] } +nix = { version = "0.28.0", features = ["signal"] } tempfile = "3.8.0" tokio = { version = "1.32.0", features = ["macros", "parking_lot", "time"] } wait-timeout = "0.2" diff --git a/cumulus/polkadot-parachain/src/chain_spec/mod.rs b/cumulus/polkadot-parachain/src/chain_spec/mod.rs index bbda334e4c66..136a19e3166b 100644 --- a/cumulus/polkadot-parachain/src/chain_spec/mod.rs +++ b/cumulus/polkadot-parachain/src/chain_spec/mod.rs @@ -37,11 +37,12 @@ const SAFE_XCM_VERSION: u32 = xcm::prelude::XCM_VERSION; /// Generic extensions for Parachain ChainSpecs. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, ChainSpecGroup, ChainSpecExtension)] -#[serde(deny_unknown_fields)] pub struct Extensions { /// The relay chain of the Parachain. + #[serde(alias = "relayChain", alias = "RelayChain")] pub relay_chain: String, /// The id of the Parachain. + #[serde(alias = "paraId", alias = "ParaId")] pub para_id: u32, } @@ -78,3 +79,22 @@ where pub fn get_collator_keys_from_seed(seed: &str) -> ::Public { get_from_seed::(seed) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn can_decode_extension_camel_and_snake_case() { + let camel_case = r#"{"relayChain":"relay","paraId":1}"#; + let snake_case = r#"{"relay_chain":"relay","para_id":1}"#; + let pascal_case = r#"{"RelayChain":"relay","ParaId":1}"#; + + let camel_case_extension: Extensions = serde_json::from_str(camel_case).unwrap(); + let snake_case_extension: Extensions = serde_json::from_str(snake_case).unwrap(); + let pascal_case_extension: Extensions = serde_json::from_str(pascal_case).unwrap(); + + assert_eq!(camel_case_extension, snake_case_extension); + assert_eq!(snake_case_extension, pascal_case_extension); + } +} diff --git a/polkadot/Cargo.toml b/polkadot/Cargo.toml index 7b5679e1084e..3aeec8d5961e 100644 --- a/polkadot/Cargo.toml +++ b/polkadot/Cargo.toml @@ -43,7 +43,7 @@ tikv-jemallocator = { version = "0.5.0", features = ["unprefixed_malloc_on_suppo [dev-dependencies] assert_cmd = "2.0.4" -nix = { version = "0.27.1", features = ["signal"] } +nix = { version = "0.28.0", features = ["signal"] } tempfile = "3.2.0" tokio = "1.37" substrate-rpc-client = { path = "../substrate/utils/frame/rpc/client" } diff --git a/polkadot/node/core/backing/Cargo.toml b/polkadot/node/core/backing/Cargo.toml index 9829f1b37cfd..f426f73284e8 100644 --- a/polkadot/node/core/backing/Cargo.toml +++ b/polkadot/node/core/backing/Cargo.toml @@ -21,7 +21,7 @@ statement-table = { package = "polkadot-statement-table", path = "../../../state bitvec = { version = "1.0.0", default-features = false, features = ["alloc"] } gum = { package = "tracing-gum", path = "../../gum" } thiserror = { workspace = true } -fatality = "0.1.0" +fatality = "0.1.1" schnellru = "0.2.1" [dev-dependencies] diff --git a/polkadot/node/core/dispute-coordinator/Cargo.toml b/polkadot/node/core/dispute-coordinator/Cargo.toml index 938fdce1cb82..dba3bcdd6430 100644 --- a/polkadot/node/core/dispute-coordinator/Cargo.toml +++ b/polkadot/node/core/dispute-coordinator/Cargo.toml @@ -16,7 +16,7 @@ parity-scale-codec = "3.6.1" kvdb = "0.13.0" thiserror = { workspace = true } schnellru = "0.2.1" -fatality = "0.1.0" +fatality = "0.1.1" polkadot-primitives = { path = "../../../primitives" } polkadot-node-primitives = { path = "../../primitives" } diff --git a/polkadot/node/core/prospective-parachains/Cargo.toml b/polkadot/node/core/prospective-parachains/Cargo.toml index 8d0aec96f010..d38a23c3fdad 100644 --- a/polkadot/node/core/prospective-parachains/Cargo.toml +++ b/polkadot/node/core/prospective-parachains/Cargo.toml @@ -14,7 +14,7 @@ futures = "0.3.30" gum = { package = "tracing-gum", path = "../../gum" } parity-scale-codec = "3.6.4" thiserror = { workspace = true } -fatality = "0.1.0" +fatality = "0.1.1" bitvec = "1" polkadot-primitives = { path = "../../../primitives" } diff --git a/polkadot/node/core/provisioner/Cargo.toml b/polkadot/node/core/provisioner/Cargo.toml index 1cd16e6599a7..d19783212644 100644 --- a/polkadot/node/core/provisioner/Cargo.toml +++ b/polkadot/node/core/provisioner/Cargo.toml @@ -19,7 +19,7 @@ polkadot-node-primitives = { path = "../../primitives" } polkadot-node-subsystem = { path = "../../subsystem" } polkadot-node-subsystem-util = { path = "../../subsystem-util" } futures-timer = "3.0.2" -fatality = "0.1.0" +fatality = "0.1.1" schnellru = "0.2.1" [dev-dependencies] diff --git a/polkadot/node/core/pvf/common/Cargo.toml b/polkadot/node/core/pvf/common/Cargo.toml index e1ce6e79cb99..adf353fe2e4f 100644 --- a/polkadot/node/core/pvf/common/Cargo.toml +++ b/polkadot/node/core/pvf/common/Cargo.toml @@ -14,7 +14,7 @@ cpu-time = "1.0.0" futures = "0.3.30" gum = { package = "tracing-gum", path = "../../../gum" } libc = "0.2.152" -nix = { version = "0.27.1", features = ["resource", "sched"] } +nix = { version = "0.28.0", features = ["resource", "sched"] } thiserror = { workspace = true } parity-scale-codec = { version = "3.6.1", default-features = false, features = [ diff --git a/polkadot/node/core/pvf/execute-worker/Cargo.toml b/polkadot/node/core/pvf/execute-worker/Cargo.toml index 04a620573b2e..3480264d1daa 100644 --- a/polkadot/node/core/pvf/execute-worker/Cargo.toml +++ b/polkadot/node/core/pvf/execute-worker/Cargo.toml @@ -13,7 +13,7 @@ workspace = true cpu-time = "1.0.0" gum = { package = "tracing-gum", path = "../../../gum" } cfg-if = "1.0" -nix = { version = "0.27.1", features = ["process", "resource", "sched"] } +nix = { version = "0.28.0", features = ["process", "resource", "sched"] } libc = "0.2.152" parity-scale-codec = { version = "3.6.1", default-features = false, features = ["derive"] } diff --git a/polkadot/node/core/pvf/prepare-worker/Cargo.toml b/polkadot/node/core/pvf/prepare-worker/Cargo.toml index 9ecf1c8af501..12628565e3a2 100644 --- a/polkadot/node/core/pvf/prepare-worker/Cargo.toml +++ b/polkadot/node/core/pvf/prepare-worker/Cargo.toml @@ -18,7 +18,7 @@ rayon = "1.5.1" tracking-allocator = { package = "staging-tracking-allocator", path = "../../../tracking-allocator" } tikv-jemalloc-ctl = { version = "0.5.0", optional = true } tikv-jemallocator = { version = "0.5.0", optional = true } -nix = { version = "0.27.1", features = ["process", "resource", "sched"] } +nix = { version = "0.28.0", features = ["process", "resource", "sched"] } parity-scale-codec = { version = "3.6.1", default-features = false, features = ["derive"] } diff --git a/polkadot/node/network/availability-distribution/Cargo.toml b/polkadot/node/network/availability-distribution/Cargo.toml index 344389d224f5..ff3529449082 100644 --- a/polkadot/node/network/availability-distribution/Cargo.toml +++ b/polkadot/node/network/availability-distribution/Cargo.toml @@ -25,7 +25,7 @@ thiserror = { workspace = true } rand = "0.8.5" derive_more = "0.99.17" schnellru = "0.2.1" -fatality = "0.1.0" +fatality = "0.1.1" [dev-dependencies] polkadot-node-subsystem-test-helpers = { path = "../../subsystem-test-helpers" } diff --git a/polkadot/node/network/availability-recovery/Cargo.toml b/polkadot/node/network/availability-recovery/Cargo.toml index b7f817d85235..d12c1b1cff95 100644 --- a/polkadot/node/network/availability-recovery/Cargo.toml +++ b/polkadot/node/network/availability-recovery/Cargo.toml @@ -14,7 +14,7 @@ futures = "0.3.30" tokio = "1.37" schnellru = "0.2.1" rand = "0.8.5" -fatality = "0.1.0" +fatality = "0.1.1" thiserror = { workspace = true } async-trait = "0.1.79" gum = { package = "tracing-gum", path = "../../gum" } diff --git a/polkadot/node/network/bridge/Cargo.toml b/polkadot/node/network/bridge/Cargo.toml index 8ab571fbe7c3..4bb49baba921 100644 --- a/polkadot/node/network/bridge/Cargo.toml +++ b/polkadot/node/network/bridge/Cargo.toml @@ -24,7 +24,7 @@ polkadot-node-subsystem = { path = "../../subsystem" } polkadot-overseer = { path = "../../overseer" } parking_lot = "0.12.1" bytes = "1" -fatality = "0.1.0" +fatality = "0.1.1" thiserror = { workspace = true } [dev-dependencies] diff --git a/polkadot/node/network/collator-protocol/Cargo.toml b/polkadot/node/network/collator-protocol/Cargo.toml index 201615ac9ffb..aa60c0166d20 100644 --- a/polkadot/node/network/collator-protocol/Cargo.toml +++ b/polkadot/node/network/collator-protocol/Cargo.toml @@ -24,7 +24,7 @@ polkadot-node-network-protocol = { path = "../protocol" } polkadot-node-primitives = { path = "../../primitives" } polkadot-node-subsystem-util = { path = "../../subsystem-util" } polkadot-node-subsystem = { path = "../../subsystem" } -fatality = "0.1.0" +fatality = "0.1.1" thiserror = { workspace = true } tokio-util = "0.7.1" diff --git a/polkadot/node/network/dispute-distribution/Cargo.toml b/polkadot/node/network/dispute-distribution/Cargo.toml index 7d9a719d8525..eb8a7606304e 100644 --- a/polkadot/node/network/dispute-distribution/Cargo.toml +++ b/polkadot/node/network/dispute-distribution/Cargo.toml @@ -25,7 +25,7 @@ sc-network = { path = "../../../../substrate/client/network" } sp-application-crypto = { path = "../../../../substrate/primitives/application-crypto" } sp-keystore = { path = "../../../../substrate/primitives/keystore" } thiserror = { workspace = true } -fatality = "0.1.0" +fatality = "0.1.1" schnellru = "0.2.1" indexmap = "2.0.0" diff --git a/polkadot/node/network/protocol/Cargo.toml b/polkadot/node/network/protocol/Cargo.toml index 29975fc735f2..2b741051b4fe 100644 --- a/polkadot/node/network/protocol/Cargo.toml +++ b/polkadot/node/network/protocol/Cargo.toml @@ -24,7 +24,7 @@ sp-runtime = { path = "../../../../substrate/primitives/runtime" } strum = { version = "0.26.2", features = ["derive"] } futures = "0.3.30" thiserror = { workspace = true } -fatality = "0.1.0" +fatality = "0.1.1" rand = "0.8" derive_more = "0.99" gum = { package = "tracing-gum", path = "../../gum" } diff --git a/polkadot/node/network/statement-distribution/Cargo.toml b/polkadot/node/network/statement-distribution/Cargo.toml index b5cfeaa0c1ed..da8c91a0a291 100644 --- a/polkadot/node/network/statement-distribution/Cargo.toml +++ b/polkadot/node/network/statement-distribution/Cargo.toml @@ -24,7 +24,7 @@ arrayvec = "0.7.4" indexmap = "2.0.0" parity-scale-codec = { version = "3.6.1", default-features = false, features = ["derive"] } thiserror = { workspace = true } -fatality = "0.1.0" +fatality = "0.1.1" bitvec = "1" [dev-dependencies] diff --git a/polkadot/node/subsystem-util/Cargo.toml b/polkadot/node/subsystem-util/Cargo.toml index b6cc368b9245..492a9847c96b 100644 --- a/polkadot/node/subsystem-util/Cargo.toml +++ b/polkadot/node/subsystem-util/Cargo.toml @@ -19,7 +19,7 @@ parking_lot = "0.12.1" pin-project = "1.0.9" rand = "0.8.5" thiserror = { workspace = true } -fatality = "0.1.0" +fatality = "0.1.1" gum = { package = "tracing-gum", path = "../gum" } derive_more = "0.99.17" schnellru = "0.2.1" diff --git a/prdoc/pr_3962.prdoc b/prdoc/pr_3962.prdoc new file mode 100644 index 000000000000..7ef59d38ce5c --- /dev/null +++ b/prdoc/pr_3962.prdoc @@ -0,0 +1,12 @@ +title: Change fork calculation algorithm. + +doc: + - audience: Node Dev + description: | + This PR changes the fork calculation and pruning algorithm to enable future block header pruning. + During the finalization of the block we prune known stale forks, so forks are pruned faster. + +crates: + - name: sc-client-api + - name: sc-client-db + - name: sp-blockchain diff --git a/substrate/bin/node/cli/Cargo.toml b/substrate/bin/node/cli/Cargo.toml index 323afa56696a..a77e197cf6f9 100644 --- a/substrate/bin/node/cli/Cargo.toml +++ b/substrate/bin/node/cli/Cargo.toml @@ -132,7 +132,7 @@ sp-crypto-hashing = { path = "../../../primitives/crypto/hashing" } futures = "0.3.30" tempfile = "3.1.0" assert_cmd = "2.0.2" -nix = { version = "0.27.1", features = ["signal"] } +nix = { version = "0.28.0", features = ["signal"] } regex = "1.6.0" platforms = "3.0" soketto = "0.7.1" diff --git a/substrate/client/api/src/in_mem.rs b/substrate/client/api/src/in_mem.rs index b933ed1f17e0..ba89aede9147 100644 --- a/substrate/client/api/src/in_mem.rs +++ b/substrate/client/api/src/in_mem.rs @@ -419,20 +419,6 @@ impl blockchain::Backend for Blockchain { Ok(self.storage.read().leaves.hashes()) } - fn displaced_leaves_after_finalizing( - &self, - block_number: NumberFor, - ) -> sp_blockchain::Result> { - Ok(self - .storage - .read() - .leaves - .displaced_by_finalize_height(block_number) - .leaves() - .cloned() - .collect::>()) - } - fn children(&self, _parent_hash: Block::Hash) -> sp_blockchain::Result> { unimplemented!() } diff --git a/substrate/client/api/src/leaves.rs b/substrate/client/api/src/leaves.rs index a8a988771e2f..e129de8bf3fa 100644 --- a/substrate/client/api/src/leaves.rs +++ b/substrate/client/api/src/leaves.rs @@ -49,7 +49,7 @@ pub struct FinalizationOutcome { removed: BTreeMap, Vec>, } -impl FinalizationOutcome { +impl FinalizationOutcome { /// Merge with another. This should only be used for displaced items that /// are produced within one transaction of each other. pub fn merge(&mut self, mut other: Self) { @@ -63,6 +63,16 @@ impl FinalizationOutcome { pub fn leaves(&self) -> impl Iterator { self.removed.values().flatten() } + + /// Constructor + pub fn new(new_displaced: impl Iterator) -> Self { + let mut removed = BTreeMap::, Vec>::new(); + for (hash, number) in new_displaced { + removed.entry(Reverse(number)).or_default().push(hash); + } + + FinalizationOutcome { removed } + } } /// list of leaf hashes ordered by number (descending). @@ -151,39 +161,12 @@ where Some(RemoveOutcome { inserted, removed: LeafSetItem { hash, number } }) } - /// Note a block height finalized, displacing all leaves with number less than the finalized - /// block's. - /// - /// Although it would be more technically correct to also prune out leaves at the - /// same number as the finalized block, but with different hashes, the current behavior - /// is simpler and our assumptions about how finalization works means that those leaves - /// will be pruned soon afterwards anyway. - pub fn finalize_height(&mut self, number: N) -> FinalizationOutcome { - let boundary = if number == N::zero() { - return FinalizationOutcome { removed: BTreeMap::new() } - } else { - number - N::one() - }; - - let below_boundary = self.storage.split_off(&Reverse(boundary)); - FinalizationOutcome { removed: below_boundary } - } - - /// The same as [`Self::finalize_height`], but it only simulates the operation. - /// - /// This means that no changes are done. - /// - /// Returns the leaves that would be displaced by finalizing the given block. - pub fn displaced_by_finalize_height(&self, number: N) -> FinalizationOutcome { - let boundary = if number == N::zero() { - return FinalizationOutcome { removed: BTreeMap::new() } - } else { - number - N::one() - }; - - let below_boundary = self.storage.range(&Reverse(boundary)..); - FinalizationOutcome { - removed: below_boundary.map(|(k, v)| (k.clone(), v.clone())).collect(), + /// Remove all leaves displaced by the last block finalization. + pub fn remove_displaced_leaves(&mut self, displaced_leaves: &FinalizationOutcome) { + for (number, hashes) in &displaced_leaves.removed { + for hash in hashes.iter() { + self.remove_leaf(number, hash); + } } } @@ -420,32 +403,6 @@ mod tests { assert!(set.contains(11, 11_2)); } - #[test] - fn finalization_works() { - let mut set = LeafSet::new(); - set.import(9_1u32, 9u32, 0u32); - set.import(10_1, 10, 9_1); - set.import(10_2, 10, 9_1); - set.import(11_1, 11, 10_1); - set.import(11_2, 11, 10_1); - set.import(12_1, 12, 11_2); - - let outcome = set.finalize_height(11); - assert_eq!(set.count(), 2); - assert!(set.contains(11, 11_1)); - assert!(set.contains(12, 12_1)); - assert_eq!( - outcome.removed, - [(Reverse(10), vec![10_2])].into_iter().collect::>(), - ); - - set.undo().undo_finalization(outcome); - assert_eq!(set.count(), 3); - assert!(set.contains(11, 11_1)); - assert!(set.contains(12, 12_1)); - assert!(set.contains(10, 10_2)); - } - #[test] fn flush_to_disk() { const PREFIX: &[u8] = b"abcdefg"; @@ -479,35 +436,4 @@ mod tests { assert!(set.contains(10, 1_2)); assert!(!set.contains(10, 1_3)); } - - #[test] - fn finalization_consistent_with_disk() { - const PREFIX: &[u8] = b"prefix"; - let db = Arc::new(sp_database::MemDb::default()); - - let mut set = LeafSet::new(); - set.import(10_1u32, 10u32, 0u32); - set.import(11_1, 11, 10_2); - set.import(11_2, 11, 10_2); - set.import(12_1, 12, 11_123); - - assert!(set.contains(10, 10_1)); - - let mut tx = Transaction::new(); - set.prepare_transaction(&mut tx, 0, PREFIX); - db.commit(tx).unwrap(); - - let _ = set.finalize_height(11); - let mut tx = Transaction::new(); - set.prepare_transaction(&mut tx, 0, PREFIX); - db.commit(tx).unwrap(); - - assert!(set.contains(11, 11_1)); - assert!(set.contains(11, 11_2)); - assert!(set.contains(12, 12_1)); - assert!(!set.contains(10, 10_1)); - - let set2 = LeafSet::read_from_db(&*db, 0, PREFIX).unwrap(); - assert_eq!(set, set2); - } } diff --git a/substrate/client/consensus/babe/src/lib.rs b/substrate/client/consensus/babe/src/lib.rs index d10bdd8c7e4c..a1ca6fd23b59 100644 --- a/substrate/client/consensus/babe/src/lib.rs +++ b/substrate/client/consensus/babe/src/lib.rs @@ -562,9 +562,10 @@ fn aux_storage_cleanup + HeaderBackend, Block: B // Cleans data for stale forks. let stale_forks = match client.expand_forks(¬ification.stale_heads) { Ok(stale_forks) => stale_forks, - Err((stale_forks, e)) => { + Err(e) => { warn!(target: LOG_TARGET, "{:?}", e); - stale_forks + + Default::default() }, }; hashes.extend(stale_forks.iter()); diff --git a/substrate/client/consensus/babe/src/tests.rs b/substrate/client/consensus/babe/src/tests.rs index 38c9e1ff6ac2..716067ae4000 100644 --- a/substrate/client/consensus/babe/src/tests.rs +++ b/substrate/client/consensus/babe/src/tests.rs @@ -1094,8 +1094,8 @@ async fn obsolete_blocks_aux_data_cleanup() { assert!(aux_data_check(&fork1_hashes[2..3], false)); // Present: A4 assert!(aux_data_check(&fork1_hashes[3..], true)); - // Present C4, C5 - assert!(aux_data_check(&fork3_hashes, true)); + // Wiped C4, C5 + assert!(aux_data_check(&fork3_hashes, false)); } #[tokio::test] diff --git a/substrate/client/db/src/lib.rs b/substrate/client/db/src/lib.rs index 0faa90dfc4f9..36f9aea817c9 100644 --- a/substrate/client/db/src/lib.rs +++ b/substrate/client/db/src/lib.rs @@ -68,8 +68,8 @@ use sc_client_api::{ use sc_state_db::{IsPruned, LastCanonicalized, StateDb}; use sp_arithmetic::traits::Saturating; use sp_blockchain::{ - Backend as _, CachedHeaderMetadata, Error as ClientError, HeaderBackend, HeaderMetadata, - HeaderMetadataCache, Result as ClientResult, + Backend as _, CachedHeaderMetadata, DisplacedLeavesAfterFinalization, Error as ClientError, + HeaderBackend, HeaderMetadata, HeaderMetadataCache, Result as ClientResult, }; use sp_core::{ offchain::OffchainOverlayedChange, @@ -747,19 +747,6 @@ impl sc_client_api::blockchain::Backend for BlockchainDb, - ) -> ClientResult> { - Ok(self - .leaves - .read() - .displaced_by_finalize_height(block_number) - .leaves() - .cloned() - .collect::>()) - } - fn children(&self, parent_hash: Block::Hash) -> ClientResult> { children::read_children(&*self.db, columns::META, meta_keys::CHILDREN_PREFIX, parent_hash) } @@ -1813,14 +1800,13 @@ impl Backend { apply_state_commit(transaction, commit); } - let new_displaced = self.blockchain.leaves.write().finalize_height(f_num); - self.prune_blocks( - transaction, - f_num, - f_hash, - &new_displaced, - current_transaction_justifications, - )?; + let new_displaced = self.blockchain.displaced_leaves_after_finalizing(f_hash, f_num)?; + let finalization_outcome = + FinalizationOutcome::new(new_displaced.displaced_leaves.clone().into_iter()); + + self.blockchain.leaves.write().remove_displaced_leaves(&finalization_outcome); + + self.prune_blocks(transaction, f_num, &new_displaced, current_transaction_justifications)?; Ok(()) } @@ -1829,8 +1815,7 @@ impl Backend { &self, transaction: &mut Transaction, finalized_number: NumberFor, - finalized_hash: Block::Hash, - displaced: &FinalizationOutcome>, + displaced: &DisplacedLeavesAfterFinalization, current_transaction_justifications: &mut HashMap, ) -> ClientResult<()> { match self.blocks_pruning { @@ -1858,10 +1843,10 @@ impl Backend { self.prune_block(transaction, BlockId::::number(number))?; } - self.prune_displaced_branches(transaction, finalized_hash, displaced)?; + self.prune_displaced_branches(transaction, displaced)?; }, BlocksPruning::KeepFinalized => { - self.prune_displaced_branches(transaction, finalized_hash, displaced)?; + self.prune_displaced_branches(transaction, displaced)?; }, } Ok(()) @@ -1870,21 +1855,13 @@ impl Backend { fn prune_displaced_branches( &self, transaction: &mut Transaction, - finalized: Block::Hash, - displaced: &FinalizationOutcome>, + displaced: &DisplacedLeavesAfterFinalization, ) -> ClientResult<()> { // Discard all blocks from displaced branches - for h in displaced.leaves() { - match sp_blockchain::tree_route(&self.blockchain, *h, finalized) { - Ok(tree_route) => - for r in tree_route.retracted() { - self.blockchain.insert_persisted_body_if_pinned(r.hash)?; - self.prune_block(transaction, BlockId::::hash(r.hash))?; - }, - Err(sp_blockchain::Error::UnknownBlock(_)) => { - // Sometimes routes can't be calculated. E.g. after warp sync. - }, - Err(e) => Err(e)?, + for (_, tree_route) in displaced.tree_routes.iter() { + for r in tree_route.retracted() { + self.blockchain.insert_persisted_body_if_pinned(r.hash)?; + self.prune_block(transaction, BlockId::::hash(r.hash))?; } } Ok(()) @@ -3190,6 +3167,9 @@ pub(crate) mod tests { #[test] fn test_leaves_pruned_on_finality() { + // / 1b - 2b - 3b + // 0 - 1a - 2a + // \ 1c let backend: Backend = Backend::new_test(10, 10); let block0 = insert_header(&backend, 0, Default::default(), None, Default::default()); @@ -3201,18 +3181,16 @@ pub(crate) mod tests { let block2_a = insert_header(&backend, 2, block1_a, None, Default::default()); let block2_b = insert_header(&backend, 2, block1_b, None, Default::default()); - let block2_c = insert_header(&backend, 2, block1_b, None, [1; 32].into()); - assert_eq!( - backend.blockchain().leaves().unwrap(), - vec![block2_a, block2_b, block2_c, block1_c] - ); + let block3_b = insert_header(&backend, 3, block2_b, None, [3; 32].into()); + + assert_eq!(backend.blockchain().leaves().unwrap(), vec![block3_b, block2_a, block1_c]); backend.finalize_block(block1_a, None).unwrap(); backend.finalize_block(block2_a, None).unwrap(); - // leaves at same height stay. Leaves at lower heights pruned. - assert_eq!(backend.blockchain().leaves().unwrap(), vec![block2_a, block2_b, block2_c]); + // All leaves are pruned that are known to not belong to canonical branch + assert_eq!(backend.blockchain().leaves().unwrap(), vec![block2_a]); } #[test] diff --git a/substrate/client/merkle-mountain-range/src/offchain_mmr.rs b/substrate/client/merkle-mountain-range/src/offchain_mmr.rs index 3c3f0beb6c6a..94593f9c2c7b 100644 --- a/substrate/client/merkle-mountain-range/src/offchain_mmr.rs +++ b/substrate/client/merkle-mountain-range/src/offchain_mmr.rs @@ -33,7 +33,7 @@ use sp_runtime::{ traits::{Block, Header, NumberFor, One}, Saturating, }; -use std::{collections::VecDeque, sync::Arc}; +use std::{collections::VecDeque, default::Default, sync::Arc}; /// `OffchainMMR` exposes MMR offchain canonicalization and pruning logic. pub struct OffchainMmr, C> { @@ -273,12 +273,11 @@ where self.write_gadget_state_or_log(); // Remove offchain MMR nodes for stale forks. - let stale_forks = self.client.expand_forks(¬ification.stale_heads).unwrap_or_else( - |(stale_forks, e)| { - warn!(target: LOG_TARGET, "{:?}", e); - stale_forks - }, - ); + let stale_forks = self.client.expand_forks(¬ification.stale_heads).unwrap_or_else(|e| { + warn!(target: LOG_TARGET, "{:?}", e); + + Default::default() + }); for hash in stale_forks.iter() { self.prune_branch(hash); } diff --git a/substrate/client/rpc-spec-v2/src/chain_head/tests.rs b/substrate/client/rpc-spec-v2/src/chain_head/tests.rs index 363d11235dda..b195e05b6649 100644 --- a/substrate/client/rpc-spec-v2/src/chain_head/tests.rs +++ b/substrate/client/rpc-spec-v2/src/chain_head/tests.rs @@ -2487,6 +2487,7 @@ async fn follow_report_multiple_pruned_block() { client.finalize_block(block_3_hash, None).unwrap(); // Finalizing block 3 directly will also result in block 1 and 2 being finalized. + // It will also mark block 2 and block 3 from the fork as pruned. let event: FollowEvent = get_next_event(&mut sub).await; let expected = FollowEvent::Finalized(Finalized { finalized_block_hashes: vec![ @@ -2494,7 +2495,7 @@ async fn follow_report_multiple_pruned_block() { format!("{:?}", block_2_hash), format!("{:?}", block_3_hash), ], - pruned_block_hashes: vec![], + pruned_block_hashes: vec![format!("{:?}", block_2_f_hash), format!("{:?}", block_3_f_hash)], }); assert_eq!(event, expected); @@ -2504,7 +2505,6 @@ async fn follow_report_multiple_pruned_block() { // ^^^ finalized // -> block 1 -> block 2_f -> block 3_f // - // Mark block 4 as finalized to force block 2_f and 3_f to get pruned. let block_4 = BlockBuilderBuilder::new(&*client) .on_parent_block(block_3.hash()) @@ -2535,11 +2535,11 @@ async fn follow_report_multiple_pruned_block() { }); assert_eq!(event, expected); - // Block 4 and 5 be reported as pruned, not just the stale head (block 5). + // Blocks from the fork were pruned earlier. let event: FollowEvent = get_next_event(&mut sub).await; let expected = FollowEvent::Finalized(Finalized { finalized_block_hashes: vec![format!("{:?}", block_4_hash)], - pruned_block_hashes: vec![format!("{:?}", block_2_f_hash), format!("{:?}", block_3_f_hash)], + pruned_block_hashes: vec![], }); assert_eq!(event, expected); } @@ -3714,16 +3714,8 @@ async fn follow_unique_pruned_blocks() { // The chainHead will see block 5 as the best block. However, the // client will finalize the block 6, which is on another fork. // - // When the block 6 is finalized, block 2 block 3 block 4 and block 5 are placed on an invalid - // fork. However, pruning of blocks happens on level N - 1. - // Therefore, no pruned blocks are reported yet. + // When the block 6 is finalized all blocks from the stale forks (2, 3, 4, 5) are pruned. // - // When the block 7 is finalized, block 3 is detected as stale. At this step, block 2 and 3 - // are reported as pruned. - // - // When the block 8 is finalized, block 5 block 4 and block 2 are detected as stale. However, - // only blocks 5 and 4 are reported as pruned. This is because the block 2 was previously - // reported. // Initial setup steps: let block_1_hash = @@ -3776,16 +3768,33 @@ async fn follow_unique_pruned_blocks() { }); assert_eq!(event, expected); - // Block 2 must be reported as pruned, even if it was the previous best. - let event: FollowEvent = get_next_event(&mut sub).await; + // All blocks from stale forks are pruned when we finalize block 6. + let mut event: FollowEvent = get_next_event(&mut sub).await; + + // Sort pruned block hashes to counter flaky test caused by event generation (get_pruned_hashes) + if let FollowEvent::Finalized(Finalized { pruned_block_hashes, .. }) = &mut event { + pruned_block_hashes.sort(); + } + let expected_pruned_block_hashes = { + let mut hashes = vec![ + format!("{:?}", block_2_hash), + format!("{:?}", block_3_hash), + format!("{:?}", block_4_hash), + format!("{:?}", block_5_hash), + ]; + hashes.sort(); + hashes + }; + let expected = FollowEvent::Finalized(Finalized { finalized_block_hashes: vec![ format!("{:?}", block_1_hash), format!("{:?}", block_2_f_hash), format!("{:?}", block_6_hash), ], - pruned_block_hashes: vec![], + pruned_block_hashes: expected_pruned_block_hashes, }); + assert_eq!(event, expected); // Pruned hash can be unpinned. @@ -3802,9 +3811,10 @@ async fn follow_unique_pruned_blocks() { client.finalize_block(block_7_hash, None).unwrap(); let event: FollowEvent = get_next_event(&mut sub).await; + // All necessary blocks were pruned on block 6 finalization. let expected = FollowEvent::Finalized(Finalized { finalized_block_hashes: vec![format!("{:?}", block_7_hash)], - pruned_block_hashes: vec![format!("{:?}", block_2_hash), format!("{:?}", block_3_hash)], + pruned_block_hashes: vec![], }); assert_eq!(event, expected); @@ -3815,10 +3825,11 @@ async fn follow_unique_pruned_blocks() { // Finalize the block 8. client.finalize_block(block_8_hash, None).unwrap(); + // All necessary blocks were pruned on block 6 finalization. let event: FollowEvent = get_next_event(&mut sub).await; let expected = FollowEvent::Finalized(Finalized { finalized_block_hashes: vec![format!("{:?}", block_8_hash)], - pruned_block_hashes: vec![format!("{:?}", block_4_hash), format!("{:?}", block_5_hash)], + pruned_block_hashes: vec![], }); assert_eq!(event, expected); } diff --git a/substrate/client/service/src/client/client.rs b/substrate/client/service/src/client/client.rs index 35e8b53a09cf..3c25c233775b 100644 --- a/substrate/client/service/src/client/client.rs +++ b/substrate/client/service/src/client/client.rs @@ -978,8 +978,12 @@ where // The stale heads are the leaves that will be displaced after the // block is finalized. - let stale_heads = - self.backend.blockchain().displaced_leaves_after_finalizing(block_number)?; + let stale_heads = self + .backend + .blockchain() + .displaced_leaves_after_finalizing(hash, block_number)? + .hashes() + .collect(); let header = self .backend diff --git a/substrate/client/service/test/src/client/mod.rs b/substrate/client/service/test/src/client/mod.rs index 4fcb7c160cb4..6542830c998b 100644 --- a/substrate/client/service/test/src/client/mod.rs +++ b/substrate/client/service/test/src/client/mod.rs @@ -1164,7 +1164,7 @@ fn finalizing_diverged_block_should_trigger_reorg() { // G -> A1 -> A2 // \ - // -> B1 -> B2 + // -> B1 -> B2 -> B3 let mut finality_notifications = client.finality_notification_stream(); @@ -1249,8 +1249,8 @@ fn finalizing_diverged_block_should_trigger_reorg() { ClientExt::finalize_block(&client, b3.hash(), None).unwrap(); - finality_notification_check(&mut finality_notifications, &[b1.hash()], &[]); - finality_notification_check(&mut finality_notifications, &[b2.hash(), b3.hash()], &[a2.hash()]); + finality_notification_check(&mut finality_notifications, &[b1.hash()], &[a2.hash()]); + finality_notification_check(&mut finality_notifications, &[b2.hash(), b3.hash()], &[]); assert!(matches!(finality_notifications.try_recv().unwrap_err(), TryRecvError::Empty)); } @@ -1371,8 +1371,12 @@ fn finality_notifications_content() { // Import and finalize D4 block_on(client.import_as_final(BlockOrigin::Own, d4.clone())).unwrap(); - finality_notification_check(&mut finality_notifications, &[a1.hash(), a2.hash()], &[c1.hash()]); - finality_notification_check(&mut finality_notifications, &[d3.hash(), d4.hash()], &[b2.hash()]); + finality_notification_check( + &mut finality_notifications, + &[a1.hash(), a2.hash()], + &[c1.hash(), b2.hash()], + ); + finality_notification_check(&mut finality_notifications, &[d3.hash(), d4.hash()], &[a3.hash()]); assert!(matches!(finality_notifications.try_recv().unwrap_err(), TryRecvError::Empty)); } @@ -1601,9 +1605,9 @@ fn doesnt_import_blocks_that_revert_finality() { block_on(client.import(BlockOrigin::Own, a3.clone())).unwrap(); ClientExt::finalize_block(&client, a3.hash(), None).unwrap(); - finality_notification_check(&mut finality_notifications, &[a1.hash(), a2.hash()], &[]); + finality_notification_check(&mut finality_notifications, &[a1.hash(), a2.hash()], &[b2.hash()]); - finality_notification_check(&mut finality_notifications, &[a3.hash()], &[b2.hash()]); + finality_notification_check(&mut finality_notifications, &[a3.hash()], &[]); assert!(matches!(finality_notifications.try_recv().unwrap_err(), TryRecvError::Empty)); } diff --git a/substrate/primitives/blockchain/src/backend.rs b/substrate/primitives/blockchain/src/backend.rs index 7a09865f858d..06e5b682964a 100644 --- a/substrate/primitives/blockchain/src/backend.rs +++ b/substrate/primitives/blockchain/src/backend.rs @@ -21,14 +21,17 @@ use log::warn; use parking_lot::RwLock; use sp_runtime::{ generic::BlockId, - traits::{Block as BlockT, Header as HeaderT, NumberFor, Saturating}, + traits::{Block as BlockT, Header as HeaderT, NumberFor, Zero}, Justifications, }; -use std::collections::btree_set::BTreeSet; +use std::collections::{btree_map::BTreeMap, btree_set::BTreeSet}; use crate::header_metadata::HeaderMetadata; -use crate::error::{Error, Result}; +use crate::{ + error::{Error, Result}, + tree_route, TreeRoute, +}; /// Blockchain database header backend. Does not perform any validation. pub trait HeaderBackend: Send + Sync { @@ -89,62 +92,32 @@ pub trait HeaderBackend: Send + Sync { pub trait ForkBackend: HeaderMetadata + HeaderBackend + Send + Sync { - /// Best effort to get all the header hashes that are part of the provided forks - /// starting only from the fork heads. + /// Returns block hashes for provided fork heads. It skips the fork if when blocks are missing + /// (e.g. warp-sync) and internal `tree_route` function fails. /// - /// The function tries to reconstruct the route from the fork head to the canonical chain. - /// If any of the hashes on the route can't be found in the db, the function won't be able - /// to reconstruct the route anymore. In this case it will give up expanding the current fork, - /// move on to the next ones and at the end it will return an error that also contains - /// the partially expanded forks. + /// Example: + /// G --- A1 --- A2 --- A3 --- A4 ( < fork1 ) + /// \-----C4 --- C5 ( < fork2 ) + /// We finalize A3 and call expand_fork(C5). Result = (C5,C4). fn expand_forks( &self, fork_heads: &[Block::Hash], - ) -> std::result::Result, (BTreeSet, Error)> { - let mut missing_blocks = vec![]; + ) -> std::result::Result, Error> { let mut expanded_forks = BTreeSet::new(); for fork_head in fork_heads { - let mut route_head = *fork_head; - // Insert stale blocks hashes until canonical chain is reached. - // If we reach a block that is already part of the `expanded_forks` we can stop - // processing the fork. - while expanded_forks.insert(route_head) { - match self.header_metadata(route_head) { - Ok(meta) => { - // If the parent is part of the canonical chain or there doesn't exist a - // block hash for the parent number (bug?!), we can abort adding blocks. - let parent_number = meta.number.saturating_sub(1u32.into()); - match self.hash(parent_number) { - Ok(Some(parent_hash)) => - if parent_hash == meta.parent { - break - }, - Ok(None) | Err(_) => { - missing_blocks.push(BlockId::::Number(parent_number)); - break - }, - } - - route_head = meta.parent; - }, - Err(_e) => { - missing_blocks.push(BlockId::::Hash(route_head)); - break - }, - } + match tree_route(self, *fork_head, self.info().finalized_hash) { + Ok(tree_route) => { + for block in tree_route.retracted() { + expanded_forks.insert(block.hash); + } + continue + }, + Err(_) => { + // There are cases when blocks are missing (e.g. warp-sync). + }, } } - if !missing_blocks.is_empty() { - return Err(( - expanded_forks, - Error::UnknownBlocks(format!( - "Missing stale headers {:?} while expanding forks {:?}.", - fork_heads, missing_blocks - )), - )) - } - Ok(expanded_forks) } } @@ -172,14 +145,6 @@ pub trait Backend: /// Results must be ordered best (longest, highest) chain first. fn leaves(&self) -> Result>; - /// Returns displaced leaves after the given block would be finalized. - /// - /// The returned leaves do not contain the leaves from the same height as `block_number`. - fn displaced_leaves_after_finalizing( - &self, - block_number: NumberFor, - ) -> Result>; - /// Return hashes of all blocks that are children of the block with `parent_hash`. fn children(&self, parent_hash: Block::Hash) -> Result>; @@ -255,6 +220,67 @@ pub trait Backend: } fn block_indexed_body(&self, hash: Block::Hash) -> Result>>>; + + /// Returns all leaves that will be displaced after the block finalization. + fn displaced_leaves_after_finalizing( + &self, + finalized_block_hash: Block::Hash, + finalized_block_number: NumberFor, + ) -> std::result::Result, Error> { + let mut result = DisplacedLeavesAfterFinalization::default(); + + if finalized_block_number == Zero::zero() { + return Ok(result) + } + + // For each leaf determine whether it belongs to a non-canonical branch. + for leaf_hash in self.leaves()? { + let leaf_block_header = self.expect_header(leaf_hash)?; + let leaf_number = *leaf_block_header.number(); + + let leaf_tree_route = match tree_route(self, leaf_hash, finalized_block_hash) { + Ok(tree_route) => tree_route, + Err(Error::UnknownBlock(_)) => { + // Sometimes routes can't be calculated. E.g. after warp sync. + continue; + }, + Err(e) => Err(e)?, + }; + + // Is it a stale fork? + let needs_pruning = leaf_tree_route.common_block().hash != finalized_block_hash; + + if needs_pruning { + result.displaced_leaves.insert(leaf_hash, leaf_number); + result.tree_routes.insert(leaf_hash, leaf_tree_route); + } + } + + Ok(result) + } +} + +/// Result of [`Backend::displaced_leaves_after_finalizing`]. +#[derive(Clone, Debug)] +pub struct DisplacedLeavesAfterFinalization { + /// A collection of hashes and block numbers for displaced leaves. + pub displaced_leaves: BTreeMap>, + + /// A collection of tree routes from the leaves to finalized block. + pub tree_routes: BTreeMap>, +} + +impl Default for DisplacedLeavesAfterFinalization { + fn default() -> Self { + Self { displaced_leaves: Default::default(), tree_routes: Default::default() } + } +} + +impl DisplacedLeavesAfterFinalization { + /// Returns a collection of hashes for the displaced leaves. + pub fn hashes(&self) -> impl Iterator + '_ { + self.displaced_leaves.keys().cloned() + } } /// Blockchain info diff --git a/substrate/primitives/blockchain/src/header_metadata.rs b/substrate/primitives/blockchain/src/header_metadata.rs index ccd640c0567a..27caaae71add 100644 --- a/substrate/primitives/blockchain/src/header_metadata.rs +++ b/substrate/primitives/blockchain/src/header_metadata.rs @@ -97,7 +97,7 @@ pub fn lowest_common_ancestor + ?Sized>( } /// Compute a tree-route between two blocks. See tree-route docs for more details. -pub fn tree_route>( +pub fn tree_route + ?Sized>( backend: &T, from: Block::Hash, to: Block::Hash, diff --git a/substrate/test-utils/cli/Cargo.toml b/substrate/test-utils/cli/Cargo.toml index c4f876710005..87c595c66f34 100644 --- a/substrate/test-utils/cli/Cargo.toml +++ b/substrate/test-utils/cli/Cargo.toml @@ -19,7 +19,7 @@ targets = ["x86_64-unknown-linux-gnu"] substrate-rpc-client = { path = "../../utils/frame/rpc/client" } sp-rpc = { path = "../../primitives/rpc" } assert_cmd = "2.0.10" -nix = { version = "0.27.1", features = ["signal"] } +nix = { version = "0.28.0", features = ["signal"] } regex = "1.7.3" tokio = { version = "1.22.0", features = ["full"] } node-primitives = { path = "../../bin/node/primitives" } diff --git a/templates/parachain/node/src/chain_spec.rs b/templates/parachain/node/src/chain_spec.rs index 16c91865cdb4..51710f1199c3 100644 --- a/templates/parachain/node/src/chain_spec.rs +++ b/templates/parachain/node/src/chain_spec.rs @@ -22,11 +22,12 @@ pub fn get_from_seed(seed: &str) -> ::Pu /// The extensions for the [`ChainSpec`]. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, ChainSpecGroup, ChainSpecExtension)] -#[serde(deny_unknown_fields)] pub struct Extensions { /// The relay chain of the Parachain. + #[serde(alias = "relayChain", alias = "RelayChain")] pub relay_chain: String, /// The id of the Parachain. + #[serde(alias = "paraId", alias = "ParaId")] pub para_id: u32, }