From 712a750a995976a66a712c15376c0cfadea6914a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 14 May 2024 16:03:25 +0000 Subject: [PATCH 1/6] Bump nix from 0.27.1 to 0.28.0 (#4438) Bumps [nix](https://github.com/nix-rust/nix) from 0.27.1 to 0.28.0.
Changelog

Sourced from nix's changelog.

[0.28.0] - 2024-02-24

Added

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=nix&package-manager=cargo&previous-version=0.27.1&new-version=0.28.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 19 ++++++++++--------- cumulus/polkadot-parachain/Cargo.toml | 2 +- polkadot/Cargo.toml | 2 +- polkadot/node/core/pvf/common/Cargo.toml | 2 +- .../node/core/pvf/execute-worker/Cargo.toml | 2 +- .../node/core/pvf/prepare-worker/Cargo.toml | 2 +- substrate/bin/node/cli/Cargo.toml | 2 +- substrate/test-utils/cli/Cargo.toml | 2 +- 8 files changed, 17 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7711d51d0df6..2b2576d1adfa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8876,12 +8876,13 @@ dependencies = [ [[package]] name = "nix" -version = "0.27.1" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053" +checksum = "ab2156c4fce2f8df6c499cc1c763e4394b7482525bf2a9701c9d79d215f519e4" dependencies = [ "bitflags 2.4.0", "cfg-if", + "cfg_aliases", "libc", ] @@ -12535,7 +12536,7 @@ version = "6.0.0" dependencies = [ "assert_cmd", "color-eyre", - "nix 0.27.1", + "nix 0.28.0", "polkadot-cli", "polkadot-core-primitives", "polkadot-node-core-pvf", @@ -13219,7 +13220,7 @@ dependencies = [ "futures", "landlock", "libc", - "nix 0.27.1", + "nix 0.28.0", "parity-scale-codec", "polkadot-parachain-primitives", "polkadot-primitives", @@ -13244,7 +13245,7 @@ dependencies = [ "cfg-if", "cpu-time", "libc", - "nix 0.27.1", + "nix 0.28.0", "parity-scale-codec", "polkadot-node-core-pvf-common", "polkadot-parachain-primitives", @@ -13260,7 +13261,7 @@ dependencies = [ "cfg-if", "criterion", "libc", - "nix 0.27.1", + "nix 0.28.0", "parity-scale-codec", "polkadot-node-core-pvf-common", "polkadot-primitives", @@ -13553,7 +13554,7 @@ dependencies = [ "hex-literal", "jsonrpsee", "log", - "nix 0.27.1", + "nix 0.28.0", "pallet-transaction-payment", "pallet-transaction-payment-rpc", "pallet-transaction-payment-rpc-runtime-api", @@ -20120,7 +20121,7 @@ dependencies = [ "kitchensink-runtime", "log", "mmr-gadget", - "nix 0.27.1", + "nix 0.28.0", "node-primitives", "node-rpc", "node-testing", @@ -20520,7 +20521,7 @@ version = "0.1.0" dependencies = [ "assert_cmd", "futures", - "nix 0.27.1", + "nix 0.28.0", "node-primitives", "regex", "sc-cli", diff --git a/cumulus/polkadot-parachain/Cargo.toml b/cumulus/polkadot-parachain/Cargo.toml index f21a5baf973b..3d0aa94e8dea 100644 --- a/cumulus/polkadot-parachain/Cargo.toml +++ b/cumulus/polkadot-parachain/Cargo.toml @@ -118,7 +118,7 @@ substrate-build-script-utils = { path = "../../substrate/utils/build-script-util [dev-dependencies] assert_cmd = "2.0" -nix = { version = "0.27.1", features = ["signal"] } +nix = { version = "0.28.0", features = ["signal"] } tempfile = "3.8.0" tokio = { version = "1.32.0", features = ["macros", "parking_lot", "time"] } wait-timeout = "0.2" diff --git a/polkadot/Cargo.toml b/polkadot/Cargo.toml index 7b5679e1084e..3aeec8d5961e 100644 --- a/polkadot/Cargo.toml +++ b/polkadot/Cargo.toml @@ -43,7 +43,7 @@ tikv-jemallocator = { version = "0.5.0", features = ["unprefixed_malloc_on_suppo [dev-dependencies] assert_cmd = "2.0.4" -nix = { version = "0.27.1", features = ["signal"] } +nix = { version = "0.28.0", features = ["signal"] } tempfile = "3.2.0" tokio = "1.37" substrate-rpc-client = { path = "../substrate/utils/frame/rpc/client" } diff --git a/polkadot/node/core/pvf/common/Cargo.toml b/polkadot/node/core/pvf/common/Cargo.toml index e1ce6e79cb99..adf353fe2e4f 100644 --- a/polkadot/node/core/pvf/common/Cargo.toml +++ b/polkadot/node/core/pvf/common/Cargo.toml @@ -14,7 +14,7 @@ cpu-time = "1.0.0" futures = "0.3.30" gum = { package = "tracing-gum", path = "../../../gum" } libc = "0.2.152" -nix = { version = "0.27.1", features = ["resource", "sched"] } +nix = { version = "0.28.0", features = ["resource", "sched"] } thiserror = { workspace = true } parity-scale-codec = { version = "3.6.1", default-features = false, features = [ diff --git a/polkadot/node/core/pvf/execute-worker/Cargo.toml b/polkadot/node/core/pvf/execute-worker/Cargo.toml index 04a620573b2e..3480264d1daa 100644 --- a/polkadot/node/core/pvf/execute-worker/Cargo.toml +++ b/polkadot/node/core/pvf/execute-worker/Cargo.toml @@ -13,7 +13,7 @@ workspace = true cpu-time = "1.0.0" gum = { package = "tracing-gum", path = "../../../gum" } cfg-if = "1.0" -nix = { version = "0.27.1", features = ["process", "resource", "sched"] } +nix = { version = "0.28.0", features = ["process", "resource", "sched"] } libc = "0.2.152" parity-scale-codec = { version = "3.6.1", default-features = false, features = ["derive"] } diff --git a/polkadot/node/core/pvf/prepare-worker/Cargo.toml b/polkadot/node/core/pvf/prepare-worker/Cargo.toml index 9ecf1c8af501..12628565e3a2 100644 --- a/polkadot/node/core/pvf/prepare-worker/Cargo.toml +++ b/polkadot/node/core/pvf/prepare-worker/Cargo.toml @@ -18,7 +18,7 @@ rayon = "1.5.1" tracking-allocator = { package = "staging-tracking-allocator", path = "../../../tracking-allocator" } tikv-jemalloc-ctl = { version = "0.5.0", optional = true } tikv-jemallocator = { version = "0.5.0", optional = true } -nix = { version = "0.27.1", features = ["process", "resource", "sched"] } +nix = { version = "0.28.0", features = ["process", "resource", "sched"] } parity-scale-codec = { version = "3.6.1", default-features = false, features = ["derive"] } diff --git a/substrate/bin/node/cli/Cargo.toml b/substrate/bin/node/cli/Cargo.toml index 323afa56696a..a77e197cf6f9 100644 --- a/substrate/bin/node/cli/Cargo.toml +++ b/substrate/bin/node/cli/Cargo.toml @@ -132,7 +132,7 @@ sp-crypto-hashing = { path = "../../../primitives/crypto/hashing" } futures = "0.3.30" tempfile = "3.1.0" assert_cmd = "2.0.2" -nix = { version = "0.27.1", features = ["signal"] } +nix = { version = "0.28.0", features = ["signal"] } regex = "1.6.0" platforms = "3.0" soketto = "0.7.1" diff --git a/substrate/test-utils/cli/Cargo.toml b/substrate/test-utils/cli/Cargo.toml index c4f876710005..87c595c66f34 100644 --- a/substrate/test-utils/cli/Cargo.toml +++ b/substrate/test-utils/cli/Cargo.toml @@ -19,7 +19,7 @@ targets = ["x86_64-unknown-linux-gnu"] substrate-rpc-client = { path = "../../utils/frame/rpc/client" } sp-rpc = { path = "../../primitives/rpc" } assert_cmd = "2.0.10" -nix = { version = "0.27.1", features = ["signal"] } +nix = { version = "0.28.0", features = ["signal"] } regex = "1.7.3" tokio = { version = "1.22.0", features = ["full"] } node-primitives = { path = "../../bin/node/primitives" } From 12ab31508cf8f03e6cc233722e535c383dc97c41 Mon Sep 17 00:00:00 2001 From: Kian Paimani <5588131+kianenigma@users.noreply.github.com> Date: Tue, 14 May 2024 21:35:04 +0100 Subject: [PATCH 2/6] make polkadot-parachain chain-spec extension more relaxed (#4452) A small step towards https://forum.polkadot.network/t/polkadot-parachain-omni-node-gathering-ideas-and-feedback/7823 and #4352. Many parachains use `camelCase` and/or `PascalCase`ing for their chain spec extension. Sometimes the only reason polkadot-parachain cannot sync them is because it cannot parse the chain spec extension. This PR relaxes the requirement for the extension to be camel case. --------- Co-authored-by: Branislav Kontur --- .../polkadot-parachain/src/chain_spec/mod.rs | 22 ++++++++++++++++++- templates/parachain/node/src/chain_spec.rs | 3 ++- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/cumulus/polkadot-parachain/src/chain_spec/mod.rs b/cumulus/polkadot-parachain/src/chain_spec/mod.rs index bbda334e4c66..136a19e3166b 100644 --- a/cumulus/polkadot-parachain/src/chain_spec/mod.rs +++ b/cumulus/polkadot-parachain/src/chain_spec/mod.rs @@ -37,11 +37,12 @@ const SAFE_XCM_VERSION: u32 = xcm::prelude::XCM_VERSION; /// Generic extensions for Parachain ChainSpecs. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, ChainSpecGroup, ChainSpecExtension)] -#[serde(deny_unknown_fields)] pub struct Extensions { /// The relay chain of the Parachain. + #[serde(alias = "relayChain", alias = "RelayChain")] pub relay_chain: String, /// The id of the Parachain. + #[serde(alias = "paraId", alias = "ParaId")] pub para_id: u32, } @@ -78,3 +79,22 @@ where pub fn get_collator_keys_from_seed(seed: &str) -> ::Public { get_from_seed::(seed) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn can_decode_extension_camel_and_snake_case() { + let camel_case = r#"{"relayChain":"relay","paraId":1}"#; + let snake_case = r#"{"relay_chain":"relay","para_id":1}"#; + let pascal_case = r#"{"RelayChain":"relay","ParaId":1}"#; + + let camel_case_extension: Extensions = serde_json::from_str(camel_case).unwrap(); + let snake_case_extension: Extensions = serde_json::from_str(snake_case).unwrap(); + let pascal_case_extension: Extensions = serde_json::from_str(pascal_case).unwrap(); + + assert_eq!(camel_case_extension, snake_case_extension); + assert_eq!(snake_case_extension, pascal_case_extension); + } +} diff --git a/templates/parachain/node/src/chain_spec.rs b/templates/parachain/node/src/chain_spec.rs index 16c91865cdb4..51710f1199c3 100644 --- a/templates/parachain/node/src/chain_spec.rs +++ b/templates/parachain/node/src/chain_spec.rs @@ -22,11 +22,12 @@ pub fn get_from_seed(seed: &str) -> ::Pu /// The extensions for the [`ChainSpec`]. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, ChainSpecGroup, ChainSpecExtension)] -#[serde(deny_unknown_fields)] pub struct Extensions { /// The relay chain of the Parachain. + #[serde(alias = "relayChain", alias = "RelayChain")] pub relay_chain: String, /// The id of the Parachain. + #[serde(alias = "paraId", alias = "ParaId")] pub para_id: u32, } From 9c69bb9850ff228f063d344f5f4beae5f2a1c793 Mon Sep 17 00:00:00 2001 From: shamil-gadelshin Date: Wed, 15 May 2024 15:52:26 +0700 Subject: [PATCH 3/6] Change forks pruning algorithm. (#3962) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR changes the fork calculation and pruning algorithm to enable future block header pruning. It's required because the previous algorithm relied on the block header persistence. It follows the [related discussion](https://github.com/paritytech/polkadot-sdk/issues/1570) The previous code contained this comment describing the situation: ``` /// Note a block height finalized, displacing all leaves with number less than the finalized /// block's. /// /// Although it would be more technically correct to also prune out leaves at the /// same number as the finalized block, but with different hashes, the current behavior /// is simpler and our assumptions about how finalization works means that those leaves /// will be pruned soon afterwards anyway. pub fn finalize_height(&mut self, number: N) -> FinalizationOutcome { ``` The previous algorithm relied on the existing block headers to prune forks later and to enable block header pruning we need to clear all obsolete forks right after the block finalization to not depend on the related block headers in the future. --------- Co-authored-by: Bastian Köcher --- prdoc/pr_3962.prdoc | 12 ++ substrate/client/api/src/in_mem.rs | 14 -- substrate/client/api/src/leaves.rs | 108 +++---------- substrate/client/consensus/babe/src/lib.rs | 5 +- substrate/client/consensus/babe/src/tests.rs | 4 +- substrate/client/db/src/lib.rs | 72 +++------ .../merkle-mountain-range/src/offchain_mmr.rs | 13 +- .../rpc-spec-v2/src/chain_head/tests.rs | 47 +++--- substrate/client/service/src/client/client.rs | 8 +- .../client/service/test/src/client/mod.rs | 18 ++- .../primitives/blockchain/src/backend.rs | 142 +++++++++++------- .../blockchain/src/header_metadata.rs | 2 +- 12 files changed, 196 insertions(+), 249 deletions(-) create mode 100644 prdoc/pr_3962.prdoc diff --git a/prdoc/pr_3962.prdoc b/prdoc/pr_3962.prdoc new file mode 100644 index 000000000000..7ef59d38ce5c --- /dev/null +++ b/prdoc/pr_3962.prdoc @@ -0,0 +1,12 @@ +title: Change fork calculation algorithm. + +doc: + - audience: Node Dev + description: | + This PR changes the fork calculation and pruning algorithm to enable future block header pruning. + During the finalization of the block we prune known stale forks, so forks are pruned faster. + +crates: + - name: sc-client-api + - name: sc-client-db + - name: sp-blockchain diff --git a/substrate/client/api/src/in_mem.rs b/substrate/client/api/src/in_mem.rs index b933ed1f17e0..ba89aede9147 100644 --- a/substrate/client/api/src/in_mem.rs +++ b/substrate/client/api/src/in_mem.rs @@ -419,20 +419,6 @@ impl blockchain::Backend for Blockchain { Ok(self.storage.read().leaves.hashes()) } - fn displaced_leaves_after_finalizing( - &self, - block_number: NumberFor, - ) -> sp_blockchain::Result> { - Ok(self - .storage - .read() - .leaves - .displaced_by_finalize_height(block_number) - .leaves() - .cloned() - .collect::>()) - } - fn children(&self, _parent_hash: Block::Hash) -> sp_blockchain::Result> { unimplemented!() } diff --git a/substrate/client/api/src/leaves.rs b/substrate/client/api/src/leaves.rs index a8a988771e2f..e129de8bf3fa 100644 --- a/substrate/client/api/src/leaves.rs +++ b/substrate/client/api/src/leaves.rs @@ -49,7 +49,7 @@ pub struct FinalizationOutcome { removed: BTreeMap, Vec>, } -impl FinalizationOutcome { +impl FinalizationOutcome { /// Merge with another. This should only be used for displaced items that /// are produced within one transaction of each other. pub fn merge(&mut self, mut other: Self) { @@ -63,6 +63,16 @@ impl FinalizationOutcome { pub fn leaves(&self) -> impl Iterator { self.removed.values().flatten() } + + /// Constructor + pub fn new(new_displaced: impl Iterator) -> Self { + let mut removed = BTreeMap::, Vec>::new(); + for (hash, number) in new_displaced { + removed.entry(Reverse(number)).or_default().push(hash); + } + + FinalizationOutcome { removed } + } } /// list of leaf hashes ordered by number (descending). @@ -151,39 +161,12 @@ where Some(RemoveOutcome { inserted, removed: LeafSetItem { hash, number } }) } - /// Note a block height finalized, displacing all leaves with number less than the finalized - /// block's. - /// - /// Although it would be more technically correct to also prune out leaves at the - /// same number as the finalized block, but with different hashes, the current behavior - /// is simpler and our assumptions about how finalization works means that those leaves - /// will be pruned soon afterwards anyway. - pub fn finalize_height(&mut self, number: N) -> FinalizationOutcome { - let boundary = if number == N::zero() { - return FinalizationOutcome { removed: BTreeMap::new() } - } else { - number - N::one() - }; - - let below_boundary = self.storage.split_off(&Reverse(boundary)); - FinalizationOutcome { removed: below_boundary } - } - - /// The same as [`Self::finalize_height`], but it only simulates the operation. - /// - /// This means that no changes are done. - /// - /// Returns the leaves that would be displaced by finalizing the given block. - pub fn displaced_by_finalize_height(&self, number: N) -> FinalizationOutcome { - let boundary = if number == N::zero() { - return FinalizationOutcome { removed: BTreeMap::new() } - } else { - number - N::one() - }; - - let below_boundary = self.storage.range(&Reverse(boundary)..); - FinalizationOutcome { - removed: below_boundary.map(|(k, v)| (k.clone(), v.clone())).collect(), + /// Remove all leaves displaced by the last block finalization. + pub fn remove_displaced_leaves(&mut self, displaced_leaves: &FinalizationOutcome) { + for (number, hashes) in &displaced_leaves.removed { + for hash in hashes.iter() { + self.remove_leaf(number, hash); + } } } @@ -420,32 +403,6 @@ mod tests { assert!(set.contains(11, 11_2)); } - #[test] - fn finalization_works() { - let mut set = LeafSet::new(); - set.import(9_1u32, 9u32, 0u32); - set.import(10_1, 10, 9_1); - set.import(10_2, 10, 9_1); - set.import(11_1, 11, 10_1); - set.import(11_2, 11, 10_1); - set.import(12_1, 12, 11_2); - - let outcome = set.finalize_height(11); - assert_eq!(set.count(), 2); - assert!(set.contains(11, 11_1)); - assert!(set.contains(12, 12_1)); - assert_eq!( - outcome.removed, - [(Reverse(10), vec![10_2])].into_iter().collect::>(), - ); - - set.undo().undo_finalization(outcome); - assert_eq!(set.count(), 3); - assert!(set.contains(11, 11_1)); - assert!(set.contains(12, 12_1)); - assert!(set.contains(10, 10_2)); - } - #[test] fn flush_to_disk() { const PREFIX: &[u8] = b"abcdefg"; @@ -479,35 +436,4 @@ mod tests { assert!(set.contains(10, 1_2)); assert!(!set.contains(10, 1_3)); } - - #[test] - fn finalization_consistent_with_disk() { - const PREFIX: &[u8] = b"prefix"; - let db = Arc::new(sp_database::MemDb::default()); - - let mut set = LeafSet::new(); - set.import(10_1u32, 10u32, 0u32); - set.import(11_1, 11, 10_2); - set.import(11_2, 11, 10_2); - set.import(12_1, 12, 11_123); - - assert!(set.contains(10, 10_1)); - - let mut tx = Transaction::new(); - set.prepare_transaction(&mut tx, 0, PREFIX); - db.commit(tx).unwrap(); - - let _ = set.finalize_height(11); - let mut tx = Transaction::new(); - set.prepare_transaction(&mut tx, 0, PREFIX); - db.commit(tx).unwrap(); - - assert!(set.contains(11, 11_1)); - assert!(set.contains(11, 11_2)); - assert!(set.contains(12, 12_1)); - assert!(!set.contains(10, 10_1)); - - let set2 = LeafSet::read_from_db(&*db, 0, PREFIX).unwrap(); - assert_eq!(set, set2); - } } diff --git a/substrate/client/consensus/babe/src/lib.rs b/substrate/client/consensus/babe/src/lib.rs index d10bdd8c7e4c..a1ca6fd23b59 100644 --- a/substrate/client/consensus/babe/src/lib.rs +++ b/substrate/client/consensus/babe/src/lib.rs @@ -562,9 +562,10 @@ fn aux_storage_cleanup + HeaderBackend, Block: B // Cleans data for stale forks. let stale_forks = match client.expand_forks(¬ification.stale_heads) { Ok(stale_forks) => stale_forks, - Err((stale_forks, e)) => { + Err(e) => { warn!(target: LOG_TARGET, "{:?}", e); - stale_forks + + Default::default() }, }; hashes.extend(stale_forks.iter()); diff --git a/substrate/client/consensus/babe/src/tests.rs b/substrate/client/consensus/babe/src/tests.rs index 38c9e1ff6ac2..716067ae4000 100644 --- a/substrate/client/consensus/babe/src/tests.rs +++ b/substrate/client/consensus/babe/src/tests.rs @@ -1094,8 +1094,8 @@ async fn obsolete_blocks_aux_data_cleanup() { assert!(aux_data_check(&fork1_hashes[2..3], false)); // Present: A4 assert!(aux_data_check(&fork1_hashes[3..], true)); - // Present C4, C5 - assert!(aux_data_check(&fork3_hashes, true)); + // Wiped C4, C5 + assert!(aux_data_check(&fork3_hashes, false)); } #[tokio::test] diff --git a/substrate/client/db/src/lib.rs b/substrate/client/db/src/lib.rs index 0faa90dfc4f9..36f9aea817c9 100644 --- a/substrate/client/db/src/lib.rs +++ b/substrate/client/db/src/lib.rs @@ -68,8 +68,8 @@ use sc_client_api::{ use sc_state_db::{IsPruned, LastCanonicalized, StateDb}; use sp_arithmetic::traits::Saturating; use sp_blockchain::{ - Backend as _, CachedHeaderMetadata, Error as ClientError, HeaderBackend, HeaderMetadata, - HeaderMetadataCache, Result as ClientResult, + Backend as _, CachedHeaderMetadata, DisplacedLeavesAfterFinalization, Error as ClientError, + HeaderBackend, HeaderMetadata, HeaderMetadataCache, Result as ClientResult, }; use sp_core::{ offchain::OffchainOverlayedChange, @@ -747,19 +747,6 @@ impl sc_client_api::blockchain::Backend for BlockchainDb, - ) -> ClientResult> { - Ok(self - .leaves - .read() - .displaced_by_finalize_height(block_number) - .leaves() - .cloned() - .collect::>()) - } - fn children(&self, parent_hash: Block::Hash) -> ClientResult> { children::read_children(&*self.db, columns::META, meta_keys::CHILDREN_PREFIX, parent_hash) } @@ -1813,14 +1800,13 @@ impl Backend { apply_state_commit(transaction, commit); } - let new_displaced = self.blockchain.leaves.write().finalize_height(f_num); - self.prune_blocks( - transaction, - f_num, - f_hash, - &new_displaced, - current_transaction_justifications, - )?; + let new_displaced = self.blockchain.displaced_leaves_after_finalizing(f_hash, f_num)?; + let finalization_outcome = + FinalizationOutcome::new(new_displaced.displaced_leaves.clone().into_iter()); + + self.blockchain.leaves.write().remove_displaced_leaves(&finalization_outcome); + + self.prune_blocks(transaction, f_num, &new_displaced, current_transaction_justifications)?; Ok(()) } @@ -1829,8 +1815,7 @@ impl Backend { &self, transaction: &mut Transaction, finalized_number: NumberFor, - finalized_hash: Block::Hash, - displaced: &FinalizationOutcome>, + displaced: &DisplacedLeavesAfterFinalization, current_transaction_justifications: &mut HashMap, ) -> ClientResult<()> { match self.blocks_pruning { @@ -1858,10 +1843,10 @@ impl Backend { self.prune_block(transaction, BlockId::::number(number))?; } - self.prune_displaced_branches(transaction, finalized_hash, displaced)?; + self.prune_displaced_branches(transaction, displaced)?; }, BlocksPruning::KeepFinalized => { - self.prune_displaced_branches(transaction, finalized_hash, displaced)?; + self.prune_displaced_branches(transaction, displaced)?; }, } Ok(()) @@ -1870,21 +1855,13 @@ impl Backend { fn prune_displaced_branches( &self, transaction: &mut Transaction, - finalized: Block::Hash, - displaced: &FinalizationOutcome>, + displaced: &DisplacedLeavesAfterFinalization, ) -> ClientResult<()> { // Discard all blocks from displaced branches - for h in displaced.leaves() { - match sp_blockchain::tree_route(&self.blockchain, *h, finalized) { - Ok(tree_route) => - for r in tree_route.retracted() { - self.blockchain.insert_persisted_body_if_pinned(r.hash)?; - self.prune_block(transaction, BlockId::::hash(r.hash))?; - }, - Err(sp_blockchain::Error::UnknownBlock(_)) => { - // Sometimes routes can't be calculated. E.g. after warp sync. - }, - Err(e) => Err(e)?, + for (_, tree_route) in displaced.tree_routes.iter() { + for r in tree_route.retracted() { + self.blockchain.insert_persisted_body_if_pinned(r.hash)?; + self.prune_block(transaction, BlockId::::hash(r.hash))?; } } Ok(()) @@ -3190,6 +3167,9 @@ pub(crate) mod tests { #[test] fn test_leaves_pruned_on_finality() { + // / 1b - 2b - 3b + // 0 - 1a - 2a + // \ 1c let backend: Backend = Backend::new_test(10, 10); let block0 = insert_header(&backend, 0, Default::default(), None, Default::default()); @@ -3201,18 +3181,16 @@ pub(crate) mod tests { let block2_a = insert_header(&backend, 2, block1_a, None, Default::default()); let block2_b = insert_header(&backend, 2, block1_b, None, Default::default()); - let block2_c = insert_header(&backend, 2, block1_b, None, [1; 32].into()); - assert_eq!( - backend.blockchain().leaves().unwrap(), - vec![block2_a, block2_b, block2_c, block1_c] - ); + let block3_b = insert_header(&backend, 3, block2_b, None, [3; 32].into()); + + assert_eq!(backend.blockchain().leaves().unwrap(), vec![block3_b, block2_a, block1_c]); backend.finalize_block(block1_a, None).unwrap(); backend.finalize_block(block2_a, None).unwrap(); - // leaves at same height stay. Leaves at lower heights pruned. - assert_eq!(backend.blockchain().leaves().unwrap(), vec![block2_a, block2_b, block2_c]); + // All leaves are pruned that are known to not belong to canonical branch + assert_eq!(backend.blockchain().leaves().unwrap(), vec![block2_a]); } #[test] diff --git a/substrate/client/merkle-mountain-range/src/offchain_mmr.rs b/substrate/client/merkle-mountain-range/src/offchain_mmr.rs index 3c3f0beb6c6a..94593f9c2c7b 100644 --- a/substrate/client/merkle-mountain-range/src/offchain_mmr.rs +++ b/substrate/client/merkle-mountain-range/src/offchain_mmr.rs @@ -33,7 +33,7 @@ use sp_runtime::{ traits::{Block, Header, NumberFor, One}, Saturating, }; -use std::{collections::VecDeque, sync::Arc}; +use std::{collections::VecDeque, default::Default, sync::Arc}; /// `OffchainMMR` exposes MMR offchain canonicalization and pruning logic. pub struct OffchainMmr, C> { @@ -273,12 +273,11 @@ where self.write_gadget_state_or_log(); // Remove offchain MMR nodes for stale forks. - let stale_forks = self.client.expand_forks(¬ification.stale_heads).unwrap_or_else( - |(stale_forks, e)| { - warn!(target: LOG_TARGET, "{:?}", e); - stale_forks - }, - ); + let stale_forks = self.client.expand_forks(¬ification.stale_heads).unwrap_or_else(|e| { + warn!(target: LOG_TARGET, "{:?}", e); + + Default::default() + }); for hash in stale_forks.iter() { self.prune_branch(hash); } diff --git a/substrate/client/rpc-spec-v2/src/chain_head/tests.rs b/substrate/client/rpc-spec-v2/src/chain_head/tests.rs index 363d11235dda..b195e05b6649 100644 --- a/substrate/client/rpc-spec-v2/src/chain_head/tests.rs +++ b/substrate/client/rpc-spec-v2/src/chain_head/tests.rs @@ -2487,6 +2487,7 @@ async fn follow_report_multiple_pruned_block() { client.finalize_block(block_3_hash, None).unwrap(); // Finalizing block 3 directly will also result in block 1 and 2 being finalized. + // It will also mark block 2 and block 3 from the fork as pruned. let event: FollowEvent = get_next_event(&mut sub).await; let expected = FollowEvent::Finalized(Finalized { finalized_block_hashes: vec![ @@ -2494,7 +2495,7 @@ async fn follow_report_multiple_pruned_block() { format!("{:?}", block_2_hash), format!("{:?}", block_3_hash), ], - pruned_block_hashes: vec![], + pruned_block_hashes: vec![format!("{:?}", block_2_f_hash), format!("{:?}", block_3_f_hash)], }); assert_eq!(event, expected); @@ -2504,7 +2505,6 @@ async fn follow_report_multiple_pruned_block() { // ^^^ finalized // -> block 1 -> block 2_f -> block 3_f // - // Mark block 4 as finalized to force block 2_f and 3_f to get pruned. let block_4 = BlockBuilderBuilder::new(&*client) .on_parent_block(block_3.hash()) @@ -2535,11 +2535,11 @@ async fn follow_report_multiple_pruned_block() { }); assert_eq!(event, expected); - // Block 4 and 5 be reported as pruned, not just the stale head (block 5). + // Blocks from the fork were pruned earlier. let event: FollowEvent = get_next_event(&mut sub).await; let expected = FollowEvent::Finalized(Finalized { finalized_block_hashes: vec![format!("{:?}", block_4_hash)], - pruned_block_hashes: vec![format!("{:?}", block_2_f_hash), format!("{:?}", block_3_f_hash)], + pruned_block_hashes: vec![], }); assert_eq!(event, expected); } @@ -3714,16 +3714,8 @@ async fn follow_unique_pruned_blocks() { // The chainHead will see block 5 as the best block. However, the // client will finalize the block 6, which is on another fork. // - // When the block 6 is finalized, block 2 block 3 block 4 and block 5 are placed on an invalid - // fork. However, pruning of blocks happens on level N - 1. - // Therefore, no pruned blocks are reported yet. + // When the block 6 is finalized all blocks from the stale forks (2, 3, 4, 5) are pruned. // - // When the block 7 is finalized, block 3 is detected as stale. At this step, block 2 and 3 - // are reported as pruned. - // - // When the block 8 is finalized, block 5 block 4 and block 2 are detected as stale. However, - // only blocks 5 and 4 are reported as pruned. This is because the block 2 was previously - // reported. // Initial setup steps: let block_1_hash = @@ -3776,16 +3768,33 @@ async fn follow_unique_pruned_blocks() { }); assert_eq!(event, expected); - // Block 2 must be reported as pruned, even if it was the previous best. - let event: FollowEvent = get_next_event(&mut sub).await; + // All blocks from stale forks are pruned when we finalize block 6. + let mut event: FollowEvent = get_next_event(&mut sub).await; + + // Sort pruned block hashes to counter flaky test caused by event generation (get_pruned_hashes) + if let FollowEvent::Finalized(Finalized { pruned_block_hashes, .. }) = &mut event { + pruned_block_hashes.sort(); + } + let expected_pruned_block_hashes = { + let mut hashes = vec![ + format!("{:?}", block_2_hash), + format!("{:?}", block_3_hash), + format!("{:?}", block_4_hash), + format!("{:?}", block_5_hash), + ]; + hashes.sort(); + hashes + }; + let expected = FollowEvent::Finalized(Finalized { finalized_block_hashes: vec![ format!("{:?}", block_1_hash), format!("{:?}", block_2_f_hash), format!("{:?}", block_6_hash), ], - pruned_block_hashes: vec![], + pruned_block_hashes: expected_pruned_block_hashes, }); + assert_eq!(event, expected); // Pruned hash can be unpinned. @@ -3802,9 +3811,10 @@ async fn follow_unique_pruned_blocks() { client.finalize_block(block_7_hash, None).unwrap(); let event: FollowEvent = get_next_event(&mut sub).await; + // All necessary blocks were pruned on block 6 finalization. let expected = FollowEvent::Finalized(Finalized { finalized_block_hashes: vec![format!("{:?}", block_7_hash)], - pruned_block_hashes: vec![format!("{:?}", block_2_hash), format!("{:?}", block_3_hash)], + pruned_block_hashes: vec![], }); assert_eq!(event, expected); @@ -3815,10 +3825,11 @@ async fn follow_unique_pruned_blocks() { // Finalize the block 8. client.finalize_block(block_8_hash, None).unwrap(); + // All necessary blocks were pruned on block 6 finalization. let event: FollowEvent = get_next_event(&mut sub).await; let expected = FollowEvent::Finalized(Finalized { finalized_block_hashes: vec![format!("{:?}", block_8_hash)], - pruned_block_hashes: vec![format!("{:?}", block_4_hash), format!("{:?}", block_5_hash)], + pruned_block_hashes: vec![], }); assert_eq!(event, expected); } diff --git a/substrate/client/service/src/client/client.rs b/substrate/client/service/src/client/client.rs index 35e8b53a09cf..3c25c233775b 100644 --- a/substrate/client/service/src/client/client.rs +++ b/substrate/client/service/src/client/client.rs @@ -978,8 +978,12 @@ where // The stale heads are the leaves that will be displaced after the // block is finalized. - let stale_heads = - self.backend.blockchain().displaced_leaves_after_finalizing(block_number)?; + let stale_heads = self + .backend + .blockchain() + .displaced_leaves_after_finalizing(hash, block_number)? + .hashes() + .collect(); let header = self .backend diff --git a/substrate/client/service/test/src/client/mod.rs b/substrate/client/service/test/src/client/mod.rs index 4fcb7c160cb4..6542830c998b 100644 --- a/substrate/client/service/test/src/client/mod.rs +++ b/substrate/client/service/test/src/client/mod.rs @@ -1164,7 +1164,7 @@ fn finalizing_diverged_block_should_trigger_reorg() { // G -> A1 -> A2 // \ - // -> B1 -> B2 + // -> B1 -> B2 -> B3 let mut finality_notifications = client.finality_notification_stream(); @@ -1249,8 +1249,8 @@ fn finalizing_diverged_block_should_trigger_reorg() { ClientExt::finalize_block(&client, b3.hash(), None).unwrap(); - finality_notification_check(&mut finality_notifications, &[b1.hash()], &[]); - finality_notification_check(&mut finality_notifications, &[b2.hash(), b3.hash()], &[a2.hash()]); + finality_notification_check(&mut finality_notifications, &[b1.hash()], &[a2.hash()]); + finality_notification_check(&mut finality_notifications, &[b2.hash(), b3.hash()], &[]); assert!(matches!(finality_notifications.try_recv().unwrap_err(), TryRecvError::Empty)); } @@ -1371,8 +1371,12 @@ fn finality_notifications_content() { // Import and finalize D4 block_on(client.import_as_final(BlockOrigin::Own, d4.clone())).unwrap(); - finality_notification_check(&mut finality_notifications, &[a1.hash(), a2.hash()], &[c1.hash()]); - finality_notification_check(&mut finality_notifications, &[d3.hash(), d4.hash()], &[b2.hash()]); + finality_notification_check( + &mut finality_notifications, + &[a1.hash(), a2.hash()], + &[c1.hash(), b2.hash()], + ); + finality_notification_check(&mut finality_notifications, &[d3.hash(), d4.hash()], &[a3.hash()]); assert!(matches!(finality_notifications.try_recv().unwrap_err(), TryRecvError::Empty)); } @@ -1601,9 +1605,9 @@ fn doesnt_import_blocks_that_revert_finality() { block_on(client.import(BlockOrigin::Own, a3.clone())).unwrap(); ClientExt::finalize_block(&client, a3.hash(), None).unwrap(); - finality_notification_check(&mut finality_notifications, &[a1.hash(), a2.hash()], &[]); + finality_notification_check(&mut finality_notifications, &[a1.hash(), a2.hash()], &[b2.hash()]); - finality_notification_check(&mut finality_notifications, &[a3.hash()], &[b2.hash()]); + finality_notification_check(&mut finality_notifications, &[a3.hash()], &[]); assert!(matches!(finality_notifications.try_recv().unwrap_err(), TryRecvError::Empty)); } diff --git a/substrate/primitives/blockchain/src/backend.rs b/substrate/primitives/blockchain/src/backend.rs index 7a09865f858d..06e5b682964a 100644 --- a/substrate/primitives/blockchain/src/backend.rs +++ b/substrate/primitives/blockchain/src/backend.rs @@ -21,14 +21,17 @@ use log::warn; use parking_lot::RwLock; use sp_runtime::{ generic::BlockId, - traits::{Block as BlockT, Header as HeaderT, NumberFor, Saturating}, + traits::{Block as BlockT, Header as HeaderT, NumberFor, Zero}, Justifications, }; -use std::collections::btree_set::BTreeSet; +use std::collections::{btree_map::BTreeMap, btree_set::BTreeSet}; use crate::header_metadata::HeaderMetadata; -use crate::error::{Error, Result}; +use crate::{ + error::{Error, Result}, + tree_route, TreeRoute, +}; /// Blockchain database header backend. Does not perform any validation. pub trait HeaderBackend: Send + Sync { @@ -89,62 +92,32 @@ pub trait HeaderBackend: Send + Sync { pub trait ForkBackend: HeaderMetadata + HeaderBackend + Send + Sync { - /// Best effort to get all the header hashes that are part of the provided forks - /// starting only from the fork heads. + /// Returns block hashes for provided fork heads. It skips the fork if when blocks are missing + /// (e.g. warp-sync) and internal `tree_route` function fails. /// - /// The function tries to reconstruct the route from the fork head to the canonical chain. - /// If any of the hashes on the route can't be found in the db, the function won't be able - /// to reconstruct the route anymore. In this case it will give up expanding the current fork, - /// move on to the next ones and at the end it will return an error that also contains - /// the partially expanded forks. + /// Example: + /// G --- A1 --- A2 --- A3 --- A4 ( < fork1 ) + /// \-----C4 --- C5 ( < fork2 ) + /// We finalize A3 and call expand_fork(C5). Result = (C5,C4). fn expand_forks( &self, fork_heads: &[Block::Hash], - ) -> std::result::Result, (BTreeSet, Error)> { - let mut missing_blocks = vec![]; + ) -> std::result::Result, Error> { let mut expanded_forks = BTreeSet::new(); for fork_head in fork_heads { - let mut route_head = *fork_head; - // Insert stale blocks hashes until canonical chain is reached. - // If we reach a block that is already part of the `expanded_forks` we can stop - // processing the fork. - while expanded_forks.insert(route_head) { - match self.header_metadata(route_head) { - Ok(meta) => { - // If the parent is part of the canonical chain or there doesn't exist a - // block hash for the parent number (bug?!), we can abort adding blocks. - let parent_number = meta.number.saturating_sub(1u32.into()); - match self.hash(parent_number) { - Ok(Some(parent_hash)) => - if parent_hash == meta.parent { - break - }, - Ok(None) | Err(_) => { - missing_blocks.push(BlockId::::Number(parent_number)); - break - }, - } - - route_head = meta.parent; - }, - Err(_e) => { - missing_blocks.push(BlockId::::Hash(route_head)); - break - }, - } + match tree_route(self, *fork_head, self.info().finalized_hash) { + Ok(tree_route) => { + for block in tree_route.retracted() { + expanded_forks.insert(block.hash); + } + continue + }, + Err(_) => { + // There are cases when blocks are missing (e.g. warp-sync). + }, } } - if !missing_blocks.is_empty() { - return Err(( - expanded_forks, - Error::UnknownBlocks(format!( - "Missing stale headers {:?} while expanding forks {:?}.", - fork_heads, missing_blocks - )), - )) - } - Ok(expanded_forks) } } @@ -172,14 +145,6 @@ pub trait Backend: /// Results must be ordered best (longest, highest) chain first. fn leaves(&self) -> Result>; - /// Returns displaced leaves after the given block would be finalized. - /// - /// The returned leaves do not contain the leaves from the same height as `block_number`. - fn displaced_leaves_after_finalizing( - &self, - block_number: NumberFor, - ) -> Result>; - /// Return hashes of all blocks that are children of the block with `parent_hash`. fn children(&self, parent_hash: Block::Hash) -> Result>; @@ -255,6 +220,67 @@ pub trait Backend: } fn block_indexed_body(&self, hash: Block::Hash) -> Result>>>; + + /// Returns all leaves that will be displaced after the block finalization. + fn displaced_leaves_after_finalizing( + &self, + finalized_block_hash: Block::Hash, + finalized_block_number: NumberFor, + ) -> std::result::Result, Error> { + let mut result = DisplacedLeavesAfterFinalization::default(); + + if finalized_block_number == Zero::zero() { + return Ok(result) + } + + // For each leaf determine whether it belongs to a non-canonical branch. + for leaf_hash in self.leaves()? { + let leaf_block_header = self.expect_header(leaf_hash)?; + let leaf_number = *leaf_block_header.number(); + + let leaf_tree_route = match tree_route(self, leaf_hash, finalized_block_hash) { + Ok(tree_route) => tree_route, + Err(Error::UnknownBlock(_)) => { + // Sometimes routes can't be calculated. E.g. after warp sync. + continue; + }, + Err(e) => Err(e)?, + }; + + // Is it a stale fork? + let needs_pruning = leaf_tree_route.common_block().hash != finalized_block_hash; + + if needs_pruning { + result.displaced_leaves.insert(leaf_hash, leaf_number); + result.tree_routes.insert(leaf_hash, leaf_tree_route); + } + } + + Ok(result) + } +} + +/// Result of [`Backend::displaced_leaves_after_finalizing`]. +#[derive(Clone, Debug)] +pub struct DisplacedLeavesAfterFinalization { + /// A collection of hashes and block numbers for displaced leaves. + pub displaced_leaves: BTreeMap>, + + /// A collection of tree routes from the leaves to finalized block. + pub tree_routes: BTreeMap>, +} + +impl Default for DisplacedLeavesAfterFinalization { + fn default() -> Self { + Self { displaced_leaves: Default::default(), tree_routes: Default::default() } + } +} + +impl DisplacedLeavesAfterFinalization { + /// Returns a collection of hashes for the displaced leaves. + pub fn hashes(&self) -> impl Iterator + '_ { + self.displaced_leaves.keys().cloned() + } } /// Blockchain info diff --git a/substrate/primitives/blockchain/src/header_metadata.rs b/substrate/primitives/blockchain/src/header_metadata.rs index ccd640c0567a..27caaae71add 100644 --- a/substrate/primitives/blockchain/src/header_metadata.rs +++ b/substrate/primitives/blockchain/src/header_metadata.rs @@ -97,7 +97,7 @@ pub fn lowest_common_ancestor + ?Sized>( } /// Compute a tree-route between two blocks. See tree-route docs for more details. -pub fn tree_route>( +pub fn tree_route + ?Sized>( backend: &T, from: Block::Hash, to: Block::Hash, From 494f12319e538f60260bcb139c8900e2eaf5c329 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe <49718502+alexggh@users.noreply.github.com> Date: Wed, 15 May 2024 11:53:53 +0200 Subject: [PATCH 4/6] Bump fatality crate to 0.1.1 (#4464) ... to get rid of the macro debug logs like this from build output ``` [/home/alexggh/.cargo/registry/src/index.crates.io-6f17d22bba15001f/fatality-proc-macro-0.1.0/src/types.rs:171:23] input = TokenStream [ Literal { kind: Str, symbol: "Error while accessing runtime information", suffix: None, span: #0 bytes(6943..6986), }, ] ``` Signed-off-by: Alexandru Gheorghe --- Cargo.lock | 10 ++++------ polkadot/node/core/backing/Cargo.toml | 2 +- polkadot/node/core/dispute-coordinator/Cargo.toml | 2 +- polkadot/node/core/prospective-parachains/Cargo.toml | 2 +- polkadot/node/core/provisioner/Cargo.toml | 2 +- .../node/network/availability-distribution/Cargo.toml | 2 +- polkadot/node/network/availability-recovery/Cargo.toml | 2 +- polkadot/node/network/bridge/Cargo.toml | 2 +- polkadot/node/network/collator-protocol/Cargo.toml | 2 +- polkadot/node/network/dispute-distribution/Cargo.toml | 2 +- polkadot/node/network/protocol/Cargo.toml | 2 +- .../node/network/statement-distribution/Cargo.toml | 2 +- polkadot/node/subsystem-util/Cargo.toml | 2 +- 13 files changed, 16 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2b2576d1adfa..7ff2da4f5472 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5362,20 +5362,19 @@ dependencies = [ [[package]] name = "fatality" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61ecdc33d04db74fc23db9f54f6f314c61d29f810d58ba423d0c204888365458" +checksum = "ec6f82451ff7f0568c6181287189126d492b5654e30a788add08027b6363d019" dependencies = [ "fatality-proc-macro", - "syn 2.0.61", "thiserror", ] [[package]] name = "fatality-proc-macro" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac67b350787fd4a934752e30ddb45569da000e14bf3e499224302778b7a918ab" +checksum = "eb42427514b063d97ce21d5199f36c0c307d981434a6be32582bc79fe5bd2303" dependencies = [ "expander", "indexmap 2.2.3", @@ -5383,7 +5382,6 @@ dependencies = [ "proc-macro2 1.0.82", "quote 1.0.35", "syn 2.0.61", - "thiserror", ] [[package]] diff --git a/polkadot/node/core/backing/Cargo.toml b/polkadot/node/core/backing/Cargo.toml index 9829f1b37cfd..f426f73284e8 100644 --- a/polkadot/node/core/backing/Cargo.toml +++ b/polkadot/node/core/backing/Cargo.toml @@ -21,7 +21,7 @@ statement-table = { package = "polkadot-statement-table", path = "../../../state bitvec = { version = "1.0.0", default-features = false, features = ["alloc"] } gum = { package = "tracing-gum", path = "../../gum" } thiserror = { workspace = true } -fatality = "0.1.0" +fatality = "0.1.1" schnellru = "0.2.1" [dev-dependencies] diff --git a/polkadot/node/core/dispute-coordinator/Cargo.toml b/polkadot/node/core/dispute-coordinator/Cargo.toml index 938fdce1cb82..dba3bcdd6430 100644 --- a/polkadot/node/core/dispute-coordinator/Cargo.toml +++ b/polkadot/node/core/dispute-coordinator/Cargo.toml @@ -16,7 +16,7 @@ parity-scale-codec = "3.6.1" kvdb = "0.13.0" thiserror = { workspace = true } schnellru = "0.2.1" -fatality = "0.1.0" +fatality = "0.1.1" polkadot-primitives = { path = "../../../primitives" } polkadot-node-primitives = { path = "../../primitives" } diff --git a/polkadot/node/core/prospective-parachains/Cargo.toml b/polkadot/node/core/prospective-parachains/Cargo.toml index 8d0aec96f010..d38a23c3fdad 100644 --- a/polkadot/node/core/prospective-parachains/Cargo.toml +++ b/polkadot/node/core/prospective-parachains/Cargo.toml @@ -14,7 +14,7 @@ futures = "0.3.30" gum = { package = "tracing-gum", path = "../../gum" } parity-scale-codec = "3.6.4" thiserror = { workspace = true } -fatality = "0.1.0" +fatality = "0.1.1" bitvec = "1" polkadot-primitives = { path = "../../../primitives" } diff --git a/polkadot/node/core/provisioner/Cargo.toml b/polkadot/node/core/provisioner/Cargo.toml index 1cd16e6599a7..d19783212644 100644 --- a/polkadot/node/core/provisioner/Cargo.toml +++ b/polkadot/node/core/provisioner/Cargo.toml @@ -19,7 +19,7 @@ polkadot-node-primitives = { path = "../../primitives" } polkadot-node-subsystem = { path = "../../subsystem" } polkadot-node-subsystem-util = { path = "../../subsystem-util" } futures-timer = "3.0.2" -fatality = "0.1.0" +fatality = "0.1.1" schnellru = "0.2.1" [dev-dependencies] diff --git a/polkadot/node/network/availability-distribution/Cargo.toml b/polkadot/node/network/availability-distribution/Cargo.toml index 344389d224f5..ff3529449082 100644 --- a/polkadot/node/network/availability-distribution/Cargo.toml +++ b/polkadot/node/network/availability-distribution/Cargo.toml @@ -25,7 +25,7 @@ thiserror = { workspace = true } rand = "0.8.5" derive_more = "0.99.17" schnellru = "0.2.1" -fatality = "0.1.0" +fatality = "0.1.1" [dev-dependencies] polkadot-node-subsystem-test-helpers = { path = "../../subsystem-test-helpers" } diff --git a/polkadot/node/network/availability-recovery/Cargo.toml b/polkadot/node/network/availability-recovery/Cargo.toml index b7f817d85235..d12c1b1cff95 100644 --- a/polkadot/node/network/availability-recovery/Cargo.toml +++ b/polkadot/node/network/availability-recovery/Cargo.toml @@ -14,7 +14,7 @@ futures = "0.3.30" tokio = "1.37" schnellru = "0.2.1" rand = "0.8.5" -fatality = "0.1.0" +fatality = "0.1.1" thiserror = { workspace = true } async-trait = "0.1.79" gum = { package = "tracing-gum", path = "../../gum" } diff --git a/polkadot/node/network/bridge/Cargo.toml b/polkadot/node/network/bridge/Cargo.toml index 8ab571fbe7c3..4bb49baba921 100644 --- a/polkadot/node/network/bridge/Cargo.toml +++ b/polkadot/node/network/bridge/Cargo.toml @@ -24,7 +24,7 @@ polkadot-node-subsystem = { path = "../../subsystem" } polkadot-overseer = { path = "../../overseer" } parking_lot = "0.12.1" bytes = "1" -fatality = "0.1.0" +fatality = "0.1.1" thiserror = { workspace = true } [dev-dependencies] diff --git a/polkadot/node/network/collator-protocol/Cargo.toml b/polkadot/node/network/collator-protocol/Cargo.toml index 201615ac9ffb..aa60c0166d20 100644 --- a/polkadot/node/network/collator-protocol/Cargo.toml +++ b/polkadot/node/network/collator-protocol/Cargo.toml @@ -24,7 +24,7 @@ polkadot-node-network-protocol = { path = "../protocol" } polkadot-node-primitives = { path = "../../primitives" } polkadot-node-subsystem-util = { path = "../../subsystem-util" } polkadot-node-subsystem = { path = "../../subsystem" } -fatality = "0.1.0" +fatality = "0.1.1" thiserror = { workspace = true } tokio-util = "0.7.1" diff --git a/polkadot/node/network/dispute-distribution/Cargo.toml b/polkadot/node/network/dispute-distribution/Cargo.toml index 7d9a719d8525..eb8a7606304e 100644 --- a/polkadot/node/network/dispute-distribution/Cargo.toml +++ b/polkadot/node/network/dispute-distribution/Cargo.toml @@ -25,7 +25,7 @@ sc-network = { path = "../../../../substrate/client/network" } sp-application-crypto = { path = "../../../../substrate/primitives/application-crypto" } sp-keystore = { path = "../../../../substrate/primitives/keystore" } thiserror = { workspace = true } -fatality = "0.1.0" +fatality = "0.1.1" schnellru = "0.2.1" indexmap = "2.0.0" diff --git a/polkadot/node/network/protocol/Cargo.toml b/polkadot/node/network/protocol/Cargo.toml index 29975fc735f2..2b741051b4fe 100644 --- a/polkadot/node/network/protocol/Cargo.toml +++ b/polkadot/node/network/protocol/Cargo.toml @@ -24,7 +24,7 @@ sp-runtime = { path = "../../../../substrate/primitives/runtime" } strum = { version = "0.26.2", features = ["derive"] } futures = "0.3.30" thiserror = { workspace = true } -fatality = "0.1.0" +fatality = "0.1.1" rand = "0.8" derive_more = "0.99" gum = { package = "tracing-gum", path = "../../gum" } diff --git a/polkadot/node/network/statement-distribution/Cargo.toml b/polkadot/node/network/statement-distribution/Cargo.toml index b5cfeaa0c1ed..da8c91a0a291 100644 --- a/polkadot/node/network/statement-distribution/Cargo.toml +++ b/polkadot/node/network/statement-distribution/Cargo.toml @@ -24,7 +24,7 @@ arrayvec = "0.7.4" indexmap = "2.0.0" parity-scale-codec = { version = "3.6.1", default-features = false, features = ["derive"] } thiserror = { workspace = true } -fatality = "0.1.0" +fatality = "0.1.1" bitvec = "1" [dev-dependencies] diff --git a/polkadot/node/subsystem-util/Cargo.toml b/polkadot/node/subsystem-util/Cargo.toml index b6cc368b9245..492a9847c96b 100644 --- a/polkadot/node/subsystem-util/Cargo.toml +++ b/polkadot/node/subsystem-util/Cargo.toml @@ -19,7 +19,7 @@ parking_lot = "0.12.1" pin-project = "1.0.9" rand = "0.8.5" thiserror = { workspace = true } -fatality = "0.1.0" +fatality = "0.1.1" gum = { package = "tracing-gum", path = "../gum" } derive_more = "0.99.17" schnellru = "0.2.1" From 005e3a77150d6bed78114763791450f54f1ba245 Mon Sep 17 00:00:00 2001 From: RadiumBlock Date: Wed, 15 May 2024 05:11:37 -0500 Subject: [PATCH 5/6] Please consider adding RadiumBlock bootnodes for Westend People and Westend Coretime (#3936) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Please add RadiumBlock bootnodes to the spec files for Westend Coretime and Westend People --------- Co-authored-by: Dónal Murray --- cumulus/parachains/chain-specs/coretime-westend.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cumulus/parachains/chain-specs/coretime-westend.json b/cumulus/parachains/chain-specs/coretime-westend.json index ab2e97fdbf41..74edd5b2cd96 100644 --- a/cumulus/parachains/chain-specs/coretime-westend.json +++ b/cumulus/parachains/chain-specs/coretime-westend.json @@ -5,6 +5,8 @@ "bootNodes": [ "/dns/westend-coretime-collator-node-0.parity-testnet.parity.io/tcp/30333/p2p/12D3KooWP93Dzk8T7GWxyWw9jhLcz8Pksokk3R9vL2eEH337bNkT", "/dns/westend-coretime-collator-node-1.parity-testnet.parity.io/tcp/30333/p2p/12D3KooWMh2imeAzsZKGQgm2cv6Uoep3GBYtwGfujt1bs5YfVzkH", + "/dns/coretime-westend-bootnode.radiumblock.com/tcp/30333/p2p/12D3KooWK7Zj1mCPg6h3eMp7v6akJ1o6AocRr59NLusDwBXQgrhw", + "/dns/coretime-westend-bootnode.radiumblock.com/tcp/30336/wss/p2p/12D3KooWK7Zj1mCPg6h3eMp7v6akJ1o6AocRr59NLusDwBXQgrhw", "/dns/westend-coretime-collator-node-0.parity-testnet.parity.io/tcp/30335/ws/p2p/12D3KooWP93Dzk8T7GWxyWw9jhLcz8Pksokk3R9vL2eEH337bNkT", "/dns/westend-coretime-collator-node-1.parity-testnet.parity.io/tcp/30335/ws/p2p/12D3KooWMh2imeAzsZKGQgm2cv6Uoep3GBYtwGfujt1bs5YfVzkH", "/dns/westend-coretime-collator-node-0.parity-testnet.parity.io/tcp/443/wss/p2p/12D3KooWP93Dzk8T7GWxyWw9jhLcz8Pksokk3R9vL2eEH337bNkT", From 3fcdada318bfee0b8da945b78496a8191375cf26 Mon Sep 17 00:00:00 2001 From: Javier Viola <363911+pepoviola@users.noreply.github.com> Date: Wed, 15 May 2024 12:16:46 +0200 Subject: [PATCH 6/6] chore: update zombienet version (#4463) Add small fixes, reorg debug definition and add more verbose debug logs in k8s logs reading. Thx! --- .gitlab/pipeline/zombienet.yml | 3 ++- .gitlab/pipeline/zombienet/cumulus.yml | 1 - .gitlab/pipeline/zombienet/polkadot.yml | 1 - .gitlab/pipeline/zombienet/substrate.yml | 1 - 4 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.gitlab/pipeline/zombienet.yml b/.gitlab/pipeline/zombienet.yml index 52948e1eb719..7be4ba1663e6 100644 --- a/.gitlab/pipeline/zombienet.yml +++ b/.gitlab/pipeline/zombienet.yml @@ -1,8 +1,9 @@ .zombienet-refs: extends: .build-refs variables: - ZOMBIENET_IMAGE: "docker.io/paritytech/zombienet:v1.3.99" + ZOMBIENET_IMAGE: "docker.io/paritytech/zombienet:v1.3.103" PUSHGATEWAY_URL: "http://zombienet-prometheus-pushgateway.managed-monitoring:9091/metrics/job/zombie-metrics" + DEBUG: "zombie,zombie::network-node,zombie::kube::client::logs" include: # substrate tests diff --git a/.gitlab/pipeline/zombienet/cumulus.yml b/.gitlab/pipeline/zombienet/cumulus.yml index c473f5c5fed7..a7f321505bac 100644 --- a/.gitlab/pipeline/zombienet/cumulus.yml +++ b/.gitlab/pipeline/zombienet/cumulus.yml @@ -15,7 +15,6 @@ - echo "${COL_IMAGE}" - echo "${GH_DIR}" - echo "${LOCAL_DIR}" - - export DEBUG=zombie - export RELAY_IMAGE=${POLKADOT_IMAGE} - export COL_IMAGE=${COL_IMAGE} diff --git a/.gitlab/pipeline/zombienet/polkadot.yml b/.gitlab/pipeline/zombienet/polkadot.yml index 38c5332f3097..a9f0eb930337 100644 --- a/.gitlab/pipeline/zombienet/polkadot.yml +++ b/.gitlab/pipeline/zombienet/polkadot.yml @@ -10,7 +10,6 @@ - if [[ $CI_COMMIT_REF_NAME == *"gh-readonly-queue"* ]]; then export DOCKER_IMAGES_VERSION="${CI_COMMIT_SHORT_SHA}"; fi - export PIPELINE_IMAGE_TAG=${DOCKER_IMAGES_VERSION} - export BUILD_RELEASE_VERSION="$(cat ./artifacts/BUILD_RELEASE_VERSION)" # from build-linux-stable job - - export DEBUG=zombie,zombie::network-node - export ZOMBIENET_INTEGRATION_TEST_IMAGE="${POLKADOT_IMAGE}":${PIPELINE_IMAGE_TAG} - export COL_IMAGE="${COLANDER_IMAGE}":${PIPELINE_IMAGE_TAG} - export CUMULUS_IMAGE="docker.io/paritypr/polkadot-parachain-debug:${DOCKER_IMAGES_VERSION}" diff --git a/.gitlab/pipeline/zombienet/substrate.yml b/.gitlab/pipeline/zombienet/substrate.yml index 8a627c454f9f..2013ffd571cf 100644 --- a/.gitlab/pipeline/zombienet/substrate.yml +++ b/.gitlab/pipeline/zombienet/substrate.yml @@ -13,7 +13,6 @@ - echo "${ZOMBIENET_IMAGE}" - echo "${GH_DIR}" - echo "${LOCAL_DIR}" - - export DEBUG=zombie,zombie::network-node - export ZOMBIENET_INTEGRATION_TEST_IMAGE="${SUBSTRATE_IMAGE}":${SUBSTRATE_IMAGE_TAG} - echo "${ZOMBIENET_INTEGRATION_TEST_IMAGE}" stage: zombienet