From 8c90e26ccd4d1fb6bac474130ea60b2ed27fb295 Mon Sep 17 00:00:00 2001
From: dastansam <dastan.samatov.33@gmail.com>
Date: Mon, 29 Apr 2024 23:16:57 +0200
Subject: [PATCH 1/4] Initial version

---
 Cargo.toml                                    |   3 +-
 .../frame/remote-externalities/src/lib.rs     |   2 +
 substrate/utils/frame/sub-du/Cargo.toml       |  28 ++
 substrate/utils/frame/sub-du/README.md        |   0
 substrate/utils/frame/sub-du/src/main.rs      | 271 ++++++++++++++++++
 substrate/utils/frame/sub-du/src/tests.rs     |   0
 6 files changed, 303 insertions(+), 1 deletion(-)
 create mode 100644 substrate/utils/frame/sub-du/Cargo.toml
 create mode 100644 substrate/utils/frame/sub-du/README.md
 create mode 100644 substrate/utils/frame/sub-du/src/main.rs
 create mode 100644 substrate/utils/frame/sub-du/src/tests.rs

diff --git a/Cargo.toml b/Cargo.toml
index 42a6bc8abe1e..ec13f65db109 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -509,10 +509,11 @@ members = [
 	"substrate/utils/frame/rpc/state-trie-migration-rpc",
 	"substrate/utils/frame/rpc/support",
 	"substrate/utils/frame/rpc/system",
+	"substrate/utils/frame/sub-du",
 	"substrate/utils/prometheus",
 	"substrate/utils/substrate-bip39",
 	"substrate/utils/wasm-builder",
-
+	
 	"templates/minimal/node",
 	"templates/minimal/pallets/template",
 	"templates/minimal/runtime",
diff --git a/substrate/utils/frame/remote-externalities/src/lib.rs b/substrate/utils/frame/remote-externalities/src/lib.rs
index 58cb901470c1..6405e09b895c 100644
--- a/substrate/utils/frame/remote-externalities/src/lib.rs
+++ b/substrate/utils/frame/remote-externalities/src/lib.rs
@@ -52,6 +52,8 @@ use std::{
 use substrate_rpc_client::{rpc_params, BatchRequestBuilder, ChainApi, ClientT, StateApi};
 use tokio_retry::{strategy::FixedInterval, Retry};
 
+pub use sp_core::twox_128;
+
 type KeyValue = (StorageKey, StorageData);
 type TopKeyValues = Vec<KeyValue>;
 type ChildKeyValues = Vec<(ChildInfo, Vec<KeyValue>)>;
diff --git a/substrate/utils/frame/sub-du/Cargo.toml b/substrate/utils/frame/sub-du/Cargo.toml
new file mode 100644
index 000000000000..ebacce129a88
--- /dev/null
+++ b/substrate/utils/frame/sub-du/Cargo.toml
@@ -0,0 +1,28 @@
+[package]
+name = "sub-du"
+version = "0.1.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+edition = "2021"
+
+[dependencies]
+codec = { package = "parity-scale-codec", version = "3.0.0" }
+jsonrpsee = { version = "0.15.1", features = ["jsonrpsee-types", "jsonrpsee-http-client", "jsonrpsee-ws-client"] }
+separator = "0.4.1"
+ansi_term = "0.12.1"
+tracing-subscriber = "0.3.16"
+log = "0.4.17"
+structopt = "0.3"
+tokio = { version = "1.21.2", features = ["rt-multi-thread", "macros"]}
+
+remote-externalities = { path = "../remote-externalities" }
+sp-runtime = { version = "6.0.0", path = "../../../primitives/runtime" }
+frame-metadata = { version = "15.0.0", default-features = false, features = ["v14", "std"] }
+sc-rpc-api = { version = "0.10.0-dev", path = "../../../client/rpc-api" }
+
+[features]
+default = []
+remote-test-kusama = []
+remote-test-polkadot = []
+
+[dev-dependencies]
+assert_cmd = "2"
\ No newline at end of file
diff --git a/substrate/utils/frame/sub-du/README.md b/substrate/utils/frame/sub-du/README.md
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/substrate/utils/frame/sub-du/src/main.rs b/substrate/utils/frame/sub-du/src/main.rs
new file mode 100644
index 000000000000..a67cd8a5ceb6
--- /dev/null
+++ b/substrate/utils/frame/sub-du/src/main.rs
@@ -0,0 +1,271 @@
+use ansi_term::{Colour::*, Style};
+use frame_metadata::{RuntimeMetadata, RuntimeMetadataPrefixed, StorageEntryType};
+use remote_externalities::{twox_128, Mode, OnlineConfig, StorageKey, Transport};
+use sc_rpc_api::state::StateApiClient;
+use separator::Separatable;
+use sp_runtime::testing::{Block as RawBlock, ExtrinsicWrapper, H256 as Hash};
+use structopt::StructOpt;
+
+const KB: usize = 1024;
+const MB: usize = KB * KB;
+const GB: usize = MB * MB;
+
+pub const LOG_TARGET: &'static str = "sub-du";
+
+fn get_prefix(indent: usize) -> &'static str {
+	match indent {
+		1 => "├─┬",
+		2 => "│ │─┬",
+		3 => "│ │ │─",
+		_ => panic!("Unexpected indent."),
+	}
+}
+
+struct Size(usize);
+
+impl std::fmt::Display for Size {
+	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+		if self.0 <= KB {
+			write!(f, "{: <4}B", self.0)?;
+		} else if self.0 <= MB {
+			write!(f, "{: <4}K", self.0 / KB)?;
+		} else if self.0 <= GB {
+			write!(f, "{: <4}M", self.0 / MB)?;
+		}
+
+		Ok(())
+	}
+}
+
+#[derive(Debug, Clone, Default)]
+struct Pallet {
+	pub name: String,
+	pub size: usize,
+	pub items: Vec<Storage>,
+}
+
+impl std::fmt::Display for Pallet {
+	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+		let mod_style = Style::new().bold().italic().fg(Green);
+		write!(
+			f,
+			"{} {} {}\n",
+			mod_style.paint(format!("{}", Size(self.size))),
+			get_prefix(2),
+			mod_style.paint(self.name.clone())
+		)?;
+		for s in self.items.iter() {
+			write!(f, "{} {} {}\n", Size(s.size), get_prefix(3), s)?;
+		}
+		Ok(())
+	}
+}
+
+impl Pallet {
+	fn new(name: String) -> Self {
+		Self { name, ..Default::default() }
+	}
+}
+
+#[derive(Debug, Copy, Clone)]
+pub enum StorageItem {
+	Value(usize),
+	Map(usize, usize),
+}
+
+impl std::fmt::Display for StorageItem {
+	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+		match self {
+			Self::Value(bytes) => write!(f, "Value({} bytes)", bytes.separated_string()),
+			Self::Map(bytes, count) => {
+				write!(f, "Map({} bytes, {} keys)", bytes.separated_string(), count)
+			},
+		}
+	}
+}
+
+impl Default for StorageItem {
+	fn default() -> Self {
+		Self::Value(0)
+	}
+}
+
+#[derive(Debug, Clone, Default)]
+struct Storage {
+	pub name: String,
+	pub size: usize,
+	pub item: StorageItem,
+}
+
+impl std::fmt::Display for Storage {
+	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+		let item_style = Style::new().italic();
+		write!(f, "{} => {}", item_style.paint(self.name.clone()), self.item)
+	}
+}
+
+impl Storage {
+	fn new(name: String, item: StorageItem) -> Self {
+		let size = match item {
+			StorageItem::Value(s) => s,
+			StorageItem::Map(s, _) => s,
+		};
+		Self { name, item, size }
+	}
+}
+
+#[derive(Debug, StructOpt)]
+#[structopt(
+	name = "sub-du",
+	about = "a du-like tool that prints the map of storage usage of a substrate chain"
+)]
+struct Opt {
+	/// The block number at which the scrap should happen. Use only the hex value, no need for a
+	/// `0x` prefix.
+	#[structopt(long)]
+	at: Option<Hash>,
+
+	/// The node to connect to.
+	#[structopt(long, default_value = "wss://rpc.polkadot.io:443")]
+	uri: String,
+
+	/// If true, intermediate values will be printed.
+	#[structopt(long, short)]
+	progress: bool,
+
+	/// Weather to scrape all pairs or just the size of them.
+	///
+	/// If enabled, the command might take longer but then the number of keys in each map is also
+	/// scraped.
+	///
+	/// # Warning
+	///
+	/// This uses an unsafe RPC call and can only be used if the target node allows it.
+	#[structopt(long, short)]
+	scrape_pairs: bool,
+}
+
+/// create key prefix for a module as vec bytes. Basically twox128 hash of the given values.
+pub fn pallet_prefix_raw(module: &[u8], storage: &[u8]) -> Vec<u8> {
+	let module_key = twox_128(module);
+	let storage_key = twox_128(storage);
+	let mut final_key = Vec::with_capacity(module_key.len() + storage_key.len());
+	final_key.extend_from_slice(&module_key);
+	final_key.extend_from_slice(&storage_key);
+	final_key
+}
+
+type Block = RawBlock<ExtrinsicWrapper<Hash>>;
+
+#[tokio::main]
+async fn main() -> () {
+	tracing_subscriber::fmt::try_init().unwrap();
+	let opt = Opt::from_args();
+
+	let mut cfg = OnlineConfig::default();
+	cfg.transport = Transport::Uri(opt.uri);
+	cfg.transport.map_uri().await.unwrap();
+
+	// connect to a node.
+	let ext = remote_externalities::Builder::<Block>::new().mode(Mode::Online(cfg));
+	let mut modules: Vec<Pallet> = vec![];
+
+	// potentially replace head with the given hash
+	let head = ext.rpc_get_head().await.unwrap();
+	let at = opt.at.or(Some(head));
+
+	let rpc_client = ext.as_online().rpc_client();
+
+	let runtime = rpc_client.runtime_version(at).await.unwrap();
+
+	println!("Scraping at block {:?} of {}({})", at, runtime.spec_name, runtime.spec_version,);
+
+	let raw_metadata = rpc_client.metadata(at).await.unwrap();
+	let prefixed_metadata = <RuntimeMetadataPrefixed as codec::Decode>::decode(&mut &*raw_metadata)
+		.expect("Runtime Metadata failed to decode");
+	let metadata = prefixed_metadata.1;
+
+	if let RuntimeMetadata::V14(inner) = metadata {
+		let pallets = inner.pallets;
+		for pallet in pallets.into_iter() {
+			let name = pallet.name;
+
+			// skip, if this module has no storage items.
+			if pallet.storage.is_none() {
+				log::warn!(
+					target: LOG_TARGET,
+					"Pallet with name {:?} seems to have no storage items.",
+					name
+				);
+				continue
+			}
+
+			let storage = pallet.storage.unwrap();
+			let prefix = storage.prefix;
+			let entries = storage.entries;
+			let mut pallet_info = Pallet::new(name.clone());
+
+			for storage_entry in entries.into_iter() {
+				let storage_name = storage_entry.name;
+				let ty = storage_entry.ty;
+				let key_prefix = pallet_prefix_raw(prefix.as_bytes(), storage_name.as_bytes());
+
+				let (pairs, size) = if opt.scrape_pairs {
+					// this should be slower but gives more detail.
+					let pairs =
+						rpc_client.storage_pairs(StorageKey(key_prefix.clone()), at).await.unwrap();
+					let pairs = pairs
+						.into_iter()
+						.map(|(k, v)| (k.0, v.0))
+						.collect::<Vec<(Vec<u8>, Vec<u8>)>>();
+					let size = pairs.iter().fold(0, |acc, x| acc + x.1.len());
+					(pairs, size)
+				} else {
+					// This should be faster
+					let size = rpc_client
+						.storage_size(StorageKey(key_prefix), at)
+						.await
+						.unwrap()
+						.unwrap_or_default() as usize;
+					let pairs: Vec<_> = vec![];
+					(pairs, size)
+				};
+
+				log::debug!(
+					target: LOG_TARGET,
+					"{:?}::{:?} => count: {}, size: {} bytes",
+					name,
+					storage_name,
+					pairs.len(),
+					size
+				);
+
+				pallet_info.size += size;
+				let item = match ty {
+					StorageEntryType::Plain(_) => StorageItem::Value(size),
+					StorageEntryType::Map { .. } => StorageItem::Map(size, pairs.len()),
+				};
+				pallet_info.items.push(Storage::new(storage_name, item));
+			}
+			pallet_info.items.sort_by_key(|x| x.size);
+			pallet_info.items.reverse();
+			println!("Scraped module {}. Total size {}.", pallet_info.name, pallet_info.size,);
+			if opt.progress {
+				print!("{}", pallet_info);
+			}
+			modules.push(pallet_info);
+		}
+
+		println!("Scraping results done. Final sorted tree:");
+		modules.sort_by_key(|m| m.size);
+		modules.reverse();
+
+		let total: usize = modules.iter().map(|m| m.size).sum();
+		println!("{} {} {}", Size(total), get_prefix(1), runtime.spec_name,);
+		modules.into_iter().for_each(|m| {
+			print!("{}", m);
+		});
+	} else {
+		panic!("Unsupported Metadata version");
+	}
+}
diff --git a/substrate/utils/frame/sub-du/src/tests.rs b/substrate/utils/frame/sub-du/src/tests.rs
new file mode 100644
index 000000000000..e69de29bb2d1

From 64541abcd8bdd126842d639ec8c31310a0c3c000 Mon Sep 17 00:00:00 2001
From: dastansam <dastan.samatov.33@gmail.com>
Date: Wed, 1 May 2024 03:22:44 +0200
Subject: [PATCH 2/4] Read keys paged instead, refactor and remove unnecessary
 parts

---
 Cargo.lock                                    | 114 +++----
 .../frame/remote-externalities/src/lib.rs     |   2 -
 substrate/utils/frame/sub-du/Cargo.toml       |  16 +-
 substrate/utils/frame/sub-du/src/main.rs      | 306 ++++++++++++------
 substrate/utils/frame/sub-du/src/tests.rs     |  24 ++
 5 files changed, 294 insertions(+), 168 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 67b0ad4def24..94dd3530224c 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1246,9 +1246,9 @@ dependencies = [
 
 [[package]]
 name = "async-task"
-version = "4.4.0"
+version = "4.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ecc7ab41815b3c653ccd2978ec3255c81349336702dfdf62ee6f7069b12a3aae"
+checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de"
 
 [[package]]
 name = "async-trait"
@@ -1384,9 +1384,9 @@ checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"
 
 [[package]]
 name = "base64"
-version = "0.21.2"
+version = "0.21.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d"
+checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567"
 
 [[package]]
 name = "base64ct"
@@ -2380,9 +2380,9 @@ checksum = "17febce684fd15d89027105661fec94afb475cb995fbc59d2865198446ba2eea"
 
 [[package]]
 name = "byteorder"
-version = "1.4.3"
+version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
+checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
 
 [[package]]
 name = "bytes"
@@ -3555,22 +3555,18 @@ dependencies = [
 
 [[package]]
 name = "crossbeam-queue"
-version = "0.3.8"
+version = "0.3.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d1cfb3ea8a53f37c40dea2c7bedcbd88bdfae54f5e2175d6ecaff1c988353add"
+checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35"
 dependencies = [
- "cfg-if",
  "crossbeam-utils",
 ]
 
 [[package]]
 name = "crossbeam-utils"
-version = "0.8.16"
+version = "0.8.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294"
-dependencies = [
- "cfg-if",
-]
+checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345"
 
 [[package]]
 name = "crunchy"
@@ -5265,23 +5261,12 @@ dependencies = [
 
 [[package]]
 name = "errno"
-version = "0.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6b30f669a7961ef1631673d2766cc92f52d64f7ef354d4fe0ddfd30ed52f0f4f"
-dependencies = [
- "errno-dragonfly",
- "libc",
- "windows-sys 0.48.0",
-]
-
-[[package]]
-name = "errno-dragonfly"
-version = "0.1.2"
+version = "0.3.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf"
+checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245"
 dependencies = [
- "cc",
  "libc",
+ "windows-sys 0.52.0",
 ]
 
 [[package]]
@@ -6109,7 +6094,7 @@ version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "29f9df8a11882c4e3335eb2d18a0137c505d9ca927470b0cac9c6f0ae07d28f7"
 dependencies = [
- "rustix 0.38.21",
+ "rustix 0.38.34",
  "windows-sys 0.48.0",
 ]
 
@@ -7011,7 +6996,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b"
 dependencies = [
  "hermit-abi 0.3.2",
- "rustix 0.38.21",
+ "rustix 0.38.34",
  "windows-sys 0.48.0",
 ]
 
@@ -7550,9 +7535,9 @@ dependencies = [
 
 [[package]]
 name = "libm"
-version = "0.2.7"
+version = "0.2.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f7012b1bbb0719e1097c47611d3898568c546d597c2e74d66f6087edd5233ff4"
+checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058"
 
 [[package]]
 name = "libnghttp2-sys"
@@ -8079,9 +8064,9 @@ checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519"
 
 [[package]]
 name = "linux-raw-sys"
-version = "0.4.10"
+version = "0.4.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "da2479e8c062e40bf0066ffa0bc823de0a9368974af99c9f6df941d2c231e03f"
+checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c"
 
 [[package]]
 name = "lioness"
@@ -11958,7 +11943,7 @@ checksum = "4e69bf016dc406eff7d53a7d3f7cf1c2e72c82b9088aac1118591e36dd2cd3e9"
 dependencies = [
  "bitcoin_hashes 0.13.0",
  "rand 0.8.5",
- "rand_core 0.5.1",
+ "rand_core 0.6.4",
  "serde",
  "unicode-normalization",
 ]
@@ -14751,7 +14736,7 @@ dependencies = [
  "hex",
  "lazy_static",
  "procfs-core",
- "rustix 0.38.21",
+ "rustix 0.38.34",
 ]
 
 [[package]]
@@ -15493,7 +15478,7 @@ version = "0.11.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3e9ad3fe7488d7e34558a2033d45a0c90b72d97b4f80705666fea71472e2e6a1"
 dependencies = [
- "base64 0.21.2",
+ "base64 0.21.7",
  "bytes",
  "encoding_rs",
  "futures-core",
@@ -16026,15 +16011,15 @@ dependencies = [
 
 [[package]]
 name = "rustix"
-version = "0.38.21"
+version = "0.38.34"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2b426b0506e5d50a7d8dafcf2e81471400deb602392c7dd110815afb4eaf02a3"
+checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f"
 dependencies = [
  "bitflags 2.4.0",
  "errno",
  "libc",
- "linux-raw-sys 0.4.10",
- "windows-sys 0.48.0",
+ "linux-raw-sys 0.4.13",
+ "windows-sys 0.52.0",
 ]
 
 [[package]]
@@ -16106,7 +16091,7 @@ version = "1.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2"
 dependencies = [
- "base64 0.21.2",
+ "base64 0.21.7",
 ]
 
 [[package]]
@@ -16115,7 +16100,7 @@ version = "2.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "35e4980fa29e4c4b212ffb3db068a564cbf560e51d3944b7c88bd8bf5bec64f4"
 dependencies = [
- "base64 0.21.2",
+ "base64 0.21.7",
  "rustls-pki-types",
 ]
 
@@ -18402,9 +18387,9 @@ dependencies = [
 
 [[package]]
 name = "smallvec"
-version = "1.11.2"
+version = "1.13.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970"
+checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67"
 
 [[package]]
 name = "smol"
@@ -18441,7 +18426,7 @@ dependencies = [
  "arrayvec 0.7.4",
  "async-lock 2.8.0",
  "atomic-take",
- "base64 0.21.2",
+ "base64 0.21.7",
  "bip39",
  "blake2-rfc",
  "bs58 0.5.0",
@@ -18494,7 +18479,7 @@ checksum = "256b5bad1d6b49045e95fe87492ce73d5af81545d8b4d8318a872d2007024c33"
 dependencies = [
  "async-channel",
  "async-lock 2.8.0",
- "base64 0.21.2",
+ "base64 0.21.7",
  "blake2-rfc",
  "derive_more",
  "either",
@@ -20490,6 +20475,27 @@ dependencies = [
  "syn 2.0.53",
 ]
 
+[[package]]
+name = "sub-du"
+version = "0.1.0"
+dependencies = [
+ "ansi_term",
+ "assert_cmd",
+ "frame-metadata",
+ "frame-remote-externalities",
+ "jsonrpsee",
+ "log",
+ "parity-scale-codec",
+ "sc-rpc-api",
+ "separator",
+ "sp-core",
+ "sp-runtime",
+ "structopt",
+ "substrate-rpc-client",
+ "tokio",
+ "tracing-subscriber 0.3.18",
+]
+
 [[package]]
 name = "subkey"
 version = "9.0.0"
@@ -21027,9 +21033,9 @@ dependencies = [
 
 [[package]]
 name = "target-lexicon"
-version = "0.12.11"
+version = "0.12.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9d0e916b1148c8e263850e1ebcbd046f333e0683c724876bb0da63ea4373dc8a"
+checksum = "e1fc403891a21bcfb7c37834ba66a547a8f402146eba7265b5a6d88059c9ff2f"
 
 [[package]]
 name = "tempfile"
@@ -21040,7 +21046,7 @@ dependencies = [
  "cfg-if",
  "fastrand 2.0.0",
  "redox_syscall 0.4.1",
- "rustix 0.38.21",
+ "rustix 0.38.34",
  "windows-sys 0.48.0",
 ]
 
@@ -21059,7 +21065,7 @@ version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "21bebf2b7c9e0a515f6e0f8c51dc0f8e4696391e6f1ff30379559f8365fb0df7"
 dependencies = [
- "rustix 0.38.21",
+ "rustix 0.38.34",
  "windows-sys 0.48.0",
 ]
 
@@ -22459,9 +22465,9 @@ dependencies = [
 
 [[package]]
 name = "wasmparser-nostd"
-version = "0.100.1"
+version = "0.100.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9157cab83003221bfd385833ab587a039f5d6fa7304854042ba358a3b09e0724"
+checksum = "d5a015fe95f3504a94bb1462c717aae75253e39b9dd6c3fb1062c934535c64aa"
 dependencies = [
  "indexmap-nostd",
 ]
@@ -22510,7 +22516,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c86437fa68626fe896e5afc69234bb2b5894949083586535f200385adfd71213"
 dependencies = [
  "anyhow",
- "base64 0.21.2",
+ "base64 0.21.7",
  "bincode",
  "directories-next",
  "file-per-thread-logger",
diff --git a/substrate/utils/frame/remote-externalities/src/lib.rs b/substrate/utils/frame/remote-externalities/src/lib.rs
index 6405e09b895c..58cb901470c1 100644
--- a/substrate/utils/frame/remote-externalities/src/lib.rs
+++ b/substrate/utils/frame/remote-externalities/src/lib.rs
@@ -52,8 +52,6 @@ use std::{
 use substrate_rpc_client::{rpc_params, BatchRequestBuilder, ChainApi, ClientT, StateApi};
 use tokio_retry::{strategy::FixedInterval, Retry};
 
-pub use sp_core::twox_128;
-
 type KeyValue = (StorageKey, StorageData);
 type TopKeyValues = Vec<KeyValue>;
 type ChildKeyValues = Vec<(ChildInfo, Vec<KeyValue>)>;
diff --git a/substrate/utils/frame/sub-du/Cargo.toml b/substrate/utils/frame/sub-du/Cargo.toml
index ebacce129a88..5dbf28a56ece 100644
--- a/substrate/utils/frame/sub-du/Cargo.toml
+++ b/substrate/utils/frame/sub-du/Cargo.toml
@@ -6,18 +6,20 @@ edition = "2021"
 
 [dependencies]
 codec = { package = "parity-scale-codec", version = "3.0.0" }
-jsonrpsee = { version = "0.15.1", features = ["jsonrpsee-types", "jsonrpsee-http-client", "jsonrpsee-ws-client"] }
+jsonrpsee = { version = "0.22", features = ["jsonrpsee-types", "jsonrpsee-http-client", "jsonrpsee-ws-client"] }
 separator = "0.4.1"
 ansi_term = "0.12.1"
 tracing-subscriber = "0.3.16"
 log = "0.4.17"
 structopt = "0.3"
-tokio = { version = "1.21.2", features = ["rt-multi-thread", "macros"]}
+tokio = { version = "1.37", features = ["rt-multi-thread", "macros"]}
 
-remote-externalities = { path = "../remote-externalities" }
-sp-runtime = { version = "6.0.0", path = "../../../primitives/runtime" }
-frame-metadata = { version = "15.0.0", default-features = false, features = ["v14", "std"] }
-sc-rpc-api = { version = "0.10.0-dev", path = "../../../client/rpc-api" }
+remote-externalities = { package = "frame-remote-externalities", path = "../remote-externalities" }
+sp-core = { path = "../../../primitives/core" }
+sp-runtime = { path = "../../../primitives/runtime" }
+frame-metadata = { version = "16.0.0", default-features = false, features = ["current", "std"] }
+sc-rpc-api = { path = "../../../client/rpc-api" }
+substrate-rpc-client = { path = "../rpc/client" }
 
 [features]
 default = []
@@ -25,4 +27,4 @@ remote-test-kusama = []
 remote-test-polkadot = []
 
 [dev-dependencies]
-assert_cmd = "2"
\ No newline at end of file
+assert_cmd = "2"
diff --git a/substrate/utils/frame/sub-du/src/main.rs b/substrate/utils/frame/sub-du/src/main.rs
index a67cd8a5ceb6..48bdca49143f 100644
--- a/substrate/utils/frame/sub-du/src/main.rs
+++ b/substrate/utils/frame/sub-du/src/main.rs
@@ -1,16 +1,64 @@
+//! # `sub-du`
+//!
+//! A tool like [`du`](https://en.wikipedia.org/wiki/Du_(Unix)) that calculates storage size of pallets of a Substrate chain.
+//!
+//! # Note
+//!
+//! Currently, when calculating the size of `Map` storages, it reads the value of first iterated key
+//! to get the size of the value and assumes all the keys have values of the same size. This is not
+//! necessarily true, but it is a trade-off between speed and accuracy. Otherwise, calculating the
+//! size of some prefixes (e.g. `System::Account`) would take quite a long time.
+//!
+//! # Usage
+//!
+//! ```sh
+//! cargo run -- --progress --uri wss://rpc.polkadot.io:443 --count 1000
+//! ```
+//!
+//! # Example Output
+//!
+//! ```sh
+//! Scraping at block Some(0xb83ec17face39e35bdf29305a9f8f568775e357fc55745132ea9bd4d8ff6d976) of polkadot(1002000)
+//! 152 K │ │─┬ System
+//! 101 M │ │ │─ Account => Map(106,025,440 bytes, 1325318 keys)
+//! 128 K │ │ │─ BlockHash => Map(131,104 bytes, 4097 keys)
+//! 24  K │ │ │─ Events => Value(24,588 bytes)
+//! 156 B │ │ │─ Digest => Value(156 bytes)
+//! 32  B │ │ │─ ParentHash => Value(32 bytes)
+//! 18  B │ │ │─ BlockWeight => Value(18 bytes)
+//! 13  B │ │ │─ LastRuntimeUpgrade => Value(13 bytes)
+//! 4   B │ │ │─ EventCount => Value(4 bytes)
+//! 4   B │ │ │─ Number => Value(4 bytes)
+//! 1   B │ │ │─ UpgradedToTripleRefCount => Value(1 bytes)
+//! 1   B │ │ │─ UpgradedToU32RefCount => Value(1 bytes)
+//! 0   B │ │ │─ AuthorizedUpgrade => Value(0 bytes)
+//! 0   B │ │ │─ ExecutionPhase => Value(0 bytes)
+//! 0   B │ │ │─ EventTopics => Map(0 bytes, 0 keys)
+//! 0   B │ │ │─ ExtrinsicData => Map(0 bytes, 0 keys)
+//! 0   B │ │ │─ AllExtrinsicsLen => Value(0 bytes)
+//! 0   B │ │ │─ ExtrinsicCount => Value(0 bytes)
+//! 1   K │ │─┬ Scheduler
+//! 1   K │ │ │─ Agenda => Map(1,551 bytes, 1551 keys)
+//! 24  B │ │ │─ Lookup => Map(24 bytes, 3 keys)
+//! ```
+
 use ansi_term::{Colour::*, Style};
-use frame_metadata::{RuntimeMetadata, RuntimeMetadataPrefixed, StorageEntryType};
-use remote_externalities::{twox_128, Mode, OnlineConfig, StorageKey, Transport};
-use sc_rpc_api::state::StateApiClient;
+use frame_metadata::{v14::StorageEntryType, RuntimeMetadata, RuntimeMetadataPrefixed};
+use sc_rpc_api::{chain::ChainApiClient, state::StateApiClient};
 use separator::Separatable;
-use sp_runtime::testing::{Block as RawBlock, ExtrinsicWrapper, H256 as Hash};
+use sp_core::{storage::StorageKey, twox_128};
+use sp_runtime::testing::{Header, H256 as Hash};
 use structopt::StructOpt;
+use substrate_rpc_client::WsClient;
+
+#[cfg(test)]
+mod tests;
 
 const KB: usize = 1024;
 const MB: usize = KB * KB;
 const GB: usize = MB * MB;
 
-pub const LOG_TARGET: &'static str = "sub-du";
+pub const LOG_TARGET: &str = "sub-du";
 
 fn get_prefix(indent: usize) -> &'static str {
 	match indent {
@@ -47,7 +95,7 @@ struct Pallet {
 impl std::fmt::Display for Pallet {
 	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
 		let mod_style = Style::new().bold().italic().fg(Green);
-		write!(
+		writeln!(
 			f,
 			"{} {} {}\n",
 			mod_style.paint(format!("{}", Size(self.size))),
@@ -55,7 +103,7 @@ impl std::fmt::Display for Pallet {
 			mod_style.paint(self.name.clone())
 		)?;
 		for s in self.items.iter() {
-			write!(f, "{} {} {}\n", Size(s.size), get_prefix(3), s)?;
+			writeln!(f, "{} {} {}", Size(s.size), get_prefix(3), s)?;
 		}
 		Ok(())
 	}
@@ -133,16 +181,9 @@ struct Opt {
 	#[structopt(long, short)]
 	progress: bool,
 
-	/// Weather to scrape all pairs or just the size of them.
-	///
-	/// If enabled, the command might take longer but then the number of keys in each map is also
-	/// scraped.
-	///
-	/// # Warning
-	///
-	/// This uses an unsafe RPC call and can only be used if the target node allows it.
-	#[structopt(long, short)]
-	scrape_pairs: bool,
+	/// Count of keys to read in one page.
+	#[structopt(long, default_value = "1000")]
+	count: u32,
 }
 
 /// create key prefix for a module as vec bytes. Basically twox128 hash of the given values.
@@ -155,26 +196,85 @@ pub fn pallet_prefix_raw(module: &[u8], storage: &[u8]) -> Vec<u8> {
 	final_key
 }
 
-type Block = RawBlock<ExtrinsicWrapper<Hash>>;
+/// Using `state_getStorageSize` RPC call times out since querying storage size of a relatively
+/// large storage prefix takes a lot of time. This function is a workaround to get the size of the
+/// storage by using paginated `state_getKeysPaged` RPC call.
+///
+/// It is a modified implementation of `state_getStorageSize` RPC call, with some major differences:
+///
+/// - uses paginated reading of keys.
+/// - only reads once for the value size, instead of reading it for all keys.
+///
+/// The latter might not be entirely accurate, since not all values might have the same size (e.g.
+/// when storage value contains `Option<T>` or unbounded data types). It is a trade-off between
+/// speed and accuracy. Otherwise, calculating the size would take quite a long time.
+async fn prefix_storage_size(
+	client: &WsClient,
+	at: Option<Hash>,
+	prefix: StorageKey,
+	count: u32,
+) -> Option<(u64, usize)> {
+	match client.storage(prefix.clone(), at).await {
+		Ok(Some(d)) => return Some((d.0.len() as u64, 0)),
+		Ok(None) => (),
+		Err(e) => {
+			log::error!(target: LOG_TARGET, "Error while reading storage: {:?}", e);
+			return None;
+		},
+	}
+
+	let mut sum = 0;
+	let mut keys_count = 0;
+	let mut value_size = None;
+
+	let mut start_key = None;
+
+	loop {
+		let keys = client
+			.storage_keys_paged(Some(prefix.clone()), count, start_key, at)
+			.await
+			.unwrap_or(vec![]);
+
+		if keys.is_empty() {
+			break;
+		}
+
+		let current_keys = keys.len();
+		keys_count += current_keys;
+
+		for key in keys.clone() {
+			// don't really need to read for all, just for the first one to get the size.
+			if let Some(size) = value_size {
+				sum += size;
+			} else if let Ok(Some(value)) = client.storage(key.clone(), at).await {
+				value_size = Some(value.0.len() as u64);
+				sum += value.0.len() as u64;
+			}
+		}
+		if current_keys < count as usize {
+			break;
+		}
+
+		start_key = if let Some(last) = keys.last() { Some(last.clone()) } else { break };
+	}
+
+	Some((sum, keys_count))
+}
 
 #[tokio::main]
-async fn main() -> () {
+async fn main() {
+	let now = std::time::Instant::now();
+
 	tracing_subscriber::fmt::try_init().unwrap();
 	let opt = Opt::from_args();
 
-	let mut cfg = OnlineConfig::default();
-	cfg.transport = Transport::Uri(opt.uri);
-	cfg.transport.map_uri().await.unwrap();
-
-	// connect to a node.
-	let ext = remote_externalities::Builder::<Block>::new().mode(Mode::Online(cfg));
+	let rpc_client = substrate_rpc_client::ws_client(opt.uri).await.unwrap();
 	let mut modules: Vec<Pallet> = vec![];
 
 	// potentially replace head with the given hash
-	let head = ext.rpc_get_head().await.unwrap();
-	let at = opt.at.or(Some(head));
+	let head = ChainApiClient::<(), _, Header, ()>::finalized_head(&rpc_client).await.unwrap();
 
-	let rpc_client = ext.as_online().rpc_client();
+	let at = opt.at.or(Some(head));
 
 	let runtime = rpc_client.runtime_version(at).await.unwrap();
 
@@ -185,87 +285,83 @@ async fn main() -> () {
 		.expect("Runtime Metadata failed to decode");
 	let metadata = prefixed_metadata.1;
 
-	if let RuntimeMetadata::V14(inner) = metadata {
-		let pallets = inner.pallets;
-		for pallet in pallets.into_iter() {
-			let name = pallet.name;
-
-			// skip, if this module has no storage items.
-			if pallet.storage.is_none() {
-				log::warn!(
-					target: LOG_TARGET,
-					"Pallet with name {:?} seems to have no storage items.",
-					name
-				);
-				continue
-			}
+	let mut total_size = 0;
+	match metadata {
+		RuntimeMetadata::V14(inner) => {
+			let pallets = inner.pallets;
+			for pallet in pallets.into_iter() {
+				let name = pallet.name;
+
+				// skip, if this module has no storage items.
+				if pallet.storage.is_none() {
+					log::warn!(
+						target: LOG_TARGET,
+						"Pallet with name {:?} seems to have no storage items.",
+						name
+					);
+					continue
+				}
+
+				let storage = pallet.storage.unwrap();
+				let prefix = storage.prefix;
+				let entries = storage.entries;
+				let mut pallet_info = Pallet::new(name.clone());
+
+				for storage_entry in entries.into_iter() {
+					let storage_name = storage_entry.name;
+
+					let ty = storage_entry.ty;
+					let key_prefix = pallet_prefix_raw(prefix.as_bytes(), storage_name.as_bytes());
 
-			let storage = pallet.storage.unwrap();
-			let prefix = storage.prefix;
-			let entries = storage.entries;
-			let mut pallet_info = Pallet::new(name.clone());
-
-			for storage_entry in entries.into_iter() {
-				let storage_name = storage_entry.name;
-				let ty = storage_entry.ty;
-				let key_prefix = pallet_prefix_raw(prefix.as_bytes(), storage_name.as_bytes());
-
-				let (pairs, size) = if opt.scrape_pairs {
-					// this should be slower but gives more detail.
-					let pairs =
-						rpc_client.storage_pairs(StorageKey(key_prefix.clone()), at).await.unwrap();
-					let pairs = pairs
-						.into_iter()
-						.map(|(k, v)| (k.0, v.0))
-						.collect::<Vec<(Vec<u8>, Vec<u8>)>>();
-					let size = pairs.iter().fold(0, |acc, x| acc + x.1.len());
-					(pairs, size)
-				} else {
 					// This should be faster
-					let size = rpc_client
-						.storage_size(StorageKey(key_prefix), at)
-						.await
-						.unwrap()
-						.unwrap_or_default() as usize;
-					let pairs: Vec<_> = vec![];
-					(pairs, size)
-				};
-
-				log::debug!(
-					target: LOG_TARGET,
-					"{:?}::{:?} => count: {}, size: {} bytes",
-					name,
-					storage_name,
-					pairs.len(),
-					size
-				);
-
-				pallet_info.size += size;
-				let item = match ty {
-					StorageEntryType::Plain(_) => StorageItem::Value(size),
-					StorageEntryType::Map { .. } => StorageItem::Map(size, pairs.len()),
-				};
-				pallet_info.items.push(Storage::new(storage_name, item));
+					let (size, pairs) = prefix_storage_size(
+						&rpc_client,
+						at,
+						StorageKey(key_prefix.clone()),
+						opt.count,
+					)
+					.await
+					.unwrap();
+
+					log::debug!(
+						target: LOG_TARGET,
+						"{:?}::{:?} => count: {}, size: {} bytes",
+						name,
+						storage_name,
+						pairs,
+						size
+					);
+
+					pallet_info.size += size as usize;
+					let item = match ty {
+						StorageEntryType::Plain(_) => StorageItem::Value(size as usize),
+						StorageEntryType::Map { .. } => StorageItem::Map(size as usize, pairs),
+					};
+					pallet_info.items.push(Storage::new(storage_name, item));
+				}
+				pallet_info.items.sort_by_key(|x| x.size);
+				pallet_info.items.reverse();
+
+				if opt.progress {
+					print!("{}", pallet_info);
+				}
+				total_size += pallet_info.size;
+				modules.push(pallet_info);
 			}
-			pallet_info.items.sort_by_key(|x| x.size);
-			pallet_info.items.reverse();
-			println!("Scraped module {}. Total size {}.", pallet_info.name, pallet_info.size,);
-			if opt.progress {
-				print!("{}", pallet_info);
+
+			modules.sort_by_key(|m| m.size);
+			modules.reverse();
+
+			if !opt.progress {
+				modules.into_iter().for_each(|m| {
+					print!("{}", m);
+				})
 			}
-			modules.push(pallet_info);
-		}
+			println!("\nTotal size: {} {}", Size(total_size), runtime.spec_name,);
+			let elapsed = now.elapsed();
+			println!("{:?}s", elapsed.as_secs_f32());
+		},
 
-		println!("Scraping results done. Final sorted tree:");
-		modules.sort_by_key(|m| m.size);
-		modules.reverse();
-
-		let total: usize = modules.iter().map(|m| m.size).sum();
-		println!("{} {} {}", Size(total), get_prefix(1), runtime.spec_name,);
-		modules.into_iter().for_each(|m| {
-			print!("{}", m);
-		});
-	} else {
-		panic!("Unsupported Metadata version");
+		_ => panic!("Unsupported metadata version."),
 	}
 }
diff --git a/substrate/utils/frame/sub-du/src/tests.rs b/substrate/utils/frame/sub-du/src/tests.rs
index e69de29bb2d1..cf0e6a3eec6d 100644
--- a/substrate/utils/frame/sub-du/src/tests.rs
+++ b/substrate/utils/frame/sub-du/src/tests.rs
@@ -0,0 +1,24 @@
+use assert_cmd::Command;
+
+#[cfg(feature = "remote-test-kusama")]
+const TEST_URI: &str = "wss://kusama-rpc.polkadot.io/";
+#[cfg(feature = "remote-test-polkadot")]
+const TEST_URI: &str = "wss://rpc.polkadot.io/";
+#[cfg(not(any(feature = "remote-test-kusama", feature = "remote-test-polkadot")))]
+const TEST_URI: &str = "ws://localhost:9944";
+
+#[test]
+fn sub_du_starts_to_scrape_normal() {
+	let mut cmd = Command::cargo_bin("sub-du").unwrap();
+	let _stdout = cmd
+		.args(["--uri", TEST_URI, "-p"])
+		.timeout(std::time::Duration::from_secs(10))
+		.output()
+		.unwrap()
+		.stdout;
+
+	#[cfg(feature = "remote-test-kusama")]
+	assert!(String::from_utf8_lossy(&stdout).contains("of kusama("));
+	#[cfg(feature = "remote-test-polkadot")]
+	assert!(String::from_utf8_lossy(&stdout).contains("of polkadot("));
+}

From 29470d260df1c578e3a1b875eaa8438a54f70115 Mon Sep 17 00:00:00 2001
From: dastansam <dastan.samatov.33@gmail.com>
Date: Wed, 1 May 2024 04:01:37 +0200
Subject: [PATCH 3/4] Fix checks

---
 Cargo.lock                                |  1 -
 prdoc/pr_4341.prdoc                       | 11 ++++++++
 substrate/utils/frame/sub-du/Cargo.toml   | 33 ++++++++++++++++-------
 substrate/utils/frame/sub-du/src/main.rs  | 19 ++++++++++---
 substrate/utils/frame/sub-du/src/tests.rs | 24 -----------------
 5 files changed, 51 insertions(+), 37 deletions(-)
 create mode 100644 prdoc/pr_4341.prdoc
 delete mode 100644 substrate/utils/frame/sub-du/src/tests.rs

diff --git a/Cargo.lock b/Cargo.lock
index 6aec0b018976..f246807493b7 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -20495,7 +20495,6 @@ dependencies = [
  "ansi_term",
  "assert_cmd",
  "frame-metadata",
- "frame-remote-externalities",
  "jsonrpsee",
  "log",
  "parity-scale-codec",
diff --git a/prdoc/pr_4341.prdoc b/prdoc/pr_4341.prdoc
new file mode 100644
index 000000000000..6efdcc0b26c2
--- /dev/null
+++ b/prdoc/pr_4341.prdoc
@@ -0,0 +1,11 @@
+# Schema: Polkadot SDK PRDoc Schema (prdoc) v1.0.0
+# See doc at https://raw.githubusercontent.com/paritytech/polkadot-sdk/master/prdoc/schema_user.json
+
+title: "Revive `sub-du`: tool for calculating storage size of pallets"
+
+doc:
+  - audience: Runtime User
+    description: |
+      A tool like [`du`](https://en.wikipedia.org/wiki/Du_(Unix)) that calculates storage size of pallets of a Substrate chain.
+
+crates: []
diff --git a/substrate/utils/frame/sub-du/Cargo.toml b/substrate/utils/frame/sub-du/Cargo.toml
index 5dbf28a56ece..393859a7c804 100644
--- a/substrate/utils/frame/sub-du/Cargo.toml
+++ b/substrate/utils/frame/sub-du/Cargo.toml
@@ -1,30 +1,45 @@
 [package]
 name = "sub-du"
 version = "0.1.0"
-authors = ["Parity Technologies <admin@parity.io>"]
-edition = "2021"
+authors.workspace = true
+edition.workspace = true
+license = "Apache-2.0"
+repository.workspace = true
+description = "A du-like tool for reading storage size of Substrate chains."
+
+[lints]
+workspace = true
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]
 
 [dependencies]
 codec = { package = "parity-scale-codec", version = "3.0.0" }
-jsonrpsee = { version = "0.22", features = ["jsonrpsee-types", "jsonrpsee-http-client", "jsonrpsee-ws-client"] }
+jsonrpsee = { version = "0.22", features = [
+	"jsonrpsee-http-client",
+	"jsonrpsee-types",
+	"jsonrpsee-ws-client",
+] }
 separator = "0.4.1"
 ansi_term = "0.12.1"
 tracing-subscriber = "0.3.16"
 log = "0.4.17"
 structopt = "0.3"
-tokio = { version = "1.37", features = ["rt-multi-thread", "macros"]}
+tokio = { version = "1.37", features = ["macros", "rt-multi-thread"] }
 
-remote-externalities = { package = "frame-remote-externalities", path = "../remote-externalities" }
 sp-core = { path = "../../../primitives/core" }
 sp-runtime = { path = "../../../primitives/runtime" }
-frame-metadata = { version = "16.0.0", default-features = false, features = ["current", "std"] }
+frame-metadata = { version = "16.0.0", default-features = false, features = [
+	"current",
+	"std",
+] }
 sc-rpc-api = { path = "../../../client/rpc-api" }
 substrate-rpc-client = { path = "../rpc/client" }
 
+[dev-dependencies]
+assert_cmd = "2"
+
 [features]
 default = []
 remote-test-kusama = []
 remote-test-polkadot = []
-
-[dev-dependencies]
-assert_cmd = "2"
diff --git a/substrate/utils/frame/sub-du/src/main.rs b/substrate/utils/frame/sub-du/src/main.rs
index 48bdca49143f..c30f867f6901 100644
--- a/substrate/utils/frame/sub-du/src/main.rs
+++ b/substrate/utils/frame/sub-du/src/main.rs
@@ -1,3 +1,19 @@
+// This file is part of Substrate.
+
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
 //! # `sub-du`
 //!
 //! A tool like [`du`](https://en.wikipedia.org/wiki/Du_(Unix)) that calculates storage size of pallets of a Substrate chain.
@@ -51,9 +67,6 @@ use sp_runtime::testing::{Header, H256 as Hash};
 use structopt::StructOpt;
 use substrate_rpc_client::WsClient;
 
-#[cfg(test)]
-mod tests;
-
 const KB: usize = 1024;
 const MB: usize = KB * KB;
 const GB: usize = MB * MB;
diff --git a/substrate/utils/frame/sub-du/src/tests.rs b/substrate/utils/frame/sub-du/src/tests.rs
deleted file mode 100644
index cf0e6a3eec6d..000000000000
--- a/substrate/utils/frame/sub-du/src/tests.rs
+++ /dev/null
@@ -1,24 +0,0 @@
-use assert_cmd::Command;
-
-#[cfg(feature = "remote-test-kusama")]
-const TEST_URI: &str = "wss://kusama-rpc.polkadot.io/";
-#[cfg(feature = "remote-test-polkadot")]
-const TEST_URI: &str = "wss://rpc.polkadot.io/";
-#[cfg(not(any(feature = "remote-test-kusama", feature = "remote-test-polkadot")))]
-const TEST_URI: &str = "ws://localhost:9944";
-
-#[test]
-fn sub_du_starts_to_scrape_normal() {
-	let mut cmd = Command::cargo_bin("sub-du").unwrap();
-	let _stdout = cmd
-		.args(["--uri", TEST_URI, "-p"])
-		.timeout(std::time::Duration::from_secs(10))
-		.output()
-		.unwrap()
-		.stdout;
-
-	#[cfg(feature = "remote-test-kusama")]
-	assert!(String::from_utf8_lossy(&stdout).contains("of kusama("));
-	#[cfg(feature = "remote-test-polkadot")]
-	assert!(String::from_utf8_lossy(&stdout).contains("of polkadot("));
-}

From 55bd5d99a1a0a3f4e300bd8c1e372b468969e7c9 Mon Sep 17 00:00:00 2001
From: dastansam <dastan.samatov.33@gmail.com>
Date: Thu, 2 May 2024 18:38:46 +0200
Subject: [PATCH 4/4] Update lock

---
 Cargo.lock | 91 +++++++++++++++++++++++++++++++-----------------------
 1 file changed, 53 insertions(+), 38 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index f73e617785f0..8007033adc48 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -9,7 +9,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3"
 dependencies = [
  "lazy_static",
- "regex", 
+ "regex",
 ]
 
 [[package]]
@@ -1237,9 +1237,9 @@ dependencies = [
 
 [[package]]
 name = "async-task"
-version = "4.7.1"
+version = "4.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de"
+checksum = "ecc7ab41815b3c653ccd2978ec3255c81349336702dfdf62ee6f7069b12a3aae"
 
 [[package]]
 name = "async-trait"
@@ -1375,9 +1375,9 @@ checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"
 
 [[package]]
 name = "base64"
-version = "0.21.7"
+version = "0.21.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567"
+checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d"
 
 [[package]]
 name = "base64ct"
@@ -2371,9 +2371,9 @@ checksum = "17febce684fd15d89027105661fec94afb475cb995fbc59d2865198446ba2eea"
 
 [[package]]
 name = "byteorder"
-version = "1.5.0"
+version = "1.4.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
+checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
 
 [[package]]
 name = "bytes"
@@ -3546,18 +3546,22 @@ dependencies = [
 
 [[package]]
 name = "crossbeam-queue"
-version = "0.3.11"
+version = "0.3.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35"
+checksum = "d1cfb3ea8a53f37c40dea2c7bedcbd88bdfae54f5e2175d6ecaff1c988353add"
 dependencies = [
+ "cfg-if",
  "crossbeam-utils",
 ]
 
 [[package]]
 name = "crossbeam-utils"
-version = "0.8.19"
+version = "0.8.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345"
+checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294"
+dependencies = [
+ "cfg-if",
+]
 
 [[package]]
 name = "crunchy"
@@ -5252,12 +5256,23 @@ dependencies = [
 
 [[package]]
 name = "errno"
-version = "0.3.8"
+version = "0.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245"
+checksum = "6b30f669a7961ef1631673d2766cc92f52d64f7ef354d4fe0ddfd30ed52f0f4f"
 dependencies = [
+ "errno-dragonfly",
+ "libc",
+ "windows-sys 0.48.0",
+]
+
+[[package]]
+name = "errno-dragonfly"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf"
+dependencies = [
+ "cc",
  "libc",
- "windows-sys 0.52.0",
 ]
 
 [[package]]
@@ -6064,7 +6079,7 @@ version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "29f9df8a11882c4e3335eb2d18a0137c505d9ca927470b0cac9c6f0ae07d28f7"
 dependencies = [
- "rustix 0.38.34",
+ "rustix 0.38.21",
  "windows-sys 0.48.0",
 ]
 
@@ -6966,7 +6981,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b"
 dependencies = [
  "hermit-abi 0.3.2",
- "rustix 0.38.34",
+ "rustix 0.38.21",
  "windows-sys 0.48.0",
 ]
 
@@ -7504,9 +7519,9 @@ dependencies = [
 
 [[package]]
 name = "libm"
-version = "0.2.8"
+version = "0.2.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058"
+checksum = "f7012b1bbb0719e1097c47611d3898568c546d597c2e74d66f6087edd5233ff4"
 
 [[package]]
 name = "libnghttp2-sys"
@@ -8033,9 +8048,9 @@ checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519"
 
 [[package]]
 name = "linux-raw-sys"
-version = "0.4.13"
+version = "0.4.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c"
+checksum = "da2479e8c062e40bf0066ffa0bc823de0a9368974af99c9f6df941d2c231e03f"
 
 [[package]]
 name = "lioness"
@@ -14727,7 +14742,7 @@ dependencies = [
  "hex",
  "lazy_static",
  "procfs-core",
- "rustix 0.38.34",
+ "rustix 0.38.21",
 ]
 
 [[package]]
@@ -15469,7 +15484,7 @@ version = "0.11.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3e9ad3fe7488d7e34558a2033d45a0c90b72d97b4f80705666fea71472e2e6a1"
 dependencies = [
- "base64 0.21.7",
+ "base64 0.21.2",
  "bytes",
  "encoding_rs",
  "futures-core",
@@ -16002,15 +16017,15 @@ dependencies = [
 
 [[package]]
 name = "rustix"
-version = "0.38.34"
+version = "0.38.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f"
+checksum = "2b426b0506e5d50a7d8dafcf2e81471400deb602392c7dd110815afb4eaf02a3"
 dependencies = [
  "bitflags 2.4.0",
  "errno",
  "libc",
- "linux-raw-sys 0.4.13",
- "windows-sys 0.52.0",
+ "linux-raw-sys 0.4.10",
+ "windows-sys 0.48.0",
 ]
 
 [[package]]
@@ -16082,7 +16097,7 @@ version = "1.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2"
 dependencies = [
- "base64 0.21.7",
+ "base64 0.21.2",
 ]
 
 [[package]]
@@ -16091,7 +16106,7 @@ version = "2.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "35e4980fa29e4c4b212ffb3db068a564cbf560e51d3944b7c88bd8bf5bec64f4"
 dependencies = [
- "base64 0.21.7",
+ "base64 0.21.2",
  "rustls-pki-types",
 ]
 
@@ -18378,9 +18393,9 @@ dependencies = [
 
 [[package]]
 name = "smallvec"
-version = "1.13.2"
+version = "1.11.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67"
+checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970"
 
 [[package]]
 name = "smol"
@@ -18417,7 +18432,7 @@ dependencies = [
  "arrayvec 0.7.4",
  "async-lock",
  "atomic-take",
- "base64 0.21.7",
+ "base64 0.21.2",
  "bip39",
  "blake2-rfc",
  "bs58 0.5.0",
@@ -21023,9 +21038,9 @@ dependencies = [
 
 [[package]]
 name = "target-lexicon"
-version = "0.12.14"
+version = "0.12.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e1fc403891a21bcfb7c37834ba66a547a8f402146eba7265b5a6d88059c9ff2f"
+checksum = "9d0e916b1148c8e263850e1ebcbd046f333e0683c724876bb0da63ea4373dc8a"
 
 [[package]]
 name = "tempfile"
@@ -21036,7 +21051,7 @@ dependencies = [
  "cfg-if",
  "fastrand 2.0.0",
  "redox_syscall 0.4.1",
- "rustix 0.38.34",
+ "rustix 0.38.21",
  "windows-sys 0.48.0",
 ]
 
@@ -21055,7 +21070,7 @@ version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "21bebf2b7c9e0a515f6e0f8c51dc0f8e4696391e6f1ff30379559f8365fb0df7"
 dependencies = [
- "rustix 0.38.34",
+ "rustix 0.38.21",
  "windows-sys 0.48.0",
 ]
 
@@ -22455,9 +22470,9 @@ dependencies = [
 
 [[package]]
 name = "wasmparser-nostd"
-version = "0.100.2"
+version = "0.100.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d5a015fe95f3504a94bb1462c717aae75253e39b9dd6c3fb1062c934535c64aa"
+checksum = "9157cab83003221bfd385833ab587a039f5d6fa7304854042ba358a3b09e0724"
 dependencies = [
  "indexmap-nostd",
 ]
@@ -22506,7 +22521,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c86437fa68626fe896e5afc69234bb2b5894949083586535f200385adfd71213"
 dependencies = [
  "anyhow",
- "base64 0.21.7",
+ "base64 0.21.2",
  "bincode",
  "directories-next",
  "file-per-thread-logger",