Expose state-db memory info (paritytech#5110)

This exposes memory statistics from the state-db.

client/api/src/client.rs

@@ -97,13 +97,56 @@ pub struct ClientInfo<Block: BlockT> {
 	pub usage: Option<UsageInfo>,
 }
 
+/// A wrapper to store the size of some memory.
+#[derive(Default, Clone, Debug, Copy)]
+pub struct MemorySize(usize);
+
+impl MemorySize {
+	/// Creates `Self` from the given `bytes` size.
+	pub fn from_bytes(bytes: usize) -> Self {
+		Self(bytes)
+	}
+
+	/// Returns the memory size as bytes.
+	pub fn as_bytes(self) -> usize {
+		self.0
+	}
+}
+
+impl fmt::Display for MemorySize {
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+		if self.0 < 1024 {
+			write!(f, "{} bytes", self.0)
+		} else if self.0 < 1024 * 1024 {
+			write!(f, "{:.2} KiB", self.0 as f64 / 1024f64)
+		} else if self.0 < 1024 * 1024 * 1024 {
+			write!(f, "{:.2} MiB", self.0 as f64 / (1024f64 * 1024f64))
+		} else {
+			write!(f, "{:.2} GiB", self.0 as f64 / (1024f64 * 1024f64 * 1024f64))
+		}
+	}
+}
+
+/// Memory statistics for state db.
+#[derive(Default, Clone, Debug)]
+pub struct StateDbMemoryInfo {
+	/// Memory usage of the non-canonical overlay
+	pub non_canonical: MemorySize,
+	/// Memory usage of the pruning window.
+	pub pruning: Option<MemorySize>,
+	/// Memory usage of the pinned blocks.
+	pub pinned: MemorySize,
+}
+
 /// Memory statistics for client instance.
 #[derive(Default, Clone, Debug)]
 pub struct MemoryInfo {
 	/// Size of state cache.
-	pub state_cache: usize,
+	pub state_cache: MemorySize,
 	/// Size of backend database cache.
-	pub database_cache: usize,
+	pub database_cache: MemorySize,
+	/// Size of the state db.
+	pub state_db: StateDbMemoryInfo,
 }
 
 /// I/O statistics for client instance.
@@ -144,10 +187,16 @@ pub struct UsageInfo {
 
 impl fmt::Display for UsageInfo {
 	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-		write!(f,
-			"caches: ({} state, {} db overlay), i/o: ({} tx, {} write, {} read, {} avg tx, {}/{} key cache reads/total, {} key writes)",
+		write!(
+			f,
+			"caches: ({} state, {} db overlay), \
+			 state db: ({} non-canonical, {} pruning, {} pinned), \
+			 i/o: ({} tx, {} write, {} read, {} avg tx, {}/{} key cache reads/total, {} key writes)",
 			self.memory.state_cache,
 			self.memory.database_cache,
+			self.memory.state_db.non_canonical,
+			self.memory.state_db.pruning.unwrap_or_default(),
+			self.memory.state_db.pinned,
 			self.io.transactions,
 			self.io.bytes_written,
 			self.io.bytes_read,

client/db/src/lib.rs

@@ -46,9 +46,11 @@ use std::path::PathBuf;
 use std::io;
 use std::collections::HashMap;
 
-use sc_client_api::{execution_extensions::ExecutionExtensions, ForkBlocks, UsageInfo, MemoryInfo, BadBlocks, IoInfo};
-use sc_client_api::backend::NewBlockState;
-use sc_client_api::backend::PrunableStateChangesTrieStorage;
+use sc_client_api::{
+	ForkBlocks, UsageInfo, MemoryInfo, BadBlocks, IoInfo, MemorySize,
+	execution_extensions::ExecutionExtensions,
+	backend::{NewBlockState, PrunableStateChangesTrieStorage},
+};
 use sp_blockchain::{
 	Result as ClientResult, Error as ClientError,
 	well_known_cache_keys, HeaderBackend,
@@ -1455,13 +1457,17 @@ impl<Block: BlockT> sc_client_api::backend::Backend<Block> for Backend<Block> {
 				self.state_usage.take(),
 			)
 		);
-		let database_cache = parity_util_mem::malloc_size(&*self.storage.db);
-		let state_cache = (*&self.shared_cache).lock().used_storage_cache_size();
+		let database_cache = MemorySize::from_bytes(parity_util_mem::malloc_size(&*self.storage.db));
+		let state_cache = MemorySize::from_bytes(
+			(*&self.shared_cache).lock().used_storage_cache_size(),
+		);
+		let state_db = self.storage.state_db.memory_info();
 
 		Some(UsageInfo {
 			memory: MemoryInfo {
 				state_cache,
 				database_cache,
+				state_db,
 			},
 			io: IoInfo {
 				transactions: io_stats.transactions,

client/db/src/light.rs

@@ -317,7 +317,7 @@ impl<Block: BlockT> LightStorage<Block> {
 			// if the header includes changes trie root, let's build a changes tries roots CHT
 			if header.digest().log(DigestItem::as_changes_trie_root).is_some() {
 				let mut current_num = new_cht_start;
-				let cht_range = ::std::iter::from_fn(|| {
+				let cht_range = std::iter::from_fn(|| {
 					let old_current_num = current_num;
 					current_num = current_num + One::one();
 					Some(old_current_num)
@@ -572,15 +572,16 @@ impl<Block> LightBlockchainStorage<Block> for LightStorage<Block>
 
 	#[cfg(not(target_os = "unknown"))]
 	fn usage_info(&self) -> Option<UsageInfo> {
-		use sc_client_api::{MemoryInfo, IoInfo};
+		use sc_client_api::{MemoryInfo, IoInfo, MemorySize};
 
-		let database_cache = parity_util_mem::malloc_size(&*self.db);
+		let database_cache = MemorySize::from_bytes(parity_util_mem::malloc_size(&*self.db));
 		let io_stats = self.io_stats.take_or_else(|| self.db.io_stats(kvdb::IoStatsKind::SincePrevious));
 
 		Some(UsageInfo {
 			memory: MemoryInfo {
 				database_cache,
-				state_cache: 0,
+				state_cache: Default::default(),
+				state_db: Default::default(),
 			},
 			io: IoInfo {
 				transactions: io_stats.transactions,

client/informant/src/lib.rs

@@ -46,7 +46,10 @@ pub fn build(service: &impl AbstractService, format: OutputFormat) -> impl futur
 			if let Some(ref usage) = info.usage {
 				trace!(target: "usage", "Usage statistics: {}", usage);
 			} else {
-				trace!(target: "usage", "Usage statistics not displayed as backend does not provide it")
+				trace!(
+					target: "usage",
+					"Usage statistics not displayed as backend does not provide it",
+				)
 			}
 			#[cfg(not(target_os = "unknown"))]
 			trace!(

client/service/src/builder.rs

@@ -1085,10 +1085,18 @@ ServiceBuilder<
 				"finalized_hash" => ?info.chain.finalized_hash,
 				"bandwidth_download" => bandwidth_download,
 				"bandwidth_upload" => bandwidth_upload,
-				"used_state_cache_size" => info.usage.as_ref().map(|usage| usage.memory.state_cache).unwrap_or(0),
-				"used_db_cache_size" => info.usage.as_ref().map(|usage| usage.memory.database_cache).unwrap_or(0),
-				"disk_read_per_sec" => info.usage.as_ref().map(|usage| usage.io.bytes_read).unwrap_or(0),
-				"disk_write_per_sec" => info.usage.as_ref().map(|usage| usage.io.bytes_written).unwrap_or(0),
+				"used_state_cache_size" => info.usage.as_ref()
+					.map(|usage| usage.memory.state_cache.as_bytes())
+					.unwrap_or(0),
+				"used_db_cache_size" => info.usage.as_ref()
+					.map(|usage| usage.memory.database_cache.as_bytes())
+					.unwrap_or(0),
+				"disk_read_per_sec" => info.usage.as_ref()
+					.map(|usage| usage.io.bytes_read)
+					.unwrap_or(0),
+				"disk_write_per_sec" => info.usage.as_ref()
+					.map(|usage| usage.io.bytes_written)
+					.unwrap_or(0),
 			);
 			if let Some(metrics) = metrics.as_ref() {
 				metrics.memory_usage_bytes.set(memory);

client/state-db/Cargo.toml

@@ -11,8 +11,11 @@ description = "State database maintenance. Handles canonicalization and pruning
 [dependencies]
 parking_lot = "0.10.0"
 log = "0.4.8"
+sc-client-api = { version = "2.0.0-alpha.2", path = "../api" }
 sp-core = { version = "2.0.0-alpha.2", path = "../../primitives/core" }
 codec = { package = "parity-scale-codec", version = "1.0.0", features = ["derive"] }
+parity-util-mem = "0.5.1"
+parity-util-mem-derive = "0.1.0"
 
 [dev-dependencies]
 env_logger = "0.7.0"