From 243b4074df9f1971986bec35509fb2a3cb6865bb Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Wed, 31 Jan 2024 14:36:56 +0100 Subject: [PATCH 1/9] Update output --- .../subsystem-bench/src/availability/mod.rs | 10 +- .../subsystem-bench/src/core/environment.rs | 111 ++++++++++++------ 2 files changed, 77 insertions(+), 44 deletions(-) diff --git a/polkadot/node/subsystem-bench/src/availability/mod.rs b/polkadot/node/subsystem-bench/src/availability/mod.rs index f9892efb3c68..f7ea50b32c54 100644 --- a/polkadot/node/subsystem-bench/src/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/availability/mod.rs @@ -477,8 +477,7 @@ pub async fn benchmark_availability_read(env: &mut TestEnvironment, mut state: T format!("{} ms", test_start.elapsed().as_millis() / env.config().num_blocks as u128).red() ); - env.display_network_usage(); - env.display_cpu_usage(&["availability-recovery"]); + println!("{}", env.collect_resource_usage(&["availability-recovery"])); env.stop().await; } @@ -634,14 +633,11 @@ pub async fn benchmark_availability_write(env: &mut TestEnvironment, mut state: format!("{} ms", test_start.elapsed().as_millis() / env.config().num_blocks as u128).red() ); - env.display_network_usage(); - - env.display_cpu_usage(&[ + println!("{}", env.collect_resource_usage(&[ "availability-distribution", "bitfield-distribution", "availability-store", - ]); - + ])); env.stop().await; } diff --git a/polkadot/node/subsystem-bench/src/core/environment.rs b/polkadot/node/subsystem-bench/src/core/environment.rs index b6846316430b..38e41cc82929 100644 --- a/polkadot/node/subsystem-bench/src/core/environment.rs +++ b/polkadot/node/subsystem-bench/src/core/environment.rs @@ -328,57 +328,94 @@ impl TestEnvironment { } } - /// Display network usage stats. - pub fn display_network_usage(&self) { - let stats = self.network().peer_stats(0); - - let total_node_received = stats.received() / 1024; - let total_node_sent = stats.sent() / 1024; - - println!( - "\nPayload bytes received from peers: {}, {}", - format!("{:.2} KiB total", total_node_received).blue(), - format!("{:.2} KiB/block", total_node_received / self.config().num_blocks) - .bright_blue() - ); + pub fn collect_resource_usage(&self, subsystems_under_test: &[&str]) -> CollectedResourceUsage { + CollectedResourceUsage { + benchmark: "Subsystem benchmarks".to_string(), + network: self.network_usage(), + cpu: self.cpu_usage(subsystems_under_test), + } + } - println!( - "Payload bytes sent to peers: {}, {}", - format!("{:.2} KiB total", total_node_sent).blue(), - format!("{:.2} KiB/block", total_node_sent / self.config().num_blocks).bright_blue() - ); + fn network_usage(&self) -> Vec { + let stats = self.network().peer_stats(0); + let total_node_received = (stats.received() / 1024) as f64; + let total_node_sent = (stats.sent() / 1024) as f64; + let num_blocks = self.config().num_blocks as f64; + + vec![ + ResourceUsage { + resource: "Received from peers".to_string(), + total: total_node_received, + per_block: total_node_received / num_blocks, + }, + ResourceUsage { + resource: "Sent to peers".to_string(), + total: total_node_sent as f64, + per_block: total_node_sent / num_blocks, + }, + ] } - /// Print CPU usage stats in the CLI. - pub fn display_cpu_usage(&self, subsystems_under_test: &[&str]) { + fn cpu_usage(&self, subsystems_under_test: &[&str]) -> Vec { let test_metrics = super::display::parse_metrics(self.registry()); + let mut usage = vec![]; + let num_blocks = self.config().num_blocks as f64; for subsystem in subsystems_under_test.iter() { let subsystem_cpu_metrics = test_metrics.subset_with_label_value("task_group", subsystem); let total_cpu = subsystem_cpu_metrics.sum_by("substrate_tasks_polling_duration_sum"); - println!( - "{} CPU usage {}", - subsystem.to_string().bright_green(), - format!("{:.3}s", total_cpu).bright_purple() - ); - println!( - "{} CPU usage per block {}", - subsystem.to_string().bright_green(), - format!("{:.3}s", total_cpu / self.config().num_blocks as f64).bright_purple() - ); + usage.push(ResourceUsage { + resource: subsystem.to_string(), + total: total_cpu, + per_block: total_cpu / num_blocks, + }); } let test_env_cpu_metrics = test_metrics.subset_with_label_value("task_group", "test-environment"); let total_cpu = test_env_cpu_metrics.sum_by("substrate_tasks_polling_duration_sum"); - println!( - "Total test environment CPU usage {}", - format!("{:.3}s", total_cpu).bright_purple() - ); - println!( - "Test environment CPU usage per block {}", - format!("{:.3}s", total_cpu / self.config().num_blocks as f64).bright_purple() + + usage.push(ResourceUsage { + resource: "Test environment".to_string(), + total: total_cpu, + per_block: total_cpu / num_blocks, + }); + + usage + } +} + +#[derive(Debug)] +pub struct CollectedResourceUsage { + benchmark: String, + network: Vec, + cpu: Vec, +} + +impl std::fmt::Display for CollectedResourceUsage { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!( + f, + "\n{}\n\n{}\n{}\n\n{}\n{}\n", + self.benchmark.blue(), + format!("{:<32}{:>12}{:>12}", "Network usage, KiB", "total", "per block").blue(), + self.network.iter().map(|v| v.to_string()).collect::>().join("\n"), + format!("{:<32}{:>12}{:>12}", "CPU usage, s", "total", "per block").blue(), + self.cpu.iter().map(|v| v.to_string()).collect::>().join("\n") ) } } + +#[derive(Debug)] +pub struct ResourceUsage { + resource: String, + total: f64, + per_block: f64, +} + +impl std::fmt::Display for ResourceUsage { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "{:<32}{:>12.3}{:>12.3}", self.resource.cyan(), self.total, self.per_block) + } +} From 3320dbe00eb9d16fdc1053b0aa0c78e0f3668685 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Wed, 31 Jan 2024 15:04:33 +0100 Subject: [PATCH 2/9] Add benchmark name based on a path --- .../subsystem-bench/src/availability/mod.rs | 8 +++---- .../subsystem-bench/src/core/environment.rs | 12 ++++++---- .../subsystem-bench/src/subsystem-bench.rs | 23 ++++++++++++++----- 3 files changed, 29 insertions(+), 14 deletions(-) diff --git a/polkadot/node/subsystem-bench/src/availability/mod.rs b/polkadot/node/subsystem-bench/src/availability/mod.rs index f7ea50b32c54..dfe71f51013b 100644 --- a/polkadot/node/subsystem-bench/src/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/availability/mod.rs @@ -417,7 +417,7 @@ impl TestState { } } -pub async fn benchmark_availability_read(env: &mut TestEnvironment, mut state: TestState) { +pub async fn benchmark_availability_read(benchmark_name: &str, env: &mut TestEnvironment, mut state: TestState) { let config = env.config().clone(); env.import_block(new_block_import_info(Hash::repeat_byte(1), 1)).await; @@ -477,11 +477,11 @@ pub async fn benchmark_availability_read(env: &mut TestEnvironment, mut state: T format!("{} ms", test_start.elapsed().as_millis() / env.config().num_blocks as u128).red() ); - println!("{}", env.collect_resource_usage(&["availability-recovery"])); + println!("{}", env.collect_resource_usage(benchmark_name, &["availability-recovery"])); env.stop().await; } -pub async fn benchmark_availability_write(env: &mut TestEnvironment, mut state: TestState) { +pub async fn benchmark_availability_write(benchmark_name: &str, env: &mut TestEnvironment, mut state: TestState) { let config = env.config().clone(); env.metrics().set_n_validators(config.n_validators); @@ -633,7 +633,7 @@ pub async fn benchmark_availability_write(env: &mut TestEnvironment, mut state: format!("{} ms", test_start.elapsed().as_millis() / env.config().num_blocks as u128).red() ); - println!("{}", env.collect_resource_usage(&[ + println!("{}", env.collect_resource_usage(benchmark_name, &[ "availability-distribution", "bitfield-distribution", "availability-store", diff --git a/polkadot/node/subsystem-bench/src/core/environment.rs b/polkadot/node/subsystem-bench/src/core/environment.rs index 38e41cc82929..5a28b00ab459 100644 --- a/polkadot/node/subsystem-bench/src/core/environment.rs +++ b/polkadot/node/subsystem-bench/src/core/environment.rs @@ -328,9 +328,13 @@ impl TestEnvironment { } } - pub fn collect_resource_usage(&self, subsystems_under_test: &[&str]) -> CollectedResourceUsage { + pub fn collect_resource_usage( + &self, + benchmark_name: &str, + subsystems_under_test: &[&str], + ) -> CollectedResourceUsage { CollectedResourceUsage { - benchmark: "Subsystem benchmarks".to_string(), + benchmark_name: benchmark_name.to_string(), network: self.network_usage(), cpu: self.cpu_usage(subsystems_under_test), } @@ -388,7 +392,7 @@ impl TestEnvironment { #[derive(Debug)] pub struct CollectedResourceUsage { - benchmark: String, + benchmark_name: String, network: Vec, cpu: Vec, } @@ -398,7 +402,7 @@ impl std::fmt::Display for CollectedResourceUsage { write!( f, "\n{}\n\n{}\n{}\n\n{}\n{}\n", - self.benchmark.blue(), + self.benchmark_name.purple(), format!("{:<32}{:>12}{:>12}", "Network usage, KiB", "total", "per block").blue(), self.network.iter().map(|v| v.to_string()).collect::>().join("\n"), format!("{:<32}{:>12}{:>12}", "CPU usage, s", "total", "per block").blue(), diff --git a/polkadot/node/subsystem-bench/src/subsystem-bench.rs b/polkadot/node/subsystem-bench/src/subsystem-bench.rs index 8633ebb703aa..11723a46d253 100644 --- a/polkadot/node/subsystem-bench/src/subsystem-bench.rs +++ b/polkadot/node/subsystem-bench/src/subsystem-bench.rs @@ -163,6 +163,7 @@ impl BenchCli { format!("Sequence contains {} step(s)", num_steps).bright_purple() ); for (index, test_config) in test_sequence.into_iter().enumerate() { + let benchmark_name = format!("{} #{}", &options.path, index + 1); gum::info!(target: LOG_TARGET, "{}", format!("Step {}/{}", index + 1, num_steps).bright_purple(),); display_configuration(&test_config); @@ -171,14 +172,18 @@ impl BenchCli { let mut state = TestState::new(&test_config); let (mut env, _protocol_config) = prepare_test(test_config, &mut state); env.runtime().block_on(availability::benchmark_availability_read( - &mut env, state, + &benchmark_name, + &mut env, + state, )); }, TestObjective::DataAvailabilityWrite => { let mut state = TestState::new(&test_config); let (mut env, _protocol_config) = prepare_test(test_config, &mut state); env.runtime().block_on(availability::benchmark_availability_write( - &mut env, state, + &benchmark_name, + &mut env, + state, )); }, _ => gum::error!("Invalid test objective in sequence"), @@ -224,12 +229,18 @@ impl BenchCli { match self.objective { TestObjective::DataAvailabilityRead(_options) => { - env.runtime() - .block_on(availability::benchmark_availability_read(&mut env, state)); + env.runtime().block_on(availability::benchmark_availability_read( + "benchmark_availability_read", + &mut env, + state, + )); }, TestObjective::DataAvailabilityWrite => { - env.runtime() - .block_on(availability::benchmark_availability_write(&mut env, state)); + env.runtime().block_on(availability::benchmark_availability_write( + "benchmark_availability_write", + &mut env, + state, + )); }, TestObjective::TestSequence(_options) => {}, } From 5a94ecf7e4fe69cc91fdbd89295b14d040cf401a Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Wed, 31 Jan 2024 16:02:26 +0100 Subject: [PATCH 3/9] Address clippy errors --- polkadot/node/subsystem-bench/src/core/environment.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polkadot/node/subsystem-bench/src/core/environment.rs b/polkadot/node/subsystem-bench/src/core/environment.rs index 5a28b00ab459..9f6c3e9aca5b 100644 --- a/polkadot/node/subsystem-bench/src/core/environment.rs +++ b/polkadot/node/subsystem-bench/src/core/environment.rs @@ -354,7 +354,7 @@ impl TestEnvironment { }, ResourceUsage { resource: "Sent to peers".to_string(), - total: total_node_sent as f64, + total: total_node_sent, per_block: total_node_sent / num_blocks, }, ] From 378bb1d49f17ebc0add56c244bbad83251cac00e Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Wed, 31 Jan 2024 16:07:01 +0100 Subject: [PATCH 4/9] Address fmt issues --- .../subsystem-bench/src/availability/mod.rs | 24 +++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/polkadot/node/subsystem-bench/src/availability/mod.rs b/polkadot/node/subsystem-bench/src/availability/mod.rs index dfe71f51013b..45e31820fe1b 100644 --- a/polkadot/node/subsystem-bench/src/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/availability/mod.rs @@ -417,7 +417,11 @@ impl TestState { } } -pub async fn benchmark_availability_read(benchmark_name: &str, env: &mut TestEnvironment, mut state: TestState) { +pub async fn benchmark_availability_read( + benchmark_name: &str, + env: &mut TestEnvironment, + mut state: TestState, +) { let config = env.config().clone(); env.import_block(new_block_import_info(Hash::repeat_byte(1), 1)).await; @@ -481,7 +485,11 @@ pub async fn benchmark_availability_read(benchmark_name: &str, env: &mut TestEnv env.stop().await; } -pub async fn benchmark_availability_write(benchmark_name: &str, env: &mut TestEnvironment, mut state: TestState) { +pub async fn benchmark_availability_write( + benchmark_name: &str, + env: &mut TestEnvironment, + mut state: TestState, +) { let config = env.config().clone(); env.metrics().set_n_validators(config.n_validators); @@ -633,11 +641,13 @@ pub async fn benchmark_availability_write(benchmark_name: &str, env: &mut TestEn format!("{} ms", test_start.elapsed().as_millis() / env.config().num_blocks as u128).red() ); - println!("{}", env.collect_resource_usage(benchmark_name, &[ - "availability-distribution", - "bitfield-distribution", - "availability-store", - ])); + println!( + "{}", + env.collect_resource_usage( + benchmark_name, + &["availability-distribution", "bitfield-distribution", "availability-store",] + ) + ); env.stop().await; } From 518c3a04db834b54726da39c18445595b200346c Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Thu, 1 Feb 2024 10:18:13 +0100 Subject: [PATCH 5/9] Prepare ci output --- .../subsystem-bench/src/availability/mod.rs | 22 +++++----- .../subsystem-bench/src/core/environment.rs | 5 ++- .../subsystem-bench/src/subsystem-bench.rs | 44 +++++++++++++------ 3 files changed, 44 insertions(+), 27 deletions(-) diff --git a/polkadot/node/subsystem-bench/src/availability/mod.rs b/polkadot/node/subsystem-bench/src/availability/mod.rs index 45e31820fe1b..112942833095 100644 --- a/polkadot/node/subsystem-bench/src/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/availability/mod.rs @@ -13,7 +13,10 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -use crate::{core::mock::ChainApiState, TestEnvironment}; +use crate::{ + core::{environment::CollectedResourceUsage, mock::ChainApiState}, + TestEnvironment, +}; use av_store::NetworkAvailabilityState; use bitvec::bitvec; use colored::Colorize; @@ -421,7 +424,7 @@ pub async fn benchmark_availability_read( benchmark_name: &str, env: &mut TestEnvironment, mut state: TestState, -) { +) -> CollectedResourceUsage { let config = env.config().clone(); env.import_block(new_block_import_info(Hash::repeat_byte(1), 1)).await; @@ -481,15 +484,15 @@ pub async fn benchmark_availability_read( format!("{} ms", test_start.elapsed().as_millis() / env.config().num_blocks as u128).red() ); - println!("{}", env.collect_resource_usage(benchmark_name, &["availability-recovery"])); env.stop().await; + env.collect_resource_usage(benchmark_name, &["availability-recovery"]) } pub async fn benchmark_availability_write( benchmark_name: &str, env: &mut TestEnvironment, mut state: TestState, -) { +) -> CollectedResourceUsage { let config = env.config().clone(); env.metrics().set_n_validators(config.n_validators); @@ -641,14 +644,11 @@ pub async fn benchmark_availability_write( format!("{} ms", test_start.elapsed().as_millis() / env.config().num_blocks as u128).red() ); - println!( - "{}", - env.collect_resource_usage( - benchmark_name, - &["availability-distribution", "bitfield-distribution", "availability-store",] - ) - ); env.stop().await; + env.collect_resource_usage( + benchmark_name, + &["availability-distribution", "bitfield-distribution", "availability-store"], + ) } pub fn peer_bitfield_message_v2( diff --git a/polkadot/node/subsystem-bench/src/core/environment.rs b/polkadot/node/subsystem-bench/src/core/environment.rs index 9f6c3e9aca5b..b034db9b1dcb 100644 --- a/polkadot/node/subsystem-bench/src/core/environment.rs +++ b/polkadot/node/subsystem-bench/src/core/environment.rs @@ -22,6 +22,7 @@ use colored::Colorize; use core::time::Duration; use futures::{Future, FutureExt}; use polkadot_overseer::{BlockInfo, Handle as OverseerHandle}; +use serde::{Deserialize, Serialize}; use polkadot_node_subsystem::{messages::AllMessages, Overseer, SpawnGlue, TimeoutExt}; use polkadot_node_subsystem_types::Hash; @@ -390,7 +391,7 @@ impl TestEnvironment { } } -#[derive(Debug)] +#[derive(Debug, Serialize, Deserialize)] pub struct CollectedResourceUsage { benchmark_name: String, network: Vec, @@ -411,7 +412,7 @@ impl std::fmt::Display for CollectedResourceUsage { } } -#[derive(Debug)] +#[derive(Debug, Serialize, Deserialize)] pub struct ResourceUsage { resource: String, total: f64, diff --git a/polkadot/node/subsystem-bench/src/subsystem-bench.rs b/polkadot/node/subsystem-bench/src/subsystem-bench.rs index 11723a46d253..ad8ac426d4ed 100644 --- a/polkadot/node/subsystem-bench/src/subsystem-bench.rs +++ b/polkadot/node/subsystem-bench/src/subsystem-bench.rs @@ -99,6 +99,10 @@ struct BenchCli { /// Enable Cache Misses Profiling with Valgrind. Linux only, Valgrind must be in the PATH pub cache_misses: bool, + #[clap(long, default_value_t = false)] + /// Shows the output in YAML format + pub ci: bool, + #[command(subcommand)] pub objective: cli::TestObjective, } @@ -167,7 +171,7 @@ impl BenchCli { gum::info!(target: LOG_TARGET, "{}", format!("Step {}/{}", index + 1, num_steps).bright_purple(),); display_configuration(&test_config); - match test_config.objective { + let usage = match test_config.objective { TestObjective::DataAvailabilityRead(ref _opts) => { let mut state = TestState::new(&test_config); let (mut env, _protocol_config) = prepare_test(test_config, &mut state); @@ -175,7 +179,7 @@ impl BenchCli { &benchmark_name, &mut env, state, - )); + )) }, TestObjective::DataAvailabilityWrite => { let mut state = TestState::new(&test_config); @@ -184,11 +188,22 @@ impl BenchCli { &benchmark_name, &mut env, state, - )); + )) + }, + _ => { + gum::error!("Invalid test objective in sequence"); + continue; }, - _ => gum::error!("Invalid test objective in sequence"), - } + }; + + let output = if self.ci { + serde_yaml::to_string(&vec![usage])? + } else { + usage.to_string() + }; + println!("{}", output); } + return Ok(()) }, TestObjective::DataAvailabilityRead(ref _options) => self.create_test_configuration(), @@ -227,29 +242,30 @@ impl BenchCli { let mut state = TestState::new(&test_config); let (mut env, _protocol_config) = prepare_test(test_config, &mut state); - match self.objective { - TestObjective::DataAvailabilityRead(_options) => { + let usage = match self.objective { + TestObjective::DataAvailabilityRead(_options) => env.runtime().block_on(availability::benchmark_availability_read( "benchmark_availability_read", &mut env, state, - )); - }, - TestObjective::DataAvailabilityWrite => { + )), + TestObjective::DataAvailabilityWrite => env.runtime().block_on(availability::benchmark_availability_write( "benchmark_availability_write", &mut env, state, - )); - }, - TestObjective::TestSequence(_options) => {}, - } + )), + TestObjective::TestSequence(_options) => todo!(), + }; if let Some(agent_running) = agent_running { let agent_ready = agent_running.stop()?; agent_ready.shutdown(); } + let output = if self.ci { serde_yaml::to_string(&vec![usage])? } else { usage.to_string() }; + println!("{}", output); + Ok(()) } } From e4265c2cf8d0ec18d2ba0bba1409f7660c146d84 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Fri, 2 Feb 2024 08:45:36 +0100 Subject: [PATCH 6/9] Rename structs --- .../subsystem-bench/src/availability/mod.rs | 6 ++-- .../subsystem-bench/src/core/environment.rs | 36 ++++++++++--------- .../subsystem-bench/src/subsystem-bench.rs | 7 ++-- 3 files changed, 27 insertions(+), 22 deletions(-) diff --git a/polkadot/node/subsystem-bench/src/availability/mod.rs b/polkadot/node/subsystem-bench/src/availability/mod.rs index 112942833095..2d33f19530d5 100644 --- a/polkadot/node/subsystem-bench/src/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/availability/mod.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . use crate::{ - core::{environment::CollectedResourceUsage, mock::ChainApiState}, + core::{environment::BenchmarkUsage, mock::ChainApiState}, TestEnvironment, }; use av_store::NetworkAvailabilityState; @@ -424,7 +424,7 @@ pub async fn benchmark_availability_read( benchmark_name: &str, env: &mut TestEnvironment, mut state: TestState, -) -> CollectedResourceUsage { +) -> BenchmarkUsage { let config = env.config().clone(); env.import_block(new_block_import_info(Hash::repeat_byte(1), 1)).await; @@ -492,7 +492,7 @@ pub async fn benchmark_availability_write( benchmark_name: &str, env: &mut TestEnvironment, mut state: TestState, -) -> CollectedResourceUsage { +) -> BenchmarkUsage { let config = env.config().clone(); env.metrics().set_n_validators(config.n_validators); diff --git a/polkadot/node/subsystem-bench/src/core/environment.rs b/polkadot/node/subsystem-bench/src/core/environment.rs index b034db9b1dcb..1edb2bea6c90 100644 --- a/polkadot/node/subsystem-bench/src/core/environment.rs +++ b/polkadot/node/subsystem-bench/src/core/environment.rs @@ -333,11 +333,11 @@ impl TestEnvironment { &self, benchmark_name: &str, subsystems_under_test: &[&str], - ) -> CollectedResourceUsage { - CollectedResourceUsage { + ) -> BenchmarkUsage { + BenchmarkUsage { benchmark_name: benchmark_name.to_string(), - network: self.network_usage(), - cpu: self.cpu_usage(subsystems_under_test), + network_usage: self.network_usage(), + cpu_usage: self.cpu_usage(subsystems_under_test), } } @@ -349,12 +349,12 @@ impl TestEnvironment { vec![ ResourceUsage { - resource: "Received from peers".to_string(), + resource_name: "Received from peers".to_string(), total: total_node_received, per_block: total_node_received / num_blocks, }, ResourceUsage { - resource: "Sent to peers".to_string(), + resource_name: "Sent to peers".to_string(), total: total_node_sent, per_block: total_node_sent / num_blocks, }, @@ -371,7 +371,7 @@ impl TestEnvironment { test_metrics.subset_with_label_value("task_group", subsystem); let total_cpu = subsystem_cpu_metrics.sum_by("substrate_tasks_polling_duration_sum"); usage.push(ResourceUsage { - resource: subsystem.to_string(), + resource_name: subsystem.to_string(), total: total_cpu, per_block: total_cpu / num_blocks, }); @@ -382,7 +382,7 @@ impl TestEnvironment { let total_cpu = test_env_cpu_metrics.sum_by("substrate_tasks_polling_duration_sum"); usage.push(ResourceUsage { - resource: "Test environment".to_string(), + resource_name: "Test environment".to_string(), total: total_cpu, per_block: total_cpu / num_blocks, }); @@ -392,35 +392,39 @@ impl TestEnvironment { } #[derive(Debug, Serialize, Deserialize)] -pub struct CollectedResourceUsage { +pub struct BenchmarkUsage { benchmark_name: String, - network: Vec, - cpu: Vec, + network_usage: Vec, + cpu_usage: Vec, } -impl std::fmt::Display for CollectedResourceUsage { +impl std::fmt::Display for BenchmarkUsage { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { write!( f, "\n{}\n\n{}\n{}\n\n{}\n{}\n", self.benchmark_name.purple(), format!("{:<32}{:>12}{:>12}", "Network usage, KiB", "total", "per block").blue(), - self.network.iter().map(|v| v.to_string()).collect::>().join("\n"), + self.network_usage + .iter() + .map(|v| v.to_string()) + .collect::>() + .join("\n"), format!("{:<32}{:>12}{:>12}", "CPU usage, s", "total", "per block").blue(), - self.cpu.iter().map(|v| v.to_string()).collect::>().join("\n") + self.cpu_usage.iter().map(|v| v.to_string()).collect::>().join("\n") ) } } #[derive(Debug, Serialize, Deserialize)] pub struct ResourceUsage { - resource: String, + resource_name: String, total: f64, per_block: f64, } impl std::fmt::Display for ResourceUsage { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - write!(f, "{:<32}{:>12.3}{:>12.3}", self.resource.cyan(), self.total, self.per_block) + write!(f, "{:<32}{:>12.3}{:>12.3}", self.resource_name.cyan(), self.total, self.per_block) } } diff --git a/polkadot/node/subsystem-bench/src/subsystem-bench.rs b/polkadot/node/subsystem-bench/src/subsystem-bench.rs index ad8ac426d4ed..03ee047406fb 100644 --- a/polkadot/node/subsystem-bench/src/subsystem-bench.rs +++ b/polkadot/node/subsystem-bench/src/subsystem-bench.rs @@ -101,7 +101,7 @@ struct BenchCli { #[clap(long, default_value_t = false)] /// Shows the output in YAML format - pub ci: bool, + pub yaml_output: bool, #[command(subcommand)] pub objective: cli::TestObjective, @@ -196,7 +196,7 @@ impl BenchCli { }, }; - let output = if self.ci { + let output = if self.yaml_output { serde_yaml::to_string(&vec![usage])? } else { usage.to_string() @@ -263,7 +263,8 @@ impl BenchCli { agent_ready.shutdown(); } - let output = if self.ci { serde_yaml::to_string(&vec![usage])? } else { usage.to_string() }; + let output = + if self.yaml_output { serde_yaml::to_string(&vec![usage])? } else { usage.to_string() }; println!("{}", output); Ok(()) From 88e082a80da7d32163b543e9327e7cf3114d1768 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Tue, 6 Feb 2024 15:12:38 +0100 Subject: [PATCH 7/9] Fix --- .../node/subsystem-bench/src/approval/mod.rs | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/polkadot/node/subsystem-bench/src/approval/mod.rs b/polkadot/node/subsystem-bench/src/approval/mod.rs index 3544ce74711e..055aeb193456 100644 --- a/polkadot/node/subsystem-bench/src/approval/mod.rs +++ b/polkadot/node/subsystem-bench/src/approval/mod.rs @@ -29,7 +29,9 @@ use crate::{ }, core::{ configuration::{TestAuthorities, TestConfiguration}, - environment::{TestEnvironment, TestEnvironmentDependencies, MAX_TIME_OF_FLIGHT}, + environment::{ + BenchmarkUsage, TestEnvironment, TestEnvironmentDependencies, MAX_TIME_OF_FLIGHT, + }, mock::{ dummy_builder, network_bridge::{MockNetworkBridgeRx, MockNetworkBridgeTx}, @@ -876,7 +878,11 @@ fn prepare_test_inner( ) } -pub async fn bench_approvals(env: &mut TestEnvironment, mut state: ApprovalTestState) { +pub async fn bench_approvals( + benchmark_name: &str, + env: &mut TestEnvironment, + mut state: ApprovalTestState, +) -> BenchmarkUsage { let producer_rx = state .start_message_production( env.network(), @@ -885,15 +891,16 @@ pub async fn bench_approvals(env: &mut TestEnvironment, mut state: ApprovalTestS env.registry().clone(), ) .await; - bench_approvals_run(env, state, producer_rx).await + bench_approvals_run(benchmark_name, env, state, producer_rx).await } /// Runs the approval benchmark. pub async fn bench_approvals_run( + benchmark_name: &str, env: &mut TestEnvironment, state: ApprovalTestState, producer_rx: oneshot::Receiver<()>, -) { +) -> BenchmarkUsage { let config = env.config().clone(); env.metrics().set_n_validators(config.n_validators); @@ -1054,6 +1061,5 @@ pub async fn bench_approvals_run( state.total_unique_messages.load(std::sync::atomic::Ordering::SeqCst) ); - env.display_network_usage(); - env.display_cpu_usage(&["approval-distribution", "approval-voting"]); + env.collect_resource_usage(benchmark_name, &["approval-distribution", "approval-voting"]) } From 46dedf3349a5b56b81f2a93291b43437cf1bf885 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Tue, 6 Feb 2024 16:42:57 +0100 Subject: [PATCH 8/9] Update polkadot/node/subsystem-bench/src/core/environment.rs Co-authored-by: Andrei Sandu <54316454+sandreim@users.noreply.github.com> --- polkadot/node/subsystem-bench/src/core/environment.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polkadot/node/subsystem-bench/src/core/environment.rs b/polkadot/node/subsystem-bench/src/core/environment.rs index 9f187cfdeeef..cffda38f2110 100644 --- a/polkadot/node/subsystem-bench/src/core/environment.rs +++ b/polkadot/node/subsystem-bench/src/core/environment.rs @@ -429,7 +429,7 @@ impl std::fmt::Display for BenchmarkUsage { .map(|v| v.to_string()) .collect::>() .join("\n"), - format!("{:<32}{:>12}{:>12}", "CPU usage, s", "total", "per block").blue(), + format!("{:<32}{:>12}{:>12}", "CPU usage in seconds", "total", "per block").blue(), self.cpu_usage.iter().map(|v| v.to_string()).collect::>().join("\n") ) } From 96bb02ea4954565d7d4a37e939db7de41affd83d Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Tue, 6 Feb 2024 17:16:35 +0100 Subject: [PATCH 9/9] Show test objective in the output --- polkadot/node/subsystem-bench/src/cli.rs | 16 ++++++++++++++ .../subsystem-bench/src/subsystem-bench.rs | 22 ++++++++----------- 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/polkadot/node/subsystem-bench/src/cli.rs b/polkadot/node/subsystem-bench/src/cli.rs index bfce8cc183a9..21f5e6a85629 100644 --- a/polkadot/node/subsystem-bench/src/cli.rs +++ b/polkadot/node/subsystem-bench/src/cli.rs @@ -40,6 +40,22 @@ pub enum TestObjective { Unimplemented, } +impl std::fmt::Display for TestObjective { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{}", + match self { + Self::DataAvailabilityRead(_) => "DataAvailabilityRead", + Self::DataAvailabilityWrite => "DataAvailabilityWrite", + Self::TestSequence(_) => "TestSequence", + Self::ApprovalVoting(_) => "ApprovalVoting", + Self::Unimplemented => "Unimplemented", + } + ) + } +} + #[derive(Debug, clap::Parser)] #[clap(rename_all = "kebab-case")] #[allow(missing_docs)] diff --git a/polkadot/node/subsystem-bench/src/subsystem-bench.rs b/polkadot/node/subsystem-bench/src/subsystem-bench.rs index 1369e2fa9bcf..433354f6525d 100644 --- a/polkadot/node/subsystem-bench/src/subsystem-bench.rs +++ b/polkadot/node/subsystem-bench/src/subsystem-bench.rs @@ -168,7 +168,8 @@ impl BenchCli { format!("Sequence contains {} step(s)", num_steps).bright_purple() ); for (index, test_config) in test_sequence.into_iter().enumerate() { - let benchmark_name = format!("{} #{}", &options.path, index + 1); + let benchmark_name = + format!("{} #{} {}", &options.path, index + 1, test_config.objective); gum::info!(target: LOG_TARGET, "{}", format!("Step {}/{}", index + 1, num_steps).bright_purple(),); display_configuration(&test_config); @@ -252,19 +253,14 @@ impl BenchCli { let mut state = TestState::new(&test_config); let (mut env, _protocol_config) = prepare_test(test_config, &mut state); + let benchmark_name = format!("{}", self.objective); let usage = match self.objective { - TestObjective::DataAvailabilityRead(_options) => - env.runtime().block_on(availability::benchmark_availability_read( - "benchmark_availability_read", - &mut env, - state, - )), - TestObjective::DataAvailabilityWrite => - env.runtime().block_on(availability::benchmark_availability_write( - "benchmark_availability_write", - &mut env, - state, - )), + TestObjective::DataAvailabilityRead(_options) => env.runtime().block_on( + availability::benchmark_availability_read(&benchmark_name, &mut env, state), + ), + TestObjective::DataAvailabilityWrite => env.runtime().block_on( + availability::benchmark_availability_write(&benchmark_name, &mut env, state), + ), TestObjective::TestSequence(_options) => todo!(), TestObjective::ApprovalVoting(_) => todo!(), TestObjective::Unimplemented => todo!(),