diff --git a/Cargo.lock b/Cargo.lock index 0a14ca9a653a..d443bdf97ed9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5865,9 +5865,11 @@ dependencies = [ "futures 0.3.18", "log", "polkadot-node-core-pvf", + "polkadot-performance-test", "polkadot-service", "sc-cli", "sc-service", + "sc-tracing", "sp-core", "sp-trie", "structopt", @@ -6540,6 +6542,20 @@ dependencies = [ "sp-std", ] +[[package]] +name = "polkadot-performance-test" +version = "0.9.13" +dependencies = [ + "env_logger 0.9.0", + "kusama-runtime", + "log", + "polkadot-erasure-coding", + "polkadot-node-core-pvf", + "polkadot-node-primitives", + "quote", + "thiserror", +] + [[package]] name = "polkadot-primitives" version = "0.9.13" diff --git a/Cargo.toml b/Cargo.toml index 9e5bd3f7b64c..2b3e51e4e83d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -84,6 +84,7 @@ members = [ "node/metrics", "node/metered-channel", "node/test/client", + "node/test/performance-test", "node/test/service", "node/test/polkadot-simnet/common", "node/test/polkadot-simnet/node", diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 8d4dff1c8e3b..5ea6f80515f0 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -21,12 +21,14 @@ futures = "0.3.17" service = { package = "polkadot-service", path = "../node/service", default-features = false, optional = true } polkadot-node-core-pvf = { path = "../node/core/pvf", optional = true } +polkadot-performance-test = { path = "../node/test/performance-test", optional = true } sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" } frame-benchmarking-cli = { git = "https://github.com/paritytech/substrate", branch = "master", optional = true } try-runtime-cli = { git = "https://github.com/paritytech/substrate", branch = "master", optional = true } sc-cli = { git = "https://github.com/paritytech/substrate", branch = "master", optional = true } sc-service = { git = "https://github.com/paritytech/substrate", branch = "master", optional = true } +sc-tracing = { git = "https://github.com/paritytech/substrate", branch = "master", optional = true } # this crate is used only to enable `trie-memory-tracker` feature # see https://github.com/paritytech/substrate/pull/6745 @@ -43,9 +45,11 @@ cli = [ "structopt", "sc-cli", "sc-service", + "sc-tracing", "frame-benchmarking-cli", "try-runtime-cli", "polkadot-node-core-pvf", + "polkadot-performance-test", ] runtime-benchmarks = [ "service/runtime-benchmarks" ] trie-memory-tracker = [ "sp-trie/memory-tracker" ] diff --git a/cli/build.rs b/cli/build.rs index a299afe20931..6eda5fb94338 100644 --- a/cli/build.rs +++ b/cli/build.rs @@ -1,4 +1,4 @@ -// Copyright 2015-2020 Parity Technologies (UK) Ltd. +// Copyright 2017-2021 Parity Technologies (UK) Ltd. // This file is part of Polkadot. // Polkadot is free software: you can redistribute it and/or modify @@ -15,5 +15,8 @@ // along with Polkadot. If not, see . fn main() { + if let Ok(profile) = std::env::var("PROFILE") { + println!("cargo:rustc-cfg=build_type=\"{}\"", profile); + } substrate_build_script_utils::generate_cargo_keys(); } diff --git a/cli/src/cli.rs b/cli/src/cli.rs index b6555db73e18..68b172892e17 100644 --- a/cli/src/cli.rs +++ b/cli/src/cli.rs @@ -54,6 +54,10 @@ pub enum Subcommand { #[structopt(name = "benchmark", about = "Benchmark runtime pallets.")] Benchmark(frame_benchmarking_cli::BenchmarkCmd), + /// Runs performance checks such as PVF compilation in order to measure machine + /// capabilities of running a validator. + HostPerfCheck, + /// Try some command against runtime state. #[cfg(feature = "try-runtime")] TryRuntime(try_runtime_cli::TryRuntimeCmd), diff --git a/cli/src/command.rs b/cli/src/command.rs index a862840cad2d..b33e1ec1d020 100644 --- a/cli/src/command.rs +++ b/cli/src/command.rs @@ -21,20 +21,8 @@ use sc_cli::{Role, RuntimeVersion, SubstrateCli}; use service::{self, IdentifyVariant}; use sp_core::crypto::Ss58AddressFormatRegistry; -#[derive(thiserror::Error, Debug)] -pub enum Error { - #[error(transparent)] - PolkadotService(#[from] service::Error), - - #[error(transparent)] - SubstrateCli(#[from] sc_cli::Error), - - #[error(transparent)] - SubstrateService(#[from] sc_service::Error), - - #[error("Other: {0}")] - Other(String), -} +pub use crate::error::Error; +pub use polkadot_performance_test::PerfCheckError; impl std::convert::From for Error { fn from(s: String) -> Self { @@ -215,6 +203,20 @@ fn ensure_dev(spec: &Box) -> std::result::Result<(), Str } } +/// Runs performance checks. +/// Should only be used in release build since the check would take too much time otherwise. +fn host_perf_check() -> Result<()> { + #[cfg(not(build_type = "release"))] + { + Err(PerfCheckError::WrongBuildType.into()) + } + #[cfg(build_type = "release")] + { + crate::host_perf_check::host_perf_check()?; + Ok(()) + } +} + /// Launch a node, accepting arguments just like a regular node, /// accepts an alternative overseer generator, to adjust behavior /// for integration tests as needed. @@ -415,6 +417,13 @@ pub fn run() -> Result<()> { #[cfg(not(feature = "polkadot-native"))] panic!("No runtime feature (polkadot, kusama, westend, rococo) is enabled") }, + Some(Subcommand::HostPerfCheck) => { + let mut builder = sc_cli::LoggerBuilder::new(""); + builder.with_colors(true); + builder.init()?; + + host_perf_check() + }, Some(Subcommand::Key(cmd)) => Ok(cmd.run(&cli)?), #[cfg(feature = "try-runtime")] Some(Subcommand::TryRuntime(cmd)) => { diff --git a/cli/src/error.rs b/cli/src/error.rs new file mode 100644 index 000000000000..92b26f51161a --- /dev/null +++ b/cli/src/error.rs @@ -0,0 +1,36 @@ +// Copyright 2017-2021 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +#[derive(thiserror::Error, Debug)] +pub enum Error { + #[error(transparent)] + PolkadotService(#[from] service::Error), + + #[error(transparent)] + SubstrateCli(#[from] sc_cli::Error), + + #[error(transparent)] + SubstrateService(#[from] sc_service::Error), + + #[error(transparent)] + SubstrateTracing(#[from] sc_tracing::logging::Error), + + #[error(transparent)] + PerfCheck(#[from] polkadot_performance_test::PerfCheckError), + + #[error("Other: {0}")] + Other(String), +} diff --git a/cli/src/host_perf_check.rs b/cli/src/host_perf_check.rs new file mode 100644 index 000000000000..23190e94712b --- /dev/null +++ b/cli/src/host_perf_check.rs @@ -0,0 +1,67 @@ +// Copyright 2017-2021 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +use log::info; +use polkadot_node_core_pvf::sp_maybe_compressed_blob; +use polkadot_performance_test::{ + measure_erasure_coding, measure_pvf_prepare, PerfCheckError, ERASURE_CODING_N_VALIDATORS, + ERASURE_CODING_TIME_LIMIT, PVF_PREPARE_TIME_LIMIT, VALIDATION_CODE_BOMB_LIMIT, +}; +use std::time::Duration; + +pub fn host_perf_check() -> Result<(), PerfCheckError> { + let wasm_code = + polkadot_performance_test::WASM_BINARY.ok_or(PerfCheckError::WasmBinaryMissing)?; + + // Decompress the code before running checks. + let code = sp_maybe_compressed_blob::decompress(wasm_code, VALIDATION_CODE_BOMB_LIMIT) + .or(Err(PerfCheckError::CodeDecompressionFailed))?; + + info!("Running the performance checks..."); + + perf_check("PVF-prepare", PVF_PREPARE_TIME_LIMIT, || measure_pvf_prepare(code.as_ref()))?; + + perf_check("Erasure-coding", ERASURE_CODING_TIME_LIMIT, || { + measure_erasure_coding(ERASURE_CODING_N_VALIDATORS, code.as_ref()) + })?; + + Ok(()) +} + +fn green_threshold(duration: Duration) -> Duration { + duration * 4 / 5 +} + +fn perf_check( + test_name: &str, + time_limit: Duration, + test: impl Fn() -> Result, +) -> Result<(), PerfCheckError> { + let elapsed = test()?; + + if elapsed < green_threshold(time_limit) { + info!("🟢 {} performance check passed, elapsed: {:?}", test_name, elapsed); + Ok(()) + } else if elapsed <= time_limit { + info!( + "🟡 {} performance check passed, {:?} limit almost exceeded, elapsed: {:?}", + test_name, time_limit, elapsed + ); + Ok(()) + } else { + Err(PerfCheckError::TimeOut { elapsed, limit: time_limit }) + } +} diff --git a/cli/src/lib.rs b/cli/src/lib.rs index c07722d97540..b4ee9f868b2e 100644 --- a/cli/src/lib.rs +++ b/cli/src/lib.rs @@ -22,6 +22,10 @@ mod cli; #[cfg(feature = "cli")] mod command; +#[cfg(feature = "cli")] +mod error; +#[cfg(all(feature = "cli", build_type = "release"))] +mod host_perf_check; #[cfg(feature = "full-node")] pub use service::RuntimeApiCollection; diff --git a/erasure-coding/src/lib.rs b/erasure-coding/src/lib.rs index 92f05fce10a4..1d2132048daa 100644 --- a/erasure-coding/src/lib.rs +++ b/erasure-coding/src/lib.rs @@ -135,7 +135,7 @@ pub fn obtain_chunks_v1(n_validators: usize, data: &AvailableData) -> Result(n_validators: usize, data: &T) -> Result>, Error> { +pub fn obtain_chunks(n_validators: usize, data: &T) -> Result>, Error> { let params = code_params(n_validators)?; let encoded = data.encode(); @@ -186,7 +186,7 @@ where /// are provided, recovery is not possible. /// /// Works only up to 65536 validators, and `n_validators` must be non-zero. -fn reconstruct<'a, I: 'a, T: Decode>(n_validators: usize, chunks: I) -> Result +pub fn reconstruct<'a, I: 'a, T: Decode>(n_validators: usize, chunks: I) -> Result where I: IntoIterator, { diff --git a/node/core/pvf/src/lib.rs b/node/core/pvf/src/lib.rs index 62d7d2c95ede..ef5f31889237 100644 --- a/node/core/pvf/src/lib.rs +++ b/node/core/pvf/src/lib.rs @@ -102,4 +102,9 @@ pub use metrics::Metrics; pub use execute::worker_entrypoint as execute_worker_entrypoint; pub use prepare::worker_entrypoint as prepare_worker_entrypoint; +pub use executor_intf::{prepare, prevalidate}; + +pub use sc_executor_common; +pub use sp_maybe_compressed_blob; + const LOG_TARGET: &str = "parachain::pvf"; diff --git a/node/test/performance-test/Cargo.toml b/node/test/performance-test/Cargo.toml new file mode 100644 index 000000000000..7b6d95441d32 --- /dev/null +++ b/node/test/performance-test/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "polkadot-performance-test" +version = "0.9.13" +authors = ["Parity Technologies "] +edition = "2018" + +[dependencies] +thiserror = "1.0.30" +quote = "1.0.10" +env_logger = "0.9" +log = "0.4" + +polkadot-node-core-pvf = { path = "../../core/pvf" } +polkadot-erasure-coding = { path = "../../../erasure-coding" } +polkadot-node-primitives = { path = "../../primitives" } + +kusama-runtime = { path = "../../../runtime/kusama" } + +[[bin]] +name = "gen-ref-constants" +path = "src/gen_ref_constants.rs" diff --git a/node/test/performance-test/build.rs b/node/test/performance-test/build.rs new file mode 100644 index 000000000000..b990c44a3143 --- /dev/null +++ b/node/test/performance-test/build.rs @@ -0,0 +1,21 @@ +// Copyright 2017-2021 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +fn main() { + if let Ok(profile) = std::env::var("PROFILE") { + println!("cargo:rustc-cfg=build_type=\"{}\"", profile); + } +} diff --git a/node/test/performance-test/src/constants.rs b/node/test/performance-test/src/constants.rs new file mode 100644 index 000000000000..0f27bb49916c --- /dev/null +++ b/node/test/performance-test/src/constants.rs @@ -0,0 +1,22 @@ +// Copyright 2017-2021 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! This file was automatically generated by `gen-ref-constants`. +//! Do not edit manually! + +use std::time::Duration; +pub const PVF_PREPARE_TIME_LIMIT: Duration = Duration::from_millis(4910u64); +pub const ERASURE_CODING_TIME_LIMIT: Duration = Duration::from_millis(466u64); diff --git a/node/test/performance-test/src/gen_ref_constants.rs b/node/test/performance-test/src/gen_ref_constants.rs new file mode 100644 index 000000000000..51327bdec693 --- /dev/null +++ b/node/test/performance-test/src/gen_ref_constants.rs @@ -0,0 +1,100 @@ +// Copyright 2021 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Generate reference performance check results. + +use polkadot_performance_test::PerfCheckError; + +fn main() -> Result<(), PerfCheckError> { + #[cfg(build_type = "release")] + { + run::run() + } + #[cfg(not(build_type = "release"))] + { + Err(PerfCheckError::WrongBuildType) + } +} + +#[cfg(build_type = "release")] +mod run { + use polkadot_node_core_pvf::sp_maybe_compressed_blob; + use polkadot_node_primitives::VALIDATION_CODE_BOMB_LIMIT; + use polkadot_performance_test::{ + measure_erasure_coding, measure_pvf_prepare, PerfCheckError, ERASURE_CODING_N_VALIDATORS, + }; + use std::{ + fs::OpenOptions, + io::{self, Write}, + time::Duration, + }; + + const WARM_UP_RUNS: usize = 16; + const FILE_HEADER: &str = include_str!("../../../../file_header.txt"); + const DOC_COMMENT: &str = "//! This file was automatically generated by `gen-ref-constants`.\n//! Do not edit manually!"; + const FILE_PATH: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/src/constants.rs"); + + fn save_constants(pvf_prepare: Duration, erasure_coding: Duration) -> io::Result<()> { + let mut output = + OpenOptions::new().truncate(true).create(true).write(true).open(FILE_PATH)?; + + writeln!(output, "{}\n\n{}\n", FILE_HEADER, DOC_COMMENT)?; + + let pvf_prepare_millis = pvf_prepare.as_millis() as u64; + let erasure_coding_millis = erasure_coding.as_millis() as u64; + + let token_stream = quote::quote! { + use std::time::Duration; + + pub const PVF_PREPARE_TIME_LIMIT: Duration = Duration::from_millis(#pvf_prepare_millis); + pub const ERASURE_CODING_TIME_LIMIT: Duration = Duration::from_millis(#erasure_coding_millis); + }; + + writeln!(output, "{}", token_stream.to_string())?; + Ok(()) + } + + pub fn run() -> Result<(), PerfCheckError> { + let _ = env_logger::builder().filter(None, log::LevelFilter::Info).try_init(); + + let wasm_code = + polkadot_performance_test::WASM_BINARY.ok_or(PerfCheckError::WasmBinaryMissing)?; + + log::info!("Running the benchmark, number of iterations: {}", WARM_UP_RUNS); + + let code = sp_maybe_compressed_blob::decompress(wasm_code, VALIDATION_CODE_BOMB_LIMIT) + .or(Err(PerfCheckError::CodeDecompressionFailed))?; + + let (pvf_prepare_time, erasure_coding_time) = (1..=WARM_UP_RUNS) + .map(|i| { + if i - 1 > 0 && (i - 1) % 5 == 0 { + log::info!("{} iterations done", i - 1); + } + ( + measure_pvf_prepare(code.as_ref()), + measure_erasure_coding(ERASURE_CODING_N_VALIDATORS, code.as_ref()), + ) + }) + .last() + .expect("`WARM_UP_RUNS` is greater than 1 and thus we have at least one element; qed"); + + save_constants(pvf_prepare_time?, erasure_coding_time?)?; + + log::info!("Successfully stored new reference values at {:?}. Make sure to format the file via `cargo +nightly fmt`", FILE_PATH); + + Ok(()) + } +} diff --git a/node/test/performance-test/src/lib.rs b/node/test/performance-test/src/lib.rs new file mode 100644 index 000000000000..e80b5e7589f2 --- /dev/null +++ b/node/test/performance-test/src/lib.rs @@ -0,0 +1,87 @@ +// Copyright 2021 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! A Polkadot performance tests utilities. + +use polkadot_erasure_coding::{obtain_chunks, reconstruct}; +use polkadot_node_core_pvf::{sc_executor_common, sp_maybe_compressed_blob}; +use std::time::{Duration, Instant}; + +mod constants; + +pub use constants::*; +pub use polkadot_node_primitives::VALIDATION_CODE_BOMB_LIMIT; + +/// Value used for reference benchmark of erasure-coding. +pub const ERASURE_CODING_N_VALIDATORS: usize = 1024; + +pub use kusama_runtime::WASM_BINARY; + +#[allow(missing_docs)] +#[derive(thiserror::Error, Debug)] +pub enum PerfCheckError { + #[error("This subcommand is only available in release mode")] + WrongBuildType, + + #[error("No wasm code found for running the performance test")] + WasmBinaryMissing, + + #[error("Failed to decompress wasm code")] + CodeDecompressionFailed, + + #[error(transparent)] + Wasm(#[from] sc_executor_common::error::WasmError), + + #[error(transparent)] + ErasureCoding(#[from] polkadot_erasure_coding::Error), + + #[error(transparent)] + Io(#[from] std::io::Error), + + #[error( + "Performance check not passed: exceeded the {limit:?} time limit, elapsed: {elapsed:?}" + )] + TimeOut { elapsed: Duration, limit: Duration }, +} + +/// Measures the time it takes to compile arbitrary wasm code. +pub fn measure_pvf_prepare(wasm_code: &[u8]) -> Result { + let start = Instant::now(); + + let code = sp_maybe_compressed_blob::decompress(wasm_code, VALIDATION_CODE_BOMB_LIMIT) + .or(Err(PerfCheckError::CodeDecompressionFailed))?; + + // Recreate the pipeline from the pvf prepare worker. + let blob = polkadot_node_core_pvf::prevalidate(code.as_ref()).map_err(PerfCheckError::from)?; + polkadot_node_core_pvf::prepare(blob).map_err(PerfCheckError::from)?; + + Ok(start.elapsed()) +} + +/// Measure the time it takes to break arbitrary data into chunks and reconstruct it back. +pub fn measure_erasure_coding( + n_validators: usize, + data: &[u8], +) -> Result { + let start = Instant::now(); + + let chunks = obtain_chunks(n_validators, &data)?; + let indexed_chunks = chunks.iter().enumerate().map(|(i, chunk)| (chunk.as_slice(), i)); + + let _: Vec = reconstruct(n_validators, indexed_chunks)?; + + Ok(start.elapsed()) +}