From 8e0d235b88622179c765651f57b286f0dbe7a17f Mon Sep 17 00:00:00 2001 From: andrejlukacovic <37964423+lukacan@users.noreply.github.com> Date: Fri, 7 Jun 2024 12:57:00 +0200 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20feat/added=20optional=20fuzzing=20s?= =?UTF-8?q?tatistics=20(#144)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Optionaly show stats during fuzzing session * 🐛 (store failed txs): Store failex txs * 🩹 Simplified stats logging * 📝 Added fuzzing stats to docs * ✨ Added failed invariants check stats --------- Co-authored-by: Ikrk --- CHANGELOG.md | 1 + Cargo.lock | 55 ++++- crates/client/Cargo.toml | 1 + .../derive/fuzz_test_executor/src/lib.rs | 52 +++-- crates/client/src/commander.rs | 221 +++++++++++++----- crates/client/src/config.rs | 143 +++++++++++- crates/client/src/fuzzer/fuzzing_stats.rs | 145 ++++++++++++ crates/client/src/fuzzer/mod.rs | 1 + crates/client/src/lib.rs | 1 + crates/client/src/templates/Trident.toml.tmpl | 3 + crates/client/src/test_generator.rs | 2 +- .../docs/fuzzing/fuzzing-run-debug.md | 37 +-- .../arbitrary-custom-types-4/Cargo.lock | 67 +++++- .../arbitrary-custom-types-4/Trident.toml | 3 + .../arbitrary-limit-inputs-5/Trident.toml | 3 + examples/fuzz-tests/hello_world/Trident.toml | 3 + .../Trident.toml | 3 + .../incorrect-ix-sequence-1/Trident.toml | 3 + .../unauthorized-access-2/Trident.toml | 3 + .../unchecked-arithmetic-0/Trident.toml | 3 + .../integration-tests/escrow/Trident.toml | 3 + .../integration-tests/turnstile/Trident.toml | 3 + 22 files changed, 660 insertions(+), 96 deletions(-) create mode 100644 crates/client/src/fuzzer/fuzzing_stats.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index c7c0f080..c84d9f5f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 incremented upon a breaking change and the patch version will be incremented for features. ## [dev] - Unreleased +- feat/fuzzer-stats-logging, an optional statistics output for fuzzing session ([#144](https://github.com/Ackee-Blockchain/trident/pull/144)) ## [0.6.0] - 2024-05-20 ### Added diff --git a/Cargo.lock b/Cargo.lock index ea513a80..fc817579 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1140,7 +1140,7 @@ version = "0.15.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb" dependencies = [ - "encode_unicode", + "encode_unicode 0.3.6", "lazy_static", "libc", "unicode-width", @@ -1273,6 +1273,27 @@ dependencies = [ "subtle", ] +[[package]] +name = "csv" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac574ff4d437a7b5ad237ef331c17ccca63c46479e5b5453eb8e10bb99a759fe" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "csv-core" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70" +dependencies = [ + "memchr", +] + [[package]] name = "ctr" version = "0.8.0" @@ -1644,6 +1665,12 @@ version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" +[[package]] +name = "encode_unicode" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" + [[package]] name = "encoding_rs" version = "0.8.33" @@ -3121,6 +3148,20 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "prettytable" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46480520d1b77c9a3482d39939fcf96831537a250ec62d4fd8fbdf8e0302e781" +dependencies = [ + "csv", + "encode_unicode 1.0.0", + "is-terminal", + "lazy_static", + "term", + "unicode-width", +] + [[package]] name = "proc-macro-crate" version = "0.1.5" @@ -5609,6 +5650,17 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "term" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f" +dependencies = [ + "dirs-next", + "rustversion", + "winapi", +] + [[package]] name = "termcolor" version = "1.4.1" @@ -6010,6 +6062,7 @@ dependencies = [ "macrotest", "pathdiff", "pretty_assertions", + "prettytable", "proc-macro2", "quinn-proto", "quote", diff --git a/crates/client/Cargo.toml b/crates/client/Cargo.toml index 6ae6cea2..4440c68f 100644 --- a/crates/client/Cargo.toml +++ b/crates/client/Cargo.toml @@ -80,3 +80,4 @@ solana-bpf-loader-program = "1.16" solana-program = "1.16" solana-sdk-macro = "1.16" solana-system-program = "1.16" +prettytable = "0.10.0" diff --git a/crates/client/derive/fuzz_test_executor/src/lib.rs b/crates/client/derive/fuzz_test_executor/src/lib.rs index 22298327..6ca24177 100644 --- a/crates/client/derive/fuzz_test_executor/src/lib.rs +++ b/crates/client/derive/fuzz_test_executor/src/lib.rs @@ -47,27 +47,47 @@ pub fn fuzz_test_executor(input: TokenStream) -> TokenStream { match duplicate_tx { Some(_) => eprintln!("\x1b[1;93mWarning\x1b[0m: Skipping duplicate instruction `{}`", self.to_context_string()), None => { + #[cfg(fuzzing_with_stats)] + let mut stats_logger = FuzzingStatistics::new(); + #[cfg(fuzzing_with_stats)] + stats_logger.increase_invoked(self.to_context_string()); + let tx_result = client.process_transaction(transaction) .map_err(|e| e.with_origin(Origin::Instruction(self.to_context_string()))); - match tx_result { - Ok(_) => { - snaphot.capture_after(client).unwrap(); - let (acc_before, acc_after) = snaphot.get_snapshot() - .map_err(|e| e.with_origin(Origin::Instruction(self.to_context_string()))) - .expect("Snapshot deserialization expect"); // we want to panic if we cannot unwrap to cause a crash + Ok(_) => { + #[cfg(fuzzing_with_stats)] + stats_logger.increase_successful(self.to_context_string()); + + snaphot.capture_after(client).unwrap(); + let (acc_before, acc_after) = snaphot.get_snapshot() + .map_err(|e| e.with_origin(Origin::Instruction(self.to_context_string()))) + .expect("Snapshot deserialization expect"); // we want to panic if we cannot unwrap to cause a crash + + if let Err(e) = ix.check(acc_before, acc_after, data).map_err(|e| e.with_origin(Origin::Instruction(self.to_context_string()))) { + #[cfg(fuzzing_with_stats)] + { + stats_logger.increase_failed_check(self.to_context_string()); + stats_logger.output_serialized(); + } + eprintln!( + "\x1b[31mCRASH DETECTED!\x1b[0m Custom check after the {} instruction did not pass!", + self.to_context_string()); + panic!("{}", e) + } + #[cfg(fuzzing_with_stats)] + stats_logger.output_serialized(); - if let Err(e) = ix.check(acc_before, acc_after, data).map_err(|e| e.with_origin(Origin::Instruction(self.to_context_string()))) { - eprintln!( - "\x1b[31mCRASH DETECTED!\x1b[0m Custom check after the {} instruction did not pass!", - self.to_context_string()); - panic!("{}", e) + }, + Err(e) => { + #[cfg(fuzzing_with_stats)] + { + stats_logger.increase_failed(self.to_context_string()); + stats_logger.output_serialized(); + } + let mut raw_accounts = snaphot.get_raw_pre_ix_accounts(); + ix.tx_error_handler(e, data, &mut raw_accounts)? } - }, - Err(e) => { - let mut raw_accounts = snaphot.get_raw_pre_ix_accounts(); - ix.tx_error_handler(e, data, &mut raw_accounts)? - } } } } diff --git a/crates/client/src/commander.rs b/crates/client/src/commander.rs index a2346ffd..b5a3c923 100644 --- a/crates/client/src/commander.rs +++ b/crates/client/src/commander.rs @@ -19,6 +19,8 @@ use tokio::{ }; use crate::constants::*; +use crate::fuzzing_stats::FuzzingStatistics; +use tokio::io::AsyncBufReadExt; #[derive(Error, Debug)] pub enum Error { @@ -153,7 +155,11 @@ impl Commander { // arguments so we need to parse the variable content. let hfuzz_run_args = std::env::var("HFUZZ_RUN_ARGS").unwrap_or_default(); - let fuzz_args = config.get_honggfuzz_args(hfuzz_run_args); + let rustflags = std::env::var("RUSTFLAGS").unwrap_or_default(); + + let rustflags = config.get_rustflags_args(rustflags); + + let mut fuzz_args = config.get_honggfuzz_args(hfuzz_run_args); // let cargo_target_dir = std::env::var("CARGO_TARGET_DIR").unwrap_or_default(); @@ -181,36 +187,34 @@ impl Commander { } } - let mut rustflags = if config.fuzz.allow_duplicate_txs { - "--cfg allow_duplicate_txs " - } else { - "" - } - .to_string(); - - rustflags.push_str(&std::env::var("RUSTFLAGS").unwrap_or_default()); - - let mut child = Command::new("cargo") - .env("HFUZZ_RUN_ARGS", fuzz_args) - .env("CARGO_TARGET_DIR", cargo_target_dir) - .env("HFUZZ_WORKSPACE", hfuzz_workspace) - .env("RUSTFLAGS", rustflags) - .arg("hfuzz") - .arg("run") - .arg(target) - .spawn()?; - - tokio::select! { - res = child.wait() => - match res { - Ok(status) => if !status.success() { - println!("Honggfuzz exited with an error!"); - }, - Err(_) => throw!(Error::FuzzingFailed), - }, - _ = signal::ctrl_c() => { - tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; - }, + match rustflags.contains("fuzzing_with_stats") { + true => { + // enforce keep output to be true + fuzz_args.push_str("--keep_output"); + let mut child = Command::new("cargo") + .env("HFUZZ_RUN_ARGS", fuzz_args) + .env("CARGO_TARGET_DIR", cargo_target_dir) + .env("HFUZZ_WORKSPACE", hfuzz_workspace) + .env("RUSTFLAGS", rustflags) + .arg("hfuzz") + .arg("run") + .arg(target) + .stdout(Stdio::piped()) + .spawn()?; + Self::handle_child_with_stats(&mut child).await?; + } + false => { + let mut child = Command::new("cargo") + .env("HFUZZ_RUN_ARGS", fuzz_args) + .env("CARGO_TARGET_DIR", cargo_target_dir) + .env("HFUZZ_WORKSPACE", hfuzz_workspace) + .env("RUSTFLAGS", rustflags) + .arg("hfuzz") + .arg("run") + .arg(target) + .spawn()?; + Self::handle_child(&mut child).await?; + } } if let Ok(crash_files) = get_crash_files(&crash_dir, &ext) { @@ -236,27 +240,54 @@ impl Commander { let hfuzz_workspace = std::env::var("HFUZZ_WORKSPACE") .unwrap_or_else(|_| config.get_env_arg("HFUZZ_WORKSPACE")); - let fuzz_args = config.get_honggfuzz_args(hfuzz_run_args); - - let mut rustflags = if config.fuzz.allow_duplicate_txs { - "--cfg allow_duplicate_txs " - } else { - "" + let mut fuzz_args = config.get_honggfuzz_args(hfuzz_run_args); + + let rustflags = std::env::var("RUSTFLAGS").unwrap_or_default(); + + let rustflags = config.get_rustflags_args(rustflags); + + match rustflags.contains("fuzzing_with_stats") { + true => { + // enforce keep output to be true + fuzz_args.push_str("--keep_output"); + let mut child = Command::new("cargo") + .env("HFUZZ_RUN_ARGS", fuzz_args) + .env("CARGO_TARGET_DIR", cargo_target_dir) + .env("HFUZZ_WORKSPACE", hfuzz_workspace) + .env("RUSTFLAGS", rustflags) + .arg("hfuzz") + .arg("run") + .arg(target) + .stdout(Stdio::piped()) + .spawn()?; + Self::handle_child_with_stats(&mut child).await?; + } + false => { + let mut child = Command::new("cargo") + .env("HFUZZ_RUN_ARGS", fuzz_args) + .env("CARGO_TARGET_DIR", cargo_target_dir) + .env("HFUZZ_WORKSPACE", hfuzz_workspace) + .env("RUSTFLAGS", rustflags) + .arg("hfuzz") + .arg("run") + .arg(target) + .spawn()?; + Self::handle_child(&mut child).await?; + } } - .to_string(); - - rustflags.push_str(&std::env::var("RUSTFLAGS").unwrap_or_default()); - - let mut child = Command::new("cargo") - .env("HFUZZ_RUN_ARGS", fuzz_args) - .env("CARGO_TARGET_DIR", cargo_target_dir) - .env("HFUZZ_WORKSPACE", hfuzz_workspace) - .env("RUSTFLAGS", rustflags) - .arg("hfuzz") - .arg("run") - .arg(target) - .spawn()?; + } + /// Manages a child process in an async context, specifically for monitoring fuzzing tasks. + /// Waits for the process to exit or a Ctrl+C signal. Prints an error message if the process + /// exits with an error, and sleeps briefly on Ctrl+C. Throws `Error::FuzzingFailed` on errors. + /// + /// # Arguments + /// * `child` - A mutable reference to a `Child` process. + /// + /// # Errors + /// * Throws `Error::FuzzingFailed` if waiting on the child process fails. + #[throws] + async fn handle_child(child: &mut Child) { tokio::select! { res = child.wait() => match res { @@ -270,6 +301,87 @@ impl Commander { }, } } + /// Asynchronously manages a child fuzzing process, collecting and logging its statistics. + /// This function spawns a new task dedicated to reading the process's standard output and logging the fuzzing statistics. + /// It waits for either the child process to exit or a Ctrl+C signal to be received. Upon process exit or Ctrl+C signal, + /// it stops the logging task and displays the collected statistics in a table format. + /// + /// The implementation ensures that the statistics logging task only stops after receiving a signal indicating the end of the fuzzing process + /// or an interrupt from the user, preventing premature termination of the logging task if scenarios where reading is faster than fuzzing, + /// which should not be common. + /// + /// # Arguments + /// * `child` - A mutable reference to a `Child` process, representing the child fuzzing process. + /// + /// # Errors + /// * `Error::FuzzingFailed` - Thrown if there's an issue with managing the child process, such as failing to wait on the child process. + #[throws] + async fn handle_child_with_stats(child: &mut Child) { + let stdout = child + .stdout + .take() + .expect("child did not have a handle to stdout"); + + let reader = tokio::io::BufReader::new(stdout); + + let fuzz_end = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let fuzz_end_clone = std::sync::Arc::clone(&fuzz_end); + + let stats_handle: tokio::task::JoinHandle> = + tokio::spawn(async move { + let mut stats_logger = FuzzingStatistics::new(); + + let mut lines = reader.lines(); + loop { + let _line = lines.next_line().await; + match _line { + Ok(__line) => match __line { + Some(content) => { + stats_logger.insert_serialized(&content); + } + None => { + if fuzz_end_clone.load(std::sync::atomic::Ordering::SeqCst) { + break; + } + } + }, + Err(e) => return Err(e), + } + } + Ok(stats_logger) + }); + + tokio::select! { + res = child.wait() =>{ + fuzz_end.store(true, std::sync::atomic::Ordering::SeqCst); + + match res { + Ok(status) => { + if !status.success() { + println!("Honggfuzz exited with an error!"); + } + }, + Err(_) => throw!(Error::FuzzingFailed), + } + }, + _ = signal::ctrl_c() => { + fuzz_end.store(true, std::sync::atomic::Ordering::SeqCst); + tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; + + }, + } + let stats_result = stats_handle + .await + .expect("Unable to obtain Statistics Handle"); + match stats_result { + Ok(stats_result) => { + stats_result.show_table(); + } + Err(e) => { + println!("Statistics thread exited with the Error: {}", e); + } + } + } /// Runs fuzzer on the given target. #[throws] @@ -286,14 +398,9 @@ impl Commander { let cargo_target_dir = std::env::var("CARGO_TARGET_DIR") .unwrap_or_else(|_| config.get_env_arg("CARGO_TARGET_DIR")); - let mut rustflags = if config.fuzz.allow_duplicate_txs { - "--cfg allow_duplicate_txs " - } else { - "" - } - .to_string(); + let rustflags = std::env::var("RUSTFLAGS").unwrap_or_default(); - rustflags.push_str(&std::env::var("RUSTFLAGS").unwrap_or_default()); + let rustflags = config.get_rustflags_args(rustflags); // using exec rather than spawn and replacing current process to avoid unflushed terminal output after ctrl+c signal std::process::Command::new("cargo") diff --git a/crates/client/src/config.rs b/crates/client/src/config.rs index 20ece1fd..1df19f12 100644 --- a/crates/client/src/config.rs +++ b/crates/client/src/config.rs @@ -42,20 +42,44 @@ impl From<_Test> for Test { } } -#[derive(Debug, Deserialize, Clone, Default)] +#[derive(Debug, Deserialize, Clone)] +pub struct Cfg { + pub cfg_identifier: String, + pub val: bool, +} + +#[derive(Debug, Deserialize, Clone)] pub struct Fuzz { - pub allow_duplicate_txs: bool, + pub rust_flags: Vec, } #[derive(Default, Debug, Deserialize, Clone)] struct _Fuzz { #[serde(default)] pub allow_duplicate_txs: Option, + #[serde(default)] + pub fuzzing_with_stats: Option, } impl From<_Fuzz> for Fuzz { - fn from(_t: _Fuzz) -> Self { - Self { - allow_duplicate_txs: _t.allow_duplicate_txs.unwrap_or(false), - } + fn from(_f: _Fuzz) -> Self { + let mut _self = Self { rust_flags: vec![] }; + + // allow_duplicate_txs + let allow_duplicate_txs = _f.allow_duplicate_txs.unwrap_or(false); + + _self.rust_flags.push(Cfg { + cfg_identifier: "allow_duplicate_txs".to_string(), + val: allow_duplicate_txs, + }); + + // fuzzing_with_stats + let fuzzing_with_stats = _f.fuzzing_with_stats.unwrap_or(false); + + _self.rust_flags.push(Cfg { + cfg_identifier: "fuzzing_with_stats".to_string(), + val: fuzzing_with_stats, + }); + + _self } } #[derive(Debug, Deserialize, Clone)] @@ -367,6 +391,22 @@ impl Config { args.push(cli_input); args.join(" ") } + pub fn get_rustflags_args(&self, cli_input: String) -> String { + let mut args: Vec = self + .fuzz + .rust_flags + .iter() + .map(|arg| { + if arg.val { + format!("--cfg {}", arg.cfg_identifier) + } else { + "".to_string() + } + }) + .collect(); + args.push(cli_input); + args.join(" ") + } pub fn get_env_arg(&self, env_variable: &str) -> String { let expect = format!("{env_variable} not found"); self.honggfuzz @@ -404,6 +444,23 @@ mod tests { } } + impl Default for Fuzz { + fn default() -> Self { + let rust_flags = vec![ + Cfg { + cfg_identifier: "allow_duplicate_txs".to_string(), + val: false, + }, + Cfg { + cfg_identifier: "fuzzing_with_stats".to_string(), + val: false, + }, + ]; + + Self { rust_flags } + } + } + use super::*; #[test] fn test_merge_and_precedence1() { @@ -514,4 +571,78 @@ mod tests { let hfuzz_workspace = config.get_env_arg(HFUZZ_WORKSPACE_ENV); assert_eq!(hfuzz_workspace, "new_value_y"); } + + #[test] + fn test_obtain_rustflags_variable1() { + let config = Config { + test: Test::default(), + honggfuzz: HonggFuzz::default(), + fuzz: Fuzz::default(), + }; + + let rustflags = config.get_rustflags_args("".to_string()); + let default_rustflags = " "; + + assert_eq!(rustflags, default_rustflags); + } + #[test] + fn test_obtain_rustflags_variable2() { + let config = Config { + test: Test::default(), + honggfuzz: HonggFuzz::default(), + fuzz: Fuzz { + rust_flags: vec![Cfg { + cfg_identifier: "fuzzing_with_stats".to_string(), + val: true, + }], + }, + }; + + let rustflags = config.get_rustflags_args("".to_string()); + let reference_rustflags = "--cfg fuzzing_with_stats "; + + assert_eq!(rustflags, reference_rustflags); + } + #[test] + fn test_obtain_rustflags_variable3() { + let config = Config { + test: Test::default(), + honggfuzz: HonggFuzz::default(), + fuzz: Fuzz { + rust_flags: vec![ + Cfg { + cfg_identifier: "allow_duplicate_txs".to_string(), + val: true, + }, + Cfg { + cfg_identifier: "fuzzing_with_stats".to_string(), + val: false, + }, + ], + }, + }; + + let rustflags = config.get_rustflags_args("".to_string()); + let reference_rustflags = "--cfg allow_duplicate_txs "; + + assert_eq!(rustflags, reference_rustflags); + } + #[test] + fn test_obtain_rustflags_variable4() { + let config = Config { + test: Test::default(), + honggfuzz: HonggFuzz::default(), + fuzz: Fuzz { + rust_flags: vec![Cfg { + cfg_identifier: "allow_duplicate_txs".to_string(), + val: true, + }], + }, + }; + + let rustflags = config.get_rustflags_args("--cfg fuzzing_with_stats".to_string()); + let reference_rustflags = "--cfg allow_duplicate_txs --cfg fuzzing_with_stats"; + + assert_eq!(rustflags, reference_rustflags); + } } diff --git a/crates/client/src/fuzzer/fuzzing_stats.rs b/crates/client/src/fuzzer/fuzzing_stats.rs new file mode 100644 index 00000000..f49403de --- /dev/null +++ b/crates/client/src/fuzzer/fuzzing_stats.rs @@ -0,0 +1,145 @@ +use prettytable::{row, Table}; +use std::collections::HashMap; + +/// Represents fuzzing statistics, specifically tracking the number of times +/// an instruction was invoked and successfully executed. +#[derive(Debug, serde::Serialize, serde::Deserialize)] +pub struct IterationStats { + pub invoked: u64, + pub successful: u64, + pub failed: u64, + pub failed_check: u64, +} + +/// Manages and aggregates statistics for fuzzing instructions. +#[derive(Debug, Default)] +pub struct FuzzingStatistics { + pub instructions: HashMap, +} + +impl FuzzingStatistics { + /// Constructs a new, empty `FuzzingStatistics`. + pub fn new() -> Self { + let empty_instructions = HashMap::::default(); + Self { + instructions: empty_instructions, + } + } + /// Outputs the statistics as a serialized JSON string. + pub fn output_serialized(&self) { + let serialized = serde_json::to_string(&self.instructions).unwrap(); + println!("{}", serialized); + } + + /// Increments the invocation count for a given instruction. + /// # Arguments + /// * `instruction` - The instruction to increment the count for. + pub fn increase_invoked(&mut self, instruction: String) { + self.instructions + .entry(instruction) + .and_modify(|iterations_stats| iterations_stats.invoked += 1) + .or_insert(IterationStats { + invoked: 1, + successful: 0, + failed: 0, + failed_check: 0, + }); + } + + /// Increments the successful invocation count for a given instruction. + /// # Arguments + /// * `instruction` - The instruction to increment the successful count for. + pub fn increase_successful(&mut self, instruction: String) { + self.instructions + .entry(instruction) + .and_modify(|iterations_stats| iterations_stats.successful += 1) + .or_insert( + // this should not occure as instruction has to be invoked + // and then successfully_invoked + IterationStats { + invoked: 1, + successful: 1, + failed: 0, + failed_check: 0, + }, + ); + } + pub fn increase_failed(&mut self, instruction: String) { + self.instructions + .entry(instruction) + .and_modify(|iterations_stats| iterations_stats.failed += 1) + .or_insert( + // this should not occure as instruction has to be invoked + // and then unsuccessfully_invoked + IterationStats { + invoked: 1, + successful: 0, + failed: 1, + failed_check: 0, + }, + ); + } + pub fn increase_failed_check(&mut self, instruction: String) { + self.instructions + .entry(instruction) + .and_modify(|iterations_stats| iterations_stats.failed_check += 1) + .or_insert( + // this should not occure as instruction has to be invoked + // and then unsuccessfully_invoked + IterationStats { + invoked: 1, + successful: 1, + failed: 0, + failed_check: 1, + }, + ); + } + + /// Inserts or updates instructions with statistics provided in a serialized string. + /// # Arguments + /// * `serialized_iteration` - The serialized statistics to insert or update. + pub fn insert_serialized(&mut self, serialized_iteration: &str) { + let result = serde_json::from_str::>(serialized_iteration); + + if let Ok(deserialized_instruction) = result { + for (key, value) in deserialized_instruction { + self.instructions + .entry(key) + .and_modify(|instruction_stats| { + instruction_stats.invoked += value.invoked; + instruction_stats.successful += value.successful; + instruction_stats.failed += value.failed; + instruction_stats.failed_check += value.failed_check; + }) + .or_insert_with(|| IterationStats { + invoked: value.invoked, + successful: value.successful, + failed: value.failed, + failed_check: value.failed_check, + }); + } + } + } + /// Displays the collected statistics in a formatted table. + pub fn show_table(&self) { + let mut table = Table::new(); + table.add_row(row![ + "Instruction", + "Invoked Total", + "Ix Success", + "Check Failed", + "Ix Failed" + ]); + for (instruction, stats) in &self.instructions { + table.add_row(row![ + instruction, + stats.invoked, + stats.successful, + stats.failed_check, + stats.failed, + ]); + } + table.printstd(); + println!("Note that unhandled panics are currently logged only as crashes and are not displayed in the table above.") + } +} diff --git a/crates/client/src/fuzzer/mod.rs b/crates/client/src/fuzzer/mod.rs index fc1a9b39..b478cf71 100644 --- a/crates/client/src/fuzzer/mod.rs +++ b/crates/client/src/fuzzer/mod.rs @@ -1,6 +1,7 @@ pub mod accounts_storage; pub mod data_builder; pub mod fuzzer_generator; +pub mod fuzzing_stats; #[cfg(feature = "fuzzing")] pub mod program_test_client_blocking; pub mod snapshot; diff --git a/crates/client/src/lib.rs b/crates/client/src/lib.rs index cb07f622..c8072b96 100644 --- a/crates/client/src/lib.rs +++ b/crates/client/src/lib.rs @@ -42,6 +42,7 @@ pub mod fuzzing { pub use super::fuzzer::accounts_storage::*; pub use super::fuzzer::data_builder::build_ix_fuzz_data; pub use super::fuzzer::data_builder::*; + pub use super::fuzzing_stats::FuzzingStatistics; pub use super::fuzzer::program_test_client_blocking::ProgramEntry; pub use super::fuzzer::program_test_client_blocking::ProgramTestClientBlocking; diff --git a/crates/client/src/templates/Trident.toml.tmpl b/crates/client/src/templates/Trident.toml.tmpl index e05d7801..a398402c 100644 --- a/crates/client/src/templates/Trident.toml.tmpl +++ b/crates/client/src/templates/Trident.toml.tmpl @@ -36,3 +36,6 @@ save_all = false [fuzz] # Allow processing of duplicate transactions. Setting to true might speed up fuzzing but can cause false positive crashes (default: false) allow_duplicate_txs = false +# Trident will show statistics after the fuzzing session. This option forces use of honggfuzz parameter +# `keep_output` as true in order to be able to catch fuzzer stdout. (default: false) +fuzzing_with_stats = false diff --git a/crates/client/src/test_generator.rs b/crates/client/src/test_generator.rs index 47b022e7..b4d7aece 100644 --- a/crates/client/src/test_generator.rs +++ b/crates/client/src/test_generator.rs @@ -609,7 +609,7 @@ impl TestGenerator { } None => { members.push(new_member); - println!("{FINISH} [{CARGO_TOML}] with [{member}]"); + println!("{FINISH} [{CARGO_TOML}] updated with [{member}]"); fs::write(cargo, content.to_string()).await?; } }; diff --git a/documentation/docs/fuzzing/fuzzing-run-debug.md b/documentation/docs/fuzzing/fuzzing-run-debug.md index f2072e47..110233c5 100644 --- a/documentation/docs/fuzzing/fuzzing-run-debug.md +++ b/documentation/docs/fuzzing/fuzzing-run-debug.md @@ -11,32 +11,29 @@ trident fuzz run Under the hood {{ config.site_name }} uses [honggfuzz-rs](https://github.com/rust-fuzz/honggfuzz-rs). -You can pass [supported parameters](https://github.com/Ackee-Blockchain/trident/blob/develop/examples/fuzz_example0/Trident.toml) via the **{{ config.site_name }}.toml** configuration file. For example: +You can pass [supported parameters](https://github.com/Ackee-Blockchain/trident/blob/develop/examples/fuzz-tests/hello_world/Trident.toml) via the **{{ config.site_name }}.toml** configuration file: ```toml # Content of {{ config.site_name }}.toml -[fuzz] +[honggfuzz] # Timeout in seconds (default: 10) timeout = 10 # Number of fuzzing iterations (default: 0 [no limit]) -iterations = 10000 +iterations = 0 # Number of concurrent fuzzing threads (default: 0 [number of CPUs / 2]) threads = 0 -# Don't close children's stdin, stdout, -# stderr; can be noisy (default: false) +# Don't close children's stdin, stdout, stderr; can be noisy (default: false) keep_output = false # Disable ANSI console; use simple log output (default: false) verbose = false # Exit upon seeing the first crash (default: false) -exit_upon_crash = true +exit_upon_crash = false # Maximal number of mutations per one run (default: 6) mutations_per_run = 6 -# Target compilation directory, -# (default: "" ["trident-tests/fuzz_tests/fuzzing/hfuzz_target"]). +# Target compilation directory, (default: "" ["trident-tests/fuzz_tests/fuzzing/hfuzz_target"]). # To not clash with cargo build's default target directory. cargo_target_dir = "" -# Honggfuzz working directory, -# (default: "" ["trident-tests/fuzz_tests/fuzzing/hfuzz_workspace"]). +# Honggfuzz working directory, (default: "" ["trident-tests/fuzz_tests/fuzzing/hfuzz_workspace"]). hfuzz_workspace = "" # Directory where crashes are saved to (default: "" [workspace directory]) crashdir = "" @@ -44,12 +41,17 @@ crashdir = "" extension = "" # Number of seconds this fuzzing session will last (default: 0 [no limit]) run_time = 0 -# Maximal size of files processed by the fuzzer -# in bytes (default: 1048576 = 1MB) +# Maximal size of files processed by the fuzzer in bytes (default: 1048576 = 1MB) max_file_size = 1048576 -# Save all test-cases (not only the unique ones) by -# appending the current time-stamp to the filenames (default: false) +# Save all test-cases (not only the unique ones) by appending the current time-stamp to the filenames (default: false) save_all = false + +[fuzz] +# Allow processing of duplicate transactions. Setting to true might speed up fuzzing but can cause false positive crashes (default: false) +allow_duplicate_txs = false +# Trident will show statistics after the fuzzing session. This option forces use of honggfuzz parameter +# `keep_output` as true in order to be able to catch fuzzer stdout. (default: false) +fuzzing_with_stats = true ``` Or you can pass any parameter via [environment variables](https://github.com/rust-fuzz/honggfuzz-rs#environment-variables). @@ -64,6 +66,13 @@ A list of hongfuzz parameters can be found in honggfuzz [usage documentation](ht HFUZZ_RUN_ARGS="-t 10 -n 1 -N 10000 -Q" trident fuzz run ``` +### Fuzzing statistics +Sometimes, it's useful to know how often a particular instruction has been invoked and how many times it has succeeded or failed. To display these statistics when fuzzing is finished or interrupted, set the `fuzzing_with_stats` option to `true` in the `[fuzz]` section of the Trident.toml configuration file. Please note that this option is disabled by default because it impacts performance. + +The statistics show the total number of invocations for each instruction, which is the sum of successful and failed invocations. Successful invocations are those that return an `Ok()` result. Failed invocations are those that return an `Err()` result. Additionally, the statistics also show as `Check Failed` the number of successful invocations that did not pass the user-defined invariants check. Note that unhandled panics are currently logged only as crashes and are not displayed in the fuzzing statistics table. + +Keep in mind that the number of fuzz iterations does not directly correspond to the total number of invocations. In one fuzz iteration, the fuzzer might be unable to deserialize fuzz data into instructions, causing the entire iteration to be skipped. + ## Debug To debug your program with values from a crash file: diff --git a/examples/fuzz-tests/arbitrary-custom-types-4/Cargo.lock b/examples/fuzz-tests/arbitrary-custom-types-4/Cargo.lock index 970c4fc5..42ab1daa 100644 --- a/examples/fuzz-tests/arbitrary-custom-types-4/Cargo.lock +++ b/examples/fuzz-tests/arbitrary-custom-types-4/Cargo.lock @@ -1096,7 +1096,7 @@ version = "0.15.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb" dependencies = [ - "encode_unicode", + "encode_unicode 0.3.6", "lazy_static", "libc", "unicode-width", @@ -1229,6 +1229,27 @@ dependencies = [ "subtle", ] +[[package]] +name = "csv" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac574ff4d437a7b5ad237ef331c17ccca63c46479e5b5453eb8e10bb99a759fe" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "csv-core" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70" +dependencies = [ + "memchr", +] + [[package]] name = "ctr" version = "0.8.0" @@ -1596,6 +1617,12 @@ version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" +[[package]] +name = "encode_unicode" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" + [[package]] name = "encoding_rs" version = "0.8.34" @@ -2278,6 +2305,17 @@ version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" +[[package]] +name = "is-terminal" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f23ff5ef2b80d608d61efee834934d862cd92461afc0560dedf493e4c033738b" +dependencies = [ + "hermit-abi 0.3.6", + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "itertools" version = "0.10.5" @@ -3083,6 +3121,20 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6fa0831dd7cc608c38a5e323422a0077678fa5744aa2be4ad91c4ece8eec8d5" +[[package]] +name = "prettytable" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46480520d1b77c9a3482d39939fcf96831537a250ec62d4fd8fbdf8e0302e781" +dependencies = [ + "csv", + "encode_unicode 1.0.0", + "is-terminal", + "lazy_static", + "term", + "unicode-width", +] + [[package]] name = "proc-macro-crate" version = "0.1.5" @@ -5607,6 +5659,17 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "term" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f" +dependencies = [ + "dirs-next", + "rustversion", + "winapi", +] + [[package]] name = "termcolor" version = "1.4.1" @@ -5999,6 +6062,7 @@ dependencies = [ "lazy_static", "log", "pathdiff", + "prettytable", "proc-macro2", "quinn-proto", "quote", @@ -6014,6 +6078,7 @@ dependencies = [ "solana-bpf-loader-program", "solana-cli-output", "solana-program", + "solana-logger", "solana-program-runtime", "solana-program-test", "solana-sdk", diff --git a/examples/fuzz-tests/arbitrary-custom-types-4/Trident.toml b/examples/fuzz-tests/arbitrary-custom-types-4/Trident.toml index e05d7801..a398402c 100644 --- a/examples/fuzz-tests/arbitrary-custom-types-4/Trident.toml +++ b/examples/fuzz-tests/arbitrary-custom-types-4/Trident.toml @@ -36,3 +36,6 @@ save_all = false [fuzz] # Allow processing of duplicate transactions. Setting to true might speed up fuzzing but can cause false positive crashes (default: false) allow_duplicate_txs = false +# Trident will show statistics after the fuzzing session. This option forces use of honggfuzz parameter +# `keep_output` as true in order to be able to catch fuzzer stdout. (default: false) +fuzzing_with_stats = false diff --git a/examples/fuzz-tests/arbitrary-limit-inputs-5/Trident.toml b/examples/fuzz-tests/arbitrary-limit-inputs-5/Trident.toml index e05d7801..a398402c 100644 --- a/examples/fuzz-tests/arbitrary-limit-inputs-5/Trident.toml +++ b/examples/fuzz-tests/arbitrary-limit-inputs-5/Trident.toml @@ -36,3 +36,6 @@ save_all = false [fuzz] # Allow processing of duplicate transactions. Setting to true might speed up fuzzing but can cause false positive crashes (default: false) allow_duplicate_txs = false +# Trident will show statistics after the fuzzing session. This option forces use of honggfuzz parameter +# `keep_output` as true in order to be able to catch fuzzer stdout. (default: false) +fuzzing_with_stats = false diff --git a/examples/fuzz-tests/hello_world/Trident.toml b/examples/fuzz-tests/hello_world/Trident.toml index 6c05d2e0..93edfd43 100644 --- a/examples/fuzz-tests/hello_world/Trident.toml +++ b/examples/fuzz-tests/hello_world/Trident.toml @@ -36,3 +36,6 @@ save_all = false [fuzz] # Allow processing of duplicate transactions. Setting to true might speed up fuzzing but can cause false positive crashes (default: false) allow_duplicate_txs = false +# Trident will show statistics after the fuzzing session. This option forces use of honggfuzz parameter +# `keep_output` as true in order to be able to catch fuzzer stdout. (default: false) +fuzzing_with_stats = false diff --git a/examples/fuzz-tests/incorrect-integer-arithmetic-3/Trident.toml b/examples/fuzz-tests/incorrect-integer-arithmetic-3/Trident.toml index e05d7801..a398402c 100644 --- a/examples/fuzz-tests/incorrect-integer-arithmetic-3/Trident.toml +++ b/examples/fuzz-tests/incorrect-integer-arithmetic-3/Trident.toml @@ -36,3 +36,6 @@ save_all = false [fuzz] # Allow processing of duplicate transactions. Setting to true might speed up fuzzing but can cause false positive crashes (default: false) allow_duplicate_txs = false +# Trident will show statistics after the fuzzing session. This option forces use of honggfuzz parameter +# `keep_output` as true in order to be able to catch fuzzer stdout. (default: false) +fuzzing_with_stats = false diff --git a/examples/fuzz-tests/incorrect-ix-sequence-1/Trident.toml b/examples/fuzz-tests/incorrect-ix-sequence-1/Trident.toml index e05d7801..a398402c 100644 --- a/examples/fuzz-tests/incorrect-ix-sequence-1/Trident.toml +++ b/examples/fuzz-tests/incorrect-ix-sequence-1/Trident.toml @@ -36,3 +36,6 @@ save_all = false [fuzz] # Allow processing of duplicate transactions. Setting to true might speed up fuzzing but can cause false positive crashes (default: false) allow_duplicate_txs = false +# Trident will show statistics after the fuzzing session. This option forces use of honggfuzz parameter +# `keep_output` as true in order to be able to catch fuzzer stdout. (default: false) +fuzzing_with_stats = false diff --git a/examples/fuzz-tests/unauthorized-access-2/Trident.toml b/examples/fuzz-tests/unauthorized-access-2/Trident.toml index e05d7801..a398402c 100644 --- a/examples/fuzz-tests/unauthorized-access-2/Trident.toml +++ b/examples/fuzz-tests/unauthorized-access-2/Trident.toml @@ -36,3 +36,6 @@ save_all = false [fuzz] # Allow processing of duplicate transactions. Setting to true might speed up fuzzing but can cause false positive crashes (default: false) allow_duplicate_txs = false +# Trident will show statistics after the fuzzing session. This option forces use of honggfuzz parameter +# `keep_output` as true in order to be able to catch fuzzer stdout. (default: false) +fuzzing_with_stats = false diff --git a/examples/fuzz-tests/unchecked-arithmetic-0/Trident.toml b/examples/fuzz-tests/unchecked-arithmetic-0/Trident.toml index e05d7801..a398402c 100644 --- a/examples/fuzz-tests/unchecked-arithmetic-0/Trident.toml +++ b/examples/fuzz-tests/unchecked-arithmetic-0/Trident.toml @@ -36,3 +36,6 @@ save_all = false [fuzz] # Allow processing of duplicate transactions. Setting to true might speed up fuzzing but can cause false positive crashes (default: false) allow_duplicate_txs = false +# Trident will show statistics after the fuzzing session. This option forces use of honggfuzz parameter +# `keep_output` as true in order to be able to catch fuzzer stdout. (default: false) +fuzzing_with_stats = false diff --git a/examples/integration-tests/escrow/Trident.toml b/examples/integration-tests/escrow/Trident.toml index e05d7801..a398402c 100644 --- a/examples/integration-tests/escrow/Trident.toml +++ b/examples/integration-tests/escrow/Trident.toml @@ -36,3 +36,6 @@ save_all = false [fuzz] # Allow processing of duplicate transactions. Setting to true might speed up fuzzing but can cause false positive crashes (default: false) allow_duplicate_txs = false +# Trident will show statistics after the fuzzing session. This option forces use of honggfuzz parameter +# `keep_output` as true in order to be able to catch fuzzer stdout. (default: false) +fuzzing_with_stats = false diff --git a/examples/integration-tests/turnstile/Trident.toml b/examples/integration-tests/turnstile/Trident.toml index e05d7801..a398402c 100644 --- a/examples/integration-tests/turnstile/Trident.toml +++ b/examples/integration-tests/turnstile/Trident.toml @@ -36,3 +36,6 @@ save_all = false [fuzz] # Allow processing of duplicate transactions. Setting to true might speed up fuzzing but can cause false positive crashes (default: false) allow_duplicate_txs = false +# Trident will show statistics after the fuzzing session. This option forces use of honggfuzz parameter +# `keep_output` as true in order to be able to catch fuzzer stdout. (default: false) +fuzzing_with_stats = false