From 7b9200b50364c01a29c34031e55adad88bdf1d7e Mon Sep 17 00:00:00 2001 From: Louis Merlin Date: Tue, 21 Nov 2023 14:29:53 +0100 Subject: [PATCH 1/9] Improve fuzzer for XCMv4 --- Cargo.lock | 14 +++-- polkadot/xcm/xcm-simulator/fuzzer/.gitignore | 6 +- polkadot/xcm/xcm-simulator/fuzzer/Cargo.toml | 2 +- polkadot/xcm/xcm-simulator/fuzzer/README.md | 16 ++--- .../fuzzer/src/{fuzz.rs => main.rs} | 61 +++++++++---------- .../xcm/xcm-simulator/fuzzer/src/parachain.rs | 5 +- 6 files changed, 49 insertions(+), 55 deletions(-) rename polkadot/xcm/xcm-simulator/fuzzer/src/{fuzz.rs => main.rs} (86%) diff --git a/Cargo.lock b/Cargo.lock index 95523039ed28..be162d03538c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1494,8 +1494,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "93f2635620bf0b9d4576eb7bb9a38a55df78bd1205d26fa994b25911a69f212f" dependencies = [ "bitcoin_hashes", - "rand 0.7.3", - "rand_core 0.5.1", + "rand 0.8.5", + "rand_core 0.6.4", "serde", "unicode-normalization", ] @@ -20009,7 +20009,7 @@ checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" dependencies = [ "cfg-if", "digest 0.10.7", - "rand 0.7.3", + "rand 0.8.5", "static_assertions", ] @@ -21574,7 +21574,6 @@ dependencies = [ "arbitrary", "frame-support", "frame-system", - "honggfuzz", "pallet-balances", "pallet-message-queue", "pallet-xcm", @@ -21591,6 +21590,7 @@ dependencies = [ "staging-xcm-builder", "staging-xcm-executor", "xcm-simulator", + "ziggy", ] [[package]] @@ -21642,6 +21642,12 @@ dependencies = [ "syn 2.0.38", ] +[[package]] +name = "ziggy" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0909a9cc2142b0b3417083b821fc570ae3c3f3c309fca92339f5a57b7c03a41b" + [[package]] name = "zombienet-backchannel" version = "1.0.0" diff --git a/polkadot/xcm/xcm-simulator/fuzzer/.gitignore b/polkadot/xcm/xcm-simulator/fuzzer/.gitignore index ec8de6fa0531..6caf68aff423 100644 --- a/polkadot/xcm/xcm-simulator/fuzzer/.gitignore +++ b/polkadot/xcm/xcm-simulator/fuzzer/.gitignore @@ -1,5 +1 @@ -hfuzz_target -hfuzz_workspace -cargo -coverage -ccov.zip +output \ No newline at end of file diff --git a/polkadot/xcm/xcm-simulator/fuzzer/Cargo.toml b/polkadot/xcm/xcm-simulator/fuzzer/Cargo.toml index acf28bec4f19..0d86ded3aeb4 100644 --- a/polkadot/xcm/xcm-simulator/fuzzer/Cargo.toml +++ b/polkadot/xcm/xcm-simulator/fuzzer/Cargo.toml @@ -9,7 +9,7 @@ publish = false [dependencies] codec = { package = "parity-scale-codec", version = "3.6.1" } -honggfuzz = "0.5.55" +ziggy = { version = "0.8", default-features = false } arbitrary = "1.2.0" scale-info = { version = "2.10.0", features = ["derive"] } diff --git a/polkadot/xcm/xcm-simulator/fuzzer/README.md b/polkadot/xcm/xcm-simulator/fuzzer/README.md index 0b3fdd8ec776..b2557d1e0f80 100644 --- a/polkadot/xcm/xcm-simulator/fuzzer/README.md +++ b/polkadot/xcm/xcm-simulator/fuzzer/README.md @@ -6,7 +6,7 @@ underflows. ## Install dependencies ``` -cargo install honggfuzz +cargo install --force ziggy cargo-afl honggfuzz grcov ``` ## Run the fuzzer @@ -14,7 +14,7 @@ cargo install honggfuzz In this directory, run this command: ``` -cargo hfuzz run xcm-fuzzer +cargo ziggy fuzz ``` ## Run a single input @@ -22,19 +22,15 @@ cargo hfuzz run xcm-fuzzer In this directory, run this command: ``` -cargo hfuzz run-debug xcm-fuzzer hfuzz_workspace/xcm-fuzzer/fuzzer_input_file +cargo ziggy run -i path/to/your/input ``` ## Generate coverage -In this directory, run these four commands: +In this directory, run this command: ``` -RUSTFLAGS="-Zprofile -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests -Cpanic=abort" \ -CARGO_INCREMENTAL=0 SKIP_WASM_BUILD=1 CARGO_HOME=./cargo cargo build -../../../target/debug/xcm-fuzzer hfuzz_workspace/xcm-fuzzer/input/ -zip -0 ccov.zip `find ../../../target/ \( -name "*.gc*" -o -name "test-*.gc*" \) -print` -grcov ccov.zip -s ../../../ -t html --llvm --branch --ignore-not-existing -o ./coverage +cargo ziggy cover ``` -The code coverage will be in `./coverage/index.html`. +The code coverage will be in `./output/xcm-simulator-fuzzer/coverage/index.html`. diff --git a/polkadot/xcm/xcm-simulator/fuzzer/src/fuzz.rs b/polkadot/xcm/xcm-simulator/fuzzer/src/main.rs similarity index 86% rename from polkadot/xcm/xcm-simulator/fuzzer/src/fuzz.rs rename to polkadot/xcm/xcm-simulator/fuzzer/src/main.rs index 7026d5467c8b..ce265ee46e8a 100644 --- a/polkadot/xcm/xcm-simulator/fuzzer/src/fuzz.rs +++ b/polkadot/xcm/xcm-simulator/fuzzer/src/main.rs @@ -98,7 +98,7 @@ impl<'a> Arbitrary<'a> for XcmMessage { if let Ok(message) = DecodeLimit::decode_with_depth_limit(MAX_XCM_DECODE_DEPTH, &mut encoded_message) { - return Ok(XcmMessage { source, destination, message }) + return Ok(XcmMessage { source, destination, message }); } Err(Error::IncorrectFormat) } @@ -155,6 +155,30 @@ fn run_input(xcm_messages: [XcmMessage; 5]) { println!(); for xcm_message in xcm_messages { + fn matches_blocklisted_messages(message: Instruction<()>) -> bool { + matches!(message, Transact { .. }) || matches!(message, SetAppendix { .. }) + } + fn matches_recursive(message: &mut Instruction<()>) -> Vec> { + match message { + SetErrorHandler(sub_m) => { + Vec::from(sub_m.inner()).iter_mut().map(matches_recursive).flatten().collect() + }, + _ => vec![message.clone()], + } + } + + if xcm_message + .message + .clone() + .iter_mut() + .map(matches_recursive) + .flatten() + .any(|m| matches_blocklisted_messages(m)) + { + println!(" skipping message\n"); + continue; + } + if xcm_message.source % 4 == 0 { // We get the destination for the message let parachain_id = (xcm_message.destination % 3) + 1; @@ -202,36 +226,7 @@ fn run_input(xcm_messages: [XcmMessage; 5]) { } fn main() { - #[cfg(fuzzing)] - { - loop { - honggfuzz::fuzz!(|xcm_messages: [XcmMessage; 5]| { - run_input(xcm_messages); - }) - } - } - #[cfg(not(fuzzing))] - { - use std::{env, fs, fs::File, io::Read}; - let args: Vec<_> = env::args().collect(); - let md = fs::metadata(&args[1]).unwrap(); - let all_files = match md.is_dir() { - true => fs::read_dir(&args[1]) - .unwrap() - .map(|x| x.unwrap().path().to_str().unwrap().to_string()) - .collect::>(), - false => (args[1..]).to_vec(), - }; - println!("All_files {:?}", all_files); - for argument in all_files { - println!("Now doing file {:?}", argument); - let mut buffer: Vec = Vec::new(); - let mut f = File::open(argument).unwrap(); - f.read_to_end(&mut buffer).unwrap(); - let mut unstructured = Unstructured::new(&buffer); - if let Ok(xcm_messages) = unstructured.arbitrary() { - run_input(xcm_messages); - } - } - } + ziggy::fuzz!(|xcm_messages: [XcmMessage; 5]| { + run_input(xcm_messages); + }); } diff --git a/polkadot/xcm/xcm-simulator/fuzzer/src/parachain.rs b/polkadot/xcm/xcm-simulator/fuzzer/src/parachain.rs index dfcb914d5d3e..3d7b8b423da8 100644 --- a/polkadot/xcm/xcm-simulator/fuzzer/src/parachain.rs +++ b/polkadot/xcm/xcm-simulator/fuzzer/src/parachain.rs @@ -249,8 +249,9 @@ pub mod mock_msg_queue { Outcome::Complete { used } => (Ok(used), Event::Success(Some(hash))), // As far as the caller is concerned, this was dispatched without error, so // we just report the weight used. - Outcome::Incomplete { used, error } => - (Ok(used), Event::Fail(Some(hash), error)), + Outcome::Incomplete { used, error } => { + (Ok(used), Event::Fail(Some(hash), error)) + }, } }, Err(()) => (Err(XcmError::UnhandledXcmVersion), Event::BadVersion(Some(hash))), From 4736947a7b9af31e2e588820bd97ef3c4ba8a74c Mon Sep 17 00:00:00 2001 From: Louis Merlin Date: Tue, 21 Nov 2023 14:39:44 +0100 Subject: [PATCH 2/9] Remove unused bin --- polkadot/xcm/xcm-simulator/fuzzer/Cargo.toml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/polkadot/xcm/xcm-simulator/fuzzer/Cargo.toml b/polkadot/xcm/xcm-simulator/fuzzer/Cargo.toml index 0d86ded3aeb4..4615d857fce1 100644 --- a/polkadot/xcm/xcm-simulator/fuzzer/Cargo.toml +++ b/polkadot/xcm/xcm-simulator/fuzzer/Cargo.toml @@ -44,7 +44,3 @@ runtime-benchmarks = [ "xcm-builder/runtime-benchmarks", "xcm-executor/runtime-benchmarks", ] - -[[bin]] -path = "src/fuzz.rs" -name = "xcm-fuzzer" From c93efe5dea613a118af83a829c4a6229d0188c19 Mon Sep 17 00:00:00 2001 From: Louis Merlin Date: Tue, 21 Nov 2023 14:46:58 +0100 Subject: [PATCH 3/9] Format code --- polkadot/xcm/xcm-simulator/fuzzer/src/main.rs | 5 ++--- polkadot/xcm/xcm-simulator/fuzzer/src/parachain.rs | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/polkadot/xcm/xcm-simulator/fuzzer/src/main.rs b/polkadot/xcm/xcm-simulator/fuzzer/src/main.rs index ce265ee46e8a..fe2274ff624c 100644 --- a/polkadot/xcm/xcm-simulator/fuzzer/src/main.rs +++ b/polkadot/xcm/xcm-simulator/fuzzer/src/main.rs @@ -160,9 +160,8 @@ fn run_input(xcm_messages: [XcmMessage; 5]) { } fn matches_recursive(message: &mut Instruction<()>) -> Vec> { match message { - SetErrorHandler(sub_m) => { - Vec::from(sub_m.inner()).iter_mut().map(matches_recursive).flatten().collect() - }, + SetErrorHandler(sub_m) => + Vec::from(sub_m.inner()).iter_mut().map(matches_recursive).flatten().collect(), _ => vec![message.clone()], } } diff --git a/polkadot/xcm/xcm-simulator/fuzzer/src/parachain.rs b/polkadot/xcm/xcm-simulator/fuzzer/src/parachain.rs index 3d7b8b423da8..dfcb914d5d3e 100644 --- a/polkadot/xcm/xcm-simulator/fuzzer/src/parachain.rs +++ b/polkadot/xcm/xcm-simulator/fuzzer/src/parachain.rs @@ -249,9 +249,8 @@ pub mod mock_msg_queue { Outcome::Complete { used } => (Ok(used), Event::Success(Some(hash))), // As far as the caller is concerned, this was dispatched without error, so // we just report the weight used. - Outcome::Incomplete { used, error } => { - (Ok(used), Event::Fail(Some(hash), error)) - }, + Outcome::Incomplete { used, error } => + (Ok(used), Event::Fail(Some(hash), error)), } }, Err(()) => (Err(XcmError::UnhandledXcmVersion), Event::BadVersion(Some(hash))), From a92dcc2cbe9ae07837af2af42a73b98f0dfc2ffb Mon Sep 17 00:00:00 2001 From: Louis Merlin Date: Tue, 21 Nov 2023 15:27:24 +0100 Subject: [PATCH 4/9] Improve filter for nested message --- polkadot/xcm/xcm-simulator/fuzzer/src/main.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/polkadot/xcm/xcm-simulator/fuzzer/src/main.rs b/polkadot/xcm/xcm-simulator/fuzzer/src/main.rs index fe2274ff624c..d526eff72365 100644 --- a/polkadot/xcm/xcm-simulator/fuzzer/src/main.rs +++ b/polkadot/xcm/xcm-simulator/fuzzer/src/main.rs @@ -162,6 +162,8 @@ fn run_input(xcm_messages: [XcmMessage; 5]) { match message { SetErrorHandler(sub_m) => Vec::from(sub_m.inner()).iter_mut().map(matches_recursive).flatten().collect(), + DepositReserveAsset { xcm, .. } => + Vec::from(xcm.inner()).iter_mut().map(matches_recursive).flatten().collect(), _ => vec![message.clone()], } } From daaa84ecdf62ec3e26594cf335ec27209c3f1937 Mon Sep 17 00:00:00 2001 From: Louis Merlin Date: Tue, 21 Nov 2023 15:30:01 +0100 Subject: [PATCH 5/9] Fix clippy warnings --- polkadot/xcm/xcm-simulator/fuzzer/src/main.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/polkadot/xcm/xcm-simulator/fuzzer/src/main.rs b/polkadot/xcm/xcm-simulator/fuzzer/src/main.rs index d526eff72365..d66eb69d234a 100644 --- a/polkadot/xcm/xcm-simulator/fuzzer/src/main.rs +++ b/polkadot/xcm/xcm-simulator/fuzzer/src/main.rs @@ -161,9 +161,9 @@ fn run_input(xcm_messages: [XcmMessage; 5]) { fn matches_recursive(message: &mut Instruction<()>) -> Vec> { match message { SetErrorHandler(sub_m) => - Vec::from(sub_m.inner()).iter_mut().map(matches_recursive).flatten().collect(), + Vec::from(sub_m.inner()).iter_mut().flat_map(matches_recursive).collect(), DepositReserveAsset { xcm, .. } => - Vec::from(xcm.inner()).iter_mut().map(matches_recursive).flatten().collect(), + Vec::from(xcm.inner()).iter_mut().flat_map(matches_recursive).collect(), _ => vec![message.clone()], } } @@ -172,8 +172,7 @@ fn run_input(xcm_messages: [XcmMessage; 5]) { .message .clone() .iter_mut() - .map(matches_recursive) - .flatten() + .flat_map(matches_recursive) .any(|m| matches_blocklisted_messages(m)) { println!(" skipping message\n"); From 48188d84ad07730c4a822b1bf38319ef2b580cd6 Mon Sep 17 00:00:00 2001 From: Louis Merlin Date: Tue, 21 Nov 2023 16:00:02 +0100 Subject: [PATCH 6/9] Tweak XCM fuzzer filters --- polkadot/xcm/xcm-simulator/fuzzer/src/main.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/polkadot/xcm/xcm-simulator/fuzzer/src/main.rs b/polkadot/xcm/xcm-simulator/fuzzer/src/main.rs index d66eb69d234a..d4449f3b34b8 100644 --- a/polkadot/xcm/xcm-simulator/fuzzer/src/main.rs +++ b/polkadot/xcm/xcm-simulator/fuzzer/src/main.rs @@ -155,14 +155,17 @@ fn run_input(xcm_messages: [XcmMessage; 5]) { println!(); for xcm_message in xcm_messages { + // We block Transact messages because they can cause panics that are due to the test + // environment, and not XCM fn matches_blocklisted_messages(message: Instruction<()>) -> bool { - matches!(message, Transact { .. }) || matches!(message, SetAppendix { .. }) + matches!(message, Transact { .. }) } + // We check XCM messages recursively for blocklisted messages fn matches_recursive(message: &mut Instruction<()>) -> Vec> { match message { - SetErrorHandler(sub_m) => + SetErrorHandler(sub_m) | SetAppendix(sub_m) => Vec::from(sub_m.inner()).iter_mut().flat_map(matches_recursive).collect(), - DepositReserveAsset { xcm, .. } => + DepositReserveAsset { xcm, .. } | InitiateReserveWithdraw { xcm, .. } => Vec::from(xcm.inner()).iter_mut().flat_map(matches_recursive).collect(), _ => vec![message.clone()], } From a32a998acab5a91420145ab2db5c5328adc6110e Mon Sep 17 00:00:00 2001 From: Louis Merlin Date: Tue, 21 Nov 2023 16:50:55 +0100 Subject: [PATCH 7/9] Add another recursive filter --- polkadot/xcm/xcm-simulator/fuzzer/src/main.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/polkadot/xcm/xcm-simulator/fuzzer/src/main.rs b/polkadot/xcm/xcm-simulator/fuzzer/src/main.rs index d4449f3b34b8..3529200b81bf 100644 --- a/polkadot/xcm/xcm-simulator/fuzzer/src/main.rs +++ b/polkadot/xcm/xcm-simulator/fuzzer/src/main.rs @@ -163,10 +163,11 @@ fn run_input(xcm_messages: [XcmMessage; 5]) { // We check XCM messages recursively for blocklisted messages fn matches_recursive(message: &mut Instruction<()>) -> Vec> { match message { - SetErrorHandler(sub_m) | SetAppendix(sub_m) => - Vec::from(sub_m.inner()).iter_mut().flat_map(matches_recursive).collect(), - DepositReserveAsset { xcm, .. } | InitiateReserveWithdraw { xcm, .. } => - Vec::from(xcm.inner()).iter_mut().flat_map(matches_recursive).collect(), + DepositReserveAsset { xcm, .. } | + InitiateReserveWithdraw { xcm, .. } | + TransferReserveAsset { xcm, .. } | + SetErrorHandler(xcm) | + SetAppendix(xcm) => Vec::from(xcm.inner()).iter_mut().flat_map(matches_recursive).collect(), _ => vec![message.clone()], } } From 40cf010e2cfb424cc1df95bf8abcd6cbea3c8a1c Mon Sep 17 00:00:00 2001 From: Louis Merlin Date: Wed, 22 Nov 2023 09:31:22 +0100 Subject: [PATCH 8/9] Add InitiateTeleport to recursive filter --- polkadot/xcm/xcm-simulator/fuzzer/src/main.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/polkadot/xcm/xcm-simulator/fuzzer/src/main.rs b/polkadot/xcm/xcm-simulator/fuzzer/src/main.rs index 3529200b81bf..18cac54fc109 100644 --- a/polkadot/xcm/xcm-simulator/fuzzer/src/main.rs +++ b/polkadot/xcm/xcm-simulator/fuzzer/src/main.rs @@ -165,6 +165,7 @@ fn run_input(xcm_messages: [XcmMessage; 5]) { match message { DepositReserveAsset { xcm, .. } | InitiateReserveWithdraw { xcm, .. } | + InitiateTeleport { xcm, .. } | TransferReserveAsset { xcm, .. } | SetErrorHandler(xcm) | SetAppendix(xcm) => Vec::from(xcm.inner()).iter_mut().flat_map(matches_recursive).collect(), From 77b8464b35e07929318043497f29f36d10ce8fa4 Mon Sep 17 00:00:00 2001 From: Louis Merlin Date: Mon, 27 Nov 2023 09:59:04 +0100 Subject: [PATCH 9/9] Add details about ziggy flags to README --- polkadot/xcm/xcm-simulator/fuzzer/README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/polkadot/xcm/xcm-simulator/fuzzer/README.md b/polkadot/xcm/xcm-simulator/fuzzer/README.md index b2557d1e0f80..f2e3fd4d05fc 100644 --- a/polkadot/xcm/xcm-simulator/fuzzer/README.md +++ b/polkadot/xcm/xcm-simulator/fuzzer/README.md @@ -17,6 +17,11 @@ In this directory, run this command: cargo ziggy fuzz ``` +You can use the following options to improve fuzzing: +- `-j number_of_jobs` for fuzzing using multiple threads +- `-G 1024` to limit the size of inputs to 1024 bytes + - this will improve fuzzing effectiveness, and you can drop the limit once you have good coverage + ## Run a single input In this directory, run this command: