From a187289f37beec236dac27d4dfa3e0fcf729a1ab Mon Sep 17 00:00:00 2001 From: Louis Date: Tue, 25 Oct 2022 11:06:35 +0200 Subject: [PATCH 1/7] Add improved XCM fuzzer --- xcm/xcm-simulator/fuzzer/.gitignore | 5 + xcm/xcm-simulator/fuzzer/Cargo.toml | 1 + xcm/xcm-simulator/fuzzer/README.md | 22 ++++ xcm/xcm-simulator/fuzzer/src/fuzz.rs | 144 ++++++++++++++++++++------- 4 files changed, 134 insertions(+), 38 deletions(-) create mode 100644 xcm/xcm-simulator/fuzzer/.gitignore create mode 100644 xcm/xcm-simulator/fuzzer/README.md diff --git a/xcm/xcm-simulator/fuzzer/.gitignore b/xcm/xcm-simulator/fuzzer/.gitignore new file mode 100644 index 000000000000..39ddd844b888 --- /dev/null +++ b/xcm/xcm-simulator/fuzzer/.gitignore @@ -0,0 +1,5 @@ +hfuzz_target +hfuzz_workspace +cargo +coverage +ccov.zip \ No newline at end of file diff --git a/xcm/xcm-simulator/fuzzer/Cargo.toml b/xcm/xcm-simulator/fuzzer/Cargo.toml index 7ecff2f8c021..03f43c5f4602 100644 --- a/xcm/xcm-simulator/fuzzer/Cargo.toml +++ b/xcm/xcm-simulator/fuzzer/Cargo.toml @@ -8,6 +8,7 @@ edition = "2021" [dependencies] codec = { package = "parity-scale-codec", version = "3.0.0" } honggfuzz = "0.5.55" +arbitrary = "1.2.0" scale-info = { version = "2.1.2", features = ["derive"] } frame-system = { git = "https://github.com/paritytech/substrate", branch = "master" } diff --git a/xcm/xcm-simulator/fuzzer/README.md b/xcm/xcm-simulator/fuzzer/README.md new file mode 100644 index 000000000000..26d6093e0b5a --- /dev/null +++ b/xcm/xcm-simulator/fuzzer/README.md @@ -0,0 +1,22 @@ +# XCM Simulator Fuzzer + +This project will fuzz-test the XCM simulator. It can catch reachable panics, timeouts as well as integer overflows and underflows. + +## Run the fuzzer + +``` +cargo hfuzz run xcm-fuzzer +``` + +## Generate coverage + +In this directory, run these four commands: + +``` +RUSTFLAGS="-Zprofile -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests -Cpanic=abort" CARGO_INCREMENTAL=0 SKIP_WASM_BUILD=1 CARGO_HOME=./cargo cargo build +../../../target/debug/xcm-fuzzer hfuzz_workspace/xcm-fuzzer/input/ +zip -0 ccov.zip `find ../../../target/ \( -name "*.gc*" -o -name "test-*.gc*" \) -print` +grcov ccov.zip -s ../../../ -t html --llvm --branch --ignore-not-existing -o ./coverage +``` + +The code coverage will be in `./coverage/index.html`. \ No newline at end of file diff --git a/xcm/xcm-simulator/fuzzer/src/fuzz.rs b/xcm/xcm-simulator/fuzzer/src/fuzz.rs index 70ad3c1e2cc6..3770acb27756 100644 --- a/xcm/xcm-simulator/fuzzer/src/fuzz.rs +++ b/xcm/xcm-simulator/fuzzer/src/fuzz.rs @@ -18,6 +18,7 @@ mod parachain; mod relay_chain; use codec::DecodeLimit; +use polkadot_core_primitives::AccountId; use polkadot_parachain::primitives::Id as ParaId; use sp_runtime::traits::AccountIdConversion; use xcm_simulator::{decl_test_network, decl_test_parachain, decl_test_relay_chain, TestExt}; @@ -25,7 +26,8 @@ use xcm_simulator::{decl_test_network, decl_test_parachain, decl_test_relay_chai use frame_support::assert_ok; use xcm::{latest::prelude::*, MAX_XCM_DECODE_DEPTH}; -pub const ALICE: sp_runtime::AccountId32 = sp_runtime::AccountId32::new([0u8; 32]); +use arbitrary::{Arbitrary, Error, Unstructured}; + pub const INITIAL_BALANCE: u128 = 1_000_000_000; decl_test_parachain! { @@ -46,6 +48,15 @@ decl_test_parachain! { } } +decl_test_parachain! { + pub struct ParaC { + Runtime = parachain::Runtime, + XcmpMessageHandler = parachain::MsgQueue, + DmpMessageHandler = parachain::MsgQueue, + new_ext = para_ext(3), + } +} + decl_test_relay_chain! { pub struct Relay { Runtime = relay_chain::Runtime, @@ -60,10 +71,35 @@ decl_test_network! { parachains = vec![ (1, ParaA), (2, ParaB), + (3, ParaC), ], } } +// An XCM message that will be generated by the fuzzer through the Arbitrary trait +struct XcmMessage { + // Source chain + source: u32, + // Destination chain + destination: u32, + // XCM message + message: Xcm<()>, +} + +impl<'a> Arbitrary<'a> for XcmMessage { + fn arbitrary(u: &mut Unstructured<'a>) -> Result { + let source: u32 = u.arbitrary()?; + let destination: u32 = u.arbitrary()?; + let mut encoded_message: &[u8] = u.arbitrary()?; + if let Ok(message) = + DecodeLimit::decode_with_depth_limit(MAX_XCM_DECODE_DEPTH, &mut encoded_message) + { + return Ok(XcmMessage { source, destination, message }) + } + Err(Error::IncorrectFormat) + } +} + pub fn para_account_id(id: u32) -> relay_chain::AccountId { ParaId::from(id).into_account_truncating() } @@ -73,9 +109,11 @@ pub fn para_ext(para_id: u32) -> sp_io::TestExternalities { let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); - pallet_balances::GenesisConfig:: { balances: vec![(ALICE, INITIAL_BALANCE)] } - .assimilate_storage(&mut t) - .unwrap(); + pallet_balances::GenesisConfig:: { + balances: (0..6).map(|i| ([i; 32].into(), INITIAL_BALANCE)).collect(), + } + .assimilate_storage(&mut t) + .unwrap(); let mut ext = sp_io::TestExternalities::new(t); ext.execute_with(|| { @@ -90,11 +128,13 @@ pub fn relay_ext() -> sp_io::TestExternalities { let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); - pallet_balances::GenesisConfig:: { - balances: vec![(ALICE, INITIAL_BALANCE), (para_account_id(1), INITIAL_BALANCE)], - } - .assimilate_storage(&mut t) - .unwrap(); + let mut balances: Vec<(AccountId, u128)> = vec![]; + balances.append(&mut (1..=3).map(|i| (para_account_id(i), INITIAL_BALANCE)).collect()); + balances.append(&mut (0..6).map(|i| ([i; 32].into(), INITIAL_BALANCE)).collect()); + + pallet_balances::GenesisConfig:: { balances } + .assimilate_storage(&mut t) + .unwrap(); let mut ext = sp_io::TestExternalities::new(t); ext.execute_with(|| System::set_block_number(1)); @@ -104,46 +144,71 @@ pub fn relay_ext() -> sp_io::TestExternalities { pub type RelayChainPalletXcm = pallet_xcm::Pallet; pub type ParachainPalletXcm = pallet_xcm::Pallet; -fn run_one_input(mut data: &[u8]) { +fn run_input(xcm_messages: [XcmMessage; 5]) { MockNet::reset(); - if let Ok(m) = Xcm::decode_all_with_depth_limit(MAX_XCM_DECODE_DEPTH, &mut data) { - #[cfg(not(fuzzing))] - { - println!("Executing message {:?}", m); + + #[cfg(not(fuzzing))] + println!(); + + for xcm_message in xcm_messages { + if xcm_message.source % 4 == 0 { + // We get the destination for the message + let parachain_id = (xcm_message.destination % 3) + 1; + let destination: MultiLocation = Parachain(parachain_id).into(); + #[cfg(not(fuzzing))] + { + println!(" source: Relay Chain"); + println!(" destination: Parachain {parachain_id}"); + println!(" message: {:?}", xcm_message.message); + } + Relay::execute_with(|| { + assert_ok!(RelayChainPalletXcm::send_xcm(Here, destination, xcm_message.message)); + }) + } else { + // We get the source's execution method + let execute_with = match xcm_message.source % 4 { + 1 => ParaA::execute_with, + 2 => ParaB::execute_with, + _ => ParaC::execute_with, + }; + // We get the destination for the message + let destination: MultiLocation = match xcm_message.destination % 4 { + n @ 1..=3 => (Parent, Parachain(n)).into(), + _ => Parent.into(), + }; + #[cfg(not(fuzzing))] + let destination_str = match xcm_message.destination % 4 { + n @ 1..=3 => format!("Parachain {n}"), + _ => "Relay Chain".to_string(), + }; + #[cfg(not(fuzzing))] + { + println!(" source: Parachain {}", xcm_message.source % 4); + println!(" destination: {}", destination_str); + println!(" message: {:?}", xcm_message.message); + } + // We execute the message with the appropriate source and destination + execute_with(|| { + assert_ok!(ParachainPalletXcm::send_xcm(Here, destination, xcm_message.message)); + }); } - ParaA::execute_with(|| { - assert_ok!(ParachainPalletXcm::send_xcm(Here, Parent, m)); - }); - Relay::execute_with(|| {}); + #[cfg(not(fuzzing))] + println!(); } + Relay::execute_with(|| {}); } fn main() { #[cfg(fuzzing)] { loop { - honggfuzz::fuzz!(|data: &[u8]| { - run_one_input(data); - }); + honggfuzz::fuzz!(|xcm_messages: [XcmMessage; 5]| { + run_input(xcm_messages); + }) } } #[cfg(not(fuzzing))] { - //This code path can be used to generate a line-code coverage report in HTML - //that depicts which lines are executed by at least one input in the current fuzzing queue. - //To generate this code coverage report, run the following commands: - /* - ``` - export CARGO_INCREMENTAL=0 - export RUSTFLAGS="-Zprofile -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests -Cpanic=abort" - export RUSTDOCFLAGS="-Cpanic=abort" - rustup override set nightly - SKIP_WASM_BUILD=1 cargo build - ./xcm/xcm-simulator/fuzzer/target/debug/xcm-fuzzer hfuzz_workspace/xcm-fuzzer/input - zip -0 ccov.zip `find ../../target/debug \( -name "*.gc*" -o -name "test-*.gc*" \) -print` - grcov ccov.zip -s / -t html --llvm --branch --ignore-not-existing -o ../../target/debug/coverage/ - ``` - */ use std::{env, fs, fs::File, io::Read}; let args: Vec<_> = env::args().collect(); let md = fs::metadata(&args[1]).unwrap(); @@ -152,7 +217,7 @@ fn main() { .unwrap() .map(|x| x.unwrap().path().to_str().unwrap().to_string()) .collect::>(), - false => (&args[1..]).to_vec(), + false => (args[1..]).to_vec(), }; println!("All_files {:?}", all_files); for argument in all_files { @@ -160,7 +225,10 @@ fn main() { let mut buffer: Vec = Vec::new(); let mut f = File::open(argument).unwrap(); f.read_to_end(&mut buffer).unwrap(); - run_one_input(&buffer.as_slice()); + let mut unstructured = Unstructured::new(&buffer); + if let Ok(xcm_messages) = unstructured.arbitrary() { + run_input(xcm_messages); + } } } } From 1d10c565c62cd3ee24446c87d0822e8896ed89fb Mon Sep 17 00:00:00 2001 From: Louis Merlin Date: Tue, 25 Oct 2022 11:46:54 +0200 Subject: [PATCH 2/7] Add command for running a single input --- xcm/xcm-simulator/fuzzer/README.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/xcm/xcm-simulator/fuzzer/README.md b/xcm/xcm-simulator/fuzzer/README.md index 26d6093e0b5a..4534d797d7b2 100644 --- a/xcm/xcm-simulator/fuzzer/README.md +++ b/xcm/xcm-simulator/fuzzer/README.md @@ -8,6 +8,12 @@ This project will fuzz-test the XCM simulator. It can catch reachable panics, ti cargo hfuzz run xcm-fuzzer ``` +## Run a single input + +``` +cargo hfuzz run-debug xcm-fuzzer hfuzz_workspace/xcm-fuzzer/fuzzer_input_file +``` + ## Generate coverage In this directory, run these four commands: @@ -19,4 +25,4 @@ zip -0 ccov.zip `find ../../../target/ \( -name "*.gc*" -o -name "test-*.gc*" \) grcov ccov.zip -s ../../../ -t html --llvm --branch --ignore-not-existing -o ./coverage ``` -The code coverage will be in `./coverage/index.html`. \ No newline at end of file +The code coverage will be in `./coverage/index.html`. From 79abd730c0a06ea1f4e056c15272db8a5cf11778 Mon Sep 17 00:00:00 2001 From: Louis Merlin Date: Wed, 26 Oct 2022 11:43:35 +0200 Subject: [PATCH 3/7] Add installation command --- xcm/xcm-simulator/fuzzer/README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/xcm/xcm-simulator/fuzzer/README.md b/xcm/xcm-simulator/fuzzer/README.md index 4534d797d7b2..7ae5fe5d7e55 100644 --- a/xcm/xcm-simulator/fuzzer/README.md +++ b/xcm/xcm-simulator/fuzzer/README.md @@ -2,6 +2,12 @@ This project will fuzz-test the XCM simulator. It can catch reachable panics, timeouts as well as integer overflows and underflows. +## Install dependencies + +``` +cargo install honggfuzz +``` + ## Run the fuzzer ``` From e0891e42bfea507b6461bbd51622715a52eb82e1 Mon Sep 17 00:00:00 2001 From: Louis Date: Fri, 28 Oct 2022 12:53:41 +0200 Subject: [PATCH 4/7] Fix @m-cat's nit --- xcm/xcm-simulator/fuzzer/src/fuzz.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/xcm/xcm-simulator/fuzzer/src/fuzz.rs b/xcm/xcm-simulator/fuzzer/src/fuzz.rs index 3770acb27756..2965b791d401 100644 --- a/xcm/xcm-simulator/fuzzer/src/fuzz.rs +++ b/xcm/xcm-simulator/fuzzer/src/fuzz.rs @@ -177,12 +177,11 @@ fn run_input(xcm_messages: [XcmMessage; 5]) { _ => Parent.into(), }; #[cfg(not(fuzzing))] - let destination_str = match xcm_message.destination % 4 { - n @ 1..=3 => format!("Parachain {n}"), - _ => "Relay Chain".to_string(), - }; - #[cfg(not(fuzzing))] { + let destination_str = match xcm_message.destination % 4 { + n @ 1..=3 => format!("Parachain {n}"), + _ => "Relay Chain".to_string(), + }; println!(" source: Parachain {}", xcm_message.source % 4); println!(" destination: {}", destination_str); println!(" message: {:?}", xcm_message.message); From 5860cefb35dfbf8367a127075699e0ba46ba6207 Mon Sep 17 00:00:00 2001 From: Keith Yeung Date: Mon, 31 Oct 2022 08:28:41 +0800 Subject: [PATCH 5/7] Add newline Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com> --- xcm/xcm-simulator/fuzzer/.gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xcm/xcm-simulator/fuzzer/.gitignore b/xcm/xcm-simulator/fuzzer/.gitignore index 39ddd844b888..ec8de6fa0531 100644 --- a/xcm/xcm-simulator/fuzzer/.gitignore +++ b/xcm/xcm-simulator/fuzzer/.gitignore @@ -2,4 +2,4 @@ hfuzz_target hfuzz_workspace cargo coverage -ccov.zip \ No newline at end of file +ccov.zip From 4b0b9806d9d077e6a45635befdd0ea7b4f1193a3 Mon Sep 17 00:00:00 2001 From: Louis Merlin Date: Wed, 2 Nov 2022 17:06:30 +0100 Subject: [PATCH 6/7] Add info about current directory in fuzzing README --- xcm/xcm-simulator/fuzzer/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/xcm/xcm-simulator/fuzzer/README.md b/xcm/xcm-simulator/fuzzer/README.md index 7ae5fe5d7e55..69e8cd377b97 100644 --- a/xcm/xcm-simulator/fuzzer/README.md +++ b/xcm/xcm-simulator/fuzzer/README.md @@ -10,12 +10,16 @@ cargo install honggfuzz ## Run the fuzzer +In this directory, run this command: + ``` cargo hfuzz run xcm-fuzzer ``` ## Run a single input +In this directory, run this command: + ``` cargo hfuzz run-debug xcm-fuzzer hfuzz_workspace/xcm-fuzzer/fuzzer_input_file ``` From 6612bcac943b919160c7b3b02d18b4bab7186ffa Mon Sep 17 00:00:00 2001 From: Louis Date: Mon, 14 Nov 2022 13:11:36 +0100 Subject: [PATCH 7/7] Update Cargo.lock --- Cargo.lock | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f80f54c13db4..25ed2831a55d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -114,9 +114,9 @@ dependencies = [ [[package]] name = "arbitrary" -version = "1.0.3" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "510c76ecefdceada737ea728f4f9a84bd2e1ef29f1ba555e560940fe279954de" +checksum = "29d47fbf90d5149a107494b15a7dc8d69b351be2db3bb9691740e88ec17fd880" [[package]] name = "array-bytes" @@ -12686,6 +12686,7 @@ dependencies = [ name = "xcm-simulator-fuzzer" version = "0.9.31" dependencies = [ + "arbitrary", "frame-support", "frame-system", "honggfuzz",