Improve fuzzer for XCMv4

polkadot/xcm/xcm-simulator/fuzzer/.gitignore

@@ -1,5 +1 @@
-hfuzz_target
-hfuzz_workspace
-cargo
-coverage
-ccov.zip
+output

polkadot/xcm/xcm-simulator/fuzzer/Cargo.toml

@@ -9,7 +9,7 @@ publish = false
 
 [dependencies]
 codec = { package = "parity-scale-codec", version = "3.6.1" }
-honggfuzz = "0.5.55"
+ziggy = { version = "0.8", default-features = false }
 arbitrary = "1.2.0"
 scale-info = { version = "2.10.0", features = ["derive"] }
 
@@ -44,7 +44,3 @@ runtime-benchmarks = [
 	"xcm-builder/runtime-benchmarks",
 	"xcm-executor/runtime-benchmarks",
 ]
-
-[[bin]]
-path = "src/fuzz.rs"
-name = "xcm-fuzzer"

polkadot/xcm/xcm-simulator/fuzzer/README.md

@@ -6,35 +6,36 @@ underflows.
 ## Install dependencies
 
 ```
-cargo install honggfuzz
+cargo install --force ziggy cargo-afl honggfuzz grcov
 ```
 
 ## Run the fuzzer
 
 In this directory, run this command:
 
 ```
-cargo hfuzz run xcm-fuzzer
+cargo ziggy fuzz
 ```
 
+You can use the following options to improve fuzzing:
+- `-j number_of_jobs` for fuzzing using multiple threads
+- `-G 1024` to limit the size of inputs to 1024 bytes
+  - this will improve fuzzing effectiveness, and you can drop the limit once you have good coverage
+
 ## Run a single input
 
 In this directory, run this command:
 
 ```
-cargo hfuzz run-debug xcm-fuzzer hfuzz_workspace/xcm-fuzzer/fuzzer_input_file
+cargo ziggy run -i path/to/your/input
 ```
 
 ## Generate coverage
 
-In this directory, run these four commands:
+In this directory, run this command:
 
 ```
-RUSTFLAGS="-Zprofile -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests -Cpanic=abort" \
-CARGO_INCREMENTAL=0 SKIP_WASM_BUILD=1 CARGO_HOME=./cargo cargo build
-../../../target/debug/xcm-fuzzer hfuzz_workspace/xcm-fuzzer/input/
-zip -0 ccov.zip `find ../../../target/ \( -name "*.gc*" -o -name "test-*.gc*" \) -print`
-grcov ccov.zip -s ../../../ -t html --llvm --branch --ignore-not-existing -o ./coverage
+cargo ziggy cover
 ```
 
-The code coverage will be in `./coverage/index.html`.
+The code coverage will be in `./output/xcm-simulator-fuzzer/coverage/index.html`.

polkadot/xcm/xcm-simulator/fuzzer/src/fuzz.rs → polkadot/xcm/xcm-simulator/fuzzer/src/main.rs

@@ -98,7 +98,7 @@ impl<'a> Arbitrary<'a> for XcmMessage {
 		if let Ok(message) =
 			DecodeLimit::decode_with_depth_limit(MAX_XCM_DECODE_DEPTH, &mut encoded_message)
 		{
-			return Ok(XcmMessage { source, destination, message })
+			return Ok(XcmMessage { source, destination, message });
 		}
 		Err(Error::IncorrectFormat)
 	}
@@ -155,6 +155,35 @@ fn run_input(xcm_messages: [XcmMessage; 5]) {
 	println!();
 
 	for xcm_message in xcm_messages {
+		// We block Transact messages because they can cause panics that are due to the test
+		// environment, and not XCM
+		fn matches_blocklisted_messages(message: Instruction<()>) -> bool {
+			matches!(message, Transact { .. })
+		}
+		// We check XCM messages recursively for blocklisted messages
+		fn matches_recursive(message: &mut Instruction<()>) -> Vec<Instruction<()>> {
+			match message {
+				DepositReserveAsset { xcm, .. } |
+				InitiateReserveWithdraw { xcm, .. } |
+				InitiateTeleport { xcm, .. } |
+				TransferReserveAsset { xcm, .. } |
+				SetErrorHandler(xcm) |
+				SetAppendix(xcm) => Vec::from(xcm.inner()).iter_mut().flat_map(matches_recursive).collect(),
+				_ => vec![message.clone()],
+			}
+		}
+
+		if xcm_message
+			.message
+			.clone()
+			.iter_mut()
+			.flat_map(matches_recursive)
+			.any(|m| matches_blocklisted_messages(m))
+		{
+			println!("  skipping message\n");
+			continue;
+		}
+
 		if xcm_message.source % 4 == 0 {
 			// We get the destination for the message
 			let parachain_id = (xcm_message.destination % 3) + 1;
@@ -202,36 +231,7 @@ fn run_input(xcm_messages: [XcmMessage; 5]) {
 }
 
 fn main() {
-	#[cfg(fuzzing)]
-	{
-		loop {
-			honggfuzz::fuzz!(|xcm_messages: [XcmMessage; 5]| {
-				run_input(xcm_messages);
-			})
-		}
-	}
-	#[cfg(not(fuzzing))]
-	{
-		use std::{env, fs, fs::File, io::Read};
-		let args: Vec<_> = env::args().collect();
-		let md = fs::metadata(&args[1]).unwrap();
-		let all_files = match md.is_dir() {
-			true => fs::read_dir(&args[1])
-				.unwrap()
-				.map(|x| x.unwrap().path().to_str().unwrap().to_string())
-				.collect::<Vec<String>>(),
-			false => (args[1..]).to_vec(),
-		};
-		println!("All_files {:?}", all_files);
-		for argument in all_files {
-			println!("Now doing file {:?}", argument);
-			let mut buffer: Vec<u8> = Vec::new();
-			let mut f = File::open(argument).unwrap();
-			f.read_to_end(&mut buffer).unwrap();
-			let mut unstructured = Unstructured::new(&buffer);
-			if let Ok(xcm_messages) = unstructured.arbitrary() {
-				run_input(xcm_messages);
-			}
-		}
-	}
+	ziggy::fuzz!(|xcm_messages: [XcmMessage; 5]| {
+		run_input(xcm_messages);
+	});
 }