From dd8bfcf0f0869cd6a256926444cd83949c36cf1f Mon Sep 17 00:00:00 2001 From: Addison Crump Date: Thu, 21 Sep 2023 23:40:31 +0200 Subject: [PATCH 1/6] improved libfuzzer corpus --- libafl/src/corpus/inmemory.rs | 16 +- libafl/src/corpus/mod.rs | 2 +- libafl/src/monitors/mod.rs | 3 +- libafl_bolts/src/lib.rs | 20 +- .../libafl_libfuzzer_runtime/Cargo.toml | 2 +- .../libafl_libfuzzer_runtime/src/corpus.rs | 321 ++++++++++++++++++ .../libafl_libfuzzer_runtime/src/feedbacks.rs | 20 +- .../libafl_libfuzzer_runtime/src/lib.rs | 33 +- .../libafl_libfuzzer_runtime/src/merge.rs | 25 +- .../libafl_libfuzzer_runtime/src/options.rs | 42 ++- .../libafl_libfuzzer_runtime/src/tmin.rs | 16 +- 11 files changed, 416 insertions(+), 84 deletions(-) create mode 100644 libafl_libfuzzer/libafl_libfuzzer_runtime/src/corpus.rs diff --git a/libafl/src/corpus/inmemory.rs b/libafl/src/corpus/inmemory.rs index b10067ef5a..2016583cb2 100644 --- a/libafl/src/corpus/inmemory.rs +++ b/libafl/src/corpus/inmemory.rs @@ -183,7 +183,7 @@ where /// Get the next id given a `CorpusId` (creation order) #[cfg(not(feature = "corpus_btreemap"))] #[must_use] - fn next(&self, idx: CorpusId) -> Option { + pub fn next(&self, idx: CorpusId) -> Option { if let Some(item) = self.map.get(&idx) { item.next } else { @@ -194,7 +194,7 @@ where /// Get the next id given a `CorpusId` (creation order) #[cfg(feature = "corpus_btreemap")] #[must_use] - fn next(&self, idx: CorpusId) -> Option { + pub fn next(&self, idx: CorpusId) -> Option { // TODO see if using self.keys is faster let mut range = self .map @@ -214,7 +214,7 @@ where /// Get the previous id given a `CorpusId` (creation order) #[cfg(not(feature = "corpus_btreemap"))] #[must_use] - fn prev(&self, idx: CorpusId) -> Option { + pub fn prev(&self, idx: CorpusId) -> Option { if let Some(item) = self.map.get(&idx) { item.prev } else { @@ -225,7 +225,7 @@ where /// Get the previous id given a `CorpusId` (creation order) #[cfg(feature = "corpus_btreemap")] #[must_use] - fn prev(&self, idx: CorpusId) -> Option { + pub fn prev(&self, idx: CorpusId) -> Option { // TODO see if using self.keys is faster let mut range = self .map @@ -245,28 +245,28 @@ where /// Get the first created id #[cfg(not(feature = "corpus_btreemap"))] #[must_use] - fn first(&self) -> Option { + pub fn first(&self) -> Option { self.first_idx } /// Get the first created id #[cfg(feature = "corpus_btreemap")] #[must_use] - fn first(&self) -> Option { + pub fn first(&self) -> Option { self.map.iter().next().map(|x| *x.0) } /// Get the last created id #[cfg(not(feature = "corpus_btreemap"))] #[must_use] - fn last(&self) -> Option { + pub fn last(&self) -> Option { self.last_idx } /// Get the last created id #[cfg(feature = "corpus_btreemap")] #[must_use] - fn last(&self) -> Option { + pub fn last(&self) -> Option { self.map.iter().next_back().map(|x| *x.0) } diff --git a/libafl/src/corpus/mod.rs b/libafl/src/corpus/mod.rs index bdbc247b37..856b80f11d 100644 --- a/libafl/src/corpus/mod.rs +++ b/libafl/src/corpus/mod.rs @@ -36,7 +36,7 @@ use crate::{inputs::UsesInput, Error}; /// An abstraction for the index that identify a testcase in the corpus #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] #[repr(transparent)] -pub struct CorpusId(pub(crate) usize); +pub struct CorpusId(pub usize); // MUST BE PUB -- downstream Corpus/Input implementations need it impl fmt::Display for CorpusId { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { diff --git a/libafl/src/monitors/mod.rs b/libafl/src/monitors/mod.rs index a488572232..e1f200d78a 100644 --- a/libafl/src/monitors/mod.rs +++ b/libafl/src/monitors/mod.rs @@ -21,7 +21,7 @@ use core::{fmt, fmt::Write, time::Duration}; #[cfg(feature = "std")] pub use disk::{OnDiskJSONMonitor, OnDiskTOMLMonitor}; use hashbrown::HashMap; -use libafl_bolts::{current_time, format_duration_hms, ClientId}; +use libafl_bolts::{current_time, format_duration_hms, ClientId, Error}; use serde::{Deserialize, Serialize}; #[cfg(feature = "afl_exec_sec")] @@ -210,6 +210,7 @@ impl ClientStats { /// Update the user-defined stat with name and value pub fn update_user_stats(&mut self, name: String, value: UserStats) { + log::info!("{}", Error::unknown("dumping backtrace for monitoring")); self.user_monitor.insert(name, value); } diff --git a/libafl_bolts/src/lib.rs b/libafl_bolts/src/lib.rs index 906d29e37f..4d24a68507 100644 --- a/libafl_bolts/src/lib.rs +++ b/libafl_bolts/src/lib.rs @@ -139,7 +139,6 @@ use alloc::vec::Vec; use core::hash::BuildHasher; #[cfg(any(feature = "xxh3", feature = "alloc"))] use core::hash::Hasher; -use core::{iter::Iterator, time}; #[cfg(feature = "std")] use std::time::{SystemTime, UNIX_EPOCH}; @@ -178,8 +177,11 @@ extern crate libafl_derive; use alloc::string::{FromUtf8Error, String}; use core::{ array::TryFromSliceError, + cell::{BorrowError, BorrowMutError}, fmt::{self, Display}, + iter::Iterator, num::{ParseIntError, TryFromIntError}, + time, }; #[cfg(feature = "std")] use std::{env::VarError, io}; @@ -443,6 +445,22 @@ impl Display for Error { } } +impl From for Error { + fn from(err: BorrowError) -> Self { + Self::illegal_state(format!( + "Couldn't borrow from a RefCell as immutable: {err:?}" + )) + } +} + +impl From for Error { + fn from(err: BorrowMutError) -> Self { + Self::illegal_state(format!( + "Couldn't borrow from a RefCell as mutable: {err:?}" + )) + } +} + /// Stringify the postcard serializer error #[cfg(feature = "alloc")] impl From for Error { diff --git a/libafl_libfuzzer/libafl_libfuzzer_runtime/Cargo.toml b/libafl_libfuzzer/libafl_libfuzzer_runtime/Cargo.toml index 160c926c33..b20aa4c1bc 100644 --- a/libafl_libfuzzer/libafl_libfuzzer_runtime/Cargo.toml +++ b/libafl_libfuzzer/libafl_libfuzzer_runtime/Cargo.toml @@ -40,7 +40,7 @@ log = "0.4.17" mimalloc = { version = "0.1.34", default-features = false, optional = true } num-traits = "0.2.15" rand = "0.8.5" -serde = { version = "1.0", default-features = false, features = ["alloc", "derive"] } # serialization lib +serde = { version = "1.0", features = ["derive"] } # serialization lib # clippy-suggested optimised byte counter bytecount = "0.6.3" diff --git a/libafl_libfuzzer/libafl_libfuzzer_runtime/src/corpus.rs b/libafl_libfuzzer/libafl_libfuzzer_runtime/src/corpus.rs new file mode 100644 index 0000000000..9e83e3c2de --- /dev/null +++ b/libafl_libfuzzer/libafl_libfuzzer_runtime/src/corpus.rs @@ -0,0 +1,321 @@ +use std::{ + cell::RefCell, + collections::{hash_map::Entry, BTreeMap, HashMap}, + io::ErrorKind, + path::PathBuf, + sync::atomic::{AtomicU64, Ordering}, +}; + +use libafl::{ + corpus::{inmemory::TestcaseStorage, Corpus, CorpusId, Testcase}, + inputs::{Input, UsesInput}, +}; +use libafl_bolts::Error; +use serde::{Deserialize, Serialize}; + +/// A corpus which attempts to mimic the behaviour of libFuzzer. +#[derive(Deserialize, Serialize, Debug)] +#[serde(bound = "I: serde::de::DeserializeOwned")] +pub struct LibfuzzerCorpus +where + I: Input + Serialize, +{ + corpus_dir: PathBuf, + loaded_mapping: RefCell>, + loaded_entries: RefCell>, + mapping: TestcaseStorage, + max_len: usize, + + current: Option, + next_recency: AtomicU64, +} + +impl LibfuzzerCorpus +where + I: Input + Serialize + for<'de> Deserialize<'de>, +{ + pub fn new(corpus_dir: PathBuf, max_len: usize) -> Self { + Self { + corpus_dir, + loaded_mapping: RefCell::new(Default::default()), + loaded_entries: RefCell::new(Default::default()), + mapping: TestcaseStorage::new(), + max_len, + current: None, + next_recency: AtomicU64::new(0), + } + } + + pub fn dir_path(&self) -> &PathBuf { + &self.corpus_dir + } + + /// Touch this index and maybe evict an entry if we have touched an input which was unloaded. + fn touch(&self, idx: CorpusId) -> Result<(), Error> { + let mut loaded_mapping = self.loaded_mapping.borrow_mut(); + let mut loaded_entries = self.loaded_entries.borrow_mut(); + match loaded_mapping.entry(idx) { + Entry::Occupied(mut e) => { + let &old = e.get(); + let new = self.next_recency.fetch_add(1, Ordering::Relaxed); + e.insert(new); + loaded_entries.remove(&old); + loaded_entries.insert(new, idx); + } + Entry::Vacant(e) => { + // new entry! send it in + let new = self.next_recency.fetch_add(1, Ordering::Relaxed); + e.insert(new); + loaded_entries.insert(new, idx); + } + } + if loaded_entries.len() > self.max_len { + let idx = loaded_entries.pop_first().unwrap().1; // cannot panic + let cell = self.mapping.get(idx).ok_or_else(|| { + Error::key_not_found(format!("Tried to evict non-existent entry {idx}")) + })?; + let mut tc = cell.try_borrow_mut()?; + let _ = tc.input_mut().take(); + } + Ok(()) + } +} + +impl UsesInput for LibfuzzerCorpus +where + I: Input + Serialize + for<'de> Deserialize<'de>, +{ + type Input = I; +} + +impl Corpus for LibfuzzerCorpus +where + I: Input + Serialize + for<'de> Deserialize<'de>, +{ + fn count(&self) -> usize { + self.mapping.map.len() + } + + fn add(&mut self, testcase: Testcase) -> Result { + let idx = self.mapping.insert(RefCell::new(testcase)); + let mut testcase = self.mapping.get(idx).unwrap().borrow_mut(); + + match testcase.file_path() { + Some(path) if path.canonicalize()?.starts_with(&self.corpus_dir) => { + // if it's already in the correct dir, we retain it + } + _ => { + let input = testcase.input().as_ref().ok_or_else(|| { + Error::empty( + "The testcase, when added to the corpus, must have an input present!", + ) + })?; + let name = input.generate_name(idx.0); + let path = self.corpus_dir.join(&name); + + match input.to_file(&path) { + Err(Error::File(e, _)) if e.kind() == ErrorKind::AlreadyExists => { + // we do not care if the file already exists; in this case, we assume it is equal + } + res => res?, + } + + // we DO NOT save metadata! + + testcase.filename_mut().replace(name); + testcase.file_path_mut().replace(path); + } + }; + + self.touch(idx)?; + Ok(idx) + } + + fn replace( + &mut self, + _idx: CorpusId, + _testcase: Testcase, + ) -> Result, Error> { + unimplemented!("It is unsafe to use this corpus variant with replace!"); + } + + fn remove(&mut self, _id: CorpusId) -> Result, Error> { + unimplemented!("It is unsafe to use this corpus variant with replace!"); + } + + fn get(&self, id: CorpusId) -> Result<&RefCell>, Error> { + self.touch(id)?; + self.mapping.map.get(&id).map(|item| &item.testcase).ok_or_else(|| Error::illegal_state("Nonexistent corpus entry {id} requested (present in loaded entries, but not the mapping?)")) + } + + fn current(&self) -> &Option { + &self.current + } + + fn current_mut(&mut self) -> &mut Option { + &mut self.current + } + + fn next(&self, id: CorpusId) -> Option { + self.mapping.next(id) + } + + fn prev(&self, id: CorpusId) -> Option { + self.mapping.prev(id) + } + + fn first(&self) -> Option { + self.mapping.first() + } + + fn last(&self) -> Option { + self.mapping.last() + } + + fn load_input_into(&self, testcase: &mut Testcase) -> Result<(), Error> { + // we don't need to update the loaded testcases because it must have already been loaded + if testcase.input().is_none() { + let path = testcase.file_path().as_ref().ok_or_else(|| { + Error::empty("The testcase, when being saved, must have a file path!") + })?; + let input = I::from_file(path)?; + testcase.input_mut().replace(input); + } + Ok(()) + } + + fn store_input_from(&self, testcase: &Testcase) -> Result<(), Error> { + let input = testcase.input().as_ref().ok_or_else(|| { + Error::empty("The testcase, when being saved, must have an input present!") + })?; + let path = testcase.file_path().as_ref().ok_or_else(|| { + Error::empty("The testcase, when being saved, must have a file path!") + })?; + match input.to_file(path) { + Err(Error::File(e, _)) if e.kind() == ErrorKind::AlreadyExists => { + // we do not care if the file already exists; in this case, we assume it is equal + Ok(()) + } + res => res, + } + } +} + +/// A corpus which attempts to mimic the behaviour of libFuzzer's crash output. +#[derive(Deserialize, Serialize, Debug)] +#[serde(bound = "I: serde::de::DeserializeOwned")] +pub struct ArtifactCorpus +where + I: Input + Serialize, +{ + last: Option>>, + count: usize, +} + +impl ArtifactCorpus +where + I: Input + Serialize + for<'de> Deserialize<'de>, +{ + pub fn new() -> Self { + Self { + last: None, + count: 0, + } + } +} + +impl UsesInput for ArtifactCorpus +where + I: Input + Serialize + for<'de> Deserialize<'de>, +{ + type Input = I; +} + +impl Corpus for ArtifactCorpus +where + I: Input + Serialize + for<'de> Deserialize<'de>, +{ + fn count(&self) -> usize { + self.count + } + + fn add(&mut self, testcase: Testcase) -> Result { + let idx = self.count; + self.count += 1; + + let input = testcase.input().as_ref().ok_or_else(|| { + Error::empty("The testcase, when added to the corpus, must have an input present!") + })?; + let path = testcase.file_path().as_ref().ok_or_else(|| { + Error::illegal_state("Should have set the path in the LibfuzzerCrashCauseFeedback.") + })?; + match input.to_file(&path) { + Err(Error::File(e, _)) if e.kind() == ErrorKind::AlreadyExists => { + // we do not care if the file already exists; in this case, we assume it is equal + } + res => res?, + } + + // we DO NOT save metadata! + + Ok(CorpusId::from(idx)) + } + + fn replace( + &mut self, + _idx: CorpusId, + _testcase: Testcase, + ) -> Result, Error> { + unimplemented!("Artifact prefix is thin and cannot get, replace, or remove.") + } + + fn remove(&mut self, _id: CorpusId) -> Result, Error> { + unimplemented!("Artifact prefix is thin and cannot get, replace, or remove.") + } + + fn get(&self, id: CorpusId) -> Result<&RefCell>, Error> { + let maybe_last = if self + .count + .checked_sub(1) + .map(CorpusId::from) + .map_or(false, |last| last == id) + { + self.last.as_ref() + } else { + None + }; + maybe_last.ok_or_else(|| Error::illegal_argument("Can only get the last corpus ID.")) + } + + fn current(&self) -> &Option { + unimplemented!("Artifact prefix is thin and cannot get, replace, or remove.") + } + + fn current_mut(&mut self) -> &mut Option { + unimplemented!("Artifact prefix is thin and cannot get, replace, or remove.") + } + + fn next(&self, _id: CorpusId) -> Option { + unimplemented!("Artifact prefix is thin and cannot get, replace, or remove.") + } + + fn prev(&self, _id: CorpusId) -> Option { + unimplemented!("Artifact prefix is thin and cannot get, replace, or remove.") + } + + fn first(&self) -> Option { + unimplemented!("Artifact prefix is thin and cannot get, replace, or remove.") + } + + fn last(&self) -> Option { + self.count.checked_sub(1).map(CorpusId::from) + } + + fn load_input_into(&self, _testcase: &mut Testcase) -> Result<(), Error> { + unimplemented!("Artifact prefix is thin and cannot get, replace, or remove.") + } + + fn store_input_from(&self, _testcase: &Testcase) -> Result<(), Error> { + unimplemented!("Artifact prefix is thin and cannot get, replace, or remove.") + } +} diff --git a/libafl_libfuzzer/libafl_libfuzzer_runtime/src/feedbacks.rs b/libafl_libfuzzer/libafl_libfuzzer_runtime/src/feedbacks.rs index 0cbdd91fc2..a987eaf703 100644 --- a/libafl_libfuzzer/libafl_libfuzzer_runtime/src/feedbacks.rs +++ b/libafl_libfuzzer/libafl_libfuzzer_runtime/src/feedbacks.rs @@ -1,6 +1,5 @@ use alloc::rc::Rc; use core::{cell::RefCell, fmt::Debug}; -use std::path::PathBuf; use libafl::{ alloc, @@ -77,12 +76,12 @@ impl LibfuzzerCrashCauseMetadata { #[derive(Debug)] pub struct LibfuzzerCrashCauseFeedback { - artifact_prefix: Option, + artifact_prefix: ArtifactPrefix, exit_kind: ExitKind, } impl LibfuzzerCrashCauseFeedback { - pub fn new(artifact_prefix: Option) -> Self { + pub fn new(artifact_prefix: ArtifactPrefix) -> Self { Self { artifact_prefix, exit_kind: ExitKind::Ok, @@ -104,17 +103,10 @@ impl LibfuzzerCrashCauseFeedback { let name = testcase.input().as_ref().unwrap().generate_name(0); name }; - let file_path = if let Some(artifact_prefix) = self.artifact_prefix.as_ref() { - if let Some(filename_prefix) = artifact_prefix.filename_prefix() { - artifact_prefix - .dir() - .join(format!("{filename_prefix}{prefix}-{base}")) - } else { - artifact_prefix.dir().join(format!("{prefix}-{base}")) - } - } else { - PathBuf::from(format!("{prefix}-{base}")) - }; + let file_path = self.artifact_prefix.dir().join(format!( + "{}{prefix}-{base}", + self.artifact_prefix.filename_prefix() + )); *testcase.file_path_mut() = Some(file_path); } } diff --git a/libafl_libfuzzer/libafl_libfuzzer_runtime/src/lib.rs b/libafl_libfuzzer/libafl_libfuzzer_runtime/src/lib.rs index 155a9d2a3f..12734b60b2 100644 --- a/libafl_libfuzzer/libafl_libfuzzer_runtime/src/lib.rs +++ b/libafl_libfuzzer/libafl_libfuzzer_runtime/src/lib.rs @@ -79,9 +79,11 @@ use libafl::{ Error, }; use libafl_bolts::AsSlice; +use libc::_exit; use crate::options::{LibfuzzerMode, LibfuzzerOptions}; +mod corpus; mod feedbacks; mod fuzz; mod merge; @@ -152,7 +154,7 @@ macro_rules! fuzz_with { AsSlice, }; use libafl::{ - corpus::{CachedOnDiskCorpus, Corpus, OnDiskCorpus}, + corpus::Corpus, executors::{ExitKind, InProcessExecutor, TimeoutExecutor}, feedback_and_fast, feedback_not, feedback_or, feedback_or_fast, feedbacks::{ConstFeedback, CrashFeedback, MaxMapFeedback, NewHashFeedback, TimeFeedback, TimeoutFeedback}, @@ -179,6 +181,7 @@ macro_rules! fuzz_with { use std::{env::temp_dir, fs::create_dir, path::PathBuf}; use crate::{BACKTRACE, CustomMutationStatus}; + use crate::corpus::{ArtifactCorpus, LibfuzzerCorpus}; use crate::feedbacks::{LibfuzzerCrashCauseFeedback, LibfuzzerKeepFeedback, ShrinkMapFeedback}; use crate::misc::should_use_grimoire; use crate::observers::{MappedEdgeMapObserver, SizeValueObserver}; @@ -243,7 +246,7 @@ macro_rules! fuzz_with { // A feedback to choose if an input is a solution or not let mut objective = feedback_or_fast!( - LibfuzzerCrashCauseFeedback::new($options.artifact_prefix().cloned()), + LibfuzzerCrashCauseFeedback::new($options.artifact_prefix().clone()), OomFeedback, feedback_and_fast!( CrashFeedback::new(), @@ -269,13 +272,7 @@ macro_rules! fuzz_with { dir }; - let crash_corpus = if let Some(prefix) = $options.artifact_prefix() { - OnDiskCorpus::with_meta_format_and_prefix(prefix.dir(), None, prefix.filename_prefix().clone(), false) - .unwrap() - } else { - OnDiskCorpus::with_meta_format_and_prefix(&std::env::current_dir().unwrap(), None, None, false) - .unwrap() - }; + let crash_corpus = ArtifactCorpus::new(); // If not restarting, create a State from scratch let mut state = state.unwrap_or_else(|| { @@ -283,7 +280,7 @@ macro_rules! fuzz_with { // RNG StdRand::with_seed(current_nanos()), // Corpus that will be evolved, we keep it in memory for performance - CachedOnDiskCorpus::with_meta_format_and_prefix(corpus_dir.clone(), 4096, None, None, true).unwrap(), + LibfuzzerCorpus::new(corpus_dir.clone(), 4096), // Corpus in which we store solutions (crashes in this example), // on disk so the user can get them after stopping the fuzzer crash_corpus, @@ -591,7 +588,21 @@ pub unsafe extern "C" fn LLVMFuzzerRunDriver( .unwrap(); if !options.unknown().is_empty() { - println!("Unrecognised options: {:?}", options.unknown()); + eprintln!("Unrecognised options: {:?}", options.unknown()); + } + + for folder in options + .dirs() + .iter() + .chain(std::iter::once(options.artifact_prefix().dir())) + { + if !folder.try_exists().unwrap_or(false) { + eprintln!( + "Required folder {} did not exist; failing fast.", + folder.to_string_lossy() + ); + _exit(1); + } } if *options.mode() != LibfuzzerMode::Tmin diff --git a/libafl_libfuzzer/libafl_libfuzzer_runtime/src/merge.rs b/libafl_libfuzzer/libafl_libfuzzer_runtime/src/merge.rs index 16f304d407..f8d26811e1 100644 --- a/libafl_libfuzzer/libafl_libfuzzer_runtime/src/merge.rs +++ b/libafl_libfuzzer/libafl_libfuzzer_runtime/src/merge.rs @@ -8,7 +8,7 @@ use std::{ }; use libafl::{ - corpus::{Corpus, OnDiskCorpus}, + corpus::Corpus, events::{EventRestarter, SimpleRestartingEventManager}, executors::{ExitKind, InProcessExecutor, TimeoutExecutor}, feedback_and_fast, feedback_or_fast, @@ -29,6 +29,7 @@ use libafl_bolts::{ use libafl_targets::{OomFeedback, OomObserver, COUNTERS_MAPS}; use crate::{ + corpus::{ArtifactCorpus, LibfuzzerCorpus}, feedbacks::{LibfuzzerCrashCauseFeedback, LibfuzzerKeepFeedback}, observers::{MappedEdgeMapObserver, SizeTimeValueObserver}, options::LibfuzzerOptions, @@ -44,23 +45,7 @@ pub fn merge( return Err(Error::illegal_argument("Missing corpora to minimize; you should provide one directory to minimize into and one-to-many from which the inputs are loaded.")); } - let crash_corpus = if let Some(prefix) = options.artifact_prefix() { - OnDiskCorpus::with_meta_format_and_prefix( - prefix.dir(), - None, - prefix.filename_prefix().clone(), - true, - ) - .unwrap() - } else { - OnDiskCorpus::with_meta_format_and_prefix( - &std::env::current_dir().unwrap(), - None, - None, - true, - ) - .unwrap() - }; + let crash_corpus = ArtifactCorpus::new(); let keep_observer = LibfuzzerKeepFeedback::new(); let keep = keep_observer.keep(); @@ -130,7 +115,7 @@ pub fn merge( // A feedback to choose if an input is a solution or not let mut objective = feedback_or_fast!( - LibfuzzerCrashCauseFeedback::new(options.artifact_prefix().cloned()), + LibfuzzerCrashCauseFeedback::new(options.artifact_prefix().clone()), OomFeedback, CrashFeedback::new(), TimeoutFeedback::new() @@ -163,7 +148,7 @@ pub fn merge( // RNG StdRand::new(), // Corpus that will be evolved, we keep it in memory for performance - OnDiskCorpus::with_meta_format_and_prefix(&corpus_dir, None, None, true).unwrap(), + LibfuzzerCorpus::new(corpus_dir, 4096), // Corpus in which we store solutions (crashes in this example), // on disk so the user can get them after stopping the fuzzer crash_corpus, diff --git a/libafl_libfuzzer/libafl_libfuzzer_runtime/src/options.rs b/libafl_libfuzzer/libafl_libfuzzer_runtime/src/options.rs index e9fdab7521..632cb7b383 100644 --- a/libafl_libfuzzer/libafl_libfuzzer_runtime/src/options.rs +++ b/libafl_libfuzzer/libafl_libfuzzer_runtime/src/options.rs @@ -2,6 +2,7 @@ use core::fmt::{Display, Formatter}; use std::{path::PathBuf, time::Duration}; use libafl::mutators::Tokens; +use serde::{Deserialize, Serialize}; use crate::options::RawOption::{Directory, Flag}; @@ -52,10 +53,10 @@ impl<'a> Display for OptionsParseError<'a> { } } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct ArtifactPrefix { dir: PathBuf, - filename_prefix: Option, + filename_prefix: String, } impl ArtifactPrefix { @@ -64,14 +65,17 @@ impl ArtifactPrefix { if path.ends_with(std::path::MAIN_SEPARATOR) { Self { dir, - filename_prefix: None, + filename_prefix: "".to_string(), } } else { - let filename_prefix = dir.file_name().map(|s| { - s.to_os_string() - .into_string() - .expect("Provided artifact prefix is not usable") - }); + let filename_prefix = dir.file_name().map_or_else( + || "".to_string(), + |s| { + s.to_os_string() + .into_string() + .expect("Provided artifact prefix is not usable") + }, + ); dir.pop(); Self { dir, @@ -84,17 +88,26 @@ impl ArtifactPrefix { &self.dir } - pub fn filename_prefix(&self) -> &Option { + pub fn filename_prefix(&self) -> &str { &self.filename_prefix } } +impl Default for ArtifactPrefix { + fn default() -> Self { + Self { + dir: std::env::current_dir().expect("Must be able to get the current directory!"), + filename_prefix: "".to_string(), + } + } +} + #[derive(Debug, Clone)] #[allow(clippy::struct_excessive_bools)] pub struct LibfuzzerOptions { fuzzer_name: String, mode: LibfuzzerMode, - artifact_prefix: Option, + artifact_prefix: ArtifactPrefix, timeout: Duration, grimoire: Option, forks: Option, @@ -140,8 +153,8 @@ impl LibfuzzerOptions { &self.mode } - pub fn artifact_prefix(&self) -> Option<&ArtifactPrefix> { - self.artifact_prefix.as_ref() + pub fn artifact_prefix(&self) -> &ArtifactPrefix { + &self.artifact_prefix } pub fn timeout(&self) -> Duration { @@ -333,7 +346,10 @@ impl<'a> LibfuzzerOptionsBuilder<'a> { LibfuzzerOptions { fuzzer_name, mode: self.mode.unwrap_or(LibfuzzerMode::Fuzz), - artifact_prefix: self.artifact_prefix.map(ArtifactPrefix::new), + artifact_prefix: self + .artifact_prefix + .map(ArtifactPrefix::new) + .unwrap_or_default(), timeout: self.timeout.unwrap_or(Duration::from_secs(1200)), grimoire: self.grimoire, forks: self.forks, diff --git a/libafl_libfuzzer/libafl_libfuzzer_runtime/src/tmin.rs b/libafl_libfuzzer/libafl_libfuzzer_runtime/src/tmin.rs index 7ad5ab5546..528e60b603 100644 --- a/libafl_libfuzzer/libafl_libfuzzer_runtime/src/tmin.rs +++ b/libafl_libfuzzer/libafl_libfuzzer_runtime/src/tmin.rs @@ -1,7 +1,6 @@ use std::{ ffi::c_int, fs::{read, write}, - path::PathBuf, }; use libafl::{ @@ -119,21 +118,10 @@ fn minimize_crash_with_mutator>( options.dirs()[0].as_path().as_os_str().to_str().unwrap() ); } else { - let (mut dest, filename_prefix) = options.artifact_prefix().map_or_else( - || (PathBuf::default(), ""), - |artifact_prefix| { - ( - artifact_prefix.dir().clone(), - artifact_prefix - .filename_prefix() - .as_ref() - .map_or("", String::as_str), - ) - }, - ); + let mut dest = options.artifact_prefix().dir().clone(); dest.push(format!( "{}minimized-from-{}", - filename_prefix, + options.artifact_prefix().filename_prefix(), options.dirs()[0].file_name().unwrap().to_str().unwrap() )); write(&dest, input)?; From 3ad4e7ca5f3961603d06ffd10a8c1ca18df2921c Mon Sep 17 00:00:00 2001 From: Addison Crump Date: Fri, 22 Sep 2023 15:09:00 +0200 Subject: [PATCH 2/6] use .into() for converting ids to usize --- libafl/src/corpus/mod.rs | 2 +- libafl_libfuzzer/libafl_libfuzzer_runtime/src/corpus.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libafl/src/corpus/mod.rs b/libafl/src/corpus/mod.rs index 856b80f11d..4406d12965 100644 --- a/libafl/src/corpus/mod.rs +++ b/libafl/src/corpus/mod.rs @@ -36,7 +36,7 @@ use crate::{inputs::UsesInput, Error}; /// An abstraction for the index that identify a testcase in the corpus #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] #[repr(transparent)] -pub struct CorpusId(pub usize); // MUST BE PUB -- downstream Corpus/Input implementations need it +pub struct CorpusId(usize); impl fmt::Display for CorpusId { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { diff --git a/libafl_libfuzzer/libafl_libfuzzer_runtime/src/corpus.rs b/libafl_libfuzzer/libafl_libfuzzer_runtime/src/corpus.rs index 9e83e3c2de..860fe466d2 100644 --- a/libafl_libfuzzer/libafl_libfuzzer_runtime/src/corpus.rs +++ b/libafl_libfuzzer/libafl_libfuzzer_runtime/src/corpus.rs @@ -110,7 +110,7 @@ where "The testcase, when added to the corpus, must have an input present!", ) })?; - let name = input.generate_name(idx.0); + let name = input.generate_name(idx.into()); let path = self.corpus_dir.join(&name); match input.to_file(&path) { From a5ca4b5c41533050a4bf1662380ae980b64acdf9 Mon Sep 17 00:00:00 2001 From: Addison Crump Date: Fri, 22 Sep 2023 15:21:07 +0200 Subject: [PATCH 3/6] oops --- libafl/src/corpus/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libafl/src/corpus/mod.rs b/libafl/src/corpus/mod.rs index 4406d12965..bdbc247b37 100644 --- a/libafl/src/corpus/mod.rs +++ b/libafl/src/corpus/mod.rs @@ -36,7 +36,7 @@ use crate::{inputs::UsesInput, Error}; /// An abstraction for the index that identify a testcase in the corpus #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] #[repr(transparent)] -pub struct CorpusId(usize); +pub struct CorpusId(pub(crate) usize); impl fmt::Display for CorpusId { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { From 2830ae5c1daf6628272399952b8efd25005cbebb Mon Sep 17 00:00:00 2001 From: Addison Crump Date: Fri, 22 Sep 2023 19:27:41 +0200 Subject: [PATCH 4/6] fix warning about unused arg --- libafl_bolts/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libafl_bolts/src/lib.rs b/libafl_bolts/src/lib.rs index 4d24a68507..2bf52ea849 100644 --- a/libafl_bolts/src/lib.rs +++ b/libafl_bolts/src/lib.rs @@ -446,7 +446,7 @@ impl Display for Error { } impl From for Error { - fn from(err: BorrowError) -> Self { + fn from(_: BorrowError) -> Self { Self::illegal_state(format!( "Couldn't borrow from a RefCell as immutable: {err:?}" )) @@ -454,7 +454,7 @@ impl From for Error { } impl From for Error { - fn from(err: BorrowMutError) -> Self { + fn from(_: BorrowMutError) -> Self { Self::illegal_state(format!( "Couldn't borrow from a RefCell as mutable: {err:?}" )) From 6cac8a3ee75772ceaae88b2132f36445203bcb1a Mon Sep 17 00:00:00 2001 From: Addison Crump Date: Fri, 22 Sep 2023 19:43:44 +0200 Subject: [PATCH 5/6] fix some lingering CI errors --- libafl_bolts/src/lib.rs | 14 +++++++++----- .../libafl_libfuzzer_runtime/src/corpus.rs | 6 +++--- .../libafl_libfuzzer_runtime/src/options.rs | 17 +++++++---------- 3 files changed, 19 insertions(+), 18 deletions(-) diff --git a/libafl_bolts/src/lib.rs b/libafl_bolts/src/lib.rs index 2bf52ea849..fdddbd7666 100644 --- a/libafl_bolts/src/lib.rs +++ b/libafl_bolts/src/lib.rs @@ -173,11 +173,8 @@ pub mod launcher {} #[allow(unused_imports)] #[macro_use] extern crate libafl_derive; -#[cfg(feature = "alloc")] -use alloc::string::{FromUtf8Error, String}; use core::{ array::TryFromSliceError, - cell::{BorrowError, BorrowMutError}, fmt::{self, Display}, iter::Iterator, num::{ParseIntError, TryFromIntError}, @@ -188,6 +185,11 @@ use std::{env::VarError, io}; #[cfg(feature = "libafl_derive")] pub use libafl_derive::SerdeAny; +#[cfg(feature = "alloc")] +use { + alloc::string::{FromUtf8Error, String}, + core::cell::{BorrowError, BorrowMutError}, +}; /// We need fixed names for many parts of this lib. pub trait Named { @@ -445,16 +447,18 @@ impl Display for Error { } } +#[cfg(feature = "alloc")] impl From for Error { - fn from(_: BorrowError) -> Self { + fn from(err: BorrowError) -> Self { Self::illegal_state(format!( "Couldn't borrow from a RefCell as immutable: {err:?}" )) } } +#[cfg(feature = "alloc")] impl From for Error { - fn from(_: BorrowMutError) -> Self { + fn from(err: BorrowMutError) -> Self { Self::illegal_state(format!( "Couldn't borrow from a RefCell as mutable: {err:?}" )) diff --git a/libafl_libfuzzer/libafl_libfuzzer_runtime/src/corpus.rs b/libafl_libfuzzer/libafl_libfuzzer_runtime/src/corpus.rs index 860fe466d2..ac7617e6ef 100644 --- a/libafl_libfuzzer/libafl_libfuzzer_runtime/src/corpus.rs +++ b/libafl_libfuzzer/libafl_libfuzzer_runtime/src/corpus.rs @@ -37,8 +37,8 @@ where pub fn new(corpus_dir: PathBuf, max_len: usize) -> Self { Self { corpus_dir, - loaded_mapping: RefCell::new(Default::default()), - loaded_entries: RefCell::new(Default::default()), + loaded_mapping: RefCell::new(HashMap::default()), + loaded_entries: RefCell::new(BTreeMap::default()), mapping: TestcaseStorage::new(), max_len, current: None, @@ -249,7 +249,7 @@ where let path = testcase.file_path().as_ref().ok_or_else(|| { Error::illegal_state("Should have set the path in the LibfuzzerCrashCauseFeedback.") })?; - match input.to_file(&path) { + match input.to_file(path) { Err(Error::File(e, _)) if e.kind() == ErrorKind::AlreadyExists => { // we do not care if the file already exists; in this case, we assume it is equal } diff --git a/libafl_libfuzzer/libafl_libfuzzer_runtime/src/options.rs b/libafl_libfuzzer/libafl_libfuzzer_runtime/src/options.rs index 632cb7b383..0e73a80483 100644 --- a/libafl_libfuzzer/libafl_libfuzzer_runtime/src/options.rs +++ b/libafl_libfuzzer/libafl_libfuzzer_runtime/src/options.rs @@ -65,17 +65,14 @@ impl ArtifactPrefix { if path.ends_with(std::path::MAIN_SEPARATOR) { Self { dir, - filename_prefix: "".to_string(), + filename_prefix: String::new(), } } else { - let filename_prefix = dir.file_name().map_or_else( - || "".to_string(), - |s| { - s.to_os_string() - .into_string() - .expect("Provided artifact prefix is not usable") - }, - ); + let filename_prefix = dir.file_name().map_or_else(String::new, |s| { + s.to_os_string() + .into_string() + .expect("Provided artifact prefix is not usable") + }); dir.pop(); Self { dir, @@ -97,7 +94,7 @@ impl Default for ArtifactPrefix { fn default() -> Self { Self { dir: std::env::current_dir().expect("Must be able to get the current directory!"), - filename_prefix: "".to_string(), + filename_prefix: String::new(), } } } From 4e46c2700588e9615a514b22e6c6ff3c24d7ba32 Mon Sep 17 00:00:00 2001 From: Addison Crump Date: Fri, 22 Sep 2023 20:42:23 +0200 Subject: [PATCH 6/6] actually save the last lmao --- libafl_libfuzzer/libafl_libfuzzer_runtime/src/corpus.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/libafl_libfuzzer/libafl_libfuzzer_runtime/src/corpus.rs b/libafl_libfuzzer/libafl_libfuzzer_runtime/src/corpus.rs index ac7617e6ef..9f473bf22c 100644 --- a/libafl_libfuzzer/libafl_libfuzzer_runtime/src/corpus.rs +++ b/libafl_libfuzzer/libafl_libfuzzer_runtime/src/corpus.rs @@ -257,6 +257,7 @@ where } // we DO NOT save metadata! + self.last = Some(RefCell::new(testcase)); Ok(CorpusId::from(idx)) }