Don't restart in deterministic stages. Don't restart where there's no…

… restart safety. Make stage names unique (#2331)

* push

* fuck

* add

* add

* api

* api

* add multi machine to workspace

* doc

* api

* api

* add

* more

* fix

* stats

* rev

* fix

* fix

* real fix

* add

* fmt

* add

* add

* fix

* a

* add

* revert workflow

---------

Co-authored-by: Your Name <you@example.com>

.github/workflows/build_and_test.yml

@@ -277,6 +277,7 @@ jobs:
       - ubuntu
       - fuzzers-preflight
     strategy:
+      fail-fast: true
       matrix:
         os: [ ubuntu-latest ]
         fuzzer:

Cargo.toml

@@ -21,6 +21,7 @@ members = [
     "utils/libafl_benches",
     "utils/gramatron/construct_automata",
     "utils/desyscall",
+    "utils/multi_machine_generator",
 ]
 default-members = [
     "libafl",

fuzzers/baby_fuzzer_minimizing/src/main.rs

@@ -125,7 +125,7 @@ pub fn main() -> Result<(), Error> {
     let mut stages = tuple_list!(StdTMinMutationalStage::new(
         minimizer,
         CrashFeedback::new(),
-        1 << 10
+        1 << 10,
     ));
 
     let scheduler = QueueScheduler::new();

fuzzers/fuzzbench_forkserver_cmplog/src/main.rs

@@ -368,7 +368,7 @@ fn fuzz(
             .build(tuple_list!(cmplog_observer))
             .unwrap();
 
-        let tracing = AFLppCmplogTracingStage::with_cmplog_observer(cmplog_executor, cmplog_ref);
+        let tracing = AFLppCmplogTracingStage::new(cmplog_executor, cmplog_ref);
 
         // Setup a randomic Input2State stage
         let rq = MultiMutationalStage::new(AFLppRedQueen::with_cmplog_options(true, true));

fuzzers/libfuzzer_stb_image_concolic/fuzzer/src/main.rs

@@ -214,12 +214,12 @@ fn fuzz(
             // Create a concolic trace
             ConcolicTracingStage::new(
                 TracingStage::new(
-                    MyCommandConfigurator.into_executor(tuple_list!(concolic_observer))
+                    MyCommandConfigurator.into_executor(tuple_list!(concolic_observer)),
                 ),
                 concolic_ref,
             ),
             // Use the concolic trace for z3-based solving
-            SimpleConcolicMutationalStage::default(),
+            SimpleConcolicMutationalStage::new(),
         );
 
         fuzzer.fuzz_loop(&mut stages, &mut executor, &mut state, &mut restarting_mgr)?;

libafl/src/fuzzer/mod.rs

@@ -27,7 +27,7 @@ use crate::{
 use crate::{monitors::PerfFeature, state::HasClientPerfMonitor};
 
 /// Send a monitor update all 15 (or more) seconds
-const STATS_TIMEOUT_DEFAULT: Duration = Duration::from_secs(1);
+const STATS_TIMEOUT_DEFAULT: Duration = Duration::from_secs(15);
 
 /// Holds a scheduler
 pub trait HasScheduler: UsesState
@@ -247,7 +247,6 @@ where
         let monitor_timeout = STATS_TIMEOUT_DEFAULT;
 
         for _ in 0..iters {
-            // log::info!("Starting another fuzz_loop");
             manager.maybe_report_progress(state, monitor_timeout)?;
             ret = Some(self.fuzz_one(stages, executor, state, manager)?);
         }

libafl/src/stages/calibrate.rs

@@ -1,6 +1,9 @@
 //! The calibration stage. The fuzzer measures the average exec time and the bitmap size.
 
-use alloc::{borrow::Cow, vec::Vec};
+use alloc::{
+    borrow::{Cow, ToOwned},
+    vec::Vec,
+};
 use core::{fmt::Debug, marker::PhantomData, time::Duration};
 
 use hashbrown::HashSet;
@@ -17,7 +20,7 @@ use crate::{
     monitors::{AggregatorOps, UserStats, UserStatsValue},
     observers::{MapObserver, ObserversTuple},
     schedulers::powersched::SchedulerMetadata,
-    stages::{ExecutionCountRestartHelper, Stage},
+    stages::{Stage, StdRestartHelper},
     state::{HasCorpus, HasCurrentTestcase, HasExecutions, UsesState},
     Error, HasMetadata, HasNamedMetadata,
 };
@@ -75,7 +78,6 @@ pub struct CalibrationStage<C, E, O, OT> {
     stage_max: usize,
     /// If we should track stability
     track_stability: bool,
-    restart_helper: ExecutionCountRestartHelper,
     phantom: PhantomData<(E, O, OT)>,
 }
 
@@ -125,8 +127,6 @@ where
 
         let mut iter = self.stage_max;
         // If we restarted after a timeout or crash, do less iterations.
-        iter -= usize::try_from(self.restart_helper.execs_since_progress_start(state)?)?;
-
         let input = state.current_input_cloned()?;
 
         // Run once to get the initial calibration map
@@ -350,14 +350,18 @@ where
         Ok(())
     }
 
-    fn restart_progress_should_run(&mut self, state: &mut Self::State) -> Result<bool, Error> {
-        // TODO: Make sure this is the correct way / there may be a better way?
-        self.restart_helper.restart_progress_should_run(state)
+    fn should_restart(&mut self, state: &mut Self::State) -> Result<bool, Error> {
+        // Calibration stage disallow restarts
+        // If a testcase that causes crash/timeout in the queue, we need to remove it from the queue immediately.
+        StdRestartHelper::no_retry(state, &self.name)
+
+        // todo
+        // remove this guy from corpus queue
     }
 
-    fn clear_restart_progress(&mut self, state: &mut Self::State) -> Result<(), Error> {
+    fn clear_progress(&mut self, state: &mut Self::State) -> Result<(), Error> {
         // TODO: Make sure this is the correct way / there may be a better way?
-        self.restart_helper.clear_restart_progress(state)
+        StdRestartHelper::clear_progress(state, &self.name)
     }
 }
 
@@ -375,14 +379,16 @@ where
     where
         F: HasObserverHandle<Observer = C> + Named,
     {
+        let map_name = map_feedback.name().clone();
         Self {
             map_observer_handle: map_feedback.observer_handle().clone(),
-            map_name: map_feedback.name().clone(),
+            map_name: map_name.clone(),
             stage_max: CAL_STAGE_START,
             track_stability: true,
-            restart_helper: ExecutionCountRestartHelper::default(),
             phantom: PhantomData,
-            name: Cow::Borrowed(CALIBRATION_STAGE_NAME),
+            name: Cow::Owned(
+                CALIBRATION_STAGE_NAME.to_owned() + ":" + map_name.into_owned().as_str(),
+            ),
         }
     }
 
@@ -392,15 +398,9 @@ where
     where
         F: HasObserverHandle<Observer = C> + Named,
     {
-        Self {
-            map_observer_handle: map_feedback.observer_handle().clone(),
-            map_name: map_feedback.name().clone(),
-            stage_max: CAL_STAGE_START,
-            track_stability: false,
-            restart_helper: ExecutionCountRestartHelper::default(),
-            phantom: PhantomData,
-            name: Cow::Borrowed(CALIBRATION_STAGE_NAME),
-        }
+        let mut ret = Self::new(map_feedback);
+        ret.track_stability = false;
+        ret
     }
 }
 

libafl/src/stages/colorization.rs

@@ -1,5 +1,9 @@
 //! The colorization stage from `colorization()` in afl++
-use alloc::{borrow::Cow, collections::binary_heap::BinaryHeap, vec::Vec};
+use alloc::{
+    borrow::{Cow, ToOwned},
+    collections::binary_heap::BinaryHeap,
+    vec::Vec,
+};
 use core::{cmp::Ordering, fmt::Debug, marker::PhantomData, ops::Range};
 
 use libafl_bolts::{
@@ -15,7 +19,7 @@ use crate::{
     inputs::HasMutatorBytes,
     mutators::mutations::buffer_copy,
     observers::{MapObserver, ObserversTuple},
-    stages::{RetryRestartHelper, Stage},
+    stages::{Stage, StdRestartHelper},
     state::{HasCorpus, HasCurrentTestcase, HasRand, UsesState},
     Error, HasMetadata, HasNamedMetadata,
 };
@@ -104,14 +108,15 @@ where
         Ok(())
     }
 
-    fn restart_progress_should_run(&mut self, state: &mut Self::State) -> Result<bool, Error> {
-        // TODO this stage needs a proper resume
-        RetryRestartHelper::restart_progress_should_run(state, self, 3)
+    fn should_restart(&mut self, state: &mut Self::State) -> Result<bool, Error> {
+        // This is a deterministic stage
+        // Once it failed, then don't retry,
+        // It will just fail again
+        StdRestartHelper::no_retry(state, &self.name)
     }
 
-    fn clear_restart_progress(&mut self, state: &mut Self::State) -> Result<(), Error> {
-        // TODO this stage needs a proper resume
-        RetryRestartHelper::clear_restart_progress(state, self)
+    fn clear_progress(&mut self, state: &mut Self::State) -> Result<(), Error> {
+        StdRestartHelper::clear_progress(state, &self.name)
     }
 }
 
@@ -309,9 +314,10 @@ where
     #[must_use]
     /// Creates a new [`ColorizationStage`]
     pub fn new(map_observer: &C) -> Self {
+        let obs_name = map_observer.name().clone().into_owned();
         Self {
             map_observer_handle: map_observer.handle(),
-            name: Cow::Borrowed(COLORIZATION_STAGE_NAME),
+            name: Cow::Owned(COLORIZATION_STAGE_NAME.to_owned() + ":" + obs_name.as_str()),
             phantom: PhantomData,
         }
     }