Reporting & replaying failing random seeds (#161)

src/lib.rs

@@ -322,13 +322,45 @@ where
 
 /// Run the given function under a randomized concurrency scheduler for some number of iterations.
 /// Each iteration will run a (potentially) different randomized schedule.
+/// When the environment variable SHUTTLE_RANDOM_SEED is set to a u64, this number will be used
+/// as the seed to initialize the random scheduler.
 pub fn check_random<F>(f: F, iterations: usize)
+where
+    F: Fn() + Send + Sync + 'static,
+{
+    use crate::scheduler::RandomScheduler;
+    use tracing::info;
+
+    let seed_env = std::env::var("SHUTTLE_RANDOM_SEED");
+    let runner = match seed_env {
+        Ok(s) => match s.as_str().parse::<u64>() {
+            Ok(seed) => {
+                info!(
+                    "Initializing RandomScheduler with the seed provided by SHUTTLE_RANDOM_SEED: {}",
+                    seed
+                );
+                Runner::new(RandomScheduler::new_from_seed(seed, iterations), Default::default())
+            }
+            Err(err) => panic!("The seed provided by SHUTTLE_RANDOM_SEED is not a valid u64: {}", err),
+        },
+        Err(_) => Runner::new(RandomScheduler::new(iterations), Default::default()),
+    };
+
+    runner.run(f);
+}
+
+/// Run function `f` using `RandomScheduler` initialized with the provided `seed` for the given
+/// `iterations`.
+/// This makes generating the random seed for each execution independent from `RandomScheduler`.
+/// Therefore, this can be used with a library (like proptest) that takes care of generating the
+/// random seeds.
+pub fn check_random_with_seed<F>(f: F, seed: u64, iterations: usize)
 where
     F: Fn() + Send + Sync + 'static,
 {
     use crate::scheduler::RandomScheduler;
 
-    let scheduler = RandomScheduler::new(iterations);
+    let scheduler = RandomScheduler::new_from_seed(seed, iterations);
     let runner = Runner::new(scheduler, Default::default());
     runner.run(f);
 }

src/scheduler/random.rs

@@ -18,6 +18,33 @@ pub struct RandomScheduler {
     rng: Pcg64Mcg,
     iterations: usize,
     data_source: RandomDataSource,
+    current_seed: CurrentSeedDropGuard,
+}
+
+#[derive(Debug, Default)]
+struct CurrentSeedDropGuard {
+    inner: Option<u64>,
+}
+
+impl CurrentSeedDropGuard {
+    fn clear(&mut self) {
+        self.inner = None
+    }
+
+    fn update(&mut self, seed: u64) {
+        self.inner = Some(seed)
+    }
+}
+
+impl Drop for CurrentSeedDropGuard {
+    fn drop(&mut self) {
+        if let Some(s) = self.inner {
+            eprintln!(
+                "failing seed:\n\"\n{}\n\"\nTo replay the failure, either:\n    1) pass the seed to `shuttle::check_random_with_seed, or\n    2) set the environment variable SHUTTLE_RANDOM_SEED to the seed and run `shuttle::check_random`.",
+                s
+            )
+        }
+    }
 }
 
 impl RandomScheduler {
@@ -37,17 +64,22 @@ impl RandomScheduler {
             rng,
             iterations: 0,
             data_source: RandomDataSource::initialize(seed),
+            current_seed: CurrentSeedDropGuard::default(),
         }
     }
 }
 
 impl Scheduler for RandomScheduler {
     fn new_execution(&mut self) -> Option<Schedule> {
         if self.iterations >= self.max_iterations {
+            self.current_seed.clear();
             None
         } else {
             self.iterations += 1;
-            Some(Schedule::new(self.data_source.reinitialize()))
+            let seed = self.data_source.reinitialize();
+            self.rng = Pcg64Mcg::seed_from_u64(seed);
+            self.current_seed.update(seed);
+            Some(Schedule::new(seed))
         }
     }
 

tests/data/random.rs

@@ -1,4 +1,4 @@
-use crate::check_replay_roundtrip;
+use crate::{check_replay_from_seed_match_schedule, check_replay_roundtrip};
 use shuttle::rand::{thread_rng, Rng};
 use shuttle::scheduler::RandomScheduler;
 use shuttle::sync::Mutex;
@@ -172,6 +172,42 @@ fn dfs_threads_decorrelated_enabled() {
     runner.run(thread_rng_decorrelated);
 }
 
+#[test]
+fn replay_from_seed_match_schedule0() {
+    check_replay_from_seed_match_schedule(
+        broken_atomic_counter_stress,
+        15603830570056246250,
+        "91022ceac7d5bcb1a7fcc5d801a8050ea528954032492693491200000000",
+    );
+}
+
+#[test]
+fn replay_from_seed_match_schedule1() {
+    check_replay_from_seed_match_schedule(
+        broken_atomic_counter_stress,
+        2185777353610950419,
+        "91023c93eecb80c29ddcaa1ef81a1c5251494a2c92928a2a954a25a904000000000000",
+    );
+}
+
+#[test]
+fn replay_from_seed_match_schedule2() {
+    check_replay_from_seed_match_schedule(
+        broken_atomic_counter_stress,
+        14231716651102207764,
+        "91024b94fed5e7c2dccdc0c501185a9c0a889e169b64ca455b2d954a52492a49a59204000000\n00000000",
+    );
+}
+
+#[test]
+fn replay_from_seed_match_schedule3() {
+    check_replay_from_seed_match_schedule(
+        broken_atomic_counter_stress,
+        14271799263003420363,
+        "910278cbcd808888bae787c601081eda4f904cb34937e96cb72db9da965c65d2969b29956dab\ne81625a54432c83469d24c020000000000000000",
+    );
+}
+
 // The DFS scheduler uses the same stream of randomness on each execution to ensure determinism
 #[test]
 fn dfs_does_not_reseed_across_executions() {

tests/mod.rs

@@ -6,7 +6,7 @@ mod demo;
 mod future;
 
 use shuttle::scheduler::{ReplayScheduler, Scheduler};
-use shuttle::{replay_from_file, Config, FailurePersistence, Runner};
+use shuttle::{check_random_with_seed, replay_from_file, Config, FailurePersistence, Runner};
 use std::panic::{self, RefUnwindSafe, UnwindSafe};
 use std::sync::Arc;
 
@@ -94,6 +94,25 @@ where
     // `FailurePersistence`s for each test
 }
 
+/// Validates that the replay from seed functionality works by running a failing seed found by a random
+/// scheduler for one iteration, expecting it to fail, comparing the new failing schedule against the
+/// previously collected one, and checking the two schedules being identical.
+fn check_replay_from_seed_match_schedule<F>(test_func: F, seed: u64, expected_schedule: &str)
+where
+    F: Fn() + Send + Sync + UnwindSafe + 'static,
+{
+    let result = {
+        panic::catch_unwind(move || {
+            check_random_with_seed(test_func, seed, 1);
+        })
+        .expect_err("replay should panic")
+    };
+    let output = result.downcast::<String>().unwrap();
+    let schedule_from_replay = parse_schedule::from_stdout(&output).expect("output should contain a schedule");
+
+    assert_eq!(schedule_from_replay, expected_schedule);
+}
+
 /// Helpers to parse schedules from different types of output (as determined by [`FailurePersistence`])
 mod parse_schedule {
     use regex::Regex;