From fc27d03a144f55d0ac7f7fff00b451cdea84d128 Mon Sep 17 00:00:00 2001 From: Yanze Li Date: Mon, 16 Sep 2024 04:39:59 -0700 Subject: [PATCH] Reporting & replaying failing random seeds (#161) --- src/lib.rs | 34 +++++++++++++++++++++++++++++++++- src/scheduler/random.rs | 34 +++++++++++++++++++++++++++++++++- tests/data/random.rs | 38 +++++++++++++++++++++++++++++++++++++- tests/mod.rs | 21 ++++++++++++++++++++- 4 files changed, 123 insertions(+), 4 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index c7c5ea85..fbb121e9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -322,13 +322,45 @@ where /// Run the given function under a randomized concurrency scheduler for some number of iterations. /// Each iteration will run a (potentially) different randomized schedule. +/// When the environment variable SHUTTLE_RANDOM_SEED is set to a u64, this number will be used +/// as the seed to initialize the random scheduler. pub fn check_random(f: F, iterations: usize) +where + F: Fn() + Send + Sync + 'static, +{ + use crate::scheduler::RandomScheduler; + use tracing::info; + + let seed_env = std::env::var("SHUTTLE_RANDOM_SEED"); + let runner = match seed_env { + Ok(s) => match s.as_str().parse::() { + Ok(seed) => { + info!( + "Initializing RandomScheduler with the seed provided by SHUTTLE_RANDOM_SEED: {}", + seed + ); + Runner::new(RandomScheduler::new_from_seed(seed, iterations), Default::default()) + } + Err(err) => panic!("The seed provided by SHUTTLE_RANDOM_SEED is not a valid u64: {}", err), + }, + Err(_) => Runner::new(RandomScheduler::new(iterations), Default::default()), + }; + + runner.run(f); +} + +/// Run function `f` using `RandomScheduler` initialized with the provided `seed` for the given +/// `iterations`. +/// This makes generating the random seed for each execution independent from `RandomScheduler`. +/// Therefore, this can be used with a library (like proptest) that takes care of generating the +/// random seeds. +pub fn check_random_with_seed(f: F, seed: u64, iterations: usize) where F: Fn() + Send + Sync + 'static, { use crate::scheduler::RandomScheduler; - let scheduler = RandomScheduler::new(iterations); + let scheduler = RandomScheduler::new_from_seed(seed, iterations); let runner = Runner::new(scheduler, Default::default()); runner.run(f); } diff --git a/src/scheduler/random.rs b/src/scheduler/random.rs index bcf582dc..e4c0d006 100644 --- a/src/scheduler/random.rs +++ b/src/scheduler/random.rs @@ -18,6 +18,33 @@ pub struct RandomScheduler { rng: Pcg64Mcg, iterations: usize, data_source: RandomDataSource, + current_seed: CurrentSeedDropGuard, +} + +#[derive(Debug, Default)] +struct CurrentSeedDropGuard { + inner: Option, +} + +impl CurrentSeedDropGuard { + fn clear(&mut self) { + self.inner = None + } + + fn update(&mut self, seed: u64) { + self.inner = Some(seed) + } +} + +impl Drop for CurrentSeedDropGuard { + fn drop(&mut self) { + if let Some(s) = self.inner { + eprintln!( + "failing seed:\n\"\n{}\n\"\nTo replay the failure, either:\n 1) pass the seed to `shuttle::check_random_with_seed, or\n 2) set the environment variable SHUTTLE_RANDOM_SEED to the seed and run `shuttle::check_random`.", + s + ) + } + } } impl RandomScheduler { @@ -37,6 +64,7 @@ impl RandomScheduler { rng, iterations: 0, data_source: RandomDataSource::initialize(seed), + current_seed: CurrentSeedDropGuard::default(), } } } @@ -44,10 +72,14 @@ impl RandomScheduler { impl Scheduler for RandomScheduler { fn new_execution(&mut self) -> Option { if self.iterations >= self.max_iterations { + self.current_seed.clear(); None } else { self.iterations += 1; - Some(Schedule::new(self.data_source.reinitialize())) + let seed = self.data_source.reinitialize(); + self.rng = Pcg64Mcg::seed_from_u64(seed); + self.current_seed.update(seed); + Some(Schedule::new(seed)) } } diff --git a/tests/data/random.rs b/tests/data/random.rs index 2d50a5c2..2daf9364 100644 --- a/tests/data/random.rs +++ b/tests/data/random.rs @@ -1,4 +1,4 @@ -use crate::check_replay_roundtrip; +use crate::{check_replay_from_seed_match_schedule, check_replay_roundtrip}; use shuttle::rand::{thread_rng, Rng}; use shuttle::scheduler::RandomScheduler; use shuttle::sync::Mutex; @@ -172,6 +172,42 @@ fn dfs_threads_decorrelated_enabled() { runner.run(thread_rng_decorrelated); } +#[test] +fn replay_from_seed_match_schedule0() { + check_replay_from_seed_match_schedule( + broken_atomic_counter_stress, + 15603830570056246250, + "91022ceac7d5bcb1a7fcc5d801a8050ea528954032492693491200000000", + ); +} + +#[test] +fn replay_from_seed_match_schedule1() { + check_replay_from_seed_match_schedule( + broken_atomic_counter_stress, + 2185777353610950419, + "91023c93eecb80c29ddcaa1ef81a1c5251494a2c92928a2a954a25a904000000000000", + ); +} + +#[test] +fn replay_from_seed_match_schedule2() { + check_replay_from_seed_match_schedule( + broken_atomic_counter_stress, + 14231716651102207764, + "91024b94fed5e7c2dccdc0c501185a9c0a889e169b64ca455b2d954a52492a49a59204000000\n00000000", + ); +} + +#[test] +fn replay_from_seed_match_schedule3() { + check_replay_from_seed_match_schedule( + broken_atomic_counter_stress, + 14271799263003420363, + "910278cbcd808888bae787c601081eda4f904cb34937e96cb72db9da965c65d2969b29956dab\ne81625a54432c83469d24c020000000000000000", + ); +} + // The DFS scheduler uses the same stream of randomness on each execution to ensure determinism #[test] fn dfs_does_not_reseed_across_executions() { diff --git a/tests/mod.rs b/tests/mod.rs index bcfa26f4..94090816 100644 --- a/tests/mod.rs +++ b/tests/mod.rs @@ -6,7 +6,7 @@ mod demo; mod future; use shuttle::scheduler::{ReplayScheduler, Scheduler}; -use shuttle::{replay_from_file, Config, FailurePersistence, Runner}; +use shuttle::{check_random_with_seed, replay_from_file, Config, FailurePersistence, Runner}; use std::panic::{self, RefUnwindSafe, UnwindSafe}; use std::sync::Arc; @@ -94,6 +94,25 @@ where // `FailurePersistence`s for each test } +/// Validates that the replay from seed functionality works by running a failing seed found by a random +/// scheduler for one iteration, expecting it to fail, comparing the new failing schedule against the +/// previously collected one, and checking the two schedules being identical. +fn check_replay_from_seed_match_schedule(test_func: F, seed: u64, expected_schedule: &str) +where + F: Fn() + Send + Sync + UnwindSafe + 'static, +{ + let result = { + panic::catch_unwind(move || { + check_random_with_seed(test_func, seed, 1); + }) + .expect_err("replay should panic") + }; + let output = result.downcast::().unwrap(); + let schedule_from_replay = parse_schedule::from_stdout(&output).expect("output should contain a schedule"); + + assert_eq!(schedule_from_replay, expected_schedule); +} + /// Helpers to parse schedules from different types of output (as determined by [`FailurePersistence`]) mod parse_schedule { use regex::Regex;