[proptest] Minimal implementation of interceptor for excessive panics outputs

proptest/CHANGELOG.md

@@ -3,6 +3,9 @@
 ### New Features
 
 - When running persisted regressions, the most recently added regression is now run first.
+- Added `handle-panics` feature which enables catching panics raised in tests and turning them into failures
+- Added `backtrace` feature which enables capturing backtraces for both test failures and panics,
+  if `handle-panics` feature is enabled
 
 ## 1.5.0
 

proptest/Cargo.toml

@@ -56,6 +56,15 @@ atomic64bit = []
 
 bit-set = [ "dep:bit-set", "dep:bit-vec" ]
 
+# Enables proper handling of panics
+# In particular, hides all intermediate panics flowing into stderr during shrink phase
+handle-panics = ["std"]
+
+# Enables gathering of failure backtraces
+# * when test failure is reported via `prop_assert_*` macro
+# * when normal assertion fails or panic fires, if `handle-panics` feature is enabled too
+backtrace = ["std"]
+
 [dependencies]
 bitflags = "2"
 unarray = "0.1.4"

proptest/src/collection.rs

@@ -705,7 +705,7 @@ mod test {
 
             match result {
                 Ok(true) => num_successes += 1,
-                Err(TestError::Fail(_, value)) => {
+                Err(TestError::Fail(_, _, value)) => {
                     // The minimal case always has between 5 (due to min
                     // length) and 9 (min element value = 1) elements, and
                     // always sums to exactly 9.

proptest/src/strategy/flatten.rs

@@ -293,7 +293,7 @@ mod test {
 
             match result {
                 Ok(_) => {}
-                Err(TestError::Fail(_, v)) => {
+                Err(TestError::Fail(_, _, v)) => {
                     failures += 1;
                     assert_eq!((10001, 10002), v);
                 }

proptest/src/strategy/unions.rs

@@ -495,8 +495,8 @@ mod test {
 
             match result {
                 Ok(true) => passed += 1,
-                Err(TestError::Fail(_, 15)) => converged_low += 1,
-                Err(TestError::Fail(_, 30)) => converged_high += 1,
+                Err(TestError::Fail(_, _, 15)) => converged_low += 1,
+                Err(TestError::Fail(_, _, 30)) => converged_high += 1,
                 e => panic!("Unexpected result: {:?}", e),
             }
         }
@@ -572,8 +572,8 @@ mod test {
 
             match result {
                 Ok(true) => passed += 1,
-                Err(TestError::Fail(_, 15)) => converged_low += 1,
-                Err(TestError::Fail(_, 30)) => converged_high += 1,
+                Err(TestError::Fail(_, _, 15)) => converged_low += 1,
+                Err(TestError::Fail(_, _, 30)) => converged_high += 1,
                 e => panic!("Unexpected result: {:?}", e),
             }
         }

proptest/src/string.rs

@@ -171,10 +171,7 @@ pub fn string_regex_parsed(expr: &Hir) -> ParseResult<String> {
 /// [`regex` crate's documentation](https://docs.rs/regex/*/regex/#opt-out-of-unicode-support)
 /// for more information.
 pub fn bytes_regex(regex: &str) -> ParseResult<Vec<u8>> {
-    let hir = ParserBuilder::new()
-        .utf8(false)
-        .build()
-        .parse(regex)?;
+    let hir = ParserBuilder::new().utf8(false).build().parse(regex)?;
     bytes_regex_parsed(&hir)
 }
 
@@ -361,8 +358,8 @@ fn unsupported<T>(error: &'static str) -> Result<T, Error> {
 mod test {
     use std::collections::HashSet;
 
-    use regex::Regex;
     use regex::bytes::Regex as BytesRegex;
+    use regex::Regex;
 
     use super::*;
 
@@ -402,7 +399,8 @@ mod test {
         max_distinct: usize,
         iterations: usize,
     ) {
-        let generated = generate_byte_values_matching_regex(pattern, iterations);
+        let generated =
+            generate_byte_values_matching_regex(pattern, iterations);
         assert!(
             generated.len() >= min_distinct,
             "Expected to generate at least {} strings, but only \
@@ -477,7 +475,8 @@ mod test {
                 if !ok {
                     panic!(
                         "Generated string {:?} which does not match {:?}",
-                        printable_ascii(&s), pattern
+                        printable_ascii(&s),
+                        pattern
                     );
                 }
 
@@ -584,10 +583,15 @@ mod test {
     fn test_non_utf8_byte_strings() {
         do_test_bytes(r"(?-u)[\xC0-\xFF]\x20", 64, 64, 512);
         do_test_bytes(r"(?-u)\x20[\x80-\xBF]", 64, 64, 512);
-        do_test_bytes(r#"(?x-u)
+        do_test_bytes(
+            r#"(?x-u)
   \xed (( ( \xa0\x80 | \xad\xbf | \xae\x80 | \xaf\xbf )
           ( \xed ( \xb0\x80 | \xbf\xbf ) )? )
-        | \xb0\x80 | \xbe\x80 | \xbf\xbf )"#, 15, 15, 120);
+        | \xb0\x80 | \xbe\x80 | \xbf\xbf )"#,
+            15,
+            15,
+            120,
+        );
     }
 
     fn assert_send_and_sync<T: Send + Sync>(_: T) {}

proptest/src/sugar.rs

@@ -754,7 +754,10 @@ macro_rules! prop_assert {
             let message = format!($($fmt)*);
             let message = format!("{} at {}:{}", message, file!(), line!());
             return ::core::result::Result::Err(
-                $crate::test_runner::TestCaseError::fail(message));
+                $crate::test_runner::TestCaseError::Fail(
+                    message.into(),
+                    $crate::test_runner::Backtrace::capture(),
+                ));
         }
     };
 }

proptest/src/test_runner/backtrace.rs

@@ -0,0 +1,135 @@
+//-
+// Copyright 2024
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+use core::fmt;
+/// Holds test failure backtrace, if captured
+///
+/// If feature `backtrace` is disabled, it's a zero-sized struct with no logic
+///
+/// If `backtrace` is enabled, attempts to capture backtrace using `std::backtrace::Backtrace` -
+/// if requested
+#[derive(Clone, Default)]
+pub struct Backtrace(internal::Backtrace);
+
+impl Backtrace {
+    /// Creates empty backtrace object
+    ///
+    /// Used when client code doesn't care
+    pub fn empty() -> Self {
+        Self(internal::Backtrace::empty())
+    }
+    /// Tells whether there's backtrace captured
+    pub fn is_empty(&self) -> bool {
+        self.0.is_empty()
+    }
+    /// Attempts to capture backtrace - but only if `backtrace` feature is enabled
+    #[inline(always)]
+    pub fn capture() -> Self {
+        Self(internal::Backtrace::capture())
+    }
+}
+
+impl fmt::Debug for Backtrace {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        fmt::Debug::fmt(&self.0, f)
+    }
+}
+
+impl fmt::Display for Backtrace {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        fmt::Display::fmt(&self.0, f)
+    }
+}
+
+#[cfg(feature = "backtrace")]
+mod internal {
+    use core::fmt;
+    use std::backtrace as bt;
+    use std::sync::Arc;
+
+    // `std::backtrace::Backtrace` isn't `Clone`, so we have
+    // to use `Arc` to also maintain `Send + Sync`
+    #[derive(Clone, Default)]
+    pub struct Backtrace(Option<Arc<bt::Backtrace>>);
+
+    impl Backtrace {
+        pub fn empty() -> Self {
+            Self(None)
+        }
+
+        pub fn is_empty(&self) -> bool {
+            self.0.is_none()
+        }
+
+        #[inline(always)]
+        pub fn capture() -> Self {
+            let bt = bt::Backtrace::capture();
+            // Store only if we have backtrace
+            if bt.status() == bt::BacktraceStatus::Captured {
+                Self(Some(Arc::new(bt)))
+            } else {
+                Self(None)
+            }
+        }
+    }
+
+    impl fmt::Debug for Backtrace {
+        fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+            if let Some(ref arc) = self.0 {
+                fmt::Debug::fmt(arc.as_ref(), f)
+            } else {
+                Ok(())
+            }
+        }
+    }
+
+    impl fmt::Display for Backtrace {
+        fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+            if let Some(ref arc) = self.0 {
+                fmt::Display::fmt(arc.as_ref(), f)
+            } else {
+                Ok(())
+            }
+        }
+    }
+}
+
+#[cfg(not(feature = "backtrace"))]
+mod internal {
+    use core::fmt;
+
+    #[derive(Clone, Default)]
+    pub struct Backtrace;
+
+    impl Backtrace {
+        pub fn empty() -> Self {
+            Self
+        }
+
+        pub fn is_empty(&self) -> bool {
+            true
+        }
+
+        pub fn capture() -> Self {
+            Self
+        }
+    }
+
+    impl fmt::Debug for Backtrace {
+        fn fmt(&self, _: &mut fmt::Formatter<'_>) -> fmt::Result {
+            Ok(())
+        }
+    }
+
+    impl fmt::Display for Backtrace {
+        fn fmt(&self, _: &mut fmt::Formatter<'_>) -> fmt::Result {
+            Ok(())
+        }
+    }
+}

proptest/src/test_runner/errors.rs

@@ -1,5 +1,5 @@
 //-
-// Copyright 2017, 2018 The proptest developers
+// Copyright 2017, 2018, 2024 The proptest developers
 //
 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
@@ -14,6 +14,8 @@ use std::string::ToString;
 
 use crate::test_runner::Reason;
 
+use super::Backtrace;
+
 /// Errors which can be returned from test cases to indicate non-successful
 /// completion.
 ///
@@ -30,7 +32,7 @@ pub enum TestCaseError {
     /// a new input and try again.
     Reject(Reason),
     /// The code under test failed the test.
-    Fail(Reason),
+    Fail(Reason, Backtrace),
 }
 
 /// Indicates the type of test that ran successfully.
@@ -76,7 +78,7 @@ impl TestCaseError {
     /// The string should indicate the location of the failure, but may
     /// generally be any string.
     pub fn fail(reason: impl Into<Reason>) -> Self {
-        TestCaseError::Fail(reason.into())
+        TestCaseError::Fail(reason.into(), Backtrace::empty())
     }
 }
 
@@ -86,7 +88,13 @@ impl fmt::Display for TestCaseError {
             TestCaseError::Reject(ref whence) => {
                 write!(f, "Input rejected at {}", whence)
             }
-            TestCaseError::Fail(ref why) => write!(f, "Case failed: {}", why),
+            TestCaseError::Fail(ref why, ref bt) => {
+                if bt.is_empty() {
+                    write!(f, "Case failed: {why}")
+                } else {
+                    write!(f, "Case failed: {why}\n{bt}")
+                }
+            }
         }
     }
 }
@@ -99,24 +107,49 @@ impl<E: ::std::error::Error> From<E> for TestCaseError {
 }
 
 /// A failure state from running test cases for a single test.
-#[derive(Debug, Clone, PartialEq, Eq)]
+#[derive(Debug, Clone)]
 pub enum TestError<T> {
     /// The test was aborted for the given reason, for example, due to too many
     /// inputs having been rejected.
     Abort(Reason),
     /// A failing test case was found. The string indicates where and/or why
     /// the test failed. The `T` is the minimal input found to reproduce the
     /// failure.
-    Fail(Reason, T),
+    Fail(Reason, Backtrace, T),
 }
 
+impl<T: PartialEq> PartialEq for TestError<T> {
+    fn eq(&self, other: &Self) -> bool {
+        match (self, other) {
+            (Self::Abort(l0), Self::Abort(r0)) => l0 == r0,
+            (Self::Fail(l0, _, l2), Self::Fail(r0, _, r2)) => {
+                l0 == r0 && l2 == r2
+            }
+            (TestError::Abort(_), TestError::Fail(_, _, _))
+            | (TestError::Fail(_, _, _), TestError::Abort(_)) => false,
+        }
+    }
+}
+
+impl<T: Eq> Eq for TestError<T> {}
+
 impl<T: fmt::Debug> fmt::Display for TestError<T> {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         match *self {
             TestError::Abort(ref why) => write!(f, "Test aborted: {}", why),
-            TestError::Fail(ref why, ref what) => {
+            TestError::Fail(ref why, ref bt, ref what) => {
                 writeln!(f, "Test failed: {}.", why)?;
-                write!(f, "minimal failing input: {:#?}", what)
+
+                if !bt.is_empty() {
+                    writeln!(f, "\nstack backtrace:\n{bt}")?;
+                    // No need for extra newline, backtrace seems to print it anyway
+                } else {
+                    // Extra empty line between failure description and minimal failing input
+                    writeln!(f)?;
+                }
+
+                writeln!(f, "minimal failing input: {:#?}", what)?;
+                Ok(())
             }
         }
     }

proptest/src/test_runner/mod.rs

@@ -1,5 +1,5 @@
 //-
-// Copyright 2017, 2018 The proptest developers
+// Copyright 2017, 2018, 2024 The proptest developers
 //
 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
@@ -12,6 +12,7 @@
 //! You do not normally need to access things in this module directly except
 //! when implementing new low-level strategies.
 
+mod backtrace;
 mod config;
 mod errors;
 mod failure_persistence;
@@ -22,6 +23,7 @@ mod result_cache;
 mod rng;
 mod runner;
 
+pub use self::backtrace::*;
 pub use self::config::*;
 pub use self::errors::*;
 pub use self::failure_persistence::*;