Provide a mechanism to skip tests at runtime

src/lib.rs

@@ -122,6 +122,31 @@ impl Trial {
         }
     }
 
+    /// Creates a (non-benchmark) test with the given name and runner.
+    ///
+    /// The runner returning `Ok(None)` is interpreted as the test passing. The runner returning
+    /// `Ok(Some(<String>))` is interpreted as the test being skipped for reasons determined at
+    /// runtime. The reason may be passed as a [`String`] if desired. If the runner returns
+    /// `Err(_)`, the test is considered failed.
+    pub fn skippable_test<R>(name: impl Into<String>, runner: R) -> Self
+    where
+        R: FnOnce() -> Result<Option<String>, Failed> + Send + 'static,
+    {
+        Self {
+            runner: Box::new(|_test_mode| match runner() {
+                Ok(None) => Outcome::Passed,
+                Ok(Some(_reason)) => Outcome::RuntimeIgnored,
+                Err(failed) => Outcome::Failed(failed),
+            }),
+            info: TestInfo {
+                name: name.into(),
+                kind: String::new(),
+                is_ignored: false,
+                is_bench: false,
+            },
+        }
+    }
+
     /// Creates a benchmark with the given name and runner.
     ///
     /// If the runner's parameter `test_mode` is `true`, the runner function
@@ -303,6 +328,9 @@ enum Outcome {
     /// The test or benchmark was ignored.
     Ignored,
 
+    /// The test or benchmark was ignored.
+    RuntimeIgnored,
+
     /// The benchmark was successfully run.
     Measured(Measurement),
 }
@@ -476,6 +504,7 @@ pub fn run(args: &Arguments, mut tests: Vec<Trial>) -> Conclusion {
             },
             Outcome::Ignored => conclusion.num_ignored += 1,
             Outcome::Measured(_) => conclusion.num_measured += 1,
+            Outcome::RuntimeIgnored => conclusion.num_ignored += 1,
         }
     };
 

src/printer.rs

@@ -158,6 +158,7 @@ impl Printer {
                         writeln!(self.out).unwrap();
                         return;
                     }
+                    Outcome::RuntimeIgnored => 'S',
                 };
 
                 self.out.set_color(&color_of_outcome(outcome)).unwrap();
@@ -184,6 +185,7 @@ impl Printer {
                             Outcome::Failed(_) => "failed",
                             Outcome::Ignored => "ignored",
                             Outcome::Measured(_) => unreachable!(),
+                            Outcome::RuntimeIgnored => "skipped",
                         },
                         match outcome {
                             Outcome::Failed(Failed { msg: Some(msg) }) => {
@@ -317,6 +319,7 @@ impl Printer {
             Outcome::Failed { .. } => "FAILED",
             Outcome::Ignored => "ignored",
             Outcome::Measured { .. } => "bench",
+            Outcome::RuntimeIgnored => "skipped",
         };
 
         self.out.set_color(&color_of_outcome(outcome)).unwrap();
@@ -354,6 +357,7 @@ fn color_of_outcome(outcome: &Outcome) -> ColorSpec {
         Outcome::Failed { .. } => Color::Red,
         Outcome::Ignored => Color::Yellow,
         Outcome::Measured { .. } => Color::Cyan,
+        Outcome::RuntimeIgnored => Color::Blue,
     };
     out.set_fg(Some(color));
     out

tests/all_passing.rs

@@ -2,7 +2,7 @@ use common::{args, check};
 use libtest_mimic::{Trial, Conclusion};
 use pretty_assertions::assert_eq;
 
-use crate::common::do_run;
+use crate::common::{assert_reordered_log, conclusion_to_output, do_run};
 
 #[macro_use]
 mod common;
@@ -13,23 +13,25 @@ fn tests() -> Vec<Trial> {
         Trial::test("foo", || Ok(())),
         Trial::test("bar", || Ok(())),
         Trial::test("barro", || Ok(())),
+        Trial::skippable_test("baz", || Ok(Some("Can't find a quux".into()))),
     ]
 }
 
 #[test]
 fn normal() {
-    check(args([]), tests, 3,
+    check(args([]), tests, 4,
         Conclusion {
             num_filtered_out: 0,
             num_passed: 3,
             num_failed: 0,
-            num_ignored: 0,
+            num_ignored: 1,
             num_measured: 0,
         },
         "
             test foo   ... ok
             test bar   ... ok
             test barro ... ok
+            test baz   ... skipped
         "
     );
 }
@@ -38,7 +40,7 @@ fn normal() {
 fn filter_one() {
     check(args(["foo"]), tests, 1,
         Conclusion {
-            num_filtered_out: 2,
+            num_filtered_out: 3,
             num_passed: 1,
             num_failed: 0,
             num_ignored: 0,
@@ -52,7 +54,7 @@ fn filter_one() {
 fn filter_two() {
     check(args(["bar"]), tests, 2,
         Conclusion {
-            num_filtered_out: 1,
+            num_filtered_out: 2,
             num_passed: 2,
             num_failed: 0,
             num_ignored: 0,
@@ -70,7 +72,7 @@ fn filter_two() {
 fn filter_exact() {
     check(args(["bar", "--exact"]), tests, 1,
         Conclusion {
-            num_filtered_out: 2,
+            num_filtered_out: 3,
             num_passed: 1,
             num_failed: 0,
             num_ignored: 0,
@@ -84,7 +86,7 @@ fn filter_exact() {
 fn filter_two_and_skip() {
     check(args(["--skip", "barro", "bar"]), tests, 1,
         Conclusion {
-            num_filtered_out: 2,
+            num_filtered_out: 3,
             num_passed: 1,
             num_failed: 0,
             num_ignored: 0,
@@ -94,51 +96,70 @@ fn filter_two_and_skip() {
     );
 }
 
+#[test]
+fn filter_runtime_ignored() {
+    check(args(["baz", "--exact"]), tests, 1,
+        Conclusion {
+            num_filtered_out: 3,
+            num_passed: 0,
+            num_failed: 0,
+            num_ignored: 1,
+            num_measured: 0,
+        },
+        "test baz ... skipped",
+    );
+}
+
 #[test]
 fn skip_nothing() {
-    check(args(["--skip", "peter"]), tests, 3,
+    check(args(["--skip", "peter"]), tests, 4,
         Conclusion {
             num_filtered_out: 0,
             num_passed: 3,
             num_failed: 0,
-            num_ignored: 0,
+            num_ignored: 1,
             num_measured: 0,
         },
         "
             test foo   ... ok
             test bar   ... ok
             test barro ... ok
+            test baz   ... skipped
         "
     );
 }
 
 #[test]
 fn skip_two() {
-    check(args(["--skip", "bar"]), tests, 1,
+    check(args(["--skip", "bar"]), tests, 2,
         Conclusion {
             num_filtered_out: 2,
             num_passed: 1,
             num_failed: 0,
-            num_ignored: 0,
+            num_ignored: 1,
             num_measured: 0,
         },
-        "test foo ... ok"
+        "
+            test foo ... ok
+            test baz ... skipped
+        "
     );
 }
 
 #[test]
 fn skip_exact() {
-    check(args(["--exact", "--skip", "bar"]), tests, 2,
+    check(args(["--exact", "--skip", "bar"]), tests, 3,
         Conclusion {
             num_filtered_out: 1,
             num_passed: 2,
             num_failed: 0,
-            num_ignored: 0,
+            num_ignored: 1,
             num_measured: 0,
         },
         "
             test foo   ... ok
             test barro ... ok
+            test baz   ... skipped
         "
     );
 }
@@ -150,13 +171,8 @@ fn terse_output() {
         num_filtered_out: 0,
         num_passed: 3,
         num_failed: 0,
-        num_ignored: 0,
+        num_ignored: 1,
         num_measured: 0,
     });
-    assert_log!(out, "
-        running 3 tests
-        ...
-        test result: ok. 3 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; \
-            finished in 0.00s
-    ");
+    assert_reordered_log(out.as_str(), 4, &["...S"], &conclusion_to_output(&c), true);
 }

tests/common/mod.rs

@@ -52,7 +52,7 @@
 
 /// Best effort tool to check certain things about a log that might have all
 /// tests randomly ordered.
-pub fn assert_reordered_log(actual: &str, num: u64, expected_lines: &[&str], tail: &str) {
+pub fn assert_reordered_log(actual: &str, num: u64, expected_lines: &[&str], tail: &str, sloppy: bool) {
     let actual = actual.trim();
     let (first_line, rest) = actual.split_once('\n').expect("log has too few lines");
     let (middle, last_line) = rest.rsplit_once('\n').expect("log has too few lines");
@@ -63,6 +63,15 @@
 
     let mut actual_lines = HashMap::new();
     for line in middle.lines().map(|l| l.trim()).filter(|l| !l.is_empty()) {
+        // If we're testing sloppily, sort before adding the line so we can compare sorted strings.
+        // This is effectively testing if the strings are anagrams/permutations.
+        let line = if sloppy {
+            let mut line_str = line.chars().collect::<Vec<_>>();
+            line_str.sort();
+            line_str.iter().collect::<String>()
+        } else {
+            line.to_string()
+        };
         *actual_lines.entry(line).or_insert(0) += 1;
     }
 
@@ -131,12 +140,19 @@
         num_running_tests,
         &expected_output.lines().collect::<Vec<_>>(),
         &conclusion_to_output(&c),
+        false,
     );
     assert_eq!(c, expected_conclusion);
 }
 
-fn conclusion_to_output(c: &Conclusion) -> String {
-    let Conclusion { num_filtered_out, num_passed, num_failed, num_ignored, num_measured } = *c;
+pub fn conclusion_to_output(c: &Conclusion) -> String {
+    let Conclusion {
+        num_filtered_out,
+        num_passed,
+        num_failed,
+        num_ignored,
+        num_measured,
+    } = *c;
     format!(
         "test result: {}. {} passed; {} failed; {} ignored; {} measured; {} filtered out;",
         if num_failed > 0 { "FAILED" } else { "ok" },