From d0b7d3c6f879686ba4180653dae804def4cf8593 Mon Sep 17 00:00:00 2001
From: Ruihang Xia <waynestxia@gmail.com>
Date: Tue, 20 Dec 2022 15:20:37 +0800
Subject: [PATCH 1/2] feat: new option  to ignore cases' runtime error

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>
---
 src/config.rs |  3 +++
 src/runner.rs | 75 ++++++++++++++++++++++++++++++++++-----------------
 2 files changed, 53 insertions(+), 25 deletions(-)

diff --git a/src/config.rs b/src/config.rs
index e96a781..d610249 100644
--- a/src/config.rs
+++ b/src/config.rs
@@ -24,4 +24,7 @@ pub struct Config {
     /// Default value: `config.toml`
     #[builder(default = "String::from(\"config.toml\")")]
     pub env_config_file: String,
+    /// Run all tests regardless of failure if true
+    #[builder(default = "false")]
+    pub no_fail_fast: bool,
 }
diff --git a/src/runner.rs b/src/runner.rs
index e686bec..6e35c39 100644
--- a/src/runner.rs
+++ b/src/runner.rs
@@ -97,30 +97,24 @@ impl<E: Environment> Runner<E> {
     async fn run_env(&self, env: &str, db: &E::DB) -> Result<()> {
         let case_paths = self.collect_case_paths(env).await?;
         let mut diff_cases = vec![];
+        let mut errors = vec![];
         let start = Instant::now();
         for path in case_paths {
-            let case_path = path.with_extension(&self.config.test_case_extension);
-            let case = TestCase::from_file(case_path, &self.config).await?;
-            let output_path = path.with_extension(&self.config.output_result_extension);
-            let mut output_file = Self::open_output_file(&output_path).await?;
-
-            let timer = Instant::now();
-            case.execute(db, &mut output_file).await?;
-            let elapsed = timer.elapsed();
-
-            output_file.flush().await?;
-            let is_different = self.compare(&path).await?;
-            if !is_different {
-                remove_file(output_path).await?;
-            } else {
-                diff_cases.push(path.as_os_str().to_str().unwrap().to_owned());
+            let case_result = self.run_single_case(db, &path).await;
+            let case_name = path.as_os_str().to_str().unwrap().to_owned();
+            match case_result {
+                Ok(true) => diff_cases.push(case_name),
+                Ok(false) => {}
+                Err(e) => {
+                    if self.config.no_fail_fast {
+                        errors.push((case_name, e))
+                    } else {
+                        println!("Case {} failed with error {:?}", case_name, e);
+                        println!("Stopping environment {} due to previous error.", env);
+                        break;
+                    }
+                }
             }
-
-            println!(
-                "Test case {:?} finished, cost: {}ms",
-                path.as_os_str(),
-                elapsed.as_millis()
-            );
         }
 
         println!(
@@ -129,17 +123,48 @@ impl<E: Environment> Runner<E> {
             start.elapsed().as_millis()
         );
 
+        let mut error_count = 0;
         if !diff_cases.is_empty() {
             println!("Different cases:");
             println!("{:#?}", diff_cases);
-            Err(SqlnessError::RunFailed {
-                count: diff_cases.len(),
-            })
-        } else {
+            error_count += diff_cases.len();
+        }
+        if !errors.is_empty() {
+            println!("Error cases:");
+            println!("{:#?}", errors);
+            error_count += errors.len();
+        }
+        if error_count == 0 {
             Ok(())
+        } else {
+            Err(SqlnessError::RunFailed { count: error_count })
         }
     }
 
+    async fn run_single_case(&self, db: &E::DB, path: &PathBuf) -> Result<bool> {
+        let case_path = path.with_extension(&self.config.test_case_extension);
+        let case = TestCase::from_file(case_path, &self.config).await?;
+        let output_path = path.with_extension(&self.config.output_result_extension);
+        let mut output_file = Self::open_output_file(&output_path).await?;
+
+        let timer = Instant::now();
+        case.execute(db, &mut output_file).await?;
+        let elapsed = timer.elapsed();
+
+        output_file.flush().await?;
+        let is_different = self.compare(&path).await?;
+        if !is_different {
+            remove_file(output_path).await?;
+        }
+
+        println!(
+            "Test case {:?} finished, cost: {}ms",
+            path.as_os_str(),
+            elapsed.as_millis()
+        );
+        Ok(is_different)
+    }
+
     async fn collect_case_paths(&self, env: &str) -> Result<Vec<PathBuf>> {
         let mut root = PathBuf::from_str(&self.config.case_dir).unwrap();
         root.push(env);

From 3d57a2f268d257d25dced455714f81441edbfc25 Mon Sep 17 00:00:00 2001
From: Ruihang Xia <waynestxia@gmail.com>
Date: Wed, 21 Dec 2022 11:14:12 +0800
Subject: [PATCH 2/2] rename to fail_fast

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>
---
 src/config.rs | 6 +++---
 src/runner.rs | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/config.rs b/src/config.rs
index d610249..f41a4c0 100644
--- a/src/config.rs
+++ b/src/config.rs
@@ -24,7 +24,7 @@ pub struct Config {
     /// Default value: `config.toml`
     #[builder(default = "String::from(\"config.toml\")")]
     pub env_config_file: String,
-    /// Run all tests regardless of failure if true
-    #[builder(default = "false")]
-    pub no_fail_fast: bool,
+    /// Fail this run as soon as one case fails if true
+    #[builder(default = "true")]
+    pub fail_fast: bool,
 }
diff --git a/src/runner.rs b/src/runner.rs
index 6e35c39..4bd78b1 100644
--- a/src/runner.rs
+++ b/src/runner.rs
@@ -106,7 +106,7 @@ impl<E: Environment> Runner<E> {
                 Ok(true) => diff_cases.push(case_name),
                 Ok(false) => {}
                 Err(e) => {
-                    if self.config.no_fail_fast {
+                    if self.config.fail_fast {
                         errors.push((case_name, e))
                     } else {
                         println!("Case {} failed with error {:?}", case_name, e);