Add testing contracts

src/engine.rs

@@ -442,6 +442,11 @@ impl Engine {
         // Effectively a dry-run
         let checkpoint = session.sandbox().take_snapshot();
         let properties = self.properties.clone();
+
+        // For each property, we will only try to break it once. If we find an argument
+        // that makes it return false, we will move on to the next property
+        // without looking for more examples. We finish the search on the first example
+        // that breaks it.
         for property in properties.iter() {
             let arguments_length = self
                 .method_info
@@ -455,9 +460,12 @@ impl Engine {
                 // Multiple arguments execute the property multiple times
                 self.config.fuzz_property_max_rounds ^ arguments_length
             };
+
+            // If the property has arguments, fuzz them
             for _round in 0..max_rounds {
                 let property_message =
                     self.generate_message(fuzzer, property, &contract_address)?;
+                // TODO: Handle ContractTrapped instead of bubbling up the error
                 let result = self.execute_message(session, &property_message);
                 assert_eq!(
                     vec![0, 0],
@@ -473,11 +481,14 @@ impl Engine {
                     }
                 };
 
+                session.sandbox().restore_snapshot(checkpoint.clone());
+
                 if failed {
+                    // If we find an argument that makes this property fail, we store it
+                    // and do not check for more
                     failed_properties.push(property_message);
+                    break;
                 }
-
-                session.sandbox().restore_snapshot(checkpoint.clone());
             }
         }
 
@@ -565,6 +576,7 @@ impl Engine {
                 // load it from the `current_state`
                 current_state = None;
 
+                // TODO: Handle ContractTrapped instead of bubbling up the error
                 // Execute the action
                 let result = self.execute_deploy(&mut session, &trace.deploy)?;
 
@@ -619,6 +631,7 @@ impl Engine {
                     current_state = None;
 
                     // Execute the action
+                    // TODO: Handle ContractTrapped instead of bubbling up the error
                     let result =
                         self.execute_message(&mut session, trace.last_message()?)?;
 

src/tests/mod.rs

@@ -1,39 +1,104 @@
 #[cfg(test)]
-pub mod ityfuzz {
+pub mod testing {
 
     use crate::{
         config::Config,
-        constants::Constants,
         engine::Engine,
     };
-    use std::{
-        default,
-        path::PathBuf,
-    };
+    use std::path::PathBuf;
+
+    fn test_contract(
+        contract_path: PathBuf,
+        should_find_broken_properties: bool,
+        config: Config,
+    ) -> Result<(), Box<dyn std::error::Error>> {
+        let mut engine = Engine::new(contract_path, config)?;
+        let campaign_result = engine.run_campaign()?;
+        engine.print_campaign_result(&campaign_result);
+
+        // Check that the campaign result is as expected
+        if should_find_broken_properties {
+            assert!(!campaign_result.failed_traces.is_empty());
+        } else {
+            assert!(campaign_result.failed_traces.is_empty());
+        }
 
-    fn test_contract(contract_path: PathBuf) -> Result<(), Box<dyn std::error::Error>> {
+        Ok(())
+    }
+
+    #[test]
+    fn fuzz_ityfuzz() -> Result<(), Box<dyn std::error::Error>> {
         // Set up the fuzzer configuration
         let config = Config {
             fail_fast: true,
             max_rounds: 1000,
             max_number_of_transactions: 50,
             ..Default::default()
         };
+        test_contract(
+            PathBuf::from("./test-contracts/ityfuzz/target/ink/ityfuzz.contract"),
+            true,
+            config,
+        )
+    }
 
-        let mut engine = Engine::new(contract_path, config)?;
-        let campaign_result = engine.run_campaign()?;
-        engine.print_campaign_result(&campaign_result);
+    #[test]
+    fn fuzz_integer_overflow_or_underflow_1_vulnerable(
+    ) -> Result<(), Box<dyn std::error::Error>> {
+        // Set up the fuzzer configuration
+        let config = Config {
+            fail_fast: true,
+            max_rounds: 100,
+            max_number_of_transactions: 50,
+            ..Default::default()
+        };
+        test_contract(PathBuf::from(
+            "./test-contracts/coinfabrik-test-contracts/integer-overflow-or-underflow-1/vulnerable-example/target/ink/integer_overflow_or_underflow.contract", 
+        ),true,config)
+    }
 
-        // Check that the campaign result found at least one failing trace
-        assert!(!campaign_result.failed_traces.is_empty());
+    #[test]
+    fn fuzz_integer_overflow_or_underflow_1_remediated(
+    ) -> Result<(), Box<dyn std::error::Error>> {
+        // Set up the fuzzer configuration
+        let config = Config {
+            fail_fast: true,
+            max_rounds: 10,
+            max_number_of_transactions: 10,
+            ..Default::default()
+        };
+        test_contract(PathBuf::from(
+            "./test-contracts/coinfabrik-test-contracts/integer-overflow-or-underflow-1/remediated-example/target/ink/integer_overflow_or_underflow.contract", 
+        ),false,config)
+    }
 
-        Ok(())
+    #[test]
+    fn fuzz_integer_overflow_or_underflow_2_vulnerable(
+    ) -> Result<(), Box<dyn std::error::Error>> {
+        // Set up the fuzzer configuration
+        let config = Config {
+            fail_fast: true,
+            max_rounds: 100,
+            max_number_of_transactions: 50,
+            ..Default::default()
+        };
+        test_contract(PathBuf::from(
+            "./test-contracts/coinfabrik-test-contracts/integer-overflow-or-underflow-2/vulnerable-example/target/ink/integer_overflow_or_underflow.contract", 
+        ),true,config)
     }
 
     #[test]
-    fn fuzz_ityfuzz() -> Result<(), Box<dyn std::error::Error>> {
+    fn fuzz_integer_overflow_or_underflow_2_remediated(
+    ) -> Result<(), Box<dyn std::error::Error>> {
+        // Set up the fuzzer configuration
+        let config = Config {
+            fail_fast: true,
+            max_rounds: 10,
+            max_number_of_transactions: 10,
+            ..Default::default()
+        };
         test_contract(PathBuf::from(
-            "./test-contracts/ityfuzz/target/ink/ityfuzz.contract",
-        ))
+            "./test-contracts/coinfabrik-test-contracts/integer-overflow-or-underflow-2/remediated-example/target/ink/integer_overflow_or_underflow.contract", 
+        ),false,config)
     }
 }

test-contracts/build.py

@@ -0,0 +1,39 @@
+import os
+import subprocess
+import json
+
+def build_contracts(current_dir):
+    # Check if the current directory has a Cargo.toml file
+    if os.path.isfile(os.path.join(current_dir, "Cargo.toml")):
+        print(f"Building contract in directory: {current_dir}")
+
+        # Execute the build command
+        subprocess.run(["cargo", "contract", "build", "--features", "fuzz-testing"], cwd=current_dir)
+
+        # Find the .contract file and modify the version field
+        contract_file = next((os.path.join(current_dir, "target", "ink", f) for f in os.listdir(os.path.join(current_dir, "target", "ink")) if f.endswith(".contract")), None)
+        if contract_file:
+            print(f"Parsing and modifying .contract file: {contract_file}")
+            with open(contract_file, "r") as f:
+                contract_data = json.load(f)
+            contract_data["version"] = int(contract_data["version"])
+            with open(f"{contract_file}.tmp", "w") as f:
+                json.dump(contract_data, f)
+            os.rename(f"{contract_file}.tmp", contract_file)
+        else:
+            print(f".contract file not found in directory: {current_dir}")
+
+    # Recursively search in all subdirectories
+    for subdir in [os.path.join(current_dir, d) for d in os.listdir(current_dir) if os.path.isdir(os.path.join(current_dir, d))]:
+        build_contracts(subdir)
+
+# Define the directory where contracts are located
+contracts_dir = "./"
+
+# Check if the directory exists
+if not os.path.isdir(contracts_dir):
+    print(f"Directory {contracts_dir} does not exist")
+    exit(1)
+
+# Call the recursive function starting from the contracts_dir
+build_contracts(contracts_dir)

test-contracts/coinfabrik-test-contracts/README.md

@@ -0,0 +1,5 @@
+# Coinfabrik Test Contracts
+
+Coinfabrik developed an Ink! bug taxonomy to test their static testing tool: Scout. In this project, we have reused the same set of test contracts from the Scout tool, with minor modifications to suit our needs. Notably, we have added additional invariants and properties to be checked by our fuzzing capabilities.
+
+By leveraging the existing test contracts from the Scout tool, we can demonstrate and test the capabilities of our fuzzing approach, while also ensuring that we are adhering to the same comprehensive set of tests as the industry-standard Scout tool. The use of these test contracts is done in compliance with the MIT license under which the Scout tool is distributed.