Add fuzz tests

Added fuzzing tests:
- As unit tests using `QuickCheck`
- As `cargo fuzz` tests

Both sets of tests use the same set fuzzing functions, running `cargo fuzz` is just
a lot more intense.

Cargo.toml

@@ -22,3 +22,4 @@ thiserror = "1.0.30"
 
 [dev-dependencies]
 hex = "0.4.3"
+quickcheck = "1"

fuzz/.gitignore

@@ -0,0 +1,4 @@
+target
+corpus
+artifacts
+coverage

fuzz/Cargo.toml

@@ -0,0 +1,51 @@
+[package]
+name = "monero-fuzz"
+version = "0.0.0"
+edition = "2021"
+publish = false
+
+[package.metadata]
+cargo-fuzz = true
+
+[dependencies]
+libfuzzer-sys = "0.4"
+
+[dependencies.randomx-rs]
+path = ".."
+
+# Prevent this from interfering with workspaces
+[workspace]
+members = ["."]
+
+[profile.release]
+debug = 1
+
+[[bin]]
+name = "randomx_alloc_cache"
+path = "fuzz_targets/randomx_alloc_cache.rs"
+test = false
+doc = false
+
+[[bin]]
+name = "randomx_create_vm_with_cache_only"
+path = "fuzz_targets/randomx_create_vm_with_cache_only.rs"
+test = false
+doc = false
+
+[[bin]]
+name = "randomx_create_vm_with_cache_and_dataset"
+path = "fuzz_targets/randomx_create_vm_with_cache_and_dataset.rs"
+test = false
+doc = false
+
+[[bin]]
+name = "randomx_vm_calculate_hash_with_cache_only"
+path = "fuzz_targets/randomx_vm_calculate_hash_with_cache_only.rs"
+test = false
+doc = false
+
+[[bin]]
+name = "randomx_vm_calculate_hash_with_cache_and_dataset"
+path = "fuzz_targets/randomx_vm_calculate_hash_with_cache_and_dataset.rs"
+test = false
+doc = false

fuzz/README.md

@@ -0,0 +1,22 @@
+# Fuzzing randomx-rs
+
+See https://rust-fuzz.github.io/book/cargo-fuzz.html for more information on fuzzing with cargo-fuzz.
+Install `cargo-fuzz` as per [installation instructions](https://rust-fuzz.github.io/book/cargo-fuzz/setup.html).
+
+
+**Note:** Fuzzing is not supported on Windows yet.
+
+To get a list of fuzz targets, from a terminal in the project root, run
+```
+cargo fuzz list
+```
+
+To run a fuzz test, from a terminal in the project root, run
+```
+cargo +nightly fuzz run --release <fuzz_target_name>
+```
+To run fuzz tests involving a cache and dataset, on error `libFuzzer: out-of-memory (malloc(2181038016))`, pass 
+`-- -rss_limit_mb=<ram_upper_limit>` as argument to allow using more than 2 GB of RAM - 3GB recommended.
+```
+cargo +nightly fuzz run --release <fuzz_target_name> -- -rss_limit_mb=3221225472
+```

fuzz/fuzz_targets/randomx_alloc_cache.rs

@@ -0,0 +1,8 @@
+#![no_main]
+
+use libfuzzer_sys::fuzz_target;
+use randomx_rs::test_utils::fuzz_randomx_alloc_cache;
+
+fuzz_target!(|data: &[u8]| {
+    fuzz_randomx_alloc_cache(data.to_vec());
+});

fuzz/fuzz_targets/randomx_create_vm_with_cache_and_dataset.rs

@@ -0,0 +1,8 @@
+#![no_main]
+
+use libfuzzer_sys::fuzz_target;
+use randomx_rs::test_utils::fuzz_randomx_create_vm_with_cache_and_dataset;
+
+fuzz_target!(|data: &[u8]| {
+    fuzz_randomx_create_vm_with_cache_and_dataset(data.to_vec());
+});

fuzz/fuzz_targets/randomx_create_vm_with_cache_only.rs

@@ -0,0 +1,8 @@
+#![no_main]
+
+use libfuzzer_sys::fuzz_target;
+use randomx_rs::test_utils::fuzz_randomx_create_vm_with_cache_only;
+
+fuzz_target!(|data: &[u8]| {
+    fuzz_randomx_create_vm_with_cache_only(data.to_vec());
+});

fuzz/fuzz_targets/randomx_vm_calculate_hash_with_cache_and_dataset.rs

@@ -0,0 +1,8 @@
+#![no_main]
+
+use libfuzzer_sys::fuzz_target;
+use randomx_rs::test_utils::fuzz_randomx_vm_calculate_hash_with_cache_and_dataset;
+
+fuzz_target!(|data: &[u8]| {
+    fuzz_randomx_vm_calculate_hash_with_cache_and_dataset(data.to_vec());
+});

fuzz/fuzz_targets/randomx_vm_calculate_hash_with_cache_only.rs

@@ -0,0 +1,8 @@
+#![no_main]
+
+use libfuzzer_sys::fuzz_target;
+use randomx_rs::test_utils::fuzz_randomx_vm_calculate_hash_with_cache_only;
+
+fuzz_target!(|data: &[u8]| {
+    fuzz_randomx_vm_calculate_hash_with_cache_only(data.to_vec());
+});

src/lib.rs

@@ -35,6 +35,9 @@
 //! [RandomX github repo]: <https://github.com/tevador/RandomX>
 //! [design document]: <https://github.com/tevador/RandomX/blob/master/doc/design.md>
 mod bindings;
+/// Test utilities for fuzzing
+pub mod test_utils;
+
 use std::{convert::TryFrom, num::TryFromIntError, ptr, sync::Arc};
 
 use bindings::{
@@ -167,11 +170,11 @@ impl RandomXCache {
         if key.is_empty() {
             Err(RandomXError::ParameterError("key is empty".to_string()))
         } else {
-            let test = unsafe { randomx_alloc_cache(flags.bits) };
-            if test.is_null() {
+            let cache_ptr = unsafe { randomx_alloc_cache(flags.bits) };
+            if cache_ptr.is_null() {
                 Err(RandomXError::CreationError("Could not allocate cache".to_string()))
             } else {
-                let inner = RandomXCacheInner { cache_ptr: test };
+                let inner = RandomXCacheInner { cache_ptr };
                 let result = RandomXCache { inner: Arc::new(inner) };
                 let key_ptr = key.as_ptr() as *mut c_void;
                 let key_size = key.len();
@@ -480,7 +483,9 @@ impl RandomXVM {
 
 #[cfg(test)]
 mod tests {
-    use crate::{RandomXCache, RandomXDataset, RandomXFlag, RandomXVM};
+    use std::{ptr, sync::Arc};
+
+    use crate::{RandomXCache, RandomXCacheInner, RandomXDataset, RandomXDatasetInner, RandomXFlag, RandomXVM};
 
     #[test]
     fn lib_alloc_cache() {
@@ -522,12 +527,33 @@ mod tests {
         let dataset = RandomXDataset::new(flags, cache.clone(), 0).unwrap();
         let memory = dataset.get_data().unwrap_or_else(|_| std::vec::Vec::new());
         assert!(!memory.is_empty(), "Failed to get dataset memory");
-        let vec = vec![0u8; memory.len() as usize];
+        let vec = vec![0u8; memory.len()];
         assert_ne!(memory, vec);
         drop(dataset);
         drop(cache);
     }
 
+    #[test]
+    fn test_null_assignments() {
+        let flags = RandomXFlag::get_recommended_flags();
+        if let Ok(mut vm) = RandomXVM::new(flags, None, None) {
+            let cache = RandomXCache {
+                inner: Arc::new(RandomXCacheInner {
+                    cache_ptr: ptr::null_mut(),
+                }),
+            };
+            assert!(vm.reinit_cache(cache.clone()).is_err());
+            let dataset = RandomXDataset {
+                inner: Arc::new(RandomXDatasetInner {
+                    dataset_ptr: ptr::null_mut(),
+                    dataset_count: 0,
+                    cache,
+                }),
+            };
+            assert!(vm.reinit_dataset(dataset.clone()).is_err());
+        }
+    }
+
     #[test]
     fn lib_calculate_hash() {
         let flags = RandomXFlag::get_recommended_flags();
@@ -537,7 +563,7 @@ mod tests {
         let cache1 = RandomXCache::new(flags, key.as_bytes()).unwrap();
         let mut vm1 = RandomXVM::new(flags, Some(cache1.clone()), None).unwrap();
         let hash1 = vm1.calculate_hash(input.as_bytes()).expect("no data");
-        let vec = vec![0u8; hash1.len() as usize];
+        let vec = vec![0u8; hash1.len()];
         assert_ne!(hash1, vec);
         let reinit_cache = vm1.reinit_cache(cache1.clone());
         assert!(reinit_cache.is_ok());
@@ -589,7 +615,7 @@ mod tests {
         assert_eq!(inputs.len(), hashes.len());
         let mut prev_hash = Vec::new();
         for (i, hash) in hashes.into_iter().enumerate() {
-            let vec = vec![0u8; hash.len() as usize];
+            let vec = vec![0u8; hash.len()];
             assert_ne!(hash, vec);
             assert_ne!(hash, prev_hash);
             let compare = vm.calculate_hash(inputs[i]).unwrap(); // sanity check