Add a subcommand to build a clusterfuzz tarball (#162)

* feat: add a build-clusterfuzz command

* feat: improve clusterfuzz error message detection

LLVMFuzzerTestOneInput is not supposed to ever return something other
than 0. Instead, the whole process should abort in case of a bug being
detected.

Hence, this replaces the previous returning of 1 (which triggered a
libfuzzer assertion downstream, that was detected by clusterfuzz as the
root cause of the error) by an abort (which should let clusterfuzz
detect the actual cause of the error in the panic message)

* put built tarball in the real target directory

* handle non-harnessed tests and add integration test

* add integration test, fix rebase mishaps

---------

Co-authored-by: Léo Gaspard <leo@gaspard.io>

bin/Cargo.toml

@@ -3,3 +3,9 @@ members = [
     "cargo-bolero",
 ]
 resolver = "2"
+
+[profile.fuzz]
+inherits = "dev"
+opt-level = 3
+incremental = false
+codegen-units = 1

bin/cargo-bolero/Cargo.toml

@@ -23,12 +23,14 @@ anyhow = "1.0"
 bit-set = "0.5"
 bolero-afl = { version = "0.9", path = "../../lib/bolero-afl", default-features = false, features = ["bin"], optional = true }
 bolero-honggfuzz = { version = "0.9", path = "../../lib/bolero-honggfuzz", default-features = false, features = ["bin"], optional = true }
+cargo_metadata = "0.18"
 humantime = "2"
 lazy_static = "1"
 rustc_version = "0.4"
 structopt = "0.3"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
+tar = "0.4"
 tempfile = "3"
 
 [dev-dependencies]
@@ -44,3 +46,7 @@ name = "fuzz_generator"
 path = "tests/fuzz_generator/fuzz_target.rs"
 harness = false
 
+[[test]]
+name = "fuzz_harnessed"
+path = "tests/fuzz_harnessed/fuzz_target.rs"
+harness = true

bin/cargo-bolero/src/build_clusterfuzz.rs

@@ -0,0 +1,122 @@
+use crate::{list::List, project::Project};
+use anyhow::{Context, Result};
+use std::{
+    collections::{hash_map::DefaultHasher, HashMap},
+    convert::TryInto,
+    hash::{Hash, Hasher},
+    path::{Path, PathBuf},
+};
+use structopt::StructOpt;
+
+/// Builds a tarball for uploading to clusterfuzz
+#[derive(Debug, StructOpt)]
+pub struct BuildClusterfuzz {
+    #[structopt(flatten)]
+    project: Project,
+}
+
+impl BuildClusterfuzz {
+    pub fn exec(&self) -> Result<()> {
+        // Find the target directory
+        let output_dir = cargo_metadata::MetadataCommand::new()
+            .exec()
+            .expect("running `cargo metadata`")
+            .target_directory
+            .into_std_path_buf()
+            .join("fuzz");
+
+        // Create the output directory, and the archive
+        std::fs::create_dir_all(&output_dir).context("creating clusterfuzz build directory")?;
+        let output_path = output_dir.join("clusterfuzz.tar");
+        let mut tarball = tar::Builder::new(
+            std::fs::File::create(&output_path)
+                .with_context(|| format!("creating {:?}", output_path))?,
+        );
+
+        // Figure out the list of fuzz targets, grouped by which test executable they use
+        let targets = List::new(self.project.clone())
+            .list()
+            .context("listing fuzz targets")?;
+        let mut targets_per_exe = HashMap::new();
+        for t in targets {
+            targets_per_exe
+                .entry(t.exe.clone())
+                .or_insert_with(Vec::new)
+                .push(t);
+        }
+
+        // Add all the targets to the archive
+        for (list_exe, tests) in targets_per_exe {
+            let mut hasher = DefaultHasher::new();
+            list_exe.hash(&mut hasher);
+            let hash = hasher.finish();
+            let list_bin = Path::new(&list_exe).file_name().unwrap().to_string_lossy();
+            let dir = PathBuf::from(format!("{}-{:x}", list_bin, hash));
+
+            let fuzz_exe =
+                crate::libfuzzer::build(self.project.clone(), tests[0].test_name.clone())
+                    .context("building to-be-fuzzed executable")?;
+            // .cargo extension is not an ALLOWED_FUZZ_TARGET_EXTENSIONS for clusterfuzz, so it doesn’t get picked up as a fuzzer
+            let fuzz_bin = format!("{}.cargo", fuzz_exe.file_name().unwrap().to_string_lossy());
+            tarball
+                .append_file(
+                    dir.join(&*fuzz_bin),
+                    &mut std::fs::File::open(&fuzz_exe)
+                        .with_context(|| format!("opening {:?}", &fuzz_exe))?,
+                )
+                .with_context(|| format!("appending {:?} to {:?}", &fuzz_exe, &output_path))?;
+
+            for t in tests {
+                // : is not in VALID_TARGET_NAME_REGEX ; so we don’t use it and make sure to end in _fuzzer so we get picked up as a fuzzer
+                let fuzzer_name = format!("{}_fuzzer", t.test_name.replace(':', "-"));
+                let path = dir.join(&fuzzer_name);
+                let contents = if t.is_harnessed {
+                    format!(
+                        r#"#!/bin/sh
+exec \
+    env BOLERO_TEST_NAME="{1}" \
+    BOLERO_LIBTEST_HARNESS=1 \
+    BOLERO_LIBFUZZER_ARGS="$*" \
+    RUST_BACKTRACE=1 \
+"$(dirname "$0")/{0}" \
+    "{1}" \
+    --exact \
+    --nocapture \
+    --quiet \
+    --test-threads 1
+"#,
+                        fuzz_bin, t.test_name,
+                    )
+                    .into_bytes()
+                } else {
+                    format!(
+                        r#"#!/bin/sh
+exec \
+env BOLERO_TEST_NAME="{1}" \
+    BOLERO_LIBFUZZER_ARGS="$*" \
+    RUST_BACKTRACE=1 \
+"$(dirname "$0")/{0}"
+"#,
+                        fuzz_bin, t.test_name,
+                    )
+                    .into_bytes()
+                };
+                let mut header = tar::Header::new_gnu();
+                header.set_mode(0o555);
+                header.set_size(contents.len().try_into().unwrap());
+                header.set_cksum();
+                tarball
+                    .append_data(&mut header, &path, &*contents)
+                    .with_context(|| {
+                        format!("adding relay script {:?} to {:?}", path, output_path)
+                    })?;
+            }
+        }
+        tarball
+            .finish()
+            .with_context(|| format!("finishing writing {:?}", output_path))?;
+
+        println!("Built the tarball in {:?}", output_path);
+        Ok(())
+    }
+}

bin/cargo-bolero/src/libfuzzer.rs

@@ -1,4 +1,4 @@
-use crate::{exec, reduce, test, Selection};
+use crate::{exec, project::Project, reduce, test, Selection};
 use anyhow::{anyhow, Result};
 use bit_set::BitSet;
 use core::cmp::Ordering;
@@ -26,6 +26,17 @@ const FLAGS: &[&str] = &[
     "-Cllvm-args=-sanitizer-coverage-stack-depth",
 ];
 
+/// test_name needs to be one cargo-bolero test in the binary described by project
+///
+/// Returns the path to the binary
+pub(crate) fn build(project: Project, test_name: String) -> Result<PathBuf> {
+    Ok(PathBuf::from(
+        Selection::new(project, test_name)
+            .test_target(FLAGS, "libfuzzer")?
+            .exe,
+    ))
+}
+
 pub(crate) fn test(selection: &Selection, test_args: &test::Args) -> Result<()> {
     let test_target = selection.test_target(FLAGS, "libfuzzer")?;
     let corpus_dir = test_args

bin/cargo-bolero/src/list.rs

@@ -11,7 +11,11 @@ pub struct List {
 }
 
 impl List {
-    pub fn exec(&self) -> Result<()> {
+    pub fn new(project: Project) -> Self {
+        Self { project }
+    }
+
+    pub fn list(&self) -> Result<Vec<TestTarget>> {
         let mut build_command = self.cmd("test", &[], None)?;
         build_command.arg("--no-run");
         exec(build_command)?;
@@ -22,10 +26,13 @@ impl List {
             .arg("--nocapture")
             .env("CARGO_BOLERO_SELECT", "all")
             .output()?;
-
         // ignore the status in case any tests failed
 
-        for target in TestTarget::all_from_stdout(&output.stdout)? {
+        TestTarget::all_from_stdout(&output.stdout)
+    }
+
+    pub fn exec(&self) -> Result<()> {
+        for target in self.list()? {
             println!("{}", target);
         }
 

bin/cargo-bolero/src/main.rs

@@ -1,10 +1,14 @@
-use crate::{list::List, new::New, reduce::Reduce, selection::Selection, test::Test};
+use crate::{
+    build_clusterfuzz::BuildClusterfuzz, list::List, new::New, reduce::Reduce,
+    selection::Selection, test::Test,
+};
 use anyhow::{anyhow, Result};
 use std::io::Write;
 use structopt::StructOpt;
 
 #[cfg(feature = "afl")]
 mod afl;
+mod build_clusterfuzz;
 mod engine;
 #[cfg(feature = "honggfuzz")]
 mod honggfuzz;
@@ -28,6 +32,7 @@ enum Commands {
     Reduce(Reduce),
     New(New),
     List(List),
+    BuildClusterfuzz(BuildClusterfuzz),
 }
 
 impl Commands {
@@ -37,6 +42,7 @@ impl Commands {
             Self::Reduce(cmd) => cmd.exec(),
             Self::New(cmd) => cmd.exec(),
             Self::List(cmd) => cmd.exec(),
+            Self::BuildClusterfuzz(cmd) => cmd.exec(),
         }
     }
 }

bin/cargo-bolero/src/project.rs

@@ -9,7 +9,7 @@ lazy_static! {
     static ref RUST_VERSION: rustc_version::VersionMeta = rustc_version::version_meta().unwrap();
 }
 
-#[derive(Debug, StructOpt)]
+#[derive(Clone, Debug, StructOpt)]
 pub struct Project {
     /// Build with the sanitizer enabled
     #[structopt(short, long, default_value = "address")]

bin/cargo-bolero/src/selection.rs

@@ -13,6 +13,10 @@ pub struct Selection {
 }
 
 impl Selection {
+    pub fn new(project: Project, test: String) -> Self {
+        Self { project, test }
+    }
+
     pub fn test_target(&self, flags: &[&str], fuzzer: &str) -> Result<TestTarget> {
         let mut build_command = self.cmd("test", flags, Some(fuzzer))?;
         build_command

bin/cargo-bolero/tests/fuzz_harnessed/fuzz_target.rs

@@ -0,0 +1,4 @@
+#[test]
+fn harnessed_fuzzer() {
+    bolero::check!().for_each(|_| {});
+}

lib/bolero-libfuzzer/src/lib.rs

@@ -20,7 +20,7 @@ pub mod fuzzer {
         pub fn LLVMFuzzerStartTest(a: c_int, b: *const *const c_char) -> c_int;
     }
 
-    type TestFn<'a> = &'a mut dyn FnMut(&[u8]) -> bool;
+    type TestFn<'a> = &'a mut dyn FnMut(&[u8]);
 
     static mut TESTFN: Option<TestFn> = None;
 
@@ -47,13 +47,12 @@ pub mod fuzzer {
             let mut report = GeneratorReport::default();
             report.spawn_timer();
 
-            start(&mut |slice: &[u8]| -> bool {
+            start(&mut |slice: &[u8]| {
                 let mut input = ByteSliceTestInput::new(slice, options);
 
                 match test.test(&mut input) {
                     Ok(is_valid) => {
                         report.on_result(is_valid);
-                        true
                     }
                     Err(error) => {
                         eprintln!("test failed; shrinking input...");
@@ -73,7 +72,7 @@ pub mod fuzzer {
                             );
                         }
 
-                        false
+                        std::process::abort();
                     }
                 }
             })
@@ -129,11 +128,9 @@ pub mod fuzzer {
         }
     }
 
-    fn start<F: FnMut(&[u8]) -> bool>(run_one_test: &mut F) -> Never {
+    fn start<F: FnMut(&[u8])>(run_one_test: &mut F) -> Never {
         unsafe {
-            TESTFN = Some(std::mem::transmute(
-                run_one_test as &mut dyn FnMut(&[u8]) -> bool,
-            ));
+            TESTFN = Some(std::mem::transmute(run_one_test as &mut dyn FnMut(&[u8])));
         }
 
         // Libfuzzer can generate multiple jobs that can make the binary recurse.
@@ -183,11 +180,8 @@ pub mod fuzzer {
     #[no_mangle]
     pub unsafe extern "C" fn LLVMFuzzerTestOneInput(data: *const u8, size: usize) -> i32 {
         let data_slice = std::slice::from_raw_parts(data, size);
-        if (TESTFN.as_mut().expect("uninitialized test function"))(data_slice) {
-            0
-        } else {
-            1
-        }
+        (TESTFN.as_mut().expect("uninitialized test function"))(data_slice);
+        0
     }
 
     #[doc(hidden)]

tests/src/cargo_bolero.rs

@@ -32,6 +32,21 @@ impl Test {
         cmd!(sh, "cargo {toolchain...} test").run()?;
         cmd!(sh, "cargo {toolchain...} build").run()?;
 
+        // Validate `cargo bolero build-clusterfuzz` runs fine
+        // This runs it in $repo/bin, which is fine as cargo-bolero does have fuzz-tests
+        cmd!(
+            sh,
+            "cargo {toolchain...} run build-clusterfuzz --rustc-bootstrap"
+        )
+        .run()?;
+
+        // Validate the built fuzzers work fine
+        sh.change_dir("target/fuzz");
+        cmd!(sh, "tar xf clusterfuzz.tar --strip-components=1").run()?;
+        cmd!(sh, "./fuzz_bytes_fuzzer -runs=10").run()?;
+        cmd!(sh, "./fuzz_generator_fuzzer -runs=10").run()?;
+        cmd!(sh, "./harnessed_fuzzer_fuzzer -runs=10").run()?;
+
         Ok(())
     }
 }