Create fuzzers for testing correctness of parsing, linting and fixing (…

…#4822)

Co-authored-by: Micha Reiser <micha@reiser.io>

.github/workflows/ci.yaml

@@ -87,6 +87,19 @@ jobs:
           name: ruff
           path: target/debug/ruff
 
+  cargo-fuzz:
+    runs-on: ubuntu-latest
+    name: "cargo fuzz"
+    steps:
+      - uses: actions/checkout@v3
+      - name: "Install Rust toolchain"
+        run: rustup show
+      - uses: Swatinem/rust-cache@v2
+      - name: "Install cargo-binstall"
+        uses: taiki-e/install-action@cargo-binstall
+      - run: cargo binstall cargo-fuzz -y
+      - run: cargo fuzz build -s none
+
   cargo-test-wasm:
     runs-on: ubuntu-latest
     name: "cargo test (wasm)"

crates/ruff/src/lib.rs

@@ -35,5 +35,5 @@ mod rule_selector;
 pub mod rules;
 pub mod settings;
 
-#[cfg(test)]
-mod test;
+#[cfg(any(test, fuzzing))]
+pub mod test;

crates/ruff/src/test.rs

@@ -1,8 +1,9 @@
-#![cfg(test)]
+#![cfg(any(test, fuzzing))]
 //! Helper functions for the tests of rule implementations.
 
 use std::path::Path;
 
+#[cfg(not(fuzzing))]
 use anyhow::Result;
 use itertools::Itertools;
 use ruff_textwrap::dedent;
@@ -21,29 +22,41 @@ use crate::registry::AsRule;
 use crate::rules::pycodestyle::rules::syntax_error;
 use crate::settings::{flags, Settings};
 
+#[cfg(not(fuzzing))]
 pub(crate) fn test_resource_path(path: impl AsRef<Path>) -> std::path::PathBuf {
     Path::new("./resources/test/").join(path)
 }
 
 /// Run [`check_path`] on a file in the `resources/test/fixtures` directory.
+#[cfg(not(fuzzing))]
 pub(crate) fn test_path(path: impl AsRef<Path>, settings: &Settings) -> Result<Vec<Message>> {
     let path = test_resource_path("fixtures").join(path);
     let contents = std::fs::read_to_string(&path)?;
     Ok(test_contents(&contents, &path, settings))
 }
 
 /// Run [`check_path`] on a snippet of Python code.
-pub(crate) fn test_snippet(contents: &str, settings: &Settings) -> Vec<Message> {
+pub fn test_snippet(contents: &str, settings: &Settings) -> Vec<Message> {
     let path = Path::new("<filename>");
     let contents = dedent(contents);
     test_contents(&contents, path, settings)
 }
 
+thread_local! {
+    static MAX_ITERATIONS: std::cell::Cell<usize> = std::cell::Cell::new(20);
+}
+
+pub fn set_max_iterations(max: usize) {
+    MAX_ITERATIONS.with(|iterations| iterations.set(max));
+}
+
+pub(crate) fn max_iterations() -> usize {
+    MAX_ITERATIONS.with(std::cell::Cell::get)
+}
+
 /// A convenient wrapper around [`check_path`], that additionally
 /// asserts that autofixes converge after a fixed number of iterations.
 fn test_contents(contents: &str, path: &Path, settings: &Settings) -> Vec<Message> {
-    static MAX_ITERATIONS: usize = 20;
-
     let tokens: Vec<LexResult> = ruff_rustpython::tokenize(contents);
     let locator = Locator::new(contents);
     let stylist = Stylist::from_tokens(&tokens, &locator);
@@ -83,14 +96,16 @@ fn test_contents(contents: &str, path: &Path, settings: &Settings) -> Vec<Messag
         let mut contents = contents.to_string();
 
         while let Some((fixed_contents, _)) = fix_file(&diagnostics, &Locator::new(&contents)) {
-            if iterations < MAX_ITERATIONS {
+            if iterations < max_iterations() {
                 iterations += 1;
             } else {
                 let output = print_diagnostics(diagnostics, path, &contents);
 
                 panic!(
-                    "Failed to converge after {MAX_ITERATIONS} iterations. This likely \
-                     indicates a bug in the implementation of the fix. Last diagnostics:\n{output}"
+                    "Failed to converge after {} iterations. This likely \
+                     indicates a bug in the implementation of the fix. Last diagnostics:\n{}",
+                    max_iterations(),
+                    output
                 );
             }
 

fuzz/.gitignore

@@ -0,0 +1,2 @@
+artifacts/
+corpus/ruff_fix_validity