feat: Add test(should_fail) attribute for tests that are meant to fail

crates/lsp/src/lib.rs

@@ -199,8 +199,8 @@ fn on_code_lens_request(
         let tests = context
             .get_all_test_functions_in_crate_matching(&crate_id, FunctionNameMatch::Anything);
 
-        for (func_name, func_id) in tests {
-            let location = context.function_meta(&func_id).name.location;
+        for (func_name, test_function) in tests {
+            let location = context.function_meta(&test_function.0).name.location;
             let file_id = location.file;
 
             // Ignore diagnostics for any file that wasn't the file we saved

crates/nargo_cli/src/cli/test_cmd.rs

@@ -94,7 +94,15 @@
             .expect("Failed to write to stdout");
         writer.flush().expect("Failed to flush writer");
 
-        match run_test(backend, &test_name, test_function, &context, show_output, compile_options) {
+        match run_test(
+            backend,
+            &test_name,
+            test_function.0,
+            test_function.1,
+            &context,
+            show_output,
+            compile_options,
+        ) {
             Ok(_) => {
                 writer
                     .set_color(ColorSpec::new().set_fg(Some(Color::Green)))
@@ -124,29 +132,53 @@
     backend: &B,
     test_name: &str,
     main: FuncId,
+    should_fail: bool,
     context: &Context,
     show_output: bool,
     config: &CompileOptions,
 ) -> Result<(), CliError<B>> {
-    let mut program = compile_no_check(context, config, main).map_err(|err| {
+    let report_error = |err| {
         noirc_errors::reporter::report_all(&context.file_manager, &[err], config.deny_warnings);
-        CliError::Generic(format!("Test '{test_name}' failed to compile"))
-    })?;
-
-    // Note: We could perform this test using the unoptimized ACIR as generated by `compile_no_check`.
-    program.circuit = optimize_circuit(backend, program.circuit).unwrap().0;
-
-    // Run the backend to ensure the PWG evaluates functions like std::hash::pedersen,
-    // otherwise constraints involving these expressions will not error.
-    match execute_circuit(backend, program.circuit, WitnessMap::new(), show_output) {
-        Ok(_) => Ok(()),
-        Err(error) => {
-            let writer = StandardStream::stderr(ColorChoice::Always);
-            let mut writer = writer.lock();
-            writer.set_color(ColorSpec::new().set_fg(Some(Color::Red))).ok();
-            writeln!(writer, "failed").ok();
-            writer.reset().ok();
-            Err(error.into())
+        Err(CliError::Generic(format!("Test '{test_name}' failed to compile")))
+    };
+    let program = compile_no_check(context, config, main);
+    match program {
+        Ok(mut program) => {
+            // Note: We could perform this test using the unoptimized ACIR as generated by `compile_no_check`.
+            program.circuit = optimize_circuit(backend, program.circuit).unwrap().0;
+            // Run the backend to ensure the PWG evaluates functions like std::hash::pedersen,
+            // otherwise constraints involving these expressions will not error.
+            let circuit_execution =
+                execute_circuit(backend, program.circuit, WitnessMap::new(), show_output);
+            if should_fail {
+                match circuit_execution {
+                    Ok(_) => Err(CliError::Generic(format!("Test '{test_name}' should fail"))),
+                    Err(_) => Ok(()),
+                }
+            } else {
+                match circuit_execution {
+                    Ok(_) => Ok(()),
+                    Err(error) => {
+                        let writer = StandardStream::stderr(ColorChoice::Always);
+                        let mut writer = writer.lock();
+                        writer.set_color(ColorSpec::new().set_fg(Some(Color::Red))).ok();
+                        writeln!(writer, "failed").ok();
+                        writer.reset().ok();
+                        Err(error.into())
+                    }
+                }
+            }
+        }
+        Err(err) => {
+            if should_fail {
+                if !err.diagnostic.message.contains("Failed constraint") {
+                    report_error(err)
+                } else {
+                    Ok(())
+                }
+            } else {
+                report_error(err)
+            }
         }
     }
 }

crates/noirc_frontend/src/ast/function.rs

@@ -83,7 +83,7 @@ impl From<FunctionDefinition> for NoirFunction {
         let kind = match fd.attribute {
             Some(Attribute::Builtin(_)) => FunctionKind::Builtin,
             Some(Attribute::Foreign(_)) => FunctionKind::LowLevel,
-            Some(Attribute::Test) => FunctionKind::Normal,
+            Some(Attribute::Test { .. }) => FunctionKind::Normal,
             Some(Attribute::Oracle(_)) => FunctionKind::Oracle,
             Some(Attribute::Deprecated(_)) | None => FunctionKind::Normal,
             Some(Attribute::Custom(_)) => FunctionKind::Normal,

crates/noirc_frontend/src/hir/def_map/mod.rs

@@ -1,7 +1,7 @@
 use crate::graph::CrateId;
 use crate::hir::def_collector::dc_crate::DefCollector;
 use crate::hir::Context;
-use crate::node_interner::{FuncId, NodeInterner};
+use crate::node_interner::{FuncId, NodeInterner, TestFunc};
 use crate::parser::{parse_program, ParsedModule};
 use crate::token::Attribute;
 use arena::{Arena, Index};
@@ -129,12 +129,26 @@ impl CrateDefMap {
     pub fn get_all_test_functions<'a>(
         &'a self,
         interner: &'a NodeInterner,
-    ) -> impl Iterator<Item = FuncId> + 'a {
+    ) -> impl Iterator<Item = TestFunc> + 'a {
         self.modules.iter().flat_map(|(_, module)| {
             module
                 .value_definitions()
                 .filter_map(|id| id.as_function())
-                .filter(|id| interner.function_meta(id).attributes == Some(Attribute::Test))
+                .filter(|id| {
+                    interner.function_meta(id).attributes
+                        == Some(Attribute::Test { expect_failure: true })
+                        || interner.function_meta(id).attributes
+                            == Some(Attribute::Test { expect_failure: true })
+                })
+                .map(|id| {
+                    if interner.function_meta(&id).attributes
+                        == Some(Attribute::Test { expect_failure: true })
+                    {
+                        (id, true)
+                    } else {
+                        (id, false)
+                    }
+                })
         })
     }
 

crates/noirc_frontend/src/hir/mod.rs

@@ -6,7 +6,7 @@ pub mod type_check;
 
 use crate::graph::{CrateGraph, CrateId};
 use crate::hir_def::function::FuncMeta;
-use crate::node_interner::{FuncId, NodeInterner};
+use crate::node_interner::{FuncId, NodeInterner, TestFunc};
 use def_map::{Contract, CrateDefMap};
 use fm::FileManager;
 use std::collections::HashMap;
@@ -107,22 +107,21 @@ impl Context {
         &self,
         crate_id: &CrateId,
         pattern: FunctionNameMatch,
-    ) -> Vec<(String, FuncId)> {
+    ) -> Vec<(String, TestFunc)> {
         let interner = &self.def_interner;
         let def_map = self.def_map(crate_id).expect("The local crate should be analyzed already");
 
         def_map
             .get_all_test_functions(interner)
-            .filter_map(|id| {
+            .filter_map(|(id, not_fail)| {
                 let fully_qualified_name = self.fully_qualified_function_name(crate_id, &id);
                 match &pattern {
-                    FunctionNameMatch::Anything => Some((fully_qualified_name, id)),
-                    FunctionNameMatch::Exact(pattern) => {
-                        (&fully_qualified_name == pattern).then_some((fully_qualified_name, id))
-                    }
-                    FunctionNameMatch::Contains(pattern) => {
-                        fully_qualified_name.contains(pattern).then_some((fully_qualified_name, id))
-                    }
+                    FunctionNameMatch::Anything => Some((fully_qualified_name, (id, not_fail))),
+                    FunctionNameMatch::Exact(pattern) => (&fully_qualified_name == pattern)
+                        .then_some((fully_qualified_name, (id, not_fail))),
+                    FunctionNameMatch::Contains(pattern) => fully_qualified_name
+                        .contains(pattern)
+                        .then_some((fully_qualified_name, (id, not_fail))),
                 }
             })
             .collect()

crates/noirc_frontend/src/hir/resolution/resolver.rs

@@ -700,7 +700,10 @@
             self.push_err(ResolverError::DistinctNotAllowed { ident: func.name_ident().clone() });
         }
 
-        if attributes == Some(Attribute::Test) && !parameters.is_empty() {
+        if ((attributes == Some(Attribute::Test { expect_failure: true }))
+            || (attributes == Some(Attribute::Test { expect_failure: false })))
+            && !parameters.is_empty()
+        {
             self.push_err(ResolverError::TestFunctionHasParameters {
                 span: func.name_ident().span(),
             });
@@ -1934,7 +1937,7 @@
                 println(f"I want to print {0}");
 
                 let new_val = 10;
                 println(f"randomstring{new_val}{new_val}");
             }
             fn println<T>(x : T) -> T {
                 x

crates/noirc_frontend/src/lexer/token.rs

@@ -325,7 +325,7 @@
     Builtin(String),
     Oracle(String),
     Deprecated(Option<String>),
-    Test,
+    Test { expect_failure: bool },
     Custom(String),
 }
 
@@ -335,7 +335,13 @@
             Attribute::Foreign(ref k) => write!(f, "#[foreign({k})]"),
             Attribute::Builtin(ref k) => write!(f, "#[builtin({k})]"),
             Attribute::Oracle(ref k) => write!(f, "#[oracle({k})]"),
-            Attribute::Test => write!(f, "#[test]"),
+            Attribute::Test { expect_failure } => {
+                if !expect_failure {
+                    write!(f, "#[test]")
+                } else {
+                    write!(f, "#[test(should_fail)]")
+                }
+            }
             Attribute::Deprecated(None) => write!(f, "#[deprecated]"),
             Attribute::Deprecated(Some(ref note)) => write!(f, r#"#[deprecated("{note}")]"#),
             Attribute::Custom(ref k) => write!(f, "#[{k}]"),
@@ -391,7 +397,17 @@
 
                 Attribute::Deprecated(name.trim_matches('"').to_string().into())
             }
-            ["test"] => Attribute::Test,
+            ["test"] => Attribute::Test { expect_failure: false },
+            ["test", name] => {
+                if name != &"should_fail" {
+                    return Err(LexerErrorKind::MalformedFuncAttribute {
+                        span,
+                        found: word.to_owned(),
+                    });
+                }
+
+                Attribute::Test { expect_failure: true }
+            }
             tokens => {
                 tokens.iter().try_for_each(|token| validate(token))?;
                 Attribute::Custom(word.to_owned())
@@ -431,7 +447,7 @@
             Attribute::Builtin(string) => string,
             Attribute::Oracle(string) => string,
             Attribute::Deprecated(Some(string)) => string,
-            Attribute::Test | Attribute::Deprecated(None) => "",
+            Attribute::Test { .. } | Attribute::Deprecated(None) => "",
             Attribute::Custom(string) => string,
         }
     }
@@ -496,7 +512,7 @@
             Keyword::Field => write!(f, "Field"),
             Keyword::Fn => write!(f, "fn"),
             Keyword::For => write!(f, "for"),
             Keyword::FormatString => write!(f, "fmtstr"),
             Keyword::Global => write!(f, "global"),
             Keyword::If => write!(f, "if"),
             Keyword::Impl => write!(f, "impl"),
@@ -538,7 +554,7 @@
             "Field" => Keyword::Field,
             "fn" => Keyword::Fn,
             "for" => Keyword::For,
             "fmtstr" => Keyword::FormatString,
             "global" => Keyword::Global,
             "if" => Keyword::If,
             "impl" => Keyword::Impl,

crates/noirc_frontend/src/node_interner.rs

@@ -138,6 +138,8 @@ impl FuncId {
     }
 }
 
+pub type TestFunc = (FuncId, /* should fail */ bool);
+
 #[derive(Debug, Eq, PartialEq, Hash, Copy, Clone)]
 pub struct StructId(pub ModuleId);