feat: lalrpop lexer prototype

compiler/noirc_frontend/Cargo.toml

@@ -24,9 +24,16 @@ small-ord-set = "0.1.3"
 regex = "1.9.1"
 tracing.workspace = true
 petgraph = "0.6"
+lalrpop-util = { version = "0.20.2", features = ["lexer"] }
 
 [dev-dependencies]
 base64.workspace = true
 strum = "0.24"
 strum_macros = "0.24"
 tempfile.workspace = true
+
+[build-dependencies]
+lalrpop = "0.20.2"
+
+[features]
+experimental_parser = []

compiler/noirc_frontend/build.rs

@@ -0,0 +1,28 @@
+use std::fs::{read_to_string, File};
+use std::io::Write;
+
+fn main() {
+    lalrpop::Configuration::new()
+        .emit_rerun_directives(true)
+        .use_cargo_dir_conventions()
+        .process()
+        .unwrap();
+
+    // here, we get a lint error from "extern crate core" so patching that until lalrpop does
+    // (adding cfg directives appears to be unsupported by lalrpop)
+    let out_dir = std::env::var("OUT_DIR").unwrap();
+    let parser_path = std::path::Path::new(&out_dir).join("noir_parser.rs");
+    let content_str = read_to_string(parser_path.clone()).unwrap();
+    let mut parser_file = File::create(parser_path).unwrap();
+    for line in content_str.lines() {
+        if line.contains("extern crate core") {
+            parser_file
+                .write_all(
+                    format!("{}\n", line.replace("extern crate core", "use core")).as_bytes(),
+                )
+                .unwrap();
+        } else {
+            parser_file.write_all(format!("{}\n", line).as_bytes()).unwrap();
+        }
+    }
+}

compiler/noirc_frontend/src/lexer/lexer.rs

@@ -2,7 +2,9 @@ use crate::token::{Attribute, DocStyle};
 
 use super::{
     errors::LexerErrorKind,
-    token::{IntType, Keyword, SpannedToken, Token, Tokens},
+    token::{
+        token_to_borrowed_token, BorrowedToken, IntType, Keyword, SpannedToken, Token, Tokens,
+    },
 };
 use acvm::FieldElement;
 use noirc_errors::{Position, Span};
@@ -21,6 +23,21 @@ pub struct Lexer<'a> {
 
 pub type SpannedTokenResult = Result<SpannedToken, LexerErrorKind>;
 
+pub(crate) fn from_spanned_token_result(
+    token_result: &SpannedTokenResult,
+) -> Result<(usize, BorrowedToken<'_>, usize), LexerErrorKind> {
+    token_result
+        .as_ref()
+        .map(|spanned_token| {
+            (
+                spanned_token.to_span().start() as usize,
+                token_to_borrowed_token(spanned_token.into()),
+                spanned_token.to_span().end() as usize,
+            )
+        })
+        .map_err(Clone::clone)
+}
+
 impl<'a> Lexer<'a> {
     /// Given a source file of noir code, return all the tokens in the file
     /// in order, along with any lexing errors that occurred.
@@ -94,7 +111,7 @@ impl<'a> Lexer<'a> {
 
     fn next_token(&mut self) -> SpannedTokenResult {
         match self.next_char() {
-            Some(x) if x.is_whitespace() => {
+            Some(x) if Self::is_code_whitespace(x) => {
                 let spanned = self.eat_whitespace(x);
                 if self.skip_whitespaces {
                     self.next_token()
@@ -560,16 +577,21 @@ impl<'a> Lexer<'a> {
         }
     }
 
+    fn is_code_whitespace(c: char) -> bool {
+        c == '\t' || c == '\n' || c == '\r' || c == ' '
+    }
+
     /// Skips white space. They are not significant in the source language
     fn eat_whitespace(&mut self, initial_char: char) -> SpannedToken {
         let start = self.position;
-        let whitespace = self.eat_while(initial_char.into(), |ch| ch.is_whitespace());
+        let whitespace = self.eat_while(initial_char.into(), Self::is_code_whitespace);
         SpannedToken::new(Token::Whitespace(whitespace), Span::inclusive(start, self.position))
     }
 }
 
 impl<'a> Iterator for Lexer<'a> {
     type Item = SpannedTokenResult;
+
     fn next(&mut self) -> Option<Self::Item> {
         if self.done {
             None
@@ -578,10 +600,12 @@ impl<'a> Iterator for Lexer<'a> {
         }
     }
 }
+
 #[cfg(test)]
 mod tests {
     use super::*;
     use crate::token::{FunctionAttribute, SecondaryAttribute, TestScope};
+
     #[test]
     fn test_single_double_char() {
         let input = "! != + ( ) { } [ ] | , ; : :: < <= > >= & - -> . .. % / * = == << >>";