fix: Handle multi-byte utf8 characters in formatter

compiler/noirc_frontend/src/lexer/errors.rs

@@ -34,6 +34,8 @@ pub enum LexerErrorKind {
     InvalidEscape { escaped: char, span: Span },
     #[error("Invalid quote delimiter `{delimiter}`, valid delimiters are `{{`, `[`, and `(`")]
     InvalidQuoteDelimiter { delimiter: SpannedToken },
+    #[error("Non-ASCII characters are invalid in comments")]
+    NonAsciiComment { span: Span },
     #[error("Expected `{end_delim}` to close this {start_delim}")]
     UnclosedQuote { start_delim: SpannedToken, end_delim: Token },
 }
@@ -65,6 +67,7 @@ impl LexerErrorKind {
             LexerErrorKind::UnterminatedStringLiteral { span } => *span,
             LexerErrorKind::InvalidEscape { span, .. } => *span,
             LexerErrorKind::InvalidQuoteDelimiter { delimiter } => delimiter.to_span(),
+            LexerErrorKind::NonAsciiComment { span, .. } => *span,
             LexerErrorKind::UnclosedQuote { start_delim, .. } => start_delim.to_span(),
         }
     }
@@ -124,6 +127,9 @@ impl LexerErrorKind {
             LexerErrorKind::InvalidQuoteDelimiter { delimiter } => {
                 (format!("Invalid quote delimiter `{delimiter}`"), "Valid delimiters are `{`, `[`, and `(`".to_string(), delimiter.to_span())
             },
+            LexerErrorKind::NonAsciiComment { span } => {
+                ("Non-ASCII character in comment".to_string(), "Invalid comment character: only ASCII is currently supported.".to_string(), *span)
+            }
             LexerErrorKind::UnclosedQuote { start_delim, end_delim } => {
                 ("Unclosed `quote` expression".to_string(), format!("Expected a `{end_delim}` to close this `{start_delim}`"), start_delim.to_span())
             }

compiler/noirc_frontend/src/lexer/lexer.rs

@@ -18,7 +18,7 @@
     position: Position,
     done: bool,
     skip_comments: bool,
     skip_whitespaces: bool,
     max_integer: BigInt,
 }
 
@@ -46,8 +46,8 @@
             position: 0,
             done: false,
             skip_comments: true,
             skip_whitespaces: true,
             max_integer: BigInt::from_biguint(num_bigint::Sign::Plus, FieldElement::modulus())
                 - BigInt::one(),
         }
     }
@@ -57,8 +57,8 @@
         self
     }
 
     pub fn skip_whitespaces(mut self, flag: bool) -> Self {
         self.skip_whitespaces = flag;
         self
     }
 
@@ -606,6 +606,11 @@
         };
         let comment = self.eat_while(None, |ch| ch != '\n');
 
+        if !comment.is_ascii() {
+            let span = Span::from(start..self.position);
+            return Err(LexerErrorKind::NonAsciiComment { span });
+        }
+
         if doc_style.is_none() && self.skip_comments {
             return self.next_token();
         }
@@ -651,6 +656,11 @@
         }
 
         if depth == 0 {
+            if !content.is_ascii() {
+                let span = Span::from(start..self.position);
+                return Err(LexerErrorKind::NonAsciiComment { span });
+            }
+
             if doc_style.is_none() && self.skip_comments {
                 return self.next_token();
             }
@@ -1331,6 +1341,7 @@
 
                             Err(LexerErrorKind::InvalidIntegerLiteral { .. })
                             | Err(LexerErrorKind::UnexpectedCharacter { .. })
+                            | Err(LexerErrorKind::NonAsciiComment { .. })
                             | Err(LexerErrorKind::UnterminatedBlockComment { .. }) => {
                                 expected_token_found = true;
                             }
@@ -1389,4 +1400,17 @@
             }
         }
     }
+
+    #[test]
+    fn test_non_ascii_comments() {
+        let cases = vec!["// 🙂", "// schön", "/* in the middle 🙂 of a comment */"];
+
+        for source in cases {
+            let mut lexer = Lexer::new(source);
+            assert!(
+                lexer.any(|token| matches!(token, Err(LexerErrorKind::NonAsciiComment { .. }))),
+                "Expected NonAsciiComment error"
+            );
+        }
+    }
 }

tooling/nargo_fmt/src/visitor.rs

@@ -36,7 +36,7 @@ impl<'me> FmtVisitor<'me> {
 
     pub(crate) fn slice(&self, span: impl Into<Span>) -> &'me str {
         let span = span.into();
-        &self.source[span.start() as usize..span.end() as usize]
+        str_slice(self.source, span.start() as usize, span.end() as usize)
     }
 
     pub(crate) fn span_after(&self, span: impl Into<Span>, token: Token) -> Span {
@@ -188,7 +188,7 @@ impl<'me> FmtVisitor<'me> {
 
             match comment.token() {
                 Token::LineComment(_, _) | Token::BlockComment(_, _) => {
-                    let comment = &slice[span.start() as usize..span.end() as usize];
+                    let comment = str_slice(slice, span.start() as usize, span.end() as usize);
                     if result.ends_with('\n') {
                         result.push_str(&indent);
                     } else if !self.at_start() {
@@ -247,6 +247,19 @@ impl<'me> FmtVisitor<'me> {
     }
 }
 
+pub(crate) fn str_slice(s: &str, start: usize, end: usize) -> &str {
+    &s[start..ceil_char_boundary(s, end)]
+}
+
+pub(crate) fn ceil_char_boundary(s: &str, byte_index: usize) -> usize {
+    for i in byte_index..s.len() {
+        if s.is_char_boundary(i) {
+            return i;
+        }
+    }
+    s.len()
+}
+
 #[derive(Clone, Copy, Debug, Default)]
 pub(crate) struct Indent {
     block_indent: usize,