Skip to content

Commit

Permalink
Add Lexer emoji test case
Browse files Browse the repository at this point in the history
  • Loading branch information
MichaReiser committed Sep 7, 2023
1 parent f1a4eb9 commit 5e83ebc
Showing 1 changed file with 18 additions and 0 deletions.
18 changes: 18 additions & 0 deletions crates/ruff_python_parser/src/lexer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1716,4 +1716,22 @@ def f(arg=%timeit a = b):
let source = "[1";
let _ = lex(source, Mode::Module).collect::<Vec<_>>();
}

/// Emoji identifiers are a non-standard python feature and are not supported by our lexer.
#[test]
fn test_emoji_identifier() {
let source = "🐦";

let lexed: Vec<_> = lex(source, Mode::Module).collect();

match lexed.as_slice() {
[Err(error)] => {
assert_eq!(
error.error,
LexicalErrorType::UnrecognizedToken { tok: '🐦' }
);
}
result => panic!("Expected an error token but found {result:?}"),
}
}
}

0 comments on commit 5e83ebc

Please sign in to comment.