Use the unicode-ident crate (#7212)

astral-sh · Sep 7, 2023 · f1a4eb9 · f1a4eb9
1 parent 041cdb9
commit f1a4eb9
Show file tree

Hide file tree

Showing 8 changed files with 10 additions and 44 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -49,7 +49,7 @@ toml = { version = "0.7.2" }
 tracing = "0.1.37"
 tracing-indicatif = "0.3.4"
 tracing-subscriber = { version = "0.3.17", features = ["env-filter"] }
-unic-ucd-ident = "0.9.0"
+unicode-ident = "1.0.11"
 unicode-width = "0.1.10"
 uuid = { version = "1.4.1", features = ["v4", "fast-rng", "macro-diagnostics", "js"] }
 wsl = { version = "0.1.0" }

diff --git a/crates/ruff_python_parser/Cargo.toml b/crates/ruff_python_parser/Cargo.toml
@@ -23,8 +23,7 @@ itertools = { workspace = true }
 lalrpop-util = { version = "0.20.0", default-features = false }
 num-bigint = { workspace = true }
 num-traits = { workspace = true }
-unic-emoji-char = "0.9.0"
-unic-ucd-ident = {  workspace = true }
+unicode-ident = { workspace = true }
 unicode_names2 = { version = "0.6.0", git = "https://github.com/youknowone/unicode_names2.git", rev = "4ce16aa85cbcdd9cc830410f1a72ef9a235f2fde" }
 rustc-hash = { workspace = true }
 static_assertions = "1.1.0"

diff --git a/crates/ruff_python_parser/src/lexer.rs b/crates/ruff_python_parser/src/lexer.rs
@@ -36,8 +36,7 @@ use num_bigint::BigInt;
 use num_traits::{Num, Zero};
 use ruff_python_ast::IpyEscapeKind;
 use ruff_text_size::{TextLen, TextRange, TextSize};
-use unic_emoji_char::is_emoji_presentation;
-use unic_ucd_ident::{is_xid_continue, is_xid_start};
+use unicode_ident::{is_xid_continue, is_xid_start};
 
 use crate::lexer::cursor::{Cursor, EOF_CHAR};
 use crate::lexer::indentation::{Indentation, Indentations};
@@ -597,15 +596,6 @@ impl<'source> Lexer<'source> {
                 self.state = State::Other;
 
                 Ok((identifier, self.token_range()))
-            } else if is_emoji_presentation(c) {
-                self.state = State::Other;
-
-                Ok((
-                    Tok::Name {
-                        name: c.to_string(),
-                    },
-                    self.token_range(),
-                ))
             } else {
                 Err(LexicalError {
                     error: LexicalErrorType::UnrecognizedToken { tok: c },

diff --git a/crates/ruff_python_stdlib/Cargo.toml b/crates/ruff_python_stdlib/Cargo.toml
@@ -13,4 +13,4 @@ license = { workspace = true }
 [lib]
 
 [dependencies]
-unic-ucd-ident = { workspace = true }
+unicode-ident = { workspace = true }
diff --git a/crates/ruff_python_stdlib/src/identifiers.rs b/crates/ruff_python_stdlib/src/identifiers.rs
@@ -1,4 +1,4 @@
-use unic_ucd_ident::{is_xid_continue, is_xid_start};
+use unicode_ident::{is_xid_continue, is_xid_start};
 
 use crate::keyword::is_keyword;
 

diff --git a/crates/ruff_python_trivia/Cargo.toml b/crates/ruff_python_trivia/Cargo.toml
@@ -18,7 +18,7 @@ ruff_source_file = { path = "../ruff_source_file" }
 
 memchr = { workspace = true }
 smallvec = { workspace = true }
-unic-ucd-ident = { workspace = true }
+unicode-ident = { workspace = true }
 
 [dev-dependencies]
 insta = { workspace = true }

diff --git a/crates/ruff_python_trivia/src/tokenizer.rs b/crates/ruff_python_trivia/src/tokenizer.rs
@@ -1,5 +1,5 @@
 use memchr::{memchr2, memchr3, memrchr3_iter};
-use unic_ucd_ident::{is_xid_continue, is_xid_start};
+use unicode_ident::{is_xid_continue, is_xid_start};
 
 use ruff_text_size::{Ranged, TextLen, TextRange, TextSize};