Auto merge of #66670 - crlf0710:normalize_ident, r=Dylan-DPC

Normalize ident Perform unicode normalization on identifiers. Resolving the first bullet point in #55467.
rust-lang · Dec 26, 2019 · acb6690 · acb6690
2 parents c0b16b4 + 27e7a1b
commit acb6690
Show file tree

Hide file tree

Showing 7 changed files with 35 additions and 6 deletions.
diff --git a/Cargo.lock b/Cargo.lock
@@ -3333,6 +3333,7 @@ dependencies = [
  "serde",
  "serde_json",
  "smallvec 0.6.10",
+ "smallvec 1.0.0",
  "syn 0.15.35",
  "url 2.1.0",
  "winapi 0.3.8",
@@ -3696,6 +3697,7 @@ dependencies = [
  "smallvec 1.0.0",
  "syntax",
  "syntax_pos",
+ "unicode-normalization",
 ]
 
 [[package]]
@@ -4913,9 +4915,12 @@ dependencies = [
 
 [[package]]
 name = "unicode-normalization"
-version = "0.1.7"
+version = "0.1.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6a0180bc61fc5a987082bfa111f4cc95c4caff7f9799f3e46df09163a937aa25"
+checksum = "b561e267b2326bb4cebfc0ef9e68355c7abe6c6f522aeac2f5bf95d56c59bdcf"
+dependencies = [
+ "smallvec 1.0.0",
+]
 
 [[package]]
 name = "unicode-segmentation"

diff --git a/src/librustc_parse/Cargo.toml b/src/librustc_parse/Cargo.toml
@@ -20,3 +20,4 @@ rustc_error_codes = { path = "../librustc_error_codes" }
 smallvec = { version = "1.0", features = ["union", "may_dangle"] }
 syntax_pos = { path = "../libsyntax_pos" }
 syntax = { path = "../libsyntax" }
+unicode-normalization = "0.1.11"
diff --git a/src/librustc_parse/lexer/mod.rs b/src/librustc_parse/lexer/mod.rs
@@ -220,8 +220,7 @@ impl<'a> StringReader<'a> {
                 if is_raw_ident {
                     ident_start = ident_start + BytePos(2);
                 }
-                // FIXME: perform NFKC normalization here. (Issue #2253)
-                let sym = self.symbol_from(ident_start);
+                let sym = self.nfc_symbol_from(ident_start);
                 if is_raw_ident {
                     let span = self.mk_sp(start, self.pos);
                     if !sym.can_be_raw() {
@@ -470,6 +469,20 @@ impl<'a> StringReader<'a> {
         Symbol::intern(self.str_from_to(start, end))
     }
 
+    /// As symbol_from, with the text normalized into Unicode NFC form.
+    fn nfc_symbol_from(&self, start: BytePos) -> Symbol {
+        use unicode_normalization::{is_nfc_quick, IsNormalized, UnicodeNormalization};
+        debug!("taking an normalized ident from {:?} to {:?}", start, self.pos);
+        let sym = self.str_from(start);
+        match is_nfc_quick(sym.chars()) {
+            IsNormalized::Yes => Symbol::intern(sym),
+            _ => {
+                let sym_str: String = sym.chars().nfc().collect();
+                Symbol::intern(&sym_str)
+            }
+        }
+    }
+
     /// Slice of the source text spanning from `start` up to but excluding `end`.
     fn str_from_to(&self, start: BytePos, end: BytePos) -> &str {
         &self.src[self.src_index(start)..self.src_index(end)]

diff --git a/src/test/ui/codemap_tests/unicode_2.stderr b/src/test/ui/codemap_tests/unicode_2.stderr
@@ -14,7 +14,7 @@ LL |     let _ = ("아あ", 1i42);
    |
    = help: valid widths are 8, 16, 32, 64 and 128
 
-error[E0425]: cannot find value `a̐é` in this scope
+error[E0425]: cannot find value `a̐é` in this scope
   --> $DIR/unicode_2.rs:6:13
    |
 LL |     let _ = a̐é;

diff --git a/src/test/ui/rfc-2457/idents-normalized.rs b/src/test/ui/rfc-2457/idents-normalized.rs
@@ -0,0 +1,8 @@
+// check-pass
+#![feature(non_ascii_idents)]
+
+struct Résumé; // ['LATIN SMALL LETTER E WITH ACUTE']
+
+fn main() {
+    let _ = Résumé; // ['LATIN SMALL LETTER E', 'COMBINING ACUTE ACCENT']
+}
diff --git a/src/tools/rustc-workspace-hack/Cargo.toml b/src/tools/rustc-workspace-hack/Cargo.toml
@@ -62,7 +62,8 @@ curl-sys = { version = "0.4.13", features = ["http2", "libnghttp2-sys"], optiona
 crossbeam-utils = { version = "0.6.5", features = ["nightly"] }
 serde = { version = "1.0.82", features = ['derive'] }
 serde_json = { version = "1.0.31", features = ["raw_value"] }
-smallvec = { version = "0.6", features = ['union', 'may_dangle'] }
+smallvec-0_6 = { package = "smallvec", version = "0.6", features = ['union', 'may_dangle'] }
+smallvec = { version = "1.0", features = ['union', 'may_dangle'] }
 url = { version = "2.0", features = ['serde'] }
 syn = { version = "0.15", features = ['full'] }
 

diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs
@@ -170,6 +170,7 @@ const WHITELIST: &[Crate<'_>] = &[
     Crate("term_size"),
     Crate("thread_local"),
     Crate("ucd-util"),
+    Crate("unicode-normalization"),
     Crate("unicode-width"),
     Crate("unicode-xid"),
     Crate("unreachable"),