Rollup merge of rust-lang#67702 - crlf0710:normalize_ident2, r=petroc…

…henkov

Add symbol normalization for proc_macro_server.

Follow up for rust-lang#66670, finishing the first bullet point in rust-lang#55467.

r? @petrochenkov

src/librustc_expand/proc_macro_server.rs

@@ -1,5 +1,6 @@
 use crate::base::ExtCtxt;
 
+use rustc_parse::lexer::nfc_normalize;
 use rustc_parse::{nt_to_tokenstream, parse_stream_from_source_str};
 use syntax::ast;
 use syntax::print::pprust;
@@ -327,6 +328,7 @@ impl Ident {
         }
     }
     fn new(sym: Symbol, is_raw: bool, span: Span) -> Ident {
+        let sym = nfc_normalize(&sym.as_str());
         let string = sym.as_str();
         if !Self::is_valid(&string) {
             panic!("`{:?}` is not a valid identifier", string)

src/librustc_parse/lexer/mod.rs

@@ -220,7 +220,7 @@ impl<'a> StringReader<'a> {
                 if is_raw_ident {
                     ident_start = ident_start + BytePos(2);
                 }
-                let sym = self.nfc_symbol_from(ident_start);
+                let sym = nfc_normalize(self.str_from(ident_start));
                 if is_raw_ident {
                     let span = self.mk_sp(start, self.pos);
                     if !sym.can_be_raw() {
@@ -469,20 +469,6 @@ impl<'a> StringReader<'a> {
         Symbol::intern(self.str_from_to(start, end))
     }
 
-    /// As symbol_from, with the text normalized into Unicode NFC form.
-    fn nfc_symbol_from(&self, start: BytePos) -> Symbol {
-        use unicode_normalization::{is_nfc_quick, IsNormalized, UnicodeNormalization};
-        debug!("taking an normalized ident from {:?} to {:?}", start, self.pos);
-        let sym = self.str_from(start);
-        match is_nfc_quick(sym.chars()) {
-            IsNormalized::Yes => Symbol::intern(sym),
-            _ => {
-                let sym_str: String = sym.chars().nfc().collect();
-                Symbol::intern(&sym_str)
-            }
-        }
-    }
-
     /// Slice of the source text spanning from `start` up to but excluding `end`.
     fn str_from_to(&self, start: BytePos, end: BytePos) -> &str {
         &self.src[self.src_index(start)..self.src_index(end)]
@@ -651,3 +637,14 @@ impl<'a> StringReader<'a> {
         }
     }
 }
+
+pub fn nfc_normalize(string: &str) -> Symbol {
+    use unicode_normalization::{is_nfc_quick, IsNormalized, UnicodeNormalization};
+    match is_nfc_quick(string.chars()) {
+        IsNormalized::Yes => Symbol::intern(string),
+        _ => {
+            let normalized_str: String = string.chars().nfc().collect();
+            Symbol::intern(&normalized_str)
+        }
+    }
+}