Rollup merge of rust-lang#69801 - petrochenkov:nonorm, r=Centril

rustc_parse: Remove `Parser::normalized(_prev)_token`

Perform the "normalization" (renamed to "uninterpolation") on the fly when necessary.

The final part of rust-lang#69579 rust-lang#69384 rust-lang#69376 rust-lang#69211 rust-lang#69034 rust-lang#69006.
r? @Centril

src/librustc_ast/attr/mod.rs

@@ -286,6 +286,10 @@ impl MetaItem {
 }
 
 impl AttrItem {
+    pub fn span(&self) -> Span {
+        self.args.span().map_or(self.path.span, |args_span| self.path.span.to(args_span))
+    }
+
     pub fn meta(&self, span: Span) -> Option<MetaItem> {
         Some(MetaItem {
             path: self.path.clone(),
@@ -437,7 +441,7 @@ impl MetaItem {
         I: Iterator<Item = TokenTree>,
     {
         // FIXME: Share code with `parse_path`.
-        let path = match tokens.next() {
+        let path = match tokens.next().map(TokenTree::uninterpolate) {
             Some(TokenTree::Token(Token { kind: kind @ token::Ident(..), span }))
             | Some(TokenTree::Token(Token { kind: kind @ token::ModSep, span })) => 'arm: {
                 let mut segments = if let token::Ident(name, _) = kind {
@@ -453,7 +457,7 @@ impl MetaItem {
                 };
                 loop {
                     if let Some(TokenTree::Token(Token { kind: token::Ident(name, _), span })) =
-                        tokens.next()
+                        tokens.next().map(TokenTree::uninterpolate)
                     {
                         segments.push(PathSegment::from_ident(Ident::new(name, span)));
                     } else {
@@ -470,7 +474,6 @@ impl MetaItem {
                 Path { span, segments }
             }
             Some(TokenTree::Token(Token { kind: token::Interpolated(nt), .. })) => match *nt {
-                token::Nonterminal::NtIdent(ident, _) => Path::from_ident(ident),
                 token::Nonterminal::NtMeta(ref item) => return item.meta(item.path.span),
                 token::Nonterminal::NtPath(ref path) => path.clone(),
                 _ => return None,

src/librustc_ast/token.rs

@@ -14,8 +14,8 @@ use rustc_macros::HashStable_Generic;
 use rustc_span::symbol::kw;
 use rustc_span::symbol::Symbol;
 use rustc_span::{self, Span, DUMMY_SP};
-use std::fmt;
-use std::mem;
+use std::borrow::Cow;
+use std::{fmt, mem};
 
 #[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)]
 #[derive(HashStable_Generic)]
@@ -225,8 +225,15 @@ pub enum TokenKind {
     /* Literals */
     Literal(Lit),
 
-    /* Name components */
+    /// Identifier token.
+    /// Do not forget about `NtIdent` when you want to match on identifiers.
+    /// It's recommended to use `Token::(ident,uninterpolate,uninterpolated_span)` to
+    /// treat regular and interpolated identifiers in the same way.
     Ident(ast::Name, /* is_raw */ bool),
+    /// Lifetime identifier token.
+    /// Do not forget about `NtLifetime` when you want to match on lifetime identifiers.
+    /// It's recommended to use `Token::(lifetime,uninterpolate,uninterpolated_span)` to
+    /// treat regular and interpolated lifetime identifiers in the same way.
     Lifetime(ast::Name),
 
     Interpolated(Lrc<Nonterminal>),
@@ -328,6 +335,19 @@ impl Token {
         mem::replace(self, Token::dummy())
     }
 
+    /// For interpolated tokens, returns a span of the fragment to which the interpolated
+    /// token refers. For all other tokens this is just a regular span.
+    /// It is particularly important to use this for identifiers and lifetimes
+    /// for which spans affect name resolution and edition checks.
+    /// Note that keywords are also identifiers, so they should use this
+    /// if they keep spans or perform edition checks.
+    pub fn uninterpolated_span(&self) -> Span {
+        match &self.kind {
+            Interpolated(nt) => nt.span(),
+            _ => self.span,
+        }
+    }
+
     pub fn is_op(&self) -> bool {
         match self.kind {
             OpenDelim(..) | CloseDelim(..) | Literal(..) | DocComment(..) | Ident(..)
@@ -345,7 +365,7 @@ impl Token {
 
     /// Returns `true` if the token can appear at the start of an expression.
     pub fn can_begin_expr(&self) -> bool {
-        match self.kind {
+        match self.uninterpolate().kind {
             Ident(name, is_raw)              =>
                 ident_can_begin_expr(name, self.span, is_raw), // value name or keyword
             OpenDelim(..)                     | // tuple, array or block
@@ -363,12 +383,10 @@ impl Token {
             Lifetime(..)                      | // labeled loop
             Pound                             => true, // expression attributes
             Interpolated(ref nt) => match **nt {
-                NtIdent(ident, is_raw) => ident_can_begin_expr(ident.name, ident.span, is_raw),
                 NtLiteral(..) |
                 NtExpr(..)    |
                 NtBlock(..)   |
-                NtPath(..)    |
-                NtLifetime(..) => true,
+                NtPath(..) => true,
                 _ => false,
             },
             _ => false,
@@ -377,7 +395,7 @@ impl Token {
 
     /// Returns `true` if the token can appear at the start of a type.
     pub fn can_begin_type(&self) -> bool {
-        match self.kind {
+        match self.uninterpolate().kind {
             Ident(name, is_raw)        =>
                 ident_can_begin_type(name, self.span, is_raw), // type name or keyword
             OpenDelim(Paren)            | // tuple
@@ -391,8 +409,7 @@ impl Token {
             Lt | BinOp(Shl)             | // associated path
             ModSep                      => true, // global path
             Interpolated(ref nt) => match **nt {
-                NtIdent(ident, is_raw) => ident_can_begin_type(ident.name, ident.span, is_raw),
-                NtTy(..) | NtPath(..) | NtLifetime(..) => true,
+                NtTy(..) | NtPath(..) => true,
                 _ => false,
             },
             _ => false,
@@ -433,38 +450,48 @@ impl Token {
     ///
     /// Keep this in sync with `Lit::from_token`.
     pub fn can_begin_literal_or_bool(&self) -> bool {
-        match self.kind {
+        match self.uninterpolate().kind {
             Literal(..) | BinOp(Minus) => true,
             Ident(name, false) if name.is_bool_lit() => true,
             Interpolated(ref nt) => match &**nt {
-                NtIdent(ident, false) if ident.name.is_bool_lit() => true,
                 NtExpr(e) | NtLiteral(e) => matches!(e.kind, ast::ExprKind::Lit(_)),
                 _ => false,
             },
             _ => false,
         }
     }
 
+    // A convenience function for matching on identifiers during parsing.
+    // Turns interpolated identifier (`$i: ident`) or lifetime (`$l: lifetime`) token
+    // into the regular identifier or lifetime token it refers to,
+    // otherwise returns the original token.
+    pub fn uninterpolate(&self) -> Cow<'_, Token> {
+        match &self.kind {
+            Interpolated(nt) => match **nt {
+                NtIdent(ident, is_raw) => {
+                    Cow::Owned(Token::new(Ident(ident.name, is_raw), ident.span))
+                }
+                NtLifetime(ident) => Cow::Owned(Token::new(Lifetime(ident.name), ident.span)),
+                _ => Cow::Borrowed(self),
+            },
+            _ => Cow::Borrowed(self),
+        }
+    }
+
     /// Returns an identifier if this token is an identifier.
     pub fn ident(&self) -> Option<(ast::Ident, /* is_raw */ bool)> {
-        match self.kind {
-            Ident(name, is_raw) => Some((ast::Ident::new(name, self.span), is_raw)),
-            Interpolated(ref nt) => match **nt {
-                NtIdent(ident, is_raw) => Some((ident, is_raw)),
-                _ => None,
-            },
+        let token = self.uninterpolate();
+        match token.kind {
+            Ident(name, is_raw) => Some((ast::Ident::new(name, token.span), is_raw)),
             _ => None,
         }
     }
 
     /// Returns a lifetime identifier if this token is a lifetime.
     pub fn lifetime(&self) -> Option<ast::Ident> {
-        match self.kind {
-            Lifetime(name) => Some(ast::Ident::new(name, self.span)),
-            Interpolated(ref nt) => match **nt {
-                NtLifetime(ident) => Some(ident),
-                _ => None,
-            },
+        let token = self.uninterpolate();
+        match token.kind {
+            Lifetime(name) => Some(ast::Ident::new(name, token.span)),
             _ => None,
         }
     }
@@ -714,6 +741,24 @@ pub enum Nonterminal {
 #[cfg(target_arch = "x86_64")]
 rustc_data_structures::static_assert_size!(Nonterminal, 40);
 
+impl Nonterminal {
+    fn span(&self) -> Span {
+        match self {
+            NtItem(item) => item.span,
+            NtBlock(block) => block.span,
+            NtStmt(stmt) => stmt.span,
+            NtPat(pat) => pat.span,
+            NtExpr(expr) | NtLiteral(expr) => expr.span,
+            NtTy(ty) => ty.span,
+            NtIdent(ident, _) | NtLifetime(ident) => ident.span,
+            NtMeta(attr_item) => attr_item.span(),
+            NtPath(path) => path.span,
+            NtVis(vis) => vis.span,
+            NtTT(tt) => tt.span(),
+        }
+    }
+}
+
 impl PartialEq for Nonterminal {
     fn eq(&self, rhs: &Self) -> bool {
         match (self, rhs) {

src/librustc_ast/tokenstream.rs

@@ -116,6 +116,13 @@ impl TokenTree {
     pub fn close_tt(span: DelimSpan, delim: DelimToken) -> TokenTree {
         TokenTree::token(token::CloseDelim(delim), span.close)
     }
+
+    pub fn uninterpolate(self) -> TokenTree {
+        match self {
+            TokenTree::Token(token) => TokenTree::Token(token.uninterpolate().into_owned()),
+            tt => tt,
+        }
+    }
 }
 
 impl<CTX> HashStable<CTX> for TokenStream

src/librustc_ast/util/literal.rs

@@ -191,23 +191,16 @@ impl Lit {
     ///
     /// Keep this in sync with `Token::can_begin_literal_or_bool`.
     pub fn from_token(token: &Token) -> Result<Lit, LitError> {
-        let lit = match token.kind {
+        let lit = match token.uninterpolate().kind {
             token::Ident(name, false) if name.is_bool_lit() => {
                 token::Lit::new(token::Bool, name, None)
             }
             token::Literal(lit) => lit,
             token::Interpolated(ref nt) => {
-                match &**nt {
-                    token::NtIdent(ident, false) if ident.name.is_bool_lit() => {
-                        let lit = token::Lit::new(token::Bool, ident.name, None);
-                        return Lit::from_lit_token(lit, ident.span);
+                if let token::NtExpr(expr) | token::NtLiteral(expr) = &**nt {
+                    if let ast::ExprKind::Lit(lit) = &expr.kind {
+                        return Ok(lit.clone());
                     }
-                    token::NtExpr(expr) | token::NtLiteral(expr) => {
-                        if let ast::ExprKind::Lit(lit) = &expr.kind {
-                            return Ok(lit.clone());
-                        }
-                    }
-                    _ => {}
                 }
                 return Err(LitError::NotLiteral);
             }

src/librustc_builtin_macros/format.rs

@@ -156,44 +156,43 @@ fn parse_args<'a>(
         if p.token == token::Eof {
             break;
         } // accept trailing commas
-        if p.token.is_ident() && p.look_ahead(1, |t| *t == token::Eq) {
-            named = true;
-            let name = if let token::Ident(name, _) = p.normalized_token.kind {
+        match p.token.ident() {
+            Some((ident, _)) if p.look_ahead(1, |t| *t == token::Eq) => {
+                named = true;
                 p.bump();
-                name
-            } else {
-                unreachable!();
-            };
+                p.expect(&token::Eq)?;
+                let e = p.parse_expr()?;
+                if let Some(prev) = names.get(&ident.name) {
+                    ecx.struct_span_err(e.span, &format!("duplicate argument named `{}`", ident))
+                        .span_label(args[*prev].span, "previously here")
+                        .span_label(e.span, "duplicate argument")
+                        .emit();
+                    continue;
+                }
 
-            p.expect(&token::Eq)?;
-            let e = p.parse_expr()?;
-            if let Some(prev) = names.get(&name) {
-                ecx.struct_span_err(e.span, &format!("duplicate argument named `{}`", name))
-                    .span_label(args[*prev].span, "previously here")
-                    .span_label(e.span, "duplicate argument")
-                    .emit();
-                continue;
+                // Resolve names into slots early.
+                // Since all the positional args are already seen at this point
+                // if the input is valid, we can simply append to the positional
+                // args. And remember the names.
+                let slot = args.len();
+                names.insert(ident.name, slot);
+                args.push(e);
             }
-
-            // Resolve names into slots early.
-            // Since all the positional args are already seen at this point
-            // if the input is valid, we can simply append to the positional
-            // args. And remember the names.
-            let slot = args.len();
-            names.insert(name, slot);
-            args.push(e);
-        } else {
-            let e = p.parse_expr()?;
-            if named {
-                let mut err = ecx
-                    .struct_span_err(e.span, "positional arguments cannot follow named arguments");
-                err.span_label(e.span, "positional arguments must be before named arguments");
-                for pos in names.values() {
-                    err.span_label(args[*pos].span, "named argument");
+            _ => {
+                let e = p.parse_expr()?;
+                if named {
+                    let mut err = ecx.struct_span_err(
+                        e.span,
+                        "positional arguments cannot follow named arguments",
+                    );
+                    err.span_label(e.span, "positional arguments must be before named arguments");
+                    for pos in names.values() {
+                        err.span_label(args[*pos].span, "named argument");
+                    }
+                    err.emit();
                 }
-                err.emit();
+                args.push(e);
             }
-            args.push(e);
         }
     }
     Ok((fmtstr, args, names))

src/librustc_expand/mbe/macro_parser.rs

@@ -750,17 +750,8 @@ pub(super) fn parse_tt(parser: &mut Cow<'_, Parser<'_>>, ms: &[TokenTree]) -> Na
 
 /// The token is an identifier, but not `_`.
 /// We prohibit passing `_` to macros expecting `ident` for now.
-fn get_macro_name(token: &Token) -> Option<(Name, bool)> {
-    match token.kind {
-        token::Ident(name, is_raw) if name != kw::Underscore => Some((name, is_raw)),
-        token::Interpolated(ref nt) => match **nt {
-            token::NtIdent(ident, is_raw) if ident.name != kw::Underscore => {
-                Some((ident.name, is_raw))
-            }
-            _ => None,
-        },
-        _ => None,
-    }
+fn get_macro_ident(token: &Token) -> Option<(Ident, bool)> {
+    token.ident().filter(|(ident, _)| ident.name != kw::Underscore)
 }
 
 /// Checks whether a non-terminal may begin with a particular token.
@@ -783,7 +774,7 @@ fn may_begin_with(token: &Token, name: Name) -> bool {
             && !token.is_keyword(kw::Let)
         }
         sym::ty => token.can_begin_type(),
-        sym::ident => get_macro_name(token).is_some(),
+        sym::ident => get_macro_ident(token).is_some(),
         sym::literal => token.can_begin_literal_or_bool(),
         sym::vis => match token.kind {
             // The follow-set of :vis + "priv" keyword + interpolated
@@ -888,9 +879,9 @@ fn parse_nt_inner<'a>(p: &mut Parser<'a>, sp: Span, name: Symbol) -> PResult<'a,
         sym::ty => token::NtTy(p.parse_ty()?),
         // this could be handled like a token, since it is one
         sym::ident => {
-            if let Some((name, is_raw)) = get_macro_name(&p.token) {
+            if let Some((ident, is_raw)) = get_macro_ident(&p.token) {
                 p.bump();
-                token::NtIdent(Ident::new(name, p.normalized_prev_token.span), is_raw)
+                token::NtIdent(ident, is_raw)
             } else {
                 let token_str = pprust::token_to_string(&p.token);
                 let msg = &format!("expected ident, found {}", &token_str);

src/librustc_parse/lib.rs

@@ -4,7 +4,7 @@
 #![feature(crate_visibility_modifier)]
 
 use rustc_ast::ast;
-use rustc_ast::token::{self, Nonterminal, Token};
+use rustc_ast::token::{self, Nonterminal};
 use rustc_ast::tokenstream::{self, TokenStream, TokenTree};
 use rustc_ast_pretty::pprust;
 use rustc_data_structures::sync::Lrc;
@@ -171,8 +171,7 @@ fn maybe_source_file_to_parser(
     let mut parser = stream_to_parser(sess, stream, None);
     parser.unclosed_delims = unclosed_delims;
     if parser.token == token::Eof {
-        let span = Span::new(end_pos, end_pos, parser.token.span.ctxt());
-        parser.set_token(Token::new(token::Eof, span));
+        parser.token.span = Span::new(end_pos, end_pos, parser.token.span.ctxt());
     }
 
     Ok(parser)