From 59261f0a7cf264f16e2b6a4e87d4249b212e920e Mon Sep 17 00:00:00 2001
From: Vadim Petrochenkov <vadim.petrochenkov@gmail.com>
Date: Sat, 22 Feb 2020 20:19:49 +0300
Subject: [PATCH 1/2] Add some missing support for `NtIdent`

---
 src/librustc_expand/mbe/macro_parser.rs  |  6 +++
 src/librustc_parse/parser/diagnostics.rs |  7 ++-
 src/libsyntax/token.rs                   | 54 +++++++++++-------------
 src/libsyntax/util/literal.rs            | 13 ++++--
 4 files changed, 44 insertions(+), 36 deletions(-)

diff --git a/src/librustc_expand/mbe/macro_parser.rs b/src/librustc_expand/mbe/macro_parser.rs
index 6599e92222c75..b0bbed6fabfdc 100644
--- a/src/librustc_expand/mbe/macro_parser.rs
+++ b/src/librustc_expand/mbe/macro_parser.rs
@@ -753,6 +753,12 @@ pub(super) fn parse_tt(parser: &mut Cow<'_, Parser<'_>>, ms: &[TokenTree]) -> Na
 fn get_macro_name(token: &Token) -> Option<(Name, bool)> {
     match token.kind {
         token::Ident(name, is_raw) if name != kw::Underscore => Some((name, is_raw)),
+        token::Interpolated(ref nt) => match **nt {
+            token::NtIdent(ident, is_raw) if ident.name != kw::Underscore => {
+                Some((ident.name, is_raw))
+            }
+            _ => None,
+        },
         _ => None,
     }
 }
diff --git a/src/librustc_parse/parser/diagnostics.rs b/src/librustc_parse/parser/diagnostics.rs
index 018aef3c13cee..09b47df2b19ba 100644
--- a/src/librustc_parse/parser/diagnostics.rs
+++ b/src/librustc_parse/parser/diagnostics.rs
@@ -13,7 +13,7 @@ use syntax::ast::{
 };
 use syntax::ast::{AttrVec, ItemKind, Mutability, Pat, PatKind, PathSegment, QSelf, Ty, TyKind};
 use syntax::ptr::P;
-use syntax::token::{self, token_can_begin_expr, TokenKind};
+use syntax::token::{self, TokenKind};
 use syntax::util::parser::AssocOp;
 
 use log::{debug, trace};
@@ -900,8 +900,7 @@ impl<'a> Parser<'a> {
         } else if !sm.is_multiline(self.prev_span.until(self.token.span)) {
             // The current token is in the same line as the prior token, not recoverable.
         } else if self.look_ahead(1, |t| {
-            t == &token::CloseDelim(token::Brace)
-                || token_can_begin_expr(t) && t.kind != token::Colon
+            t == &token::CloseDelim(token::Brace) || t.can_begin_expr() && t.kind != token::Colon
         }) && [token::Comma, token::Colon].contains(&self.token.kind)
         {
             // Likely typo: `,` → `;` or `:` → `;`. This is triggered if the current token is
@@ -919,7 +918,7 @@ impl<'a> Parser<'a> {
         } else if self.look_ahead(0, |t| {
             t == &token::CloseDelim(token::Brace)
                 || (
-                    token_can_begin_expr(t) && t != &token::Semi && t != &token::Pound
+                    t.can_begin_expr() && t != &token::Semi && t != &token::Pound
                     // Avoid triggering with too many trailing `#` in raw string.
                 )
         }) {
diff --git a/src/libsyntax/token.rs b/src/libsyntax/token.rs
index 6eeee49881579..bfb1a0e70f656 100644
--- a/src/libsyntax/token.rs
+++ b/src/libsyntax/token.rs
@@ -147,36 +147,30 @@ impl Lit {
 
 pub fn ident_can_begin_expr(name: ast::Name, span: Span, is_raw: bool) -> bool {
     let ident_token = Token::new(Ident(name, is_raw), span);
-    token_can_begin_expr(&ident_token)
-}
 
-pub fn token_can_begin_expr(ident_token: &Token) -> bool {
     !ident_token.is_reserved_ident()
         || ident_token.is_path_segment_keyword()
-        || match ident_token.kind {
-            TokenKind::Ident(ident, _) => [
-                kw::Async,
-                kw::Do,
-                kw::Box,
-                kw::Break,
-                kw::Continue,
-                kw::False,
-                kw::For,
-                kw::If,
-                kw::Let,
-                kw::Loop,
-                kw::Match,
-                kw::Move,
-                kw::Return,
-                kw::True,
-                kw::Unsafe,
-                kw::While,
-                kw::Yield,
-                kw::Static,
-            ]
-            .contains(&ident),
-            _ => false,
-        }
+        || [
+            kw::Async,
+            kw::Do,
+            kw::Box,
+            kw::Break,
+            kw::Continue,
+            kw::False,
+            kw::For,
+            kw::If,
+            kw::Let,
+            kw::Loop,
+            kw::Match,
+            kw::Move,
+            kw::Return,
+            kw::True,
+            kw::Unsafe,
+            kw::While,
+            kw::Yield,
+            kw::Static,
+        ]
+        .contains(&name)
 }
 
 fn ident_can_begin_type(name: ast::Name, span: Span, is_raw: bool) -> bool {
@@ -369,8 +363,8 @@ impl Token {
             Lifetime(..)                      | // labeled loop
             Pound                             => true, // expression attributes
             Interpolated(ref nt) => match **nt {
+                NtIdent(ident, is_raw) => ident_can_begin_expr(ident.name, ident.span, is_raw),
                 NtLiteral(..) |
-                NtIdent(..)   |
                 NtExpr(..)    |
                 NtBlock(..)   |
                 NtPath(..)    |
@@ -397,7 +391,8 @@ impl Token {
             Lt | BinOp(Shl)             | // associated path
             ModSep                      => true, // global path
             Interpolated(ref nt) => match **nt {
-                NtIdent(..) | NtTy(..) | NtPath(..) | NtLifetime(..) => true,
+                NtIdent(ident, is_raw) => ident_can_begin_type(ident.name, ident.span, is_raw),
+                NtTy(..) | NtPath(..) | NtLifetime(..) => true,
                 _ => false,
             },
             _ => false,
@@ -442,6 +437,7 @@ impl Token {
             Literal(..) | BinOp(Minus) => true,
             Ident(name, false) if name.is_bool_lit() => true,
             Interpolated(ref nt) => match &**nt {
+                NtIdent(ident, false) if ident.name.is_bool_lit() => true,
                 NtExpr(e) | NtLiteral(e) => matches!(e.kind, ast::ExprKind::Lit(_)),
                 _ => false,
             },
diff --git a/src/libsyntax/util/literal.rs b/src/libsyntax/util/literal.rs
index 0c611adc06bcf..ecf17efc4e0d1 100644
--- a/src/libsyntax/util/literal.rs
+++ b/src/libsyntax/util/literal.rs
@@ -197,10 +197,17 @@ impl Lit {
             }
             token::Literal(lit) => lit,
             token::Interpolated(ref nt) => {
-                if let token::NtExpr(expr) | token::NtLiteral(expr) = &**nt {
-                    if let ast::ExprKind::Lit(lit) = &expr.kind {
-                        return Ok(lit.clone());
+                match &**nt {
+                    token::NtIdent(ident, false) if ident.name.is_bool_lit() => {
+                        let lit = token::Lit::new(token::Bool, ident.name, None);
+                        return Lit::from_lit_token(lit, ident.span);
                     }
+                    token::NtExpr(expr) | token::NtLiteral(expr) => {
+                        if let ast::ExprKind::Lit(lit) = &expr.kind {
+                            return Ok(lit.clone());
+                        }
+                    }
+                    _ => {}
                 }
                 return Err(LitError::NotLiteral);
             }

From b2605c118de31db9d61a9f7d0d158b9bf35f7a4a Mon Sep 17 00:00:00 2001
From: Vadim Petrochenkov <vadim.petrochenkov@gmail.com>
Date: Mon, 24 Feb 2020 13:04:13 +0300
Subject: [PATCH 2/2] parser: `token` -> `normalized_token`,
 `nonnormalized_token` -> `token`

---
 src/librustc_builtin_macros/format.rs         |  2 +-
 src/librustc_expand/mbe/macro_parser.rs       |  3 +-
 src/librustc_parse/parser/diagnostics.rs      |  6 +--
 src/librustc_parse/parser/expr.rs             | 39 +++++++--------
 src/librustc_parse/parser/item.rs             | 14 +++---
 src/librustc_parse/parser/mod.rs              | 48 +++++++++----------
 src/librustc_parse/parser/path.rs             |  7 ++-
 src/librustc_parse/parser/ty.rs               |  7 ++-
 .../ui/borrowck/move-error-snippets.stderr    | 20 ++++----
 .../directory_ownership/macro-expanded-mod.rs |  3 +-
 .../macro-expanded-mod.stderr                 |  9 +++-
 src/test/ui/hygiene/fields-definition.stderr  |  4 +-
 src/test/ui/issues/issue-39848.rs             |  3 +-
 src/test/ui/issues/issue-39848.stderr         | 12 +++--
 14 files changed, 88 insertions(+), 89 deletions(-)

diff --git a/src/librustc_builtin_macros/format.rs b/src/librustc_builtin_macros/format.rs
index a9298abe2d759..072c987a5230e 100644
--- a/src/librustc_builtin_macros/format.rs
+++ b/src/librustc_builtin_macros/format.rs
@@ -158,7 +158,7 @@ fn parse_args<'a>(
         } // accept trailing commas
         if p.token.is_ident() && p.look_ahead(1, |t| *t == token::Eq) {
             named = true;
-            let name = if let token::Ident(name, _) = p.token.kind {
+            let name = if let token::Ident(name, _) = p.normalized_token.kind {
                 p.bump();
                 name
             } else {
diff --git a/src/librustc_expand/mbe/macro_parser.rs b/src/librustc_expand/mbe/macro_parser.rs
index b0bbed6fabfdc..2a53d600c5bcf 100644
--- a/src/librustc_expand/mbe/macro_parser.rs
+++ b/src/librustc_expand/mbe/macro_parser.rs
@@ -889,9 +889,8 @@ fn parse_nt_inner<'a>(p: &mut Parser<'a>, sp: Span, name: Symbol) -> PResult<'a,
         // this could be handled like a token, since it is one
         sym::ident => {
             if let Some((name, is_raw)) = get_macro_name(&p.token) {
-                let span = p.token.span;
                 p.bump();
-                token::NtIdent(Ident::new(name, span), is_raw)
+                token::NtIdent(Ident::new(name, p.normalized_prev_token.span), is_raw)
             } else {
                 let token_str = pprust::token_to_string(&p.token);
                 let msg = &format!("expected ident, found {}", &token_str);
diff --git a/src/librustc_parse/parser/diagnostics.rs b/src/librustc_parse/parser/diagnostics.rs
index 09b47df2b19ba..00f5fb9705286 100644
--- a/src/librustc_parse/parser/diagnostics.rs
+++ b/src/librustc_parse/parser/diagnostics.rs
@@ -192,12 +192,12 @@ impl<'a> Parser<'a> {
             TokenKind::CloseDelim(token::DelimToken::Brace),
             TokenKind::CloseDelim(token::DelimToken::Paren),
         ];
-        if let token::Ident(name, false) = self.token.kind {
-            if Ident::new(name, self.token.span).is_raw_guess()
+        if let token::Ident(name, false) = self.normalized_token.kind {
+            if Ident::new(name, self.normalized_token.span).is_raw_guess()
                 && self.look_ahead(1, |t| valid_follow.contains(&t.kind))
             {
                 err.span_suggestion(
-                    self.token.span,
+                    self.normalized_token.span,
                     "you can escape reserved keywords to use them as identifiers",
                     format!("r#{}", name),
                     Applicability::MaybeIncorrect,
diff --git a/src/librustc_parse/parser/expr.rs b/src/librustc_parse/parser/expr.rs
index 3ae97ed5f8822..859e53127b7b0 100644
--- a/src/librustc_parse/parser/expr.rs
+++ b/src/librustc_parse/parser/expr.rs
@@ -97,15 +97,14 @@ impl<'a> Parser<'a> {
     fn parse_expr_catch_underscore(&mut self) -> PResult<'a, P<Expr>> {
         match self.parse_expr() {
             Ok(expr) => Ok(expr),
-            Err(mut err) => match self.token.kind {
+            Err(mut err) => match self.normalized_token.kind {
                 token::Ident(name, false)
                     if name == kw::Underscore && self.look_ahead(1, |t| t == &token::Comma) =>
                 {
                     // Special-case handling of `foo(_, _, _)`
                     err.emit();
-                    let sp = self.token.span;
                     self.bump();
-                    Ok(self.mk_expr(sp, ExprKind::Err, AttrVec::new()))
+                    Ok(self.mk_expr(self.prev_token.span, ExprKind::Err, AttrVec::new()))
                 }
                 _ => Err(err),
             },
@@ -166,7 +165,7 @@ impl<'a> Parser<'a> {
         while let Some(op) = self.check_assoc_op() {
             // Adjust the span for interpolated LHS to point to the `$lhs` token
             // and not to what it refers to.
-            let lhs_span = match self.unnormalized_prev_token.kind {
+            let lhs_span = match self.prev_token.kind {
                 TokenKind::Interpolated(..) => self.prev_span,
                 _ => lhs.span,
             };
@@ -337,7 +336,7 @@ impl<'a> Parser<'a> {
     /// Also performs recovery for `and` / `or` which are mistaken for `&&` and `||` respectively.
     fn check_assoc_op(&self) -> Option<Spanned<AssocOp>> {
         Some(Spanned {
-            node: match (AssocOp::from_token(&self.token), &self.token.kind) {
+            node: match (AssocOp::from_token(&self.token), &self.normalized_token.kind) {
                 (Some(op), _) => op,
                 (None, token::Ident(sym::and, false)) => {
                     self.error_bad_logical_op("and", "&&", "conjunction");
@@ -349,7 +348,7 @@ impl<'a> Parser<'a> {
                 }
                 _ => return None,
             },
-            span: self.token.span,
+            span: self.normalized_token.span,
         })
     }
 
@@ -441,7 +440,7 @@ impl<'a> Parser<'a> {
         let attrs = self.parse_or_use_outer_attributes(attrs)?;
         let lo = self.token.span;
         // Note: when adding new unary operators, don't forget to adjust TokenKind::can_begin_expr()
-        let (hi, ex) = match self.token.kind {
+        let (hi, ex) = match self.normalized_token.kind {
             token::Not => self.parse_unary_expr(lo, UnOp::Not), // `!expr`
             token::Tilde => self.recover_tilde_expr(lo),        // `~expr`
             token::BinOp(token::Minus) => self.parse_unary_expr(lo, UnOp::Neg), // `-expr`
@@ -527,7 +526,7 @@ impl<'a> Parser<'a> {
     ) -> PResult<'a, (Span, P<Expr>)> {
         expr.map(|e| {
             (
-                match self.unnormalized_prev_token.kind {
+                match self.prev_token.kind {
                     TokenKind::Interpolated(..) => self.prev_span,
                     _ => e.span,
                 },
@@ -708,7 +707,7 @@ impl<'a> Parser<'a> {
     }
 
     fn parse_dot_suffix_expr(&mut self, lo: Span, base: P<Expr>) -> PResult<'a, P<Expr>> {
-        match self.token.kind {
+        match self.normalized_token.kind {
             token::Ident(..) => self.parse_dot_suffix(base, lo),
             token::Literal(token::Lit { kind: token::Integer, symbol, suffix }) => {
                 Ok(self.parse_tuple_field_access_expr(lo, base, symbol, suffix))
@@ -777,8 +776,8 @@ impl<'a> Parser<'a> {
         field: Symbol,
         suffix: Option<Symbol>,
     ) -> P<Expr> {
-        let span = self.token.span;
         self.bump();
+        let span = self.prev_token.span;
         let field = ExprKind::Field(base, Ident::new(field, span));
         self.expect_no_suffix(span, "a tuple index", suffix);
         self.mk_expr(lo.to(span), field, AttrVec::new())
@@ -802,7 +801,7 @@ impl<'a> Parser<'a> {
 
     /// Assuming we have just parsed `.`, continue parsing into an expression.
     fn parse_dot_suffix(&mut self, self_arg: P<Expr>, lo: Span) -> PResult<'a, P<Expr>> {
-        if self.token.span.rust_2018() && self.eat_keyword(kw::Await) {
+        if self.normalized_token.span.rust_2018() && self.eat_keyword(kw::Await) {
             return self.mk_await_expr(self_arg, lo);
         }
 
@@ -916,7 +915,7 @@ impl<'a> Parser<'a> {
             //       |             ^ expected expression
             self.bump();
             Ok(self.mk_expr_err(self.token.span))
-        } else if self.token.span.rust_2018() {
+        } else if self.normalized_token.span.rust_2018() {
             // `Span::rust_2018()` is somewhat expensive; don't get it repeatedly.
             if self.check_keyword(kw::Async) {
                 if self.is_async_block() {
@@ -1346,7 +1345,7 @@ impl<'a> Parser<'a> {
             if self.eat_keyword(kw::Static) { Movability::Static } else { Movability::Movable };
 
         let asyncness =
-            if self.token.span.rust_2018() { self.parse_asyncness() } else { Async::No };
+            if self.normalized_token.span.rust_2018() { self.parse_asyncness() } else { Async::No };
         if asyncness.is_async() {
             // Feature-gate `async ||` closures.
             self.sess.gated_spans.gate(sym::async_closure, self.prev_span);
@@ -1560,9 +1559,8 @@ impl<'a> Parser<'a> {
 
     fn eat_label(&mut self) -> Option<Label> {
         self.token.lifetime().map(|ident| {
-            let span = self.token.span;
             self.bump();
-            Label { ident: Ident::new(ident.name, span) }
+            Label { ident }
         })
     }
 
@@ -1704,7 +1702,7 @@ impl<'a> Parser<'a> {
     fn is_try_block(&self) -> bool {
         self.token.is_keyword(kw::Try) &&
         self.look_ahead(1, |t| *t == token::OpenDelim(token::Brace)) &&
-        self.token.span.rust_2018() &&
+        self.normalized_token.span.rust_2018() &&
         // Prevent `while try {} {}`, `if try {} {} else {}`, etc.
         !self.restrictions.contains(Restrictions::NO_STRUCT_LITERAL)
     }
@@ -1854,13 +1852,12 @@ impl<'a> Parser<'a> {
 
     /// Use in case of error after field-looking code: `S { foo: () with a }`.
     fn find_struct_error_after_field_looking_code(&self) -> Option<Field> {
-        if let token::Ident(name, _) = self.token.kind {
+        if let token::Ident(name, _) = self.normalized_token.kind {
             if !self.token.is_reserved_ident() && self.look_ahead(1, |t| *t == token::Colon) {
-                let span = self.token.span;
                 return Some(ast::Field {
-                    ident: Ident::new(name, span),
-                    span,
-                    expr: self.mk_expr_err(span),
+                    ident: Ident::new(name, self.normalized_token.span),
+                    span: self.token.span,
+                    expr: self.mk_expr_err(self.token.span),
                     is_shorthand: false,
                     attrs: AttrVec::new(),
                     id: DUMMY_NODE_ID,
diff --git a/src/librustc_parse/parser/item.rs b/src/librustc_parse/parser/item.rs
index d6da627054196..d62687f8945df 100644
--- a/src/librustc_parse/parser/item.rs
+++ b/src/librustc_parse/parser/item.rs
@@ -747,11 +747,10 @@ impl<'a> Parser<'a> {
     }
 
     fn parse_ident_or_underscore(&mut self) -> PResult<'a, ast::Ident> {
-        match self.token.kind {
+        match self.normalized_token.kind {
             token::Ident(name @ kw::Underscore, false) => {
-                let span = self.token.span;
                 self.bump();
-                Ok(Ident::new(name, span))
+                Ok(Ident::new(name, self.normalized_prev_token.span))
             }
             _ => self.parse_ident(),
         }
@@ -1545,7 +1544,7 @@ impl<'a> Parser<'a> {
 
         let is_name_required = match self.token.kind {
             token::DotDotDot => false,
-            _ => req_name(&self.token),
+            _ => req_name(&self.normalized_token),
         };
         let (pat, ty) = if is_name_required || self.is_named_param() {
             debug!("parse_param_general parse_pat (is_name_required:{})", is_name_required);
@@ -1611,12 +1610,11 @@ impl<'a> Parser<'a> {
     fn parse_self_param(&mut self) -> PResult<'a, Option<Param>> {
         // Extract an identifier *after* having confirmed that the token is one.
         let expect_self_ident = |this: &mut Self| {
-            match this.token.kind {
+            match this.normalized_token.kind {
                 // Preserve hygienic context.
                 token::Ident(name, _) => {
-                    let span = this.token.span;
                     this.bump();
-                    Ident::new(name, span)
+                    Ident::new(name, this.normalized_prev_token.span)
                 }
                 _ => unreachable!(),
             }
@@ -1653,7 +1651,7 @@ impl<'a> Parser<'a> {
         // Only a limited set of initial token sequences is considered `self` parameters; anything
         // else is parsed as a normal function parameter list, so some lookahead is required.
         let eself_lo = self.token.span;
-        let (eself, eself_ident, eself_hi) = match self.token.kind {
+        let (eself, eself_ident, eself_hi) = match self.normalized_token.kind {
             token::BinOp(token::And) => {
                 let eself = if is_isolated_self(self, 1) {
                     // `&self`
diff --git a/src/librustc_parse/parser/mod.rs b/src/librustc_parse/parser/mod.rs
index 75d4b3750f164..cf940233b59f8 100644
--- a/src/librustc_parse/parser/mod.rs
+++ b/src/librustc_parse/parser/mod.rs
@@ -86,23 +86,22 @@ macro_rules! maybe_recover_from_interpolated_ty_qpath {
 #[derive(Clone)]
 pub struct Parser<'a> {
     pub sess: &'a ParseSess,
+    /// The current non-normalized token.
+    pub token: Token,
     /// The current normalized token.
     /// "Normalized" means that some interpolated tokens
     /// (`$i: ident` and `$l: lifetime` meta-variables) are replaced
     /// with non-interpolated identifier and lifetime tokens they refer to.
-    /// Use span from this token if you need an isolated span.
-    pub token: Token,
-    /// The current non-normalized token if it's different from `token`.
-    /// Use span from this token if you need to concatenate it with some neighbouring spans.
-    unnormalized_token: Token,
+    /// Use this if you need to check for `token::Ident` or `token::Lifetime` specifically,
+    /// this also includes edition checks for edition-specific keyword identifiers.
+    pub normalized_token: Token,
+    /// The previous non-normalized token.
+    pub prev_token: Token,
     /// The previous normalized token.
-    /// Use span from this token if you need an isolated span.
-    prev_token: Token,
-    /// The previous non-normalized token if it's different from `prev_token`.
-    /// Use span from this token if you need to concatenate it with some neighbouring spans.
-    unnormalized_prev_token: Token,
-    /// Equivalent to `unnormalized_prev_token.span`.
-    /// FIXME: Remove in favor of `(unnormalized_)prev_token.span`.
+    /// Use this if you need to check for `token::Ident` or `token::Lifetime` specifically,
+    /// this also includes edition checks for edition-specific keyword identifiers.
+    pub normalized_prev_token: Token,
+    /// FIXME: Remove in favor of the equivalent `prev_token.span`.
     pub prev_span: Span,
     restrictions: Restrictions,
     /// Used to determine the path to externally loaded source files.
@@ -376,9 +375,9 @@ impl<'a> Parser<'a> {
         let mut parser = Parser {
             sess,
             token: Token::dummy(),
-            unnormalized_token: Token::dummy(),
+            normalized_token: Token::dummy(),
             prev_token: Token::dummy(),
-            unnormalized_prev_token: Token::dummy(),
+            normalized_prev_token: Token::dummy(),
             prev_span: DUMMY_SP,
             restrictions: Restrictions::empty(),
             recurse_into_file_modules,
@@ -483,7 +482,7 @@ impl<'a> Parser<'a> {
     }
 
     fn parse_ident_common(&mut self, recover: bool) -> PResult<'a, ast::Ident> {
-        match self.token.kind {
+        match self.normalized_token.kind {
             token::Ident(name, _) => {
                 if self.token.is_reserved_ident() {
                     let mut err = self.expected_ident_found();
@@ -493,9 +492,8 @@ impl<'a> Parser<'a> {
                         return Err(err);
                     }
                 }
-                let span = self.token.span;
                 self.bump();
-                Ok(Ident::new(name, span))
+                Ok(Ident::new(name, self.normalized_prev_token.span))
             }
             _ => Err(match self.prev_token.kind {
                 TokenKind::DocComment(..) => {
@@ -825,16 +823,16 @@ impl<'a> Parser<'a> {
     // tokens are replaced with usual identifier and lifetime tokens,
     // so the former are never encountered during normal parsing.
     crate fn set_token(&mut self, token: Token) {
-        self.unnormalized_token = token;
-        self.token = match &self.unnormalized_token.kind {
+        self.token = token;
+        self.normalized_token = match &self.token.kind {
             token::Interpolated(nt) => match **nt {
                 token::NtIdent(ident, is_raw) => {
                     Token::new(token::Ident(ident.name, is_raw), ident.span)
                 }
                 token::NtLifetime(ident) => Token::new(token::Lifetime(ident.name), ident.span),
-                _ => self.unnormalized_token.clone(),
+                _ => self.token.clone(),
             },
-            _ => self.unnormalized_token.clone(),
+            _ => self.token.clone(),
         }
     }
 
@@ -848,11 +846,11 @@ impl<'a> Parser<'a> {
 
         // Update the current and previous tokens.
         self.prev_token = self.token.take();
-        self.unnormalized_prev_token = self.unnormalized_token.take();
+        self.normalized_prev_token = self.normalized_token.take();
         self.set_token(next_token);
 
         // Update fields derived from the previous token.
-        self.prev_span = self.unnormalized_prev_token.span;
+        self.prev_span = self.prev_token.span;
 
         // Diagnostics.
         self.expected_tokens.clear();
@@ -860,7 +858,7 @@ impl<'a> Parser<'a> {
 
     /// Advance the parser by one token.
     pub fn bump(&mut self) {
-        let next_token = self.next_tok(self.unnormalized_token.span);
+        let next_token = self.next_tok(self.token.span);
         self.bump_with(next_token);
     }
 
@@ -891,7 +889,7 @@ impl<'a> Parser<'a> {
     /// Parses asyncness: `async` or nothing.
     fn parse_asyncness(&mut self) -> Async {
         if self.eat_keyword(kw::Async) {
-            let span = self.prev_span;
+            let span = self.normalized_prev_token.span;
             Async::Yes { span, closure_id: DUMMY_NODE_ID, return_impl_trait_id: DUMMY_NODE_ID }
         } else {
             Async::No
diff --git a/src/librustc_parse/parser/path.rs b/src/librustc_parse/parser/path.rs
index 18e57c6a5d49f..f3a61ad441978 100644
--- a/src/librustc_parse/parser/path.rs
+++ b/src/librustc_parse/parser/path.rs
@@ -134,7 +134,7 @@ impl<'a> Parser<'a> {
             path
         });
 
-        let lo = self.unnormalized_token.span;
+        let lo = self.token.span;
         let mut segments = Vec::new();
         let mod_sep_ctxt = self.token.span.ctxt();
         if self.eat(&token::ModSep) {
@@ -238,11 +238,10 @@ impl<'a> Parser<'a> {
     }
 
     pub(super) fn parse_path_segment_ident(&mut self) -> PResult<'a, Ident> {
-        match self.token.kind {
+        match self.normalized_token.kind {
             token::Ident(name, _) if name.is_path_segment_keyword() => {
-                let span = self.token.span;
                 self.bump();
-                Ok(Ident::new(name, span))
+                Ok(Ident::new(name, self.normalized_prev_token.span))
             }
             _ => self.parse_ident(),
         }
diff --git a/src/librustc_parse/parser/ty.rs b/src/librustc_parse/parser/ty.rs
index 29615ac14703c..7b2fdebcfb910 100644
--- a/src/librustc_parse/parser/ty.rs
+++ b/src/librustc_parse/parser/ty.rs
@@ -5,7 +5,7 @@ use crate::{maybe_recover_from_interpolated_ty_qpath, maybe_whole};
 use rustc_errors::{pluralize, struct_span_err, Applicability, PResult};
 use rustc_span::source_map::Span;
 use rustc_span::symbol::{kw, sym};
-use syntax::ast::{self, BareFnTy, FnRetTy, GenericParam, Ident, Lifetime, MutTy, Ty, TyKind};
+use syntax::ast::{self, BareFnTy, FnRetTy, GenericParam, Lifetime, MutTy, Ty, TyKind};
 use syntax::ast::{
     GenericBound, GenericBounds, PolyTraitRef, TraitBoundModifier, TraitObjectSyntax,
 };
@@ -323,7 +323,7 @@ impl<'a> Parser<'a> {
     /// Is a `dyn B0 + ... + Bn` type allowed here?
     fn is_explicit_dyn_type(&mut self) -> bool {
         self.check_keyword(kw::Dyn)
-            && (self.token.span.rust_2018()
+            && (self.normalized_token.span.rust_2018()
                 || self.look_ahead(1, |t| {
                     t.can_begin_bound() && !can_continue_type_after_non_fn_ident(t)
                 }))
@@ -604,9 +604,8 @@ impl<'a> Parser<'a> {
     /// Parses a single lifetime `'a` or panics.
     pub fn expect_lifetime(&mut self) -> Lifetime {
         if let Some(ident) = self.token.lifetime() {
-            let span = self.token.span;
             self.bump();
-            Lifetime { ident: Ident::new(ident.name, span), id: ast::DUMMY_NODE_ID }
+            Lifetime { ident, id: ast::DUMMY_NODE_ID }
         } else {
             self.span_bug(self.token.span, "not a lifetime")
         }
diff --git a/src/test/ui/borrowck/move-error-snippets.stderr b/src/test/ui/borrowck/move-error-snippets.stderr
index e0acd45957145..58a90b2fca2ca 100644
--- a/src/test/ui/borrowck/move-error-snippets.stderr
+++ b/src/test/ui/borrowck/move-error-snippets.stderr
@@ -1,14 +1,16 @@
 error[E0507]: cannot move out of static item `D`
-  --> $DIR/move-error-snippets.rs:16:18
+  --> $DIR/move-error-snippets-ext.rs:5:17
    |
-LL | | #[macro_use]
-   | |__________________^ move occurs because `D` has type `A`, which does not implement the `Copy` trait
-...
-LL |               aaa!(D);
-   |  __________________^
-...
-LL |   sss!();
-   |   ------- in this macro invocation
+LL |         let a = $c;
+   |                 ^^
+   |                 |
+   |                 move occurs because `D` has type `A`, which does not implement the `Copy` trait
+   |                 help: consider borrowing here: `&$c`
+   | 
+  ::: $DIR/move-error-snippets.rs:21:1
+   |
+LL | sss!();
+   | ------- in this macro invocation
    |
    = note: this error originates in a macro (in Nightly builds, run with -Z macro-backtrace for more info)
 
diff --git a/src/test/ui/directory_ownership/macro-expanded-mod.rs b/src/test/ui/directory_ownership/macro-expanded-mod.rs
index 9752a64162ebc..376c1a9cd6627 100644
--- a/src/test/ui/directory_ownership/macro-expanded-mod.rs
+++ b/src/test/ui/directory_ownership/macro-expanded-mod.rs
@@ -1,7 +1,7 @@
 // Test that macro-expanded non-inline modules behave correctly
 
 macro_rules! mod_decl {
-    ($i:ident) => { mod $i; }
+    ($i:ident) => { mod $i; } //~ ERROR Cannot declare a non-inline module inside a block
 }
 
 mod macro_expanded_mod_helper {
@@ -10,5 +10,4 @@ mod macro_expanded_mod_helper {
 
 fn main() {
     mod_decl!(foo);
-    //~^ ERROR Cannot declare a non-inline module inside a block
 }
diff --git a/src/test/ui/directory_ownership/macro-expanded-mod.stderr b/src/test/ui/directory_ownership/macro-expanded-mod.stderr
index 620a00f89bb95..c7780c869d635 100644
--- a/src/test/ui/directory_ownership/macro-expanded-mod.stderr
+++ b/src/test/ui/directory_ownership/macro-expanded-mod.stderr
@@ -1,8 +1,13 @@
 error: Cannot declare a non-inline module inside a block unless it has a path attribute
-  --> $DIR/macro-expanded-mod.rs:12:15
+  --> $DIR/macro-expanded-mod.rs:4:25
    |
+LL |     ($i:ident) => { mod $i; }
+   |                         ^^
+...
 LL |     mod_decl!(foo);
-   |               ^^^
+   |     --------------- in this macro invocation
+   |
+   = note: this error originates in a macro (in Nightly builds, run with -Z macro-backtrace for more info)
 
 error: aborting due to previous error
 
diff --git a/src/test/ui/hygiene/fields-definition.stderr b/src/test/ui/hygiene/fields-definition.stderr
index 8070ffdfdeb89..fd8846bb13c88 100644
--- a/src/test/ui/hygiene/fields-definition.stderr
+++ b/src/test/ui/hygiene/fields-definition.stderr
@@ -1,10 +1,10 @@
 error[E0124]: field `a` is already declared
-  --> $DIR/fields-definition.rs:14:17
+  --> $DIR/fields-definition.rs:14:13
    |
 LL |             a: u8,
    |             ----- `a` first declared here
 LL |             $a: u8,
-   |                 ^^ field already declared
+   |             ^^^^^^ field already declared
 ...
 LL | legacy!(a);
    | ----------- in this macro invocation
diff --git a/src/test/ui/issues/issue-39848.rs b/src/test/ui/issues/issue-39848.rs
index 5d1db7be85733..1964d739989eb 100644
--- a/src/test/ui/issues/issue-39848.rs
+++ b/src/test/ui/issues/issue-39848.rs
@@ -1,10 +1,9 @@
 macro_rules! get_opt {
     ($tgt:expr, $field:ident) => {
-        if $tgt.has_$field() {}
+        if $tgt.has_$field() {} //~ ERROR expected `{`, found `foo`
     }
 }
 
 fn main() {
     get_opt!(bar, foo);
-    //~^ ERROR expected `{`, found `foo`
 }
diff --git a/src/test/ui/issues/issue-39848.stderr b/src/test/ui/issues/issue-39848.stderr
index 11b145d6e0dc9..0250c6b1fdd0f 100644
--- a/src/test/ui/issues/issue-39848.stderr
+++ b/src/test/ui/issues/issue-39848.stderr
@@ -1,13 +1,17 @@
 error: expected `{`, found `foo`
-  --> $DIR/issue-39848.rs:8:19
+  --> $DIR/issue-39848.rs:3:21
    |
 LL |         if $tgt.has_$field() {}
-   |         --                -- help: try placing this code inside a block: `{ () }`
-   |         |
+   |         --          ^^^^^^--
+   |         |           |
+   |         |           expected `{`
+   |         |           help: try placing this code inside a block: `{ $field() }`
    |         this `if` expression has a condition, but no block
 ...
 LL |     get_opt!(bar, foo);
-   |                   ^^^ expected `{`
+   |     ------------------- in this macro invocation
+   |
+   = note: this error originates in a macro (in Nightly builds, run with -Z macro-backtrace for more info)
 
 error: aborting due to previous error