From 115f10b99418f0b55ea0c7cfaef3988210231d75 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Mon, 2 May 2022 09:51:09 +1000 Subject: [PATCH] Remove `NtIdent` and `NtLifetime`. There's no need to use Interpolated for these because they both consist of a single token, and so there's no operator precedence problems with inserting them directly into the token stream. --- compiler/rustc_ast/src/ast_like.rs | 9 +-- compiler/rustc_ast/src/attr/mod.rs | 5 +- compiler/rustc_ast/src/mut_visit.rs | 2 - compiler/rustc_ast/src/token.rs | 53 +++------------ compiler/rustc_ast/src/tokenstream.rs | 7 -- compiler/rustc_ast/src/util/literal.rs | 2 +- compiler/rustc_ast_pretty/src/pprust/state.rs | 2 - compiler/rustc_builtin_macros/src/asm.rs | 2 +- compiler/rustc_expand/src/mbe/macro_parser.rs | 4 ++ compiler/rustc_expand/src/mbe/transcribe.rs | 44 +++++++------ .../rustc_expand/src/proc_macro_server.rs | 3 - compiler/rustc_parse/src/lib.rs | 6 -- .../rustc_parse/src/parser/attr_wrapper.rs | 4 +- compiler/rustc_parse/src/parser/expr.rs | 6 +- compiler/rustc_parse/src/parser/item.rs | 2 +- compiler/rustc_parse/src/parser/mod.rs | 2 + .../rustc_parse/src/parser/nonterminal.rs | 65 +++++++++---------- compiler/rustc_parse/src/parser/pat.rs | 2 +- compiler/rustc_resolve/src/late.rs | 26 ++++++-- .../rustc_resolve/src/late/diagnostics.rs | 4 ++ compiler/rustc_session/src/utils.rs | 3 - 21 files changed, 109 insertions(+), 144 deletions(-) diff --git a/compiler/rustc_ast/src/ast_like.rs b/compiler/rustc_ast/src/ast_like.rs index 1a271b0adef9..eae10156d165 100644 --- a/compiler/rustc_ast/src/ast_like.rs +++ b/compiler/rustc_ast/src/ast_like.rs @@ -51,9 +51,7 @@ impl AstLike for crate::token::Nonterminal { | Nonterminal::NtMeta(_) | Nonterminal::NtPath(_) | Nonterminal::NtVis(_) - | Nonterminal::NtBlock(_) - | Nonterminal::NtIdent(..) - | Nonterminal::NtLifetime(_) => &[], + | Nonterminal::NtBlock(_) => &[], } } fn visit_attrs(&mut self, f: impl FnOnce(&mut Vec)) { @@ -66,9 +64,7 @@ impl AstLike for crate::token::Nonterminal { | Nonterminal::NtMeta(_) | Nonterminal::NtPath(_) | Nonterminal::NtVis(_) - | Nonterminal::NtBlock(_) - | Nonterminal::NtIdent(..) - | Nonterminal::NtLifetime(_) => {} + | Nonterminal::NtBlock(_) => {} } } fn tokens_mut(&mut self) -> Option<&mut Option> { @@ -82,7 +78,6 @@ impl AstLike for crate::token::Nonterminal { Nonterminal::NtPath(path) => path.tokens_mut(), Nonterminal::NtVis(vis) => vis.tokens_mut(), Nonterminal::NtBlock(block) => block.tokens_mut(), - Nonterminal::NtIdent(..) | Nonterminal::NtLifetime(..) => None, } } } diff --git a/compiler/rustc_ast/src/attr/mod.rs b/compiler/rustc_ast/src/attr/mod.rs index b14367aa1c2c..7d27d8714f48 100644 --- a/compiler/rustc_ast/src/attr/mod.rs +++ b/compiler/rustc_ast/src/attr/mod.rs @@ -408,7 +408,7 @@ impl MetaItem { I: Iterator, { // FIXME: Share code with `parse_path`. - let path = match tokens.next().map(TokenTree::uninterpolate) { + let path = match tokens.next() { Some(TokenTree::Token(Token { kind: kind @ (token::Ident(..) | token::ModSep), span, @@ -426,7 +426,7 @@ impl MetaItem { }; loop { if let Some(TokenTree::Token(Token { kind: token::Ident(name, _), span })) = - tokens.next().map(TokenTree::uninterpolate) + tokens.next() { segments.push(PathSegment::from_ident(Ident::new(name, span))); } else { @@ -449,6 +449,7 @@ impl MetaItem { }, _ => return None, }; + //eprintln!("A1 {:?}", path); let list_closing_paren_pos = tokens.peek().map(|tt| tt.span().hi()); let kind = MetaItemKind::from_tokens(tokens)?; let hi = match kind { diff --git a/compiler/rustc_ast/src/mut_visit.rs b/compiler/rustc_ast/src/mut_visit.rs index cba49835f69c..320de5229e5f 100644 --- a/compiler/rustc_ast/src/mut_visit.rs +++ b/compiler/rustc_ast/src/mut_visit.rs @@ -787,8 +787,6 @@ pub fn visit_interpolated(nt: &mut token::Nonterminal, vis: &mut token::NtPat(pat) => vis.visit_pat(pat), token::NtExpr(expr) => vis.visit_expr(expr), token::NtTy(ty) => vis.visit_ty(ty), - token::NtIdent(ident, _is_raw) => vis.visit_ident(ident), - token::NtLifetime(ident) => vis.visit_ident(ident), token::NtLiteral(expr) => vis.visit_expr(expr), token::NtMeta(item) => { let AttrItem { path, args, tokens } = item.deref_mut(); diff --git a/compiler/rustc_ast/src/token.rs b/compiler/rustc_ast/src/token.rs index 1589a882f089..18e6b8eb28f0 100644 --- a/compiler/rustc_ast/src/token.rs +++ b/compiler/rustc_ast/src/token.rs @@ -12,7 +12,6 @@ use rustc_macros::HashStable_Generic; use rustc_span::symbol::{kw, sym}; use rustc_span::symbol::{Ident, Symbol}; use rustc_span::{self, edition::Edition, Span, DUMMY_SP}; -use std::borrow::Cow; use std::{fmt, mem}; #[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)] @@ -227,14 +226,8 @@ pub enum TokenKind { Literal(Lit), /// Identifier token. - /// Do not forget about `NtIdent` when you want to match on identifiers. - /// It's recommended to use `Token::(ident,uninterpolate,uninterpolated_span)` to - /// treat regular and interpolated identifiers in the same way. Ident(Symbol, /* is_raw */ bool), /// Lifetime identifier token. - /// Do not forget about `NtLifetime` when you want to match on lifetime identifiers. - /// It's recommended to use `Token::(lifetime,uninterpolate,uninterpolated_span)` to - /// treat regular and interpolated lifetime identifiers in the same way. Lifetime(Symbol), Interpolated(Lrc), @@ -364,7 +357,7 @@ impl Token { /// Returns `true` if the token can appear at the start of an expression. pub fn can_begin_expr(&self) -> bool { - match self.uninterpolate().kind { + match self.kind { Ident(name, is_raw) => ident_can_begin_expr(name, self.span, is_raw), // value name or keyword OpenDelim(..) | // tuple, array or block @@ -391,7 +384,7 @@ impl Token { /// Returns `true` if the token can appear at the start of a type. pub fn can_begin_type(&self) -> bool { - match self.uninterpolate().kind { + match self.kind { Ident(name, is_raw) => ident_can_begin_type(name, self.span, is_raw), // type name or keyword OpenDelim(Delimiter::Parenthesis) | // tuple @@ -439,7 +432,7 @@ impl Token { /// /// Keep this in sync with and `Lit::from_token`, excluding unary negation. pub fn can_begin_literal_maybe_minus(&self) -> bool { - match self.uninterpolate().kind { + match self.kind { Literal(..) | BinOp(Minus) => true, Ident(name, false) if name.is_bool_lit() => true, Interpolated(ref nt) => match &**nt { @@ -457,37 +450,18 @@ impl Token { } } - // A convenience function for matching on identifiers during parsing. - // Turns interpolated identifier (`$i: ident`) or lifetime (`$l: lifetime`) token - // into the regular identifier or lifetime token it refers to, - // otherwise returns the original token. - pub fn uninterpolate(&self) -> Cow<'_, Token> { - match &self.kind { - Interpolated(nt) => match **nt { - NtIdent(ident, is_raw) => { - Cow::Owned(Token::new(Ident(ident.name, is_raw), ident.span)) - } - NtLifetime(ident) => Cow::Owned(Token::new(Lifetime(ident.name), ident.span)), - _ => Cow::Borrowed(self), - }, - _ => Cow::Borrowed(self), - } - } - /// Returns an identifier if this token is an identifier. pub fn ident(&self) -> Option<(Ident, /* is_raw */ bool)> { - let token = self.uninterpolate(); - match token.kind { - Ident(name, is_raw) => Some((Ident::new(name, token.span), is_raw)), + match self.kind { + Ident(name, is_raw) => Some((Ident::new(name, self.span), is_raw)), _ => None, } } /// Returns a lifetime identifier if this token is a lifetime. pub fn lifetime(&self) -> Option { - let token = self.uninterpolate(); - match token.kind { - Lifetime(name) => Some(Ident::new(name, token.span)), + match self.kind { + Lifetime(name) => Some(Ident::new(name, self.span)), _ => None, } } @@ -521,7 +495,7 @@ impl Token { /// (which happens while parsing the result of macro expansion)? pub fn is_whole_expr(&self) -> bool { if let Interpolated(ref nt) = self.kind - && let NtExpr(_) | NtLiteral(_) | NtPath(_) | NtIdent(..) | NtBlock(_) = **nt + && let NtExpr(_) | NtLiteral(_) | NtPath(_) | NtBlock(_) = **nt { return true; } @@ -679,8 +653,6 @@ pub enum Nonterminal { NtPat(P), NtExpr(P), NtTy(P), - NtIdent(Ident, /* is_raw */ bool), - NtLifetime(Ident), NtLiteral(P), /// Stuff inside brackets for attributes NtMeta(P), @@ -779,7 +751,6 @@ impl Nonterminal { NtPat(pat) => pat.span, NtExpr(expr) | NtLiteral(expr) => expr.span, NtTy(ty) => ty.span, - NtIdent(ident, _) | NtLifetime(ident) => ident.span, NtMeta(attr_item) => attr_item.span(), NtPath(path) => path.span, NtVis(vis) => vis.span, @@ -790,11 +761,7 @@ impl Nonterminal { impl PartialEq for Nonterminal { fn eq(&self, rhs: &Self) -> bool { match (self, rhs) { - (NtIdent(ident_lhs, is_raw_lhs), NtIdent(ident_rhs, is_raw_rhs)) => { - ident_lhs == ident_rhs && is_raw_lhs == is_raw_rhs - } - (NtLifetime(ident_lhs), NtLifetime(ident_rhs)) => ident_lhs == ident_rhs, - // FIXME: Assume that all "complex" nonterminal are not equal, we can't compare them + // FIXME: Assume that all nonterminals are not equal, because we can't compare them // correctly based on data from AST. This will prevent them from matching each other // in macros. The comparison will become possible only when each nonterminal has an // attached token stream from which it was parsed. @@ -812,12 +779,10 @@ impl fmt::Debug for Nonterminal { NtPat(..) => f.pad("NtPat(..)"), NtExpr(..) => f.pad("NtExpr(..)"), NtTy(..) => f.pad("NtTy(..)"), - NtIdent(..) => f.pad("NtIdent(..)"), NtLiteral(..) => f.pad("NtLiteral(..)"), NtMeta(..) => f.pad("NtMeta(..)"), NtPath(..) => f.pad("NtPath(..)"), NtVis(..) => f.pad("NtVis(..)"), - NtLifetime(..) => f.pad("NtLifetime(..)"), } } } diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs index a8f29f334070..02eaeda89995 100644 --- a/compiler/rustc_ast/src/tokenstream.rs +++ b/compiler/rustc_ast/src/tokenstream.rs @@ -93,13 +93,6 @@ impl TokenTree { pub fn token(kind: TokenKind, span: Span) -> TokenTree { TokenTree::Token(Token::new(kind, span)) } - - pub fn uninterpolate(self) -> TokenTree { - match self { - TokenTree::Token(token) => TokenTree::Token(token.uninterpolate().into_owned()), - tt => tt, - } - } } impl HashStable for TokenStream diff --git a/compiler/rustc_ast/src/util/literal.rs b/compiler/rustc_ast/src/util/literal.rs index 9c18f55c03b4..9c30c57aedec 100644 --- a/compiler/rustc_ast/src/util/literal.rs +++ b/compiler/rustc_ast/src/util/literal.rs @@ -216,7 +216,7 @@ impl Lit { /// /// Keep this in sync with `Token::can_begin_literal_or_bool` excluding unary negation. pub fn from_token(token: &Token) -> Result { - let lit = match token.uninterpolate().kind { + let lit = match token.kind { token::Ident(name, false) if name.is_bool_lit() => { token::Lit::new(token::Bool, name, None) } diff --git a/compiler/rustc_ast_pretty/src/pprust/state.rs b/compiler/rustc_ast_pretty/src/pprust/state.rs index a2ebe3048cee..dc66807f6c52 100644 --- a/compiler/rustc_ast_pretty/src/pprust/state.rs +++ b/compiler/rustc_ast_pretty/src/pprust/state.rs @@ -713,8 +713,6 @@ pub trait PrintState<'a>: std::ops::Deref + std::ops::Dere token::NtBlock(ref e) => self.block_to_string(e), token::NtStmt(ref e) => self.stmt_to_string(e), token::NtPat(ref e) => self.pat_to_string(e), - token::NtIdent(e, is_raw) => IdentPrinter::for_ast_ident(e, is_raw).to_string(), - token::NtLifetime(e) => e.to_string(), token::NtLiteral(ref e) => self.expr_to_string(e), token::NtVis(ref e) => self.vis_to_string(e), } diff --git a/compiler/rustc_builtin_macros/src/asm.rs b/compiler/rustc_builtin_macros/src/asm.rs index e9e3307ca95d..cc701a85863b 100644 --- a/compiler/rustc_builtin_macros/src/asm.rs +++ b/compiler/rustc_builtin_macros/src/asm.rs @@ -502,7 +502,7 @@ fn parse_reg<'a>( explicit_reg: &mut bool, ) -> PResult<'a, ast::InlineAsmRegOrRegClass> { p.expect(&token::OpenDelim(Delimiter::Parenthesis))?; - let result = match p.token.uninterpolate().kind { + let result = match p.token.kind { token::Ident(name, false) => ast::InlineAsmRegOrRegClass::RegClass(name), token::Literal(token::Lit { kind: token::LitKind::Str, symbol, suffix: _ }) => { *explicit_reg = true; diff --git a/compiler/rustc_expand/src/mbe/macro_parser.rs b/compiler/rustc_expand/src/mbe/macro_parser.rs index 8f260e1cdb5c..e33cb30bfd85 100644 --- a/compiler/rustc_expand/src/mbe/macro_parser.rs +++ b/compiler/rustc_expand/src/mbe/macro_parser.rs @@ -344,6 +344,9 @@ crate enum NamedMatch { // A metavar match of type `tt`. MatchedTokenTree(rustc_ast::tokenstream::TokenTree), + // njn: comment + MatchedToken(token::Token), + // A metavar match of any type other than `tt`. MatchedNonterminal(Lrc), } @@ -624,6 +627,7 @@ impl TtParser { let m = match nt { NtOrTt::Nt(nt) => MatchedNonterminal(Lrc::new(nt)), NtOrTt::Tt(tt) => MatchedTokenTree(tt), + NtOrTt::Token(token) => MatchedToken(token), }; mp.push_match(next_metavar, seq_depth, m); mp.idx += 1; diff --git a/compiler/rustc_expand/src/mbe/transcribe.rs b/compiler/rustc_expand/src/mbe/transcribe.rs index 94b6c3153ca3..93044f0cea3f 100644 --- a/compiler/rustc_expand/src/mbe/transcribe.rs +++ b/compiler/rustc_expand/src/mbe/transcribe.rs @@ -1,5 +1,5 @@ use crate::base::ExtCtxt; -use crate::mbe::macro_parser::{MatchedNonterminal, MatchedSeq, MatchedTokenTree, NamedMatch}; +use crate::mbe::macro_parser::{NamedMatch, NamedMatch::*}; use crate::mbe::{self, MetaVarExpr}; use rustc_ast::mut_visit::{self, MutVisitor}; use rustc_ast::token::{self, Delimiter, Token, TokenKind}; @@ -226,16 +226,22 @@ pub(super) fn transcribe<'a>( MatchedTokenTree(ref tt) => { // `tt`s are emitted into the output stream directly as "raw tokens", // without wrapping them into groups. - let token = tt.clone(); - result.push(token.into()); + let tt = tt.clone(); + result.push(tt.into()); + } + MatchedToken(ref token) => { + // njn: this doesn't quite work, but adding + // visit_token makes things worse + let tt = TokenTree::Token(token.clone()); + result.push(tt.into()); } MatchedNonterminal(ref nt) => { // Other variables are emitted into the output stream as groups with // `Delimiter::Invisible` to maintain parsing priorities. // `Interpolated` is currently used for such groups in rustc parser. marker.visit_span(&mut sp); - let token = TokenTree::token(token::Interpolated(nt.clone()), sp); - result.push(token.into()); + let tt = TokenTree::token(token::Interpolated(nt.clone()), sp); + result.push(tt.into()); } MatchedSeq(..) => { // We were unable to descend far enough. This is an error. @@ -306,8 +312,8 @@ fn lookup_cur_matched<'a>( let mut matched = matched; for &(idx, _) in repeats { match matched { - MatchedTokenTree(_) | MatchedNonterminal(_) => break, MatchedSeq(ref ads) => matched = ads.get(idx).unwrap(), + _ => break, } } @@ -396,8 +402,8 @@ fn lockstep_iter_size( let name = MacroRulesNormalizedIdent::new(name); match lookup_cur_matched(name, interpolations, repeats) { Some(matched) => match matched { - MatchedTokenTree(_) | MatchedNonterminal(_) => LockstepIterSize::Unconstrained, MatchedSeq(ref ads) => LockstepIterSize::Constraint(ads.len(), name), + _ => LockstepIterSize::Unconstrained, }, _ => LockstepIterSize::Unconstrained, } @@ -443,18 +449,6 @@ fn count_repetitions<'a>( sp: &DelimSpan, ) -> PResult<'a, usize> { match matched { - MatchedTokenTree(_) | MatchedNonterminal(_) => { - if declared_lhs_depth == 0 { - return Err(cx.struct_span_err( - sp.entire(), - "`count` can not be placed inside the inner-most repetition", - )); - } - match depth_opt { - None => Ok(1), - Some(_) => Err(out_of_bounds_err(cx, declared_lhs_depth, sp.entire(), "count")), - } - } MatchedSeq(ref named_matches) => { let new_declared_lhs_depth = declared_lhs_depth + 1; match depth_opt { @@ -469,6 +463,18 @@ fn count_repetitions<'a>( .sum(), } } + _ => { + if declared_lhs_depth == 0 { + return Err(cx.struct_span_err( + sp.entire(), + "`count` can not be placed inside the inner-most repetition", + )); + } + match depth_opt { + None => Ok(1), + Some(_) => Err(out_of_bounds_err(cx, declared_lhs_depth, sp.entire(), "count")), + } + } } } // `repeats` records all of the nested levels at which we are currently diff --git a/compiler/rustc_expand/src/proc_macro_server.rs b/compiler/rustc_expand/src/proc_macro_server.rs index b7230cec3e4a..c89b2f468b23 100644 --- a/compiler/rustc_expand/src/proc_macro_server.rs +++ b/compiler/rustc_expand/src/proc_macro_server.rs @@ -175,9 +175,6 @@ impl FromInternal<(TreeAndSpacing, &'_ mut Vec, &mut Rustc<'_, '_>)> tt!(Punct::new('#', false)) } - Interpolated(nt) if let NtIdent(ident, is_raw) = *nt => { - TokenTree::Ident(Ident::new(rustc.sess(), ident.name, is_raw, ident.span)) - } Interpolated(nt) => { let stream = nt_to_tokenstream(&nt, rustc.sess(), CanSynthesizeMissingTokens::No); TokenTree::Group(Group { diff --git a/compiler/rustc_parse/src/lib.rs b/compiler/rustc_parse/src/lib.rs index 28c2a63db27f..45bc76fe2bef 100644 --- a/compiler/rustc_parse/src/lib.rs +++ b/compiler/rustc_parse/src/lib.rs @@ -281,12 +281,6 @@ pub fn nt_to_tokenstream( Nonterminal::NtStmt(ref stmt) => prepend_attrs(&stmt.attrs(), stmt.tokens()), Nonterminal::NtPat(ref pat) => convert_tokens(pat.tokens.as_ref()), Nonterminal::NtTy(ref ty) => convert_tokens(ty.tokens.as_ref()), - Nonterminal::NtIdent(ident, is_raw) => { - Some(tokenstream::TokenTree::token(token::Ident(ident.name, is_raw), ident.span).into()) - } - Nonterminal::NtLifetime(ident) => { - Some(tokenstream::TokenTree::token(token::Lifetime(ident.name), ident.span).into()) - } Nonterminal::NtMeta(ref attr) => convert_tokens(attr.tokens.as_ref()), Nonterminal::NtPath(ref path) => convert_tokens(path.tokens.as_ref()), Nonterminal::NtVis(ref vis) => convert_tokens(vis.tokens.as_ref()), diff --git a/compiler/rustc_parse/src/parser/attr_wrapper.rs b/compiler/rustc_parse/src/parser/attr_wrapper.rs index a12621564ab7..cc17fa00e167 100644 --- a/compiler/rustc_parse/src/parser/attr_wrapper.rs +++ b/compiler/rustc_parse/src/parser/attr_wrapper.rs @@ -229,8 +229,8 @@ impl<'a> Parser<'a> { let (mut ret, trailing) = ret?; // When we're not in `capture-cfg` mode, then bail out early if: - // 1. Our target doesn't support tokens at all (e.g we're parsing an `NtIdent`) - // so there's nothing for us to do. + // 1. Our target doesn't support tokens at all so there's nothing for + // us to do. // 2. Our target already has tokens set (e.g. we've parsed something // like `#[my_attr] $item`. The actual parsing code takes care of prepending // any attributes to the nonterminal, so we don't need to modify the diff --git a/compiler/rustc_parse/src/parser/expr.rs b/compiler/rustc_parse/src/parser/expr.rs index bdbc25af7fa1..c4362464c223 100644 --- a/compiler/rustc_parse/src/parser/expr.rs +++ b/compiler/rustc_parse/src/parser/expr.rs @@ -558,7 +558,7 @@ impl<'a> Parser<'a> { let this = self; // Note: when adding new unary operators, don't forget to adjust TokenKind::can_begin_expr() - match this.token.uninterpolate().kind { + match this.token.kind { token::Not => make_it!(this, attrs, |this, _| this.parse_unary_expr(lo, UnOp::Not)), // `!expr` token::Tilde => make_it!(this, attrs, |this, _| this.recover_tilde_expr(lo)), // `~expr` token::BinOp(token::Minus) => { @@ -647,7 +647,7 @@ impl<'a> Parser<'a> { } fn is_mistaken_not_ident_negation(&self) -> bool { - let token_cannot_continue_expr = |t: &Token| match t.uninterpolate().kind { + let token_cannot_continue_expr = |t: &Token| match t.kind { // These tokens can start an expression after `!`, but // can't continue an expression after an ident token::Ident(name, is_raw) => token::ident_can_begin_expr(name, t.span, is_raw), @@ -1005,7 +1005,7 @@ impl<'a> Parser<'a> { } fn parse_dot_suffix_expr(&mut self, lo: Span, base: P) -> PResult<'a, P> { - match self.token.uninterpolate().kind { + match self.token.kind { token::Ident(..) => self.parse_dot_suffix(base, lo), token::Literal(token::Lit { kind: token::Integer, symbol, suffix }) => { Ok(self.parse_tuple_field_access_expr(lo, base, symbol, suffix, None)) diff --git a/compiler/rustc_parse/src/parser/item.rs b/compiler/rustc_parse/src/parser/item.rs index 746861933d2f..bc074c1daa91 100644 --- a/compiler/rustc_parse/src/parser/item.rs +++ b/compiler/rustc_parse/src/parser/item.rs @@ -2303,7 +2303,7 @@ impl<'a> Parser<'a> { // Only a limited set of initial token sequences is considered `self` parameters; anything // else is parsed as a normal function parameter list, so some lookahead is required. let eself_lo = self.token.span; - let (eself, eself_ident, eself_hi) = match self.token.uninterpolate().kind { + let (eself, eself_ident, eself_hi) = match self.token.kind { token::BinOp(token::And) => { let eself = if is_isolated_self(self, 1) { // `&self` diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs index 96cca68257e7..1a79329f55fa 100644 --- a/compiler/rustc_parse/src/parser/mod.rs +++ b/compiler/rustc_parse/src/parser/mod.rs @@ -1482,8 +1482,10 @@ pub enum FlatToken { Empty, } +// njn: rename? simplify? remove? just a subset of NamedMatch... #[derive(Debug)] pub enum NtOrTt { Nt(Nonterminal), Tt(TokenTree), + Token(Token), } diff --git a/compiler/rustc_parse/src/parser/nonterminal.rs b/compiler/rustc_parse/src/parser/nonterminal.rs index 6974f318f949..bdce3c68b5d8 100644 --- a/compiler/rustc_parse/src/parser/nonterminal.rs +++ b/compiler/rustc_parse/src/parser/nonterminal.rs @@ -19,9 +19,7 @@ impl<'a> Parser<'a> { /// Checks whether the non-terminal may contain a single (non-keyword) identifier. fn may_be_ident(nt: &token::Nonterminal) -> bool { match *nt { - token::NtItem(_) | token::NtBlock(_) | token::NtVis(_) | token::NtLifetime(_) => { - false - } + token::NtBlock(_) | token::NtVis(_) => false, _ => true, } } @@ -49,7 +47,6 @@ impl<'a> Parser<'a> { token::NtItem(_) | token::NtPat(_) | token::NtTy(_) - | token::NtIdent(..) | token::NtMeta(_) | token::NtPath(_) | token::NtVis(_) @@ -86,10 +83,7 @@ impl<'a> Parser<'a> { } NonterminalKind::Lifetime => match token.kind { token::Lifetime(_) => true, - token::Interpolated(ref nt) => { - matches!(**nt, token::NtLifetime(_)) - } - _ => false, + _ => false, // njn: is this reachable? }, NonterminalKind::TT | NonterminalKind::Item | NonterminalKind::Stmt => { !matches!(token.kind, token::CloseDelim(_)) @@ -108,8 +102,29 @@ impl<'a> Parser<'a> { // in advance whether or not a proc-macro will be (transitively) invoked, // we always capture tokens for any `Nonterminal` which needs them. let mut nt = match kind { - // Note that TT is treated differently to all the others. NonterminalKind::TT => return Ok(NtOrTt::Tt(self.parse_token_tree())), + NonterminalKind::Ident => { + return if let Some((ident, is_raw)) = get_macro_ident(&self.token) { + self.bump(); + let token = Token::new(token::Ident(ident.name, is_raw), ident.span); + Ok(NtOrTt::Token(token)) + } else { + let token_str = pprust::token_to_string(&self.token); + let msg = &format!("expected ident, found {}", &token_str); + Err(self.struct_span_err(self.token.span, msg)) + }; + } + NonterminalKind::Lifetime => { + return if self.check_lifetime() { + let ident = self.expect_lifetime().ident; + let token = Token::new(token::Lifetime(ident.name), ident.span); + Ok(NtOrTt::Token(token)) + } else { + let token_str = pprust::token_to_string(&self.token); + let msg = &format!("expected a lifetime, found `{}`", &token_str); + Err(self.struct_span_err(self.token.span, msg)) + }; + } NonterminalKind::Item => match self.parse_item(ForceCollect::Yes)? { Some(item) => token::NtItem(item), None => { @@ -152,33 +167,13 @@ impl<'a> Parser<'a> { self.collect_tokens_no_attrs(|this| this.parse_no_question_mark_recover())?, ), - // this could be handled like a token, since it is one - NonterminalKind::Ident - if let Some((ident, is_raw)) = get_macro_ident(&self.token) => - { - self.bump(); - token::NtIdent(ident, is_raw) - } - NonterminalKind::Ident => { - let token_str = pprust::token_to_string(&self.token); - let msg = &format!("expected ident, found {}", &token_str); - return Err(self.struct_span_err(self.token.span, msg)); - } - NonterminalKind::Path => token::NtPath( - P(self.collect_tokens_no_attrs(|this| this.parse_path(PathStyle::Type))?), - ), + NonterminalKind::Path => token::NtPath(P( + self.collect_tokens_no_attrs(|this| this.parse_path(PathStyle::Type))? + )), NonterminalKind::Meta => token::NtMeta(P(self.parse_attr_item(true)?)), - NonterminalKind::Vis => token::NtVis( - P(self.collect_tokens_no_attrs(|this| this.parse_visibility(FollowedByType::Yes))?), - ), - NonterminalKind::Lifetime => { - if self.check_lifetime() { - token::NtLifetime(self.expect_lifetime().ident) - } else { - let token_str = pprust::token_to_string(&self.token); - let msg = &format!("expected a lifetime, found `{}`", &token_str); - return Err(self.struct_span_err(self.token.span, msg)); - } + NonterminalKind::Vis => { + token::NtVis(P(self + .collect_tokens_no_attrs(|this| this.parse_visibility(FollowedByType::Yes))?)) } }; diff --git a/compiler/rustc_parse/src/parser/pat.rs b/compiler/rustc_parse/src/parser/pat.rs index c7b929bf367d..241cb12bd42d 100644 --- a/compiler/rustc_parse/src/parser/pat.rs +++ b/compiler/rustc_parse/src/parser/pat.rs @@ -253,7 +253,7 @@ impl<'a> Parser<'a> { fn recover_trailing_vert(&mut self, lo: Option) -> bool { let is_end_ahead = self.look_ahead(1, |token| { matches!( - &token.uninterpolate().kind, + &token.kind, token::FatArrow // e.g. `a | => 0,`. | token::Ident(kw::If, false) // e.g. `a | if expr`. | token::Eq // e.g. `let a | = 0`. diff --git a/compiler/rustc_resolve/src/late.rs b/compiler/rustc_resolve/src/late.rs index 9ea66c0b59d9..34c5d7b8fd34 100644 --- a/compiler/rustc_resolve/src/late.rs +++ b/compiler/rustc_resolve/src/late.rs @@ -558,7 +558,7 @@ impl<'a: 'ast, 'ast> Visitor<'ast> for LateResolutionVisitor<'a, '_, 'ast> { } TyKind::Path(ref qself, ref path) => { self.diagnostic_metadata.current_type_path = Some(ty); - self.smart_resolve_path(ty.id, qself.as_ref(), path, PathSource::Type); + self.smart_resolve_path(2, ty.id, qself.as_ref(), path, PathSource::Type); } TyKind::ImplicitSelf => { let self_ty = Ident::with_dummy_span(kw::SelfUpper); @@ -622,6 +622,7 @@ impl<'a: 'ast, 'ast> Visitor<'ast> for LateResolutionVisitor<'a, '_, 'ast> { |this| { this.visit_generic_param_vec(&tref.bound_generic_params, false); this.smart_resolve_path( + 3, tref.trait_ref.ref_id, None, &tref.trait_ref.path, @@ -816,6 +817,7 @@ impl<'a: 'ast, 'ast> Visitor<'ast> for LateResolutionVisitor<'a, '_, 'ast> { // non-trivial constants this is doesn't matter. self.with_constant_rib(IsRepeatExpr::No, true, None, |this| { this.smart_resolve_path( + 4, ty.id, qself.as_ref(), path, @@ -901,6 +903,7 @@ impl<'a: 'ast, 'ast> Visitor<'ast> for LateResolutionVisitor<'a, '_, 'ast> { this.with_rib(TypeNS, InlineAsmSymRibKind, |this| { this.with_label_rib(InlineAsmSymRibKind, |this| { this.smart_resolve_path( + 5, sym.id, sym.qself.as_ref(), &sym.path, @@ -1946,6 +1949,7 @@ impl<'a: 'ast, 'b, 'ast> LateResolutionVisitor<'a, 'b, 'ast> { let mut new_id = None; if let Some(trait_ref) = opt_trait_ref { let path: Vec<_> = Segment::from_path(&trait_ref.path); + //eprintln!("sr2"); let res = self.smart_resolve_path_fragment( None, &path, @@ -2408,6 +2412,7 @@ impl<'a: 'ast, 'b, 'ast> LateResolutionVisitor<'a, 'b, 'ast> { } PatKind::TupleStruct(ref qself, ref path, ref sub_patterns) => { self.smart_resolve_path( + 6, pat.id, qself.as_ref(), path, @@ -2418,10 +2423,10 @@ impl<'a: 'ast, 'b, 'ast> LateResolutionVisitor<'a, 'b, 'ast> { ); } PatKind::Path(ref qself, ref path) => { - self.smart_resolve_path(pat.id, qself.as_ref(), path, PathSource::Pat); + self.smart_resolve_path(7, pat.id, qself.as_ref(), path, PathSource::Pat); } PatKind::Struct(ref qself, ref path, ..) => { - self.smart_resolve_path(pat.id, qself.as_ref(), path, PathSource::Struct); + self.smart_resolve_path(8, pat.id, qself.as_ref(), path, PathSource::Struct); } PatKind::Or(ref ps) => { // Add a new set of bindings to the stack. `Or` here records that when a @@ -2613,11 +2618,13 @@ impl<'a: 'ast, 'b, 'ast> LateResolutionVisitor<'a, 'b, 'ast> { // errors in user friendly way. fn smart_resolve_path( &mut self, + _x: u32, id: NodeId, qself: Option<&QSelf>, path: &Path, source: PathSource<'ast>, ) { + //eprintln!("sr3 {}", x); self.smart_resolve_path_fragment( qself, &Segment::from_path(path), @@ -2645,6 +2652,7 @@ impl<'a: 'ast, 'b, 'ast> LateResolutionVisitor<'a, 'b, 'ast> { finalize.node_id_and_path_span().expect("unexpected speculative resolution"); let report_errors = |this: &mut Self, res: Option| { if this.should_report_errs() { + //eprintln!("smart1"); let (err, candidates) = this.smart_resolve_report_errors(path, path_span, source, res); @@ -2683,6 +2691,7 @@ impl<'a: 'ast, 'b, 'ast> LateResolutionVisitor<'a, 'b, 'ast> { _ => return Some(parent_err), }; + //eprintln!("smart2"); let (mut err, candidates) = this.smart_resolve_report_errors(path, path_span, PathSource::Type, None); @@ -2909,6 +2918,7 @@ impl<'a: 'ast, 'b, 'ast> LateResolutionVisitor<'a, 'b, 'ast> { // name from a fully qualified path, and this also // contains the full span (the `Finalize::QPathTrait`). let ns = if qself.position + 1 == path.len() { ns } else { TypeNS }; + //eprintln!("sr1"); let partial_res = self.smart_resolve_path_fragment( None, &path[..=qself.position], @@ -3076,12 +3086,18 @@ impl<'a: 'ast, 'b, 'ast> LateResolutionVisitor<'a, 'b, 'ast> { // Next, resolve the node. match expr.kind { ExprKind::Path(ref qself, ref path) => { - self.smart_resolve_path(expr.id, qself.as_ref(), path, PathSource::Expr(parent)); + self.smart_resolve_path(0, expr.id, qself.as_ref(), path, PathSource::Expr(parent)); visit::walk_expr(self, expr); } ExprKind::Struct(ref se) => { - self.smart_resolve_path(expr.id, se.qself.as_ref(), &se.path, PathSource::Struct); + self.smart_resolve_path( + 1, + expr.id, + se.qself.as_ref(), + &se.path, + PathSource::Struct, + ); visit::walk_expr(self, expr); } diff --git a/compiler/rustc_resolve/src/late/diagnostics.rs b/compiler/rustc_resolve/src/late/diagnostics.rs index a9edb713b0da..9dcd80eebb60 100644 --- a/compiler/rustc_resolve/src/late/diagnostics.rs +++ b/compiler/rustc_resolve/src/late/diagnostics.rs @@ -138,6 +138,10 @@ impl<'a: 'ast, 'ast> LateResolutionVisitor<'a, '_, 'ast> { source: PathSource<'_>, res: Option, ) -> (DiagnosticBuilder<'a, ErrorGuaranteed>, Vec) { + //eprintln!("smart:\n- {:?}\n- {:?}", path, span); + for _seg in path { + //eprintln!("* {} {:?}", seg.ident, seg.ident.span); + } let ident_span = path.last().map_or(span, |ident| ident.ident.span); let ns = source.namespace(); let is_expected = &|res| source.is_expected(res); diff --git a/compiler/rustc_session/src/utils.rs b/compiler/rustc_session/src/utils.rs index db755ccd1d51..c02f37ac099e 100644 --- a/compiler/rustc_session/src/utils.rs +++ b/compiler/rustc_session/src/utils.rs @@ -130,9 +130,6 @@ impl<'a> FlattenNonterminals<'a> { pub fn process_token(&mut self, token: Token) -> TokenStream { match token.kind { - token::Interpolated(nt) if let token::NtIdent(ident, is_raw) = *nt => { - TokenTree::Token(Token::new(token::Ident(ident.name, is_raw), ident.span)).into() - } token::Interpolated(nt) => { let tts = (self.nt_to_tokenstream)(&nt, self.parse_sess, self.synthesize_tokens); TokenTree::Delimited(