Auto merge of #40597 - jseyfried:improve_span_expn_info, r=jseyfried

macros: improve `Span`'s expansion information

This PR improves `Span`'s expansion information. More specifically:
 - It refactors AST node span construction to preserve expansion information.
   - Today, we only use the underlying tokens' `BytePos`s, throwing away the `ExpnId`s.
   - This improves the accuracy of AST nodes' expansion information, fixing #30506.
 - It refactors `span.expn_id: ExpnId` to `span.ctxt: SyntaxContext` and removes `ExpnId`.
   - This gives all tokens as much hygiene information as `Ident`s.
   - This is groundwork for procedural macros 2.0 `TokenStream` API.
   - This is also groundwork for declarative macros 2.0, which will need this hygiene information for some non-`Ident` tokens.
 - It simplifies processing of spans' expansion information throughout the compiler.
 - It fixes #40649.
 - It fixes #39450 and fixes part of #23480.

r? @nrc

src/librustc/hir/lowering.rs

@@ -57,6 +57,7 @@ use std::mem;
 use syntax::attr;
 use syntax::ast::*;
 use syntax::errors;
+use syntax::ext::hygiene::{Mark, SyntaxContext};
 use syntax::ptr::P;
 use syntax::codemap::{self, respan, Spanned};
 use syntax::std_inject;
@@ -392,14 +393,16 @@ impl<'a> LoweringContext<'a> {
     }
 
     fn allow_internal_unstable(&self, reason: &'static str, mut span: Span) -> Span {
-        span.expn_id = self.sess.codemap().record_expansion(codemap::ExpnInfo {
+        let mark = Mark::fresh();
+        mark.set_expn_info(codemap::ExpnInfo {
             call_site: span,
             callee: codemap::NameAndSpan {
                 format: codemap::CompilerDesugaring(Symbol::intern(reason)),
                 span: Some(span),
                 allow_internal_unstable: true,
             },
         });
+        span.ctxt = SyntaxContext::empty().apply_mark(mark);
         span
     }
 
@@ -1993,7 +1996,7 @@ impl<'a> LoweringContext<'a> {
                     volatile: asm.volatile,
                     alignstack: asm.alignstack,
                     dialect: asm.dialect,
-                    expn_id: asm.expn_id,
+                    ctxt: asm.ctxt,
                 };
                 let outputs =
                     asm.outputs.iter().map(|out| self.lower_expr(&out.expr)).collect();

src/librustc/hir/map/def_collector.rs

@@ -92,7 +92,7 @@ impl<'a> DefCollector<'a> {
     fn visit_macro_invoc(&mut self, id: NodeId, const_expr: bool) {
         if let Some(ref mut visit) = self.visit_macro_invoc {
             visit(MacroInvocationData {
-                mark: Mark::from_placeholder_id(id),
+                mark: id.placeholder_to_mark(),
                 const_expr: const_expr,
                 def_index: self.parent_def.unwrap(),
             })

src/librustc/hir/mod.rs

@@ -33,11 +33,12 @@ use hir::def::Def;
 use hir::def_id::{DefId, DefIndex, CRATE_DEF_INDEX};
 use util::nodemap::{NodeMap, FxHashSet};
 
-use syntax_pos::{Span, ExpnId, DUMMY_SP};
+use syntax_pos::{Span, DUMMY_SP};
 use syntax::codemap::{self, Spanned};
 use syntax::abi::Abi;
 use syntax::ast::{Ident, Name, NodeId, DUMMY_NODE_ID, AsmDialect};
 use syntax::ast::{Attribute, Lit, StrStyle, FloatTy, IntTy, UintTy, MetaItem};
+use syntax::ext::hygiene::SyntaxContext;
 use syntax::ptr::P;
 use syntax::symbol::{Symbol, keywords};
 use syntax::tokenstream::TokenStream;
@@ -1367,7 +1368,7 @@ pub struct InlineAsm {
     pub volatile: bool,
     pub alignstack: bool,
     pub dialect: AsmDialect,
-    pub expn_id: ExpnId,
+    pub ctxt: SyntaxContext,
 }
 
 /// represents an argument in a function header

src/librustc/ich/caching_codemap_view.rs

@@ -47,10 +47,6 @@ impl<'tcx> CachingCodemapView<'tcx> {
         }
     }
 
-    pub fn codemap(&self) -> &'tcx CodeMap {
-        self.codemap
-    }
-
     pub fn byte_pos_to_line_and_col(&mut self,
                                     pos: BytePos)
                                     -> Option<(Rc<FileMap>, usize, BytePos)> {

src/librustc/middle/region.rs

@@ -236,7 +236,7 @@ impl CodeExtent {
                         // (This is the special case aluded to in the
                         // doc-comment for this method)
                         let stmt_span = blk.stmts[r.first_statement_index as usize].span;
-                        Some(Span { lo: stmt_span.hi, hi: blk.span.hi, expn_id: stmt_span.expn_id })
+                        Some(Span { lo: stmt_span.hi, hi: blk.span.hi, ctxt: stmt_span.ctxt })
                     }
                 }
             }

src/librustc/middle/resolve_lifetime.rs

@@ -29,7 +29,7 @@ use syntax::ast;
 use syntax::attr;
 use syntax::ptr::P;
 use syntax::symbol::keywords;
-use syntax_pos::{mk_sp, Span};
+use syntax_pos::Span;
 use errors::DiagnosticBuilder;
 use util::nodemap::{NodeMap, NodeSet, FxHashSet, FxHashMap, DefIdMap};
 use rustc_back::slice;
@@ -1468,8 +1468,7 @@ impl<'a, 'tcx> LifetimeContext<'a, 'tcx> {
                     self.resolve_lifetime_ref(bound);
                 } else {
                     self.insert_lifetime(bound, Region::Static);
-                    let full_span = mk_sp(lifetime_i.lifetime.span.lo, bound.span.hi);
-                    self.sess.struct_span_warn(full_span,
+                    self.sess.struct_span_warn(lifetime_i.lifetime.span.to(bound.span),
                         &format!("unnecessary lifetime parameter `{}`", lifetime_i.lifetime.name))
                         .help(&format!("you can use the `'static` lifetime directly, in place \
                                         of `{}`", lifetime_i.lifetime.name))

src/librustc/middle/stability.rs

@@ -467,7 +467,7 @@ impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> {
     }
 
     pub fn check_stability(self, def_id: DefId, id: NodeId, span: Span) {
-        if self.sess.codemap().span_allows_unstable(span) {
+        if span.allows_unstable() {
             debug!("stability: \
                     skipping span={:?} since it is internal", span);
             return;

src/librustc_driver/driver.rs

@@ -580,7 +580,7 @@ pub fn phase_2_configure_and_expand<F>(sess: &Session,
 
     krate = time(time_passes, "crate injection", || {
         let alt_std_name = sess.opts.alt_std_name.clone();
-        syntax::std_inject::maybe_inject_crates_ref(&sess.parse_sess, krate, alt_std_name)
+        syntax::std_inject::maybe_inject_crates_ref(krate, alt_std_name)
     });
 
     let mut addl_plugins = Some(addl_plugins);
@@ -798,7 +798,7 @@ pub fn phase_2_configure_and_expand<F>(sess: &Session,
 
     // Discard hygiene data, which isn't required after lowering to HIR.
     if !keep_hygiene_data(sess) {
-        syntax::ext::hygiene::reset_hygiene_data();
+        syntax::ext::hygiene::clear_markings();
     }
 
     Ok(ExpansionResult {

src/librustc_errors/emitter.rs

@@ -10,7 +10,7 @@
 
 use self::Destination::*;
 
-use syntax_pos::{COMMAND_LINE_SP, DUMMY_SP, FileMap, Span, MultiSpan, CharPos};
+use syntax_pos::{DUMMY_SP, FileMap, Span, MultiSpan, CharPos};
 
 use {Level, CodeSuggestion, DiagnosticBuilder, SubDiagnostic, CodeMapper};
 use RenderSpan::*;
@@ -151,7 +151,7 @@ impl EmitterWriter {
 
         if let Some(ref cm) = self.cm {
             for span_label in msp.span_labels() {
-                if span_label.span == DUMMY_SP || span_label.span == COMMAND_LINE_SP {
+                if span_label.span == DUMMY_SP {
                     continue;
                 }
                 let lo = cm.lookup_char_pos(span_label.span.lo);
@@ -615,15 +615,15 @@ impl EmitterWriter {
         let mut max = 0;
         if let Some(ref cm) = self.cm {
             for primary_span in msp.primary_spans() {
-                if primary_span != &DUMMY_SP && primary_span != &COMMAND_LINE_SP {
+                if primary_span != &DUMMY_SP {
                     let hi = cm.lookup_char_pos(primary_span.hi);
                     if hi.line > max {
                         max = hi.line;
                     }
                 }
             }
             for span_label in msp.span_labels() {
-                if span_label.span != DUMMY_SP && span_label.span != COMMAND_LINE_SP {
+                if span_label.span != DUMMY_SP {
                     let hi = cm.lookup_char_pos(span_label.span.hi);
                     if hi.line > max {
                         max = hi.line;
@@ -659,20 +659,20 @@ impl EmitterWriter {
 
             // First, find all the spans in <*macros> and point instead at their use site
             for sp in span.primary_spans() {
-                if (*sp == COMMAND_LINE_SP) || (*sp == DUMMY_SP) {
+                if *sp == DUMMY_SP {
                     continue;
                 }
                 if cm.span_to_filename(sp.clone()).contains("macros>") {
-                    let v = cm.macro_backtrace(sp.clone());
+                    let v = sp.macro_backtrace();
                     if let Some(use_site) = v.last() {
                         before_after.push((sp.clone(), use_site.call_site.clone()));
                     }
                 }
-                for trace in cm.macro_backtrace(sp.clone()).iter().rev() {
+                for trace in sp.macro_backtrace().iter().rev() {
                     // Only show macro locations that are local
                     // and display them like a span_note
                     if let Some(def_site) = trace.def_site_span {
-                        if (def_site == COMMAND_LINE_SP) || (def_site == DUMMY_SP) {
+                        if def_site == DUMMY_SP {
                             continue;
                         }
                         // Check to make sure we're not in any <*macros>
@@ -689,11 +689,11 @@ impl EmitterWriter {
                 span.push_span_label(label_span, label_text);
             }
             for sp_label in span.span_labels() {
-                if (sp_label.span == COMMAND_LINE_SP) || (sp_label.span == DUMMY_SP) {
+                if sp_label.span == DUMMY_SP {
                     continue;
                 }
                 if cm.span_to_filename(sp_label.span.clone()).contains("macros>") {
-                    let v = cm.macro_backtrace(sp_label.span.clone());
+                    let v = sp_label.span.macro_backtrace();
                     if let Some(use_site) = v.last() {
                         before_after.push((sp_label.span.clone(), use_site.call_site.clone()));
                     }
@@ -848,7 +848,7 @@ impl EmitterWriter {
         // Make sure our primary file comes first
         let primary_lo = if let (Some(ref cm), Some(ref primary_span)) =
             (self.cm.as_ref(), msp.primary_span().as_ref()) {
-            if primary_span != &&DUMMY_SP && primary_span != &&COMMAND_LINE_SP {
+            if primary_span != &&DUMMY_SP {
                 cm.lookup_char_pos(primary_span.lo)
             } else {
                 emit_to_destination(&buffer.render(), level, &mut self.dst)?;

src/librustc_errors/lib.rs

@@ -48,7 +48,6 @@ pub mod styled_buffer;
 mod lock;
 
 use syntax_pos::{BytePos, Loc, FileLinesResult, FileName, MultiSpan, Span, NO_EXPANSION};
-use syntax_pos::MacroBacktrace;
 
 #[derive(Clone, Debug, PartialEq, RustcEncodable, RustcDecodable)]
 pub enum RenderSpan {
@@ -75,7 +74,6 @@ pub trait CodeMapper {
     fn span_to_lines(&self, sp: Span) -> FileLinesResult;
     fn span_to_string(&self, sp: Span) -> String;
     fn span_to_filename(&self, sp: Span) -> FileName;
-    fn macro_backtrace(&self, span: Span) -> Vec<MacroBacktrace>;
     fn merge_spans(&self, sp_lhs: Span, sp_rhs: Span) -> Option<Span>;
 }
 
@@ -120,7 +118,7 @@ impl CodeSuggestion {
         let bounding_span = Span {
             lo: lo,
             hi: hi,
-            expn_id: NO_EXPANSION,
+            ctxt: NO_EXPANSION,
         };
         let lines = cm.span_to_lines(bounding_span).unwrap();
         assert!(!lines.lines.is_empty());

src/librustc_incremental/calculate_svh/svh_visitor.rs

@@ -17,9 +17,10 @@ use self::SawTraitOrImplItemComponent::*;
 use syntax::abi::Abi;
 use syntax::ast::{self, Name, NodeId};
 use syntax::attr;
+use syntax::ext::hygiene::SyntaxContext;
 use syntax::parse::token;
 use syntax::symbol::InternedString;
-use syntax_pos::{Span, NO_EXPANSION, COMMAND_LINE_EXPN, BytePos};
+use syntax_pos::{Span, BytePos};
 use syntax::tokenstream;
 use rustc::hir;
 use rustc::hir::*;
@@ -92,10 +93,10 @@ impl<'a, 'hash, 'tcx> StrictVersionHashVisitor<'a, 'hash, 'tcx> {
             span.hi
         };
 
-        let expn_kind = match span.expn_id {
-            NO_EXPANSION => SawSpanExpnKind::NoExpansion,
-            COMMAND_LINE_EXPN => SawSpanExpnKind::CommandLine,
-            _ => SawSpanExpnKind::SomeExpansion,
+        let expn_kind = if span.ctxt == SyntaxContext::empty() {
+            SawSpanExpnKind::NoExpansion
+        } else {
+            SawSpanExpnKind::SomeExpansion
         };
 
         let loc1 = self.codemap.byte_pos_to_line_and_col(span.lo);
@@ -121,8 +122,7 @@ impl<'a, 'hash, 'tcx> StrictVersionHashVisitor<'a, 'hash, 'tcx> {
         saw.hash(self.st);
 
         if expn_kind == SawSpanExpnKind::SomeExpansion {
-            let call_site = self.codemap.codemap().source_callsite(span);
-            self.hash_span(call_site);
+            self.hash_span(span.source_callsite());
         }
     }
 
@@ -483,7 +483,6 @@ fn saw_impl_item(ii: &ImplItemKind) -> SawTraitOrImplItemComponent {
 #[derive(Clone, Copy, Hash, Eq, PartialEq)]
 enum SawSpanExpnKind {
     NoExpansion,
-    CommandLine,
     SomeExpansion,
 }
 
@@ -501,7 +500,7 @@ impl<'a> Hash for StableInlineAsm<'a> {
             volatile,
             alignstack,
             dialect,
-            expn_id: _, // This is used for error reporting
+            ctxt: _, // This is used for error reporting
         } = *self.0;
 
         asm.as_str().hash(state);

src/librustc_metadata/cstore_impl.rs

@@ -36,7 +36,7 @@ use syntax::ast;
 use syntax::attr;
 use syntax::parse::filemap_to_stream;
 use syntax::symbol::Symbol;
-use syntax_pos::{mk_sp, Span};
+use syntax_pos::{Span, NO_EXPANSION};
 use rustc::hir::svh::Svh;
 use rustc_back::target::Target;
 use rustc::hir;
@@ -395,7 +395,7 @@ impl CrateStore for cstore::CStore {
         let source_name = format!("<{} macros>", name);
 
         let filemap = sess.parse_sess.codemap().new_filemap(source_name, None, def.body);
-        let local_span = mk_sp(filemap.start_pos, filemap.end_pos);
+        let local_span = Span { lo: filemap.start_pos, hi: filemap.end_pos, ctxt: NO_EXPANSION };
         let body = filemap_to_stream(&sess.parse_sess, filemap);
 
         // Mark the attrs as used

src/librustc_metadata/decoder.rs

@@ -39,7 +39,7 @@ use syntax::attr;
 use syntax::ast;
 use syntax::codemap;
 use syntax::ext::base::MacroKind;
-use syntax_pos::{self, Span, BytePos, Pos, DUMMY_SP};
+use syntax_pos::{self, Span, BytePos, Pos, DUMMY_SP, NO_EXPANSION};
 
 pub struct DecodeContext<'a, 'tcx: 'a> {
     opaque: opaque::Decoder<'a>,
@@ -243,7 +243,7 @@ impl<'a, 'tcx> SpecializedDecoder<Span> for DecodeContext<'a, 'tcx> {
         let sess = if let Some(sess) = self.sess {
             sess
         } else {
-            return Ok(syntax_pos::mk_sp(lo, hi));
+            return Ok(Span { lo: lo, hi: hi, ctxt: NO_EXPANSION });
         };
 
         let (lo, hi) = if lo > hi {
@@ -290,7 +290,7 @@ impl<'a, 'tcx> SpecializedDecoder<Span> for DecodeContext<'a, 'tcx> {
         let lo = (lo - filemap.original_start_pos) + filemap.translated_filemap.start_pos;
         let hi = (hi - filemap.original_start_pos) + filemap.translated_filemap.start_pos;
 
-        Ok(syntax_pos::mk_sp(lo, hi))
+        Ok(Span { lo: lo, hi: hi, ctxt: NO_EXPANSION })
     }
 }
 

src/librustc_mir/transform/qualify_consts.rs

@@ -223,7 +223,7 @@ impl<'a, 'tcx> Qualifier<'a, 'tcx, 'tcx> {
             }
 
             // This comes from a macro that has #[allow_internal_unstable].
-            if self.tcx.sess.codemap().span_allows_unstable(self.span) {
+            if self.span.allows_unstable() {
                 return;
             }
 
@@ -805,7 +805,7 @@ impl<'a, 'tcx> Visitor<'tcx> for Qualifier<'a, 'tcx, 'tcx> {
                     self.def_id.is_local() &&
 
                     // this doesn't come from a macro that has #[allow_internal_unstable]
-                    !self.tcx.sess.codemap().span_allows_unstable(self.span)
+                    !self.span.allows_unstable()
                 {
                     let mut err = self.tcx.sess.struct_span_err(self.span,
                         "const fns are an unstable feature");

src/librustc_plugin/load.rs

@@ -20,7 +20,7 @@ use std::env;
 use std::mem;
 use std::path::PathBuf;
 use syntax::ast;
-use syntax_pos::{Span, COMMAND_LINE_SP};
+use syntax_pos::{Span, DUMMY_SP};
 
 /// Pointer to a registrar function.
 pub type PluginRegistrarFun =
@@ -81,7 +81,7 @@ pub fn load_plugins(sess: &Session,
 
     if let Some(plugins) = addl_plugins {
         for plugin in plugins {
-            loader.load_plugin(COMMAND_LINE_SP, &plugin, vec![]);
+            loader.load_plugin(DUMMY_SP, &plugin, vec![]);
         }
     }
 

src/librustc_resolve/build_reduced_graph.rs

@@ -680,7 +680,7 @@ pub struct BuildReducedGraphVisitor<'a, 'b: 'a> {
 
 impl<'a, 'b> BuildReducedGraphVisitor<'a, 'b> {
     fn visit_invoc(&mut self, id: ast::NodeId) -> &'b InvocationData<'b> {
-        let mark = Mark::from_placeholder_id(id);
+        let mark = id.placeholder_to_mark();
         self.resolver.current_module.unresolved_invocations.borrow_mut().insert(mark);
         let invocation = self.resolver.invocations[&mark];
         invocation.module.set(self.resolver.current_module);