Auto merge of #60679 - petrochenkov:lit2, r=<try>

Keep original literal tokens in AST

The original literal tokens (`token::Lit`) are kept in AST until lowering to HIR.

The tokens are kept together with their lowered "semantic" representation (`ast::LitKind`), so the size of `ast::Lit` is increased (this also increases the size of meta-item structs used for processing built-in attributes).
However, the size of `ast::Expr` stays the same.

The intent is to remove the "semantic" representation from AST eventually and keep literals as tokens until lowering to HIR (at least), and I'm going to work on that, but it would be good to land this sooner to unblock progress on the [lexer refactoring](#59706).

Fixes a part of #43081 (literal tokens that are passed to proc macros are always precise, including hexadecimal numbers, strings with their original escaping, etc)
Fixes a part of #60495 (everything except for proc macro API doesn't need escaping anymore)
This also allows to eliminate a certain hack from the lexer (https://rust-lang.zulipchat.com/#narrow/stream/131828-t-compiler/topic/pretty-printing.20comments/near/165005357).

cc @matklad

src/librustc/hir/lowering.rs

@@ -4124,7 +4124,7 @@ impl<'a> LoweringContext<'a> {
                 let ohs = P(self.lower_expr(ohs));
                 hir::ExprKind::Unary(op, ohs)
             }
-            ExprKind::Lit(ref l) => hir::ExprKind::Lit((*l).clone()),
+            ExprKind::Lit(ref l) => hir::ExprKind::Lit(respan(l.span, l.node.clone())),
             ExprKind::Cast(ref expr, ref ty) => {
                 let expr = P(self.lower_expr(expr));
                 hir::ExprKind::Cast(expr, self.lower_ty(ty, ImplTraitContext::disallowed()))

src/librustc/hir/mod.rs

@@ -20,7 +20,7 @@ use syntax_pos::{Span, DUMMY_SP, symbol::InternedString};
 use syntax::source_map::Spanned;
 use rustc_target::spec::abi::Abi;
 use syntax::ast::{self, CrateSugar, Ident, Name, NodeId, AsmDialect};
-use syntax::ast::{Attribute, Label, Lit, StrStyle, FloatTy, IntTy, UintTy};
+use syntax::ast::{Attribute, Label, LitKind, StrStyle, FloatTy, IntTy, UintTy};
 use syntax::attr::{InlineAttr, OptimizeAttr};
 use syntax::ext::hygiene::SyntaxContext;
 use syntax::ptr::P;
@@ -1331,6 +1331,9 @@ impl BodyOwnerKind {
     }
 }
 
+/// A literal.
+pub type Lit = Spanned<LitKind>;
+
 /// A constant (expression) that's not an item or associated item,
 /// but needs its own `DefId` for type-checking, const-eval, etc.
 /// These are usually found nested inside types (e.g., array lengths)

src/librustc/hir/print.rs

@@ -15,10 +15,10 @@ use crate::hir;
 use crate::hir::{PatKind, GenericBound, TraitBoundModifier, RangeEnd};
 use crate::hir::{GenericParam, GenericParamKind, GenericArg};
 
+use std::ascii;
 use std::borrow::Cow;
 use std::cell::Cell;
 use std::io::{self, Write, Read};
-use std::iter::Peekable;
 use std::vec;
 
 pub enum AnnNode<'a> {
@@ -76,7 +76,6 @@ pub struct State<'a> {
     pub s: pp::Printer<'a>,
     cm: Option<&'a SourceMap>,
     comments: Option<Vec<comments::Comment>>,
-    literals: Peekable<vec::IntoIter<comments::Literal>>,
     cur_cmnt: usize,
     boxes: Vec<pp::Breaks>,
     ann: &'a (dyn PpAnn + 'a),
@@ -98,14 +97,6 @@ impl<'a> PrintState<'a> for State<'a> {
     fn cur_cmnt(&mut self) -> &mut usize {
         &mut self.cur_cmnt
     }
-
-    fn cur_lit(&mut self) -> Option<&comments::Literal> {
-        self.literals.peek()
-    }
-
-    fn bump_lit(&mut self) -> Option<comments::Literal> {
-        self.literals.next()
-    }
 }
 
 #[allow(non_upper_case_globals)]
@@ -116,18 +107,16 @@ pub const default_columns: usize = 78;
 
 
 /// Requires you to pass an input filename and reader so that
-/// it can scan the input text for comments and literals to
-/// copy forward.
+/// it can scan the input text for comments to copy forward.
 pub fn print_crate<'a>(cm: &'a SourceMap,
                        sess: &ParseSess,
                        krate: &hir::Crate,
                        filename: FileName,
                        input: &mut dyn Read,
                        out: Box<dyn Write + 'a>,
-                       ann: &'a dyn PpAnn,
-                       is_expanded: bool)
+                       ann: &'a dyn PpAnn)
                        -> io::Result<()> {
-    let mut s = State::new_from_input(cm, sess, filename, input, out, ann, is_expanded);
+    let mut s = State::new_from_input(cm, sess, filename, input, out, ann);
 
     // When printing the AST, we sometimes need to inject `#[no_std]` here.
     // Since you can't compile the HIR, it's not necessary.
@@ -143,36 +132,21 @@ impl<'a> State<'a> {
                           filename: FileName,
                           input: &mut dyn Read,
                           out: Box<dyn Write + 'a>,
-                          ann: &'a dyn PpAnn,
-                          is_expanded: bool)
+                          ann: &'a dyn PpAnn)
                           -> State<'a> {
-        let (cmnts, lits) = comments::gather_comments_and_literals(sess, filename, input);
-
-        State::new(cm,
-                   out,
-                   ann,
-                   Some(cmnts),
-                   // If the code is post expansion, don't use the table of
-                   // literals, since it doesn't correspond with the literals
-                   // in the AST anymore.
-                   if is_expanded {
-                       None
-                   } else {
-                       Some(lits)
-                   })
+        let comments = comments::gather_comments(sess, filename, input);
+        State::new(cm, out, ann, Some(comments))
     }
 
     pub fn new(cm: &'a SourceMap,
                out: Box<dyn Write + 'a>,
                ann: &'a dyn PpAnn,
-               comments: Option<Vec<comments::Comment>>,
-               literals: Option<Vec<comments::Literal>>)
+               comments: Option<Vec<comments::Comment>>)
                -> State<'a> {
         State {
             s: pp::mk_printer(out, default_columns),
             cm: Some(cm),
             comments,
-            literals: literals.unwrap_or_default().into_iter().peekable(),
             cur_cmnt: 0,
             boxes: Vec::new(),
             ann,
@@ -189,7 +163,6 @@ pub fn to_string<F>(ann: &dyn PpAnn, f: F) -> String
             s: pp::mk_printer(Box::new(&mut wr), default_columns),
             cm: None,
             comments: None,
-            literals: vec![].into_iter().peekable(),
             cur_cmnt: 0,
             boxes: Vec::new(),
             ann,
@@ -1335,6 +1308,61 @@ impl<'a> State<'a> {
         self.print_expr_maybe_paren(expr, parser::PREC_PREFIX)
     }
 
+    fn print_literal(&mut self, lit: &hir::Lit) -> io::Result<()> {
+        self.maybe_print_comment(lit.span.lo())?;
+        match lit.node {
+            hir::LitKind::Str(st, style) => self.print_string(&st.as_str(), style),
+            hir::LitKind::Err(st) => {
+                let st = st.as_str().escape_debug().to_string();
+                let mut res = String::with_capacity(st.len() + 2);
+                res.push('\'');
+                res.push_str(&st);
+                res.push('\'');
+                self.writer().word(res)
+            }
+            hir::LitKind::Byte(byte) => {
+                let mut res = String::from("b'");
+                res.extend(ascii::escape_default(byte).map(|c| c as char));
+                res.push('\'');
+                self.writer().word(res)
+            }
+            hir::LitKind::Char(ch) => {
+                let mut res = String::from("'");
+                res.extend(ch.escape_default());
+                res.push('\'');
+                self.writer().word(res)
+            }
+            hir::LitKind::Int(i, t) => {
+                match t {
+                    ast::LitIntType::Signed(st) => {
+                        self.writer().word(st.val_to_string(i as i128))
+                    }
+                    ast::LitIntType::Unsigned(ut) => {
+                        self.writer().word(ut.val_to_string(i))
+                    }
+                    ast::LitIntType::Unsuffixed => {
+                        self.writer().word(i.to_string())
+                    }
+                }
+            }
+            hir::LitKind::Float(ref f, t) => {
+                self.writer().word(format!("{}{}", &f, t.ty_to_string()))
+            }
+            hir::LitKind::FloatUnsuffixed(ref f) => self.writer().word(f.as_str().to_string()),
+            hir::LitKind::Bool(val) => {
+                if val { self.writer().word("true") } else { self.writer().word("false") }
+            }
+            hir::LitKind::ByteStr(ref v) => {
+                let mut escaped: String = String::new();
+                for &ch in v.iter() {
+                    escaped.extend(ascii::escape_default(ch)
+                                         .map(|c| c as char));
+                }
+                self.writer().word(format!("b\"{}\"", escaped))
+            }
+        }
+    }
+
     pub fn print_expr(&mut self, expr: &hir::Expr) -> io::Result<()> {
         self.maybe_print_comment(expr.span.lo())?;
         self.print_outer_attributes(&expr.attrs)?;

src/librustc/ich/impls_syntax.rs

@@ -162,7 +162,13 @@ impl_stable_hash_for!(enum ::syntax::ast::LitIntType {
     Unsuffixed
 });
 
-impl_stable_hash_for_spanned!(::syntax::ast::LitKind);
+impl_stable_hash_for!(struct ::syntax::ast::Lit {
+    node,
+    token,
+    suffix,
+    span
+});
+
 impl_stable_hash_for!(enum ::syntax::ast::LitKind {
     Str(value, style),
     Err(value),
@@ -175,6 +181,8 @@ impl_stable_hash_for!(enum ::syntax::ast::LitKind {
     Bool(value)
 });
 
+impl_stable_hash_for_spanned!(::syntax::ast::LitKind);
+
 impl_stable_hash_for!(enum ::syntax::ast::IntTy { Isize, I8, I16, I32, I64, I128 });
 impl_stable_hash_for!(enum ::syntax::ast::UintTy { Usize, U8, U16, U32, U64, U128 });
 impl_stable_hash_for!(enum ::syntax::ast::FloatTy { F32, F64 });
@@ -280,6 +288,19 @@ for tokenstream::TokenStream {
     }
 }
 
+impl_stable_hash_for!(enum token::Lit {
+    Bool(val),
+    Byte(val),
+    Char(val),
+    Err(val),
+    Integer(val),
+    Float(val),
+    Str_(val),
+    ByteStr(val),
+    StrRaw(val, n),
+    ByteStrRaw(val, n)
+});
+
 fn hash_token<'a, 'gcx, W: StableHasherResult>(
     token: &token::Token,
     hcx: &mut StableHashingContext<'a>,
@@ -327,22 +348,8 @@ fn hash_token<'a, 'gcx, W: StableHasherResult>(
         token::Token::CloseDelim(delim_token) => {
             std_hash::Hash::hash(&delim_token, hasher);
         }
-        token::Token::Literal(ref lit, ref opt_name) => {
-            mem::discriminant(lit).hash_stable(hcx, hasher);
-            match *lit {
-                token::Lit::Byte(val) |
-                token::Lit::Char(val) |
-                token::Lit::Err(val) |
-                token::Lit::Integer(val) |
-                token::Lit::Float(val) |
-                token::Lit::Str_(val) |
-                token::Lit::ByteStr(val) => val.hash_stable(hcx, hasher),
-                token::Lit::StrRaw(val, n) |
-                token::Lit::ByteStrRaw(val, n) => {
-                    val.hash_stable(hcx, hasher);
-                    n.hash_stable(hcx, hasher);
-                }
-            };
+        token::Token::Literal(lit, opt_name) => {
+            lit.hash_stable(hcx, hasher);
             opt_name.hash_stable(hcx, hasher);
         }
 

src/librustc_driver/pretty.rs

@@ -806,8 +806,7 @@ pub fn print_after_hir_lowering<'tcx>(
                                             src_name,
                                             &mut rdr,
                                             box out,
-                                            annotation.pp_ann(),
-                                            true)
+                                            annotation.pp_ann())
                 })
             }
 
@@ -830,8 +829,7 @@ pub fn print_after_hir_lowering<'tcx>(
                                                                          src_name,
                                                                          &mut rdr,
                                                                          box out,
-                                                                         annotation.pp_ann(),
-                                                                         true);
+                                                                         annotation.pp_ann());
                     for node_id in uii.all_matching_node_ids(hir_map) {
                         let node = hir_map.get(node_id);
                         pp_state.print_node(node)?;

src/librustc_lint/types.rs

@@ -62,7 +62,7 @@ impl TypeLimits {
 /// Returns `true` iff the lint was overridden.
 fn lint_overflowing_range_endpoint<'a, 'tcx>(
     cx: &LateContext<'a, 'tcx>,
-    lit: &ast::Lit,
+    lit: &hir::Lit,
     lit_val: u128,
     max: u128,
     expr: &'tcx hir::Expr,
@@ -132,7 +132,7 @@ fn uint_ty_range(uint_ty: ast::UintTy) -> (u128, u128) {
     }
 }
 
-fn get_bin_hex_repr(cx: &LateContext<'_, '_>, lit: &ast::Lit) -> Option<String> {
+fn get_bin_hex_repr(cx: &LateContext<'_, '_>, lit: &hir::Lit) -> Option<String> {
     let src = cx.sess().source_map().span_to_snippet(lit.span).ok()?;
     let firstch = src.chars().next()?;
 
@@ -249,7 +249,7 @@ fn lint_int_literal<'a, 'tcx>(
     cx: &LateContext<'a, 'tcx>,
     type_limits: &TypeLimits,
     e: &'tcx hir::Expr,
-    lit: &ast::Lit,
+    lit: &hir::Lit,
     t: ast::IntTy,
     v: u128,
 ) {
@@ -301,7 +301,7 @@ fn lint_int_literal<'a, 'tcx>(
 fn lint_uint_literal<'a, 'tcx>(
     cx: &LateContext<'a, 'tcx>,
     e: &'tcx hir::Expr,
-    lit: &ast::Lit,
+    lit: &hir::Lit,
     t: ast::UintTy,
 ) {
     let uint_type = if let ast::UintTy::Usize = t {
@@ -363,7 +363,7 @@ fn lint_literal<'a, 'tcx>(
     cx: &LateContext<'a, 'tcx>,
     type_limits: &TypeLimits,
     e: &'tcx hir::Expr,
-    lit: &ast::Lit,
+    lit: &hir::Lit,
 ) {
     match cx.tables.node_type(e.hir_id).sty {
         ty::Int(t) => {

src/librustc_typeck/check/mod.rs

@@ -3084,7 +3084,7 @@ impl<'a, 'gcx, 'tcx> FnCtxt<'a, 'gcx, 'tcx> {
 
     // AST fragment checking
     fn check_lit(&self,
-                 lit: &ast::Lit,
+                 lit: &hir::Lit,
                  expected: Expectation<'tcx>)
                  -> Ty<'tcx>
     {

src/librustdoc/clean/cfg.rs

@@ -414,10 +414,9 @@ impl<'a> fmt::Display for Html<'a> {
 mod test {
     use super::Cfg;
 
-    use syntax::symbol::Symbol;
-    use syntax::ast::*;
-    use syntax::source_map::dummy_spanned;
     use syntax_pos::DUMMY_SP;
+    use syntax::ast::*;
+    use syntax::symbol::Symbol;
     use syntax::with_globals;
 
     fn word_cfg(s: &str) -> Cfg {
@@ -592,12 +591,11 @@ mod test {
             let mi = dummy_meta_item_word("all");
             assert_eq!(Cfg::parse(&mi), Ok(word_cfg("all")));
 
+            let node = LitKind::Str(Symbol::intern("done"), StrStyle::Cooked);
+            let (token, suffix) =   node.lit_token();
             let mi = MetaItem {
                 path: Path::from_ident(Ident::from_str("all")),
-                node: MetaItemKind::NameValue(dummy_spanned(LitKind::Str(
-                    Symbol::intern("done"),
-                    StrStyle::Cooked,
-                ))),
+                node: MetaItemKind::NameValue(Lit { node, token, suffix, span: DUMMY_SP }),
                 span: DUMMY_SP,
             };
             assert_eq!(Cfg::parse(&mi), Ok(name_value_cfg("all", "done")));
@@ -627,9 +625,11 @@ mod test {
     #[test]
     fn test_parse_err() {
         with_globals(|| {
+            let node = LitKind::Bool(false);
+            let (token, suffix) = node.lit_token();
             let mi = MetaItem {
                 path: Path::from_ident(Ident::from_str("foo")),
-                node: MetaItemKind::NameValue(dummy_spanned(LitKind::Bool(false))),
+                node: MetaItemKind::NameValue(Lit { node, token, suffix, span: DUMMY_SP }),
                 span: DUMMY_SP,
             };
             assert!(Cfg::parse(&mi).is_err());

src/librustdoc/html/highlight.rs

@@ -318,6 +318,8 @@ impl<'a> Classifier<'a> {
 
                     // Number literals.
                     token::Integer(..) | token::Float(..) => Class::Number,
+
+                    token::Bool(..) => panic!("literal token contains `Lit::Bool`"),
                 }
             }