diff --git a/Cargo.lock b/Cargo.lock index 8db8a56eaa8ec..c9d86349c279c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3840,6 +3840,7 @@ dependencies = [ name = "rustc_expand" version = "0.0.0" dependencies = [ + "crossbeam-channel", "rustc_ast", "rustc_ast_passes", "rustc_ast_pretty", @@ -3856,6 +3857,7 @@ dependencies = [ "rustc_span", "smallvec", "tracing", + "unicode-normalization", ] [[package]] diff --git a/compiler/rustc_expand/Cargo.toml b/compiler/rustc_expand/Cargo.toml index 59c1604e8444c..d85460756a545 100644 --- a/compiler/rustc_expand/Cargo.toml +++ b/compiler/rustc_expand/Cargo.toml @@ -25,3 +25,5 @@ rustc_parse = { path = "../rustc_parse" } rustc_session = { path = "../rustc_session" } smallvec = { version = "1.6.1", features = ["union", "may_dangle"] } rustc_ast = { path = "../rustc_ast" } +crossbeam-channel = "0.5.0" +unicode-normalization = "0.1.11" diff --git a/compiler/rustc_expand/src/proc_macro.rs b/compiler/rustc_expand/src/proc_macro.rs index 494b3fb61ee97..e597e9a58b6af 100644 --- a/compiler/rustc_expand/src/proc_macro.rs +++ b/compiler/rustc_expand/src/proc_macro.rs @@ -12,7 +12,32 @@ use rustc_parse::parser::ForceCollect; use rustc_span::def_id::CrateNum; use rustc_span::{Span, DUMMY_SP}; -const EXEC_STRATEGY: pm::bridge::server::SameThread = pm::bridge::server::SameThread; +struct CrossbeamMessagePipe { + tx: crossbeam_channel::Sender, + rx: crossbeam_channel::Receiver, +} + +impl pm::bridge::server::MessagePipe for CrossbeamMessagePipe { + fn new() -> (Self, Self) { + let (tx1, rx1) = crossbeam_channel::unbounded(); + let (tx2, rx2) = crossbeam_channel::unbounded(); + (CrossbeamMessagePipe { tx: tx1, rx: rx2 }, CrossbeamMessagePipe { tx: tx2, rx: rx1 }) + } + + fn send(&mut self, value: T) { + self.tx.send(value).unwrap(); + } + + fn recv(&mut self) -> Option { + self.rx.recv().ok() + } +} + +fn exec_strategy(ecx: &ExtCtxt<'_>) -> impl pm::bridge::server::ExecutionStrategy { + >>::new( + ecx.sess.opts.debugging_opts.proc_macro_cross_thread, + ) +} pub struct BangProcMacro { pub client: pm::bridge::client::Client pm::TokenStream>, @@ -27,14 +52,16 @@ impl base::ProcMacro for BangProcMacro { input: TokenStream, ) -> Result { let server = proc_macro_server::Rustc::new(ecx, self.krate); - self.client.run(&EXEC_STRATEGY, server, input, ecx.ecfg.proc_macro_backtrace).map_err(|e| { - let mut err = ecx.struct_span_err(span, "proc macro panicked"); - if let Some(s) = e.as_str() { - err.help(&format!("message: {}", s)); - } - err.emit(); - ErrorReported - }) + self.client.run(&exec_strategy(ecx), server, input, ecx.ecfg.proc_macro_backtrace).map_err( + |e| { + let mut err = ecx.struct_span_err(span, "proc macro panicked"); + if let Some(s) = e.as_str() { + err.help(&format!("message: {}", s)); + } + err.emit(); + ErrorReported + }, + ) } } @@ -53,7 +80,7 @@ impl base::AttrProcMacro for AttrProcMacro { ) -> Result { let server = proc_macro_server::Rustc::new(ecx, self.krate); self.client - .run(&EXEC_STRATEGY, server, annotation, annotated, ecx.ecfg.proc_macro_backtrace) + .run(&exec_strategy(ecx), server, annotation, annotated, ecx.ecfg.proc_macro_backtrace) .map_err(|e| { let mut err = ecx.struct_span_err(span, "custom attribute panicked"); if let Some(s) = e.as_str() { @@ -102,18 +129,22 @@ impl MultiItemModifier for ProcMacroDerive { }; let server = proc_macro_server::Rustc::new(ecx, self.krate); - let stream = - match self.client.run(&EXEC_STRATEGY, server, input, ecx.ecfg.proc_macro_backtrace) { - Ok(stream) => stream, - Err(e) => { - let mut err = ecx.struct_span_err(span, "proc-macro derive panicked"); - if let Some(s) = e.as_str() { - err.help(&format!("message: {}", s)); - } - err.emit(); - return ExpandResult::Ready(vec![]); + let stream = match self.client.run( + &exec_strategy(ecx), + server, + input, + ecx.ecfg.proc_macro_backtrace, + ) { + Ok(stream) => stream, + Err(e) => { + let mut err = ecx.struct_span_err(span, "proc-macro derive panicked"); + if let Some(s) = e.as_str() { + err.help(&format!("message: {}", s)); } - }; + err.emit(); + return ExpandResult::Ready(vec![]); + } + }; let error_count_before = ecx.sess.parse_sess.span_diagnostic.err_count(); let mut parser = diff --git a/compiler/rustc_expand/src/proc_macro_server.rs b/compiler/rustc_expand/src/proc_macro_server.rs index 92315c4d4f6c7..5e09566d97514 100644 --- a/compiler/rustc_expand/src/proc_macro_server.rs +++ b/compiler/rustc_expand/src/proc_macro_server.rs @@ -3,26 +3,24 @@ use crate::base::{ExtCtxt, ResolverExpand}; use rustc_ast as ast; use rustc_ast::token::{self, Nonterminal, NtIdent, TokenKind}; use rustc_ast::tokenstream::{self, CanSynthesizeMissingTokens}; -use rustc_ast::tokenstream::{DelimSpan, Spacing::*, TokenStream, TreeAndSpacing}; +use rustc_ast::tokenstream::{Spacing::*, TokenStream}; use rustc_ast_pretty::pprust; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::sync::Lrc; use rustc_errors::Diagnostic; use rustc_lint_defs::builtin::PROC_MACRO_BACK_COMPAT; use rustc_lint_defs::BuiltinLintDiagnostics; -use rustc_parse::lexer::nfc_normalize; use rustc_parse::{nt_to_tokenstream, parse_stream_from_source_str}; use rustc_session::parse::ParseSess; use rustc_span::def_id::CrateNum; use rustc_span::hygiene::ExpnId; use rustc_span::hygiene::ExpnKind; -use rustc_span::symbol::{self, kw, sym, Symbol}; +use rustc_span::symbol::{self, sym, Symbol}; use rustc_span::{BytePos, FileName, MultiSpan, Pos, RealFileName, SourceFile, Span}; -use pm::bridge::{server, TokenTree}; -use pm::{Delimiter, Level, LineColumn, Spacing}; +use pm::bridge::{server, DelimSpan, Group, Ident, LitKind, Literal, Punct, TokenTree}; +use pm::{Delimiter, Level, LineColumn}; use std::ops::Bound; -use std::{ascii, panic}; trait FromInternal { fn from_internal(x: T) -> Self; @@ -54,166 +52,224 @@ impl ToInternal for Delimiter { } } -impl FromInternal<(TreeAndSpacing, &'_ mut Vec, &mut Rustc<'_>)> - for TokenTree -{ - fn from_internal( - ((tree, spacing), stack, rustc): (TreeAndSpacing, &mut Vec, &mut Rustc<'_>), - ) -> Self { - use rustc_ast::token::*; - - let joint = spacing == Joint; - let Token { kind, span } = match tree { - tokenstream::TokenTree::Delimited(span, delim, tts) => { - let delimiter = Delimiter::from_internal(delim); - return TokenTree::Group(Group { delimiter, stream: tts, span, flatten: false }); - } - tokenstream::TokenTree::Token(token) => token, - }; +impl FromInternal for LitKind { + fn from_internal(kind: token::LitKind) -> Self { + match kind { + token::Byte => LitKind::Byte, + token::Char => LitKind::Char, + token::Integer => LitKind::Integer, + token::Float => LitKind::Float, + token::Str => LitKind::Str, + token::StrRaw(n) => LitKind::StrRaw(n), + token::ByteStr => LitKind::ByteStr, + token::ByteStrRaw(n) => LitKind::ByteStrRaw(n), + token::Err => LitKind::Err, + token::Bool => unreachable!(), + } + } +} - macro_rules! tt { - ($ty:ident { $($field:ident $(: $value:expr)*),+ $(,)? }) => ( - TokenTree::$ty(self::$ty { - $($field $(: $value)*,)+ - span, - }) - ); - ($ty:ident::$method:ident($($value:expr),*)) => ( - TokenTree::$ty(self::$ty::$method($($value,)* span)) - ); +impl ToInternal for LitKind { + fn to_internal(self) -> token::LitKind { + match self { + LitKind::Byte => token::Byte, + LitKind::Char => token::Char, + LitKind::Integer => token::Integer, + LitKind::Float => token::Float, + LitKind::Str => token::Str, + LitKind::StrRaw(n) => token::StrRaw(n), + LitKind::ByteStr => token::ByteStr, + LitKind::ByteStrRaw(n) => token::ByteStrRaw(n), + LitKind::Err => token::Err, } - macro_rules! op { - ($a:expr) => { - tt!(Punct::new($a, joint)) + } +} + +impl FromInternal<(TokenStream, &mut Rustc<'_>)> for Vec> { + fn from_internal((stream, rustc): (TokenStream, &mut Rustc<'_>)) -> Self { + use rustc_ast::token::*; + + let mut cursor = stream.into_trees(); + let mut trees = Vec::new(); + + while let Some((tree, spacing)) = cursor.next_with_spacing() { + let joint = spacing == Joint; + let Token { kind, span } = match tree { + tokenstream::TokenTree::Delimited(span, delim, tts) => { + let delimiter = Delimiter::from_internal(delim); + trees.push(TokenTree::Group(Group { + delimiter, + stream: Some(tts), + span: DelimSpan { + open: span.open, + close: span.close, + entire: span.entire(), + }, + })); + continue; + } + tokenstream::TokenTree::Token(token) => token, }; - ($a:expr, $b:expr) => {{ - stack.push(tt!(Punct::new($b, joint))); - tt!(Punct::new($a, true)) - }}; - ($a:expr, $b:expr, $c:expr) => {{ - stack.push(tt!(Punct::new($c, joint))); - stack.push(tt!(Punct::new($b, true))); - tt!(Punct::new($a, true)) - }}; - } - match kind { - Eq => op!('='), - Lt => op!('<'), - Le => op!('<', '='), - EqEq => op!('=', '='), - Ne => op!('!', '='), - Ge => op!('>', '='), - Gt => op!('>'), - AndAnd => op!('&', '&'), - OrOr => op!('|', '|'), - Not => op!('!'), - Tilde => op!('~'), - BinOp(Plus) => op!('+'), - BinOp(Minus) => op!('-'), - BinOp(Star) => op!('*'), - BinOp(Slash) => op!('/'), - BinOp(Percent) => op!('%'), - BinOp(Caret) => op!('^'), - BinOp(And) => op!('&'), - BinOp(Or) => op!('|'), - BinOp(Shl) => op!('<', '<'), - BinOp(Shr) => op!('>', '>'), - BinOpEq(Plus) => op!('+', '='), - BinOpEq(Minus) => op!('-', '='), - BinOpEq(Star) => op!('*', '='), - BinOpEq(Slash) => op!('/', '='), - BinOpEq(Percent) => op!('%', '='), - BinOpEq(Caret) => op!('^', '='), - BinOpEq(And) => op!('&', '='), - BinOpEq(Or) => op!('|', '='), - BinOpEq(Shl) => op!('<', '<', '='), - BinOpEq(Shr) => op!('>', '>', '='), - At => op!('@'), - Dot => op!('.'), - DotDot => op!('.', '.'), - DotDotDot => op!('.', '.', '.'), - DotDotEq => op!('.', '.', '='), - Comma => op!(','), - Semi => op!(';'), - Colon => op!(':'), - ModSep => op!(':', ':'), - RArrow => op!('-', '>'), - LArrow => op!('<', '-'), - FatArrow => op!('=', '>'), - Pound => op!('#'), - Dollar => op!('$'), - Question => op!('?'), - SingleQuote => op!('\''), - - Ident(name, false) if name == kw::DollarCrate => tt!(Ident::dollar_crate()), - Ident(name, is_raw) => tt!(Ident::new(rustc.sess, name, is_raw)), - Lifetime(name) => { - let ident = symbol::Ident::new(name, span).without_first_quote(); - stack.push(tt!(Ident::new(rustc.sess, ident.name, false))); - tt!(Punct::new('\'', true)) + macro_rules! tt { + ($ty:ident { $($field:ident $(: $value:expr)*),+ $(,)? }) => ( + trees.push(TokenTree::$ty(self::$ty { + $($field $(: $value)*,)+ + span, + })) + ); + ($ty:ident::$method:ident($($value:expr),*)) => ( + trees.push(TokenTree::$ty(self::$ty::$method($($value,)* span))) + ); } - Literal(lit) => tt!(Literal { lit }), - DocComment(_, attr_style, data) => { - let mut escaped = String::new(); - for ch in data.as_str().chars() { - escaped.extend(ch.escape_debug()); - } - let stream = vec![ - Ident(sym::doc, false), - Eq, - TokenKind::lit(token::Str, Symbol::intern(&escaped), None), - ] - .into_iter() - .map(|kind| tokenstream::TokenTree::token(kind, span)) - .collect(); - stack.push(TokenTree::Group(Group { - delimiter: Delimiter::Bracket, - stream, - span: DelimSpan::from_single(span), - flatten: false, - })); - if attr_style == ast::AttrStyle::Inner { - stack.push(tt!(Punct::new('!', false))); - } - tt!(Punct::new('#', false)) + macro_rules! op { + ($a:expr) => {{ + tt!(Punct { ch: $a, joint }); + }}; + ($a:expr, $b:expr) => {{ + tt!(Punct { ch: $a, joint: true }); + tt!(Punct { ch: $b, joint }); + }}; + ($a:expr, $b:expr, $c:expr) => {{ + tt!(Punct { ch: $a, joint: true }); + tt!(Punct { ch: $b, joint: true }); + tt!(Punct { ch: $c, joint }); + }}; } - Interpolated(nt) => { - if let Some((name, is_raw)) = ident_name_compatibility_hack(&nt, span, rustc) { - TokenTree::Ident(Ident::new(rustc.sess, name.name, is_raw, name.span)) - } else { - let stream = nt_to_tokenstream(&nt, rustc.sess, CanSynthesizeMissingTokens::No); - TokenTree::Group(Group { - delimiter: Delimiter::None, - stream, + match kind { + Eq => op!('='), + Lt => op!('<'), + Le => op!('<', '='), + EqEq => op!('=', '='), + Ne => op!('!', '='), + Ge => op!('>', '='), + Gt => op!('>'), + AndAnd => op!('&', '&'), + OrOr => op!('|', '|'), + Not => op!('!'), + Tilde => op!('~'), + BinOp(Plus) => op!('+'), + BinOp(Minus) => op!('-'), + BinOp(Star) => op!('*'), + BinOp(Slash) => op!('/'), + BinOp(Percent) => op!('%'), + BinOp(Caret) => op!('^'), + BinOp(And) => op!('&'), + BinOp(Or) => op!('|'), + BinOp(Shl) => op!('<', '<'), + BinOp(Shr) => op!('>', '>'), + BinOpEq(Plus) => op!('+', '='), + BinOpEq(Minus) => op!('-', '='), + BinOpEq(Star) => op!('*', '='), + BinOpEq(Slash) => op!('/', '='), + BinOpEq(Percent) => op!('%', '='), + BinOpEq(Caret) => op!('^', '='), + BinOpEq(And) => op!('&', '='), + BinOpEq(Or) => op!('|', '='), + BinOpEq(Shl) => op!('<', '<', '='), + BinOpEq(Shr) => op!('>', '>', '='), + At => op!('@'), + Dot => op!('.'), + DotDot => op!('.', '.'), + DotDotDot => op!('.', '.', '.'), + DotDotEq => op!('.', '.', '='), + Comma => op!(','), + Semi => op!(';'), + Colon => op!(':'), + ModSep => op!(':', ':'), + RArrow => op!('-', '>'), + LArrow => op!('<', '-'), + FatArrow => op!('=', '>'), + Pound => op!('#'), + Dollar => op!('$'), + Question => op!('?'), + SingleQuote => op!('\''), + + Ident(sym, is_raw) => tt!(Ident { sym, is_raw }), + Lifetime(name) => { + let ident = symbol::Ident::new(name, span).without_first_quote(); + tt!(Punct { ch: '\'', joint: true }); + tt!(Ident { sym: ident.name, is_raw: false }); + } + Literal(token::Lit { kind, symbol, suffix }) => { + tt!(Literal { kind: FromInternal::from_internal(kind), symbol, suffix }); + } + DocComment(_, attr_style, data) => { + let mut escaped = String::new(); + for ch in data.as_str().chars() { + escaped.extend(ch.escape_debug()); + } + let stream = vec![ + Ident(sym::doc, false), + Eq, + TokenKind::lit(token::Str, Symbol::intern(&escaped), None), + ] + .into_iter() + .map(|kind| tokenstream::TokenTree::token(kind, span)) + .collect(); + tt!(Punct { ch: '#', joint: false }); + if attr_style == ast::AttrStyle::Inner { + tt!(Punct { ch: '!', joint: false }); + } + trees.push(TokenTree::Group(Group { + delimiter: Delimiter::Bracket, + stream: Some(stream), span: DelimSpan::from_single(span), - flatten: crate::base::pretty_printing_compatibility_hack(&nt, rustc.sess), - }) + })); } - } - OpenDelim(..) | CloseDelim(..) => unreachable!(), - Eof => unreachable!(), + Interpolated(nt) => { + if let Some((name, is_raw)) = ident_name_compatibility_hack(&nt, span, rustc) { + trees.push(TokenTree::Ident(Ident { + sym: name.name, + is_raw, + span: name.span, + })); + } else { + let stream = + nt_to_tokenstream(&nt, rustc.sess, CanSynthesizeMissingTokens::No); + if crate::base::pretty_printing_compatibility_hack(&nt, rustc.sess) { + cursor.append(stream); + } else { + trees.push(TokenTree::Group(Group { + delimiter: Delimiter::None, + stream: Some(stream), + span: DelimSpan::from_single(span), + })) + } + } + } + + OpenDelim(..) | CloseDelim(..) => unreachable!(), + Eof => unreachable!(), + } } + trees } } -impl ToInternal for TokenTree { +impl ToInternal for TokenTree { fn to_internal(self) -> TokenStream { use rustc_ast::token::*; let (ch, joint, span) = match self { TokenTree::Punct(Punct { ch, joint, span }) => (ch, joint, span), - TokenTree::Group(Group { delimiter, stream, span, .. }) => { - return tokenstream::TokenTree::Delimited(span, delimiter.to_internal(), stream) - .into(); + TokenTree::Group(Group { delimiter, stream, span: DelimSpan { open, close, .. } }) => { + return tokenstream::TokenTree::Delimited( + tokenstream::DelimSpan { open, close }, + delimiter.to_internal(), + stream.unwrap_or_default(), + ) + .into(); } TokenTree::Ident(self::Ident { sym, is_raw, span }) => { return tokenstream::TokenTree::token(Ident(sym, is_raw), span).into(); } TokenTree::Literal(self::Literal { - lit: token::Lit { kind: token::Integer, symbol, suffix }, + kind: self::LitKind::Integer, + symbol, + suffix, span, }) if symbol.as_str().starts_with('-') => { let minus = BinOp(BinOpToken::Minus); @@ -224,7 +280,9 @@ impl ToInternal for TokenTree { return vec![a, b].into_iter().collect(); } TokenTree::Literal(self::Literal { - lit: token::Lit { kind: token::Float, symbol, suffix }, + kind: self::LitKind::Float, + symbol, + suffix, span, }) if symbol.as_str().starts_with('-') => { let minus = BinOp(BinOpToken::Minus); @@ -234,8 +292,12 @@ impl ToInternal for TokenTree { let b = tokenstream::TokenTree::token(float, span); return vec![a, b].into_iter().collect(); } - TokenTree::Literal(self::Literal { lit, span }) => { - return tokenstream::TokenTree::token(Literal(lit), span).into(); + TokenTree::Literal(self::Literal { kind, symbol, suffix, span }) => { + return tokenstream::TokenTree::token( + TokenKind::lit(kind.to_internal(), symbol, suffix), + span, + ) + .into(); } }; @@ -284,77 +346,6 @@ impl ToInternal for Level { pub struct FreeFunctions; -#[derive(Clone)] -pub struct TokenStreamIter { - cursor: tokenstream::Cursor, - stack: Vec>, -} - -#[derive(Clone)] -pub struct Group { - delimiter: Delimiter, - stream: TokenStream, - span: DelimSpan, - /// A hack used to pass AST fragments to attribute and derive macros - /// as a single nonterminal token instead of a token stream. - /// FIXME: It needs to be removed, but there are some compatibility issues (see #73345). - flatten: bool, -} - -#[derive(Copy, Clone, PartialEq, Eq, Hash)] -pub struct Punct { - ch: char, - // NB. not using `Spacing` here because it doesn't implement `Hash`. - joint: bool, - span: Span, -} - -impl Punct { - fn new(ch: char, joint: bool, span: Span) -> Punct { - const LEGAL_CHARS: &[char] = &[ - '=', '<', '>', '!', '~', '+', '-', '*', '/', '%', '^', '&', '|', '@', '.', ',', ';', - ':', '#', '$', '?', '\'', - ]; - if !LEGAL_CHARS.contains(&ch) { - panic!("unsupported character `{:?}`", ch) - } - Punct { ch, joint, span } - } -} - -#[derive(Copy, Clone, PartialEq, Eq, Hash)] -pub struct Ident { - sym: Symbol, - is_raw: bool, - span: Span, -} - -impl Ident { - fn new(sess: &ParseSess, sym: Symbol, is_raw: bool, span: Span) -> Ident { - let sym = nfc_normalize(&sym.as_str()); - let string = sym.as_str(); - if !rustc_lexer::is_ident(&string) { - panic!("`{:?}` is not a valid identifier", string) - } - if is_raw && !sym.can_be_raw() { - panic!("`{}` cannot be a raw identifier", string); - } - sess.symbol_gallery.insert(sym, span); - Ident { sym, is_raw, span } - } - fn dollar_crate(span: Span) -> Ident { - // `$crate` is accepted as an ident only if it comes from the compiler. - Ident { sym: kw::DollarCrate, is_raw: false, span } - } -} - -// FIXME(eddyb) `Literal` should not expose internal `Debug` impls. -#[derive(Clone, Debug)] -pub struct Literal { - lit: token::Lit, - span: Span, -} - pub(crate) struct Rustc<'a> { resolver: &'a dyn ResolverExpand, sess: &'a ParseSess, @@ -386,160 +377,24 @@ impl<'a> Rustc<'a> { rebased_spans: FxHashMap::default(), } } - - fn lit(&mut self, kind: token::LitKind, symbol: Symbol, suffix: Option) -> Literal { - Literal { lit: token::Lit::new(kind, symbol, suffix), span: server::Span::call_site(self) } - } } impl server::Types for Rustc<'_> { type FreeFunctions = FreeFunctions; type TokenStream = TokenStream; - type TokenStreamBuilder = tokenstream::TokenStreamBuilder; - type TokenStreamIter = TokenStreamIter; - type Group = Group; - type Punct = Punct; - type Ident = Ident; - type Literal = Literal; type SourceFile = Lrc; type MultiSpan = Vec; type Diagnostic = Diagnostic; type Span = Span; + type Symbol = Symbol; } impl server::FreeFunctions for Rustc<'_> { fn track_env_var(&mut self, var: &str, value: Option<&str>) { self.sess.env_depinfo.borrow_mut().insert((Symbol::intern(var), value.map(Symbol::intern))); } -} -impl server::TokenStream for Rustc<'_> { - fn new(&mut self) -> Self::TokenStream { - TokenStream::default() - } - fn is_empty(&mut self, stream: &Self::TokenStream) -> bool { - stream.is_empty() - } - fn from_str(&mut self, src: &str) -> Self::TokenStream { - parse_stream_from_source_str( - FileName::proc_macro_source_code(src), - src.to_string(), - self.sess, - Some(self.call_site), - ) - } - fn to_string(&mut self, stream: &Self::TokenStream) -> String { - pprust::tts_to_string(stream) - } - fn from_token_tree( - &mut self, - tree: TokenTree, - ) -> Self::TokenStream { - tree.to_internal() - } - fn into_iter(&mut self, stream: Self::TokenStream) -> Self::TokenStreamIter { - TokenStreamIter { cursor: stream.trees(), stack: vec![] } - } -} - -impl server::TokenStreamBuilder for Rustc<'_> { - fn new(&mut self) -> Self::TokenStreamBuilder { - tokenstream::TokenStreamBuilder::new() - } - fn push(&mut self, builder: &mut Self::TokenStreamBuilder, stream: Self::TokenStream) { - builder.push(stream); - } - fn build(&mut self, builder: Self::TokenStreamBuilder) -> Self::TokenStream { - builder.build() - } -} - -impl server::TokenStreamIter for Rustc<'_> { - fn next( - &mut self, - iter: &mut Self::TokenStreamIter, - ) -> Option> { - loop { - let tree = iter.stack.pop().or_else(|| { - let next = iter.cursor.next_with_spacing()?; - Some(TokenTree::from_internal((next, &mut iter.stack, self))) - })?; - // A hack used to pass AST fragments to attribute and derive macros - // as a single nonterminal token instead of a token stream. - // Such token needs to be "unwrapped" and not represented as a delimited group. - // FIXME: It needs to be removed, but there are some compatibility issues (see #73345). - if let TokenTree::Group(ref group) = tree { - if group.flatten { - iter.cursor.append(group.stream.clone()); - continue; - } - } - return Some(tree); - } - } -} - -impl server::Group for Rustc<'_> { - fn new(&mut self, delimiter: Delimiter, stream: Self::TokenStream) -> Self::Group { - Group { - delimiter, - stream, - span: DelimSpan::from_single(server::Span::call_site(self)), - flatten: false, - } - } - fn delimiter(&mut self, group: &Self::Group) -> Delimiter { - group.delimiter - } - fn stream(&mut self, group: &Self::Group) -> Self::TokenStream { - group.stream.clone() - } - fn span(&mut self, group: &Self::Group) -> Self::Span { - group.span.entire() - } - fn span_open(&mut self, group: &Self::Group) -> Self::Span { - group.span.open - } - fn span_close(&mut self, group: &Self::Group) -> Self::Span { - group.span.close - } - fn set_span(&mut self, group: &mut Self::Group, span: Self::Span) { - group.span = DelimSpan::from_single(span); - } -} - -impl server::Punct for Rustc<'_> { - fn new(&mut self, ch: char, spacing: Spacing) -> Self::Punct { - Punct::new(ch, spacing == Spacing::Joint, server::Span::call_site(self)) - } - fn as_char(&mut self, punct: Self::Punct) -> char { - punct.ch - } - fn spacing(&mut self, punct: Self::Punct) -> Spacing { - if punct.joint { Spacing::Joint } else { Spacing::Alone } - } - fn span(&mut self, punct: Self::Punct) -> Self::Span { - punct.span - } - fn with_span(&mut self, punct: Self::Punct, span: Self::Span) -> Self::Punct { - Punct { span, ..punct } - } -} - -impl server::Ident for Rustc<'_> { - fn new(&mut self, string: &str, span: Self::Span, is_raw: bool) -> Self::Ident { - Ident::new(self.sess, Symbol::intern(string), is_raw, span) - } - fn span(&mut self, ident: Self::Ident) -> Self::Span { - ident.span - } - fn with_span(&mut self, ident: Self::Ident, span: Self::Span) -> Self::Ident { - Ident { span, ..ident } - } -} - -impl server::Literal for Rustc<'_> { - fn from_str(&mut self, s: &str) -> Result { + fn literal_from_str(&mut self, s: &str) -> Result, ()> { let override_span = None; let stream = parse_stream_from_source_str( FileName::proc_macro_source_code(s), @@ -560,66 +415,21 @@ impl server::Literal for Rustc<'_> { // There is a comment or whitespace adjacent to the literal. return Err(()); } - let lit = match token.kind { + let token::Lit { kind, symbol, suffix } = match token.kind { TokenKind::Literal(lit) => lit, _ => return Err(()), }; - Ok(Literal { lit, span: self.call_site }) - } - fn debug_kind(&mut self, literal: &Self::Literal) -> String { - format!("{:?}", literal.lit.kind) - } - fn symbol(&mut self, literal: &Self::Literal) -> String { - literal.lit.symbol.to_string() - } - fn suffix(&mut self, literal: &Self::Literal) -> Option { - literal.lit.suffix.as_ref().map(Symbol::to_string) - } - fn integer(&mut self, n: &str) -> Self::Literal { - self.lit(token::Integer, Symbol::intern(n), None) - } - fn typed_integer(&mut self, n: &str, kind: &str) -> Self::Literal { - self.lit(token::Integer, Symbol::intern(n), Some(Symbol::intern(kind))) - } - fn float(&mut self, n: &str) -> Self::Literal { - self.lit(token::Float, Symbol::intern(n), None) - } - fn f32(&mut self, n: &str) -> Self::Literal { - self.lit(token::Float, Symbol::intern(n), Some(sym::f32)) - } - fn f64(&mut self, n: &str) -> Self::Literal { - self.lit(token::Float, Symbol::intern(n), Some(sym::f64)) - } - fn string(&mut self, string: &str) -> Self::Literal { - let mut escaped = String::new(); - for ch in string.chars() { - escaped.extend(ch.escape_debug()); - } - self.lit(token::Str, Symbol::intern(&escaped), None) - } - fn character(&mut self, ch: char) -> Self::Literal { - let mut escaped = String::new(); - escaped.extend(ch.escape_unicode()); - self.lit(token::Char, Symbol::intern(&escaped), None) - } - fn byte_string(&mut self, bytes: &[u8]) -> Self::Literal { - let string = bytes - .iter() - .cloned() - .flat_map(ascii::escape_default) - .map(Into::::into) - .collect::(); - self.lit(token::ByteStr, Symbol::intern(&string), None) - } - fn span(&mut self, literal: &Self::Literal) -> Self::Span { - literal.span - } - fn set_span(&mut self, literal: &mut Self::Literal, span: Self::Span) { - literal.span = span; + Ok(Literal { + kind: FromInternal::from_internal(kind), + symbol, + suffix, + span: self.call_site, + }) } - fn subspan( + + fn literal_subspan( &mut self, - literal: &Self::Literal, + literal: Literal, start: Bound, end: Bound, ) -> Option { @@ -655,6 +465,63 @@ impl server::Literal for Rustc<'_> { } } +impl server::TokenStream for Rustc<'_> { + fn is_empty(&mut self, stream: &Self::TokenStream) -> bool { + stream.is_empty() + } + fn from_str(&mut self, src: &str) -> Self::TokenStream { + parse_stream_from_source_str( + FileName::proc_macro_source_code(src), + src.to_string(), + self.sess, + Some(self.call_site), + ) + } + fn to_string(&mut self, stream: &Self::TokenStream) -> String { + pprust::tts_to_string(stream) + } + fn from_token_tree( + &mut self, + tree: TokenTree, + ) -> Self::TokenStream { + tree.to_internal() + } + fn concat_trees( + &mut self, + base: Option, + trees: Vec>, + ) -> Self::TokenStream { + let mut builder = tokenstream::TokenStreamBuilder::new(); + if let Some(base) = base { + builder.push(base); + } + for tree in trees { + builder.push(tree.to_internal()); + } + builder.build() + } + fn concat_streams( + &mut self, + base: Option, + streams: Vec, + ) -> Self::TokenStream { + let mut builder = tokenstream::TokenStreamBuilder::new(); + if let Some(base) = base { + builder.push(base); + } + for stream in streams { + builder.push(stream); + } + builder.build() + } + fn into_iter( + &mut self, + stream: Self::TokenStream, + ) -> Vec> { + FromInternal::from_internal((stream, self)) + } +} + impl server::SourceFile for Rustc<'_> { fn eq(&mut self, file1: &Self::SourceFile, file2: &Self::SourceFile) -> bool { Lrc::ptr_eq(file1, file2) @@ -712,15 +579,6 @@ impl server::Span for Rustc<'_> { format!("{:?} bytes({}..{})", span.ctxt(), span.lo().0, span.hi().0) } } - fn def_site(&mut self) -> Self::Span { - self.def_site - } - fn call_site(&mut self) -> Self::Span { - self.call_site - } - fn mixed_site(&mut self) -> Self::Span { - self.mixed_site - } fn source_file(&mut self, span: Self::Span) -> Self::SourceFile { self.sess.source_map().lookup_char_pos(span.lo()).file } @@ -801,6 +659,40 @@ impl server::Span for Rustc<'_> { } } +impl server::Context for Rustc<'_> { + fn def_site(&mut self) -> Self::Span { + self.def_site + } + fn call_site(&mut self) -> Self::Span { + self.call_site + } + fn mixed_site(&mut self) -> Self::Span { + self.mixed_site + } + + // NOTE: May be run on any thread, so cannot use `nfc_normalize` + fn validate_ident(s: &str) -> Result, ()> { + use unicode_normalization::{is_nfc_quick, IsNormalized, UnicodeNormalization}; + let normalized: Option = match is_nfc_quick(s.chars()) { + IsNormalized::Yes => None, + _ => Some(s.chars().nfc().collect()), + }; + if rustc_lexer::is_ident(normalized.as_ref().map(|s| &s[..]).unwrap_or(s)) { + Ok(normalized) + } else { + Err(()) + } + } + + fn intern_symbol(string: &str) -> Self::Symbol { + Symbol::intern(string) + } + + fn with_symbol_string(symbol: &Self::Symbol, f: impl FnOnce(&str)) { + f(&symbol.as_str()) + } +} + // See issue #74616 for details fn ident_name_compatibility_hack( nt: &Nonterminal, diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs index 4c40d0c367eca..04f1f063a0126 100644 --- a/compiler/rustc_session/src/options.rs +++ b/compiler/rustc_session/src/options.rs @@ -1207,6 +1207,8 @@ options! { "print layout information for each type encountered (default: no)"), proc_macro_backtrace: bool = (false, parse_bool, [UNTRACKED], "show backtraces for panics during proc-macro execution (default: no)"), + proc_macro_cross_thread: bool = (false, parse_bool, [UNTRACKED], + "run proc-macro code on a separate thread (default: no)"), profile: bool = (false, parse_bool, [TRACKED], "insert profiling code (default: no)"), profile_closures: bool = (false, parse_no_flag, [UNTRACKED], diff --git a/library/proc_macro/src/bridge/buffer.rs b/library/proc_macro/src/bridge/buffer.rs index d82669d3e2336..ea5de80da12b2 100644 --- a/library/proc_macro/src/bridge/buffer.rs +++ b/library/proc_macro/src/bridge/buffer.rs @@ -5,6 +5,35 @@ use std::mem; use std::ops::{Deref, DerefMut}; use std::slice; +#[repr(C)] +pub struct Slice<'a, T> { + data: &'a [T; 0], + len: usize, +} + +unsafe impl<'a, T: Sync> Sync for Slice<'a, T> {} +unsafe impl<'a, T: Sync> Send for Slice<'a, T> {} + +impl Copy for Slice<'a, T> {} +impl Clone for Slice<'a, T> { + fn clone(&self) -> Self { + *self + } +} + +impl From<&'a [T]> for Slice<'a, T> { + fn from(xs: &'a [T]) -> Self { + Slice { data: unsafe { &*(xs.as_ptr() as *const [T; 0]) }, len: xs.len() } + } +} + +impl Deref for Slice<'a, T> { + type Target = [T]; + fn deref(&self) -> &[T] { + unsafe { slice::from_raw_parts(self.data.as_ptr(), self.len) } + } +} + #[repr(C)] pub struct Buffer { data: *mut T, diff --git a/library/proc_macro/src/bridge/client.rs b/library/proc_macro/src/bridge/client.rs index c6bec5a6fbdc2..c2a0886367190 100644 --- a/library/proc_macro/src/bridge/client.rs +++ b/library/proc_macro/src/bridge/client.rs @@ -2,6 +2,24 @@ use super::*; +use std::rc::Rc; +use std::sync::atomic::Ordering; + +trait OwnedHandle { + /// Create a new handle of this type from the client side, which may be used + /// later by the server. + /// + /// Should only be called on the client. + fn next_raw_handle() -> handle::Handle; + + /// Create an instance of the owning handle object for this raw handle. The + /// raw handle should've previously been created with `next_raw_handle`, and + /// the corresponding message should've been sent to the server. + /// + /// Should only be called on the client. + fn from_raw_handle(handle: handle::Handle) -> Self; +} + macro_rules! define_handles { ( 'owned: $($oty:ident,)* @@ -116,6 +134,25 @@ macro_rules! define_handles { $oty(handle::Handle::decode(r, s)) } } + + impl server::InitOwnedHandle>> + for Marked + { + fn init_handle(self, raw_handle: handle::Handle, s: &mut HandleStore>) { + s.$oty.init(raw_handle, self); + } + } + + impl OwnedHandle for $oty { + fn next_raw_handle() -> handle::Handle { + let counter = HandleCounters::get().$oty.fetch_add(1, Ordering::SeqCst); + handle::Handle::new(counter as u32).expect("`proc_macro` handle counter overflowed") + } + + fn from_raw_handle(handle: handle::Handle) -> $oty { + $oty(handle) + } + } )* $( @@ -143,7 +180,7 @@ macro_rules! define_handles { for Marked { fn encode(self, w: &mut Writer, s: &mut HandleStore>) { - s.$ity.alloc(self).encode(w, s); + s.$ity.alloc(&self).encode(w, s); } } @@ -159,17 +196,11 @@ define_handles! { 'owned: FreeFunctions, TokenStream, - TokenStreamBuilder, - TokenStreamIter, - Group, - Literal, SourceFile, MultiSpan, Diagnostic, 'interned: - Punct, - Ident, Span, } @@ -185,76 +216,197 @@ impl Clone for TokenStream { } } -impl Clone for TokenStreamIter { +impl Clone for SourceFile { fn clone(&self) -> Self { self.clone() } } -impl Clone for Group { - fn clone(&self) -> Self { - self.clone() +impl Span { + pub(crate) fn def_site() -> Span { + Bridge::with(|bridge| bridge.context.def_site) } -} -impl Clone for Literal { - fn clone(&self) -> Self { - self.clone() + pub(crate) fn call_site() -> Span { + Bridge::with(|bridge| bridge.context.call_site) + } + + pub(crate) fn mixed_site() -> Span { + Bridge::with(|bridge| bridge.context.mixed_site) } } -impl fmt::Debug for Literal { +impl fmt::Debug for Span { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("Literal") - // format the kind without quotes, as in `kind: Float` - .field("kind", &format_args!("{}", &self.debug_kind())) - .field("symbol", &self.symbol()) - // format `Some("...")` on one line even in {:#?} mode - .field("suffix", &format_args!("{:?}", &self.suffix())) - .field("span", &self.span()) - .finish() + f.write_str(&self.debug()) } } -impl Clone for SourceFile { - fn clone(&self) -> Self { - self.clone() +#[repr(C)] +#[derive(Copy, Clone, PartialEq, Eq, Hash)] +pub(crate) struct Symbol(handle::Handle); + +impl Symbol { + /// Intern a new `Symbol` + pub(crate) fn new(string: &str) -> Self { + Symbol(Bridge::with(|bridge| bridge.symbols.alloc(string))) + } + + /// Create a new `Symbol` for an identifier. + /// + /// Validates and normalizes before converting it to a symbol. + pub(crate) fn new_ident(string: &str, is_raw: bool) -> Self { + Symbol(Bridge::with(|bridge| { + let mut normalized = Buffer::new(); + if !(bridge.validate_ident)(string.as_bytes().into(), &mut normalized) { + panic!("`{:?}` is not a valid identifier", string) + } + let string = if normalized.len() > 0 { + std::str::from_utf8(&normalized[..]).unwrap() + } else { + string + }; + if is_raw && !Self::can_be_raw(string) { + panic!("`{:?}` cannot be a raw identifier", string); + } + bridge.symbols.alloc(string) + })) + } + + // Mimics the behaviour of `Symbol::can_be_raw` from `rustc_span` + fn can_be_raw(string: &str) -> bool { + match string { + "" | "_" | "super" | "self" | "Self" | "crate" | "$crate" | "{{root}}" => false, + _ => true, + } } } -impl fmt::Debug for Span { +impl !Send for Symbol {} +impl !Sync for Symbol {} + +impl fmt::Debug for Symbol { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(&self.debug()) + Bridge::with(|bridge| fmt::Debug::fmt(&bridge.symbols[self.0], f)) } } -macro_rules! define_client_side { - ($($name:ident { - $(fn $method:ident($($arg:ident: $arg_ty:ty),* $(,)?) $(-> $ret_ty:ty)*;)* - }),* $(,)?) => { - $(impl $name { - $(pub(crate) fn $method($($arg: $arg_ty),*) $(-> $ret_ty)* { - Bridge::with(|bridge| { - let mut b = bridge.cached_buffer.take(); +impl fmt::Display for Symbol { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + Bridge::with(|bridge| fmt::Display::fmt(&bridge.symbols[self.0], f)) + } +} + +impl Encode> for Symbol { + fn encode(self, w: &mut Writer, s: &mut Bridge<'_>) { + s.symbols[self.0][..].encode(w, &mut ()); + } +} + +impl DecodeMut<'_, '_, HandleStore>> + for Marked +{ + fn decode(r: &mut Reader<'_>, s: &mut HandleStore>) -> Self { + Mark::mark(S::intern_symbol(<&str>::decode(r, s))) + } +} + +impl Encode>> + for Marked +{ + fn encode(self, w: &mut Writer, s: &mut HandleStore>) { + S::with_symbol_string(&self.unmark(), |sym| sym.encode(w, s)) + } +} + +impl DecodeMut<'_, '_, Bridge<'_>> for Symbol { + fn decode(r: &mut Reader<'_>, s: &mut Bridge<'_>) -> Self { + Symbol(s.symbols.alloc(<&str>::decode(r, &mut ()))) + } +} - b.clear(); - api_tags::Method::$name(api_tags::$name::$method).encode(&mut b, &mut ()); - reverse_encode!(b; $($arg),*); +macro_rules! client_send_impl { + (wait $name:ident :: $method:ident($($arg:ident),*) $(-> $ret_ty:ty)?) => { + Bridge::with(|bridge| { + let mut b = bridge.cached_buffer.take(); - b = bridge.dispatch.call(b); + b.clear(); + api_tags::Method::$name(api_tags::$name::$method).encode(&mut b, &mut ()); + reverse_encode!(b, bridge; $($arg),*); + + b = bridge.dispatch.call(b); - let r = Result::<_, PanicMessage>::decode(&mut &b[..], &mut ()); + let r = Result::<_, PanicMessage>::decode(&mut &b[..], bridge); - bridge.cached_buffer = b; + bridge.cached_buffer = b; + + r.unwrap_or_else(|e| panic::resume_unwind(e.into())) + }) + }; + + (nowait $name:ident :: $method:ident($($arg:ident),*) $(-> $ret_ty:ty)?) => { + Bridge::with(|bridge| { + let mut b = bridge.cached_buffer.take(); + + b.clear(); + api_tags::Method::$name(api_tags::$name::$method).encode(&mut b, &mut ()); + reverse_encode!(b, bridge; $($arg),*); - r.unwrap_or_else(|e| panic::resume_unwind(e.into())) - }) + $( + let raw_handle = <$ret_ty as OwnedHandle>::next_raw_handle(); + raw_handle.encode(&mut b, &mut ()); + )? + + b = bridge.dispatch.call(b); + + let r = if b.len() > 0 { + Result::<(), PanicMessage>::decode(&mut &b[..], &mut ()) + } else { + Ok(()) + }; + + bridge.cached_buffer = b; + + r.unwrap_or_else(|e| panic::resume_unwind(e.into())); + $(<$ret_ty as OwnedHandle>::from_raw_handle(raw_handle))? + }) + }; +} + +macro_rules! define_client_side { + ($($name:ident { + $($wait:ident fn $method:ident($($arg:ident: $arg_ty:ty),* $(,)?) $(-> $ret_ty:ty)?;)* + }),* $(,)?) => { + $(impl $name { + $(pub(crate) fn $method($($arg: $arg_ty),*) $(-> $ret_ty)? { + client_send_impl!($wait $name :: $method ($($arg),*) $(-> $ret_ty)?) })* })* } } with_api!(self, self, define_client_side); +struct Bridge<'a> { + /// Reusable buffer (only `clear`-ed, never shrunk), primarily + /// used for making requests. + cached_buffer: Buffer, + + /// Server-side function that the client uses to make requests. + dispatch: closure::Closure<'a, Buffer, Buffer>, + + /// Server-side function to validate and normalize an ident. + validate_ident: extern "C" fn(buffer::Slice<'_, u8>, &mut Buffer) -> bool, + + /// Interned store for storing symbols within the client. + symbols: handle::InternedStore>, + + /// Provided context for this macro expansion. + context: ExpnContext, +} + +impl<'a> !Send for Bridge<'a> {} +impl<'a> !Sync for Bridge<'a> {} + enum BridgeState<'a> { /// No server is currently connected to this client. NotConnected, @@ -297,34 +449,6 @@ impl BridgeState<'_> { } impl Bridge<'_> { - pub(crate) fn is_available() -> bool { - BridgeState::with(|state| match state { - BridgeState::Connected(_) | BridgeState::InUse => true, - BridgeState::NotConnected => false, - }) - } - - fn enter(self, f: impl FnOnce() -> R) -> R { - let force_show_panics = self.force_show_panics; - // Hide the default panic output within `proc_macro` expansions. - // NB. the server can't do this because it may use a different libstd. - static HIDE_PANICS_DURING_EXPANSION: Once = Once::new(); - HIDE_PANICS_DURING_EXPANSION.call_once(|| { - let prev = panic::take_hook(); - panic::set_hook(Box::new(move |info| { - let show = BridgeState::with(|state| match state { - BridgeState::NotConnected => true, - BridgeState::Connected(_) | BridgeState::InUse => force_show_panics, - }); - if show { - prev(info) - } - })); - }); - - BRIDGE_STATE.with(|state| state.set(BridgeState::Connected(self), f)) - } - fn with(f: impl FnOnce(&mut Bridge<'_>) -> R) -> R { BridgeState::with(|state| match state { BridgeState::NotConnected => { @@ -338,6 +462,13 @@ impl Bridge<'_> { } } +pub(crate) fn is_available() -> bool { + BridgeState::with(|state| match state { + BridgeState::Connected(_) | BridgeState::InUse => true, + BridgeState::NotConnected => false, + }) +} + /// A client-side "global object" (usually a function pointer), /// which may be using a different `proc_macro` from the one /// used by the server, but can be interacted with compatibly. @@ -352,44 +483,73 @@ pub struct Client { // FIXME(eddyb) use a reference to the `static COUNTERS`, instead of // a wrapper `fn` pointer, once `const fn` can reference `static`s. pub(super) get_handle_counters: extern "C" fn() -> &'static HandleCounters, - pub(super) run: extern "C" fn(Bridge<'_>, F) -> Buffer, + pub(super) run: extern "C" fn(BridgeConfig<'_>, F) -> Buffer, pub(super) f: F, } +fn maybe_install_panic_hook(force_show_panics: bool) { + // Hide the default panic output within `proc_macro` expansions. + // NB. the server can't do this because it may use a different libstd. + static HIDE_PANICS_DURING_EXPANSION: Once = Once::new(); + HIDE_PANICS_DURING_EXPANSION.call_once(|| { + let prev = panic::take_hook(); + panic::set_hook(Box::new(move |info| { + let show = BridgeState::with(|state| match state { + BridgeState::NotConnected => true, + BridgeState::Connected(_) | BridgeState::InUse => force_show_panics, + }); + if show { + prev(info) + } + })); + }); +} + +static SYMBOL_COUNTER: AtomicUsize = AtomicUsize::new(1); + /// Client-side helper for handling client panics, entering the bridge, /// deserializing input and serializing output. // FIXME(eddyb) maybe replace `Bridge::enter` with this? fn run_client DecodeMut<'a, 's, ()>, R: Encode<()>>( - mut bridge: Bridge<'_>, + config: BridgeConfig<'_>, f: impl FnOnce(A) -> R, ) -> Buffer { - // The initial `cached_buffer` contains the input. - let mut b = bridge.cached_buffer.take(); + let BridgeConfig { input: mut b, dispatch, validate_ident, force_show_panics } = config; panic::catch_unwind(panic::AssertUnwindSafe(|| { - bridge.enter(|| { - let reader = &mut &b[..]; - let input = A::decode(reader, &mut ()); - - // Put the `cached_buffer` back in the `Bridge`, for requests. - Bridge::with(|bridge| bridge.cached_buffer = b.take()); - - let output = f(input); - - // Take the `cached_buffer` back out, for the output value. - b = Bridge::with(|bridge| bridge.cached_buffer.take()); - - // HACK(eddyb) Separate encoding a success value (`Ok(output)`) - // from encoding a panic (`Err(e: PanicMessage)`) to avoid - // having handles outside the `bridge.enter(|| ...)` scope, and - // to catch panics that could happen while encoding the success. - // - // Note that panics should be impossible beyond this point, but - // this is defensively trying to avoid any accidental panicking - // reaching the `extern "C"` (which should `abort` but may not - // at the moment, so this is also potentially preventing UB). - b.clear(); - Ok::<_, ()>(output).encode(&mut b, &mut ()); + maybe_install_panic_hook(force_show_panics); + + let reader = &mut &b[..]; + let (input, context) = <(A, ExpnContext)>::decode(reader, &mut ()); + + // Put the buffer we used for input back in the `Bridge` for requests. + let new_state = BridgeState::Connected(Bridge { + cached_buffer: b.take(), + dispatch, + validate_ident, + symbols: handle::InternedStore::new(&SYMBOL_COUNTER), + context, + }); + + BRIDGE_STATE.with(|state| { + state.set(new_state, || { + let output = f(input); + + // Take the `cached_buffer` back out, for the output value. + b = Bridge::with(|bridge| bridge.cached_buffer.take()); + + // HACK(eddyb) Separate encoding a success value (`Ok(output)`) + // from encoding a panic (`Err(e: PanicMessage)`) to avoid + // having handles outside the `bridge.enter(|| ...)` scope, and + // to catch panics that could happen while encoding the success. + // + // Note that panics should be impossible beyond this point, but + // this is defensively trying to avoid any accidental panicking + // reaching the `extern "C"` (which should `abort` but may not + // at the moment, so this is also potentially preventing UB). + b.clear(); + Ok::<_, ()>(output).encode(&mut b, &mut ()); + }) }) })) .map_err(PanicMessage::from) @@ -404,10 +564,14 @@ impl Client crate::TokenStream> { #[rustc_allow_const_fn_unstable(const_fn)] pub const fn expand1(f: fn(crate::TokenStream) -> crate::TokenStream) -> Self { extern "C" fn run( - bridge: Bridge<'_>, + bridge: BridgeConfig<'_>, f: impl FnOnce(crate::TokenStream) -> crate::TokenStream, ) -> Buffer { - run_client(bridge, |input| f(crate::TokenStream(input)).0) + run_client(bridge, |input| { + f(crate::TokenStream(Some(input))) + .0 + .unwrap_or_else(|| TokenStream::concat_streams(None, vec![])) + }) } Client { get_handle_counters: HandleCounters::get, run, f } } @@ -419,11 +583,13 @@ impl Client crate::TokenStream> { f: fn(crate::TokenStream, crate::TokenStream) -> crate::TokenStream, ) -> Self { extern "C" fn run( - bridge: Bridge<'_>, + bridge: BridgeConfig<'_>, f: impl FnOnce(crate::TokenStream, crate::TokenStream) -> crate::TokenStream, ) -> Buffer { run_client(bridge, |(input, input2)| { - f(crate::TokenStream(input), crate::TokenStream(input2)).0 + f(crate::TokenStream(Some(input)), crate::TokenStream(Some(input2))) + .0 + .unwrap_or_else(|| TokenStream::concat_streams(None, vec![])) }) } Client { get_handle_counters: HandleCounters::get, run, f } diff --git a/library/proc_macro/src/bridge/handle.rs b/library/proc_macro/src/bridge/handle.rs index bcbb86812470a..9b0475f7049bd 100644 --- a/library/proc_macro/src/bridge/handle.rs +++ b/library/proc_macro/src/bridge/handle.rs @@ -1,9 +1,11 @@ //! Server-side handles and storage for per-handle data. +use std::borrow::Borrow; use std::collections::{BTreeMap, HashMap}; use std::hash::Hash; use std::num::NonZeroU32; use std::ops::{Index, IndexMut}; +use std::rc::Rc; use std::sync::atomic::{AtomicUsize, Ordering}; pub(super) type Handle = NonZeroU32; @@ -27,10 +29,14 @@ impl OwnedStore { pub(super) fn alloc(&mut self, x: T) -> Handle { let counter = self.counter.fetch_add(1, Ordering::SeqCst); let handle = Handle::new(counter as u32).expect("`proc_macro` handle counter overflowed"); - assert!(self.data.insert(handle, x).is_none()); + self.init(handle, x); handle } + pub(super) fn init(&mut self, h: Handle, x: T) { + assert!(self.data.insert(h, x).is_none()); + } + pub(super) fn take(&mut self, h: Handle) -> T { self.data.remove(&h).expect("use-after-free in `proc_macro` handle") } @@ -49,22 +55,60 @@ impl IndexMut for OwnedStore { } } +pub(super) trait FromKey { + fn from_key(key: &Q) -> Self; +} + +impl FromKey for T { + fn from_key(key: &T) -> T { + key.clone() + } +} + +impl FromKey for Rc +where + Rc: for<'a> From<&'a T>, +{ + fn from_key(key: &T) -> Rc { + key.into() + } +} + pub(super) struct InternedStore { owned: OwnedStore, interner: HashMap, } -impl InternedStore { +impl InternedStore { pub(super) fn new(counter: &'static AtomicUsize) -> Self { InternedStore { owned: OwnedStore::new(counter), interner: HashMap::new() } } - pub(super) fn alloc(&mut self, x: T) -> Handle { + pub(super) fn alloc<'a, Q: ?Sized>(&mut self, x: &'a Q) -> Handle + where + T: Borrow + FromKey, + Q: Hash + Eq, + { let owned = &mut self.owned; - *self.interner.entry(x).or_insert_with(|| owned.alloc(x)) + *self + .interner + .raw_entry_mut() + .from_key(x) + .or_insert_with(|| { + let own = T::from_key(x); + (own.clone(), owned.alloc(own)) + }) + .1 } pub(super) fn copy(&mut self, h: Handle) -> T { - self.owned[h] + self.owned[h].clone() + } +} + +impl Index for InternedStore { + type Output = T; + fn index(&self, h: Handle) -> &T { + self.owned.index(h) } } diff --git a/library/proc_macro/src/bridge/mod.rs b/library/proc_macro/src/bridge/mod.rs index a2953b68564a8..b74894dd20c0c 100644 --- a/library/proc_macro/src/bridge/mod.rs +++ b/library/proc_macro/src/bridge/mod.rs @@ -28,10 +28,10 @@ use std::thread; /// // ... /// Literal { /// // ... -/// fn character(ch: char) -> MySelf::Literal; +/// wait fn character(ch: char) -> MySelf::Literal; /// // ... -/// fn span(my_self: &MySelf::Literal) -> MySelf::Span; -/// fn set_span(my_self: &mut MySelf::Literal, span: MySelf::Span); +/// wait fn span(my_self: &MySelf::Literal) -> MySelf::Span; +/// nowait fn set_span(my_self: &mut MySelf::Literal, span: MySelf::Span); /// }, /// // ... /// } @@ -49,122 +49,77 @@ use std::thread; /// a trait or a trait impl, where the trait has associated types /// for each of the API types. If non-associated types are desired, /// a module name (`self` in practice) can be used instead of `Self`. +/// +/// If the `nowait` modifier is used, the server implementation may not +/// panic, and the client will continue immediately without waiting for +/// a response from the server when in multithreaded mode. If a return +/// type is present, it must be an owning IPC handle. Other return types +/// are not supported with `nowait`. macro_rules! with_api { ($S:ident, $self:ident, $m:ident) => { $m! { FreeFunctions { - fn drop($self: $S::FreeFunctions); - fn track_env_var(var: &str, value: Option<&str>); + nowait fn drop($self: $S::FreeFunctions); + nowait fn track_env_var(var: &str, value: Option<&str>); + wait fn literal_from_str(s: &str) -> Result, ()>; + wait fn literal_subspan(lit: Literal<$S::Span, $S::Symbol>, start: Bound, end: Bound) -> Option<$S::Span>; }, TokenStream { - fn drop($self: $S::TokenStream); - fn clone($self: &$S::TokenStream) -> $S::TokenStream; - fn new() -> $S::TokenStream; - fn is_empty($self: &$S::TokenStream) -> bool; - fn from_str(src: &str) -> $S::TokenStream; - fn to_string($self: &$S::TokenStream) -> String; - fn from_token_tree( - tree: TokenTree<$S::Group, $S::Punct, $S::Ident, $S::Literal>, + nowait fn drop($self: $S::TokenStream); + nowait fn clone($self: &$S::TokenStream) -> $S::TokenStream; + wait fn is_empty($self: &$S::TokenStream) -> bool; + wait fn from_str(src: &str) -> $S::TokenStream; + wait fn to_string($self: &$S::TokenStream) -> String; + nowait fn from_token_tree( + tree: TokenTree<$S::TokenStream, $S::Span, $S::Symbol>, ) -> $S::TokenStream; - fn into_iter($self: $S::TokenStream) -> $S::TokenStreamIter; - }, - TokenStreamBuilder { - fn drop($self: $S::TokenStreamBuilder); - fn new() -> $S::TokenStreamBuilder; - fn push($self: &mut $S::TokenStreamBuilder, stream: $S::TokenStream); - fn build($self: $S::TokenStreamBuilder) -> $S::TokenStream; - }, - TokenStreamIter { - fn drop($self: $S::TokenStreamIter); - fn clone($self: &$S::TokenStreamIter) -> $S::TokenStreamIter; - fn next( - $self: &mut $S::TokenStreamIter, - ) -> Option>; - }, - Group { - fn drop($self: $S::Group); - fn clone($self: &$S::Group) -> $S::Group; - fn new(delimiter: Delimiter, stream: $S::TokenStream) -> $S::Group; - fn delimiter($self: &$S::Group) -> Delimiter; - fn stream($self: &$S::Group) -> $S::TokenStream; - fn span($self: &$S::Group) -> $S::Span; - fn span_open($self: &$S::Group) -> $S::Span; - fn span_close($self: &$S::Group) -> $S::Span; - fn set_span($self: &mut $S::Group, span: $S::Span); - }, - Punct { - fn new(ch: char, spacing: Spacing) -> $S::Punct; - fn as_char($self: $S::Punct) -> char; - fn spacing($self: $S::Punct) -> Spacing; - fn span($self: $S::Punct) -> $S::Span; - fn with_span($self: $S::Punct, span: $S::Span) -> $S::Punct; - }, - Ident { - fn new(string: &str, span: $S::Span, is_raw: bool) -> $S::Ident; - fn span($self: $S::Ident) -> $S::Span; - fn with_span($self: $S::Ident, span: $S::Span) -> $S::Ident; - }, - Literal { - fn drop($self: $S::Literal); - fn clone($self: &$S::Literal) -> $S::Literal; - fn from_str(s: &str) -> Result<$S::Literal, ()>; - fn debug_kind($self: &$S::Literal) -> String; - fn symbol($self: &$S::Literal) -> String; - fn suffix($self: &$S::Literal) -> Option; - fn integer(n: &str) -> $S::Literal; - fn typed_integer(n: &str, kind: &str) -> $S::Literal; - fn float(n: &str) -> $S::Literal; - fn f32(n: &str) -> $S::Literal; - fn f64(n: &str) -> $S::Literal; - fn string(string: &str) -> $S::Literal; - fn character(ch: char) -> $S::Literal; - fn byte_string(bytes: &[u8]) -> $S::Literal; - fn span($self: &$S::Literal) -> $S::Span; - fn set_span($self: &mut $S::Literal, span: $S::Span); - fn subspan( - $self: &$S::Literal, - start: Bound, - end: Bound, - ) -> Option<$S::Span>; + nowait fn concat_trees( + base: Option<$S::TokenStream>, + trees: Vec>, + ) -> $S::TokenStream; + nowait fn concat_streams( + base: Option<$S::TokenStream>, + trees: Vec<$S::TokenStream>, + ) -> $S::TokenStream; + wait fn into_iter( + $self: $S::TokenStream + ) -> Vec>; }, SourceFile { - fn drop($self: $S::SourceFile); - fn clone($self: &$S::SourceFile) -> $S::SourceFile; - fn eq($self: &$S::SourceFile, other: &$S::SourceFile) -> bool; - fn path($self: &$S::SourceFile) -> String; - fn is_real($self: &$S::SourceFile) -> bool; + nowait fn drop($self: $S::SourceFile); + nowait fn clone($self: &$S::SourceFile) -> $S::SourceFile; + wait fn eq($self: &$S::SourceFile, other: &$S::SourceFile) -> bool; + wait fn path($self: &$S::SourceFile) -> String; + wait fn is_real($self: &$S::SourceFile) -> bool; }, MultiSpan { - fn drop($self: $S::MultiSpan); - fn new() -> $S::MultiSpan; - fn push($self: &mut $S::MultiSpan, span: $S::Span); + nowait fn drop($self: $S::MultiSpan); + nowait fn new() -> $S::MultiSpan; + wait fn push($self: &mut $S::MultiSpan, span: $S::Span); }, Diagnostic { - fn drop($self: $S::Diagnostic); - fn new(level: Level, msg: &str, span: $S::MultiSpan) -> $S::Diagnostic; - fn sub( + wait fn drop($self: $S::Diagnostic); + wait fn new(level: Level, msg: &str, span: $S::MultiSpan) -> $S::Diagnostic; + wait fn sub( $self: &mut $S::Diagnostic, level: Level, msg: &str, span: $S::MultiSpan, ); - fn emit($self: $S::Diagnostic); + wait fn emit($self: $S::Diagnostic); }, Span { - fn debug($self: $S::Span) -> String; - fn def_site() -> $S::Span; - fn call_site() -> $S::Span; - fn mixed_site() -> $S::Span; - fn source_file($self: $S::Span) -> $S::SourceFile; - fn parent($self: $S::Span) -> Option<$S::Span>; - fn source($self: $S::Span) -> $S::Span; - fn start($self: $S::Span) -> LineColumn; - fn end($self: $S::Span) -> LineColumn; - fn join($self: $S::Span, other: $S::Span) -> Option<$S::Span>; - fn resolved_at($self: $S::Span, at: $S::Span) -> $S::Span; - fn source_text($self: $S::Span) -> Option; - fn save_span($self: $S::Span) -> usize; - fn recover_proc_macro_span(id: usize) -> $S::Span; + wait fn debug($self: $S::Span) -> String; + wait fn source_file($self: $S::Span) -> $S::SourceFile; + wait fn parent($self: $S::Span) -> Option<$S::Span>; + wait fn source($self: $S::Span) -> $S::Span; + wait fn start($self: $S::Span) -> LineColumn; + wait fn end($self: $S::Span) -> LineColumn; + wait fn join($self: $S::Span, other: $S::Span) -> Option<$S::Span>; + wait fn resolved_at($self: $S::Span, at: $S::Span) -> $S::Span; + wait fn source_text($self: $S::Span) -> Option; + wait fn save_span($self: $S::Span) -> usize; + wait fn recover_proc_macro_span(id: usize) -> $S::Span; }, } }; @@ -173,10 +128,10 @@ macro_rules! with_api { // FIXME(eddyb) this calls `encode` for each argument, but in reverse, // to avoid borrow conflicts from borrows started by `&mut` arguments. macro_rules! reverse_encode { - ($writer:ident;) => {}; - ($writer:ident; $first:ident $(, $rest:ident)*) => { - reverse_encode!($writer; $($rest),*); - $first.encode(&mut $writer, &mut ()); + ($writer:ident, $s:ident;) => {}; + ($writer:ident, $s:ident; $first:ident $(, $rest:ident)*) => { + reverse_encode!($writer, $s; $($rest),*); + $first.encode(&mut $writer, $s); } } @@ -210,35 +165,46 @@ use buffer::Buffer; pub use rpc::PanicMessage; use rpc::{Decode, DecodeMut, Encode, Reader, Writer}; -/// An active connection between a server and a client. -/// The server creates the bridge (`Bridge::run_server` in `server.rs`), -/// then passes it to the client through the function pointer in the `run` -/// field of `client::Client`. The client holds its copy of the `Bridge` +/// Configuration for establishing an active connection between a server and a +/// client. The server creates the bridge config (`run_server` in `server.rs`), +/// then passes it to the client through the function pointer in the `run` field +/// of `client::Client`. The client constructs a local `Bridge` from the config /// in TLS during its execution (`Bridge::{enter, with}` in `client.rs`). #[repr(C)] -pub struct Bridge<'a> { - /// Reusable buffer (only `clear`-ed, never shrunk), primarily - /// used for making requests, but also for passing input to client. - cached_buffer: Buffer, +pub struct BridgeConfig<'a> { + /// Buffer used to pass initial input to the client. + input: Buffer, /// Server-side function that the client uses to make requests. dispatch: closure::Closure<'a, Buffer, Buffer>, + /// Server-side function to validate and normalize an ident. + validate_ident: extern "C" fn(buffer::Slice<'_, u8>, &mut Buffer) -> bool, + /// If 'true', always invoke the default panic hook force_show_panics: bool, } -impl<'a> !Sync for Bridge<'a> {} -impl<'a> !Send for Bridge<'a> {} +impl<'a> !Sync for BridgeConfig<'a> {} +impl<'a> !Send for BridgeConfig<'a> {} #[forbid(unsafe_code)] #[allow(non_camel_case_types)] mod api_tags { use super::rpc::{DecodeMut, Encode, Reader, Writer}; + macro_rules! should_wait_impl { + (wait) => { + true + }; + (nowait) => { + false + }; + } + macro_rules! declare_tags { ($($name:ident { - $(fn $method:ident($($arg:ident: $arg_ty:ty),* $(,)?) $(-> $ret_ty:ty)*;)* + $($wait:ident fn $method:ident($($arg:ident: $arg_ty:ty),* $(,)?) $(-> $ret_ty:ty)*;)* }),* $(,)?) => { $( pub(super) enum $name { @@ -252,6 +218,16 @@ mod api_tags { $($name($name)),* } rpc_encode_decode!(enum Method { $($name(m)),* }); + + impl Method { + pub(super) fn should_wait(&self) -> bool { + match self { + $($( + Method::$name($name::$method) => should_wait_impl!($wait), + )*)* + } + } + } } } with_api!(self, self, declare_tags); @@ -329,6 +305,21 @@ impl Unmark for Result { } } +impl Mark for Vec { + type Unmarked = Vec; + fn mark(unmarked: Self::Unmarked) -> Self { + // Should be a no-op due to std's in-place collect optimizations. + unmarked.into_iter().map(T::mark).collect() + } +} +impl Unmark for Vec { + type Unmarked = Vec; + fn unmark(self) -> Self::Unmarked { + // Should be a no-op due to std's in-place collect optimizations. + self.into_iter().map(T::unmark).collect() + } +} + macro_rules! mark_noop { ($($ty:ty),* $(,)?) => { $( @@ -356,6 +347,7 @@ mark_noop! { String, usize, Delimiter, + LitKind, Level, LineColumn, Spacing, @@ -386,42 +378,163 @@ rpc_encode_decode!( } ); -#[derive(Clone)] -pub enum TokenTree { - Group(G), - Punct(P), - Ident(I), - Literal(L), +#[derive(Copy, Clone, Eq, PartialEq, Debug)] +pub enum LitKind { + Byte, + Char, + Integer, + Float, + Str, + StrRaw(u16), + ByteStr, + ByteStrRaw(u16), + Err, } -impl Mark for TokenTree { - type Unmarked = TokenTree; - fn mark(unmarked: Self::Unmarked) -> Self { - match unmarked { - TokenTree::Group(tt) => TokenTree::Group(G::mark(tt)), - TokenTree::Punct(tt) => TokenTree::Punct(P::mark(tt)), - TokenTree::Ident(tt) => TokenTree::Ident(I::mark(tt)), - TokenTree::Literal(tt) => TokenTree::Literal(L::mark(tt)), +rpc_encode_decode!( + enum LitKind { + Byte, + Char, + Integer, + Float, + Str, + StrRaw(n), + ByteStr, + ByteStrRaw(n), + Err, + } +); + +macro_rules! mark_compound { + (struct $name:ident <$($T:ident),+> { $($field:ident),* $(,)? }) => { + impl<$($T: Mark),+> Mark for $name <$($T),+> { + type Unmarked = $name <$($T::Unmarked),+>; + fn mark(unmarked: Self::Unmarked) -> Self { + $name { + $($field: Mark::mark(unmarked.$field)),* + } + } + } + + impl<$($T: Unmark),+> Unmark for $name <$($T),+> { + type Unmarked = $name <$($T::Unmarked),+>; + fn unmark(self) -> Self::Unmarked { + $name { + $($field: Unmark::unmark(self.$field)),* + } + } + } + }; + (enum $name:ident <$($T:ident),+> { $($variant:ident $(($field:ident))?),* $(,)? }) => { + impl<$($T: Mark),+> Mark for $name <$($T),+> { + type Unmarked = $name <$($T::Unmarked),+>; + fn mark(unmarked: Self::Unmarked) -> Self { + match unmarked { + $($name::$variant $(($field))? => { + $name::$variant $((Mark::mark($field)))? + })* + } + } + } + + impl<$($T: Unmark),+> Unmark for $name <$($T),+> { + type Unmarked = $name <$($T::Unmarked),+>; + fn unmark(self) -> Self::Unmarked { + match self { + $($name::$variant $(($field))? => { + $name::$variant $((Unmark::unmark($field)))? + })* + } + } } } } -impl Unmark for TokenTree { - type Unmarked = TokenTree; - fn unmark(self) -> Self::Unmarked { - match self { - TokenTree::Group(tt) => TokenTree::Group(tt.unmark()), - TokenTree::Punct(tt) => TokenTree::Punct(tt.unmark()), - TokenTree::Ident(tt) => TokenTree::Ident(tt.unmark()), - TokenTree::Literal(tt) => TokenTree::Literal(tt.unmark()), - } + +macro_rules! compound_traits { + ($($t:tt)*) => { + rpc_encode_decode!($($t)*); + mark_compound!($($t)*); + }; +} + +#[derive(Copy, Clone)] +pub struct DelimSpan { + pub open: Sp, + pub close: Sp, + pub entire: Sp, +} + +impl DelimSpan { + pub fn from_single(span: Sp) -> Self { + DelimSpan { open: span, close: span, entire: span } } } -rpc_encode_decode!( - enum TokenTree { +compound_traits!(struct DelimSpan { open, close, entire }); + +#[derive(Clone)] +pub struct Group { + pub delimiter: Delimiter, + pub stream: Option, + pub span: DelimSpan, +} + +compound_traits!(struct Group { delimiter, stream, span }); + +#[derive(Clone)] +pub struct Punct { + pub ch: char, + pub joint: bool, + pub span: Sp, +} + +compound_traits!(struct Punct { ch, joint, span }); + +#[derive(Copy, Clone, Eq, PartialEq)] +pub struct Ident { + pub sym: Sy, + pub is_raw: bool, + pub span: Sp, +} + +compound_traits!(struct Ident { sym, is_raw, span }); + +#[derive(Clone, Eq, PartialEq)] +pub struct Literal { + pub kind: LitKind, + pub symbol: Sy, + pub suffix: Option, + pub span: Sp, +} + +compound_traits!(struct Literal { kind, symbol, suffix, span }); + +#[derive(Clone)] +pub enum TokenTree { + Group(Group), + Punct(Punct), + Ident(Ident), + Literal(Literal), +} + +compound_traits!( + enum TokenTree { Group(tt), Punct(tt), Ident(tt), Literal(tt), } ); + +/// Context provided alongside the initial inputs for a macro expansion. +/// Provides values such as spans which are used frequently to avoid RPC. +#[derive(Clone)] +struct ExpnContext { + def_site: Sp, + call_site: Sp, + mixed_site: Sp, +} + +compound_traits!( + struct ExpnContext { def_site, call_site, mixed_site } +); diff --git a/library/proc_macro/src/bridge/rpc.rs b/library/proc_macro/src/bridge/rpc.rs index 42432563faf33..66512ac71e427 100644 --- a/library/proc_macro/src/bridge/rpc.rs +++ b/library/proc_macro/src/bridge/rpc.rs @@ -43,15 +43,17 @@ macro_rules! rpc_encode_decode { } } }; - (struct $name:ident { $($field:ident),* $(,)? }) => { - impl Encode for $name { + (struct $name:ident $(<$($T:ident),+>)? { $($field:ident),* $(,)? }) => { + impl),+)?> Encode for $name $(<$($T),+>)? { fn encode(self, w: &mut Writer, s: &mut S) { $(self.$field.encode(w, s);)* } } - impl DecodeMut<'_, '_, S> for $name { - fn decode(r: &mut Reader<'_>, s: &mut S) -> Self { + impl DecodeMut<'a, 's, S>),+)?> DecodeMut<'a, '_, S> + for $name $(<$($T),+>)? + { + fn decode(r: &mut Reader<'a>, s: &mut S) -> Self { $name { $($field: DecodeMut::decode(r, s)),* } @@ -126,6 +128,7 @@ impl DecodeMut<'_, '_, S> for u8 { } } +rpc_encode_decode!(le u16); rpc_encode_decode!(le u32); rpc_encode_decode!(le usize); @@ -246,6 +249,26 @@ impl DecodeMut<'_, '_, S> for String { } } +impl> Encode for Vec { + fn encode(self, w: &mut Writer, s: &mut S) { + self.len().encode(w, s); + for x in self { + x.encode(w, s); + } + } +} + +impl DecodeMut<'a, 's, S>> DecodeMut<'a, '_, S> for Vec { + fn decode(r: &mut Reader<'a>, s: &mut S) -> Self { + let len = usize::decode(r, s); + let mut vec = Vec::with_capacity(len); + for _ in 0..len { + vec.push(T::decode(r, s)); + } + vec + } +} + /// Simplified version of panic payloads, ignoring /// types other than `&'static str` and `String`. pub enum PanicMessage { diff --git a/library/proc_macro/src/bridge/server.rs b/library/proc_macro/src/bridge/server.rs index 1b3ccf4c18e70..6875fd0a0520c 100644 --- a/library/proc_macro/src/bridge/server.rs +++ b/library/proc_macro/src/bridge/server.rs @@ -2,6 +2,8 @@ use super::*; +use std::marker::PhantomData; + // FIXME(eddyb) generate the definition of `HandleStore` in `server.rs`. use super::client::HandleStore; @@ -12,18 +14,6 @@ macro_rules! associated_item { (type FreeFunctions: 'static;); (type TokenStream) => (type TokenStream: 'static + Clone;); - (type TokenStreamBuilder) => - (type TokenStreamBuilder: 'static;); - (type TokenStreamIter) => - (type TokenStreamIter: 'static + Clone;); - (type Group) => - (type Group: 'static + Clone;); - (type Punct) => - (type Punct: 'static + Copy + Eq + Hash;); - (type Ident) => - (type Ident: 'static + Copy + Eq + Hash;); - (type Literal) => - (type Literal: 'static + Clone;); (type SourceFile) => (type SourceFile: 'static + Clone;); (type MultiSpan) => @@ -39,32 +29,77 @@ macro_rules! associated_item { ($($item:tt)*) => ($($item)*;) } +/// Helper methods defined by `Server` types not invoked over RPC. +pub trait Context: Types { + fn def_site(&mut self) -> Self::Span; + fn call_site(&mut self) -> Self::Span; + fn mixed_site(&mut self) -> Self::Span; + + /// Check if an identifier is valid, and return `Ok(...)` if it is. + /// + /// May be called on any thread. + /// + /// Returns `Ok(Some(str))` with a normalized version of the identifier if + /// normalization is required, and `Ok(None)` if the existing identifier is + /// already normalized. + fn validate_ident(ident: &str) -> Result, ()>; + + /// Intern a symbol received from RPC + fn intern_symbol(ident: &str) -> Self::Symbol; + + /// Recover the string value of a symbol, and invoke a callback with it. + fn with_symbol_string(symbol: &Self::Symbol, f: impl FnOnce(&str)); +} + macro_rules! declare_server_traits { ($($name:ident { - $(fn $method:ident($($arg:ident: $arg_ty:ty),* $(,)?) $(-> $ret_ty:ty)?;)* + $($wait:ident fn $method:ident($($arg:ident: $arg_ty:ty),* $(,)?) $(-> $ret_ty:ty)?;)* }),* $(,)?) => { pub trait Types { $(associated_item!(type $name);)* + type Symbol: 'static + Copy + Eq + Hash; } $(pub trait $name: Types { $(associated_item!(fn $method(&mut self, $($arg: $arg_ty),*) $(-> $ret_ty)?);)* })* - pub trait Server: Types $(+ $name)* {} - impl Server for S {} + pub trait Server: Types + Context $(+ $name)* {} + impl Server for S {} } } with_api!(Self, self_, declare_server_traits); pub(super) struct MarkedTypes(S); +impl Context for MarkedTypes { + fn def_site(&mut self) -> Self::Span { + <_>::mark(Context::def_site(&mut self.0)) + } + fn call_site(&mut self) -> Self::Span { + <_>::mark(Context::call_site(&mut self.0)) + } + fn mixed_site(&mut self) -> Self::Span { + <_>::mark(Context::mixed_site(&mut self.0)) + } + fn validate_ident(ident: &str) -> Result, ()> { + S::validate_ident(ident) + } + fn intern_symbol(ident: &str) -> Self::Symbol { + <_>::mark(S::intern_symbol(ident)) + } + fn with_symbol_string(symbol: &Self::Symbol, f: impl FnOnce(&str)) { + S::with_symbol_string(symbol.unmark(), f) + } +} + macro_rules! define_mark_types_impls { ($($name:ident { - $(fn $method:ident($($arg:ident: $arg_ty:ty),* $(,)?) $(-> $ret_ty:ty)?;)* + $($wait:ident fn $method:ident($($arg:ident: $arg_ty:ty),* $(,)?) $(-> $ret_ty:ty)?;)* }),* $(,)?) => { impl Types for MarkedTypes { $(type $name = Marked;)* + type Symbol = Marked; } $(impl $name for MarkedTypes { @@ -76,24 +111,43 @@ macro_rules! define_mark_types_impls { } with_api!(Self, self_, define_mark_types_impls); +pub(super) trait InitOwnedHandle { + fn init_handle(self, raw_handle: handle::Handle, s: &mut S); +} + struct Dispatcher { handle_store: HandleStore, server: S, } +macro_rules! maybe_handle_nowait_reply { + (wait, $reader:ident, $r:ident, $handle_store:ident, $ret_ty:ty) => {}; + (nowait, $reader:ident, $r:ident, $handle_store:ident, $ret_ty:ty) => { + let $r = $r.map(|r| { + let raw_handle = handle::Handle::decode(&mut $reader, &mut ()); + <$ret_ty as InitOwnedHandle<_>>::init_handle(r, raw_handle, $handle_store); + }); + }; +} + macro_rules! define_dispatcher_impl { ($($name:ident { - $(fn $method:ident($($arg:ident: $arg_ty:ty),* $(,)?) $(-> $ret_ty:ty)?;)* + $($wait:ident fn $method:ident($($arg:ident: $arg_ty:ty),* $(,)?) $(-> $ret_ty:ty)?;)* }),* $(,)?) => { // FIXME(eddyb) `pub` only for `ExecutionStrategy` below. pub trait DispatcherTrait { // HACK(eddyb) these are here to allow `Self::$name` to work below. $(type $name;)* + type Symbol; + fn dispatch(&mut self, b: Buffer) -> Buffer; + fn validate_ident(ident: &str) -> Result, ()>; } impl DispatcherTrait for Dispatcher> { $(type $name = as Types>::$name;)* + type Symbol = as Types>::Symbol; + fn dispatch(&mut self, mut b: Buffer) -> Buffer { let Dispatcher { handle_store, server } = self; @@ -116,6 +170,8 @@ macro_rules! define_dispatcher_impl { .map_err(PanicMessage::from) }; + $(maybe_handle_nowait_reply!($wait, reader, r, handle_store, $ret_ty);)? + b.clear(); r.encode(&mut b, handle_store); })* @@ -123,148 +179,202 @@ macro_rules! define_dispatcher_impl { } b } + fn validate_ident(ident: &str) -> Result, ()> { + S::validate_ident(ident) + } } } } with_api!(Self, self_, define_dispatcher_impl); +extern "C" fn validate_ident_impl( + string: buffer::Slice<'_, u8>, + normalized: &mut Buffer, +) -> bool { + match std::str::from_utf8(&string[..]).map_err(|_| ()).and_then(D::validate_ident) { + Ok(Some(norm)) => { + *normalized = norm.into_bytes().into(); + true + } + Ok(None) => true, + Err(_) => false, + } +} + pub trait ExecutionStrategy { - fn run_bridge_and_client( + fn run_bridge_and_client( &self, - dispatcher: &mut impl DispatcherTrait, + dispatcher: &mut D, input: Buffer, - run_client: extern "C" fn(Bridge<'_>, D) -> Buffer, - client_data: D, + run_client: extern "C" fn(BridgeConfig<'_>, T) -> Buffer, + client_data: T, force_show_panics: bool, ) -> Buffer; } +pub struct MaybeCrossThread

{ + cross_thread: bool, + marker: PhantomData

, +} + +impl

MaybeCrossThread

{ + pub const fn new(cross_thread: bool) -> Self { + MaybeCrossThread { cross_thread, marker: PhantomData } + } +} + +impl

ExecutionStrategy for MaybeCrossThread

+where + P: MessagePipe> + Send + 'static, +{ + fn run_bridge_and_client( + &self, + dispatcher: &mut D, + input: Buffer, + run_client: extern "C" fn(BridgeConfig<'_>, T) -> Buffer, + client_data: T, + force_show_panics: bool, + ) -> Buffer { + if self.cross_thread { + >::new().run_bridge_and_client( + dispatcher, + input, + run_client, + client_data, + force_show_panics, + ) + } else { + SameThread.run_bridge_and_client( + dispatcher, + input, + run_client, + client_data, + force_show_panics, + ) + } + } +} + +#[derive(Default)] pub struct SameThread; impl ExecutionStrategy for SameThread { - fn run_bridge_and_client( + fn run_bridge_and_client( &self, - dispatcher: &mut impl DispatcherTrait, + dispatcher: &mut D, input: Buffer, - run_client: extern "C" fn(Bridge<'_>, D) -> Buffer, - client_data: D, + run_client: extern "C" fn(BridgeConfig<'_>, T) -> Buffer, + client_data: T, force_show_panics: bool, ) -> Buffer { let mut dispatch = |b| dispatcher.dispatch(b); run_client( - Bridge { cached_buffer: input, dispatch: (&mut dispatch).into(), force_show_panics }, + BridgeConfig { + input, + dispatch: (&mut dispatch).into(), + validate_ident: validate_ident_impl::, + force_show_panics, + }, client_data, ) } } -// NOTE(eddyb) Two implementations are provided, the second one is a bit -// faster but neither is anywhere near as fast as same-thread execution. +pub struct CrossThread

(PhantomData

); -pub struct CrossThread1; +impl

CrossThread

{ + pub const fn new() -> Self { + CrossThread(PhantomData) + } +} -impl ExecutionStrategy for CrossThread1 { - fn run_bridge_and_client( +impl

ExecutionStrategy for CrossThread

+where + P: MessagePipe> + Send + 'static, +{ + fn run_bridge_and_client( &self, - dispatcher: &mut impl DispatcherTrait, + dispatcher: &mut D, input: Buffer, - run_client: extern "C" fn(Bridge<'_>, D) -> Buffer, - client_data: D, + run_client: extern "C" fn(BridgeConfig<'_>, T) -> Buffer, + client_data: T, force_show_panics: bool, ) -> Buffer { - use std::sync::mpsc::channel; - - let (req_tx, req_rx) = channel(); - let (res_tx, res_rx) = channel(); + let (mut server, mut client) = P::new(); let join_handle = thread::spawn(move || { - let mut dispatch = |b| { - req_tx.send(b).unwrap(); - res_rx.recv().unwrap() + let mut dispatch = |b: Buffer| -> Buffer { + let method_tag = api_tags::Method::decode(&mut &b[..], &mut ()); + client.send(b); + + if method_tag.should_wait() { + client.recv().expect("server died while client waiting for reply") + } else { + Buffer::new() + } }; run_client( - Bridge { - cached_buffer: input, + BridgeConfig { + input, dispatch: (&mut dispatch).into(), + validate_ident: validate_ident_impl::, force_show_panics, }, client_data, ) }); - for b in req_rx { - res_tx.send(dispatcher.dispatch(b)).unwrap(); + while let Some(b) = server.recv() { + let method_tag = api_tags::Method::decode(&mut &b[..], &mut ()); + let b = dispatcher.dispatch(b); + + if method_tag.should_wait() { + server.send(b); + } else if let Err(err) = >::decode(&mut &b[..], &mut ()) { + panic::resume_unwind(err.into()); + } } join_handle.join().unwrap() } } -pub struct CrossThread2; - -impl ExecutionStrategy for CrossThread2 { - fn run_bridge_and_client( - &self, - dispatcher: &mut impl DispatcherTrait, - input: Buffer, - run_client: extern "C" fn(Bridge<'_>, D) -> Buffer, - client_data: D, - force_show_panics: bool, - ) -> Buffer { - use std::sync::{Arc, Mutex}; - - enum State { - Req(T), - Res(T), - } +/// A message pipe used for communicating between server and client threads. +pub trait MessagePipe: Sized { + /// Create a new pair of endpoints for the message pipe. + fn new() -> (Self, Self); - let mut state = Arc::new(Mutex::new(State::Res(Buffer::new()))); + /// Send a message to the other endpoint of this pipe. + fn send(&mut self, value: T); - let server_thread = thread::current(); - let state2 = state.clone(); - let join_handle = thread::spawn(move || { - let mut dispatch = |b| { - *state2.lock().unwrap() = State::Req(b); - server_thread.unpark(); - loop { - thread::park(); - if let State::Res(b) = &mut *state2.lock().unwrap() { - break b.take(); - } - } - }; - - let r = run_client( - Bridge { - cached_buffer: input, - dispatch: (&mut dispatch).into(), - force_show_panics, - }, - client_data, - ); + /// Receive a message from the other endpoint of this pipe. + /// + /// Returns `None` if the other end of the pipe has been destroyed, and no + /// message was received. + fn recv(&mut self) -> Option; +} - // Wake up the server so it can exit the dispatch loop. - drop(state2); - server_thread.unpark(); +/// Implementation of `MessagePipe` using `std::sync::mpsc` +pub struct StdMessagePipe { + tx: std::sync::mpsc::Sender, + rx: std::sync::mpsc::Receiver, +} - r - }); +impl MessagePipe for StdMessagePipe { + fn new() -> (Self, Self) { + let (tx1, rx1) = std::sync::mpsc::channel(); + let (tx2, rx2) = std::sync::mpsc::channel(); + (StdMessagePipe { tx: tx1, rx: rx2 }, StdMessagePipe { tx: tx2, rx: rx1 }) + } - // Check whether `state2` was dropped, to know when to stop. - while Arc::get_mut(&mut state).is_none() { - thread::park(); - let mut b = match &mut *state.lock().unwrap() { - State::Req(b) => b.take(), - _ => continue, - }; - b = dispatcher.dispatch(b.take()); - *state.lock().unwrap() = State::Res(b); - join_handle.thread().unpark(); - } + fn send(&mut self, v: T) { + self.tx.send(v).unwrap(); + } - join_handle.join().unwrap() + fn recv(&mut self) -> Option { + self.rx.recv().ok() } } @@ -278,15 +388,21 @@ fn run_server< handle_counters: &'static client::HandleCounters, server: S, input: I, - run_client: extern "C" fn(Bridge<'_>, D) -> Buffer, + run_client: extern "C" fn(BridgeConfig<'_>, D) -> Buffer, client_data: D, force_show_panics: bool, ) -> Result { let mut dispatcher = Dispatcher { handle_store: HandleStore::new(handle_counters), server: MarkedTypes(server) }; + let expn_context = ExpnContext { + def_site: dispatcher.server.def_site(), + call_site: dispatcher.server.call_site(), + mixed_site: dispatcher.server.mixed_site(), + }; + let mut b = Buffer::new(); - input.encode(&mut b, &mut dispatcher.handle_store); + (input, expn_context).encode(&mut b, &mut dispatcher.handle_store); b = strategy.run_bridge_and_client( &mut dispatcher, diff --git a/library/proc_macro/src/lib.rs b/library/proc_macro/src/lib.rs index 9b155db6d7b1b..fd275cb8cfb53 100644 --- a/library/proc_macro/src/lib.rs +++ b/library/proc_macro/src/lib.rs @@ -31,6 +31,7 @@ #![feature(restricted_std)] #![feature(rustc_attrs)] #![feature(min_specialization)] +#![feature(hash_raw_entry)] #![recursion_limit = "256"] #[unstable(feature = "proc_macro_internals", issue = "27812")] @@ -46,7 +47,7 @@ use std::cmp::Ordering; use std::ops::RangeBounds; use std::path::PathBuf; use std::str::FromStr; -use std::{error, fmt, iter, mem}; +use std::{error, fmt, iter}; /// Determines whether proc_macro has been made accessible to the currently /// running program. @@ -63,7 +64,7 @@ use std::{error, fmt, iter, mem}; /// inside of a procedural macro, false if invoked from any other binary. #[unstable(feature = "proc_macro_is_available", issue = "71436")] pub fn is_available() -> bool { - bridge::Bridge::is_available() + bridge::client::is_available() } /// The main type provided by this crate, representing an abstract stream of @@ -75,7 +76,7 @@ pub fn is_available() -> bool { /// and `#[proc_macro_derive]` definitions. #[stable(feature = "proc_macro_lib", since = "1.15.0")] #[derive(Clone)] -pub struct TokenStream(bridge::client::TokenStream); +pub struct TokenStream(Option); #[stable(feature = "proc_macro_lib", since = "1.15.0")] impl !Send for TokenStream {} @@ -113,13 +114,13 @@ impl TokenStream { /// Returns an empty `TokenStream` containing no token trees. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn new() -> TokenStream { - TokenStream(bridge::client::TokenStream::new()) + TokenStream(None) } /// Checks if this `TokenStream` is empty. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn is_empty(&self) -> bool { - self.0.is_empty() + self.0.as_ref().map(|h| h.is_empty()).unwrap_or(true) } } @@ -135,7 +136,7 @@ impl FromStr for TokenStream { type Err = LexError; fn from_str(src: &str) -> Result { - Ok(TokenStream(bridge::client::TokenStream::from_str(src))) + Ok(TokenStream(Some(bridge::client::TokenStream::from_str(src)))) } } @@ -144,7 +145,7 @@ impl FromStr for TokenStream { #[stable(feature = "proc_macro_lib", since = "1.15.0")] impl ToString for TokenStream { fn to_string(&self) -> String { - self.0.to_string() + self.0.as_ref().map(|t| t.to_string()).unwrap_or_default() } } @@ -177,16 +178,22 @@ impl Default for TokenStream { #[unstable(feature = "proc_macro_quote", issue = "54722")] pub use quote::{quote, quote_span}; +fn tree_to_bridge_tree( + tree: TokenTree, +) -> bridge::TokenTree { + match tree { + TokenTree::Group(tt) => bridge::TokenTree::Group(tt.0), + TokenTree::Punct(tt) => bridge::TokenTree::Punct(tt.0), + TokenTree::Ident(tt) => bridge::TokenTree::Ident(tt.0), + TokenTree::Literal(tt) => bridge::TokenTree::Literal(tt.0), + } +} + /// Creates a token stream containing a single token tree. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] impl From for TokenStream { fn from(tree: TokenTree) -> TokenStream { - TokenStream(bridge::client::TokenStream::from_token_tree(match tree { - TokenTree::Group(tt) => bridge::TokenTree::Group(tt.0), - TokenTree::Punct(tt) => bridge::TokenTree::Punct(tt.0), - TokenTree::Ident(tt) => bridge::TokenTree::Ident(tt.0), - TokenTree::Literal(tt) => bridge::TokenTree::Literal(tt.0), - })) + TokenStream(Some(bridge::client::TokenStream::from_token_tree(tree_to_bridge_tree(tree)))) } } @@ -194,7 +201,10 @@ impl From for TokenStream { #[stable(feature = "proc_macro_lib2", since = "1.29.0")] impl iter::FromIterator for TokenStream { fn from_iter>(trees: I) -> Self { - trees.into_iter().map(TokenStream::from).collect() + TokenStream(Some(bridge::client::TokenStream::concat_trees( + None, + trees.into_iter().map(tree_to_bridge_tree).collect(), + ))) } } @@ -203,24 +213,30 @@ impl iter::FromIterator for TokenStream { #[stable(feature = "proc_macro_lib", since = "1.15.0")] impl iter::FromIterator for TokenStream { fn from_iter>(streams: I) -> Self { - let mut builder = bridge::client::TokenStreamBuilder::new(); - streams.into_iter().for_each(|stream| builder.push(stream.0)); - TokenStream(builder.build()) + TokenStream(Some(bridge::client::TokenStream::concat_streams( + None, + streams.into_iter().filter_map(|stream| stream.0).collect(), + ))) } } #[stable(feature = "token_stream_extend", since = "1.30.0")] impl Extend for TokenStream { fn extend>(&mut self, trees: I) { - self.extend(trees.into_iter().map(TokenStream::from)); + *self = TokenStream(Some(bridge::client::TokenStream::concat_trees( + self.0.take(), + trees.into_iter().map(|tree| tree_to_bridge_tree(tree)).collect(), + ))); } } #[stable(feature = "token_stream_extend", since = "1.30.0")] impl Extend for TokenStream { fn extend>(&mut self, streams: I) { - // FIXME(eddyb) Use an optimized implementation if/when possible. - *self = iter::once(mem::replace(self, Self::new())).chain(streams).collect(); + *self = TokenStream(Some(bridge::client::TokenStream::concat_streams( + self.0.take(), + streams.into_iter().filter_map(|stream| stream.0).collect(), + ))); } } @@ -234,7 +250,15 @@ pub mod token_stream { /// and returns whole groups as token trees. #[derive(Clone)] #[stable(feature = "proc_macro_lib2", since = "1.29.0")] - pub struct IntoIter(bridge::client::TokenStreamIter); + pub struct IntoIter( + std::vec::IntoIter< + bridge::TokenTree< + bridge::client::TokenStream, + bridge::client::Span, + bridge::client::Symbol, + >, + >, + ); #[stable(feature = "proc_macro_lib2", since = "1.29.0")] impl Iterator for IntoIter { @@ -256,7 +280,7 @@ pub mod token_stream { type IntoIter = IntoIter; fn into_iter(self) -> IntoIter { - IntoIter(self.0.into_iter()) + IntoIter(self.0.map(|v| v.into_iter()).unwrap_or_default().into_iter()) } } } @@ -633,7 +657,7 @@ impl fmt::Display for TokenTree { /// A `Group` internally contains a `TokenStream` which is surrounded by `Delimiter`s. #[derive(Clone)] #[stable(feature = "proc_macro_lib2", since = "1.29.0")] -pub struct Group(bridge::client::Group); +pub struct Group(bridge::Group); #[stable(feature = "proc_macro_lib2", since = "1.29.0")] impl !Send for Group {} @@ -670,13 +694,17 @@ impl Group { /// method below. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn new(delimiter: Delimiter, stream: TokenStream) -> Group { - Group(bridge::client::Group::new(delimiter, stream.0)) + Group(bridge::Group { + delimiter, + stream: stream.0, + span: bridge::DelimSpan::from_single(Span::call_site().0), + }) } /// Returns the delimiter of this `Group` #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn delimiter(&self) -> Delimiter { - self.0.delimiter() + self.0.delimiter } /// Returns the `TokenStream` of tokens that are delimited in this `Group`. @@ -685,7 +713,7 @@ impl Group { /// returned above. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn stream(&self) -> TokenStream { - TokenStream(self.0.stream()) + TokenStream(self.0.stream.clone()) } /// Returns the span for the delimiters of this token stream, spanning the @@ -697,7 +725,7 @@ impl Group { /// ``` #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn span(&self) -> Span { - Span(self.0.span()) + Span(self.0.span.entire) } /// Returns the span pointing to the opening delimiter of this group. @@ -708,7 +736,7 @@ impl Group { /// ``` #[stable(feature = "proc_macro_group_span", since = "1.55.0")] pub fn span_open(&self) -> Span { - Span(self.0.span_open()) + Span(self.0.span.open) } /// Returns the span pointing to the closing delimiter of this group. @@ -719,7 +747,7 @@ impl Group { /// ``` #[stable(feature = "proc_macro_group_span", since = "1.55.0")] pub fn span_close(&self) -> Span { - Span(self.0.span_close()) + Span(self.0.span.close) } /// Configures the span for this `Group`'s delimiters, but not its internal @@ -730,7 +758,7 @@ impl Group { /// tokens at the level of the `Group`. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn set_span(&mut self, span: Span) { - self.0.set_span(span.0); + self.0.span = bridge::DelimSpan::from_single(span.0); } } @@ -770,7 +798,7 @@ impl fmt::Debug for Group { /// forms of `Spacing` returned. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] #[derive(Clone)] -pub struct Punct(bridge::client::Punct); +pub struct Punct(bridge::Punct); #[stable(feature = "proc_macro_lib2", since = "1.29.0")] impl !Send for Punct {} @@ -803,13 +831,20 @@ impl Punct { /// which can be further configured with the `set_span` method below. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn new(ch: char, spacing: Spacing) -> Punct { - Punct(bridge::client::Punct::new(ch, spacing)) + const LEGAL_CHARS: &[char] = &[ + '=', '<', '>', '!', '~', '+', '-', '*', '/', '%', '^', '&', '|', '@', '.', ',', ';', + ':', '#', '$', '?', '\'', + ]; + if !LEGAL_CHARS.contains(&ch) { + panic!("unsupported character `{:?}`", ch); + } + Punct(bridge::Punct { ch, joint: spacing == Spacing::Joint, span: Span::call_site().0 }) } /// Returns the value of this punctuation character as `char`. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn as_char(&self) -> char { - self.0.as_char() + self.0.ch } /// Returns the spacing of this punctuation character, indicating whether it's immediately @@ -818,28 +853,19 @@ impl Punct { /// (`Alone`) so the operator has certainly ended. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn spacing(&self) -> Spacing { - self.0.spacing() + if self.0.joint { Spacing::Joint } else { Spacing::Alone } } /// Returns the span for this punctuation character. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn span(&self) -> Span { - Span(self.0.span()) + Span(self.0.span) } /// Configure the span for this punctuation character. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn set_span(&mut self, span: Span) { - self.0 = self.0.with_span(span.0); - } -} - -// N.B., the bridge only provides `to_string`, implement `fmt::Display` -// based on it (the reverse of the usual relationship between the two). -#[stable(feature = "proc_macro_lib", since = "1.15.0")] -impl ToString for Punct { - fn to_string(&self) -> String { - TokenStream::from(TokenTree::from(self.clone())).to_string() + self.0.span = span.0; } } @@ -848,7 +874,7 @@ impl ToString for Punct { #[stable(feature = "proc_macro_lib2", since = "1.29.0")] impl fmt::Display for Punct { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(&self.to_string()) + write!(f, "{}", self.as_char()) } } @@ -880,7 +906,7 @@ impl PartialEq for char { /// An identifier (`ident`). #[derive(Clone)] #[stable(feature = "proc_macro_lib2", since = "1.29.0")] -pub struct Ident(bridge::client::Ident); +pub struct Ident(bridge::Ident); impl Ident { /// Creates a new `Ident` with the given `string` as well as the specified @@ -904,7 +930,11 @@ impl Ident { /// tokens, requires a `Span` to be specified at construction. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn new(string: &str, span: Span) -> Ident { - Ident(bridge::client::Ident::new(string, span.0, false)) + Ident(bridge::Ident { + sym: bridge::client::Symbol::new_ident(string, false), + is_raw: false, + span: span.0, + }) } /// Same as `Ident::new`, but creates a raw identifier (`r#ident`). @@ -913,29 +943,24 @@ impl Ident { /// (e.g. `self`, `super`) are not supported, and will cause a panic. #[stable(feature = "proc_macro_raw_ident", since = "1.47.0")] pub fn new_raw(string: &str, span: Span) -> Ident { - Ident(bridge::client::Ident::new(string, span.0, true)) + Ident(bridge::Ident { + sym: bridge::client::Symbol::new_ident(string, true), + is_raw: true, + span: span.0, + }) } /// Returns the span of this `Ident`, encompassing the entire string returned /// by [`to_string`](Self::to_string). #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn span(&self) -> Span { - Span(self.0.span()) + Span(self.0.span) } /// Configures the span of this `Ident`, possibly changing its hygiene context. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn set_span(&mut self, span: Span) { - self.0 = self.0.with_span(span.0); - } -} - -// N.B., the bridge only provides `to_string`, implement `fmt::Display` -// based on it (the reverse of the usual relationship between the two). -#[stable(feature = "proc_macro_lib", since = "1.15.0")] -impl ToString for Ident { - fn to_string(&self) -> String { - TokenStream::from(TokenTree::from(self.clone())).to_string() + self.0.span = span.0; } } @@ -944,7 +969,7 @@ impl ToString for Ident { #[stable(feature = "proc_macro_lib2", since = "1.29.0")] impl fmt::Display for Ident { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(&self.to_string()) + if self.0.is_raw { write!(f, "r#{}", self.0.sym) } else { write!(f, "{}", self.0.sym) } } } @@ -964,7 +989,7 @@ impl fmt::Debug for Ident { /// Boolean literals like `true` and `false` do not belong here, they are `Ident`s. #[derive(Clone)] #[stable(feature = "proc_macro_lib2", since = "1.29.0")] -pub struct Literal(bridge::client::Literal); +pub struct Literal(bridge::Literal); macro_rules! suffixed_int_literals { ($($name:ident => $kind:ident,)*) => ($( @@ -981,7 +1006,12 @@ macro_rules! suffixed_int_literals { /// below. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn $name(n: $kind) -> Literal { - Literal(bridge::client::Literal::typed_integer(&n.to_string(), stringify!($kind))) + Literal(bridge::Literal { + kind: bridge::LitKind::Integer, + symbol: bridge::client::Symbol::new(&n.to_string()), + suffix: Some(bridge::client::Symbol::new(stringify!($kind))), + span: Span::call_site().0, + }) } )*) } @@ -1003,12 +1033,26 @@ macro_rules! unsuffixed_int_literals { /// below. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn $name(n: $kind) -> Literal { - Literal(bridge::client::Literal::integer(&n.to_string())) + Literal(bridge::Literal { + kind: bridge::LitKind::Integer, + symbol: bridge::client::Symbol::new(&n.to_string()), + suffix: None, + span: Span::call_site().0, + }) } )*) } impl Literal { + fn new(kind: bridge::LitKind, value: &str, suffix: Option<&str>) -> Self { + Literal(bridge::Literal { + kind, + symbol: bridge::client::Symbol::new(value), + suffix: suffix.map(bridge::client::Symbol::new), + span: Span::call_site().0, + }) + } + suffixed_int_literals! { u8_suffixed => u8, u16_suffixed => u16, @@ -1056,7 +1100,7 @@ impl Literal { if !n.is_finite() { panic!("Invalid float literal {}", n); } - Literal(bridge::client::Literal::float(&n.to_string())) + Literal::new(bridge::LitKind::Float, &n.to_string(), None) } /// Creates a new suffixed floating-point literal. @@ -1077,7 +1121,7 @@ impl Literal { if !n.is_finite() { panic!("Invalid float literal {}", n); } - Literal(bridge::client::Literal::f32(&n.to_string())) + Literal::new(bridge::LitKind::Float, &n.to_string(), Some("f32")) } /// Creates a new unsuffixed floating-point literal. @@ -1097,7 +1141,7 @@ impl Literal { if !n.is_finite() { panic!("Invalid float literal {}", n); } - Literal(bridge::client::Literal::float(&n.to_string())) + Literal::new(bridge::LitKind::Float, &n.to_string(), None) } /// Creates a new suffixed floating-point literal. @@ -1118,37 +1162,49 @@ impl Literal { if !n.is_finite() { panic!("Invalid float literal {}", n); } - Literal(bridge::client::Literal::f64(&n.to_string())) + Literal::new(bridge::LitKind::Float, &n.to_string(), Some("f64")) } /// String literal. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn string(string: &str) -> Literal { - Literal(bridge::client::Literal::string(string)) + let mut escaped = String::new(); + for ch in string.chars() { + escaped.extend(ch.escape_debug()); + } + Literal::new(bridge::LitKind::Str, &escaped, None) } /// Character literal. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn character(ch: char) -> Literal { - Literal(bridge::client::Literal::character(ch)) + let mut escaped = String::new(); + escaped.extend(ch.escape_unicode()); + Literal::new(bridge::LitKind::Char, &escaped, None) } /// Byte string literal. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn byte_string(bytes: &[u8]) -> Literal { - Literal(bridge::client::Literal::byte_string(bytes)) + let string = bytes + .iter() + .cloned() + .flat_map(std::ascii::escape_default) + .map(Into::::into) + .collect::(); + Literal::new(bridge::LitKind::ByteStr, &string, None) } /// Returns the span encompassing this literal. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn span(&self) -> Span { - Span(self.0.span()) + Span(self.0.span) } /// Configures the span associated for this literal. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn set_span(&mut self, span: Span) { - self.0.set_span(span.0); + self.0.span = span.0; } /// Returns a `Span` that is a subset of `self.span()` containing only the @@ -1164,7 +1220,12 @@ impl Literal { // was 'c' or whether it was '\u{63}'. #[unstable(feature = "proc_macro_span", issue = "54725")] pub fn subspan>(&self, range: R) -> Option { - self.0.subspan(range.start_bound().cloned(), range.end_bound().cloned()).map(Span) + bridge::client::FreeFunctions::literal_subspan( + self.0.clone(), + range.start_bound().cloned(), + range.end_bound().cloned(), + ) + .map(Span) } } @@ -1183,35 +1244,56 @@ impl FromStr for Literal { type Err = LexError; fn from_str(src: &str) -> Result { - match bridge::client::Literal::from_str(src) { + match bridge::client::FreeFunctions::literal_from_str(src) { Ok(literal) => Ok(Literal(literal)), Err(()) => Err(LexError::new()), } } } -// N.B., the bridge only provides `to_string`, implement `fmt::Display` -// based on it (the reverse of the usual relationship between the two). -#[stable(feature = "proc_macro_lib", since = "1.15.0")] -impl ToString for Literal { - fn to_string(&self) -> String { - TokenStream::from(TokenTree::from(self.clone())).to_string() - } -} - /// Prints the literal as a string that should be losslessly convertible /// back into the same literal (except for possible rounding for floating point literals). #[stable(feature = "proc_macro_lib2", since = "1.29.0")] impl fmt::Display for Literal { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(&self.to_string()) + // Based on `literal_to_string` from `pprust/state.rs` + match self.0.kind { + bridge::LitKind::Byte => write!(f, "b'{}'", self.0.symbol)?, + bridge::LitKind::Char => write!(f, "'{}'", self.0.symbol)?, + bridge::LitKind::Str => write!(f, "\"{}\"", self.0.symbol)?, + bridge::LitKind::StrRaw(n) => write!( + f, + "r{delim}\"{string}\"{delim}", + delim = "#".repeat(n as usize), + string = self.0.symbol + )?, + bridge::LitKind::ByteStr => write!(f, "b\"{}\"", self.0.symbol)?, + bridge::LitKind::ByteStrRaw(n) => write!( + f, + "br{delim}\"{string}\"{delim}", + delim = "#".repeat(n as usize), + string = self.0.symbol + )?, + _ => write!(f, "{}", self.0.symbol)?, + } + if let Some(suffix) = self.0.suffix { + write!(f, "{}", suffix)?; + } + Ok(()) } } #[stable(feature = "proc_macro_lib2", since = "1.29.0")] impl fmt::Debug for Literal { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.0.fmt(f) + f.debug_struct("Literal") + // format the kind without quotes, as in `kind: Float` + .field("kind", &self.0.kind) + .field("symbol", &self.0.symbol) + // format `Some("...")` on one line even in {:#?} mode + .field("suffix", &format_args!("{:?}", &self.0.suffix)) + .field("span", &self.0.span) + .finish() } } diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs index ea587210b4f4e..578f56a7a8630 100644 --- a/src/tools/tidy/src/deps.rs +++ b/src/tools/tidy/src/deps.rs @@ -97,6 +97,7 @@ const PERMITTED_DEPENDENCIES: &[&str] = &[ "compiler_builtins", "cpuid-bool", "crc32fast", + "crossbeam-channel", "crossbeam-deque", "crossbeam-epoch", "crossbeam-queue",