From 674525c7631711872a099622c0fdfca6e42520b2 Mon Sep 17 00:00:00 2001 From: Nika Layzell Date: Mon, 28 Jun 2021 19:43:40 -0400 Subject: [PATCH 01/12] proc_macro: support encoding/decoding structs with type parameters --- library/proc_macro/src/bridge/rpc.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/library/proc_macro/src/bridge/rpc.rs b/library/proc_macro/src/bridge/rpc.rs index 42432563faf33..564a7824a730c 100644 --- a/library/proc_macro/src/bridge/rpc.rs +++ b/library/proc_macro/src/bridge/rpc.rs @@ -43,15 +43,17 @@ macro_rules! rpc_encode_decode { } } }; - (struct $name:ident { $($field:ident),* $(,)? }) => { - impl Encode for $name { + (struct $name:ident $(<$($T:ident),+>)? { $($field:ident),* $(,)? }) => { + impl),+)?> Encode for $name $(<$($T),+>)? { fn encode(self, w: &mut Writer, s: &mut S) { $(self.$field.encode(w, s);)* } } - impl DecodeMut<'_, '_, S> for $name { - fn decode(r: &mut Reader<'_>, s: &mut S) -> Self { + impl DecodeMut<'a, 's, S>),+)?> DecodeMut<'a, '_, S> + for $name $(<$($T),+>)? + { + fn decode(r: &mut Reader<'a>, s: &mut S) -> Self { $name { $($field: DecodeMut::decode(r, s)),* } From 0f4f8e511d4f12675d6e931f4c3c26344a92db0d Mon Sep 17 00:00:00 2001 From: Nika Layzell Date: Thu, 1 Jul 2021 12:56:07 -0400 Subject: [PATCH 02/12] proc_macro: support encoding/decoding Vec --- library/proc_macro/src/bridge/mod.rs | 15 +++++++++++++++ library/proc_macro/src/bridge/rpc.rs | 20 ++++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/library/proc_macro/src/bridge/mod.rs b/library/proc_macro/src/bridge/mod.rs index a2953b68564a8..09052f96d7e8a 100644 --- a/library/proc_macro/src/bridge/mod.rs +++ b/library/proc_macro/src/bridge/mod.rs @@ -329,6 +329,21 @@ impl Unmark for Result { } } +impl Mark for Vec { + type Unmarked = Vec; + fn mark(unmarked: Self::Unmarked) -> Self { + // Should be a no-op due to std's in-place collect optimizations. + unmarked.into_iter().map(T::mark).collect() + } +} +impl Unmark for Vec { + type Unmarked = Vec; + fn unmark(self) -> Self::Unmarked { + // Should be a no-op due to std's in-place collect optimizations. + self.into_iter().map(T::unmark).collect() + } +} + macro_rules! mark_noop { ($($ty:ty),* $(,)?) => { $( diff --git a/library/proc_macro/src/bridge/rpc.rs b/library/proc_macro/src/bridge/rpc.rs index 564a7824a730c..1dca14ea5d8db 100644 --- a/library/proc_macro/src/bridge/rpc.rs +++ b/library/proc_macro/src/bridge/rpc.rs @@ -248,6 +248,26 @@ impl DecodeMut<'_, '_, S> for String { } } +impl> Encode for Vec { + fn encode(self, w: &mut Writer, s: &mut S) { + self.len().encode(w, s); + for x in self { + x.encode(w, s); + } + } +} + +impl DecodeMut<'a, 's, S>> DecodeMut<'a, '_, S> for Vec { + fn decode(r: &mut Reader<'a>, s: &mut S) -> Self { + let len = usize::decode(r, s); + let mut vec = Vec::with_capacity(len); + for _ in 0..len { + vec.push(T::decode(r, s)); + } + vec + } +} + /// Simplified version of panic payloads, ignoring /// types other than `&'static str` and `String`. pub enum PanicMessage { From d5f2ff5c972f0ec45604a14b76fe2cf605d11be3 Mon Sep 17 00:00:00 2001 From: Nika Layzell Date: Tue, 29 Jun 2021 16:36:10 -0400 Subject: [PATCH 03/12] proc_macro: use macros to simplify aggregate Mark/Unmark definitions --- library/proc_macro/src/bridge/mod.rs | 77 +++++++++++++++++++--------- 1 file changed, 53 insertions(+), 24 deletions(-) diff --git a/library/proc_macro/src/bridge/mod.rs b/library/proc_macro/src/bridge/mod.rs index 09052f96d7e8a..e13eb4aa1e411 100644 --- a/library/proc_macro/src/bridge/mod.rs +++ b/library/proc_macro/src/bridge/mod.rs @@ -401,6 +401,58 @@ rpc_encode_decode!( } ); +macro_rules! mark_compound { + (struct $name:ident <$($T:ident),+> { $($field:ident),* $(,)? }) => { + impl<$($T: Mark),+> Mark for $name <$($T),+> { + type Unmarked = $name <$($T::Unmarked),+>; + fn mark(unmarked: Self::Unmarked) -> Self { + $name { + $($field: Mark::mark(unmarked.$field)),* + } + } + } + + impl<$($T: Unmark),+> Unmark for $name <$($T),+> { + type Unmarked = $name <$($T::Unmarked),+>; + fn unmark(self) -> Self::Unmarked { + $name { + $($field: Unmark::unmark(self.$field)),* + } + } + } + }; + (enum $name:ident <$($T:ident),+> { $($variant:ident $(($field:ident))?),* $(,)? }) => { + impl<$($T: Mark),+> Mark for $name <$($T),+> { + type Unmarked = $name <$($T::Unmarked),+>; + fn mark(unmarked: Self::Unmarked) -> Self { + match unmarked { + $($name::$variant $(($field))? => { + $name::$variant $((Mark::mark($field)))? + })* + } + } + } + + impl<$($T: Unmark),+> Unmark for $name <$($T),+> { + type Unmarked = $name <$($T::Unmarked),+>; + fn unmark(self) -> Self::Unmarked { + match self { + $($name::$variant $(($field))? => { + $name::$variant $((Unmark::unmark($field)))? + })* + } + } + } + } +} + +macro_rules! compound_traits { + ($($t:tt)*) => { + rpc_encode_decode!($($t)*); + mark_compound!($($t)*); + }; +} + #[derive(Clone)] pub enum TokenTree { Group(G), @@ -409,30 +461,7 @@ pub enum TokenTree { Literal(L), } -impl Mark for TokenTree { - type Unmarked = TokenTree; - fn mark(unmarked: Self::Unmarked) -> Self { - match unmarked { - TokenTree::Group(tt) => TokenTree::Group(G::mark(tt)), - TokenTree::Punct(tt) => TokenTree::Punct(P::mark(tt)), - TokenTree::Ident(tt) => TokenTree::Ident(I::mark(tt)), - TokenTree::Literal(tt) => TokenTree::Literal(L::mark(tt)), - } - } -} -impl Unmark for TokenTree { - type Unmarked = TokenTree; - fn unmark(self) -> Self::Unmarked { - match self { - TokenTree::Group(tt) => TokenTree::Group(tt.unmark()), - TokenTree::Punct(tt) => TokenTree::Punct(tt.unmark()), - TokenTree::Ident(tt) => TokenTree::Ident(tt.unmark()), - TokenTree::Literal(tt) => TokenTree::Literal(tt.unmark()), - } - } -} - -rpc_encode_decode!( +compound_traits!( enum TokenTree { Group(tt), Punct(tt), From afc36cc1af9dbb18a7b6d10b7f8af95e081beff1 Mon Sep 17 00:00:00 2001 From: Nika Layzell Date: Tue, 29 Jun 2021 14:49:54 -0400 Subject: [PATCH 04/12] proc_macro: cache static spans in client's thread-local state This greatly improves the performance of the very frequently called `call_site()` macro when running in a cross-thread configuration. --- .../rustc_expand/src/proc_macro_server.rs | 30 ++-- library/proc_macro/src/bridge/client.rs | 144 +++++++++++------- library/proc_macro/src/bridge/mod.rs | 35 +++-- library/proc_macro/src/bridge/server.rs | 55 ++++--- library/proc_macro/src/lib.rs | 2 +- 5 files changed, 164 insertions(+), 102 deletions(-) diff --git a/compiler/rustc_expand/src/proc_macro_server.rs b/compiler/rustc_expand/src/proc_macro_server.rs index 92315c4d4f6c7..97163ce31af34 100644 --- a/compiler/rustc_expand/src/proc_macro_server.rs +++ b/compiler/rustc_expand/src/proc_macro_server.rs @@ -388,7 +388,10 @@ impl<'a> Rustc<'a> { } fn lit(&mut self, kind: token::LitKind, symbol: Symbol, suffix: Option) -> Literal { - Literal { lit: token::Lit::new(kind, symbol, suffix), span: server::Span::call_site(self) } + Literal { + lit: token::Lit::new(kind, symbol, suffix), + span: server::Context::call_site(self), + } } } @@ -484,7 +487,7 @@ impl server::Group for Rustc<'_> { Group { delimiter, stream, - span: DelimSpan::from_single(server::Span::call_site(self)), + span: DelimSpan::from_single(server::Context::call_site(self)), flatten: false, } } @@ -510,7 +513,7 @@ impl server::Group for Rustc<'_> { impl server::Punct for Rustc<'_> { fn new(&mut self, ch: char, spacing: Spacing) -> Self::Punct { - Punct::new(ch, spacing == Spacing::Joint, server::Span::call_site(self)) + Punct::new(ch, spacing == Spacing::Joint, server::Context::call_site(self)) } fn as_char(&mut self, punct: Self::Punct) -> char { punct.ch @@ -712,15 +715,6 @@ impl server::Span for Rustc<'_> { format!("{:?} bytes({}..{})", span.ctxt(), span.lo().0, span.hi().0) } } - fn def_site(&mut self) -> Self::Span { - self.def_site - } - fn call_site(&mut self) -> Self::Span { - self.call_site - } - fn mixed_site(&mut self) -> Self::Span { - self.mixed_site - } fn source_file(&mut self, span: Self::Span) -> Self::SourceFile { self.sess.source_map().lookup_char_pos(span.lo()).file } @@ -801,6 +795,18 @@ impl server::Span for Rustc<'_> { } } +impl server::Context for Rustc<'_> { + fn def_site(&mut self) -> Self::Span { + self.def_site + } + fn call_site(&mut self) -> Self::Span { + self.call_site + } + fn mixed_site(&mut self) -> Self::Span { + self.mixed_site + } +} + // See issue #74616 for details fn ident_name_compatibility_hack( nt: &Nonterminal, diff --git a/library/proc_macro/src/bridge/client.rs b/library/proc_macro/src/bridge/client.rs index c6bec5a6fbdc2..19b991a260454 100644 --- a/library/proc_macro/src/bridge/client.rs +++ b/library/proc_macro/src/bridge/client.rs @@ -222,6 +222,20 @@ impl Clone for SourceFile { } } +impl Span { + pub(crate) fn def_site() -> Span { + Bridge::with(|bridge| bridge.context.def_site) + } + + pub(crate) fn call_site() -> Span { + Bridge::with(|bridge| bridge.context.call_site) + } + + pub(crate) fn mixed_site() -> Span { + Bridge::with(|bridge| bridge.context.mixed_site) + } +} + impl fmt::Debug for Span { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.write_str(&self.debug()) @@ -255,6 +269,21 @@ macro_rules! define_client_side { } with_api!(self, self, define_client_side); +struct Bridge<'a> { + /// Reusable buffer (only `clear`-ed, never shrunk), primarily + /// used for making requests. + cached_buffer: Buffer, + + /// Server-side function that the client uses to make requests. + dispatch: closure::Closure<'a, Buffer, Buffer>, + + /// Provided context for this macro expansion. + context: ExpnContext, +} + +impl<'a> !Send for Bridge<'a> {} +impl<'a> !Sync for Bridge<'a> {} + enum BridgeState<'a> { /// No server is currently connected to this client. NotConnected, @@ -297,34 +326,6 @@ impl BridgeState<'_> { } impl Bridge<'_> { - pub(crate) fn is_available() -> bool { - BridgeState::with(|state| match state { - BridgeState::Connected(_) | BridgeState::InUse => true, - BridgeState::NotConnected => false, - }) - } - - fn enter(self, f: impl FnOnce() -> R) -> R { - let force_show_panics = self.force_show_panics; - // Hide the default panic output within `proc_macro` expansions. - // NB. the server can't do this because it may use a different libstd. - static HIDE_PANICS_DURING_EXPANSION: Once = Once::new(); - HIDE_PANICS_DURING_EXPANSION.call_once(|| { - let prev = panic::take_hook(); - panic::set_hook(Box::new(move |info| { - let show = BridgeState::with(|state| match state { - BridgeState::NotConnected => true, - BridgeState::Connected(_) | BridgeState::InUse => force_show_panics, - }); - if show { - prev(info) - } - })); - }); - - BRIDGE_STATE.with(|state| state.set(BridgeState::Connected(self), f)) - } - fn with(f: impl FnOnce(&mut Bridge<'_>) -> R) -> R { BridgeState::with(|state| match state { BridgeState::NotConnected => { @@ -338,6 +339,13 @@ impl Bridge<'_> { } } +pub(crate) fn is_available() -> bool { + BridgeState::with(|state| match state { + BridgeState::Connected(_) | BridgeState::InUse => true, + BridgeState::NotConnected => false, + }) +} + /// A client-side "global object" (usually a function pointer), /// which may be using a different `proc_macro` from the one /// used by the server, but can be interacted with compatibly. @@ -352,44 +360,66 @@ pub struct Client { // FIXME(eddyb) use a reference to the `static COUNTERS`, instead of // a wrapper `fn` pointer, once `const fn` can reference `static`s. pub(super) get_handle_counters: extern "C" fn() -> &'static HandleCounters, - pub(super) run: extern "C" fn(Bridge<'_>, F) -> Buffer, + pub(super) run: extern "C" fn(BridgeConfig<'_>, F) -> Buffer, pub(super) f: F, } +fn maybe_install_panic_hook(force_show_panics: bool) { + // Hide the default panic output within `proc_macro` expansions. + // NB. the server can't do this because it may use a different libstd. + static HIDE_PANICS_DURING_EXPANSION: Once = Once::new(); + HIDE_PANICS_DURING_EXPANSION.call_once(|| { + let prev = panic::take_hook(); + panic::set_hook(Box::new(move |info| { + let show = BridgeState::with(|state| match state { + BridgeState::NotConnected => true, + BridgeState::Connected(_) | BridgeState::InUse => force_show_panics, + }); + if show { + prev(info) + } + })); + }); +} + /// Client-side helper for handling client panics, entering the bridge, /// deserializing input and serializing output. // FIXME(eddyb) maybe replace `Bridge::enter` with this? fn run_client DecodeMut<'a, 's, ()>, R: Encode<()>>( - mut bridge: Bridge<'_>, + config: BridgeConfig<'_>, f: impl FnOnce(A) -> R, ) -> Buffer { - // The initial `cached_buffer` contains the input. - let mut b = bridge.cached_buffer.take(); + let BridgeConfig { input: mut b, dispatch, force_show_panics } = config; panic::catch_unwind(panic::AssertUnwindSafe(|| { - bridge.enter(|| { - let reader = &mut &b[..]; - let input = A::decode(reader, &mut ()); - - // Put the `cached_buffer` back in the `Bridge`, for requests. - Bridge::with(|bridge| bridge.cached_buffer = b.take()); - - let output = f(input); - - // Take the `cached_buffer` back out, for the output value. - b = Bridge::with(|bridge| bridge.cached_buffer.take()); - - // HACK(eddyb) Separate encoding a success value (`Ok(output)`) - // from encoding a panic (`Err(e: PanicMessage)`) to avoid - // having handles outside the `bridge.enter(|| ...)` scope, and - // to catch panics that could happen while encoding the success. - // - // Note that panics should be impossible beyond this point, but - // this is defensively trying to avoid any accidental panicking - // reaching the `extern "C"` (which should `abort` but may not - // at the moment, so this is also potentially preventing UB). - b.clear(); - Ok::<_, ()>(output).encode(&mut b, &mut ()); + maybe_install_panic_hook(force_show_panics); + + let reader = &mut &b[..]; + let (input, context) = <(A, ExpnContext)>::decode(reader, &mut ()); + + // Put the buffer we used for input back in the `Bridge` for requests. + let new_state = + BridgeState::Connected(Bridge { cached_buffer: b.take(), dispatch, context }); + + BRIDGE_STATE.with(|state| { + state.set(new_state, || { + let output = f(input); + + // Take the `cached_buffer` back out, for the output value. + b = Bridge::with(|bridge| bridge.cached_buffer.take()); + + // HACK(eddyb) Separate encoding a success value (`Ok(output)`) + // from encoding a panic (`Err(e: PanicMessage)`) to avoid + // having handles outside the `bridge.enter(|| ...)` scope, and + // to catch panics that could happen while encoding the success. + // + // Note that panics should be impossible beyond this point, but + // this is defensively trying to avoid any accidental panicking + // reaching the `extern "C"` (which should `abort` but may not + // at the moment, so this is also potentially preventing UB). + b.clear(); + Ok::<_, ()>(output).encode(&mut b, &mut ()); + }) }) })) .map_err(PanicMessage::from) @@ -404,7 +434,7 @@ impl Client crate::TokenStream> { #[rustc_allow_const_fn_unstable(const_fn)] pub const fn expand1(f: fn(crate::TokenStream) -> crate::TokenStream) -> Self { extern "C" fn run( - bridge: Bridge<'_>, + bridge: BridgeConfig<'_>, f: impl FnOnce(crate::TokenStream) -> crate::TokenStream, ) -> Buffer { run_client(bridge, |input| f(crate::TokenStream(input)).0) @@ -419,7 +449,7 @@ impl Client crate::TokenStream> { f: fn(crate::TokenStream, crate::TokenStream) -> crate::TokenStream, ) -> Self { extern "C" fn run( - bridge: Bridge<'_>, + bridge: BridgeConfig<'_>, f: impl FnOnce(crate::TokenStream, crate::TokenStream) -> crate::TokenStream, ) -> Buffer { run_client(bridge, |(input, input2)| { diff --git a/library/proc_macro/src/bridge/mod.rs b/library/proc_macro/src/bridge/mod.rs index e13eb4aa1e411..c88bbdcd462fd 100644 --- a/library/proc_macro/src/bridge/mod.rs +++ b/library/proc_macro/src/bridge/mod.rs @@ -152,9 +152,6 @@ macro_rules! with_api { }, Span { fn debug($self: $S::Span) -> String; - fn def_site() -> $S::Span; - fn call_site() -> $S::Span; - fn mixed_site() -> $S::Span; fn source_file($self: $S::Span) -> $S::SourceFile; fn parent($self: $S::Span) -> Option<$S::Span>; fn source($self: $S::Span) -> $S::Span; @@ -210,16 +207,15 @@ use buffer::Buffer; pub use rpc::PanicMessage; use rpc::{Decode, DecodeMut, Encode, Reader, Writer}; -/// An active connection between a server and a client. -/// The server creates the bridge (`Bridge::run_server` in `server.rs`), -/// then passes it to the client through the function pointer in the `run` -/// field of `client::Client`. The client holds its copy of the `Bridge` +/// Configuration for establishing an active connection between a server and a +/// client. The server creates the bridge config (`run_server` in `server.rs`), +/// then passes it to the client through the function pointer in the `run` field +/// of `client::Client`. The client constructs a local `Bridge` from the config /// in TLS during its execution (`Bridge::{enter, with}` in `client.rs`). #[repr(C)] -pub struct Bridge<'a> { - /// Reusable buffer (only `clear`-ed, never shrunk), primarily - /// used for making requests, but also for passing input to client. - cached_buffer: Buffer, +pub struct BridgeConfig<'a> { + /// Buffer used to pass initial input to the client. + input: Buffer, /// Server-side function that the client uses to make requests. dispatch: closure::Closure<'a, Buffer, Buffer>, @@ -228,8 +224,8 @@ pub struct Bridge<'a> { force_show_panics: bool, } -impl<'a> !Sync for Bridge<'a> {} -impl<'a> !Send for Bridge<'a> {} +impl<'a> !Sync for BridgeConfig<'a> {} +impl<'a> !Send for BridgeConfig<'a> {} #[forbid(unsafe_code)] #[allow(non_camel_case_types)] @@ -469,3 +465,16 @@ compound_traits!( Literal(tt), } ); + +/// Context provided alongside the initial inputs for a macro expansion. +/// Provides values such as spans which are used frequently to avoid RPC. +#[derive(Clone)] +struct ExpnContext { + def_site: S, + call_site: S, + mixed_site: S, +} + +compound_traits!( + struct ExpnContext { def_site, call_site, mixed_site } +); diff --git a/library/proc_macro/src/bridge/server.rs b/library/proc_macro/src/bridge/server.rs index 1b3ccf4c18e70..b4b16e8ad0cf7 100644 --- a/library/proc_macro/src/bridge/server.rs +++ b/library/proc_macro/src/bridge/server.rs @@ -39,6 +39,13 @@ macro_rules! associated_item { ($($item:tt)*) => ($($item)*;) } +/// Helper methods defined by `Server` types not invoked over RPC. +pub trait Context: Types { + fn def_site(&mut self) -> Self::Span; + fn call_site(&mut self) -> Self::Span; + fn mixed_site(&mut self) -> Self::Span; +} + macro_rules! declare_server_traits { ($($name:ident { $(fn $method:ident($($arg:ident: $arg_ty:ty),* $(,)?) $(-> $ret_ty:ty)?;)* @@ -51,14 +58,26 @@ macro_rules! declare_server_traits { $(associated_item!(fn $method(&mut self, $($arg: $arg_ty),*) $(-> $ret_ty)?);)* })* - pub trait Server: Types $(+ $name)* {} - impl Server for S {} + pub trait Server: Types + Context $(+ $name)* {} + impl Server for S {} } } with_api!(Self, self_, declare_server_traits); pub(super) struct MarkedTypes(S); +impl Context for MarkedTypes { + fn def_site(&mut self) -> Self::Span { + <_>::mark(Context::def_site(&mut self.0)) + } + fn call_site(&mut self) -> Self::Span { + <_>::mark(Context::call_site(&mut self.0)) + } + fn mixed_site(&mut self) -> Self::Span { + <_>::mark(Context::mixed_site(&mut self.0)) + } +} + macro_rules! define_mark_types_impls { ($($name:ident { $(fn $method:ident($($arg:ident: $arg_ty:ty),* $(,)?) $(-> $ret_ty:ty)?;)* @@ -133,7 +152,7 @@ pub trait ExecutionStrategy { &self, dispatcher: &mut impl DispatcherTrait, input: Buffer, - run_client: extern "C" fn(Bridge<'_>, D) -> Buffer, + run_client: extern "C" fn(BridgeConfig<'_>, D) -> Buffer, client_data: D, force_show_panics: bool, ) -> Buffer; @@ -146,14 +165,14 @@ impl ExecutionStrategy for SameThread { &self, dispatcher: &mut impl DispatcherTrait, input: Buffer, - run_client: extern "C" fn(Bridge<'_>, D) -> Buffer, + run_client: extern "C" fn(BridgeConfig<'_>, D) -> Buffer, client_data: D, force_show_panics: bool, ) -> Buffer { let mut dispatch = |b| dispatcher.dispatch(b); run_client( - Bridge { cached_buffer: input, dispatch: (&mut dispatch).into(), force_show_panics }, + BridgeConfig { input, dispatch: (&mut dispatch).into(), force_show_panics }, client_data, ) } @@ -169,7 +188,7 @@ impl ExecutionStrategy for CrossThread1 { &self, dispatcher: &mut impl DispatcherTrait, input: Buffer, - run_client: extern "C" fn(Bridge<'_>, D) -> Buffer, + run_client: extern "C" fn(BridgeConfig<'_>, D) -> Buffer, client_data: D, force_show_panics: bool, ) -> Buffer { @@ -185,11 +204,7 @@ impl ExecutionStrategy for CrossThread1 { }; run_client( - Bridge { - cached_buffer: input, - dispatch: (&mut dispatch).into(), - force_show_panics, - }, + BridgeConfig { input, dispatch: (&mut dispatch).into(), force_show_panics }, client_data, ) }); @@ -209,7 +224,7 @@ impl ExecutionStrategy for CrossThread2 { &self, dispatcher: &mut impl DispatcherTrait, input: Buffer, - run_client: extern "C" fn(Bridge<'_>, D) -> Buffer, + run_client: extern "C" fn(BridgeConfig<'_>, D) -> Buffer, client_data: D, force_show_panics: bool, ) -> Buffer { @@ -237,11 +252,7 @@ impl ExecutionStrategy for CrossThread2 { }; let r = run_client( - Bridge { - cached_buffer: input, - dispatch: (&mut dispatch).into(), - force_show_panics, - }, + BridgeConfig { input, dispatch: (&mut dispatch).into(), force_show_panics }, client_data, ); @@ -278,15 +289,21 @@ fn run_server< handle_counters: &'static client::HandleCounters, server: S, input: I, - run_client: extern "C" fn(Bridge<'_>, D) -> Buffer, + run_client: extern "C" fn(BridgeConfig<'_>, D) -> Buffer, client_data: D, force_show_panics: bool, ) -> Result { let mut dispatcher = Dispatcher { handle_store: HandleStore::new(handle_counters), server: MarkedTypes(server) }; + let expn_context = ExpnContext { + def_site: dispatcher.server.def_site(), + call_site: dispatcher.server.call_site(), + mixed_site: dispatcher.server.mixed_site(), + }; + let mut b = Buffer::new(); - input.encode(&mut b, &mut dispatcher.handle_store); + (input, expn_context).encode(&mut b, &mut dispatcher.handle_store); b = strategy.run_bridge_and_client( &mut dispatcher, diff --git a/library/proc_macro/src/lib.rs b/library/proc_macro/src/lib.rs index 9b155db6d7b1b..353eb3c45c1fe 100644 --- a/library/proc_macro/src/lib.rs +++ b/library/proc_macro/src/lib.rs @@ -63,7 +63,7 @@ use std::{error, fmt, iter, mem}; /// inside of a procedural macro, false if invoked from any other binary. #[unstable(feature = "proc_macro_is_available", issue = "71436")] pub fn is_available() -> bool { - bridge::Bridge::is_available() + bridge::client::is_available() } /// The main type provided by this crate, representing an abstract stream of From b6f8dc97ef66d132cad2827db981da737c9b2a21 Mon Sep 17 00:00:00 2001 From: Nika Layzell Date: Tue, 29 Jun 2021 22:17:12 -0400 Subject: [PATCH 05/12] proc_macro: add support for sending messages without waiting for a reply This has minimal impact without changes from a later part, as it also requires support from the ExecutionStrategy. Messages are able to return owning handles without waiting by allocating an ID from the client thread, and passing it up to the server to fill with a response. If the server panics, it will be an ICE, and no further requests will be handled. --- library/proc_macro/src/bridge/client.rs | 98 ++++++++++--- library/proc_macro/src/bridge/handle.rs | 6 +- library/proc_macro/src/bridge/mod.rs | 181 ++++++++++++++---------- library/proc_macro/src/bridge/server.rs | 22 ++- 4 files changed, 209 insertions(+), 98 deletions(-) diff --git a/library/proc_macro/src/bridge/client.rs b/library/proc_macro/src/bridge/client.rs index 19b991a260454..9d41497fb28c9 100644 --- a/library/proc_macro/src/bridge/client.rs +++ b/library/proc_macro/src/bridge/client.rs @@ -2,6 +2,23 @@ use super::*; +use std::sync::atomic::Ordering; + +trait OwnedHandle { + /// Create a new handle of this type from the client side, which may be used + /// later by the server. + /// + /// Should only be called on the client. + fn next_raw_handle() -> handle::Handle; + + /// Create an instance of the owning handle object for this raw handle. The + /// raw handle should've previously been created with `next_raw_handle`, and + /// the corresponding message should've been sent to the server. + /// + /// Should only be called on the client. + fn from_raw_handle(handle: handle::Handle) -> Self; +} + macro_rules! define_handles { ( 'owned: $($oty:ident,)* @@ -116,6 +133,25 @@ macro_rules! define_handles { $oty(handle::Handle::decode(r, s)) } } + + impl server::InitOwnedHandle>> + for Marked + { + fn init_handle(self, raw_handle: handle::Handle, s: &mut HandleStore>) { + s.$oty.init(raw_handle, self); + } + } + + impl OwnedHandle for $oty { + fn next_raw_handle() -> handle::Handle { + let counter = HandleCounters::get().$oty.fetch_add(1, Ordering::SeqCst); + handle::Handle::new(counter as u32).expect("`proc_macro` handle counter overflowed") + } + + fn from_raw_handle(handle: handle::Handle) -> $oty { + $oty(handle) + } + } )* $( @@ -242,27 +278,57 @@ impl fmt::Debug for Span { } } -macro_rules! define_client_side { - ($($name:ident { - $(fn $method:ident($($arg:ident: $arg_ty:ty),* $(,)?) $(-> $ret_ty:ty)*;)* - }),* $(,)?) => { - $(impl $name { - $(pub(crate) fn $method($($arg: $arg_ty),*) $(-> $ret_ty)* { - Bridge::with(|bridge| { - let mut b = bridge.cached_buffer.take(); +macro_rules! client_send_impl { + (wait $name:ident :: $method:ident($($arg:ident),*) $(-> $ret_ty:ty)?) => { + Bridge::with(|bridge| { + let mut b = bridge.cached_buffer.take(); + + b.clear(); + api_tags::Method::$name(api_tags::$name::$method).encode(&mut b, &mut ()); + reverse_encode!(b; $($arg),*); + + b = bridge.dispatch.call(b); + + let r = Result::<_, PanicMessage>::decode(&mut &b[..], &mut ()); + + bridge.cached_buffer = b; + + r.unwrap_or_else(|e| panic::resume_unwind(e.into())) + }) + }; + + (nowait $name:ident :: $method:ident($($arg:ident),*) $(-> $ret_ty:ty)?) => { + Bridge::with(|bridge| { + let mut b = bridge.cached_buffer.take(); - b.clear(); - api_tags::Method::$name(api_tags::$name::$method).encode(&mut b, &mut ()); - reverse_encode!(b; $($arg),*); + b.clear(); + api_tags::Method::$name(api_tags::$name::$method).encode(&mut b, &mut ()); + reverse_encode!(b; $($arg),*); - b = bridge.dispatch.call(b); + $( + let raw_handle = <$ret_ty as OwnedHandle>::next_raw_handle(); + raw_handle.encode(&mut b, &mut ()); + )? - let r = Result::<_, PanicMessage>::decode(&mut &b[..], &mut ()); + b = bridge.dispatch.call(b); - bridge.cached_buffer = b; + let r = Result::<(), PanicMessage>::decode(&mut &b[..], &mut ()); - r.unwrap_or_else(|e| panic::resume_unwind(e.into())) - }) + bridge.cached_buffer = b; + + r.unwrap_or_else(|e| panic::resume_unwind(e.into())); + $(<$ret_ty as OwnedHandle>::from_raw_handle(raw_handle))? + }) + }; +} + +macro_rules! define_client_side { + ($($name:ident { + $($wait:ident fn $method:ident($($arg:ident: $arg_ty:ty),* $(,)?) $(-> $ret_ty:ty)?;)* + }),* $(,)?) => { + $(impl $name { + $(pub(crate) fn $method($($arg: $arg_ty),*) $(-> $ret_ty)? { + client_send_impl!($wait $name :: $method ($($arg),*) $(-> $ret_ty)?) })* })* } diff --git a/library/proc_macro/src/bridge/handle.rs b/library/proc_macro/src/bridge/handle.rs index bcbb86812470a..9c57a9c3d2a7c 100644 --- a/library/proc_macro/src/bridge/handle.rs +++ b/library/proc_macro/src/bridge/handle.rs @@ -27,10 +27,14 @@ impl OwnedStore { pub(super) fn alloc(&mut self, x: T) -> Handle { let counter = self.counter.fetch_add(1, Ordering::SeqCst); let handle = Handle::new(counter as u32).expect("`proc_macro` handle counter overflowed"); - assert!(self.data.insert(handle, x).is_none()); + self.init(handle, x); handle } + pub(super) fn init(&mut self, h: Handle, x: T) { + assert!(self.data.insert(h, x).is_none()); + } + pub(super) fn take(&mut self, h: Handle) -> T { self.data.remove(&h).expect("use-after-free in `proc_macro` handle") } diff --git a/library/proc_macro/src/bridge/mod.rs b/library/proc_macro/src/bridge/mod.rs index c88bbdcd462fd..17538ecde3904 100644 --- a/library/proc_macro/src/bridge/mod.rs +++ b/library/proc_macro/src/bridge/mod.rs @@ -28,10 +28,10 @@ use std::thread; /// // ... /// Literal { /// // ... -/// fn character(ch: char) -> MySelf::Literal; +/// wait fn character(ch: char) -> MySelf::Literal; /// // ... -/// fn span(my_self: &MySelf::Literal) -> MySelf::Span; -/// fn set_span(my_self: &mut MySelf::Literal, span: MySelf::Span); +/// wait fn span(my_self: &MySelf::Literal) -> MySelf::Span; +/// nowait fn set_span(my_self: &mut MySelf::Literal, span: MySelf::Span); /// }, /// // ... /// } @@ -49,119 +49,125 @@ use std::thread; /// a trait or a trait impl, where the trait has associated types /// for each of the API types. If non-associated types are desired, /// a module name (`self` in practice) can be used instead of `Self`. +/// +/// If the `nowait` modifier is used, the server implementation may not +/// panic, and the client will continue immediately without waiting for +/// a response from the server when in multithreaded mode. If a return +/// type is present, it must be an owning IPC handle. Other return types +/// are not supported with `nowait`. macro_rules! with_api { ($S:ident, $self:ident, $m:ident) => { $m! { FreeFunctions { - fn drop($self: $S::FreeFunctions); - fn track_env_var(var: &str, value: Option<&str>); + nowait fn drop($self: $S::FreeFunctions); + nowait fn track_env_var(var: &str, value: Option<&str>); }, TokenStream { - fn drop($self: $S::TokenStream); - fn clone($self: &$S::TokenStream) -> $S::TokenStream; - fn new() -> $S::TokenStream; - fn is_empty($self: &$S::TokenStream) -> bool; - fn from_str(src: &str) -> $S::TokenStream; - fn to_string($self: &$S::TokenStream) -> String; - fn from_token_tree( + nowait fn drop($self: $S::TokenStream); + nowait fn clone($self: &$S::TokenStream) -> $S::TokenStream; + nowait fn new() -> $S::TokenStream; + wait fn is_empty($self: &$S::TokenStream) -> bool; + wait fn from_str(src: &str) -> $S::TokenStream; + wait fn to_string($self: &$S::TokenStream) -> String; + nowait fn from_token_tree( tree: TokenTree<$S::Group, $S::Punct, $S::Ident, $S::Literal>, ) -> $S::TokenStream; - fn into_iter($self: $S::TokenStream) -> $S::TokenStreamIter; + nowait fn into_iter($self: $S::TokenStream) -> $S::TokenStreamIter; }, TokenStreamBuilder { - fn drop($self: $S::TokenStreamBuilder); - fn new() -> $S::TokenStreamBuilder; - fn push($self: &mut $S::TokenStreamBuilder, stream: $S::TokenStream); - fn build($self: $S::TokenStreamBuilder) -> $S::TokenStream; + nowait fn drop($self: $S::TokenStreamBuilder); + nowait fn new() -> $S::TokenStreamBuilder; + nowait fn push($self: &mut $S::TokenStreamBuilder, stream: $S::TokenStream); + nowait fn build($self: $S::TokenStreamBuilder) -> $S::TokenStream; }, TokenStreamIter { - fn drop($self: $S::TokenStreamIter); - fn clone($self: &$S::TokenStreamIter) -> $S::TokenStreamIter; - fn next( + nowait fn drop($self: $S::TokenStreamIter); + nowait fn clone($self: &$S::TokenStreamIter) -> $S::TokenStreamIter; + wait fn next( $self: &mut $S::TokenStreamIter, ) -> Option>; }, Group { - fn drop($self: $S::Group); - fn clone($self: &$S::Group) -> $S::Group; - fn new(delimiter: Delimiter, stream: $S::TokenStream) -> $S::Group; - fn delimiter($self: &$S::Group) -> Delimiter; - fn stream($self: &$S::Group) -> $S::TokenStream; - fn span($self: &$S::Group) -> $S::Span; - fn span_open($self: &$S::Group) -> $S::Span; - fn span_close($self: &$S::Group) -> $S::Span; - fn set_span($self: &mut $S::Group, span: $S::Span); + nowait fn drop($self: $S::Group); + nowait fn clone($self: &$S::Group) -> $S::Group; + nowait fn new(delimiter: Delimiter, stream: $S::TokenStream) -> $S::Group; + wait fn delimiter($self: &$S::Group) -> Delimiter; + nowait fn stream($self: &$S::Group) -> $S::TokenStream; + wait fn span($self: &$S::Group) -> $S::Span; + wait fn span_open($self: &$S::Group) -> $S::Span; + wait fn span_close($self: &$S::Group) -> $S::Span; + nowait fn set_span($self: &mut $S::Group, span: $S::Span); }, Punct { - fn new(ch: char, spacing: Spacing) -> $S::Punct; - fn as_char($self: $S::Punct) -> char; - fn spacing($self: $S::Punct) -> Spacing; - fn span($self: $S::Punct) -> $S::Span; - fn with_span($self: $S::Punct, span: $S::Span) -> $S::Punct; + wait fn new(ch: char, spacing: Spacing) -> $S::Punct; + wait fn as_char($self: $S::Punct) -> char; + wait fn spacing($self: $S::Punct) -> Spacing; + wait fn span($self: $S::Punct) -> $S::Span; + wait fn with_span($self: $S::Punct, span: $S::Span) -> $S::Punct; }, Ident { - fn new(string: &str, span: $S::Span, is_raw: bool) -> $S::Ident; - fn span($self: $S::Ident) -> $S::Span; - fn with_span($self: $S::Ident, span: $S::Span) -> $S::Ident; + wait fn new(string: &str, span: $S::Span, is_raw: bool) -> $S::Ident; + wait fn span($self: $S::Ident) -> $S::Span; + wait fn with_span($self: $S::Ident, span: $S::Span) -> $S::Ident; }, Literal { - fn drop($self: $S::Literal); - fn clone($self: &$S::Literal) -> $S::Literal; - fn from_str(s: &str) -> Result<$S::Literal, ()>; - fn debug_kind($self: &$S::Literal) -> String; - fn symbol($self: &$S::Literal) -> String; - fn suffix($self: &$S::Literal) -> Option; - fn integer(n: &str) -> $S::Literal; - fn typed_integer(n: &str, kind: &str) -> $S::Literal; - fn float(n: &str) -> $S::Literal; - fn f32(n: &str) -> $S::Literal; - fn f64(n: &str) -> $S::Literal; - fn string(string: &str) -> $S::Literal; - fn character(ch: char) -> $S::Literal; - fn byte_string(bytes: &[u8]) -> $S::Literal; - fn span($self: &$S::Literal) -> $S::Span; - fn set_span($self: &mut $S::Literal, span: $S::Span); - fn subspan( + nowait fn drop($self: $S::Literal); + nowait fn clone($self: &$S::Literal) -> $S::Literal; + wait fn from_str(s: &str) -> Result<$S::Literal, ()>; + wait fn debug_kind($self: &$S::Literal) -> String; + wait fn symbol($self: &$S::Literal) -> String; + wait fn suffix($self: &$S::Literal) -> Option; + nowait fn integer(n: &str) -> $S::Literal; + nowait fn typed_integer(n: &str, kind: &str) -> $S::Literal; + nowait fn float(n: &str) -> $S::Literal; + nowait fn f32(n: &str) -> $S::Literal; + nowait fn f64(n: &str) -> $S::Literal; + nowait fn string(string: &str) -> $S::Literal; + nowait fn character(ch: char) -> $S::Literal; + nowait fn byte_string(bytes: &[u8]) -> $S::Literal; + wait fn span($self: &$S::Literal) -> $S::Span; + nowait fn set_span($self: &mut $S::Literal, span: $S::Span); + wait fn subspan( $self: &$S::Literal, start: Bound, end: Bound, ) -> Option<$S::Span>; }, SourceFile { - fn drop($self: $S::SourceFile); - fn clone($self: &$S::SourceFile) -> $S::SourceFile; - fn eq($self: &$S::SourceFile, other: &$S::SourceFile) -> bool; - fn path($self: &$S::SourceFile) -> String; - fn is_real($self: &$S::SourceFile) -> bool; + nowait fn drop($self: $S::SourceFile); + nowait fn clone($self: &$S::SourceFile) -> $S::SourceFile; + wait fn eq($self: &$S::SourceFile, other: &$S::SourceFile) -> bool; + wait fn path($self: &$S::SourceFile) -> String; + wait fn is_real($self: &$S::SourceFile) -> bool; }, MultiSpan { - fn drop($self: $S::MultiSpan); - fn new() -> $S::MultiSpan; - fn push($self: &mut $S::MultiSpan, span: $S::Span); + nowait fn drop($self: $S::MultiSpan); + nowait fn new() -> $S::MultiSpan; + wait fn push($self: &mut $S::MultiSpan, span: $S::Span); }, Diagnostic { - fn drop($self: $S::Diagnostic); - fn new(level: Level, msg: &str, span: $S::MultiSpan) -> $S::Diagnostic; - fn sub( + wait fn drop($self: $S::Diagnostic); + wait fn new(level: Level, msg: &str, span: $S::MultiSpan) -> $S::Diagnostic; + wait fn sub( $self: &mut $S::Diagnostic, level: Level, msg: &str, span: $S::MultiSpan, ); - fn emit($self: $S::Diagnostic); + wait fn emit($self: $S::Diagnostic); }, Span { - fn debug($self: $S::Span) -> String; - fn source_file($self: $S::Span) -> $S::SourceFile; - fn parent($self: $S::Span) -> Option<$S::Span>; - fn source($self: $S::Span) -> $S::Span; - fn start($self: $S::Span) -> LineColumn; - fn end($self: $S::Span) -> LineColumn; - fn join($self: $S::Span, other: $S::Span) -> Option<$S::Span>; - fn resolved_at($self: $S::Span, at: $S::Span) -> $S::Span; - fn source_text($self: $S::Span) -> Option; - fn save_span($self: $S::Span) -> usize; - fn recover_proc_macro_span(id: usize) -> $S::Span; + wait fn debug($self: $S::Span) -> String; + wait fn source_file($self: $S::Span) -> $S::SourceFile; + wait fn parent($self: $S::Span) -> Option<$S::Span>; + wait fn source($self: $S::Span) -> $S::Span; + wait fn start($self: $S::Span) -> LineColumn; + wait fn end($self: $S::Span) -> LineColumn; + wait fn join($self: $S::Span, other: $S::Span) -> Option<$S::Span>; + wait fn resolved_at($self: $S::Span, at: $S::Span) -> $S::Span; + wait fn source_text($self: $S::Span) -> Option; + wait fn save_span($self: $S::Span) -> usize; + wait fn recover_proc_macro_span(id: usize) -> $S::Span; }, } }; @@ -232,9 +238,18 @@ impl<'a> !Send for BridgeConfig<'a> {} mod api_tags { use super::rpc::{DecodeMut, Encode, Reader, Writer}; + macro_rules! should_wait_impl { + (wait) => { + true + }; + (nowait) => { + false + }; + } + macro_rules! declare_tags { ($($name:ident { - $(fn $method:ident($($arg:ident: $arg_ty:ty),* $(,)?) $(-> $ret_ty:ty)*;)* + $($wait:ident fn $method:ident($($arg:ident: $arg_ty:ty),* $(,)?) $(-> $ret_ty:ty)*;)* }),* $(,)?) => { $( pub(super) enum $name { @@ -248,6 +263,16 @@ mod api_tags { $($name($name)),* } rpc_encode_decode!(enum Method { $($name(m)),* }); + + impl Method { + pub(super) fn should_wait(&self) -> bool { + match self { + $($( + Method::$name($name::$method) => should_wait_impl!($wait), + )*)* + } + } + } } } with_api!(self, self, declare_tags); diff --git a/library/proc_macro/src/bridge/server.rs b/library/proc_macro/src/bridge/server.rs index b4b16e8ad0cf7..3b9210ea55d29 100644 --- a/library/proc_macro/src/bridge/server.rs +++ b/library/proc_macro/src/bridge/server.rs @@ -48,7 +48,7 @@ pub trait Context: Types { macro_rules! declare_server_traits { ($($name:ident { - $(fn $method:ident($($arg:ident: $arg_ty:ty),* $(,)?) $(-> $ret_ty:ty)?;)* + $($wait:ident fn $method:ident($($arg:ident: $arg_ty:ty),* $(,)?) $(-> $ret_ty:ty)?;)* }),* $(,)?) => { pub trait Types { $(associated_item!(type $name);)* @@ -80,7 +80,7 @@ impl Context for MarkedTypes { macro_rules! define_mark_types_impls { ($($name:ident { - $(fn $method:ident($($arg:ident: $arg_ty:ty),* $(,)?) $(-> $ret_ty:ty)?;)* + $($wait:ident fn $method:ident($($arg:ident: $arg_ty:ty),* $(,)?) $(-> $ret_ty:ty)?;)* }),* $(,)?) => { impl Types for MarkedTypes { $(type $name = Marked;)* @@ -95,14 +95,28 @@ macro_rules! define_mark_types_impls { } with_api!(Self, self_, define_mark_types_impls); +pub(super) trait InitOwnedHandle { + fn init_handle(self, raw_handle: handle::Handle, s: &mut S); +} + struct Dispatcher { handle_store: HandleStore, server: S, } +macro_rules! maybe_handle_nowait_reply { + (wait, $reader:ident, $r:ident, $handle_store:ident, $ret_ty:ty) => {}; + (nowait, $reader:ident, $r:ident, $handle_store:ident, $ret_ty:ty) => { + let $r = $r.map(|r| { + let raw_handle = handle::Handle::decode(&mut $reader, &mut ()); + <$ret_ty as InitOwnedHandle<_>>::init_handle(r, raw_handle, $handle_store); + }); + }; +} + macro_rules! define_dispatcher_impl { ($($name:ident { - $(fn $method:ident($($arg:ident: $arg_ty:ty),* $(,)?) $(-> $ret_ty:ty)?;)* + $($wait:ident fn $method:ident($($arg:ident: $arg_ty:ty),* $(,)?) $(-> $ret_ty:ty)?;)* }),* $(,)?) => { // FIXME(eddyb) `pub` only for `ExecutionStrategy` below. pub trait DispatcherTrait { @@ -135,6 +149,8 @@ macro_rules! define_dispatcher_impl { .map_err(PanicMessage::from) }; + $(maybe_handle_nowait_reply!($wait, reader, r, handle_store, $ret_ty);)? + b.clear(); r.encode(&mut b, handle_store); })* From 6ce595a65e79e9f7fb2201b299bcb5bcc23c7e22 Mon Sep 17 00:00:00 2001 From: Nika Layzell Date: Thu, 1 Jul 2021 15:13:42 -0400 Subject: [PATCH 06/12] proc_macro: add an optimized CrossThread execution strategy, and a debug flag to use it This new strategy supports avoiding waiting for a reply for noblock messages. This strategy requires using a channel-like approach (similar to the previous CrossThread1 approach). This new CrossThread execution strategy takes a type parameter for the channel to use, allowing rustc to use a more efficient channel which the proc_macro crate could not declare as a dependency. --- compiler/rustc_expand/src/proc_macro.rs | 52 +++++--- compiler/rustc_session/src/options.rs | 2 + library/proc_macro/src/bridge/client.rs | 6 +- library/proc_macro/src/bridge/server.rs | 169 +++++++++++++++--------- 4 files changed, 141 insertions(+), 88 deletions(-) diff --git a/compiler/rustc_expand/src/proc_macro.rs b/compiler/rustc_expand/src/proc_macro.rs index 494b3fb61ee97..a23d28bf81f7c 100644 --- a/compiler/rustc_expand/src/proc_macro.rs +++ b/compiler/rustc_expand/src/proc_macro.rs @@ -12,7 +12,11 @@ use rustc_parse::parser::ForceCollect; use rustc_span::def_id::CrateNum; use rustc_span::{Span, DUMMY_SP}; -const EXEC_STRATEGY: pm::bridge::server::SameThread = pm::bridge::server::SameThread; +fn exec_strategy(ecx: &ExtCtxt<'_>) -> impl pm::bridge::server::ExecutionStrategy { + >>::new( + ecx.sess.opts.debugging_opts.proc_macro_cross_thread, + ) +} pub struct BangProcMacro { pub client: pm::bridge::client::Client pm::TokenStream>, @@ -27,14 +31,16 @@ impl base::ProcMacro for BangProcMacro { input: TokenStream, ) -> Result { let server = proc_macro_server::Rustc::new(ecx, self.krate); - self.client.run(&EXEC_STRATEGY, server, input, ecx.ecfg.proc_macro_backtrace).map_err(|e| { - let mut err = ecx.struct_span_err(span, "proc macro panicked"); - if let Some(s) = e.as_str() { - err.help(&format!("message: {}", s)); - } - err.emit(); - ErrorReported - }) + self.client.run(&exec_strategy(ecx), server, input, ecx.ecfg.proc_macro_backtrace).map_err( + |e| { + let mut err = ecx.struct_span_err(span, "proc macro panicked"); + if let Some(s) = e.as_str() { + err.help(&format!("message: {}", s)); + } + err.emit(); + ErrorReported + }, + ) } } @@ -53,7 +59,7 @@ impl base::AttrProcMacro for AttrProcMacro { ) -> Result { let server = proc_macro_server::Rustc::new(ecx, self.krate); self.client - .run(&EXEC_STRATEGY, server, annotation, annotated, ecx.ecfg.proc_macro_backtrace) + .run(&exec_strategy(ecx), server, annotation, annotated, ecx.ecfg.proc_macro_backtrace) .map_err(|e| { let mut err = ecx.struct_span_err(span, "custom attribute panicked"); if let Some(s) = e.as_str() { @@ -102,18 +108,22 @@ impl MultiItemModifier for ProcMacroDerive { }; let server = proc_macro_server::Rustc::new(ecx, self.krate); - let stream = - match self.client.run(&EXEC_STRATEGY, server, input, ecx.ecfg.proc_macro_backtrace) { - Ok(stream) => stream, - Err(e) => { - let mut err = ecx.struct_span_err(span, "proc-macro derive panicked"); - if let Some(s) = e.as_str() { - err.help(&format!("message: {}", s)); - } - err.emit(); - return ExpandResult::Ready(vec![]); + let stream = match self.client.run( + &exec_strategy(ecx), + server, + input, + ecx.ecfg.proc_macro_backtrace, + ) { + Ok(stream) => stream, + Err(e) => { + let mut err = ecx.struct_span_err(span, "proc-macro derive panicked"); + if let Some(s) = e.as_str() { + err.help(&format!("message: {}", s)); } - }; + err.emit(); + return ExpandResult::Ready(vec![]); + } + }; let error_count_before = ecx.sess.parse_sess.span_diagnostic.err_count(); let mut parser = diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs index 4c40d0c367eca..04f1f063a0126 100644 --- a/compiler/rustc_session/src/options.rs +++ b/compiler/rustc_session/src/options.rs @@ -1207,6 +1207,8 @@ options! { "print layout information for each type encountered (default: no)"), proc_macro_backtrace: bool = (false, parse_bool, [UNTRACKED], "show backtraces for panics during proc-macro execution (default: no)"), + proc_macro_cross_thread: bool = (false, parse_bool, [UNTRACKED], + "run proc-macro code on a separate thread (default: no)"), profile: bool = (false, parse_bool, [TRACKED], "insert profiling code (default: no)"), profile_closures: bool = (false, parse_no_flag, [UNTRACKED], diff --git a/library/proc_macro/src/bridge/client.rs b/library/proc_macro/src/bridge/client.rs index 9d41497fb28c9..ee40a92d9ef3b 100644 --- a/library/proc_macro/src/bridge/client.rs +++ b/library/proc_macro/src/bridge/client.rs @@ -312,7 +312,11 @@ macro_rules! client_send_impl { b = bridge.dispatch.call(b); - let r = Result::<(), PanicMessage>::decode(&mut &b[..], &mut ()); + let r = if b.len() > 0 { + Result::<(), PanicMessage>::decode(&mut &b[..], &mut ()) + } else { + Ok(()) + }; bridge.cached_buffer = b; diff --git a/library/proc_macro/src/bridge/server.rs b/library/proc_macro/src/bridge/server.rs index 3b9210ea55d29..8f64d860fe41d 100644 --- a/library/proc_macro/src/bridge/server.rs +++ b/library/proc_macro/src/bridge/server.rs @@ -2,6 +2,8 @@ use super::*; +use std::marker::PhantomData; + // FIXME(eddyb) generate the definition of `HandleStore` in `server.rs`. use super::client::HandleStore; @@ -174,6 +176,50 @@ pub trait ExecutionStrategy { ) -> Buffer; } +pub struct MaybeCrossThread

{ + cross_thread: bool, + marker: PhantomData

, +} + +impl

MaybeCrossThread

{ + pub const fn new(cross_thread: bool) -> Self { + MaybeCrossThread { cross_thread, marker: PhantomData } + } +} + +impl

ExecutionStrategy for MaybeCrossThread

+where + P: MessagePipe> + Send + 'static, +{ + fn run_bridge_and_client( + &self, + dispatcher: &mut impl DispatcherTrait, + input: Buffer, + run_client: extern "C" fn(BridgeConfig<'_>, D) -> Buffer, + client_data: D, + force_show_panics: bool, + ) -> Buffer { + if self.cross_thread { + >::new().run_bridge_and_client( + dispatcher, + input, + run_client, + client_data, + force_show_panics, + ) + } else { + SameThread.run_bridge_and_client( + dispatcher, + input, + run_client, + client_data, + force_show_panics, + ) + } + } +} + +#[derive(Default)] pub struct SameThread; impl ExecutionStrategy for SameThread { @@ -194,12 +240,18 @@ impl ExecutionStrategy for SameThread { } } -// NOTE(eddyb) Two implementations are provided, the second one is a bit -// faster but neither is anywhere near as fast as same-thread execution. +pub struct CrossThread

(PhantomData

); -pub struct CrossThread1; +impl

CrossThread

{ + pub const fn new() -> Self { + CrossThread(PhantomData) + } +} -impl ExecutionStrategy for CrossThread1 { +impl

ExecutionStrategy for CrossThread

+where + P: MessagePipe> + Send + 'static, +{ fn run_bridge_and_client( &self, dispatcher: &mut impl DispatcherTrait, @@ -208,15 +260,18 @@ impl ExecutionStrategy for CrossThread1 { client_data: D, force_show_panics: bool, ) -> Buffer { - use std::sync::mpsc::channel; - - let (req_tx, req_rx) = channel(); - let (res_tx, res_rx) = channel(); + let (mut server, mut client) = P::new(); let join_handle = thread::spawn(move || { - let mut dispatch = |b| { - req_tx.send(b).unwrap(); - res_rx.recv().unwrap() + let mut dispatch = |b: Buffer| -> Buffer { + let method_tag = api_tags::Method::decode(&mut &b[..], &mut ()); + client.send(b); + + if method_tag.should_wait() { + client.recv().expect("server died while client waiting for reply") + } else { + Buffer::new() + } }; run_client( @@ -225,73 +280,55 @@ impl ExecutionStrategy for CrossThread1 { ) }); - for b in req_rx { - res_tx.send(dispatcher.dispatch(b)).unwrap(); + while let Some(b) = server.recv() { + let method_tag = api_tags::Method::decode(&mut &b[..], &mut ()); + let b = dispatcher.dispatch(b); + + if method_tag.should_wait() { + server.send(b); + } else if let Err(err) = >::decode(&mut &b[..], &mut ()) { + panic::resume_unwind(err.into()); + } } join_handle.join().unwrap() } } -pub struct CrossThread2; - -impl ExecutionStrategy for CrossThread2 { - fn run_bridge_and_client( - &self, - dispatcher: &mut impl DispatcherTrait, - input: Buffer, - run_client: extern "C" fn(BridgeConfig<'_>, D) -> Buffer, - client_data: D, - force_show_panics: bool, - ) -> Buffer { - use std::sync::{Arc, Mutex}; - - enum State { - Req(T), - Res(T), - } - - let mut state = Arc::new(Mutex::new(State::Res(Buffer::new()))); +/// A message pipe used for communicating between server and client threads. +pub trait MessagePipe: Sized { + /// Create a new pair of endpoints for the message pipe. + fn new() -> (Self, Self); - let server_thread = thread::current(); - let state2 = state.clone(); - let join_handle = thread::spawn(move || { - let mut dispatch = |b| { - *state2.lock().unwrap() = State::Req(b); - server_thread.unpark(); - loop { - thread::park(); - if let State::Res(b) = &mut *state2.lock().unwrap() { - break b.take(); - } - } - }; + /// Send a message to the other endpoint of this pipe. + fn send(&mut self, value: T); - let r = run_client( - BridgeConfig { input, dispatch: (&mut dispatch).into(), force_show_panics }, - client_data, - ); + /// Receive a message from the other endpoint of this pipe. + /// + /// Returns `None` if the other end of the pipe has been destroyed, and no + /// message was received. + fn recv(&mut self) -> Option; +} - // Wake up the server so it can exit the dispatch loop. - drop(state2); - server_thread.unpark(); +/// Implementation of `MessagePipe` using `std::sync::mpsc` +pub struct StdMessagePipe { + tx: std::sync::mpsc::Sender, + rx: std::sync::mpsc::Receiver, +} - r - }); +impl MessagePipe for StdMessagePipe { + fn new() -> (Self, Self) { + let (tx1, rx1) = std::sync::mpsc::channel(); + let (tx2, rx2) = std::sync::mpsc::channel(); + (StdMessagePipe { tx: tx1, rx: rx2 }, StdMessagePipe { tx: tx2, rx: rx1 }) + } - // Check whether `state2` was dropped, to know when to stop. - while Arc::get_mut(&mut state).is_none() { - thread::park(); - let mut b = match &mut *state.lock().unwrap() { - State::Req(b) => b.take(), - _ => continue, - }; - b = dispatcher.dispatch(b.take()); - *state.lock().unwrap() = State::Res(b); - join_handle.thread().unpark(); - } + fn send(&mut self, v: T) { + self.tx.send(v).unwrap(); + } - join_handle.join().unwrap() + fn recv(&mut self) -> Option { + self.rx.recv().ok() } } From 8db4f28c53176ef190b7cba1fdc2b95d66af030d Mon Sep 17 00:00:00 2001 From: Nika Layzell Date: Thu, 1 Jul 2021 12:31:36 -0400 Subject: [PATCH 07/12] proc_macro: use crossbeam-channel for the CrossThread execution strategy Compared to mpsc::channel, crossbeam-channel has significantly lower overhead. --- Cargo.lock | 1 + compiler/rustc_expand/Cargo.toml | 1 + compiler/rustc_expand/src/proc_macro.rs | 23 ++++++++++++++++++++++- src/tools/tidy/src/deps.rs | 1 + 4 files changed, 25 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 8db8a56eaa8ec..060d9b033c223 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3840,6 +3840,7 @@ dependencies = [ name = "rustc_expand" version = "0.0.0" dependencies = [ + "crossbeam-channel", "rustc_ast", "rustc_ast_passes", "rustc_ast_pretty", diff --git a/compiler/rustc_expand/Cargo.toml b/compiler/rustc_expand/Cargo.toml index 59c1604e8444c..e217d8ad23aa7 100644 --- a/compiler/rustc_expand/Cargo.toml +++ b/compiler/rustc_expand/Cargo.toml @@ -25,3 +25,4 @@ rustc_parse = { path = "../rustc_parse" } rustc_session = { path = "../rustc_session" } smallvec = { version = "1.6.1", features = ["union", "may_dangle"] } rustc_ast = { path = "../rustc_ast" } +crossbeam-channel = "0.5.0" diff --git a/compiler/rustc_expand/src/proc_macro.rs b/compiler/rustc_expand/src/proc_macro.rs index a23d28bf81f7c..e597e9a58b6af 100644 --- a/compiler/rustc_expand/src/proc_macro.rs +++ b/compiler/rustc_expand/src/proc_macro.rs @@ -12,8 +12,29 @@ use rustc_parse::parser::ForceCollect; use rustc_span::def_id::CrateNum; use rustc_span::{Span, DUMMY_SP}; +struct CrossbeamMessagePipe { + tx: crossbeam_channel::Sender, + rx: crossbeam_channel::Receiver, +} + +impl pm::bridge::server::MessagePipe for CrossbeamMessagePipe { + fn new() -> (Self, Self) { + let (tx1, rx1) = crossbeam_channel::unbounded(); + let (tx2, rx2) = crossbeam_channel::unbounded(); + (CrossbeamMessagePipe { tx: tx1, rx: rx2 }, CrossbeamMessagePipe { tx: tx2, rx: rx1 }) + } + + fn send(&mut self, value: T) { + self.tx.send(value).unwrap(); + } + + fn recv(&mut self) -> Option { + self.rx.recv().ok() + } +} + fn exec_strategy(ecx: &ExtCtxt<'_>) -> impl pm::bridge::server::ExecutionStrategy { - >>::new( + >>::new( ecx.sess.opts.debugging_opts.proc_macro_cross_thread, ) } diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs index ea587210b4f4e..578f56a7a8630 100644 --- a/src/tools/tidy/src/deps.rs +++ b/src/tools/tidy/src/deps.rs @@ -97,6 +97,7 @@ const PERMITTED_DEPENDENCIES: &[&str] = &[ "compiler_builtins", "cpuid-bool", "crc32fast", + "crossbeam-channel", "crossbeam-deque", "crossbeam-epoch", "crossbeam-queue", From b0ca197d35197432c3aa2a072ab0df9426f96c1b Mon Sep 17 00:00:00 2001 From: Nika Layzell Date: Thu, 1 Jul 2021 15:03:51 -0400 Subject: [PATCH 08/12] proc_macro: reduce the number of messages required to create, extend, and iterate TokenStreams This significantly reduces the cost of common interactions with TokenStream when running with the CrossThread execution strategy, by reducing the number of RPC calls required. --- .../rustc_expand/src/proc_macro_server.rs | 104 ++++++++++-------- library/proc_macro/src/bridge/client.rs | 18 ++- library/proc_macro/src/bridge/mod.rs | 28 ++--- library/proc_macro/src/bridge/server.rs | 4 - library/proc_macro/src/lib.rs | 73 ++++++++---- 5 files changed, 130 insertions(+), 97 deletions(-) diff --git a/compiler/rustc_expand/src/proc_macro_server.rs b/compiler/rustc_expand/src/proc_macro_server.rs index 97163ce31af34..2b8d5cd61225a 100644 --- a/compiler/rustc_expand/src/proc_macro_server.rs +++ b/compiler/rustc_expand/src/proc_macro_server.rs @@ -284,12 +284,6 @@ impl ToInternal for Level { pub struct FreeFunctions; -#[derive(Clone)] -pub struct TokenStreamIter { - cursor: tokenstream::Cursor, - stack: Vec>, -} - #[derive(Clone)] pub struct Group { delimiter: Delimiter, @@ -398,8 +392,6 @@ impl<'a> Rustc<'a> { impl server::Types for Rustc<'_> { type FreeFunctions = FreeFunctions; type TokenStream = TokenStream; - type TokenStreamBuilder = tokenstream::TokenStreamBuilder; - type TokenStreamIter = TokenStreamIter; type Group = Group; type Punct = Punct; type Ident = Ident; @@ -417,9 +409,6 @@ impl server::FreeFunctions for Rustc<'_> { } impl server::TokenStream for Rustc<'_> { - fn new(&mut self) -> Self::TokenStream { - TokenStream::default() - } fn is_empty(&mut self, stream: &Self::TokenStream) -> bool { stream.is_empty() } @@ -440,53 +429,74 @@ impl server::TokenStream for Rustc<'_> { ) -> Self::TokenStream { tree.to_internal() } - fn into_iter(&mut self, stream: Self::TokenStream) -> Self::TokenStreamIter { - TokenStreamIter { cursor: stream.trees(), stack: vec![] } - } -} - -impl server::TokenStreamBuilder for Rustc<'_> { - fn new(&mut self) -> Self::TokenStreamBuilder { - tokenstream::TokenStreamBuilder::new() - } - fn push(&mut self, builder: &mut Self::TokenStreamBuilder, stream: Self::TokenStream) { - builder.push(stream); + fn concat_trees( + &mut self, + base: Option, + trees: Vec>, + ) -> Self::TokenStream { + let mut builder = tokenstream::TokenStreamBuilder::new(); + if let Some(base) = base { + builder.push(base); + } + for tree in trees { + builder.push(tree.to_internal()); + } + builder.build() } - fn build(&mut self, builder: Self::TokenStreamBuilder) -> Self::TokenStream { + fn concat_streams( + &mut self, + base: Option, + streams: Vec, + ) -> Self::TokenStream { + let mut builder = tokenstream::TokenStreamBuilder::new(); + if let Some(base) = base { + builder.push(base); + } + for stream in streams { + builder.push(stream); + } builder.build() } -} - -impl server::TokenStreamIter for Rustc<'_> { - fn next( + fn into_iter( &mut self, - iter: &mut Self::TokenStreamIter, - ) -> Option> { + stream: Self::TokenStream, + ) -> Vec> { + // XXX: This is a raw port of the previous approach, and can probably be + // optimized. + let mut cursor = stream.trees(); + let mut stack = Vec::new(); + let mut tts = Vec::new(); loop { - let tree = iter.stack.pop().or_else(|| { - let next = iter.cursor.next_with_spacing()?; - Some(TokenTree::from_internal((next, &mut iter.stack, self))) - })?; - // A hack used to pass AST fragments to attribute and derive macros - // as a single nonterminal token instead of a token stream. - // Such token needs to be "unwrapped" and not represented as a delimited group. - // FIXME: It needs to be removed, but there are some compatibility issues (see #73345). - if let TokenTree::Group(ref group) = tree { - if group.flatten { - iter.cursor.append(group.stream.clone()); - continue; + let next = stack.pop().or_else(|| { + let next = cursor.next_with_spacing()?; + Some(TokenTree::from_internal((next, &mut stack, self))) + }); + match next { + Some(TokenTree::Group(group)) => { + // A hack used to pass AST fragments to attribute and derive + // macros as a single nonterminal token instead of a token + // stream. Such token needs to be "unwrapped" and not + // represented as a delimited group. + // FIXME: It needs to be removed, but there are some + // compatibility issues (see #73345). + if group.flatten { + cursor.append(group.stream); + continue; + } + tts.push(TokenTree::Group(group)); } + Some(tt) => tts.push(tt), + None => return tts, } - return Some(tree); } } } impl server::Group for Rustc<'_> { - fn new(&mut self, delimiter: Delimiter, stream: Self::TokenStream) -> Self::Group { + fn new(&mut self, delimiter: Delimiter, stream: Option) -> Self::Group { Group { delimiter, - stream, + stream: stream.unwrap_or_default(), span: DelimSpan::from_single(server::Context::call_site(self)), flatten: false, } @@ -519,7 +529,11 @@ impl server::Punct for Rustc<'_> { punct.ch } fn spacing(&mut self, punct: Self::Punct) -> Spacing { - if punct.joint { Spacing::Joint } else { Spacing::Alone } + if punct.joint { + Spacing::Joint + } else { + Spacing::Alone + } } fn span(&mut self, punct: Self::Punct) -> Self::Span { punct.span diff --git a/library/proc_macro/src/bridge/client.rs b/library/proc_macro/src/bridge/client.rs index ee40a92d9ef3b..5c6f4a7921425 100644 --- a/library/proc_macro/src/bridge/client.rs +++ b/library/proc_macro/src/bridge/client.rs @@ -195,8 +195,6 @@ define_handles! { 'owned: FreeFunctions, TokenStream, - TokenStreamBuilder, - TokenStreamIter, Group, Literal, SourceFile, @@ -221,12 +219,6 @@ impl Clone for TokenStream { } } -impl Clone for TokenStreamIter { - fn clone(&self) -> Self { - self.clone() - } -} - impl Clone for Group { fn clone(&self) -> Self { self.clone() @@ -507,7 +499,11 @@ impl Client crate::TokenStream> { bridge: BridgeConfig<'_>, f: impl FnOnce(crate::TokenStream) -> crate::TokenStream, ) -> Buffer { - run_client(bridge, |input| f(crate::TokenStream(input)).0) + run_client(bridge, |input| { + f(crate::TokenStream(Some(input))) + .0 + .unwrap_or_else(|| TokenStream::concat_streams(None, vec![])) + }) } Client { get_handle_counters: HandleCounters::get, run, f } } @@ -523,7 +519,9 @@ impl Client crate::TokenStream> { f: impl FnOnce(crate::TokenStream, crate::TokenStream) -> crate::TokenStream, ) -> Buffer { run_client(bridge, |(input, input2)| { - f(crate::TokenStream(input), crate::TokenStream(input2)).0 + f(crate::TokenStream(Some(input)), crate::TokenStream(Some(input2))) + .0 + .unwrap_or_else(|| TokenStream::concat_streams(None, vec![])) }) } Client { get_handle_counters: HandleCounters::get, run, f } diff --git a/library/proc_macro/src/bridge/mod.rs b/library/proc_macro/src/bridge/mod.rs index 17538ecde3904..7ed525a060575 100644 --- a/library/proc_macro/src/bridge/mod.rs +++ b/library/proc_macro/src/bridge/mod.rs @@ -65,32 +65,28 @@ macro_rules! with_api { TokenStream { nowait fn drop($self: $S::TokenStream); nowait fn clone($self: &$S::TokenStream) -> $S::TokenStream; - nowait fn new() -> $S::TokenStream; wait fn is_empty($self: &$S::TokenStream) -> bool; wait fn from_str(src: &str) -> $S::TokenStream; wait fn to_string($self: &$S::TokenStream) -> String; nowait fn from_token_tree( tree: TokenTree<$S::Group, $S::Punct, $S::Ident, $S::Literal>, ) -> $S::TokenStream; - nowait fn into_iter($self: $S::TokenStream) -> $S::TokenStreamIter; - }, - TokenStreamBuilder { - nowait fn drop($self: $S::TokenStreamBuilder); - nowait fn new() -> $S::TokenStreamBuilder; - nowait fn push($self: &mut $S::TokenStreamBuilder, stream: $S::TokenStream); - nowait fn build($self: $S::TokenStreamBuilder) -> $S::TokenStream; - }, - TokenStreamIter { - nowait fn drop($self: $S::TokenStreamIter); - nowait fn clone($self: &$S::TokenStreamIter) -> $S::TokenStreamIter; - wait fn next( - $self: &mut $S::TokenStreamIter, - ) -> Option>; + nowait fn concat_trees( + base: Option<$S::TokenStream>, + trees: Vec>, + ) -> $S::TokenStream; + nowait fn concat_streams( + base: Option<$S::TokenStream>, + trees: Vec<$S::TokenStream>, + ) -> $S::TokenStream; + wait fn into_iter( + $self: $S::TokenStream + ) -> Vec>; }, Group { nowait fn drop($self: $S::Group); nowait fn clone($self: &$S::Group) -> $S::Group; - nowait fn new(delimiter: Delimiter, stream: $S::TokenStream) -> $S::Group; + nowait fn new(delimiter: Delimiter, stream: Option<$S::TokenStream>) -> $S::Group; wait fn delimiter($self: &$S::Group) -> Delimiter; nowait fn stream($self: &$S::Group) -> $S::TokenStream; wait fn span($self: &$S::Group) -> $S::Span; diff --git a/library/proc_macro/src/bridge/server.rs b/library/proc_macro/src/bridge/server.rs index 8f64d860fe41d..e8a3df09f5fc6 100644 --- a/library/proc_macro/src/bridge/server.rs +++ b/library/proc_macro/src/bridge/server.rs @@ -14,10 +14,6 @@ macro_rules! associated_item { (type FreeFunctions: 'static;); (type TokenStream) => (type TokenStream: 'static + Clone;); - (type TokenStreamBuilder) => - (type TokenStreamBuilder: 'static;); - (type TokenStreamIter) => - (type TokenStreamIter: 'static + Clone;); (type Group) => (type Group: 'static + Clone;); (type Punct) => diff --git a/library/proc_macro/src/lib.rs b/library/proc_macro/src/lib.rs index 353eb3c45c1fe..722c0fb0c3ba2 100644 --- a/library/proc_macro/src/lib.rs +++ b/library/proc_macro/src/lib.rs @@ -46,7 +46,7 @@ use std::cmp::Ordering; use std::ops::RangeBounds; use std::path::PathBuf; use std::str::FromStr; -use std::{error, fmt, iter, mem}; +use std::{error, fmt, iter}; /// Determines whether proc_macro has been made accessible to the currently /// running program. @@ -75,7 +75,7 @@ pub fn is_available() -> bool { /// and `#[proc_macro_derive]` definitions. #[stable(feature = "proc_macro_lib", since = "1.15.0")] #[derive(Clone)] -pub struct TokenStream(bridge::client::TokenStream); +pub struct TokenStream(Option); #[stable(feature = "proc_macro_lib", since = "1.15.0")] impl !Send for TokenStream {} @@ -113,13 +113,13 @@ impl TokenStream { /// Returns an empty `TokenStream` containing no token trees. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn new() -> TokenStream { - TokenStream(bridge::client::TokenStream::new()) + TokenStream(None) } /// Checks if this `TokenStream` is empty. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn is_empty(&self) -> bool { - self.0.is_empty() + self.0.as_ref().map(|h| h.is_empty()).unwrap_or(true) } } @@ -135,7 +135,7 @@ impl FromStr for TokenStream { type Err = LexError; fn from_str(src: &str) -> Result { - Ok(TokenStream(bridge::client::TokenStream::from_str(src))) + Ok(TokenStream(Some(bridge::client::TokenStream::from_str(src)))) } } @@ -144,7 +144,7 @@ impl FromStr for TokenStream { #[stable(feature = "proc_macro_lib", since = "1.15.0")] impl ToString for TokenStream { fn to_string(&self) -> String { - self.0.to_string() + self.0.as_ref().map(|t| t.to_string()).unwrap_or_default() } } @@ -177,16 +177,27 @@ impl Default for TokenStream { #[unstable(feature = "proc_macro_quote", issue = "54722")] pub use quote::{quote, quote_span}; +fn tree_to_bridge_tree( + tree: TokenTree, +) -> bridge::TokenTree< + bridge::client::Group, + bridge::client::Punct, + bridge::client::Ident, + bridge::client::Literal, +> { + match tree { + TokenTree::Group(tt) => bridge::TokenTree::Group(tt.0), + TokenTree::Punct(tt) => bridge::TokenTree::Punct(tt.0), + TokenTree::Ident(tt) => bridge::TokenTree::Ident(tt.0), + TokenTree::Literal(tt) => bridge::TokenTree::Literal(tt.0), + } +} + /// Creates a token stream containing a single token tree. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] impl From for TokenStream { fn from(tree: TokenTree) -> TokenStream { - TokenStream(bridge::client::TokenStream::from_token_tree(match tree { - TokenTree::Group(tt) => bridge::TokenTree::Group(tt.0), - TokenTree::Punct(tt) => bridge::TokenTree::Punct(tt.0), - TokenTree::Ident(tt) => bridge::TokenTree::Ident(tt.0), - TokenTree::Literal(tt) => bridge::TokenTree::Literal(tt.0), - })) + TokenStream(Some(bridge::client::TokenStream::from_token_tree(tree_to_bridge_tree(tree)))) } } @@ -194,7 +205,10 @@ impl From for TokenStream { #[stable(feature = "proc_macro_lib2", since = "1.29.0")] impl iter::FromIterator for TokenStream { fn from_iter>(trees: I) -> Self { - trees.into_iter().map(TokenStream::from).collect() + TokenStream(Some(bridge::client::TokenStream::concat_trees( + None, + trees.into_iter().map(tree_to_bridge_tree).collect(), + ))) } } @@ -203,24 +217,30 @@ impl iter::FromIterator for TokenStream { #[stable(feature = "proc_macro_lib", since = "1.15.0")] impl iter::FromIterator for TokenStream { fn from_iter>(streams: I) -> Self { - let mut builder = bridge::client::TokenStreamBuilder::new(); - streams.into_iter().for_each(|stream| builder.push(stream.0)); - TokenStream(builder.build()) + TokenStream(Some(bridge::client::TokenStream::concat_streams( + None, + streams.into_iter().filter_map(|stream| stream.0).collect(), + ))) } } #[stable(feature = "token_stream_extend", since = "1.30.0")] impl Extend for TokenStream { fn extend>(&mut self, trees: I) { - self.extend(trees.into_iter().map(TokenStream::from)); + *self = TokenStream(Some(bridge::client::TokenStream::concat_trees( + self.0.take(), + trees.into_iter().map(|tree| tree_to_bridge_tree(tree)).collect(), + ))); } } #[stable(feature = "token_stream_extend", since = "1.30.0")] impl Extend for TokenStream { fn extend>(&mut self, streams: I) { - // FIXME(eddyb) Use an optimized implementation if/when possible. - *self = iter::once(mem::replace(self, Self::new())).chain(streams).collect(); + *self = TokenStream(Some(bridge::client::TokenStream::concat_streams( + self.0.take(), + streams.into_iter().filter_map(|stream| stream.0).collect(), + ))); } } @@ -234,7 +254,16 @@ pub mod token_stream { /// and returns whole groups as token trees. #[derive(Clone)] #[stable(feature = "proc_macro_lib2", since = "1.29.0")] - pub struct IntoIter(bridge::client::TokenStreamIter); + pub struct IntoIter( + std::vec::IntoIter< + bridge::TokenTree< + bridge::client::Group, + bridge::client::Punct, + bridge::client::Ident, + bridge::client::Literal, + >, + >, + ); #[stable(feature = "proc_macro_lib2", since = "1.29.0")] impl Iterator for IntoIter { @@ -256,7 +285,7 @@ pub mod token_stream { type IntoIter = IntoIter; fn into_iter(self) -> IntoIter { - IntoIter(self.0.into_iter()) + IntoIter(self.0.map(|v| v.into_iter()).unwrap_or_default().into_iter()) } } } @@ -685,7 +714,7 @@ impl Group { /// returned above. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn stream(&self) -> TokenStream { - TokenStream(self.0.stream()) + TokenStream(Some(self.0.stream())) } /// Returns the span for the delimiters of this token stream, spanning the From dc613dc3fb9e4743f4c6689013fa171e7fc3ba87 Mon Sep 17 00:00:00 2001 From: Nika Layzell Date: Mon, 28 Jun 2021 22:30:55 -0400 Subject: [PATCH 09/12] proc_macro: stop using a remote object handle for Punct This greatly reduces round-trips to fetch relevant extra information about the token in proc macro code, and avoids RPC messages to create Punct tokens. --- .../rustc_expand/src/proc_macro_server.rs | 76 ++++--------------- library/proc_macro/src/bridge/client.rs | 1 - library/proc_macro/src/bridge/mod.rs | 28 +++---- library/proc_macro/src/bridge/server.rs | 2 - library/proc_macro/src/lib.rs | 34 ++++----- 5 files changed, 47 insertions(+), 94 deletions(-) diff --git a/compiler/rustc_expand/src/proc_macro_server.rs b/compiler/rustc_expand/src/proc_macro_server.rs index 2b8d5cd61225a..857271776b345 100644 --- a/compiler/rustc_expand/src/proc_macro_server.rs +++ b/compiler/rustc_expand/src/proc_macro_server.rs @@ -19,8 +19,8 @@ use rustc_span::hygiene::ExpnKind; use rustc_span::symbol::{self, kw, sym, Symbol}; use rustc_span::{BytePos, FileName, MultiSpan, Pos, RealFileName, SourceFile, Span}; -use pm::bridge::{server, TokenTree}; -use pm::{Delimiter, Level, LineColumn, Spacing}; +use pm::bridge::{server, Punct, TokenTree}; +use pm::{Delimiter, Level, LineColumn}; use std::ops::Bound; use std::{ascii, panic}; @@ -55,7 +55,7 @@ impl ToInternal for Delimiter { } impl FromInternal<(TreeAndSpacing, &'_ mut Vec, &mut Rustc<'_>)> - for TokenTree + for TokenTree { fn from_internal( ((tree, spacing), stack, rustc): (TreeAndSpacing, &mut Vec, &mut Rustc<'_>), @@ -84,16 +84,16 @@ impl FromInternal<(TreeAndSpacing, &'_ mut Vec, &mut Rustc<'_>)> } macro_rules! op { ($a:expr) => { - tt!(Punct::new($a, joint)) + tt!(Punct { ch: $a, joint }) }; ($a:expr, $b:expr) => {{ - stack.push(tt!(Punct::new($b, joint))); - tt!(Punct::new($a, true)) + stack.push(tt!(Punct { ch: $b, joint })); + tt!(Punct { ch: $a, joint: true }) }}; ($a:expr, $b:expr, $c:expr) => {{ - stack.push(tt!(Punct::new($c, joint))); - stack.push(tt!(Punct::new($b, true))); - tt!(Punct::new($a, true)) + stack.push(tt!(Punct { ch: $c, joint })); + stack.push(tt!(Punct { ch: $b, joint: true })); + tt!(Punct { ch: $a, joint: true }) }}; } @@ -151,7 +151,7 @@ impl FromInternal<(TreeAndSpacing, &'_ mut Vec, &mut Rustc<'_>)> Lifetime(name) => { let ident = symbol::Ident::new(name, span).without_first_quote(); stack.push(tt!(Ident::new(rustc.sess, ident.name, false))); - tt!(Punct::new('\'', true)) + tt!(Punct { ch: '\'', joint: true }) } Literal(lit) => tt!(Literal { lit }), DocComment(_, attr_style, data) => { @@ -174,9 +174,9 @@ impl FromInternal<(TreeAndSpacing, &'_ mut Vec, &mut Rustc<'_>)> flatten: false, })); if attr_style == ast::AttrStyle::Inner { - stack.push(tt!(Punct::new('!', false))); + stack.push(tt!(Punct { ch: '!', joint: false })); } - tt!(Punct::new('#', false)) + tt!(Punct { ch: '#', joint: false }) } Interpolated(nt) => { @@ -199,7 +199,7 @@ impl FromInternal<(TreeAndSpacing, &'_ mut Vec, &mut Rustc<'_>)> } } -impl ToInternal for TokenTree { +impl ToInternal for TokenTree { fn to_internal(self) -> TokenStream { use rustc_ast::token::*; @@ -295,27 +295,6 @@ pub struct Group { flatten: bool, } -#[derive(Copy, Clone, PartialEq, Eq, Hash)] -pub struct Punct { - ch: char, - // NB. not using `Spacing` here because it doesn't implement `Hash`. - joint: bool, - span: Span, -} - -impl Punct { - fn new(ch: char, joint: bool, span: Span) -> Punct { - const LEGAL_CHARS: &[char] = &[ - '=', '<', '>', '!', '~', '+', '-', '*', '/', '%', '^', '&', '|', '@', '.', ',', ';', - ':', '#', '$', '?', '\'', - ]; - if !LEGAL_CHARS.contains(&ch) { - panic!("unsupported character `{:?}`", ch) - } - Punct { ch, joint, span } - } -} - #[derive(Copy, Clone, PartialEq, Eq, Hash)] pub struct Ident { sym: Symbol, @@ -393,7 +372,6 @@ impl server::Types for Rustc<'_> { type FreeFunctions = FreeFunctions; type TokenStream = TokenStream; type Group = Group; - type Punct = Punct; type Ident = Ident; type Literal = Literal; type SourceFile = Lrc; @@ -425,14 +403,14 @@ impl server::TokenStream for Rustc<'_> { } fn from_token_tree( &mut self, - tree: TokenTree, + tree: TokenTree, ) -> Self::TokenStream { tree.to_internal() } fn concat_trees( &mut self, base: Option, - trees: Vec>, + trees: Vec>, ) -> Self::TokenStream { let mut builder = tokenstream::TokenStreamBuilder::new(); if let Some(base) = base { @@ -460,7 +438,7 @@ impl server::TokenStream for Rustc<'_> { fn into_iter( &mut self, stream: Self::TokenStream, - ) -> Vec> { + ) -> Vec> { // XXX: This is a raw port of the previous approach, and can probably be // optimized. let mut cursor = stream.trees(); @@ -521,28 +499,6 @@ impl server::Group for Rustc<'_> { } } -impl server::Punct for Rustc<'_> { - fn new(&mut self, ch: char, spacing: Spacing) -> Self::Punct { - Punct::new(ch, spacing == Spacing::Joint, server::Context::call_site(self)) - } - fn as_char(&mut self, punct: Self::Punct) -> char { - punct.ch - } - fn spacing(&mut self, punct: Self::Punct) -> Spacing { - if punct.joint { - Spacing::Joint - } else { - Spacing::Alone - } - } - fn span(&mut self, punct: Self::Punct) -> Self::Span { - punct.span - } - fn with_span(&mut self, punct: Self::Punct, span: Self::Span) -> Self::Punct { - Punct { span, ..punct } - } -} - impl server::Ident for Rustc<'_> { fn new(&mut self, string: &str, span: Self::Span, is_raw: bool) -> Self::Ident { Ident::new(self.sess, Symbol::intern(string), is_raw, span) diff --git a/library/proc_macro/src/bridge/client.rs b/library/proc_macro/src/bridge/client.rs index 5c6f4a7921425..defc1eb53946c 100644 --- a/library/proc_macro/src/bridge/client.rs +++ b/library/proc_macro/src/bridge/client.rs @@ -202,7 +202,6 @@ define_handles! { Diagnostic, 'interned: - Punct, Ident, Span, } diff --git a/library/proc_macro/src/bridge/mod.rs b/library/proc_macro/src/bridge/mod.rs index 7ed525a060575..e52e51802e3ef 100644 --- a/library/proc_macro/src/bridge/mod.rs +++ b/library/proc_macro/src/bridge/mod.rs @@ -69,11 +69,11 @@ macro_rules! with_api { wait fn from_str(src: &str) -> $S::TokenStream; wait fn to_string($self: &$S::TokenStream) -> String; nowait fn from_token_tree( - tree: TokenTree<$S::Group, $S::Punct, $S::Ident, $S::Literal>, + tree: TokenTree<$S::Span, $S::Group, $S::Ident, $S::Literal>, ) -> $S::TokenStream; nowait fn concat_trees( base: Option<$S::TokenStream>, - trees: Vec>, + trees: Vec>, ) -> $S::TokenStream; nowait fn concat_streams( base: Option<$S::TokenStream>, @@ -81,7 +81,7 @@ macro_rules! with_api { ) -> $S::TokenStream; wait fn into_iter( $self: $S::TokenStream - ) -> Vec>; + ) -> Vec>; }, Group { nowait fn drop($self: $S::Group); @@ -94,13 +94,6 @@ macro_rules! with_api { wait fn span_close($self: &$S::Group) -> $S::Span; nowait fn set_span($self: &mut $S::Group, span: $S::Span); }, - Punct { - wait fn new(ch: char, spacing: Spacing) -> $S::Punct; - wait fn as_char($self: $S::Punct) -> char; - wait fn spacing($self: $S::Punct) -> Spacing; - wait fn span($self: $S::Punct) -> $S::Span; - wait fn with_span($self: $S::Punct, span: $S::Span) -> $S::Punct; - }, Ident { wait fn new(string: &str, span: $S::Span, is_raw: bool) -> $S::Ident; wait fn span($self: $S::Ident) -> $S::Span; @@ -471,15 +464,24 @@ macro_rules! compound_traits { } #[derive(Clone)] -pub enum TokenTree { +pub struct Punct { + pub ch: char, + pub joint: bool, + pub span: S, +} + +compound_traits!(struct Punct { ch, joint, span }); + +#[derive(Clone)] +pub enum TokenTree { Group(G), - Punct(P), + Punct(Punct), Ident(I), Literal(L), } compound_traits!( - enum TokenTree { + enum TokenTree { Group(tt), Punct(tt), Ident(tt), diff --git a/library/proc_macro/src/bridge/server.rs b/library/proc_macro/src/bridge/server.rs index e8a3df09f5fc6..17075166373d9 100644 --- a/library/proc_macro/src/bridge/server.rs +++ b/library/proc_macro/src/bridge/server.rs @@ -16,8 +16,6 @@ macro_rules! associated_item { (type TokenStream: 'static + Clone;); (type Group) => (type Group: 'static + Clone;); - (type Punct) => - (type Punct: 'static + Copy + Eq + Hash;); (type Ident) => (type Ident: 'static + Copy + Eq + Hash;); (type Literal) => diff --git a/library/proc_macro/src/lib.rs b/library/proc_macro/src/lib.rs index 722c0fb0c3ba2..8792063ed9e16 100644 --- a/library/proc_macro/src/lib.rs +++ b/library/proc_macro/src/lib.rs @@ -180,8 +180,8 @@ pub use quote::{quote, quote_span}; fn tree_to_bridge_tree( tree: TokenTree, ) -> bridge::TokenTree< + bridge::client::Span, bridge::client::Group, - bridge::client::Punct, bridge::client::Ident, bridge::client::Literal, > { @@ -257,8 +257,8 @@ pub mod token_stream { pub struct IntoIter( std::vec::IntoIter< bridge::TokenTree< + bridge::client::Span, bridge::client::Group, - bridge::client::Punct, bridge::client::Ident, bridge::client::Literal, >, @@ -799,7 +799,7 @@ impl fmt::Debug for Group { /// forms of `Spacing` returned. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] #[derive(Clone)] -pub struct Punct(bridge::client::Punct); +pub struct Punct(bridge::Punct); #[stable(feature = "proc_macro_lib2", since = "1.29.0")] impl !Send for Punct {} @@ -832,13 +832,20 @@ impl Punct { /// which can be further configured with the `set_span` method below. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn new(ch: char, spacing: Spacing) -> Punct { - Punct(bridge::client::Punct::new(ch, spacing)) + const LEGAL_CHARS: &[char] = &[ + '=', '<', '>', '!', '~', '+', '-', '*', '/', '%', '^', '&', '|', '@', '.', ',', ';', + ':', '#', '$', '?', '\'', + ]; + if !LEGAL_CHARS.contains(&ch) { + panic!("unsupported character `{:?}`", ch); + } + Punct(bridge::Punct { ch, joint: spacing == Spacing::Joint, span: Span::call_site().0 }) } /// Returns the value of this punctuation character as `char`. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn as_char(&self) -> char { - self.0.as_char() + self.0.ch } /// Returns the spacing of this punctuation character, indicating whether it's immediately @@ -847,28 +854,19 @@ impl Punct { /// (`Alone`) so the operator has certainly ended. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn spacing(&self) -> Spacing { - self.0.spacing() + if self.0.joint { Spacing::Joint } else { Spacing::Alone } } /// Returns the span for this punctuation character. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn span(&self) -> Span { - Span(self.0.span()) + Span(self.0.span) } /// Configure the span for this punctuation character. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn set_span(&mut self, span: Span) { - self.0 = self.0.with_span(span.0); - } -} - -// N.B., the bridge only provides `to_string`, implement `fmt::Display` -// based on it (the reverse of the usual relationship between the two). -#[stable(feature = "proc_macro_lib", since = "1.15.0")] -impl ToString for Punct { - fn to_string(&self) -> String { - TokenStream::from(TokenTree::from(self.clone())).to_string() + self.0.span = span.0; } } @@ -877,7 +875,7 @@ impl ToString for Punct { #[stable(feature = "proc_macro_lib2", since = "1.29.0")] impl fmt::Display for Punct { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(&self.to_string()) + write!(f, "{}", self.as_char()) } } From c8ea717ee6970f5b0a215928778a945bc05512cf Mon Sep 17 00:00:00 2001 From: Nika Layzell Date: Thu, 1 Jul 2021 17:36:38 -0400 Subject: [PATCH 10/12] proc_macro: stop using a remote object handle for Group This greatly reduces round-trips to fetch relevant extra information about the token in proc macro code, and avoids RPC messages to create Group tokens. --- .../rustc_expand/src/proc_macro_server.rs | 370 ++++++++---------- library/proc_macro/src/bridge/client.rs | 7 - library/proc_macro/src/bridge/mod.rs | 47 ++- library/proc_macro/src/bridge/server.rs | 2 - library/proc_macro/src/lib.rs | 24 +- 5 files changed, 206 insertions(+), 244 deletions(-) diff --git a/compiler/rustc_expand/src/proc_macro_server.rs b/compiler/rustc_expand/src/proc_macro_server.rs index 857271776b345..0349418521c49 100644 --- a/compiler/rustc_expand/src/proc_macro_server.rs +++ b/compiler/rustc_expand/src/proc_macro_server.rs @@ -3,7 +3,7 @@ use crate::base::{ExtCtxt, ResolverExpand}; use rustc_ast as ast; use rustc_ast::token::{self, Nonterminal, NtIdent, TokenKind}; use rustc_ast::tokenstream::{self, CanSynthesizeMissingTokens}; -use rustc_ast::tokenstream::{DelimSpan, Spacing::*, TokenStream, TreeAndSpacing}; +use rustc_ast::tokenstream::{Spacing::*, TokenStream}; use rustc_ast_pretty::pprust; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::sync::Lrc; @@ -19,7 +19,7 @@ use rustc_span::hygiene::ExpnKind; use rustc_span::symbol::{self, kw, sym, Symbol}; use rustc_span::{BytePos, FileName, MultiSpan, Pos, RealFileName, SourceFile, Span}; -use pm::bridge::{server, Punct, TokenTree}; +use pm::bridge::{server, DelimSpan, Group, Punct, TokenTree}; use pm::{Delimiter, Level, LineColumn}; use std::ops::Bound; use std::{ascii, panic}; @@ -54,160 +54,182 @@ impl ToInternal for Delimiter { } } -impl FromInternal<(TreeAndSpacing, &'_ mut Vec, &mut Rustc<'_>)> - for TokenTree +impl FromInternal<(TokenStream, &mut Rustc<'_>)> + for Vec> { - fn from_internal( - ((tree, spacing), stack, rustc): (TreeAndSpacing, &mut Vec, &mut Rustc<'_>), - ) -> Self { + fn from_internal((stream, rustc): (TokenStream, &mut Rustc<'_>)) -> Self { use rustc_ast::token::*; - let joint = spacing == Joint; - let Token { kind, span } = match tree { - tokenstream::TokenTree::Delimited(span, delim, tts) => { - let delimiter = Delimiter::from_internal(delim); - return TokenTree::Group(Group { delimiter, stream: tts, span, flatten: false }); - } - tokenstream::TokenTree::Token(token) => token, - }; - - macro_rules! tt { - ($ty:ident { $($field:ident $(: $value:expr)*),+ $(,)? }) => ( - TokenTree::$ty(self::$ty { - $($field $(: $value)*,)+ - span, - }) - ); - ($ty:ident::$method:ident($($value:expr),*)) => ( - TokenTree::$ty(self::$ty::$method($($value,)* span)) - ); - } - macro_rules! op { - ($a:expr) => { - tt!(Punct { ch: $a, joint }) + let mut cursor = stream.into_trees(); + let mut trees = Vec::new(); + + while let Some((tree, spacing)) = cursor.next_with_spacing() { + let joint = spacing == Joint; + let Token { kind, span } = match tree { + tokenstream::TokenTree::Delimited(span, delim, tts) => { + let delimiter = Delimiter::from_internal(delim); + trees.push(TokenTree::Group(Group { + delimiter, + stream: Some(tts), + span: DelimSpan { + open: span.open, + close: span.close, + entire: span.entire(), + }, + })); + continue; + } + tokenstream::TokenTree::Token(token) => token, }; - ($a:expr, $b:expr) => {{ - stack.push(tt!(Punct { ch: $b, joint })); - tt!(Punct { ch: $a, joint: true }) - }}; - ($a:expr, $b:expr, $c:expr) => {{ - stack.push(tt!(Punct { ch: $c, joint })); - stack.push(tt!(Punct { ch: $b, joint: true })); - tt!(Punct { ch: $a, joint: true }) - }}; - } - match kind { - Eq => op!('='), - Lt => op!('<'), - Le => op!('<', '='), - EqEq => op!('=', '='), - Ne => op!('!', '='), - Ge => op!('>', '='), - Gt => op!('>'), - AndAnd => op!('&', '&'), - OrOr => op!('|', '|'), - Not => op!('!'), - Tilde => op!('~'), - BinOp(Plus) => op!('+'), - BinOp(Minus) => op!('-'), - BinOp(Star) => op!('*'), - BinOp(Slash) => op!('/'), - BinOp(Percent) => op!('%'), - BinOp(Caret) => op!('^'), - BinOp(And) => op!('&'), - BinOp(Or) => op!('|'), - BinOp(Shl) => op!('<', '<'), - BinOp(Shr) => op!('>', '>'), - BinOpEq(Plus) => op!('+', '='), - BinOpEq(Minus) => op!('-', '='), - BinOpEq(Star) => op!('*', '='), - BinOpEq(Slash) => op!('/', '='), - BinOpEq(Percent) => op!('%', '='), - BinOpEq(Caret) => op!('^', '='), - BinOpEq(And) => op!('&', '='), - BinOpEq(Or) => op!('|', '='), - BinOpEq(Shl) => op!('<', '<', '='), - BinOpEq(Shr) => op!('>', '>', '='), - At => op!('@'), - Dot => op!('.'), - DotDot => op!('.', '.'), - DotDotDot => op!('.', '.', '.'), - DotDotEq => op!('.', '.', '='), - Comma => op!(','), - Semi => op!(';'), - Colon => op!(':'), - ModSep => op!(':', ':'), - RArrow => op!('-', '>'), - LArrow => op!('<', '-'), - FatArrow => op!('=', '>'), - Pound => op!('#'), - Dollar => op!('$'), - Question => op!('?'), - SingleQuote => op!('\''), - - Ident(name, false) if name == kw::DollarCrate => tt!(Ident::dollar_crate()), - Ident(name, is_raw) => tt!(Ident::new(rustc.sess, name, is_raw)), - Lifetime(name) => { - let ident = symbol::Ident::new(name, span).without_first_quote(); - stack.push(tt!(Ident::new(rustc.sess, ident.name, false))); - tt!(Punct { ch: '\'', joint: true }) + macro_rules! tt { + ($ty:ident { $($field:ident $(: $value:expr)*),+ $(,)? }) => ( + trees.push(TokenTree::$ty(self::$ty { + $($field $(: $value)*,)+ + span, + })) + ); + ($ty:ident::$method:ident($($value:expr),*)) => ( + trees.push(TokenTree::$ty(self::$ty::$method($($value,)* span))) + ); } - Literal(lit) => tt!(Literal { lit }), - DocComment(_, attr_style, data) => { - let mut escaped = String::new(); - for ch in data.as_str().chars() { - escaped.extend(ch.escape_debug()); - } - let stream = vec![ - Ident(sym::doc, false), - Eq, - TokenKind::lit(token::Str, Symbol::intern(&escaped), None), - ] - .into_iter() - .map(|kind| tokenstream::TokenTree::token(kind, span)) - .collect(); - stack.push(TokenTree::Group(Group { - delimiter: Delimiter::Bracket, - stream, - span: DelimSpan::from_single(span), - flatten: false, - })); - if attr_style == ast::AttrStyle::Inner { - stack.push(tt!(Punct { ch: '!', joint: false })); - } - tt!(Punct { ch: '#', joint: false }) + macro_rules! op { + ($a:expr) => {{ + tt!(Punct { ch: $a, joint }); + }}; + ($a:expr, $b:expr) => {{ + tt!(Punct { ch: $a, joint: true }); + tt!(Punct { ch: $b, joint }); + }}; + ($a:expr, $b:expr, $c:expr) => {{ + tt!(Punct { ch: $a, joint: true }); + tt!(Punct { ch: $b, joint: true }); + tt!(Punct { ch: $c, joint }); + }}; } - Interpolated(nt) => { - if let Some((name, is_raw)) = ident_name_compatibility_hack(&nt, span, rustc) { - TokenTree::Ident(Ident::new(rustc.sess, name.name, is_raw, name.span)) - } else { - let stream = nt_to_tokenstream(&nt, rustc.sess, CanSynthesizeMissingTokens::No); - TokenTree::Group(Group { - delimiter: Delimiter::None, - stream, + match kind { + Eq => op!('='), + Lt => op!('<'), + Le => op!('<', '='), + EqEq => op!('=', '='), + Ne => op!('!', '='), + Ge => op!('>', '='), + Gt => op!('>'), + AndAnd => op!('&', '&'), + OrOr => op!('|', '|'), + Not => op!('!'), + Tilde => op!('~'), + BinOp(Plus) => op!('+'), + BinOp(Minus) => op!('-'), + BinOp(Star) => op!('*'), + BinOp(Slash) => op!('/'), + BinOp(Percent) => op!('%'), + BinOp(Caret) => op!('^'), + BinOp(And) => op!('&'), + BinOp(Or) => op!('|'), + BinOp(Shl) => op!('<', '<'), + BinOp(Shr) => op!('>', '>'), + BinOpEq(Plus) => op!('+', '='), + BinOpEq(Minus) => op!('-', '='), + BinOpEq(Star) => op!('*', '='), + BinOpEq(Slash) => op!('/', '='), + BinOpEq(Percent) => op!('%', '='), + BinOpEq(Caret) => op!('^', '='), + BinOpEq(And) => op!('&', '='), + BinOpEq(Or) => op!('|', '='), + BinOpEq(Shl) => op!('<', '<', '='), + BinOpEq(Shr) => op!('>', '>', '='), + At => op!('@'), + Dot => op!('.'), + DotDot => op!('.', '.'), + DotDotDot => op!('.', '.', '.'), + DotDotEq => op!('.', '.', '='), + Comma => op!(','), + Semi => op!(';'), + Colon => op!(':'), + ModSep => op!(':', ':'), + RArrow => op!('-', '>'), + LArrow => op!('<', '-'), + FatArrow => op!('=', '>'), + Pound => op!('#'), + Dollar => op!('$'), + Question => op!('?'), + SingleQuote => op!('\''), + + Ident(name, false) if name == kw::DollarCrate => tt!(Ident::dollar_crate()), + Ident(name, is_raw) => tt!(Ident::new(rustc.sess, name, is_raw)), + Lifetime(name) => { + let ident = symbol::Ident::new(name, span).without_first_quote(); + tt!(Punct { ch: '\'', joint: true }); + tt!(Ident::new(rustc.sess, ident.name, false)); + } + Literal(lit) => tt!(Literal { lit }), + DocComment(_, attr_style, data) => { + let mut escaped = String::new(); + for ch in data.as_str().chars() { + escaped.extend(ch.escape_debug()); + } + let stream = vec![ + Ident(sym::doc, false), + Eq, + TokenKind::lit(token::Str, Symbol::intern(&escaped), None), + ] + .into_iter() + .map(|kind| tokenstream::TokenTree::token(kind, span)) + .collect(); + tt!(Punct { ch: '#', joint: false }); + if attr_style == ast::AttrStyle::Inner { + tt!(Punct { ch: '!', joint: false }); + } + trees.push(TokenTree::Group(Group { + delimiter: Delimiter::Bracket, + stream: Some(stream), span: DelimSpan::from_single(span), - flatten: crate::base::pretty_printing_compatibility_hack(&nt, rustc.sess), - }) + })); + } + + Interpolated(nt) => { + if let Some((name, is_raw)) = ident_name_compatibility_hack(&nt, span, rustc) { + trees.push(TokenTree::Ident(Ident::new( + rustc.sess, name.name, is_raw, name.span, + ))); + } else { + let stream = + nt_to_tokenstream(&nt, rustc.sess, CanSynthesizeMissingTokens::No); + if crate::base::pretty_printing_compatibility_hack(&nt, rustc.sess) { + cursor.append(stream); + } else { + trees.push(TokenTree::Group(Group { + delimiter: Delimiter::None, + stream: Some(stream), + span: DelimSpan::from_single(span), + })) + } + } } - } - OpenDelim(..) | CloseDelim(..) => unreachable!(), - Eof => unreachable!(), + OpenDelim(..) | CloseDelim(..) => unreachable!(), + Eof => unreachable!(), + } } + trees } } -impl ToInternal for TokenTree { +impl ToInternal for TokenTree { fn to_internal(self) -> TokenStream { use rustc_ast::token::*; let (ch, joint, span) = match self { TokenTree::Punct(Punct { ch, joint, span }) => (ch, joint, span), - TokenTree::Group(Group { delimiter, stream, span, .. }) => { - return tokenstream::TokenTree::Delimited(span, delimiter.to_internal(), stream) - .into(); + TokenTree::Group(Group { delimiter, stream, span: DelimSpan { open, close, .. } }) => { + return tokenstream::TokenTree::Delimited( + tokenstream::DelimSpan { open, close }, + delimiter.to_internal(), + stream.unwrap_or_default(), + ) + .into(); } TokenTree::Ident(self::Ident { sym, is_raw, span }) => { return tokenstream::TokenTree::token(Ident(sym, is_raw), span).into(); @@ -284,17 +306,6 @@ impl ToInternal for Level { pub struct FreeFunctions; -#[derive(Clone)] -pub struct Group { - delimiter: Delimiter, - stream: TokenStream, - span: DelimSpan, - /// A hack used to pass AST fragments to attribute and derive macros - /// as a single nonterminal token instead of a token stream. - /// FIXME: It needs to be removed, but there are some compatibility issues (see #73345). - flatten: bool, -} - #[derive(Copy, Clone, PartialEq, Eq, Hash)] pub struct Ident { sym: Symbol, @@ -371,7 +382,6 @@ impl<'a> Rustc<'a> { impl server::Types for Rustc<'_> { type FreeFunctions = FreeFunctions; type TokenStream = TokenStream; - type Group = Group; type Ident = Ident; type Literal = Literal; type SourceFile = Lrc; @@ -403,14 +413,14 @@ impl server::TokenStream for Rustc<'_> { } fn from_token_tree( &mut self, - tree: TokenTree, + tree: TokenTree, ) -> Self::TokenStream { tree.to_internal() } fn concat_trees( &mut self, base: Option, - trees: Vec>, + trees: Vec>, ) -> Self::TokenStream { let mut builder = tokenstream::TokenStreamBuilder::new(); if let Some(base) = base { @@ -438,64 +448,8 @@ impl server::TokenStream for Rustc<'_> { fn into_iter( &mut self, stream: Self::TokenStream, - ) -> Vec> { - // XXX: This is a raw port of the previous approach, and can probably be - // optimized. - let mut cursor = stream.trees(); - let mut stack = Vec::new(); - let mut tts = Vec::new(); - loop { - let next = stack.pop().or_else(|| { - let next = cursor.next_with_spacing()?; - Some(TokenTree::from_internal((next, &mut stack, self))) - }); - match next { - Some(TokenTree::Group(group)) => { - // A hack used to pass AST fragments to attribute and derive - // macros as a single nonterminal token instead of a token - // stream. Such token needs to be "unwrapped" and not - // represented as a delimited group. - // FIXME: It needs to be removed, but there are some - // compatibility issues (see #73345). - if group.flatten { - cursor.append(group.stream); - continue; - } - tts.push(TokenTree::Group(group)); - } - Some(tt) => tts.push(tt), - None => return tts, - } - } - } -} - -impl server::Group for Rustc<'_> { - fn new(&mut self, delimiter: Delimiter, stream: Option) -> Self::Group { - Group { - delimiter, - stream: stream.unwrap_or_default(), - span: DelimSpan::from_single(server::Context::call_site(self)), - flatten: false, - } - } - fn delimiter(&mut self, group: &Self::Group) -> Delimiter { - group.delimiter - } - fn stream(&mut self, group: &Self::Group) -> Self::TokenStream { - group.stream.clone() - } - fn span(&mut self, group: &Self::Group) -> Self::Span { - group.span.entire() - } - fn span_open(&mut self, group: &Self::Group) -> Self::Span { - group.span.open - } - fn span_close(&mut self, group: &Self::Group) -> Self::Span { - group.span.close - } - fn set_span(&mut self, group: &mut Self::Group, span: Self::Span) { - group.span = DelimSpan::from_single(span); + ) -> Vec> { + FromInternal::from_internal((stream, self)) } } diff --git a/library/proc_macro/src/bridge/client.rs b/library/proc_macro/src/bridge/client.rs index defc1eb53946c..49175a5c476ed 100644 --- a/library/proc_macro/src/bridge/client.rs +++ b/library/proc_macro/src/bridge/client.rs @@ -195,7 +195,6 @@ define_handles! { 'owned: FreeFunctions, TokenStream, - Group, Literal, SourceFile, MultiSpan, @@ -218,12 +217,6 @@ impl Clone for TokenStream { } } -impl Clone for Group { - fn clone(&self) -> Self { - self.clone() - } -} - impl Clone for Literal { fn clone(&self) -> Self { self.clone() diff --git a/library/proc_macro/src/bridge/mod.rs b/library/proc_macro/src/bridge/mod.rs index e52e51802e3ef..44b5e28d0fbf2 100644 --- a/library/proc_macro/src/bridge/mod.rs +++ b/library/proc_macro/src/bridge/mod.rs @@ -69,11 +69,11 @@ macro_rules! with_api { wait fn from_str(src: &str) -> $S::TokenStream; wait fn to_string($self: &$S::TokenStream) -> String; nowait fn from_token_tree( - tree: TokenTree<$S::Span, $S::Group, $S::Ident, $S::Literal>, + tree: TokenTree<$S::TokenStream, $S::Span, $S::Ident, $S::Literal>, ) -> $S::TokenStream; nowait fn concat_trees( base: Option<$S::TokenStream>, - trees: Vec>, + trees: Vec>, ) -> $S::TokenStream; nowait fn concat_streams( base: Option<$S::TokenStream>, @@ -81,18 +81,7 @@ macro_rules! with_api { ) -> $S::TokenStream; wait fn into_iter( $self: $S::TokenStream - ) -> Vec>; - }, - Group { - nowait fn drop($self: $S::Group); - nowait fn clone($self: &$S::Group) -> $S::Group; - nowait fn new(delimiter: Delimiter, stream: Option<$S::TokenStream>) -> $S::Group; - wait fn delimiter($self: &$S::Group) -> Delimiter; - nowait fn stream($self: &$S::Group) -> $S::TokenStream; - wait fn span($self: &$S::Group) -> $S::Span; - wait fn span_open($self: &$S::Group) -> $S::Span; - wait fn span_close($self: &$S::Group) -> $S::Span; - nowait fn set_span($self: &mut $S::Group, span: $S::Span); + ) -> Vec>; }, Ident { wait fn new(string: &str, span: $S::Span, is_raw: bool) -> $S::Ident; @@ -463,6 +452,30 @@ macro_rules! compound_traits { }; } +#[derive(Copy, Clone)] +pub struct DelimSpan { + pub open: S, + pub close: S, + pub entire: S, +} + +impl DelimSpan { + pub fn from_single(span: S) -> Self { + DelimSpan { open: span, close: span, entire: span } + } +} + +compound_traits!(struct DelimSpan { open, close, entire }); + +#[derive(Clone)] +pub struct Group { + pub delimiter: Delimiter, + pub stream: Option, + pub span: DelimSpan, +} + +compound_traits!(struct Group { delimiter, stream, span }); + #[derive(Clone)] pub struct Punct { pub ch: char, @@ -473,15 +486,15 @@ pub struct Punct { compound_traits!(struct Punct { ch, joint, span }); #[derive(Clone)] -pub enum TokenTree { - Group(G), +pub enum TokenTree { + Group(Group), Punct(Punct), Ident(I), Literal(L), } compound_traits!( - enum TokenTree { + enum TokenTree { Group(tt), Punct(tt), Ident(tt), diff --git a/library/proc_macro/src/bridge/server.rs b/library/proc_macro/src/bridge/server.rs index 17075166373d9..a4452b3651e57 100644 --- a/library/proc_macro/src/bridge/server.rs +++ b/library/proc_macro/src/bridge/server.rs @@ -14,8 +14,6 @@ macro_rules! associated_item { (type FreeFunctions: 'static;); (type TokenStream) => (type TokenStream: 'static + Clone;); - (type Group) => - (type Group: 'static + Clone;); (type Ident) => (type Ident: 'static + Copy + Eq + Hash;); (type Literal) => diff --git a/library/proc_macro/src/lib.rs b/library/proc_macro/src/lib.rs index 8792063ed9e16..37d0d93714252 100644 --- a/library/proc_macro/src/lib.rs +++ b/library/proc_macro/src/lib.rs @@ -180,8 +180,8 @@ pub use quote::{quote, quote_span}; fn tree_to_bridge_tree( tree: TokenTree, ) -> bridge::TokenTree< + bridge::client::TokenStream, bridge::client::Span, - bridge::client::Group, bridge::client::Ident, bridge::client::Literal, > { @@ -257,8 +257,8 @@ pub mod token_stream { pub struct IntoIter( std::vec::IntoIter< bridge::TokenTree< + bridge::client::TokenStream, bridge::client::Span, - bridge::client::Group, bridge::client::Ident, bridge::client::Literal, >, @@ -662,7 +662,7 @@ impl fmt::Display for TokenTree { /// A `Group` internally contains a `TokenStream` which is surrounded by `Delimiter`s. #[derive(Clone)] #[stable(feature = "proc_macro_lib2", since = "1.29.0")] -pub struct Group(bridge::client::Group); +pub struct Group(bridge::Group); #[stable(feature = "proc_macro_lib2", since = "1.29.0")] impl !Send for Group {} @@ -699,13 +699,17 @@ impl Group { /// method below. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn new(delimiter: Delimiter, stream: TokenStream) -> Group { - Group(bridge::client::Group::new(delimiter, stream.0)) + Group(bridge::Group { + delimiter, + stream: stream.0, + span: bridge::DelimSpan::from_single(Span::call_site().0), + }) } /// Returns the delimiter of this `Group` #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn delimiter(&self) -> Delimiter { - self.0.delimiter() + self.0.delimiter } /// Returns the `TokenStream` of tokens that are delimited in this `Group`. @@ -714,7 +718,7 @@ impl Group { /// returned above. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn stream(&self) -> TokenStream { - TokenStream(Some(self.0.stream())) + TokenStream(self.0.stream.clone()) } /// Returns the span for the delimiters of this token stream, spanning the @@ -726,7 +730,7 @@ impl Group { /// ``` #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn span(&self) -> Span { - Span(self.0.span()) + Span(self.0.span.entire) } /// Returns the span pointing to the opening delimiter of this group. @@ -737,7 +741,7 @@ impl Group { /// ``` #[stable(feature = "proc_macro_group_span", since = "1.55.0")] pub fn span_open(&self) -> Span { - Span(self.0.span_open()) + Span(self.0.span.open) } /// Returns the span pointing to the closing delimiter of this group. @@ -748,7 +752,7 @@ impl Group { /// ``` #[stable(feature = "proc_macro_group_span", since = "1.55.0")] pub fn span_close(&self) -> Span { - Span(self.0.span_close()) + Span(self.0.span.close) } /// Configures the span for this `Group`'s delimiters, but not its internal @@ -759,7 +763,7 @@ impl Group { /// tokens at the level of the `Group`. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn set_span(&mut self, span: Span) { - self.0.set_span(span.0); + self.0.span = bridge::DelimSpan::from_single(span.0); } } From 11f70c84a96d63cfe4ea91630c699a92a3651b07 Mon Sep 17 00:00:00 2001 From: Nika Layzell Date: Thu, 1 Jul 2021 22:59:51 -0400 Subject: [PATCH 11/12] proc_macro: stop using a remote object handle for Ident This requires a dependency on `unicode-normalization` and `rustc_lexer`, which is currently not possible for `proc_macro`. Instead, a second `extern "C" fn` is provided by the compiler server to perform these steps from any thread. String values are interned in both the server and client, meaning that identifiers can be stringified without any RPC roundtrips without substantially inflating their size. RPC messages passing symbols include the full un-interned value, and are re-interned on the receiving side. This could potentially be optimized in the future. The symbol infrastructure will alwo be used for literals in a following part. --- Cargo.lock | 1 + compiler/rustc_expand/Cargo.toml | 1 + .../rustc_expand/src/proc_macro_server.rs | 90 ++++++--------- library/proc_macro/src/bridge/buffer.rs | 29 +++++ library/proc_macro/src/bridge/client.rs | 107 ++++++++++++++++-- library/proc_macro/src/bridge/handle.rs | 48 +++++++- library/proc_macro/src/bridge/mod.rs | 69 ++++++----- library/proc_macro/src/bridge/server.rs | 96 ++++++++++++---- library/proc_macro/src/lib.rs | 34 +++--- 9 files changed, 342 insertions(+), 133 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 060d9b033c223..c9d86349c279c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3857,6 +3857,7 @@ dependencies = [ "rustc_span", "smallvec", "tracing", + "unicode-normalization", ] [[package]] diff --git a/compiler/rustc_expand/Cargo.toml b/compiler/rustc_expand/Cargo.toml index e217d8ad23aa7..d85460756a545 100644 --- a/compiler/rustc_expand/Cargo.toml +++ b/compiler/rustc_expand/Cargo.toml @@ -26,3 +26,4 @@ rustc_session = { path = "../rustc_session" } smallvec = { version = "1.6.1", features = ["union", "may_dangle"] } rustc_ast = { path = "../rustc_ast" } crossbeam-channel = "0.5.0" +unicode-normalization = "0.1.11" diff --git a/compiler/rustc_expand/src/proc_macro_server.rs b/compiler/rustc_expand/src/proc_macro_server.rs index 0349418521c49..89a69bacb0f37 100644 --- a/compiler/rustc_expand/src/proc_macro_server.rs +++ b/compiler/rustc_expand/src/proc_macro_server.rs @@ -10,7 +10,6 @@ use rustc_data_structures::sync::Lrc; use rustc_errors::Diagnostic; use rustc_lint_defs::builtin::PROC_MACRO_BACK_COMPAT; use rustc_lint_defs::BuiltinLintDiagnostics; -use rustc_parse::lexer::nfc_normalize; use rustc_parse::{nt_to_tokenstream, parse_stream_from_source_str}; use rustc_session::parse::ParseSess; use rustc_span::def_id::CrateNum; @@ -19,10 +18,10 @@ use rustc_span::hygiene::ExpnKind; use rustc_span::symbol::{self, kw, sym, Symbol}; use rustc_span::{BytePos, FileName, MultiSpan, Pos, RealFileName, SourceFile, Span}; -use pm::bridge::{server, DelimSpan, Group, Punct, TokenTree}; +use pm::bridge::{server, DelimSpan, Group, Ident, Punct, TokenTree}; use pm::{Delimiter, Level, LineColumn}; +use std::ascii; use std::ops::Bound; -use std::{ascii, panic}; trait FromInternal { fn from_internal(x: T) -> Self; @@ -55,7 +54,7 @@ impl ToInternal for Delimiter { } impl FromInternal<(TokenStream, &mut Rustc<'_>)> - for Vec> + for Vec> { fn from_internal((stream, rustc): (TokenStream, &mut Rustc<'_>)) -> Self { use rustc_ast::token::*; @@ -157,12 +156,11 @@ impl FromInternal<(TokenStream, &mut Rustc<'_>)> Question => op!('?'), SingleQuote => op!('\''), - Ident(name, false) if name == kw::DollarCrate => tt!(Ident::dollar_crate()), - Ident(name, is_raw) => tt!(Ident::new(rustc.sess, name, is_raw)), + Ident(sym, is_raw) => tt!(Ident { sym, is_raw }), Lifetime(name) => { let ident = symbol::Ident::new(name, span).without_first_quote(); tt!(Punct { ch: '\'', joint: true }); - tt!(Ident::new(rustc.sess, ident.name, false)); + tt!(Ident { sym: ident.name, is_raw: false }); } Literal(lit) => tt!(Literal { lit }), DocComment(_, attr_style, data) => { @@ -191,9 +189,11 @@ impl FromInternal<(TokenStream, &mut Rustc<'_>)> Interpolated(nt) => { if let Some((name, is_raw)) = ident_name_compatibility_hack(&nt, span, rustc) { - trees.push(TokenTree::Ident(Ident::new( - rustc.sess, name.name, is_raw, name.span, - ))); + trees.push(TokenTree::Ident(Ident { + sym: name.name, + is_raw, + span: name.span, + })); } else { let stream = nt_to_tokenstream(&nt, rustc.sess, CanSynthesizeMissingTokens::No); @@ -217,7 +217,7 @@ impl FromInternal<(TokenStream, &mut Rustc<'_>)> } } -impl ToInternal for TokenTree { +impl ToInternal for TokenTree { fn to_internal(self) -> TokenStream { use rustc_ast::token::*; @@ -306,32 +306,6 @@ impl ToInternal for Level { pub struct FreeFunctions; -#[derive(Copy, Clone, PartialEq, Eq, Hash)] -pub struct Ident { - sym: Symbol, - is_raw: bool, - span: Span, -} - -impl Ident { - fn new(sess: &ParseSess, sym: Symbol, is_raw: bool, span: Span) -> Ident { - let sym = nfc_normalize(&sym.as_str()); - let string = sym.as_str(); - if !rustc_lexer::is_ident(&string) { - panic!("`{:?}` is not a valid identifier", string) - } - if is_raw && !sym.can_be_raw() { - panic!("`{}` cannot be a raw identifier", string); - } - sess.symbol_gallery.insert(sym, span); - Ident { sym, is_raw, span } - } - fn dollar_crate(span: Span) -> Ident { - // `$crate` is accepted as an ident only if it comes from the compiler. - Ident { sym: kw::DollarCrate, is_raw: false, span } - } -} - // FIXME(eddyb) `Literal` should not expose internal `Debug` impls. #[derive(Clone, Debug)] pub struct Literal { @@ -382,12 +356,12 @@ impl<'a> Rustc<'a> { impl server::Types for Rustc<'_> { type FreeFunctions = FreeFunctions; type TokenStream = TokenStream; - type Ident = Ident; type Literal = Literal; type SourceFile = Lrc; type MultiSpan = Vec; type Diagnostic = Diagnostic; type Span = Span; + type Symbol = Symbol; } impl server::FreeFunctions for Rustc<'_> { @@ -413,14 +387,14 @@ impl server::TokenStream for Rustc<'_> { } fn from_token_tree( &mut self, - tree: TokenTree, + tree: TokenTree, ) -> Self::TokenStream { tree.to_internal() } fn concat_trees( &mut self, base: Option, - trees: Vec>, + trees: Vec>, ) -> Self::TokenStream { let mut builder = tokenstream::TokenStreamBuilder::new(); if let Some(base) = base { @@ -448,23 +422,11 @@ impl server::TokenStream for Rustc<'_> { fn into_iter( &mut self, stream: Self::TokenStream, - ) -> Vec> { + ) -> Vec> { FromInternal::from_internal((stream, self)) } } -impl server::Ident for Rustc<'_> { - fn new(&mut self, string: &str, span: Self::Span, is_raw: bool) -> Self::Ident { - Ident::new(self.sess, Symbol::intern(string), is_raw, span) - } - fn span(&mut self, ident: Self::Ident) -> Self::Span { - ident.span - } - fn with_span(&mut self, ident: Self::Ident, span: Self::Span) -> Self::Ident { - Ident { span, ..ident } - } -} - impl server::Literal for Rustc<'_> { fn from_str(&mut self, s: &str) -> Result { let override_span = None; @@ -729,6 +691,28 @@ impl server::Context for Rustc<'_> { fn mixed_site(&mut self) -> Self::Span { self.mixed_site } + + // NOTE: May be run on any thread, so cannot use `nfc_normalize` + fn validate_ident(s: &str) -> Result, ()> { + use unicode_normalization::{is_nfc_quick, IsNormalized, UnicodeNormalization}; + let normalized: Option = match is_nfc_quick(s.chars()) { + IsNormalized::Yes => None, + _ => Some(s.chars().nfc().collect()), + }; + if rustc_lexer::is_ident(normalized.as_ref().map(|s| &s[..]).unwrap_or(s)) { + Ok(normalized) + } else { + Err(()) + } + } + + fn intern_symbol(string: &str) -> Self::Symbol { + Symbol::intern(string) + } + + fn with_symbol_string(symbol: &Self::Symbol, f: impl FnOnce(&str)) { + f(&symbol.as_str()) + } } // See issue #74616 for details diff --git a/library/proc_macro/src/bridge/buffer.rs b/library/proc_macro/src/bridge/buffer.rs index d82669d3e2336..ea5de80da12b2 100644 --- a/library/proc_macro/src/bridge/buffer.rs +++ b/library/proc_macro/src/bridge/buffer.rs @@ -5,6 +5,35 @@ use std::mem; use std::ops::{Deref, DerefMut}; use std::slice; +#[repr(C)] +pub struct Slice<'a, T> { + data: &'a [T; 0], + len: usize, +} + +unsafe impl<'a, T: Sync> Sync for Slice<'a, T> {} +unsafe impl<'a, T: Sync> Send for Slice<'a, T> {} + +impl Copy for Slice<'a, T> {} +impl Clone for Slice<'a, T> { + fn clone(&self) -> Self { + *self + } +} + +impl From<&'a [T]> for Slice<'a, T> { + fn from(xs: &'a [T]) -> Self { + Slice { data: unsafe { &*(xs.as_ptr() as *const [T; 0]) }, len: xs.len() } + } +} + +impl Deref for Slice<'a, T> { + type Target = [T]; + fn deref(&self) -> &[T] { + unsafe { slice::from_raw_parts(self.data.as_ptr(), self.len) } + } +} + #[repr(C)] pub struct Buffer { data: *mut T, diff --git a/library/proc_macro/src/bridge/client.rs b/library/proc_macro/src/bridge/client.rs index 49175a5c476ed..eeb0d3e613a20 100644 --- a/library/proc_macro/src/bridge/client.rs +++ b/library/proc_macro/src/bridge/client.rs @@ -2,6 +2,7 @@ use super::*; +use std::rc::Rc; use std::sync::atomic::Ordering; trait OwnedHandle { @@ -179,7 +180,7 @@ macro_rules! define_handles { for Marked { fn encode(self, w: &mut Writer, s: &mut HandleStore>) { - s.$ity.alloc(self).encode(w, s); + s.$ity.alloc(&self).encode(w, s); } } @@ -201,7 +202,6 @@ define_handles! { Diagnostic, 'interned: - Ident, Span, } @@ -262,6 +262,84 @@ impl fmt::Debug for Span { } } +#[repr(C)] +#[derive(Copy, Clone, PartialEq, Eq, Hash)] +pub(crate) struct Symbol(handle::Handle); + +impl Symbol { + /// Create a new `Symbol` for an identifier. + /// + /// Validates and normalizes before converting it to a symbol. + pub(crate) fn new_ident(string: &str, is_raw: bool) -> Self { + Symbol(Bridge::with(|bridge| { + let mut normalized = Buffer::new(); + if !(bridge.validate_ident)(string.as_bytes().into(), &mut normalized) { + panic!("`{:?}` is not a valid identifier", string) + } + let string = if normalized.len() > 0 { + std::str::from_utf8(&normalized[..]).unwrap() + } else { + string + }; + if is_raw && !Self::can_be_raw(string) { + panic!("`{:?}` cannot be a raw identifier", string); + } + bridge.symbols.alloc(string) + })) + } + + // Mimics the behaviour of `Symbol::can_be_raw` from `rustc_span` + fn can_be_raw(string: &str) -> bool { + match string { + "" | "_" | "super" | "self" | "Self" | "crate" | "$crate" | "{{root}}" => false, + _ => true, + } + } +} + +impl !Send for Symbol {} +impl !Sync for Symbol {} + +impl fmt::Debug for Symbol { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + Bridge::with(|bridge| fmt::Debug::fmt(&bridge.symbols[self.0], f)) + } +} + +impl fmt::Display for Symbol { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + Bridge::with(|bridge| fmt::Display::fmt(&bridge.symbols[self.0], f)) + } +} + +impl Encode> for Symbol { + fn encode(self, w: &mut Writer, s: &mut Bridge<'_>) { + s.symbols[self.0][..].encode(w, &mut ()); + } +} + +impl DecodeMut<'_, '_, HandleStore>> + for Marked +{ + fn decode(r: &mut Reader<'_>, s: &mut HandleStore>) -> Self { + Mark::mark(S::intern_symbol(<&str>::decode(r, s))) + } +} + +impl Encode>> + for Marked +{ + fn encode(self, w: &mut Writer, s: &mut HandleStore>) { + S::with_symbol_string(&self.unmark(), |sym| sym.encode(w, s)) + } +} + +impl DecodeMut<'_, '_, Bridge<'_>> for Symbol { + fn decode(r: &mut Reader<'_>, s: &mut Bridge<'_>) -> Self { + Symbol(s.symbols.alloc(<&str>::decode(r, &mut ()))) + } +} + macro_rules! client_send_impl { (wait $name:ident :: $method:ident($($arg:ident),*) $(-> $ret_ty:ty)?) => { Bridge::with(|bridge| { @@ -269,11 +347,11 @@ macro_rules! client_send_impl { b.clear(); api_tags::Method::$name(api_tags::$name::$method).encode(&mut b, &mut ()); - reverse_encode!(b; $($arg),*); + reverse_encode!(b, bridge; $($arg),*); b = bridge.dispatch.call(b); - let r = Result::<_, PanicMessage>::decode(&mut &b[..], &mut ()); + let r = Result::<_, PanicMessage>::decode(&mut &b[..], bridge); bridge.cached_buffer = b; @@ -287,7 +365,7 @@ macro_rules! client_send_impl { b.clear(); api_tags::Method::$name(api_tags::$name::$method).encode(&mut b, &mut ()); - reverse_encode!(b; $($arg),*); + reverse_encode!(b, bridge; $($arg),*); $( let raw_handle = <$ret_ty as OwnedHandle>::next_raw_handle(); @@ -331,6 +409,12 @@ struct Bridge<'a> { /// Server-side function that the client uses to make requests. dispatch: closure::Closure<'a, Buffer, Buffer>, + /// Server-side function to validate and normalize an ident. + validate_ident: extern "C" fn(buffer::Slice<'_, u8>, &mut Buffer) -> bool, + + /// Interned store for storing symbols within the client. + symbols: handle::InternedStore>, + /// Provided context for this macro expansion. context: ExpnContext, } @@ -436,6 +520,8 @@ fn maybe_install_panic_hook(force_show_panics: bool) { }); } +static SYMBOL_COUNTER: AtomicUsize = AtomicUsize::new(1); + /// Client-side helper for handling client panics, entering the bridge, /// deserializing input and serializing output. // FIXME(eddyb) maybe replace `Bridge::enter` with this? @@ -443,7 +529,7 @@ fn run_client DecodeMut<'a, 's, ()>, R: Encode<()>>( config: BridgeConfig<'_>, f: impl FnOnce(A) -> R, ) -> Buffer { - let BridgeConfig { input: mut b, dispatch, force_show_panics } = config; + let BridgeConfig { input: mut b, dispatch, validate_ident, force_show_panics } = config; panic::catch_unwind(panic::AssertUnwindSafe(|| { maybe_install_panic_hook(force_show_panics); @@ -452,8 +538,13 @@ fn run_client DecodeMut<'a, 's, ()>, R: Encode<()>>( let (input, context) = <(A, ExpnContext)>::decode(reader, &mut ()); // Put the buffer we used for input back in the `Bridge` for requests. - let new_state = - BridgeState::Connected(Bridge { cached_buffer: b.take(), dispatch, context }); + let new_state = BridgeState::Connected(Bridge { + cached_buffer: b.take(), + dispatch, + validate_ident, + symbols: handle::InternedStore::new(&SYMBOL_COUNTER), + context, + }); BRIDGE_STATE.with(|state| { state.set(new_state, || { diff --git a/library/proc_macro/src/bridge/handle.rs b/library/proc_macro/src/bridge/handle.rs index 9c57a9c3d2a7c..9b0475f7049bd 100644 --- a/library/proc_macro/src/bridge/handle.rs +++ b/library/proc_macro/src/bridge/handle.rs @@ -1,9 +1,11 @@ //! Server-side handles and storage for per-handle data. +use std::borrow::Borrow; use std::collections::{BTreeMap, HashMap}; use std::hash::Hash; use std::num::NonZeroU32; use std::ops::{Index, IndexMut}; +use std::rc::Rc; use std::sync::atomic::{AtomicUsize, Ordering}; pub(super) type Handle = NonZeroU32; @@ -53,22 +55,60 @@ impl IndexMut for OwnedStore { } } +pub(super) trait FromKey { + fn from_key(key: &Q) -> Self; +} + +impl FromKey for T { + fn from_key(key: &T) -> T { + key.clone() + } +} + +impl FromKey for Rc +where + Rc: for<'a> From<&'a T>, +{ + fn from_key(key: &T) -> Rc { + key.into() + } +} + pub(super) struct InternedStore { owned: OwnedStore, interner: HashMap, } -impl InternedStore { +impl InternedStore { pub(super) fn new(counter: &'static AtomicUsize) -> Self { InternedStore { owned: OwnedStore::new(counter), interner: HashMap::new() } } - pub(super) fn alloc(&mut self, x: T) -> Handle { + pub(super) fn alloc<'a, Q: ?Sized>(&mut self, x: &'a Q) -> Handle + where + T: Borrow + FromKey, + Q: Hash + Eq, + { let owned = &mut self.owned; - *self.interner.entry(x).or_insert_with(|| owned.alloc(x)) + *self + .interner + .raw_entry_mut() + .from_key(x) + .or_insert_with(|| { + let own = T::from_key(x); + (own.clone(), owned.alloc(own)) + }) + .1 } pub(super) fn copy(&mut self, h: Handle) -> T { - self.owned[h] + self.owned[h].clone() + } +} + +impl Index for InternedStore { + type Output = T; + fn index(&self, h: Handle) -> &T { + self.owned.index(h) } } diff --git a/library/proc_macro/src/bridge/mod.rs b/library/proc_macro/src/bridge/mod.rs index 44b5e28d0fbf2..fbfc723da4d0b 100644 --- a/library/proc_macro/src/bridge/mod.rs +++ b/library/proc_macro/src/bridge/mod.rs @@ -69,11 +69,11 @@ macro_rules! with_api { wait fn from_str(src: &str) -> $S::TokenStream; wait fn to_string($self: &$S::TokenStream) -> String; nowait fn from_token_tree( - tree: TokenTree<$S::TokenStream, $S::Span, $S::Ident, $S::Literal>, + tree: TokenTree<$S::TokenStream, $S::Span, $S::Symbol, $S::Literal>, ) -> $S::TokenStream; nowait fn concat_trees( base: Option<$S::TokenStream>, - trees: Vec>, + trees: Vec>, ) -> $S::TokenStream; nowait fn concat_streams( base: Option<$S::TokenStream>, @@ -81,12 +81,7 @@ macro_rules! with_api { ) -> $S::TokenStream; wait fn into_iter( $self: $S::TokenStream - ) -> Vec>; - }, - Ident { - wait fn new(string: &str, span: $S::Span, is_raw: bool) -> $S::Ident; - wait fn span($self: $S::Ident) -> $S::Span; - wait fn with_span($self: $S::Ident, span: $S::Span) -> $S::Ident; + ) -> Vec>; }, Literal { nowait fn drop($self: $S::Literal); @@ -154,10 +149,10 @@ macro_rules! with_api { // FIXME(eddyb) this calls `encode` for each argument, but in reverse, // to avoid borrow conflicts from borrows started by `&mut` arguments. macro_rules! reverse_encode { - ($writer:ident;) => {}; - ($writer:ident; $first:ident $(, $rest:ident)*) => { - reverse_encode!($writer; $($rest),*); - $first.encode(&mut $writer, &mut ()); + ($writer:ident, $s:ident;) => {}; + ($writer:ident, $s:ident; $first:ident $(, $rest:ident)*) => { + reverse_encode!($writer, $s; $($rest),*); + $first.encode(&mut $writer, $s); } } @@ -204,6 +199,9 @@ pub struct BridgeConfig<'a> { /// Server-side function that the client uses to make requests. dispatch: closure::Closure<'a, Buffer, Buffer>, + /// Server-side function to validate and normalize an ident. + validate_ident: extern "C" fn(buffer::Slice<'_, u8>, &mut Buffer) -> bool, + /// If 'true', always invoke the default panic hook force_show_panics: bool, } @@ -453,14 +451,14 @@ macro_rules! compound_traits { } #[derive(Copy, Clone)] -pub struct DelimSpan { - pub open: S, - pub close: S, - pub entire: S, +pub struct DelimSpan { + pub open: Sp, + pub close: Sp, + pub entire: Sp, } -impl DelimSpan { - pub fn from_single(span: S) -> Self { +impl DelimSpan { + pub fn from_single(span: Sp) -> Self { DelimSpan { open: span, close: span, entire: span } } } @@ -468,33 +466,42 @@ impl DelimSpan { compound_traits!(struct DelimSpan { open, close, entire }); #[derive(Clone)] -pub struct Group { +pub struct Group { pub delimiter: Delimiter, pub stream: Option, - pub span: DelimSpan, + pub span: DelimSpan, } compound_traits!(struct Group { delimiter, stream, span }); #[derive(Clone)] -pub struct Punct { +pub struct Punct { pub ch: char, pub joint: bool, - pub span: S, + pub span: Sp, } compound_traits!(struct Punct { ch, joint, span }); +#[derive(Copy, Clone, Eq, PartialEq)] +pub struct Ident { + pub sym: Sy, + pub is_raw: bool, + pub span: Sp, +} + +compound_traits!(struct Ident { sym, is_raw, span }); + #[derive(Clone)] -pub enum TokenTree { - Group(Group), - Punct(Punct), - Ident(I), +pub enum TokenTree { + Group(Group), + Punct(Punct), + Ident(Ident), Literal(L), } compound_traits!( - enum TokenTree { + enum TokenTree { Group(tt), Punct(tt), Ident(tt), @@ -505,10 +512,10 @@ compound_traits!( /// Context provided alongside the initial inputs for a macro expansion. /// Provides values such as spans which are used frequently to avoid RPC. #[derive(Clone)] -struct ExpnContext { - def_site: S, - call_site: S, - mixed_site: S, +struct ExpnContext { + def_site: Sp, + call_site: Sp, + mixed_site: Sp, } compound_traits!( diff --git a/library/proc_macro/src/bridge/server.rs b/library/proc_macro/src/bridge/server.rs index a4452b3651e57..bb4026db6f30a 100644 --- a/library/proc_macro/src/bridge/server.rs +++ b/library/proc_macro/src/bridge/server.rs @@ -14,8 +14,6 @@ macro_rules! associated_item { (type FreeFunctions: 'static;); (type TokenStream) => (type TokenStream: 'static + Clone;); - (type Ident) => - (type Ident: 'static + Copy + Eq + Hash;); (type Literal) => (type Literal: 'static + Clone;); (type SourceFile) => @@ -38,6 +36,21 @@ pub trait Context: Types { fn def_site(&mut self) -> Self::Span; fn call_site(&mut self) -> Self::Span; fn mixed_site(&mut self) -> Self::Span; + + /// Check if an identifier is valid, and return `Ok(...)` if it is. + /// + /// May be called on any thread. + /// + /// Returns `Ok(Some(str))` with a normalized version of the identifier if + /// normalization is required, and `Ok(None)` if the existing identifier is + /// already normalized. + fn validate_ident(ident: &str) -> Result, ()>; + + /// Intern a symbol received from RPC + fn intern_symbol(ident: &str) -> Self::Symbol; + + /// Recover the string value of a symbol, and invoke a callback with it. + fn with_symbol_string(symbol: &Self::Symbol, f: impl FnOnce(&str)); } macro_rules! declare_server_traits { @@ -46,6 +59,7 @@ macro_rules! declare_server_traits { }),* $(,)?) => { pub trait Types { $(associated_item!(type $name);)* + type Symbol: 'static + Copy + Eq + Hash; } $(pub trait $name: Types { @@ -70,6 +84,15 @@ impl Context for MarkedTypes { fn mixed_site(&mut self) -> Self::Span { <_>::mark(Context::mixed_site(&mut self.0)) } + fn validate_ident(ident: &str) -> Result, ()> { + S::validate_ident(ident) + } + fn intern_symbol(ident: &str) -> Self::Symbol { + <_>::mark(S::intern_symbol(ident)) + } + fn with_symbol_string(symbol: &Self::Symbol, f: impl FnOnce(&str)) { + S::with_symbol_string(symbol.unmark(), f) + } } macro_rules! define_mark_types_impls { @@ -78,6 +101,7 @@ macro_rules! define_mark_types_impls { }),* $(,)?) => { impl Types for MarkedTypes { $(type $name = Marked;)* + type Symbol = Marked; } $(impl $name for MarkedTypes { @@ -116,11 +140,16 @@ macro_rules! define_dispatcher_impl { pub trait DispatcherTrait { // HACK(eddyb) these are here to allow `Self::$name` to work below. $(type $name;)* + type Symbol; + fn dispatch(&mut self, b: Buffer) -> Buffer; + fn validate_ident(ident: &str) -> Result, ()>; } impl DispatcherTrait for Dispatcher> { $(type $name = as Types>::$name;)* + type Symbol = as Types>::Symbol; + fn dispatch(&mut self, mut b: Buffer) -> Buffer { let Dispatcher { handle_store, server } = self; @@ -152,18 +181,35 @@ macro_rules! define_dispatcher_impl { } b } + fn validate_ident(ident: &str) -> Result, ()> { + S::validate_ident(ident) + } } } } with_api!(Self, self_, define_dispatcher_impl); +extern "C" fn validate_ident_impl( + string: buffer::Slice<'_, u8>, + normalized: &mut Buffer, +) -> bool { + match std::str::from_utf8(&string[..]).map_err(|_| ()).and_then(D::validate_ident) { + Ok(Some(norm)) => { + *normalized = norm.into_bytes().into(); + true + } + Ok(None) => true, + Err(_) => false, + } +} + pub trait ExecutionStrategy { - fn run_bridge_and_client( + fn run_bridge_and_client( &self, - dispatcher: &mut impl DispatcherTrait, + dispatcher: &mut D, input: Buffer, - run_client: extern "C" fn(BridgeConfig<'_>, D) -> Buffer, - client_data: D, + run_client: extern "C" fn(BridgeConfig<'_>, T) -> Buffer, + client_data: T, force_show_panics: bool, ) -> Buffer; } @@ -183,12 +229,12 @@ impl

ExecutionStrategy for MaybeCrossThread

where P: MessagePipe> + Send + 'static, { - fn run_bridge_and_client( + fn run_bridge_and_client( &self, - dispatcher: &mut impl DispatcherTrait, + dispatcher: &mut D, input: Buffer, - run_client: extern "C" fn(BridgeConfig<'_>, D) -> Buffer, - client_data: D, + run_client: extern "C" fn(BridgeConfig<'_>, T) -> Buffer, + client_data: T, force_show_panics: bool, ) -> Buffer { if self.cross_thread { @@ -215,18 +261,23 @@ where pub struct SameThread; impl ExecutionStrategy for SameThread { - fn run_bridge_and_client( + fn run_bridge_and_client( &self, - dispatcher: &mut impl DispatcherTrait, + dispatcher: &mut D, input: Buffer, - run_client: extern "C" fn(BridgeConfig<'_>, D) -> Buffer, - client_data: D, + run_client: extern "C" fn(BridgeConfig<'_>, T) -> Buffer, + client_data: T, force_show_panics: bool, ) -> Buffer { let mut dispatch = |b| dispatcher.dispatch(b); run_client( - BridgeConfig { input, dispatch: (&mut dispatch).into(), force_show_panics }, + BridgeConfig { + input, + dispatch: (&mut dispatch).into(), + validate_ident: validate_ident_impl::, + force_show_panics, + }, client_data, ) } @@ -244,12 +295,12 @@ impl

ExecutionStrategy for CrossThread

where P: MessagePipe> + Send + 'static, { - fn run_bridge_and_client( + fn run_bridge_and_client( &self, - dispatcher: &mut impl DispatcherTrait, + dispatcher: &mut D, input: Buffer, - run_client: extern "C" fn(BridgeConfig<'_>, D) -> Buffer, - client_data: D, + run_client: extern "C" fn(BridgeConfig<'_>, T) -> Buffer, + client_data: T, force_show_panics: bool, ) -> Buffer { let (mut server, mut client) = P::new(); @@ -267,7 +318,12 @@ where }; run_client( - BridgeConfig { input, dispatch: (&mut dispatch).into(), force_show_panics }, + BridgeConfig { + input, + dispatch: (&mut dispatch).into(), + validate_ident: validate_ident_impl::, + force_show_panics, + }, client_data, ) }); diff --git a/library/proc_macro/src/lib.rs b/library/proc_macro/src/lib.rs index 37d0d93714252..c6cd76c1cc936 100644 --- a/library/proc_macro/src/lib.rs +++ b/library/proc_macro/src/lib.rs @@ -31,6 +31,7 @@ #![feature(restricted_std)] #![feature(rustc_attrs)] #![feature(min_specialization)] +#![feature(hash_raw_entry)] #![recursion_limit = "256"] #[unstable(feature = "proc_macro_internals", issue = "27812")] @@ -182,7 +183,7 @@ fn tree_to_bridge_tree( ) -> bridge::TokenTree< bridge::client::TokenStream, bridge::client::Span, - bridge::client::Ident, + bridge::client::Symbol, bridge::client::Literal, > { match tree { @@ -259,7 +260,7 @@ pub mod token_stream { bridge::TokenTree< bridge::client::TokenStream, bridge::client::Span, - bridge::client::Ident, + bridge::client::Symbol, bridge::client::Literal, >, >, @@ -911,7 +912,7 @@ impl PartialEq for char { /// An identifier (`ident`). #[derive(Clone)] #[stable(feature = "proc_macro_lib2", since = "1.29.0")] -pub struct Ident(bridge::client::Ident); +pub struct Ident(bridge::Ident); impl Ident { /// Creates a new `Ident` with the given `string` as well as the specified @@ -935,7 +936,11 @@ impl Ident { /// tokens, requires a `Span` to be specified at construction. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn new(string: &str, span: Span) -> Ident { - Ident(bridge::client::Ident::new(string, span.0, false)) + Ident(bridge::Ident { + sym: bridge::client::Symbol::new_ident(string, false), + is_raw: false, + span: span.0, + }) } /// Same as `Ident::new`, but creates a raw identifier (`r#ident`). @@ -944,29 +949,24 @@ impl Ident { /// (e.g. `self`, `super`) are not supported, and will cause a panic. #[stable(feature = "proc_macro_raw_ident", since = "1.47.0")] pub fn new_raw(string: &str, span: Span) -> Ident { - Ident(bridge::client::Ident::new(string, span.0, true)) + Ident(bridge::Ident { + sym: bridge::client::Symbol::new_ident(string, true), + is_raw: true, + span: span.0, + }) } /// Returns the span of this `Ident`, encompassing the entire string returned /// by [`to_string`](Self::to_string). #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn span(&self) -> Span { - Span(self.0.span()) + Span(self.0.span) } /// Configures the span of this `Ident`, possibly changing its hygiene context. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn set_span(&mut self, span: Span) { - self.0 = self.0.with_span(span.0); - } -} - -// N.B., the bridge only provides `to_string`, implement `fmt::Display` -// based on it (the reverse of the usual relationship between the two). -#[stable(feature = "proc_macro_lib", since = "1.15.0")] -impl ToString for Ident { - fn to_string(&self) -> String { - TokenStream::from(TokenTree::from(self.clone())).to_string() + self.0.span = span.0; } } @@ -975,7 +975,7 @@ impl ToString for Ident { #[stable(feature = "proc_macro_lib2", since = "1.29.0")] impl fmt::Display for Ident { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(&self.to_string()) + if self.0.is_raw { write!(f, "r#{}", self.0.sym) } else { write!(f, "{}", self.0.sym) } } } From 4f83573d41889ccebd613019f4156568228aa805 Mon Sep 17 00:00:00 2001 From: Nika Layzell Date: Fri, 2 Jul 2021 00:40:26 -0400 Subject: [PATCH 12/12] proc_macro: stop using a remote object handle for Literal This builds on the symbol infrastructure built for ident to replicate the `LitKind` and `Lit` structures in rustc within the `proc_macro` client, allowing literals to be fully created and interacted with from the client thread. Only parsing and subspan operations still require sync RPC. --- .../rustc_expand/src/proc_macro_server.rs | 262 ++++++++---------- library/proc_macro/src/bridge/client.rs | 25 +- library/proc_macro/src/bridge/mod.rs | 75 +++-- library/proc_macro/src/bridge/rpc.rs | 1 + library/proc_macro/src/bridge/server.rs | 2 - library/proc_macro/src/lib.rs | 115 +++++--- 6 files changed, 255 insertions(+), 225 deletions(-) diff --git a/compiler/rustc_expand/src/proc_macro_server.rs b/compiler/rustc_expand/src/proc_macro_server.rs index 89a69bacb0f37..5e09566d97514 100644 --- a/compiler/rustc_expand/src/proc_macro_server.rs +++ b/compiler/rustc_expand/src/proc_macro_server.rs @@ -15,12 +15,11 @@ use rustc_session::parse::ParseSess; use rustc_span::def_id::CrateNum; use rustc_span::hygiene::ExpnId; use rustc_span::hygiene::ExpnKind; -use rustc_span::symbol::{self, kw, sym, Symbol}; +use rustc_span::symbol::{self, sym, Symbol}; use rustc_span::{BytePos, FileName, MultiSpan, Pos, RealFileName, SourceFile, Span}; -use pm::bridge::{server, DelimSpan, Group, Ident, Punct, TokenTree}; +use pm::bridge::{server, DelimSpan, Group, Ident, LitKind, Literal, Punct, TokenTree}; use pm::{Delimiter, Level, LineColumn}; -use std::ascii; use std::ops::Bound; trait FromInternal { @@ -53,9 +52,40 @@ impl ToInternal for Delimiter { } } -impl FromInternal<(TokenStream, &mut Rustc<'_>)> - for Vec> -{ +impl FromInternal for LitKind { + fn from_internal(kind: token::LitKind) -> Self { + match kind { + token::Byte => LitKind::Byte, + token::Char => LitKind::Char, + token::Integer => LitKind::Integer, + token::Float => LitKind::Float, + token::Str => LitKind::Str, + token::StrRaw(n) => LitKind::StrRaw(n), + token::ByteStr => LitKind::ByteStr, + token::ByteStrRaw(n) => LitKind::ByteStrRaw(n), + token::Err => LitKind::Err, + token::Bool => unreachable!(), + } + } +} + +impl ToInternal for LitKind { + fn to_internal(self) -> token::LitKind { + match self { + LitKind::Byte => token::Byte, + LitKind::Char => token::Char, + LitKind::Integer => token::Integer, + LitKind::Float => token::Float, + LitKind::Str => token::Str, + LitKind::StrRaw(n) => token::StrRaw(n), + LitKind::ByteStr => token::ByteStr, + LitKind::ByteStrRaw(n) => token::ByteStrRaw(n), + LitKind::Err => token::Err, + } + } +} + +impl FromInternal<(TokenStream, &mut Rustc<'_>)> for Vec> { fn from_internal((stream, rustc): (TokenStream, &mut Rustc<'_>)) -> Self { use rustc_ast::token::*; @@ -162,7 +192,9 @@ impl FromInternal<(TokenStream, &mut Rustc<'_>)> tt!(Punct { ch: '\'', joint: true }); tt!(Ident { sym: ident.name, is_raw: false }); } - Literal(lit) => tt!(Literal { lit }), + Literal(token::Lit { kind, symbol, suffix }) => { + tt!(Literal { kind: FromInternal::from_internal(kind), symbol, suffix }); + } DocComment(_, attr_style, data) => { let mut escaped = String::new(); for ch in data.as_str().chars() { @@ -217,7 +249,7 @@ impl FromInternal<(TokenStream, &mut Rustc<'_>)> } } -impl ToInternal for TokenTree { +impl ToInternal for TokenTree { fn to_internal(self) -> TokenStream { use rustc_ast::token::*; @@ -235,7 +267,9 @@ impl ToInternal for TokenTree { return tokenstream::TokenTree::token(Ident(sym, is_raw), span).into(); } TokenTree::Literal(self::Literal { - lit: token::Lit { kind: token::Integer, symbol, suffix }, + kind: self::LitKind::Integer, + symbol, + suffix, span, }) if symbol.as_str().starts_with('-') => { let minus = BinOp(BinOpToken::Minus); @@ -246,7 +280,9 @@ impl ToInternal for TokenTree { return vec![a, b].into_iter().collect(); } TokenTree::Literal(self::Literal { - lit: token::Lit { kind: token::Float, symbol, suffix }, + kind: self::LitKind::Float, + symbol, + suffix, span, }) if symbol.as_str().starts_with('-') => { let minus = BinOp(BinOpToken::Minus); @@ -256,8 +292,12 @@ impl ToInternal for TokenTree { let b = tokenstream::TokenTree::token(float, span); return vec![a, b].into_iter().collect(); } - TokenTree::Literal(self::Literal { lit, span }) => { - return tokenstream::TokenTree::token(Literal(lit), span).into(); + TokenTree::Literal(self::Literal { kind, symbol, suffix, span }) => { + return tokenstream::TokenTree::token( + TokenKind::lit(kind.to_internal(), symbol, suffix), + span, + ) + .into(); } }; @@ -306,13 +346,6 @@ impl ToInternal for Level { pub struct FreeFunctions; -// FIXME(eddyb) `Literal` should not expose internal `Debug` impls. -#[derive(Clone, Debug)] -pub struct Literal { - lit: token::Lit, - span: Span, -} - pub(crate) struct Rustc<'a> { resolver: &'a dyn ResolverExpand, sess: &'a ParseSess, @@ -344,19 +377,11 @@ impl<'a> Rustc<'a> { rebased_spans: FxHashMap::default(), } } - - fn lit(&mut self, kind: token::LitKind, symbol: Symbol, suffix: Option) -> Literal { - Literal { - lit: token::Lit::new(kind, symbol, suffix), - span: server::Context::call_site(self), - } - } } impl server::Types for Rustc<'_> { type FreeFunctions = FreeFunctions; type TokenStream = TokenStream; - type Literal = Literal; type SourceFile = Lrc; type MultiSpan = Vec; type Diagnostic = Diagnostic; @@ -368,67 +393,8 @@ impl server::FreeFunctions for Rustc<'_> { fn track_env_var(&mut self, var: &str, value: Option<&str>) { self.sess.env_depinfo.borrow_mut().insert((Symbol::intern(var), value.map(Symbol::intern))); } -} - -impl server::TokenStream for Rustc<'_> { - fn is_empty(&mut self, stream: &Self::TokenStream) -> bool { - stream.is_empty() - } - fn from_str(&mut self, src: &str) -> Self::TokenStream { - parse_stream_from_source_str( - FileName::proc_macro_source_code(src), - src.to_string(), - self.sess, - Some(self.call_site), - ) - } - fn to_string(&mut self, stream: &Self::TokenStream) -> String { - pprust::tts_to_string(stream) - } - fn from_token_tree( - &mut self, - tree: TokenTree, - ) -> Self::TokenStream { - tree.to_internal() - } - fn concat_trees( - &mut self, - base: Option, - trees: Vec>, - ) -> Self::TokenStream { - let mut builder = tokenstream::TokenStreamBuilder::new(); - if let Some(base) = base { - builder.push(base); - } - for tree in trees { - builder.push(tree.to_internal()); - } - builder.build() - } - fn concat_streams( - &mut self, - base: Option, - streams: Vec, - ) -> Self::TokenStream { - let mut builder = tokenstream::TokenStreamBuilder::new(); - if let Some(base) = base { - builder.push(base); - } - for stream in streams { - builder.push(stream); - } - builder.build() - } - fn into_iter( - &mut self, - stream: Self::TokenStream, - ) -> Vec> { - FromInternal::from_internal((stream, self)) - } -} -impl server::Literal for Rustc<'_> { - fn from_str(&mut self, s: &str) -> Result { + fn literal_from_str(&mut self, s: &str) -> Result, ()> { let override_span = None; let stream = parse_stream_from_source_str( FileName::proc_macro_source_code(s), @@ -449,66 +415,21 @@ impl server::Literal for Rustc<'_> { // There is a comment or whitespace adjacent to the literal. return Err(()); } - let lit = match token.kind { + let token::Lit { kind, symbol, suffix } = match token.kind { TokenKind::Literal(lit) => lit, _ => return Err(()), }; - Ok(Literal { lit, span: self.call_site }) - } - fn debug_kind(&mut self, literal: &Self::Literal) -> String { - format!("{:?}", literal.lit.kind) - } - fn symbol(&mut self, literal: &Self::Literal) -> String { - literal.lit.symbol.to_string() - } - fn suffix(&mut self, literal: &Self::Literal) -> Option { - literal.lit.suffix.as_ref().map(Symbol::to_string) - } - fn integer(&mut self, n: &str) -> Self::Literal { - self.lit(token::Integer, Symbol::intern(n), None) - } - fn typed_integer(&mut self, n: &str, kind: &str) -> Self::Literal { - self.lit(token::Integer, Symbol::intern(n), Some(Symbol::intern(kind))) - } - fn float(&mut self, n: &str) -> Self::Literal { - self.lit(token::Float, Symbol::intern(n), None) - } - fn f32(&mut self, n: &str) -> Self::Literal { - self.lit(token::Float, Symbol::intern(n), Some(sym::f32)) - } - fn f64(&mut self, n: &str) -> Self::Literal { - self.lit(token::Float, Symbol::intern(n), Some(sym::f64)) - } - fn string(&mut self, string: &str) -> Self::Literal { - let mut escaped = String::new(); - for ch in string.chars() { - escaped.extend(ch.escape_debug()); - } - self.lit(token::Str, Symbol::intern(&escaped), None) - } - fn character(&mut self, ch: char) -> Self::Literal { - let mut escaped = String::new(); - escaped.extend(ch.escape_unicode()); - self.lit(token::Char, Symbol::intern(&escaped), None) - } - fn byte_string(&mut self, bytes: &[u8]) -> Self::Literal { - let string = bytes - .iter() - .cloned() - .flat_map(ascii::escape_default) - .map(Into::::into) - .collect::(); - self.lit(token::ByteStr, Symbol::intern(&string), None) - } - fn span(&mut self, literal: &Self::Literal) -> Self::Span { - literal.span - } - fn set_span(&mut self, literal: &mut Self::Literal, span: Self::Span) { - literal.span = span; + Ok(Literal { + kind: FromInternal::from_internal(kind), + symbol, + suffix, + span: self.call_site, + }) } - fn subspan( + + fn literal_subspan( &mut self, - literal: &Self::Literal, + literal: Literal, start: Bound, end: Bound, ) -> Option { @@ -544,6 +465,63 @@ impl server::Literal for Rustc<'_> { } } +impl server::TokenStream for Rustc<'_> { + fn is_empty(&mut self, stream: &Self::TokenStream) -> bool { + stream.is_empty() + } + fn from_str(&mut self, src: &str) -> Self::TokenStream { + parse_stream_from_source_str( + FileName::proc_macro_source_code(src), + src.to_string(), + self.sess, + Some(self.call_site), + ) + } + fn to_string(&mut self, stream: &Self::TokenStream) -> String { + pprust::tts_to_string(stream) + } + fn from_token_tree( + &mut self, + tree: TokenTree, + ) -> Self::TokenStream { + tree.to_internal() + } + fn concat_trees( + &mut self, + base: Option, + trees: Vec>, + ) -> Self::TokenStream { + let mut builder = tokenstream::TokenStreamBuilder::new(); + if let Some(base) = base { + builder.push(base); + } + for tree in trees { + builder.push(tree.to_internal()); + } + builder.build() + } + fn concat_streams( + &mut self, + base: Option, + streams: Vec, + ) -> Self::TokenStream { + let mut builder = tokenstream::TokenStreamBuilder::new(); + if let Some(base) = base { + builder.push(base); + } + for stream in streams { + builder.push(stream); + } + builder.build() + } + fn into_iter( + &mut self, + stream: Self::TokenStream, + ) -> Vec> { + FromInternal::from_internal((stream, self)) + } +} + impl server::SourceFile for Rustc<'_> { fn eq(&mut self, file1: &Self::SourceFile, file2: &Self::SourceFile) -> bool { Lrc::ptr_eq(file1, file2) diff --git a/library/proc_macro/src/bridge/client.rs b/library/proc_macro/src/bridge/client.rs index eeb0d3e613a20..c2a0886367190 100644 --- a/library/proc_macro/src/bridge/client.rs +++ b/library/proc_macro/src/bridge/client.rs @@ -196,7 +196,6 @@ define_handles! { 'owned: FreeFunctions, TokenStream, - Literal, SourceFile, MultiSpan, Diagnostic, @@ -217,25 +216,6 @@ impl Clone for TokenStream { } } -impl Clone for Literal { - fn clone(&self) -> Self { - self.clone() - } -} - -impl fmt::Debug for Literal { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("Literal") - // format the kind without quotes, as in `kind: Float` - .field("kind", &format_args!("{}", &self.debug_kind())) - .field("symbol", &self.symbol()) - // format `Some("...")` on one line even in {:#?} mode - .field("suffix", &format_args!("{:?}", &self.suffix())) - .field("span", &self.span()) - .finish() - } -} - impl Clone for SourceFile { fn clone(&self) -> Self { self.clone() @@ -267,6 +247,11 @@ impl fmt::Debug for Span { pub(crate) struct Symbol(handle::Handle); impl Symbol { + /// Intern a new `Symbol` + pub(crate) fn new(string: &str) -> Self { + Symbol(Bridge::with(|bridge| bridge.symbols.alloc(string))) + } + /// Create a new `Symbol` for an identifier. /// /// Validates and normalizes before converting it to a symbol. diff --git a/library/proc_macro/src/bridge/mod.rs b/library/proc_macro/src/bridge/mod.rs index fbfc723da4d0b..b74894dd20c0c 100644 --- a/library/proc_macro/src/bridge/mod.rs +++ b/library/proc_macro/src/bridge/mod.rs @@ -61,6 +61,8 @@ macro_rules! with_api { FreeFunctions { nowait fn drop($self: $S::FreeFunctions); nowait fn track_env_var(var: &str, value: Option<&str>); + wait fn literal_from_str(s: &str) -> Result, ()>; + wait fn literal_subspan(lit: Literal<$S::Span, $S::Symbol>, start: Bound, end: Bound) -> Option<$S::Span>; }, TokenStream { nowait fn drop($self: $S::TokenStream); @@ -69,11 +71,11 @@ macro_rules! with_api { wait fn from_str(src: &str) -> $S::TokenStream; wait fn to_string($self: &$S::TokenStream) -> String; nowait fn from_token_tree( - tree: TokenTree<$S::TokenStream, $S::Span, $S::Symbol, $S::Literal>, + tree: TokenTree<$S::TokenStream, $S::Span, $S::Symbol>, ) -> $S::TokenStream; nowait fn concat_trees( base: Option<$S::TokenStream>, - trees: Vec>, + trees: Vec>, ) -> $S::TokenStream; nowait fn concat_streams( base: Option<$S::TokenStream>, @@ -81,30 +83,7 @@ macro_rules! with_api { ) -> $S::TokenStream; wait fn into_iter( $self: $S::TokenStream - ) -> Vec>; - }, - Literal { - nowait fn drop($self: $S::Literal); - nowait fn clone($self: &$S::Literal) -> $S::Literal; - wait fn from_str(s: &str) -> Result<$S::Literal, ()>; - wait fn debug_kind($self: &$S::Literal) -> String; - wait fn symbol($self: &$S::Literal) -> String; - wait fn suffix($self: &$S::Literal) -> Option; - nowait fn integer(n: &str) -> $S::Literal; - nowait fn typed_integer(n: &str, kind: &str) -> $S::Literal; - nowait fn float(n: &str) -> $S::Literal; - nowait fn f32(n: &str) -> $S::Literal; - nowait fn f64(n: &str) -> $S::Literal; - nowait fn string(string: &str) -> $S::Literal; - nowait fn character(ch: char) -> $S::Literal; - nowait fn byte_string(bytes: &[u8]) -> $S::Literal; - wait fn span($self: &$S::Literal) -> $S::Span; - nowait fn set_span($self: &mut $S::Literal, span: $S::Span); - wait fn subspan( - $self: &$S::Literal, - start: Bound, - end: Bound, - ) -> Option<$S::Span>; + ) -> Vec>; }, SourceFile { nowait fn drop($self: $S::SourceFile); @@ -368,6 +347,7 @@ mark_noop! { String, usize, Delimiter, + LitKind, Level, LineColumn, Spacing, @@ -398,6 +378,33 @@ rpc_encode_decode!( } ); +#[derive(Copy, Clone, Eq, PartialEq, Debug)] +pub enum LitKind { + Byte, + Char, + Integer, + Float, + Str, + StrRaw(u16), + ByteStr, + ByteStrRaw(u16), + Err, +} + +rpc_encode_decode!( + enum LitKind { + Byte, + Char, + Integer, + Float, + Str, + StrRaw(n), + ByteStr, + ByteStrRaw(n), + Err, + } +); + macro_rules! mark_compound { (struct $name:ident <$($T:ident),+> { $($field:ident),* $(,)? }) => { impl<$($T: Mark),+> Mark for $name <$($T),+> { @@ -492,16 +499,26 @@ pub struct Ident { compound_traits!(struct Ident { sym, is_raw, span }); +#[derive(Clone, Eq, PartialEq)] +pub struct Literal { + pub kind: LitKind, + pub symbol: Sy, + pub suffix: Option, + pub span: Sp, +} + +compound_traits!(struct Literal { kind, symbol, suffix, span }); + #[derive(Clone)] -pub enum TokenTree { +pub enum TokenTree { Group(Group), Punct(Punct), Ident(Ident), - Literal(L), + Literal(Literal), } compound_traits!( - enum TokenTree { + enum TokenTree { Group(tt), Punct(tt), Ident(tt), diff --git a/library/proc_macro/src/bridge/rpc.rs b/library/proc_macro/src/bridge/rpc.rs index 1dca14ea5d8db..66512ac71e427 100644 --- a/library/proc_macro/src/bridge/rpc.rs +++ b/library/proc_macro/src/bridge/rpc.rs @@ -128,6 +128,7 @@ impl DecodeMut<'_, '_, S> for u8 { } } +rpc_encode_decode!(le u16); rpc_encode_decode!(le u32); rpc_encode_decode!(le usize); diff --git a/library/proc_macro/src/bridge/server.rs b/library/proc_macro/src/bridge/server.rs index bb4026db6f30a..6875fd0a0520c 100644 --- a/library/proc_macro/src/bridge/server.rs +++ b/library/proc_macro/src/bridge/server.rs @@ -14,8 +14,6 @@ macro_rules! associated_item { (type FreeFunctions: 'static;); (type TokenStream) => (type TokenStream: 'static + Clone;); - (type Literal) => - (type Literal: 'static + Clone;); (type SourceFile) => (type SourceFile: 'static + Clone;); (type MultiSpan) => diff --git a/library/proc_macro/src/lib.rs b/library/proc_macro/src/lib.rs index c6cd76c1cc936..fd275cb8cfb53 100644 --- a/library/proc_macro/src/lib.rs +++ b/library/proc_macro/src/lib.rs @@ -180,12 +180,7 @@ pub use quote::{quote, quote_span}; fn tree_to_bridge_tree( tree: TokenTree, -) -> bridge::TokenTree< - bridge::client::TokenStream, - bridge::client::Span, - bridge::client::Symbol, - bridge::client::Literal, -> { +) -> bridge::TokenTree { match tree { TokenTree::Group(tt) => bridge::TokenTree::Group(tt.0), TokenTree::Punct(tt) => bridge::TokenTree::Punct(tt.0), @@ -261,7 +256,6 @@ pub mod token_stream { bridge::client::TokenStream, bridge::client::Span, bridge::client::Symbol, - bridge::client::Literal, >, >, ); @@ -995,7 +989,7 @@ impl fmt::Debug for Ident { /// Boolean literals like `true` and `false` do not belong here, they are `Ident`s. #[derive(Clone)] #[stable(feature = "proc_macro_lib2", since = "1.29.0")] -pub struct Literal(bridge::client::Literal); +pub struct Literal(bridge::Literal); macro_rules! suffixed_int_literals { ($($name:ident => $kind:ident,)*) => ($( @@ -1012,7 +1006,12 @@ macro_rules! suffixed_int_literals { /// below. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn $name(n: $kind) -> Literal { - Literal(bridge::client::Literal::typed_integer(&n.to_string(), stringify!($kind))) + Literal(bridge::Literal { + kind: bridge::LitKind::Integer, + symbol: bridge::client::Symbol::new(&n.to_string()), + suffix: Some(bridge::client::Symbol::new(stringify!($kind))), + span: Span::call_site().0, + }) } )*) } @@ -1034,12 +1033,26 @@ macro_rules! unsuffixed_int_literals { /// below. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn $name(n: $kind) -> Literal { - Literal(bridge::client::Literal::integer(&n.to_string())) + Literal(bridge::Literal { + kind: bridge::LitKind::Integer, + symbol: bridge::client::Symbol::new(&n.to_string()), + suffix: None, + span: Span::call_site().0, + }) } )*) } impl Literal { + fn new(kind: bridge::LitKind, value: &str, suffix: Option<&str>) -> Self { + Literal(bridge::Literal { + kind, + symbol: bridge::client::Symbol::new(value), + suffix: suffix.map(bridge::client::Symbol::new), + span: Span::call_site().0, + }) + } + suffixed_int_literals! { u8_suffixed => u8, u16_suffixed => u16, @@ -1087,7 +1100,7 @@ impl Literal { if !n.is_finite() { panic!("Invalid float literal {}", n); } - Literal(bridge::client::Literal::float(&n.to_string())) + Literal::new(bridge::LitKind::Float, &n.to_string(), None) } /// Creates a new suffixed floating-point literal. @@ -1108,7 +1121,7 @@ impl Literal { if !n.is_finite() { panic!("Invalid float literal {}", n); } - Literal(bridge::client::Literal::f32(&n.to_string())) + Literal::new(bridge::LitKind::Float, &n.to_string(), Some("f32")) } /// Creates a new unsuffixed floating-point literal. @@ -1128,7 +1141,7 @@ impl Literal { if !n.is_finite() { panic!("Invalid float literal {}", n); } - Literal(bridge::client::Literal::float(&n.to_string())) + Literal::new(bridge::LitKind::Float, &n.to_string(), None) } /// Creates a new suffixed floating-point literal. @@ -1149,37 +1162,49 @@ impl Literal { if !n.is_finite() { panic!("Invalid float literal {}", n); } - Literal(bridge::client::Literal::f64(&n.to_string())) + Literal::new(bridge::LitKind::Float, &n.to_string(), Some("f64")) } /// String literal. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn string(string: &str) -> Literal { - Literal(bridge::client::Literal::string(string)) + let mut escaped = String::new(); + for ch in string.chars() { + escaped.extend(ch.escape_debug()); + } + Literal::new(bridge::LitKind::Str, &escaped, None) } /// Character literal. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn character(ch: char) -> Literal { - Literal(bridge::client::Literal::character(ch)) + let mut escaped = String::new(); + escaped.extend(ch.escape_unicode()); + Literal::new(bridge::LitKind::Char, &escaped, None) } /// Byte string literal. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn byte_string(bytes: &[u8]) -> Literal { - Literal(bridge::client::Literal::byte_string(bytes)) + let string = bytes + .iter() + .cloned() + .flat_map(std::ascii::escape_default) + .map(Into::::into) + .collect::(); + Literal::new(bridge::LitKind::ByteStr, &string, None) } /// Returns the span encompassing this literal. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn span(&self) -> Span { - Span(self.0.span()) + Span(self.0.span) } /// Configures the span associated for this literal. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn set_span(&mut self, span: Span) { - self.0.set_span(span.0); + self.0.span = span.0; } /// Returns a `Span` that is a subset of `self.span()` containing only the @@ -1195,7 +1220,12 @@ impl Literal { // was 'c' or whether it was '\u{63}'. #[unstable(feature = "proc_macro_span", issue = "54725")] pub fn subspan>(&self, range: R) -> Option { - self.0.subspan(range.start_bound().cloned(), range.end_bound().cloned()).map(Span) + bridge::client::FreeFunctions::literal_subspan( + self.0.clone(), + range.start_bound().cloned(), + range.end_bound().cloned(), + ) + .map(Span) } } @@ -1214,35 +1244,56 @@ impl FromStr for Literal { type Err = LexError; fn from_str(src: &str) -> Result { - match bridge::client::Literal::from_str(src) { + match bridge::client::FreeFunctions::literal_from_str(src) { Ok(literal) => Ok(Literal(literal)), Err(()) => Err(LexError::new()), } } } -// N.B., the bridge only provides `to_string`, implement `fmt::Display` -// based on it (the reverse of the usual relationship between the two). -#[stable(feature = "proc_macro_lib", since = "1.15.0")] -impl ToString for Literal { - fn to_string(&self) -> String { - TokenStream::from(TokenTree::from(self.clone())).to_string() - } -} - /// Prints the literal as a string that should be losslessly convertible /// back into the same literal (except for possible rounding for floating point literals). #[stable(feature = "proc_macro_lib2", since = "1.29.0")] impl fmt::Display for Literal { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(&self.to_string()) + // Based on `literal_to_string` from `pprust/state.rs` + match self.0.kind { + bridge::LitKind::Byte => write!(f, "b'{}'", self.0.symbol)?, + bridge::LitKind::Char => write!(f, "'{}'", self.0.symbol)?, + bridge::LitKind::Str => write!(f, "\"{}\"", self.0.symbol)?, + bridge::LitKind::StrRaw(n) => write!( + f, + "r{delim}\"{string}\"{delim}", + delim = "#".repeat(n as usize), + string = self.0.symbol + )?, + bridge::LitKind::ByteStr => write!(f, "b\"{}\"", self.0.symbol)?, + bridge::LitKind::ByteStrRaw(n) => write!( + f, + "br{delim}\"{string}\"{delim}", + delim = "#".repeat(n as usize), + string = self.0.symbol + )?, + _ => write!(f, "{}", self.0.symbol)?, + } + if let Some(suffix) = self.0.suffix { + write!(f, "{}", suffix)?; + } + Ok(()) } } #[stable(feature = "proc_macro_lib2", since = "1.29.0")] impl fmt::Debug for Literal { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.0.fmt(f) + f.debug_struct("Literal") + // format the kind without quotes, as in `kind: Float` + .field("kind", &self.0.kind) + .field("symbol", &self.0.symbol) + // format `Some("...")` on one line even in {:#?} mode + .field("suffix", &format_args!("{:?}", &self.0.suffix)) + .field("span", &self.0.span) + .finish() } }