From 7678e6ad8560cb559e1ca0b37e362890c25fe92b Mon Sep 17 00:00:00 2001 From: Nika Layzell Date: Mon, 28 Jun 2021 19:43:40 -0400 Subject: [PATCH 01/11] proc_macro: support encoding/decoding structs with type parameters --- library/proc_macro/src/bridge/rpc.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/library/proc_macro/src/bridge/rpc.rs b/library/proc_macro/src/bridge/rpc.rs index f79e016400fb1..e19fcf68c5c0c 100644 --- a/library/proc_macro/src/bridge/rpc.rs +++ b/library/proc_macro/src/bridge/rpc.rs @@ -43,15 +43,17 @@ macro_rules! rpc_encode_decode { } } }; - (struct $name:ident { $($field:ident),* $(,)? }) => { - impl Encode for $name { + (struct $name:ident $(<$($T:ident),+>)? { $($field:ident),* $(,)? }) => { + impl),+)?> Encode for $name $(<$($T),+>)? { fn encode(self, w: &mut Writer, s: &mut S) { $(self.$field.encode(w, s);)* } } - impl DecodeMut<'_, '_, S> for $name { - fn decode(r: &mut Reader<'_>, s: &mut S) -> Self { + impl<'a, S, $($($T: for<'s> DecodeMut<'a, 's, S>),+)?> DecodeMut<'a, '_, S> + for $name $(<$($T),+>)? + { + fn decode(r: &mut Reader<'a>, s: &mut S) -> Self { $name { $($field: DecodeMut::decode(r, s)),* } From 1793ee06589193a33f7f3d6670928dcb5a0f4742 Mon Sep 17 00:00:00 2001 From: Nika Layzell Date: Thu, 1 Jul 2021 12:56:07 -0400 Subject: [PATCH 02/11] proc_macro: support encoding/decoding Vec --- library/proc_macro/src/bridge/mod.rs | 15 +++++++++++++++ library/proc_macro/src/bridge/rpc.rs | 20 ++++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/library/proc_macro/src/bridge/mod.rs b/library/proc_macro/src/bridge/mod.rs index 4d3e89ba09356..aca43945491fa 100644 --- a/library/proc_macro/src/bridge/mod.rs +++ b/library/proc_macro/src/bridge/mod.rs @@ -337,6 +337,21 @@ impl Unmark for Result { } } +impl Mark for Vec { + type Unmarked = Vec; + fn mark(unmarked: Self::Unmarked) -> Self { + // Should be a no-op due to std's in-place collect optimizations. + unmarked.into_iter().map(T::mark).collect() + } +} +impl Unmark for Vec { + type Unmarked = Vec; + fn unmark(self) -> Self::Unmarked { + // Should be a no-op due to std's in-place collect optimizations. + self.into_iter().map(T::unmark).collect() + } +} + macro_rules! mark_noop { ($($ty:ty),* $(,)?) => { $( diff --git a/library/proc_macro/src/bridge/rpc.rs b/library/proc_macro/src/bridge/rpc.rs index e19fcf68c5c0c..a94334e07362e 100644 --- a/library/proc_macro/src/bridge/rpc.rs +++ b/library/proc_macro/src/bridge/rpc.rs @@ -248,6 +248,26 @@ impl DecodeMut<'_, '_, S> for String { } } +impl> Encode for Vec { + fn encode(self, w: &mut Writer, s: &mut S) { + self.len().encode(w, s); + for x in self { + x.encode(w, s); + } + } +} + +impl<'a, S, T: for<'s> DecodeMut<'a, 's, S>> DecodeMut<'a, '_, S> for Vec { + fn decode(r: &mut Reader<'a>, s: &mut S) -> Self { + let len = usize::decode(r, s); + let mut vec = Vec::with_capacity(len); + for _ in 0..len { + vec.push(T::decode(r, s)); + } + vec + } +} + /// Simplified version of panic payloads, ignoring /// types other than `&'static str` and `String`. pub enum PanicMessage { From cc698dce1575b76a906091e4dc4647304595dec9 Mon Sep 17 00:00:00 2001 From: Nika Layzell Date: Tue, 29 Jun 2021 16:36:10 -0400 Subject: [PATCH 03/11] proc_macro: use macros to simplify aggregate Mark/Unmark definitions --- library/proc_macro/src/bridge/mod.rs | 77 +++++++++++++++++++--------- 1 file changed, 53 insertions(+), 24 deletions(-) diff --git a/library/proc_macro/src/bridge/mod.rs b/library/proc_macro/src/bridge/mod.rs index aca43945491fa..82f12eacada23 100644 --- a/library/proc_macro/src/bridge/mod.rs +++ b/library/proc_macro/src/bridge/mod.rs @@ -409,6 +409,58 @@ rpc_encode_decode!( } ); +macro_rules! mark_compound { + (struct $name:ident <$($T:ident),+> { $($field:ident),* $(,)? }) => { + impl<$($T: Mark),+> Mark for $name <$($T),+> { + type Unmarked = $name <$($T::Unmarked),+>; + fn mark(unmarked: Self::Unmarked) -> Self { + $name { + $($field: Mark::mark(unmarked.$field)),* + } + } + } + + impl<$($T: Unmark),+> Unmark for $name <$($T),+> { + type Unmarked = $name <$($T::Unmarked),+>; + fn unmark(self) -> Self::Unmarked { + $name { + $($field: Unmark::unmark(self.$field)),* + } + } + } + }; + (enum $name:ident <$($T:ident),+> { $($variant:ident $(($field:ident))?),* $(,)? }) => { + impl<$($T: Mark),+> Mark for $name <$($T),+> { + type Unmarked = $name <$($T::Unmarked),+>; + fn mark(unmarked: Self::Unmarked) -> Self { + match unmarked { + $($name::$variant $(($field))? => { + $name::$variant $((Mark::mark($field)))? + })* + } + } + } + + impl<$($T: Unmark),+> Unmark for $name <$($T),+> { + type Unmarked = $name <$($T::Unmarked),+>; + fn unmark(self) -> Self::Unmarked { + match self { + $($name::$variant $(($field))? => { + $name::$variant $((Unmark::unmark($field)))? + })* + } + } + } + } +} + +macro_rules! compound_traits { + ($($t:tt)*) => { + rpc_encode_decode!($($t)*); + mark_compound!($($t)*); + }; +} + #[derive(Clone)] pub enum TokenTree { Group(G), @@ -417,30 +469,7 @@ pub enum TokenTree { Literal(L), } -impl Mark for TokenTree { - type Unmarked = TokenTree; - fn mark(unmarked: Self::Unmarked) -> Self { - match unmarked { - TokenTree::Group(tt) => TokenTree::Group(G::mark(tt)), - TokenTree::Punct(tt) => TokenTree::Punct(P::mark(tt)), - TokenTree::Ident(tt) => TokenTree::Ident(I::mark(tt)), - TokenTree::Literal(tt) => TokenTree::Literal(L::mark(tt)), - } - } -} -impl Unmark for TokenTree { - type Unmarked = TokenTree; - fn unmark(self) -> Self::Unmarked { - match self { - TokenTree::Group(tt) => TokenTree::Group(tt.unmark()), - TokenTree::Punct(tt) => TokenTree::Punct(tt.unmark()), - TokenTree::Ident(tt) => TokenTree::Ident(tt.unmark()), - TokenTree::Literal(tt) => TokenTree::Literal(tt.unmark()), - } - } -} - -rpc_encode_decode!( +compound_traits!( enum TokenTree { Group(tt), Punct(tt), From 8315ce998db369e62724c316dcb08f1cd6bf7232 Mon Sep 17 00:00:00 2001 From: Nika Layzell Date: Tue, 29 Jun 2021 14:49:54 -0400 Subject: [PATCH 04/11] proc_macro: cache static spans in client's thread-local state This greatly improves the performance of the very frequently called `call_site()` macro when running in a cross-thread configuration. --- .../rustc_expand/src/proc_macro_server.rs | 30 ++-- library/proc_macro/src/bridge/client.rs | 140 +++++++++++------- library/proc_macro/src/bridge/mod.rs | 31 ++-- .../proc_macro/src/bridge/selfless_reify.rs | 11 +- library/proc_macro/src/bridge/server.rs | 53 +++++-- library/proc_macro/src/lib.rs | 2 +- 6 files changed, 169 insertions(+), 98 deletions(-) diff --git a/compiler/rustc_expand/src/proc_macro_server.rs b/compiler/rustc_expand/src/proc_macro_server.rs index d4407c03d03f5..0de8f2edd31ea 100644 --- a/compiler/rustc_expand/src/proc_macro_server.rs +++ b/compiler/rustc_expand/src/proc_macro_server.rs @@ -375,7 +375,10 @@ impl<'a, 'b> Rustc<'a, 'b> { } fn lit(&mut self, kind: token::LitKind, symbol: Symbol, suffix: Option) -> Literal { - Literal { lit: token::Lit::new(kind, symbol, suffix), span: server::Span::call_site(self) } + Literal { + lit: token::Lit::new(kind, symbol, suffix), + span: server::Context::call_site(self), + } } } @@ -528,7 +531,7 @@ impl server::Group for Rustc<'_, '_> { Group { delimiter, stream, - span: DelimSpan::from_single(server::Span::call_site(self)), + span: DelimSpan::from_single(server::Context::call_site(self)), flatten: false, } } @@ -554,7 +557,7 @@ impl server::Group for Rustc<'_, '_> { impl server::Punct for Rustc<'_, '_> { fn new(&mut self, ch: char, spacing: Spacing) -> Self::Punct { - Punct::new(ch, spacing == Spacing::Joint, server::Span::call_site(self)) + Punct::new(ch, spacing == Spacing::Joint, server::Context::call_site(self)) } fn as_char(&mut self, punct: Self::Punct) -> char { punct.ch @@ -777,15 +780,6 @@ impl server::Span for Rustc<'_, '_> { format!("{:?} bytes({}..{})", span.ctxt(), span.lo().0, span.hi().0) } } - fn def_site(&mut self) -> Self::Span { - self.def_site - } - fn call_site(&mut self) -> Self::Span { - self.call_site - } - fn mixed_site(&mut self) -> Self::Span { - self.mixed_site - } fn source_file(&mut self, span: Self::Span) -> Self::SourceFile { self.sess().source_map().lookup_char_pos(span.lo()).file } @@ -861,3 +855,15 @@ impl server::Span for Rustc<'_, '_> { }) } } + +impl server::Context for Rustc<'_, '_> { + fn def_site(&mut self) -> Self::Span { + self.def_site + } + fn call_site(&mut self) -> Self::Span { + self.call_site + } + fn mixed_site(&mut self) -> Self::Span { + self.mixed_site + } +} diff --git a/library/proc_macro/src/bridge/client.rs b/library/proc_macro/src/bridge/client.rs index c38457ac6712d..a605d2eb4707d 100644 --- a/library/proc_macro/src/bridge/client.rs +++ b/library/proc_macro/src/bridge/client.rs @@ -241,6 +241,20 @@ impl Clone for SourceFile { } } +impl Span { + pub(crate) fn def_site() -> Span { + Bridge::with(|bridge| bridge.context.def_site) + } + + pub(crate) fn call_site() -> Span { + Bridge::with(|bridge| bridge.context.call_site) + } + + pub(crate) fn mixed_site() -> Span { + Bridge::with(|bridge| bridge.context.mixed_site) + } +} + impl fmt::Debug for Span { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.write_str(&self.debug()) @@ -274,6 +288,21 @@ macro_rules! define_client_side { } with_api!(self, self, define_client_side); +struct Bridge<'a> { + /// Reusable buffer (only `clear`-ed, never shrunk), primarily + /// used for making requests. + cached_buffer: Buffer, + + /// Server-side function that the client uses to make requests. + dispatch: closure::Closure<'a, Buffer, Buffer>, + + /// Provided context for this macro expansion. + context: ExpnContext, +} + +impl<'a> !Send for Bridge<'a> {} +impl<'a> !Sync for Bridge<'a> {} + enum BridgeState<'a> { /// No server is currently connected to this client. NotConnected, @@ -316,34 +345,6 @@ impl BridgeState<'_> { } impl Bridge<'_> { - pub(crate) fn is_available() -> bool { - BridgeState::with(|state| match state { - BridgeState::Connected(_) | BridgeState::InUse => true, - BridgeState::NotConnected => false, - }) - } - - fn enter(self, f: impl FnOnce() -> R) -> R { - let force_show_panics = self.force_show_panics; - // Hide the default panic output within `proc_macro` expansions. - // NB. the server can't do this because it may use a different libstd. - static HIDE_PANICS_DURING_EXPANSION: Once = Once::new(); - HIDE_PANICS_DURING_EXPANSION.call_once(|| { - let prev = panic::take_hook(); - panic::set_hook(Box::new(move |info| { - let show = BridgeState::with(|state| match state { - BridgeState::NotConnected => true, - BridgeState::Connected(_) | BridgeState::InUse => force_show_panics, - }); - if show { - prev(info) - } - })); - }); - - BRIDGE_STATE.with(|state| state.set(BridgeState::Connected(self), f)) - } - fn with(f: impl FnOnce(&mut Bridge<'_>) -> R) -> R { BridgeState::with(|state| match state { BridgeState::NotConnected => { @@ -357,6 +358,13 @@ impl Bridge<'_> { } } +pub(crate) fn is_available() -> bool { + BridgeState::with(|state| match state { + BridgeState::Connected(_) | BridgeState::InUse => true, + BridgeState::NotConnected => false, + }) +} + /// A client-side RPC entry-point, which may be using a different `proc_macro` /// from the one used by the server, but can be invoked compatibly. /// @@ -374,7 +382,7 @@ pub struct Client { // a wrapper `fn` pointer, once `const fn` can reference `static`s. pub(super) get_handle_counters: extern "C" fn() -> &'static HandleCounters, - pub(super) run: extern "C" fn(Bridge<'_>) -> Buffer, + pub(super) run: extern "C" fn(BridgeConfig<'_>) -> Buffer, pub(super) _marker: PhantomData O>, } @@ -386,40 +394,62 @@ impl Clone for Client { } } +fn maybe_install_panic_hook(force_show_panics: bool) { + // Hide the default panic output within `proc_macro` expansions. + // NB. the server can't do this because it may use a different libstd. + static HIDE_PANICS_DURING_EXPANSION: Once = Once::new(); + HIDE_PANICS_DURING_EXPANSION.call_once(|| { + let prev = panic::take_hook(); + panic::set_hook(Box::new(move |info| { + let show = BridgeState::with(|state| match state { + BridgeState::NotConnected => true, + BridgeState::Connected(_) | BridgeState::InUse => force_show_panics, + }); + if show { + prev(info) + } + })); + }); +} + /// Client-side helper for handling client panics, entering the bridge, /// deserializing input and serializing output. // FIXME(eddyb) maybe replace `Bridge::enter` with this? fn run_client DecodeMut<'a, 's, ()>, R: Encode<()>>( - mut bridge: Bridge<'_>, + config: BridgeConfig<'_>, f: impl FnOnce(A) -> R, ) -> Buffer { - // The initial `cached_buffer` contains the input. - let mut buf = bridge.cached_buffer.take(); + let BridgeConfig { input: mut buf, dispatch, force_show_panics, .. } = config; panic::catch_unwind(panic::AssertUnwindSafe(|| { - bridge.enter(|| { - let reader = &mut &buf[..]; - let input = A::decode(reader, &mut ()); - - // Put the `cached_buffer` back in the `Bridge`, for requests. - Bridge::with(|bridge| bridge.cached_buffer = buf.take()); - - let output = f(input); - - // Take the `cached_buffer` back out, for the output value. - buf = Bridge::with(|bridge| bridge.cached_buffer.take()); - - // HACK(eddyb) Separate encoding a success value (`Ok(output)`) - // from encoding a panic (`Err(e: PanicMessage)`) to avoid - // having handles outside the `bridge.enter(|| ...)` scope, and - // to catch panics that could happen while encoding the success. - // - // Note that panics should be impossible beyond this point, but - // this is defensively trying to avoid any accidental panicking - // reaching the `extern "C"` (which should `abort` but might not - // at the moment, so this is also potentially preventing UB). - buf.clear(); - Ok::<_, ()>(output).encode(&mut buf, &mut ()); + maybe_install_panic_hook(force_show_panics); + + let reader = &mut &buf[..]; + let (input, context) = <(A, ExpnContext)>::decode(reader, &mut ()); + + // Put the buffer we used for input back in the `Bridge` for requests. + let new_state = + BridgeState::Connected(Bridge { cached_buffer: buf.take(), dispatch, context }); + + BRIDGE_STATE.with(|state| { + state.set(new_state, || { + let output = f(input); + + // Take the `cached_buffer` back out, for the output value. + buf = Bridge::with(|bridge| bridge.cached_buffer.take()); + + // HACK(eddyb) Separate encoding a success value (`Ok(output)`) + // from encoding a panic (`Err(e: PanicMessage)`) to avoid + // having handles outside the `bridge.enter(|| ...)` scope, and + // to catch panics that could happen while encoding the success. + // + // Note that panics should be impossible beyond this point, but + // this is defensively trying to avoid any accidental panicking + // reaching the `extern "C"` (which should `abort` but might not + // at the moment, so this is also potentially preventing UB). + buf.clear(); + Ok::<_, ()>(output).encode(&mut buf, &mut ()); + }) }) })) .map_err(PanicMessage::from) diff --git a/library/proc_macro/src/bridge/mod.rs b/library/proc_macro/src/bridge/mod.rs index 82f12eacada23..984afa7b44a61 100644 --- a/library/proc_macro/src/bridge/mod.rs +++ b/library/proc_macro/src/bridge/mod.rs @@ -155,9 +155,6 @@ macro_rules! with_api { }, Span { fn debug($self: $S::Span) -> String; - fn def_site() -> $S::Span; - fn call_site() -> $S::Span; - fn mixed_site() -> $S::Span; fn source_file($self: $S::Span) -> $S::SourceFile; fn parent($self: $S::Span) -> Option<$S::Span>; fn source($self: $S::Span) -> $S::Span; @@ -217,16 +214,15 @@ use buffer::Buffer; pub use rpc::PanicMessage; use rpc::{Decode, DecodeMut, Encode, Reader, Writer}; -/// An active connection between a server and a client. -/// The server creates the bridge (`Bridge::run_server` in `server.rs`), -/// then passes it to the client through the function pointer in the `run` -/// field of `client::Client`. The client holds its copy of the `Bridge` +/// Configuration for establishing an active connection between a server and a +/// client. The server creates the bridge config (`run_server` in `server.rs`), +/// then passes it to the client through the function pointer in the `run` field +/// of `client::Client`. The client constructs a local `Bridge` from the config /// in TLS during its execution (`Bridge::{enter, with}` in `client.rs`). #[repr(C)] -pub struct Bridge<'a> { - /// Reusable buffer (only `clear`-ed, never shrunk), primarily - /// used for making requests, but also for passing input to client. - cached_buffer: Buffer, +pub struct BridgeConfig<'a> { + /// Buffer used to pass initial input to the client. + input: Buffer, /// Server-side function that the client uses to make requests. dispatch: closure::Closure<'a, Buffer, Buffer>, @@ -477,3 +473,16 @@ compound_traits!( Literal(tt), } ); + +/// Context provided alongside the initial inputs for a macro expansion. +/// Provides values such as spans which are used frequently to avoid RPC. +#[derive(Clone)] +struct ExpnContext { + def_site: S, + call_site: S, + mixed_site: S, +} + +compound_traits!( + struct ExpnContext { def_site, call_site, mixed_site } +); diff --git a/library/proc_macro/src/bridge/selfless_reify.rs b/library/proc_macro/src/bridge/selfless_reify.rs index 4ee4bb87c2bbd..907ad256e4b43 100644 --- a/library/proc_macro/src/bridge/selfless_reify.rs +++ b/library/proc_macro/src/bridge/selfless_reify.rs @@ -75,9 +75,10 @@ macro_rules! define_reify_functions { define_reify_functions! { fn _reify_to_extern_c_fn_unary for extern "C" fn(arg: A) -> R; - // HACK(eddyb) this abstraction is used with `for<'a> fn(Bridge<'a>) -> T` - // but that doesn't work with just `reify_to_extern_c_fn_unary` because of - // the `fn` pointer type being "higher-ranked" (i.e. the `for<'a>` binder). - // FIXME(eddyb) try to remove the lifetime from `Bridge`, that'd help. - fn reify_to_extern_c_fn_hrt_bridge for extern "C" fn(bridge: super::Bridge<'_>) -> R; + // HACK(eddyb) this abstraction is used with `for<'a> fn(BridgeConfig<'a>) + // -> T` but that doesn't work with just `reify_to_extern_c_fn_unary` + // because of the `fn` pointer type being "higher-ranked" (i.e. the + // `for<'a>` binder). + // FIXME(eddyb) try to remove the lifetime from `BridgeConfig`, that'd help. + fn reify_to_extern_c_fn_hrt_bridge for extern "C" fn(bridge: super::BridgeConfig<'_>) -> R; } diff --git a/library/proc_macro/src/bridge/server.rs b/library/proc_macro/src/bridge/server.rs index cbddf39da44d2..99c8a9f6845a1 100644 --- a/library/proc_macro/src/bridge/server.rs +++ b/library/proc_macro/src/bridge/server.rs @@ -32,6 +32,13 @@ macro_rules! associated_fn { ($($item:tt)*) => ($($item)*;) } +/// Helper methods defined by `Server` types not invoked over RPC. +pub trait Context: Types { + fn def_site(&mut self) -> Self::Span; + fn call_site(&mut self) -> Self::Span; + fn mixed_site(&mut self) -> Self::Span; +} + macro_rules! declare_server_traits { ($($name:ident { $(fn $method:ident($($arg:ident: $arg_ty:ty),* $(,)?) $(-> $ret_ty:ty)?;)* @@ -40,14 +47,26 @@ macro_rules! declare_server_traits { $(associated_fn!(fn $method(&mut self, $($arg: $arg_ty),*) $(-> $ret_ty)?);)* })* - pub trait Server: Types $(+ $name)* {} - impl Server for S {} + pub trait Server: Types + Context $(+ $name)* {} + impl Server for S {} } } with_api!(Self, self_, declare_server_traits); pub(super) struct MarkedTypes(S); +impl Context for MarkedTypes { + fn def_site(&mut self) -> Self::Span { + <_>::mark(Context::def_site(&mut self.0)) + } + fn call_site(&mut self) -> Self::Span { + <_>::mark(Context::call_site(&mut self.0)) + } + fn mixed_site(&mut self) -> Self::Span { + <_>::mark(Context::mixed_site(&mut self.0)) + } +} + macro_rules! define_mark_types_impls { ($($name:ident { $(fn $method:ident($($arg:ident: $arg_ty:ty),* $(,)?) $(-> $ret_ty:ty)?;)* @@ -122,7 +141,7 @@ pub trait ExecutionStrategy { &self, dispatcher: &mut impl DispatcherTrait, input: Buffer, - run_client: extern "C" fn(Bridge<'_>) -> Buffer, + run_client: extern "C" fn(BridgeConfig<'_>) -> Buffer, force_show_panics: bool, ) -> Buffer; } @@ -134,13 +153,13 @@ impl ExecutionStrategy for SameThread { &self, dispatcher: &mut impl DispatcherTrait, input: Buffer, - run_client: extern "C" fn(Bridge<'_>) -> Buffer, + run_client: extern "C" fn(BridgeConfig<'_>) -> Buffer, force_show_panics: bool, ) -> Buffer { let mut dispatch = |buf| dispatcher.dispatch(buf); - run_client(Bridge { - cached_buffer: input, + run_client(BridgeConfig { + input, dispatch: (&mut dispatch).into(), force_show_panics, _marker: marker::PhantomData, @@ -158,7 +177,7 @@ impl ExecutionStrategy for CrossThread1 { &self, dispatcher: &mut impl DispatcherTrait, input: Buffer, - run_client: extern "C" fn(Bridge<'_>) -> Buffer, + run_client: extern "C" fn(BridgeConfig<'_>) -> Buffer, force_show_panics: bool, ) -> Buffer { use std::sync::mpsc::channel; @@ -172,8 +191,8 @@ impl ExecutionStrategy for CrossThread1 { res_rx.recv().unwrap() }; - run_client(Bridge { - cached_buffer: input, + run_client(BridgeConfig { + input, dispatch: (&mut dispatch).into(), force_show_panics, _marker: marker::PhantomData, @@ -195,7 +214,7 @@ impl ExecutionStrategy for CrossThread2 { &self, dispatcher: &mut impl DispatcherTrait, input: Buffer, - run_client: extern "C" fn(Bridge<'_>) -> Buffer, + run_client: extern "C" fn(BridgeConfig<'_>) -> Buffer, force_show_panics: bool, ) -> Buffer { use std::sync::{Arc, Mutex}; @@ -221,8 +240,8 @@ impl ExecutionStrategy for CrossThread2 { } }; - let r = run_client(Bridge { - cached_buffer: input, + let r = run_client(BridgeConfig { + input, dispatch: (&mut dispatch).into(), force_show_panics, _marker: marker::PhantomData, @@ -260,14 +279,20 @@ fn run_server< handle_counters: &'static client::HandleCounters, server: S, input: I, - run_client: extern "C" fn(Bridge<'_>) -> Buffer, + run_client: extern "C" fn(BridgeConfig<'_>) -> Buffer, force_show_panics: bool, ) -> Result { let mut dispatcher = Dispatcher { handle_store: HandleStore::new(handle_counters), server: MarkedTypes(server) }; + let expn_context = ExpnContext { + def_site: dispatcher.server.def_site(), + call_site: dispatcher.server.call_site(), + mixed_site: dispatcher.server.mixed_site(), + }; + let mut buf = Buffer::new(); - input.encode(&mut buf, &mut dispatcher.handle_store); + (input, expn_context).encode(&mut buf, &mut dispatcher.handle_store); buf = strategy.run_bridge_and_client(&mut dispatcher, buf, run_client, force_show_panics); diff --git a/library/proc_macro/src/lib.rs b/library/proc_macro/src/lib.rs index 30ad3d2388082..d03f3dae1c200 100644 --- a/library/proc_macro/src/lib.rs +++ b/library/proc_macro/src/lib.rs @@ -60,7 +60,7 @@ use std::{error, fmt, iter, mem}; /// inside of a procedural macro, false if invoked from any other binary. #[stable(feature = "proc_macro_is_available", since = "1.57.0")] pub fn is_available() -> bool { - bridge::Bridge::is_available() + bridge::client::is_available() } /// The main type provided by this crate, representing an abstract stream of From 29d8b9c6a4f9ff3288c0559f3be5bea45bc121c7 Mon Sep 17 00:00:00 2001 From: Nika Layzell Date: Thu, 1 Jul 2021 15:03:51 -0400 Subject: [PATCH 05/11] proc_macro: reduce the number of messages required to create, extend, and iterate TokenStreams This significantly reduces the cost of common interactions with TokenStream when running with the CrossThread execution strategy, by reducing the number of RPC calls required. --- .../rustc_expand/src/proc_macro_server.rs | 98 ++++++++++--------- library/proc_macro/src/bridge/client.rs | 18 ++-- library/proc_macro/src/bridge/mod.rs | 28 +++--- library/proc_macro/src/bridge/server.rs | 2 - library/proc_macro/src/lib.rs | 78 ++++++++++----- 5 files changed, 128 insertions(+), 96 deletions(-) diff --git a/compiler/rustc_expand/src/proc_macro_server.rs b/compiler/rustc_expand/src/proc_macro_server.rs index 0de8f2edd31ea..f214ee6ae2da7 100644 --- a/compiler/rustc_expand/src/proc_macro_server.rs +++ b/compiler/rustc_expand/src/proc_macro_server.rs @@ -277,12 +277,6 @@ impl ToInternal for Level { pub struct FreeFunctions; -#[derive(Clone)] -pub struct TokenStreamIter { - cursor: tokenstream::Cursor, - stack: Vec>, -} - #[derive(Clone)] pub struct Group { delimiter: Delimiter, @@ -385,8 +379,6 @@ impl<'a, 'b> Rustc<'a, 'b> { impl server::Types for Rustc<'_, '_> { type FreeFunctions = FreeFunctions; type TokenStream = TokenStream; - type TokenStreamBuilder = tokenstream::TokenStreamBuilder; - type TokenStreamIter = TokenStreamIter; type Group = Group; type Punct = Punct; type Ident = Ident; @@ -411,9 +403,6 @@ impl server::FreeFunctions for Rustc<'_, '_> { } impl server::TokenStream for Rustc<'_, '_> { - fn new(&mut self) -> Self::TokenStream { - TokenStream::default() - } fn is_empty(&mut self, stream: &Self::TokenStream) -> bool { stream.is_empty() } @@ -484,53 +473,74 @@ impl server::TokenStream for Rustc<'_, '_> { ) -> Self::TokenStream { tree.to_internal() } - fn into_iter(&mut self, stream: Self::TokenStream) -> Self::TokenStreamIter { - TokenStreamIter { cursor: stream.into_trees(), stack: vec![] } - } -} - -impl server::TokenStreamBuilder for Rustc<'_, '_> { - fn new(&mut self) -> Self::TokenStreamBuilder { - tokenstream::TokenStreamBuilder::new() - } - fn push(&mut self, builder: &mut Self::TokenStreamBuilder, stream: Self::TokenStream) { - builder.push(stream); + fn concat_trees( + &mut self, + base: Option, + trees: Vec>, + ) -> Self::TokenStream { + let mut builder = tokenstream::TokenStreamBuilder::new(); + if let Some(base) = base { + builder.push(base); + } + for tree in trees { + builder.push(tree.to_internal()); + } + builder.build() } - fn build(&mut self, builder: Self::TokenStreamBuilder) -> Self::TokenStream { + fn concat_streams( + &mut self, + base: Option, + streams: Vec, + ) -> Self::TokenStream { + let mut builder = tokenstream::TokenStreamBuilder::new(); + if let Some(base) = base { + builder.push(base); + } + for stream in streams { + builder.push(stream); + } builder.build() } -} - -impl server::TokenStreamIter for Rustc<'_, '_> { - fn next( + fn into_iter( &mut self, - iter: &mut Self::TokenStreamIter, - ) -> Option> { + stream: Self::TokenStream, + ) -> Vec> { + // XXX: This is a raw port of the previous approach, and can probably be + // optimized. + let mut cursor = stream.into_trees(); + let mut stack = Vec::new(); + let mut tts = Vec::new(); loop { - let tree = iter.stack.pop().or_else(|| { - let next = iter.cursor.next_with_spacing()?; - Some(TokenTree::from_internal((next, &mut iter.stack, self))) - })?; - // A hack used to pass AST fragments to attribute and derive macros - // as a single nonterminal token instead of a token stream. - // Such token needs to be "unwrapped" and not represented as a delimited group. - // FIXME: It needs to be removed, but there are some compatibility issues (see #73345). - if let TokenTree::Group(ref group) = tree { - if group.flatten { - iter.cursor.append(group.stream.clone()); - continue; + let next = stack.pop().or_else(|| { + let next = cursor.next_with_spacing()?; + Some(TokenTree::from_internal((next, &mut stack, self))) + }); + match next { + Some(TokenTree::Group(group)) => { + // A hack used to pass AST fragments to attribute and derive + // macros as a single nonterminal token instead of a token + // stream. Such token needs to be "unwrapped" and not + // represented as a delimited group. + // FIXME: It needs to be removed, but there are some + // compatibility issues (see #73345). + if group.flatten { + cursor.append(group.stream); + continue; + } + tts.push(TokenTree::Group(group)); } + Some(tt) => tts.push(tt), + None => return tts, } - return Some(tree); } } } impl server::Group for Rustc<'_, '_> { - fn new(&mut self, delimiter: Delimiter, stream: Self::TokenStream) -> Self::Group { + fn new(&mut self, delimiter: Delimiter, stream: Option) -> Self::Group { Group { delimiter, - stream, + stream: stream.unwrap_or_default(), span: DelimSpan::from_single(server::Context::call_site(self)), flatten: false, } diff --git a/library/proc_macro/src/bridge/client.rs b/library/proc_macro/src/bridge/client.rs index a605d2eb4707d..a0070f2f0e685 100644 --- a/library/proc_macro/src/bridge/client.rs +++ b/library/proc_macro/src/bridge/client.rs @@ -178,8 +178,6 @@ define_handles! { 'owned: FreeFunctions, TokenStream, - TokenStreamBuilder, - TokenStreamIter, Group, Literal, SourceFile, @@ -204,12 +202,6 @@ impl Clone for TokenStream { } } -impl Clone for TokenStreamIter { - fn clone(&self) -> Self { - self.clone() - } -} - impl Clone for Group { fn clone(&self) -> Self { self.clone() @@ -465,7 +457,11 @@ impl Client { Client { get_handle_counters: HandleCounters::get, run: super::selfless_reify::reify_to_extern_c_fn_hrt_bridge(move |bridge| { - run_client(bridge, |input| f(crate::TokenStream(input)).0) + run_client(bridge, |input| { + f(crate::TokenStream(Some(input))) + .0 + .unwrap_or_else(|| TokenStream::concat_streams(None, vec![])) + }) }), _marker: PhantomData, } @@ -480,7 +476,9 @@ impl Client<(crate::TokenStream, crate::TokenStream), crate::TokenStream> { get_handle_counters: HandleCounters::get, run: super::selfless_reify::reify_to_extern_c_fn_hrt_bridge(move |bridge| { run_client(bridge, |(input, input2)| { - f(crate::TokenStream(input), crate::TokenStream(input2)).0 + f(crate::TokenStream(Some(input)), crate::TokenStream(Some(input2))) + .0 + .unwrap_or_else(|| TokenStream::concat_streams(None, vec![])) }) }), _marker: PhantomData, diff --git a/library/proc_macro/src/bridge/mod.rs b/library/proc_macro/src/bridge/mod.rs index 984afa7b44a61..44d695b17d45f 100644 --- a/library/proc_macro/src/bridge/mod.rs +++ b/library/proc_macro/src/bridge/mod.rs @@ -60,7 +60,6 @@ macro_rules! with_api { TokenStream { fn drop($self: $S::TokenStream); fn clone($self: &$S::TokenStream) -> $S::TokenStream; - fn new() -> $S::TokenStream; fn is_empty($self: &$S::TokenStream) -> bool; fn expand_expr($self: &$S::TokenStream) -> Result<$S::TokenStream, ()>; fn from_str(src: &str) -> $S::TokenStream; @@ -68,25 +67,22 @@ macro_rules! with_api { fn from_token_tree( tree: TokenTree<$S::Group, $S::Punct, $S::Ident, $S::Literal>, ) -> $S::TokenStream; - fn into_iter($self: $S::TokenStream) -> $S::TokenStreamIter; - }, - TokenStreamBuilder { - fn drop($self: $S::TokenStreamBuilder); - fn new() -> $S::TokenStreamBuilder; - fn push($self: &mut $S::TokenStreamBuilder, stream: $S::TokenStream); - fn build($self: $S::TokenStreamBuilder) -> $S::TokenStream; - }, - TokenStreamIter { - fn drop($self: $S::TokenStreamIter); - fn clone($self: &$S::TokenStreamIter) -> $S::TokenStreamIter; - fn next( - $self: &mut $S::TokenStreamIter, - ) -> Option>; + fn concat_trees( + base: Option<$S::TokenStream>, + trees: Vec>, + ) -> $S::TokenStream; + fn concat_streams( + base: Option<$S::TokenStream>, + trees: Vec<$S::TokenStream>, + ) -> $S::TokenStream; + fn into_iter( + $self: $S::TokenStream + ) -> Vec>; }, Group { fn drop($self: $S::Group); fn clone($self: &$S::Group) -> $S::Group; - fn new(delimiter: Delimiter, stream: $S::TokenStream) -> $S::Group; + fn new(delimiter: Delimiter, stream: Option<$S::TokenStream>) -> $S::Group; fn delimiter($self: &$S::Group) -> Delimiter; fn stream($self: &$S::Group) -> $S::TokenStream; fn span($self: &$S::Group) -> $S::Span; diff --git a/library/proc_macro/src/bridge/server.rs b/library/proc_macro/src/bridge/server.rs index 99c8a9f6845a1..a04d1b0dd0fb3 100644 --- a/library/proc_macro/src/bridge/server.rs +++ b/library/proc_macro/src/bridge/server.rs @@ -8,8 +8,6 @@ use super::client::HandleStore; pub trait Types { type FreeFunctions: 'static; type TokenStream: 'static + Clone; - type TokenStreamBuilder: 'static; - type TokenStreamIter: 'static + Clone; type Group: 'static + Clone; type Punct: 'static + Copy + Eq + Hash; type Ident: 'static + Copy + Eq + Hash; diff --git a/library/proc_macro/src/lib.rs b/library/proc_macro/src/lib.rs index d03f3dae1c200..559c7ff4be517 100644 --- a/library/proc_macro/src/lib.rs +++ b/library/proc_macro/src/lib.rs @@ -43,7 +43,7 @@ use std::cmp::Ordering; use std::ops::RangeBounds; use std::path::PathBuf; use std::str::FromStr; -use std::{error, fmt, iter, mem}; +use std::{error, fmt, iter}; /// Determines whether proc_macro has been made accessible to the currently /// running program. @@ -72,7 +72,7 @@ pub fn is_available() -> bool { /// and `#[proc_macro_derive]` definitions. #[stable(feature = "proc_macro_lib", since = "1.15.0")] #[derive(Clone)] -pub struct TokenStream(bridge::client::TokenStream); +pub struct TokenStream(Option); #[stable(feature = "proc_macro_lib", since = "1.15.0")] impl !Send for TokenStream {} @@ -126,13 +126,13 @@ impl TokenStream { /// Returns an empty `TokenStream` containing no token trees. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn new() -> TokenStream { - TokenStream(bridge::client::TokenStream::new()) + TokenStream(None) } /// Checks if this `TokenStream` is empty. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn is_empty(&self) -> bool { - self.0.is_empty() + self.0.as_ref().map(|h| h.is_empty()).unwrap_or(true) } /// Parses this `TokenStream` as an expression and attempts to expand any @@ -147,8 +147,9 @@ impl TokenStream { /// considered errors, is unspecified and may change in the future. #[unstable(feature = "proc_macro_expand", issue = "90765")] pub fn expand_expr(&self) -> Result { - match bridge::client::TokenStream::expand_expr(&self.0) { - Ok(stream) => Ok(TokenStream(stream)), + let stream = self.0.as_ref().ok_or(ExpandError)?; + match bridge::client::TokenStream::expand_expr(stream) { + Ok(stream) => Ok(TokenStream(Some(stream))), Err(_) => Err(ExpandError), } } @@ -166,7 +167,7 @@ impl FromStr for TokenStream { type Err = LexError; fn from_str(src: &str) -> Result { - Ok(TokenStream(bridge::client::TokenStream::from_str(src))) + Ok(TokenStream(Some(bridge::client::TokenStream::from_str(src)))) } } @@ -175,7 +176,7 @@ impl FromStr for TokenStream { #[stable(feature = "proc_macro_lib", since = "1.15.0")] impl ToString for TokenStream { fn to_string(&self) -> String { - self.0.to_string() + self.0.as_ref().map(|t| t.to_string()).unwrap_or_default() } } @@ -208,16 +209,27 @@ impl Default for TokenStream { #[unstable(feature = "proc_macro_quote", issue = "54722")] pub use quote::{quote, quote_span}; +fn tree_to_bridge_tree( + tree: TokenTree, +) -> bridge::TokenTree< + bridge::client::Group, + bridge::client::Punct, + bridge::client::Ident, + bridge::client::Literal, +> { + match tree { + TokenTree::Group(tt) => bridge::TokenTree::Group(tt.0), + TokenTree::Punct(tt) => bridge::TokenTree::Punct(tt.0), + TokenTree::Ident(tt) => bridge::TokenTree::Ident(tt.0), + TokenTree::Literal(tt) => bridge::TokenTree::Literal(tt.0), + } +} + /// Creates a token stream containing a single token tree. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] impl From for TokenStream { fn from(tree: TokenTree) -> TokenStream { - TokenStream(bridge::client::TokenStream::from_token_tree(match tree { - TokenTree::Group(tt) => bridge::TokenTree::Group(tt.0), - TokenTree::Punct(tt) => bridge::TokenTree::Punct(tt.0), - TokenTree::Ident(tt) => bridge::TokenTree::Ident(tt.0), - TokenTree::Literal(tt) => bridge::TokenTree::Literal(tt.0), - })) + TokenStream(Some(bridge::client::TokenStream::from_token_tree(tree_to_bridge_tree(tree)))) } } @@ -225,7 +237,10 @@ impl From for TokenStream { #[stable(feature = "proc_macro_lib2", since = "1.29.0")] impl iter::FromIterator for TokenStream { fn from_iter>(trees: I) -> Self { - trees.into_iter().map(TokenStream::from).collect() + TokenStream(Some(bridge::client::TokenStream::concat_trees( + None, + trees.into_iter().map(tree_to_bridge_tree).collect(), + ))) } } @@ -234,24 +249,30 @@ impl iter::FromIterator for TokenStream { #[stable(feature = "proc_macro_lib", since = "1.15.0")] impl iter::FromIterator for TokenStream { fn from_iter>(streams: I) -> Self { - let mut builder = bridge::client::TokenStreamBuilder::new(); - streams.into_iter().for_each(|stream| builder.push(stream.0)); - TokenStream(builder.build()) + TokenStream(Some(bridge::client::TokenStream::concat_streams( + None, + streams.into_iter().filter_map(|stream| stream.0).collect(), + ))) } } #[stable(feature = "token_stream_extend", since = "1.30.0")] impl Extend for TokenStream { fn extend>(&mut self, trees: I) { - self.extend(trees.into_iter().map(TokenStream::from)); + *self = TokenStream(Some(bridge::client::TokenStream::concat_trees( + self.0.take(), + trees.into_iter().map(|tree| tree_to_bridge_tree(tree)).collect(), + ))); } } #[stable(feature = "token_stream_extend", since = "1.30.0")] impl Extend for TokenStream { fn extend>(&mut self, streams: I) { - // FIXME(eddyb) Use an optimized implementation if/when possible. - *self = iter::once(mem::replace(self, Self::new())).chain(streams).collect(); + *self = TokenStream(Some(bridge::client::TokenStream::concat_streams( + self.0.take(), + streams.into_iter().filter_map(|stream| stream.0).collect(), + ))); } } @@ -265,7 +286,16 @@ pub mod token_stream { /// and returns whole groups as token trees. #[derive(Clone)] #[stable(feature = "proc_macro_lib2", since = "1.29.0")] - pub struct IntoIter(bridge::client::TokenStreamIter); + pub struct IntoIter( + std::vec::IntoIter< + bridge::TokenTree< + bridge::client::Group, + bridge::client::Punct, + bridge::client::Ident, + bridge::client::Literal, + >, + >, + ); #[stable(feature = "proc_macro_lib2", since = "1.29.0")] impl Iterator for IntoIter { @@ -287,7 +317,7 @@ pub mod token_stream { type IntoIter = IntoIter; fn into_iter(self) -> IntoIter { - IntoIter(self.0.into_iter()) + IntoIter(self.0.map(|v| v.into_iter()).unwrap_or_default().into_iter()) } } } @@ -734,7 +764,7 @@ impl Group { /// returned above. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn stream(&self) -> TokenStream { - TokenStream(self.0.stream()) + TokenStream(Some(self.0.stream())) } /// Returns the span for the delimiters of this token stream, spanning the From 801c77bc31bdee53fe7f02e4040ff7318c4fe5f4 Mon Sep 17 00:00:00 2001 From: Nika Layzell Date: Mon, 28 Jun 2021 22:30:55 -0400 Subject: [PATCH 06/11] proc_macro: stop using a remote object handle for Punct This greatly reduces round-trips to fetch relevant extra information about the token in proc macro code, and avoids RPC messages to create Punct tokens. --- .../rustc_expand/src/proc_macro_server.rs | 72 +++++-------------- library/proc_macro/src/bridge/client.rs | 1 - library/proc_macro/src/bridge/mod.rs | 28 ++++---- library/proc_macro/src/bridge/server.rs | 1 - library/proc_macro/src/lib.rs | 34 +++++---- .../ui/proc-macro/invalid-punct-ident-1.rs | 2 +- 6 files changed, 48 insertions(+), 90 deletions(-) diff --git a/compiler/rustc_expand/src/proc_macro_server.rs b/compiler/rustc_expand/src/proc_macro_server.rs index f214ee6ae2da7..f078749742cdf 100644 --- a/compiler/rustc_expand/src/proc_macro_server.rs +++ b/compiler/rustc_expand/src/proc_macro_server.rs @@ -14,8 +14,8 @@ use rustc_span::def_id::CrateNum; use rustc_span::symbol::{self, kw, sym, Symbol}; use rustc_span::{BytePos, FileName, Pos, SourceFile, Span}; -use pm::bridge::{server, TokenTree}; -use pm::{Delimiter, Level, LineColumn, Spacing}; +use pm::bridge::{server, Punct, TokenTree}; +use pm::{Delimiter, Level, LineColumn}; use std::ops::Bound; use std::{ascii, panic}; @@ -50,7 +50,7 @@ impl ToInternal for Delimiter { } impl FromInternal<(TreeAndSpacing, &'_ mut Vec, &mut Rustc<'_, '_>)> - for TokenTree + for TokenTree { fn from_internal( ((tree, spacing), stack, rustc): (TreeAndSpacing, &mut Vec, &mut Rustc<'_, '_>), @@ -79,16 +79,16 @@ impl FromInternal<(TreeAndSpacing, &'_ mut Vec, &mut Rustc<'_, '_>)> } macro_rules! op { ($a:expr) => { - tt!(Punct::new($a, joint)) + tt!(Punct { ch: $a, joint }) }; ($a:expr, $b:expr) => {{ - stack.push(tt!(Punct::new($b, joint))); - tt!(Punct::new($a, true)) + stack.push(tt!(Punct { ch: $b, joint })); + tt!(Punct { ch: $a, joint: true }) }}; ($a:expr, $b:expr, $c:expr) => {{ - stack.push(tt!(Punct::new($c, joint))); - stack.push(tt!(Punct::new($b, true))); - tt!(Punct::new($a, true)) + stack.push(tt!(Punct { ch: $c, joint })); + stack.push(tt!(Punct { ch: $b, joint: true })); + tt!(Punct { ch: $a, joint: true }) }}; } @@ -146,7 +146,7 @@ impl FromInternal<(TreeAndSpacing, &'_ mut Vec, &mut Rustc<'_, '_>)> Lifetime(name) => { let ident = symbol::Ident::new(name, span).without_first_quote(); stack.push(tt!(Ident::new(rustc.sess(), ident.name, false))); - tt!(Punct::new('\'', true)) + tt!(Punct { ch: '\'', joint: true }) } Literal(lit) => tt!(Literal { lit }), DocComment(_, attr_style, data) => { @@ -169,9 +169,9 @@ impl FromInternal<(TreeAndSpacing, &'_ mut Vec, &mut Rustc<'_, '_>)> flatten: false, })); if attr_style == ast::AttrStyle::Inner { - stack.push(tt!(Punct::new('!', false))); + stack.push(tt!(Punct { ch: '!', joint: false })); } - tt!(Punct::new('#', false)) + tt!(Punct { ch: '#', joint: false }) } Interpolated(nt) if let NtIdent(ident, is_raw) = *nt => { @@ -192,7 +192,7 @@ impl FromInternal<(TreeAndSpacing, &'_ mut Vec, &mut Rustc<'_, '_>)> } } -impl ToInternal for TokenTree { +impl ToInternal for TokenTree { fn to_internal(self) -> TokenStream { use rustc_ast::token::*; @@ -288,27 +288,6 @@ pub struct Group { flatten: bool, } -#[derive(Copy, Clone, PartialEq, Eq, Hash)] -pub struct Punct { - ch: char, - // NB. not using `Spacing` here because it doesn't implement `Hash`. - joint: bool, - span: Span, -} - -impl Punct { - fn new(ch: char, joint: bool, span: Span) -> Punct { - const LEGAL_CHARS: &[char] = &[ - '=', '<', '>', '!', '~', '+', '-', '*', '/', '%', '^', '&', '|', '@', '.', ',', ';', - ':', '#', '$', '?', '\'', - ]; - if !LEGAL_CHARS.contains(&ch) { - panic!("unsupported character `{:?}`", ch) - } - Punct { ch, joint, span } - } -} - #[derive(Copy, Clone, PartialEq, Eq, Hash)] pub struct Ident { sym: Symbol, @@ -380,7 +359,6 @@ impl server::Types for Rustc<'_, '_> { type FreeFunctions = FreeFunctions; type TokenStream = TokenStream; type Group = Group; - type Punct = Punct; type Ident = Ident; type Literal = Literal; type SourceFile = Lrc; @@ -469,14 +447,14 @@ impl server::TokenStream for Rustc<'_, '_> { } fn from_token_tree( &mut self, - tree: TokenTree, + tree: TokenTree, ) -> Self::TokenStream { tree.to_internal() } fn concat_trees( &mut self, base: Option, - trees: Vec>, + trees: Vec>, ) -> Self::TokenStream { let mut builder = tokenstream::TokenStreamBuilder::new(); if let Some(base) = base { @@ -504,7 +482,7 @@ impl server::TokenStream for Rustc<'_, '_> { fn into_iter( &mut self, stream: Self::TokenStream, - ) -> Vec> { + ) -> Vec> { // XXX: This is a raw port of the previous approach, and can probably be // optimized. let mut cursor = stream.into_trees(); @@ -565,24 +543,6 @@ impl server::Group for Rustc<'_, '_> { } } -impl server::Punct for Rustc<'_, '_> { - fn new(&mut self, ch: char, spacing: Spacing) -> Self::Punct { - Punct::new(ch, spacing == Spacing::Joint, server::Context::call_site(self)) - } - fn as_char(&mut self, punct: Self::Punct) -> char { - punct.ch - } - fn spacing(&mut self, punct: Self::Punct) -> Spacing { - if punct.joint { Spacing::Joint } else { Spacing::Alone } - } - fn span(&mut self, punct: Self::Punct) -> Self::Span { - punct.span - } - fn with_span(&mut self, punct: Self::Punct, span: Self::Span) -> Self::Punct { - Punct { span, ..punct } - } -} - impl server::Ident for Rustc<'_, '_> { fn new(&mut self, string: &str, span: Self::Span, is_raw: bool) -> Self::Ident { Ident::new(self.sess(), Symbol::intern(string), is_raw, span) diff --git a/library/proc_macro/src/bridge/client.rs b/library/proc_macro/src/bridge/client.rs index a0070f2f0e685..b34454989fe3b 100644 --- a/library/proc_macro/src/bridge/client.rs +++ b/library/proc_macro/src/bridge/client.rs @@ -185,7 +185,6 @@ define_handles! { Diagnostic, 'interned: - Punct, Ident, Span, } diff --git a/library/proc_macro/src/bridge/mod.rs b/library/proc_macro/src/bridge/mod.rs index 44d695b17d45f..361a8d8103482 100644 --- a/library/proc_macro/src/bridge/mod.rs +++ b/library/proc_macro/src/bridge/mod.rs @@ -65,11 +65,11 @@ macro_rules! with_api { fn from_str(src: &str) -> $S::TokenStream; fn to_string($self: &$S::TokenStream) -> String; fn from_token_tree( - tree: TokenTree<$S::Group, $S::Punct, $S::Ident, $S::Literal>, + tree: TokenTree<$S::Span, $S::Group, $S::Ident, $S::Literal>, ) -> $S::TokenStream; fn concat_trees( base: Option<$S::TokenStream>, - trees: Vec>, + trees: Vec>, ) -> $S::TokenStream; fn concat_streams( base: Option<$S::TokenStream>, @@ -77,7 +77,7 @@ macro_rules! with_api { ) -> $S::TokenStream; fn into_iter( $self: $S::TokenStream - ) -> Vec>; + ) -> Vec>; }, Group { fn drop($self: $S::Group); @@ -90,13 +90,6 @@ macro_rules! with_api { fn span_close($self: &$S::Group) -> $S::Span; fn set_span($self: &mut $S::Group, span: $S::Span); }, - Punct { - fn new(ch: char, spacing: Spacing) -> $S::Punct; - fn as_char($self: $S::Punct) -> char; - fn spacing($self: $S::Punct) -> Spacing; - fn span($self: $S::Punct) -> $S::Span; - fn with_span($self: $S::Punct, span: $S::Span) -> $S::Punct; - }, Ident { fn new(string: &str, span: $S::Span, is_raw: bool) -> $S::Ident; fn span($self: $S::Ident) -> $S::Span; @@ -454,15 +447,24 @@ macro_rules! compound_traits { } #[derive(Clone)] -pub enum TokenTree { +pub struct Punct { + pub ch: char, + pub joint: bool, + pub span: S, +} + +compound_traits!(struct Punct { ch, joint, span }); + +#[derive(Clone)] +pub enum TokenTree { Group(G), - Punct(P), + Punct(Punct), Ident(I), Literal(L), } compound_traits!( - enum TokenTree { + enum TokenTree { Group(tt), Punct(tt), Ident(tt), diff --git a/library/proc_macro/src/bridge/server.rs b/library/proc_macro/src/bridge/server.rs index a04d1b0dd0fb3..140276e177758 100644 --- a/library/proc_macro/src/bridge/server.rs +++ b/library/proc_macro/src/bridge/server.rs @@ -9,7 +9,6 @@ pub trait Types { type FreeFunctions: 'static; type TokenStream: 'static + Clone; type Group: 'static + Clone; - type Punct: 'static + Copy + Eq + Hash; type Ident: 'static + Copy + Eq + Hash; type Literal: 'static + Clone; type SourceFile: 'static + Clone; diff --git a/library/proc_macro/src/lib.rs b/library/proc_macro/src/lib.rs index 559c7ff4be517..22883850d8477 100644 --- a/library/proc_macro/src/lib.rs +++ b/library/proc_macro/src/lib.rs @@ -212,8 +212,8 @@ pub use quote::{quote, quote_span}; fn tree_to_bridge_tree( tree: TokenTree, ) -> bridge::TokenTree< + bridge::client::Span, bridge::client::Group, - bridge::client::Punct, bridge::client::Ident, bridge::client::Literal, > { @@ -289,8 +289,8 @@ pub mod token_stream { pub struct IntoIter( std::vec::IntoIter< bridge::TokenTree< + bridge::client::Span, bridge::client::Group, - bridge::client::Punct, bridge::client::Ident, bridge::client::Literal, >, @@ -849,7 +849,7 @@ impl fmt::Debug for Group { /// forms of `Spacing` returned. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] #[derive(Clone)] -pub struct Punct(bridge::client::Punct); +pub struct Punct(bridge::Punct); #[stable(feature = "proc_macro_lib2", since = "1.29.0")] impl !Send for Punct {} @@ -882,13 +882,20 @@ impl Punct { /// which can be further configured with the `set_span` method below. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn new(ch: char, spacing: Spacing) -> Punct { - Punct(bridge::client::Punct::new(ch, spacing)) + const LEGAL_CHARS: &[char] = &[ + '=', '<', '>', '!', '~', '+', '-', '*', '/', '%', '^', '&', '|', '@', '.', ',', ';', + ':', '#', '$', '?', '\'', + ]; + if !LEGAL_CHARS.contains(&ch) { + panic!("unsupported character `{:?}`", ch); + } + Punct(bridge::Punct { ch, joint: spacing == Spacing::Joint, span: Span::call_site().0 }) } /// Returns the value of this punctuation character as `char`. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn as_char(&self) -> char { - self.0.as_char() + self.0.ch } /// Returns the spacing of this punctuation character, indicating whether it's immediately @@ -897,28 +904,19 @@ impl Punct { /// (`Alone`) so the operator has certainly ended. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn spacing(&self) -> Spacing { - self.0.spacing() + if self.0.joint { Spacing::Joint } else { Spacing::Alone } } /// Returns the span for this punctuation character. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn span(&self) -> Span { - Span(self.0.span()) + Span(self.0.span) } /// Configure the span for this punctuation character. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn set_span(&mut self, span: Span) { - self.0 = self.0.with_span(span.0); - } -} - -// N.B., the bridge only provides `to_string`, implement `fmt::Display` -// based on it (the reverse of the usual relationship between the two). -#[stable(feature = "proc_macro_lib", since = "1.15.0")] -impl ToString for Punct { - fn to_string(&self) -> String { - TokenStream::from(TokenTree::from(self.clone())).to_string() + self.0.span = span.0; } } @@ -927,7 +925,7 @@ impl ToString for Punct { #[stable(feature = "proc_macro_lib2", since = "1.29.0")] impl fmt::Display for Punct { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(&self.to_string()) + write!(f, "{}", self.as_char()) } } diff --git a/src/test/ui/proc-macro/invalid-punct-ident-1.rs b/src/test/ui/proc-macro/invalid-punct-ident-1.rs index a3133a1a79070..ecbb6ebf55b9a 100644 --- a/src/test/ui/proc-macro/invalid-punct-ident-1.rs +++ b/src/test/ui/proc-macro/invalid-punct-ident-1.rs @@ -2,7 +2,7 @@ // rustc-env:RUST_BACKTRACE=0 // FIXME https://github.com/rust-lang/rust/issues/59998 -// normalize-stderr-test "thread.*panicked.*proc_macro_server.rs.*\n" -> "" +// normalize-stderr-test "thread.*panicked.*proc_macro.*lib.rs.*\n" -> "" // normalize-stderr-test "note:.*RUST_BACKTRACE=1.*\n" -> "" // normalize-stderr-test "\nerror: internal compiler error.*\n\n" -> "" // normalize-stderr-test "note:.*unexpectedly panicked.*\n\n" -> "" From ff17109eece8815fb400e53089d5b62bbc2e3927 Mon Sep 17 00:00:00 2001 From: Nika Layzell Date: Thu, 1 Jul 2021 17:36:38 -0400 Subject: [PATCH 07/11] proc_macro: stop using a remote object handle for Group This greatly reduces round-trips to fetch relevant extra information about the token in proc macro code, and avoids RPC messages to create Group tokens. --- .../rustc_expand/src/proc_macro_server.rs | 367 ++++++++---------- library/proc_macro/src/bridge/client.rs | 7 - library/proc_macro/src/bridge/mod.rs | 47 ++- library/proc_macro/src/bridge/server.rs | 1 - library/proc_macro/src/lib.rs | 24 +- 5 files changed, 204 insertions(+), 242 deletions(-) diff --git a/compiler/rustc_expand/src/proc_macro_server.rs b/compiler/rustc_expand/src/proc_macro_server.rs index f078749742cdf..6b89c972ca96c 100644 --- a/compiler/rustc_expand/src/proc_macro_server.rs +++ b/compiler/rustc_expand/src/proc_macro_server.rs @@ -2,7 +2,7 @@ use crate::base::ExtCtxt; use rustc_ast as ast; use rustc_ast::token; -use rustc_ast::tokenstream::{self, DelimSpan, Spacing::*, TokenStream, TreeAndSpacing}; +use rustc_ast::tokenstream::{self, Spacing::*, TokenStream}; use rustc_ast_pretty::pprust; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::sync::Lrc; @@ -14,7 +14,7 @@ use rustc_span::def_id::CrateNum; use rustc_span::symbol::{self, kw, sym, Symbol}; use rustc_span::{BytePos, FileName, Pos, SourceFile, Span}; -use pm::bridge::{server, Punct, TokenTree}; +use pm::bridge::{server, DelimSpan, Group, Punct, TokenTree}; use pm::{Delimiter, Level, LineColumn}; use std::ops::Bound; use std::{ascii, panic}; @@ -49,158 +49,179 @@ impl ToInternal for Delimiter { } } -impl FromInternal<(TreeAndSpacing, &'_ mut Vec, &mut Rustc<'_, '_>)> - for TokenTree +impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> + for Vec> { - fn from_internal( - ((tree, spacing), stack, rustc): (TreeAndSpacing, &mut Vec, &mut Rustc<'_, '_>), - ) -> Self { + fn from_internal((stream, rustc): (TokenStream, &mut Rustc<'_, '_>)) -> Self { use rustc_ast::token::*; - let joint = spacing == Joint; - let Token { kind, span } = match tree { - tokenstream::TokenTree::Delimited(span, delim, tts) => { - let delimiter = pm::Delimiter::from_internal(delim); - return TokenTree::Group(Group { delimiter, stream: tts, span, flatten: false }); - } - tokenstream::TokenTree::Token(token) => token, - }; - - macro_rules! tt { - ($ty:ident { $($field:ident $(: $value:expr)*),+ $(,)? }) => ( - TokenTree::$ty(self::$ty { - $($field $(: $value)*,)+ - span, - }) - ); - ($ty:ident::$method:ident($($value:expr),*)) => ( - TokenTree::$ty(self::$ty::$method($($value,)* span)) - ); - } - macro_rules! op { - ($a:expr) => { - tt!(Punct { ch: $a, joint }) + let mut cursor = stream.into_trees(); + let mut trees = Vec::new(); + + while let Some((tree, spacing)) = cursor.next_with_spacing() { + let joint = spacing == Joint; + let Token { kind, span } = match tree { + tokenstream::TokenTree::Delimited(span, delim, tts) => { + let delimiter = pm::Delimiter::from_internal(delim); + trees.push(TokenTree::Group(Group { + delimiter, + stream: Some(tts), + span: DelimSpan { + open: span.open, + close: span.close, + entire: span.entire(), + }, + })); + continue; + } + tokenstream::TokenTree::Token(token) => token, }; - ($a:expr, $b:expr) => {{ - stack.push(tt!(Punct { ch: $b, joint })); - tt!(Punct { ch: $a, joint: true }) - }}; - ($a:expr, $b:expr, $c:expr) => {{ - stack.push(tt!(Punct { ch: $c, joint })); - stack.push(tt!(Punct { ch: $b, joint: true })); - tt!(Punct { ch: $a, joint: true }) - }}; - } - match kind { - Eq => op!('='), - Lt => op!('<'), - Le => op!('<', '='), - EqEq => op!('=', '='), - Ne => op!('!', '='), - Ge => op!('>', '='), - Gt => op!('>'), - AndAnd => op!('&', '&'), - OrOr => op!('|', '|'), - Not => op!('!'), - Tilde => op!('~'), - BinOp(Plus) => op!('+'), - BinOp(Minus) => op!('-'), - BinOp(Star) => op!('*'), - BinOp(Slash) => op!('/'), - BinOp(Percent) => op!('%'), - BinOp(Caret) => op!('^'), - BinOp(And) => op!('&'), - BinOp(Or) => op!('|'), - BinOp(Shl) => op!('<', '<'), - BinOp(Shr) => op!('>', '>'), - BinOpEq(Plus) => op!('+', '='), - BinOpEq(Minus) => op!('-', '='), - BinOpEq(Star) => op!('*', '='), - BinOpEq(Slash) => op!('/', '='), - BinOpEq(Percent) => op!('%', '='), - BinOpEq(Caret) => op!('^', '='), - BinOpEq(And) => op!('&', '='), - BinOpEq(Or) => op!('|', '='), - BinOpEq(Shl) => op!('<', '<', '='), - BinOpEq(Shr) => op!('>', '>', '='), - At => op!('@'), - Dot => op!('.'), - DotDot => op!('.', '.'), - DotDotDot => op!('.', '.', '.'), - DotDotEq => op!('.', '.', '='), - Comma => op!(','), - Semi => op!(';'), - Colon => op!(':'), - ModSep => op!(':', ':'), - RArrow => op!('-', '>'), - LArrow => op!('<', '-'), - FatArrow => op!('=', '>'), - Pound => op!('#'), - Dollar => op!('$'), - Question => op!('?'), - SingleQuote => op!('\''), - - Ident(name, false) if name == kw::DollarCrate => tt!(Ident::dollar_crate()), - Ident(name, is_raw) => tt!(Ident::new(rustc.sess(), name, is_raw)), - Lifetime(name) => { - let ident = symbol::Ident::new(name, span).without_first_quote(); - stack.push(tt!(Ident::new(rustc.sess(), ident.name, false))); - tt!(Punct { ch: '\'', joint: true }) + macro_rules! tt { + ($ty:ident { $($field:ident $(: $value:expr)*),+ $(,)? }) => ( + trees.push(TokenTree::$ty(self::$ty { + $($field $(: $value)*,)+ + span, + })) + ); + ($ty:ident::$method:ident($($value:expr),*)) => ( + trees.push(TokenTree::$ty(self::$ty::$method($($value,)* span))) + ); + } + macro_rules! op { + ($a:expr) => {{ + tt!(Punct { ch: $a, joint }); + }}; + ($a:expr, $b:expr) => {{ + tt!(Punct { ch: $a, joint: true }); + tt!(Punct { ch: $b, joint }); + }}; + ($a:expr, $b:expr, $c:expr) => {{ + tt!(Punct { ch: $a, joint: true }); + tt!(Punct { ch: $b, joint: true }); + tt!(Punct { ch: $c, joint }); + }}; } - Literal(lit) => tt!(Literal { lit }), - DocComment(_, attr_style, data) => { - let mut escaped = String::new(); - for ch in data.as_str().chars() { - escaped.extend(ch.escape_debug()); + + match kind { + Eq => op!('='), + Lt => op!('<'), + Le => op!('<', '='), + EqEq => op!('=', '='), + Ne => op!('!', '='), + Ge => op!('>', '='), + Gt => op!('>'), + AndAnd => op!('&', '&'), + OrOr => op!('|', '|'), + Not => op!('!'), + Tilde => op!('~'), + BinOp(Plus) => op!('+'), + BinOp(Minus) => op!('-'), + BinOp(Star) => op!('*'), + BinOp(Slash) => op!('/'), + BinOp(Percent) => op!('%'), + BinOp(Caret) => op!('^'), + BinOp(And) => op!('&'), + BinOp(Or) => op!('|'), + BinOp(Shl) => op!('<', '<'), + BinOp(Shr) => op!('>', '>'), + BinOpEq(Plus) => op!('+', '='), + BinOpEq(Minus) => op!('-', '='), + BinOpEq(Star) => op!('*', '='), + BinOpEq(Slash) => op!('/', '='), + BinOpEq(Percent) => op!('%', '='), + BinOpEq(Caret) => op!('^', '='), + BinOpEq(And) => op!('&', '='), + BinOpEq(Or) => op!('|', '='), + BinOpEq(Shl) => op!('<', '<', '='), + BinOpEq(Shr) => op!('>', '>', '='), + At => op!('@'), + Dot => op!('.'), + DotDot => op!('.', '.'), + DotDotDot => op!('.', '.', '.'), + DotDotEq => op!('.', '.', '='), + Comma => op!(','), + Semi => op!(';'), + Colon => op!(':'), + ModSep => op!(':', ':'), + RArrow => op!('-', '>'), + LArrow => op!('<', '-'), + FatArrow => op!('=', '>'), + Pound => op!('#'), + Dollar => op!('$'), + Question => op!('?'), + SingleQuote => op!('\''), + + Ident(name, false) if name == kw::DollarCrate => tt!(Ident::dollar_crate()), + Ident(name, is_raw) => tt!(Ident::new(rustc.sess(), name, is_raw)), + Lifetime(name) => { + let ident = symbol::Ident::new(name, span).without_first_quote(); + tt!(Punct { ch: '\'', joint: true }); + tt!(Ident::new(rustc.sess(), ident.name, false)); } - let stream = [ - Ident(sym::doc, false), - Eq, - TokenKind::lit(token::Str, Symbol::intern(&escaped), None), - ] - .into_iter() - .map(|kind| tokenstream::TokenTree::token(kind, span)) - .collect(); - stack.push(TokenTree::Group(Group { - delimiter: pm::Delimiter::Bracket, - stream, - span: DelimSpan::from_single(span), - flatten: false, - })); - if attr_style == ast::AttrStyle::Inner { - stack.push(tt!(Punct { ch: '!', joint: false })); + Literal(lit) => tt!(Literal { lit }), + DocComment(_, attr_style, data) => { + let mut escaped = String::new(); + for ch in data.as_str().chars() { + escaped.extend(ch.escape_debug()); + } + let stream = vec![ + Ident(sym::doc, false), + Eq, + TokenKind::lit(token::Str, Symbol::intern(&escaped), None), + ] + .into_iter() + .map(|kind| tokenstream::TokenTree::token(kind, span)) + .collect(); + tt!(Punct { ch: '#', joint: false }); + if attr_style == ast::AttrStyle::Inner { + tt!(Punct { ch: '!', joint: false }); + } + trees.push(TokenTree::Group(Group { + delimiter: pm::Delimiter::Bracket, + stream: Some(stream), + span: DelimSpan::from_single(span), + })); } - tt!(Punct { ch: '#', joint: false }) - } - Interpolated(nt) if let NtIdent(ident, is_raw) = *nt => { - TokenTree::Ident(Ident::new(rustc.sess(), ident.name, is_raw, ident.span)) - } - Interpolated(nt) => { - TokenTree::Group(Group { - delimiter: pm::Delimiter::None, - stream: TokenStream::from_nonterminal_ast(&nt), - span: DelimSpan::from_single(span), - flatten: crate::base::nt_pretty_printing_compatibility_hack(&nt, rustc.sess()), - }) - } + Interpolated(nt) if let NtIdent(ident, is_raw) = *nt => { + trees.push(TokenTree::Ident(Ident::new(rustc.sess(), ident.name, is_raw, ident.span))) + } + + Interpolated(nt) => { + let stream = TokenStream::from_nonterminal_ast(&nt); + if crate::base::nt_pretty_printing_compatibility_hack(&nt, rustc.sess()) { + cursor.append(stream); + } else { + trees.push(TokenTree::Group(Group { + delimiter: pm::Delimiter::None, + stream: Some(stream), + span: DelimSpan::from_single(span), + })) + } + } - OpenDelim(..) | CloseDelim(..) => unreachable!(), - Eof => unreachable!(), + OpenDelim(..) | CloseDelim(..) => unreachable!(), + Eof => unreachable!(), + } } + trees } } -impl ToInternal for TokenTree { +impl ToInternal for TokenTree { fn to_internal(self) -> TokenStream { use rustc_ast::token::*; let (ch, joint, span) = match self { TokenTree::Punct(Punct { ch, joint, span }) => (ch, joint, span), - TokenTree::Group(Group { delimiter, stream, span, .. }) => { - return tokenstream::TokenTree::Delimited(span, delimiter.to_internal(), stream) - .into(); + TokenTree::Group(Group { delimiter, stream, span: DelimSpan { open, close, .. } }) => { + return tokenstream::TokenTree::Delimited( + tokenstream::DelimSpan { open, close }, + delimiter.to_internal(), + stream.unwrap_or_default(), + ) + .into(); } TokenTree::Ident(self::Ident { sym, is_raw, span }) => { return tokenstream::TokenTree::token(Ident(sym, is_raw), span).into(); @@ -277,17 +298,6 @@ impl ToInternal for Level { pub struct FreeFunctions; -#[derive(Clone)] -pub struct Group { - delimiter: Delimiter, - stream: TokenStream, - span: DelimSpan, - /// A hack used to pass AST fragments to attribute and derive macros - /// as a single nonterminal token instead of a token stream. - /// FIXME: It needs to be removed, but there are some compatibility issues (see #73345). - flatten: bool, -} - #[derive(Copy, Clone, PartialEq, Eq, Hash)] pub struct Ident { sym: Symbol, @@ -358,7 +368,6 @@ impl<'a, 'b> Rustc<'a, 'b> { impl server::Types for Rustc<'_, '_> { type FreeFunctions = FreeFunctions; type TokenStream = TokenStream; - type Group = Group; type Ident = Ident; type Literal = Literal; type SourceFile = Lrc; @@ -447,14 +456,14 @@ impl server::TokenStream for Rustc<'_, '_> { } fn from_token_tree( &mut self, - tree: TokenTree, + tree: TokenTree, ) -> Self::TokenStream { tree.to_internal() } fn concat_trees( &mut self, base: Option, - trees: Vec>, + trees: Vec>, ) -> Self::TokenStream { let mut builder = tokenstream::TokenStreamBuilder::new(); if let Some(base) = base { @@ -482,64 +491,8 @@ impl server::TokenStream for Rustc<'_, '_> { fn into_iter( &mut self, stream: Self::TokenStream, - ) -> Vec> { - // XXX: This is a raw port of the previous approach, and can probably be - // optimized. - let mut cursor = stream.into_trees(); - let mut stack = Vec::new(); - let mut tts = Vec::new(); - loop { - let next = stack.pop().or_else(|| { - let next = cursor.next_with_spacing()?; - Some(TokenTree::from_internal((next, &mut stack, self))) - }); - match next { - Some(TokenTree::Group(group)) => { - // A hack used to pass AST fragments to attribute and derive - // macros as a single nonterminal token instead of a token - // stream. Such token needs to be "unwrapped" and not - // represented as a delimited group. - // FIXME: It needs to be removed, but there are some - // compatibility issues (see #73345). - if group.flatten { - cursor.append(group.stream); - continue; - } - tts.push(TokenTree::Group(group)); - } - Some(tt) => tts.push(tt), - None => return tts, - } - } - } -} - -impl server::Group for Rustc<'_, '_> { - fn new(&mut self, delimiter: Delimiter, stream: Option) -> Self::Group { - Group { - delimiter, - stream: stream.unwrap_or_default(), - span: DelimSpan::from_single(server::Context::call_site(self)), - flatten: false, - } - } - fn delimiter(&mut self, group: &Self::Group) -> Delimiter { - group.delimiter - } - fn stream(&mut self, group: &Self::Group) -> Self::TokenStream { - group.stream.clone() - } - fn span(&mut self, group: &Self::Group) -> Self::Span { - group.span.entire() - } - fn span_open(&mut self, group: &Self::Group) -> Self::Span { - group.span.open - } - fn span_close(&mut self, group: &Self::Group) -> Self::Span { - group.span.close - } - fn set_span(&mut self, group: &mut Self::Group, span: Self::Span) { - group.span = DelimSpan::from_single(span); + ) -> Vec> { + FromInternal::from_internal((stream, self)) } } diff --git a/library/proc_macro/src/bridge/client.rs b/library/proc_macro/src/bridge/client.rs index b34454989fe3b..9f8fbcbf67ff0 100644 --- a/library/proc_macro/src/bridge/client.rs +++ b/library/proc_macro/src/bridge/client.rs @@ -178,7 +178,6 @@ define_handles! { 'owned: FreeFunctions, TokenStream, - Group, Literal, SourceFile, MultiSpan, @@ -201,12 +200,6 @@ impl Clone for TokenStream { } } -impl Clone for Group { - fn clone(&self) -> Self { - self.clone() - } -} - impl Clone for Literal { fn clone(&self) -> Self { self.clone() diff --git a/library/proc_macro/src/bridge/mod.rs b/library/proc_macro/src/bridge/mod.rs index 361a8d8103482..244285cb349d4 100644 --- a/library/proc_macro/src/bridge/mod.rs +++ b/library/proc_macro/src/bridge/mod.rs @@ -65,11 +65,11 @@ macro_rules! with_api { fn from_str(src: &str) -> $S::TokenStream; fn to_string($self: &$S::TokenStream) -> String; fn from_token_tree( - tree: TokenTree<$S::Span, $S::Group, $S::Ident, $S::Literal>, + tree: TokenTree<$S::TokenStream, $S::Span, $S::Ident, $S::Literal>, ) -> $S::TokenStream; fn concat_trees( base: Option<$S::TokenStream>, - trees: Vec>, + trees: Vec>, ) -> $S::TokenStream; fn concat_streams( base: Option<$S::TokenStream>, @@ -77,18 +77,7 @@ macro_rules! with_api { ) -> $S::TokenStream; fn into_iter( $self: $S::TokenStream - ) -> Vec>; - }, - Group { - fn drop($self: $S::Group); - fn clone($self: &$S::Group) -> $S::Group; - fn new(delimiter: Delimiter, stream: Option<$S::TokenStream>) -> $S::Group; - fn delimiter($self: &$S::Group) -> Delimiter; - fn stream($self: &$S::Group) -> $S::TokenStream; - fn span($self: &$S::Group) -> $S::Span; - fn span_open($self: &$S::Group) -> $S::Span; - fn span_close($self: &$S::Group) -> $S::Span; - fn set_span($self: &mut $S::Group, span: $S::Span); + ) -> Vec>; }, Ident { fn new(string: &str, span: $S::Span, is_raw: bool) -> $S::Ident; @@ -446,6 +435,30 @@ macro_rules! compound_traits { }; } +#[derive(Copy, Clone)] +pub struct DelimSpan { + pub open: S, + pub close: S, + pub entire: S, +} + +impl DelimSpan { + pub fn from_single(span: S) -> Self { + DelimSpan { open: span, close: span, entire: span } + } +} + +compound_traits!(struct DelimSpan { open, close, entire }); + +#[derive(Clone)] +pub struct Group { + pub delimiter: Delimiter, + pub stream: Option, + pub span: DelimSpan, +} + +compound_traits!(struct Group { delimiter, stream, span }); + #[derive(Clone)] pub struct Punct { pub ch: char, @@ -456,15 +469,15 @@ pub struct Punct { compound_traits!(struct Punct { ch, joint, span }); #[derive(Clone)] -pub enum TokenTree { - Group(G), +pub enum TokenTree { + Group(Group), Punct(Punct), Ident(I), Literal(L), } compound_traits!( - enum TokenTree { + enum TokenTree { Group(tt), Punct(tt), Ident(tt), diff --git a/library/proc_macro/src/bridge/server.rs b/library/proc_macro/src/bridge/server.rs index 140276e177758..8fbe010939684 100644 --- a/library/proc_macro/src/bridge/server.rs +++ b/library/proc_macro/src/bridge/server.rs @@ -8,7 +8,6 @@ use super::client::HandleStore; pub trait Types { type FreeFunctions: 'static; type TokenStream: 'static + Clone; - type Group: 'static + Clone; type Ident: 'static + Copy + Eq + Hash; type Literal: 'static + Clone; type SourceFile: 'static + Clone; diff --git a/library/proc_macro/src/lib.rs b/library/proc_macro/src/lib.rs index 22883850d8477..68db0ba3dc49b 100644 --- a/library/proc_macro/src/lib.rs +++ b/library/proc_macro/src/lib.rs @@ -212,8 +212,8 @@ pub use quote::{quote, quote_span}; fn tree_to_bridge_tree( tree: TokenTree, ) -> bridge::TokenTree< + bridge::client::TokenStream, bridge::client::Span, - bridge::client::Group, bridge::client::Ident, bridge::client::Literal, > { @@ -289,8 +289,8 @@ pub mod token_stream { pub struct IntoIter( std::vec::IntoIter< bridge::TokenTree< + bridge::client::TokenStream, bridge::client::Span, - bridge::client::Group, bridge::client::Ident, bridge::client::Literal, >, @@ -712,7 +712,7 @@ impl fmt::Display for TokenTree { /// A `Group` internally contains a `TokenStream` which is surrounded by `Delimiter`s. #[derive(Clone)] #[stable(feature = "proc_macro_lib2", since = "1.29.0")] -pub struct Group(bridge::client::Group); +pub struct Group(bridge::Group); #[stable(feature = "proc_macro_lib2", since = "1.29.0")] impl !Send for Group {} @@ -749,13 +749,17 @@ impl Group { /// method below. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn new(delimiter: Delimiter, stream: TokenStream) -> Group { - Group(bridge::client::Group::new(delimiter, stream.0)) + Group(bridge::Group { + delimiter, + stream: stream.0, + span: bridge::DelimSpan::from_single(Span::call_site().0), + }) } /// Returns the delimiter of this `Group` #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn delimiter(&self) -> Delimiter { - self.0.delimiter() + self.0.delimiter } /// Returns the `TokenStream` of tokens that are delimited in this `Group`. @@ -764,7 +768,7 @@ impl Group { /// returned above. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn stream(&self) -> TokenStream { - TokenStream(Some(self.0.stream())) + TokenStream(self.0.stream.clone()) } /// Returns the span for the delimiters of this token stream, spanning the @@ -776,7 +780,7 @@ impl Group { /// ``` #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn span(&self) -> Span { - Span(self.0.span()) + Span(self.0.span.entire) } /// Returns the span pointing to the opening delimiter of this group. @@ -787,7 +791,7 @@ impl Group { /// ``` #[stable(feature = "proc_macro_group_span", since = "1.55.0")] pub fn span_open(&self) -> Span { - Span(self.0.span_open()) + Span(self.0.span.open) } /// Returns the span pointing to the closing delimiter of this group. @@ -798,7 +802,7 @@ impl Group { /// ``` #[stable(feature = "proc_macro_group_span", since = "1.55.0")] pub fn span_close(&self) -> Span { - Span(self.0.span_close()) + Span(self.0.span.close) } /// Configures the span for this `Group`'s delimiters, but not its internal @@ -809,7 +813,7 @@ impl Group { /// tokens at the level of the `Group`. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn set_span(&mut self, span: Span) { - self.0.set_span(span.0); + self.0.span = bridge::DelimSpan::from_single(span.0); } } From 67bd4241fb875078db4179cb5dccdbeb0d26cae7 Mon Sep 17 00:00:00 2001 From: Nika Layzell Date: Thu, 1 Jul 2021 22:59:51 -0400 Subject: [PATCH 08/11] proc_macro: stop using a remote object handle for Ident This requires a dependency on `unicode-normalization` and `rustc_lexer`, which is currently not possible for `proc_macro`. Instead, a second `extern "C" fn` is provided by the compiler server to perform these steps from any thread. String values are interned in both the server and client, meaning that identifiers can be stringified without any RPC roundtrips without substantially inflating their size. RPC messages passing symbols include the full un-interned value, and are re-interned on the receiving side. This could potentially be optimized in the future. The symbol infrastructure will alwo be used for literals in a following part. --- Cargo.lock | 1 + compiler/rustc_expand/Cargo.toml | 1 + .../rustc_expand/src/proc_macro_server.rs | 89 ++++++--------- library/proc_macro/src/bridge/buffer.rs | 31 ++++++ library/proc_macro/src/bridge/client.rs | 105 ++++++++++++++++-- library/proc_macro/src/bridge/handle.rs | 48 +++++++- library/proc_macro/src/bridge/mod.rs | 69 ++++++------ library/proc_macro/src/bridge/server.rs | 68 ++++++++++-- library/proc_macro/src/lib.rs | 36 +++--- .../ui/proc-macro/invalid-punct-ident-2.rs | 2 +- .../ui/proc-macro/invalid-punct-ident-3.rs | 2 +- 11 files changed, 326 insertions(+), 126 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f586e9facc62a..2aeb5c043420d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3886,6 +3886,7 @@ dependencies = [ "rustc_span", "smallvec", "tracing", + "unicode-normalization", ] [[package]] diff --git a/compiler/rustc_expand/Cargo.toml b/compiler/rustc_expand/Cargo.toml index 45237ab2e9f25..44f773cfcd120 100644 --- a/compiler/rustc_expand/Cargo.toml +++ b/compiler/rustc_expand/Cargo.toml @@ -24,3 +24,4 @@ rustc_parse = { path = "../rustc_parse" } rustc_session = { path = "../rustc_session" } smallvec = { version = "1.6.1", features = ["union", "may_dangle"] } rustc_ast = { path = "../rustc_ast" } +unicode-normalization = "0.1.11" diff --git a/compiler/rustc_expand/src/proc_macro_server.rs b/compiler/rustc_expand/src/proc_macro_server.rs index 6b89c972ca96c..0a9952f773cef 100644 --- a/compiler/rustc_expand/src/proc_macro_server.rs +++ b/compiler/rustc_expand/src/proc_macro_server.rs @@ -7,17 +7,16 @@ use rustc_ast_pretty::pprust; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::sync::Lrc; use rustc_errors::{Diagnostic, MultiSpan, PResult}; -use rustc_parse::lexer::nfc_normalize; use rustc_parse::parse_stream_from_source_str; use rustc_session::parse::ParseSess; use rustc_span::def_id::CrateNum; -use rustc_span::symbol::{self, kw, sym, Symbol}; +use rustc_span::symbol::{self, sym, Symbol}; use rustc_span::{BytePos, FileName, Pos, SourceFile, Span}; -use pm::bridge::{server, DelimSpan, Group, Punct, TokenTree}; +use pm::bridge::{server, DelimSpan, Group, Ident, Punct, TokenTree}; use pm::{Delimiter, Level, LineColumn}; +use std::ascii; use std::ops::Bound; -use std::{ascii, panic}; trait FromInternal { fn from_internal(x: T) -> Self; @@ -50,7 +49,7 @@ impl ToInternal for Delimiter { } impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> - for Vec> + for Vec> { fn from_internal((stream, rustc): (TokenStream, &mut Rustc<'_, '_>)) -> Self { use rustc_ast::token::*; @@ -84,9 +83,6 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> span, })) ); - ($ty:ident::$method:ident($($value:expr),*)) => ( - trees.push(TokenTree::$ty(self::$ty::$method($($value,)* span))) - ); } macro_rules! op { ($a:expr) => {{ @@ -152,12 +148,11 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> Question => op!('?'), SingleQuote => op!('\''), - Ident(name, false) if name == kw::DollarCrate => tt!(Ident::dollar_crate()), - Ident(name, is_raw) => tt!(Ident::new(rustc.sess(), name, is_raw)), + Ident(sym, is_raw) => tt!(Ident { sym, is_raw }), Lifetime(name) => { let ident = symbol::Ident::new(name, span).without_first_quote(); tt!(Punct { ch: '\'', joint: true }); - tt!(Ident::new(rustc.sess(), ident.name, false)); + tt!(Ident { sym: ident.name, is_raw: false }); } Literal(lit) => tt!(Literal { lit }), DocComment(_, attr_style, data) => { @@ -185,7 +180,7 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> } Interpolated(nt) if let NtIdent(ident, is_raw) = *nt => { - trees.push(TokenTree::Ident(Ident::new(rustc.sess(), ident.name, is_raw, ident.span))) + trees.push(TokenTree::Ident(Ident { sym: ident.name, is_raw, span: ident.span })) } Interpolated(nt) => { @@ -209,7 +204,7 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> } } -impl ToInternal for TokenTree { +impl ToInternal for TokenTree { fn to_internal(self) -> TokenStream { use rustc_ast::token::*; @@ -298,32 +293,6 @@ impl ToInternal for Level { pub struct FreeFunctions; -#[derive(Copy, Clone, PartialEq, Eq, Hash)] -pub struct Ident { - sym: Symbol, - is_raw: bool, - span: Span, -} - -impl Ident { - fn new(sess: &ParseSess, sym: Symbol, is_raw: bool, span: Span) -> Ident { - let sym = nfc_normalize(sym.as_str()); - let string = sym.as_str(); - if !rustc_lexer::is_ident(string) { - panic!("`{:?}` is not a valid identifier", string) - } - if is_raw && !sym.can_be_raw() { - panic!("`{}` cannot be a raw identifier", string); - } - sess.symbol_gallery.insert(sym, span); - Ident { sym, is_raw, span } - } - fn dollar_crate(span: Span) -> Ident { - // `$crate` is accepted as an ident only if it comes from the compiler. - Ident { sym: kw::DollarCrate, is_raw: false, span } - } -} - // FIXME(eddyb) `Literal` should not expose internal `Debug` impls. #[derive(Clone, Debug)] pub struct Literal { @@ -368,12 +337,12 @@ impl<'a, 'b> Rustc<'a, 'b> { impl server::Types for Rustc<'_, '_> { type FreeFunctions = FreeFunctions; type TokenStream = TokenStream; - type Ident = Ident; type Literal = Literal; type SourceFile = Lrc; type MultiSpan = Vec; type Diagnostic = Diagnostic; type Span = Span; + type Symbol = Symbol; } impl server::FreeFunctions for Rustc<'_, '_> { @@ -456,14 +425,14 @@ impl server::TokenStream for Rustc<'_, '_> { } fn from_token_tree( &mut self, - tree: TokenTree, + tree: TokenTree, ) -> Self::TokenStream { tree.to_internal() } fn concat_trees( &mut self, base: Option, - trees: Vec>, + trees: Vec>, ) -> Self::TokenStream { let mut builder = tokenstream::TokenStreamBuilder::new(); if let Some(base) = base { @@ -491,23 +460,11 @@ impl server::TokenStream for Rustc<'_, '_> { fn into_iter( &mut self, stream: Self::TokenStream, - ) -> Vec> { + ) -> Vec> { FromInternal::from_internal((stream, self)) } } -impl server::Ident for Rustc<'_, '_> { - fn new(&mut self, string: &str, span: Self::Span, is_raw: bool) -> Self::Ident { - Ident::new(self.sess(), Symbol::intern(string), is_raw, span) - } - fn span(&mut self, ident: Self::Ident) -> Self::Span { - ident.span - } - fn with_span(&mut self, ident: Self::Ident, span: Self::Span) -> Self::Ident { - Ident { span, ..ident } - } -} - impl server::Literal for Rustc<'_, '_> { fn from_str(&mut self, s: &str) -> Result { let name = FileName::proc_macro_source_code(s); @@ -789,4 +746,26 @@ impl server::Context for Rustc<'_, '_> { fn mixed_site(&mut self) -> Self::Span { self.mixed_site } + + // NOTE: May be run on any thread, so cannot use `nfc_normalize` + fn validate_ident(s: &str) -> Result, ()> { + use unicode_normalization::{is_nfc_quick, IsNormalized, UnicodeNormalization}; + let normalized: Option = match is_nfc_quick(s.chars()) { + IsNormalized::Yes => None, + _ => Some(s.chars().nfc().collect()), + }; + if rustc_lexer::is_ident(normalized.as_ref().map(|s| &s[..]).unwrap_or(s)) { + Ok(normalized) + } else { + Err(()) + } + } + + fn intern_symbol(string: &str) -> Self::Symbol { + Symbol::intern(string) + } + + fn with_symbol_string(symbol: &Self::Symbol, f: impl FnOnce(&str)) { + f(&symbol.as_str()) + } } diff --git a/library/proc_macro/src/bridge/buffer.rs b/library/proc_macro/src/bridge/buffer.rs index 48030f8d82dca..3d8832d64ba8c 100644 --- a/library/proc_macro/src/bridge/buffer.rs +++ b/library/proc_macro/src/bridge/buffer.rs @@ -1,10 +1,41 @@ //! Buffer management for same-process client<->server communication. use std::io::{self, Write}; +use std::marker::PhantomData; use std::mem; use std::ops::{Deref, DerefMut}; use std::slice; +#[repr(C)] +pub struct Slice<'a> { + data: *const u8, + len: usize, + _marker: PhantomData<&'a [u8]>, +} + +unsafe impl<'a> Send for Slice<'a> {} +unsafe impl<'a> Sync for Slice<'a> {} + +impl<'a> Copy for Slice<'a> {} +impl<'a> Clone for Slice<'a> { + fn clone(&self) -> Self { + *self + } +} + +impl<'a> From<&'a [u8]> for Slice<'a> { + fn from(xs: &'a [u8]) -> Self { + Slice { data: xs.as_ptr(), len: xs.len(), _marker: PhantomData } + } +} + +impl<'a> Deref for Slice<'a> { + type Target = [u8]; + fn deref(&self) -> &[u8] { + unsafe { slice::from_raw_parts(self.data, self.len) } + } +} + #[repr(C)] pub struct Buffer { data: *mut u8, diff --git a/library/proc_macro/src/bridge/client.rs b/library/proc_macro/src/bridge/client.rs index 9f8fbcbf67ff0..eb6e351095b69 100644 --- a/library/proc_macro/src/bridge/client.rs +++ b/library/proc_macro/src/bridge/client.rs @@ -3,6 +3,7 @@ use super::*; use std::marker::PhantomData; +use std::rc::Rc; macro_rules! define_handles { ( @@ -159,7 +160,7 @@ macro_rules! define_handles { for Marked { fn encode(self, w: &mut Writer, s: &mut HandleStore>) { - s.$ity.alloc(self).encode(w, s); + s.$ity.alloc(&self).encode(w, s); } } @@ -184,7 +185,6 @@ define_handles! { Diagnostic, 'interned: - Ident, Span, } @@ -245,6 +245,84 @@ impl fmt::Debug for Span { } } +#[repr(C)] +#[derive(Copy, Clone, PartialEq, Eq, Hash)] +pub(crate) struct Symbol(handle::Handle); + +impl Symbol { + /// Create a new `Symbol` for an identifier. + /// + /// Validates and normalizes before converting it to a symbol. + pub(crate) fn new_ident(string: &str, is_raw: bool) -> Self { + Symbol(Bridge::with(|bridge| { + let mut normalized = Buffer::new(); + if !(bridge.validate_ident)(string.as_bytes().into(), &mut normalized) { + panic!("`{:?}` is not a valid identifier", string) + } + let string = if normalized.len() > 0 { + std::str::from_utf8(&normalized[..]).unwrap() + } else { + string + }; + if is_raw && !Self::can_be_raw(string) { + panic!("`{}` cannot be a raw identifier", string); + } + bridge.symbols.alloc(string) + })) + } + + // Mimics the behaviour of `Symbol::can_be_raw` from `rustc_span` + fn can_be_raw(string: &str) -> bool { + match string { + "" | "_" | "super" | "self" | "Self" | "crate" | "$crate" | "{{root}}" => false, + _ => true, + } + } +} + +impl !Send for Symbol {} +impl !Sync for Symbol {} + +impl fmt::Debug for Symbol { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + Bridge::with(|bridge| fmt::Debug::fmt(&bridge.symbols[self.0], f)) + } +} + +impl fmt::Display for Symbol { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + Bridge::with(|bridge| fmt::Display::fmt(&bridge.symbols[self.0], f)) + } +} + +impl Encode> for Symbol { + fn encode(self, w: &mut Writer, s: &mut Bridge<'_>) { + s.symbols[self.0][..].encode(w, &mut ()); + } +} + +impl DecodeMut<'_, '_, HandleStore>> + for Marked +{ + fn decode(r: &mut Reader<'_>, s: &mut HandleStore>) -> Self { + Mark::mark(S::intern_symbol(<&str>::decode(r, s))) + } +} + +impl Encode>> + for Marked +{ + fn encode(self, w: &mut Writer, s: &mut HandleStore>) { + S::with_symbol_string(&self.unmark(), |sym| sym.encode(w, s)) + } +} + +impl DecodeMut<'_, '_, Bridge<'_>> for Symbol { + fn decode(r: &mut Reader<'_>, s: &mut Bridge<'_>) -> Self { + Symbol(s.symbols.alloc(<&str>::decode(r, &mut ()))) + } +} + macro_rules! define_client_side { ($($name:ident { $(fn $method:ident($($arg:ident: $arg_ty:ty),* $(,)?) $(-> $ret_ty:ty)*;)* @@ -256,11 +334,11 @@ macro_rules! define_client_side { buf.clear(); api_tags::Method::$name(api_tags::$name::$method).encode(&mut buf, &mut ()); - reverse_encode!(buf; $($arg),*); + reverse_encode!(buf, bridge; $($arg),*); buf = bridge.dispatch.call(buf); - let r = Result::<_, PanicMessage>::decode(&mut &buf[..], &mut ()); + let r = Result::<_, PanicMessage>::decode(&mut &buf[..], bridge); bridge.cached_buffer = buf; @@ -280,6 +358,12 @@ struct Bridge<'a> { /// Server-side function that the client uses to make requests. dispatch: closure::Closure<'a, Buffer, Buffer>, + /// Server-side function to validate and normalize an ident. + validate_ident: extern "C" fn(buffer::Slice<'_>, &mut Buffer) -> bool, + + /// Interned store for storing symbols within the client. + symbols: handle::InternedStore>, + /// Provided context for this macro expansion. context: ExpnContext, } @@ -396,6 +480,8 @@ fn maybe_install_panic_hook(force_show_panics: bool) { }); } +static SYMBOL_COUNTER: AtomicUsize = AtomicUsize::new(1); + /// Client-side helper for handling client panics, entering the bridge, /// deserializing input and serializing output. // FIXME(eddyb) maybe replace `Bridge::enter` with this? @@ -403,7 +489,7 @@ fn run_client DecodeMut<'a, 's, ()>, R: Encode<()>>( config: BridgeConfig<'_>, f: impl FnOnce(A) -> R, ) -> Buffer { - let BridgeConfig { input: mut buf, dispatch, force_show_panics, .. } = config; + let BridgeConfig { input: mut buf, dispatch, validate_ident, force_show_panics, .. } = config; panic::catch_unwind(panic::AssertUnwindSafe(|| { maybe_install_panic_hook(force_show_panics); @@ -412,8 +498,13 @@ fn run_client DecodeMut<'a, 's, ()>, R: Encode<()>>( let (input, context) = <(A, ExpnContext)>::decode(reader, &mut ()); // Put the buffer we used for input back in the `Bridge` for requests. - let new_state = - BridgeState::Connected(Bridge { cached_buffer: buf.take(), dispatch, context }); + let new_state = BridgeState::Connected(Bridge { + cached_buffer: buf.take(), + dispatch, + validate_ident, + symbols: handle::InternedStore::new(&SYMBOL_COUNTER), + context, + }); BRIDGE_STATE.with(|state| { state.set(new_state, || { diff --git a/library/proc_macro/src/bridge/handle.rs b/library/proc_macro/src/bridge/handle.rs index c219a9465d39f..b28adeaef26f7 100644 --- a/library/proc_macro/src/bridge/handle.rs +++ b/library/proc_macro/src/bridge/handle.rs @@ -1,9 +1,11 @@ //! Server-side handles and storage for per-handle data. +use std::borrow::Borrow; use std::collections::{BTreeMap, HashMap}; use std::hash::{BuildHasher, Hash}; use std::num::NonZeroU32; use std::ops::{Index, IndexMut}; +use std::rc::Rc; use std::sync::atomic::{AtomicUsize, Ordering}; pub(super) type Handle = NonZeroU32; @@ -64,13 +66,32 @@ impl BuildHasher for NonRandomState { } } +pub(super) trait FromKey { + fn from_key(key: &Q) -> Self; +} + +impl FromKey for T { + fn from_key(key: &T) -> T { + key.clone() + } +} + +impl FromKey for Rc +where + Rc: for<'a> From<&'a T>, +{ + fn from_key(key: &T) -> Rc { + key.into() + } +} + /// Like `OwnedStore`, but avoids storing any value more than once. pub(super) struct InternedStore { owned: OwnedStore, interner: HashMap, } -impl InternedStore { +impl InternedStore { pub(super) fn new(counter: &'static AtomicUsize) -> Self { InternedStore { owned: OwnedStore::new(counter), @@ -78,12 +99,31 @@ impl InternedStore { } } - pub(super) fn alloc(&mut self, x: T) -> Handle { + pub(super) fn alloc<'a, Q: ?Sized>(&mut self, x: &'a Q) -> Handle + where + T: Borrow + FromKey, + Q: Hash + Eq, + { let owned = &mut self.owned; - *self.interner.entry(x).or_insert_with(|| owned.alloc(x)) + *self + .interner + .raw_entry_mut() + .from_key(x) + .or_insert_with(|| { + let own = T::from_key(x); + (own.clone(), owned.alloc(own)) + }) + .1 } pub(super) fn copy(&mut self, h: Handle) -> T { - self.owned[h] + self.owned[h].clone() + } +} + +impl Index for InternedStore { + type Output = T; + fn index(&self, h: Handle) -> &T { + self.owned.index(h) } } diff --git a/library/proc_macro/src/bridge/mod.rs b/library/proc_macro/src/bridge/mod.rs index 244285cb349d4..c6803ea25a5fe 100644 --- a/library/proc_macro/src/bridge/mod.rs +++ b/library/proc_macro/src/bridge/mod.rs @@ -65,11 +65,11 @@ macro_rules! with_api { fn from_str(src: &str) -> $S::TokenStream; fn to_string($self: &$S::TokenStream) -> String; fn from_token_tree( - tree: TokenTree<$S::TokenStream, $S::Span, $S::Ident, $S::Literal>, + tree: TokenTree<$S::TokenStream, $S::Span, $S::Symbol, $S::Literal>, ) -> $S::TokenStream; fn concat_trees( base: Option<$S::TokenStream>, - trees: Vec>, + trees: Vec>, ) -> $S::TokenStream; fn concat_streams( base: Option<$S::TokenStream>, @@ -77,12 +77,7 @@ macro_rules! with_api { ) -> $S::TokenStream; fn into_iter( $self: $S::TokenStream - ) -> Vec>; - }, - Ident { - fn new(string: &str, span: $S::Span, is_raw: bool) -> $S::Ident; - fn span($self: $S::Ident) -> $S::Span; - fn with_span($self: $S::Ident, span: $S::Span) -> $S::Ident; + ) -> Vec>; }, Literal { fn drop($self: $S::Literal); @@ -153,10 +148,10 @@ macro_rules! with_api { // FIXME(eddyb) this calls `encode` for each argument, but in reverse, // to match the ordering in `reverse_decode`. macro_rules! reverse_encode { - ($writer:ident;) => {}; - ($writer:ident; $first:ident $(, $rest:ident)*) => { - reverse_encode!($writer; $($rest),*); - $first.encode(&mut $writer, &mut ()); + ($writer:ident, $s:ident;) => {}; + ($writer:ident, $s:ident; $first:ident $(, $rest:ident)*) => { + reverse_encode!($writer, $s; $($rest),*); + $first.encode(&mut $writer, $s); } } @@ -205,6 +200,9 @@ pub struct BridgeConfig<'a> { /// Server-side function that the client uses to make requests. dispatch: closure::Closure<'a, Buffer, Buffer>, + /// Server-side function to validate and normalize an ident. + validate_ident: extern "C" fn(buffer::Slice<'_>, &mut Buffer) -> bool, + /// If 'true', always invoke the default panic hook force_show_panics: bool, @@ -436,14 +434,14 @@ macro_rules! compound_traits { } #[derive(Copy, Clone)] -pub struct DelimSpan { - pub open: S, - pub close: S, - pub entire: S, +pub struct DelimSpan { + pub open: Sp, + pub close: Sp, + pub entire: Sp, } -impl DelimSpan { - pub fn from_single(span: S) -> Self { +impl DelimSpan { + pub fn from_single(span: Sp) -> Self { DelimSpan { open: span, close: span, entire: span } } } @@ -451,33 +449,42 @@ impl DelimSpan { compound_traits!(struct DelimSpan { open, close, entire }); #[derive(Clone)] -pub struct Group { +pub struct Group { pub delimiter: Delimiter, pub stream: Option, - pub span: DelimSpan, + pub span: DelimSpan, } compound_traits!(struct Group { delimiter, stream, span }); #[derive(Clone)] -pub struct Punct { +pub struct Punct { pub ch: char, pub joint: bool, - pub span: S, + pub span: Sp, } compound_traits!(struct Punct { ch, joint, span }); +#[derive(Copy, Clone, Eq, PartialEq)] +pub struct Ident { + pub sym: Sy, + pub is_raw: bool, + pub span: Sp, +} + +compound_traits!(struct Ident { sym, is_raw, span }); + #[derive(Clone)] -pub enum TokenTree { - Group(Group), - Punct(Punct), - Ident(I), +pub enum TokenTree { + Group(Group), + Punct(Punct), + Ident(Ident), Literal(L), } compound_traits!( - enum TokenTree { + enum TokenTree { Group(tt), Punct(tt), Ident(tt), @@ -488,10 +495,10 @@ compound_traits!( /// Context provided alongside the initial inputs for a macro expansion. /// Provides values such as spans which are used frequently to avoid RPC. #[derive(Clone)] -struct ExpnContext { - def_site: S, - call_site: S, - mixed_site: S, +struct ExpnContext { + def_site: Sp, + call_site: Sp, + mixed_site: Sp, } compound_traits!( diff --git a/library/proc_macro/src/bridge/server.rs b/library/proc_macro/src/bridge/server.rs index 8fbe010939684..0431354010951 100644 --- a/library/proc_macro/src/bridge/server.rs +++ b/library/proc_macro/src/bridge/server.rs @@ -8,12 +8,12 @@ use super::client::HandleStore; pub trait Types { type FreeFunctions: 'static; type TokenStream: 'static + Clone; - type Ident: 'static + Copy + Eq + Hash; type Literal: 'static + Clone; type SourceFile: 'static + Clone; type MultiSpan: 'static; type Diagnostic: 'static; type Span: 'static + Copy + Eq + Hash; + type Symbol: 'static + Copy + Eq + Hash; } /// Declare an associated fn of one of the traits below, adding necessary @@ -33,6 +33,21 @@ pub trait Context: Types { fn def_site(&mut self) -> Self::Span; fn call_site(&mut self) -> Self::Span; fn mixed_site(&mut self) -> Self::Span; + + /// Check if an identifier is valid, and return `Ok(...)` if it is. + /// + /// May be called on any thread. + /// + /// Returns `Ok(Some(str))` with a normalized version of the identifier if + /// normalization is required, and `Ok(None)` if the existing identifier is + /// already normalized. + fn validate_ident(ident: &str) -> Result, ()>; + + /// Intern a symbol received from RPC + fn intern_symbol(ident: &str) -> Self::Symbol; + + /// Recover the string value of a symbol, and invoke a callback with it. + fn with_symbol_string(symbol: &Self::Symbol, f: impl FnOnce(&str)); } macro_rules! declare_server_traits { @@ -61,6 +76,15 @@ impl Context for MarkedTypes { fn mixed_site(&mut self) -> Self::Span { <_>::mark(Context::mixed_site(&mut self.0)) } + fn validate_ident(ident: &str) -> Result, ()> { + S::validate_ident(ident) + } + fn intern_symbol(ident: &str) -> Self::Symbol { + <_>::mark(S::intern_symbol(ident)) + } + fn with_symbol_string(symbol: &Self::Symbol, f: impl FnOnce(&str)) { + S::with_symbol_string(symbol.unmark(), f) + } } macro_rules! define_mark_types_impls { @@ -69,6 +93,7 @@ macro_rules! define_mark_types_impls { }),* $(,)?) => { impl Types for MarkedTypes { $(type $name = Marked;)* + type Symbol = Marked; } $(impl $name for MarkedTypes { @@ -93,11 +118,16 @@ macro_rules! define_dispatcher_impl { pub trait DispatcherTrait { // HACK(eddyb) these are here to allow `Self::$name` to work below. $(type $name;)* + type Symbol; + fn dispatch(&mut self, buf: Buffer) -> Buffer; + fn validate_ident(ident: &str) -> Result, ()>; } impl DispatcherTrait for Dispatcher> { $(type $name = as Types>::$name;)* + type Symbol = as Types>::Symbol; + fn dispatch(&mut self, mut buf: Buffer) -> Buffer { let Dispatcher { handle_store, server } = self; @@ -127,15 +157,32 @@ macro_rules! define_dispatcher_impl { } buf } + fn validate_ident(ident: &str) -> Result, ()> { + S::validate_ident(ident) + } } } } with_api!(Self, self_, define_dispatcher_impl); +extern "C" fn validate_ident_impl( + string: buffer::Slice<'_>, + normalized: &mut Buffer, +) -> bool { + match std::str::from_utf8(&string[..]).map_err(|_| ()).and_then(D::validate_ident) { + Ok(Some(norm)) => { + *normalized = norm.into_bytes().into(); + true + } + Ok(None) => true, + Err(_) => false, + } +} + pub trait ExecutionStrategy { - fn run_bridge_and_client( + fn run_bridge_and_client( &self, - dispatcher: &mut impl DispatcherTrait, + dispatcher: &mut D, input: Buffer, run_client: extern "C" fn(BridgeConfig<'_>) -> Buffer, force_show_panics: bool, @@ -145,9 +192,9 @@ pub trait ExecutionStrategy { pub struct SameThread; impl ExecutionStrategy for SameThread { - fn run_bridge_and_client( + fn run_bridge_and_client( &self, - dispatcher: &mut impl DispatcherTrait, + dispatcher: &mut D, input: Buffer, run_client: extern "C" fn(BridgeConfig<'_>) -> Buffer, force_show_panics: bool, @@ -157,6 +204,7 @@ impl ExecutionStrategy for SameThread { run_client(BridgeConfig { input, dispatch: (&mut dispatch).into(), + validate_ident: validate_ident_impl::, force_show_panics, _marker: marker::PhantomData, }) @@ -169,9 +217,9 @@ impl ExecutionStrategy for SameThread { pub struct CrossThread1; impl ExecutionStrategy for CrossThread1 { - fn run_bridge_and_client( + fn run_bridge_and_client( &self, - dispatcher: &mut impl DispatcherTrait, + dispatcher: &mut D, input: Buffer, run_client: extern "C" fn(BridgeConfig<'_>) -> Buffer, force_show_panics: bool, @@ -190,6 +238,7 @@ impl ExecutionStrategy for CrossThread1 { run_client(BridgeConfig { input, dispatch: (&mut dispatch).into(), + validate_ident: validate_ident_impl::, force_show_panics, _marker: marker::PhantomData, }) @@ -206,9 +255,9 @@ impl ExecutionStrategy for CrossThread1 { pub struct CrossThread2; impl ExecutionStrategy for CrossThread2 { - fn run_bridge_and_client( + fn run_bridge_and_client( &self, - dispatcher: &mut impl DispatcherTrait, + dispatcher: &mut D, input: Buffer, run_client: extern "C" fn(BridgeConfig<'_>) -> Buffer, force_show_panics: bool, @@ -239,6 +288,7 @@ impl ExecutionStrategy for CrossThread2 { let r = run_client(BridgeConfig { input, dispatch: (&mut dispatch).into(), + validate_ident: validate_ident_impl::, force_show_panics, _marker: marker::PhantomData, }); diff --git a/library/proc_macro/src/lib.rs b/library/proc_macro/src/lib.rs index 68db0ba3dc49b..843842fd28e7a 100644 --- a/library/proc_macro/src/lib.rs +++ b/library/proc_macro/src/lib.rs @@ -28,6 +28,7 @@ #![feature(restricted_std)] #![feature(rustc_attrs)] #![feature(min_specialization)] +#![feature(hash_raw_entry)] #![recursion_limit = "256"] #[unstable(feature = "proc_macro_internals", issue = "27812")] @@ -214,7 +215,7 @@ fn tree_to_bridge_tree( ) -> bridge::TokenTree< bridge::client::TokenStream, bridge::client::Span, - bridge::client::Ident, + bridge::client::Symbol, bridge::client::Literal, > { match tree { @@ -291,7 +292,7 @@ pub mod token_stream { bridge::TokenTree< bridge::client::TokenStream, bridge::client::Span, - bridge::client::Ident, + bridge::client::Symbol, bridge::client::Literal, >, >, @@ -961,7 +962,7 @@ impl PartialEq for char { /// An identifier (`ident`). #[derive(Clone)] #[stable(feature = "proc_macro_lib2", since = "1.29.0")] -pub struct Ident(bridge::client::Ident); +pub struct Ident(bridge::Ident); impl Ident { /// Creates a new `Ident` with the given `string` as well as the specified @@ -985,7 +986,11 @@ impl Ident { /// tokens, requires a `Span` to be specified at construction. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn new(string: &str, span: Span) -> Ident { - Ident(bridge::client::Ident::new(string, span.0, false)) + Ident(bridge::Ident { + sym: bridge::client::Symbol::new_ident(string, false), + is_raw: false, + span: span.0, + }) } /// Same as `Ident::new`, but creates a raw identifier (`r#ident`). @@ -994,29 +999,24 @@ impl Ident { /// (e.g. `self`, `super`) are not supported, and will cause a panic. #[stable(feature = "proc_macro_raw_ident", since = "1.47.0")] pub fn new_raw(string: &str, span: Span) -> Ident { - Ident(bridge::client::Ident::new(string, span.0, true)) + Ident(bridge::Ident { + sym: bridge::client::Symbol::new_ident(string, true), + is_raw: true, + span: span.0, + }) } /// Returns the span of this `Ident`, encompassing the entire string returned - /// by [`to_string`](Self::to_string). + /// by [`to_string`](ToString::to_string). #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn span(&self) -> Span { - Span(self.0.span()) + Span(self.0.span) } /// Configures the span of this `Ident`, possibly changing its hygiene context. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn set_span(&mut self, span: Span) { - self.0 = self.0.with_span(span.0); - } -} - -// N.B., the bridge only provides `to_string`, implement `fmt::Display` -// based on it (the reverse of the usual relationship between the two). -#[stable(feature = "proc_macro_lib", since = "1.15.0")] -impl ToString for Ident { - fn to_string(&self) -> String { - TokenStream::from(TokenTree::from(self.clone())).to_string() + self.0.span = span.0; } } @@ -1025,7 +1025,7 @@ impl ToString for Ident { #[stable(feature = "proc_macro_lib2", since = "1.29.0")] impl fmt::Display for Ident { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(&self.to_string()) + if self.0.is_raw { write!(f, "r#{}", self.0.sym) } else { write!(f, "{}", self.0.sym) } } } diff --git a/src/test/ui/proc-macro/invalid-punct-ident-2.rs b/src/test/ui/proc-macro/invalid-punct-ident-2.rs index 04a0a8733115a..465cae3aeaf9c 100644 --- a/src/test/ui/proc-macro/invalid-punct-ident-2.rs +++ b/src/test/ui/proc-macro/invalid-punct-ident-2.rs @@ -2,7 +2,7 @@ // rustc-env:RUST_BACKTRACE=0 // FIXME https://github.com/rust-lang/rust/issues/59998 -// normalize-stderr-test "thread.*panicked.*proc_macro_server.rs.*\n" -> "" +// normalize-stderr-test "thread.*panicked.*proc_macro.*client.rs.*\n" -> "" // normalize-stderr-test "note:.*RUST_BACKTRACE=1.*\n" -> "" // normalize-stderr-test "\nerror: internal compiler error.*\n\n" -> "" // normalize-stderr-test "note:.*unexpectedly panicked.*\n\n" -> "" diff --git a/src/test/ui/proc-macro/invalid-punct-ident-3.rs b/src/test/ui/proc-macro/invalid-punct-ident-3.rs index aebba341625ae..7f53b1f0027db 100644 --- a/src/test/ui/proc-macro/invalid-punct-ident-3.rs +++ b/src/test/ui/proc-macro/invalid-punct-ident-3.rs @@ -2,7 +2,7 @@ // rustc-env:RUST_BACKTRACE=0 // FIXME https://github.com/rust-lang/rust/issues/59998 -// normalize-stderr-test "thread.*panicked.*proc_macro_server.rs.*\n" -> "" +// normalize-stderr-test "thread.*panicked.*proc_macro.*client.rs.*\n" -> "" // normalize-stderr-test "note:.*RUST_BACKTRACE=1.*\n" -> "" // normalize-stderr-test "\nerror: internal compiler error.*\n\n" -> "" // normalize-stderr-test "note:.*unexpectedly panicked.*\n\n" -> "" From dd916c7880d9fbc9a3afd4412564bc60a764404e Mon Sep 17 00:00:00 2001 From: Nika Layzell Date: Fri, 2 Jul 2021 00:40:26 -0400 Subject: [PATCH 09/11] proc_macro: stop using a remote object handle for Literal This builds on the symbol infrastructure built for ident to replicate the `LitKind` and `Lit` structures in rustc within the `proc_macro` client, allowing literals to be fully created and interacted with from the client thread. Only parsing and subspan operations still require sync RPC. --- .../rustc_expand/src/proc_macro_server.rs | 309 ++++++++---------- library/proc_macro/src/bridge/client.rs | 25 +- library/proc_macro/src/bridge/mod.rs | 76 +++-- library/proc_macro/src/bridge/rpc.rs | 1 + library/proc_macro/src/bridge/server.rs | 1 - library/proc_macro/src/lib.rs | 115 +++++-- 6 files changed, 277 insertions(+), 250 deletions(-) diff --git a/compiler/rustc_expand/src/proc_macro_server.rs b/compiler/rustc_expand/src/proc_macro_server.rs index 0a9952f773cef..edf0d8812d45e 100644 --- a/compiler/rustc_expand/src/proc_macro_server.rs +++ b/compiler/rustc_expand/src/proc_macro_server.rs @@ -13,9 +13,8 @@ use rustc_span::def_id::CrateNum; use rustc_span::symbol::{self, sym, Symbol}; use rustc_span::{BytePos, FileName, Pos, SourceFile, Span}; -use pm::bridge::{server, DelimSpan, Group, Ident, Punct, TokenTree}; +use pm::bridge::{server, DelimSpan, Group, Ident, LitKind, Literal, Punct, TokenTree}; use pm::{Delimiter, Level, LineColumn}; -use std::ascii; use std::ops::Bound; trait FromInternal { @@ -48,9 +47,40 @@ impl ToInternal for Delimiter { } } -impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> - for Vec> -{ +impl FromInternal for LitKind { + fn from_internal(kind: token::LitKind) -> Self { + match kind { + token::Byte => LitKind::Byte, + token::Char => LitKind::Char, + token::Integer => LitKind::Integer, + token::Float => LitKind::Float, + token::Str => LitKind::Str, + token::StrRaw(n) => LitKind::StrRaw(n), + token::ByteStr => LitKind::ByteStr, + token::ByteStrRaw(n) => LitKind::ByteStrRaw(n), + token::Err => LitKind::Err, + token::Bool => unreachable!(), + } + } +} + +impl ToInternal for LitKind { + fn to_internal(self) -> token::LitKind { + match self { + LitKind::Byte => token::Byte, + LitKind::Char => token::Char, + LitKind::Integer => token::Integer, + LitKind::Float => token::Float, + LitKind::Str => token::Str, + LitKind::StrRaw(n) => token::StrRaw(n), + LitKind::ByteStr => token::ByteStr, + LitKind::ByteStrRaw(n) => token::ByteStrRaw(n), + LitKind::Err => token::Err, + } + } +} + +impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec> { fn from_internal((stream, rustc): (TokenStream, &mut Rustc<'_, '_>)) -> Self { use rustc_ast::token::*; @@ -154,7 +184,9 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> tt!(Punct { ch: '\'', joint: true }); tt!(Ident { sym: ident.name, is_raw: false }); } - Literal(lit) => tt!(Literal { lit }), + Literal(token::Lit { kind, symbol, suffix }) => { + tt!(Literal { kind: FromInternal::from_internal(kind), symbol, suffix }); + } DocComment(_, attr_style, data) => { let mut escaped = String::new(); for ch in data.as_str().chars() { @@ -204,7 +236,7 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> } } -impl ToInternal for TokenTree { +impl ToInternal for TokenTree { fn to_internal(self) -> TokenStream { use rustc_ast::token::*; @@ -222,7 +254,9 @@ impl ToInternal for TokenTree { return tokenstream::TokenTree::token(Ident(sym, is_raw), span).into(); } TokenTree::Literal(self::Literal { - lit: token::Lit { kind: token::Integer, symbol, suffix }, + kind: self::LitKind::Integer, + symbol, + suffix, span, }) if symbol.as_str().starts_with('-') => { let minus = BinOp(BinOpToken::Minus); @@ -233,7 +267,9 @@ impl ToInternal for TokenTree { return [a, b].into_iter().collect(); } TokenTree::Literal(self::Literal { - lit: token::Lit { kind: token::Float, symbol, suffix }, + kind: self::LitKind::Float, + symbol, + suffix, span, }) if symbol.as_str().starts_with('-') => { let minus = BinOp(BinOpToken::Minus); @@ -243,8 +279,12 @@ impl ToInternal for TokenTree { let b = tokenstream::TokenTree::token(float, span); return [a, b].into_iter().collect(); } - TokenTree::Literal(self::Literal { lit, span }) => { - return tokenstream::TokenTree::token(Literal(lit), span).into(); + TokenTree::Literal(self::Literal { kind, symbol, suffix, span }) => { + return tokenstream::TokenTree::token( + TokenKind::lit(kind.to_internal(), symbol, suffix), + span, + ) + .into(); } }; @@ -293,13 +333,6 @@ impl ToInternal for Level { pub struct FreeFunctions; -// FIXME(eddyb) `Literal` should not expose internal `Debug` impls. -#[derive(Clone, Debug)] -pub struct Literal { - lit: token::Lit, - span: Span, -} - pub(crate) struct Rustc<'a, 'b> { ecx: &'a mut ExtCtxt<'b>, def_site: Span, @@ -325,19 +358,11 @@ impl<'a, 'b> Rustc<'a, 'b> { fn sess(&self) -> &ParseSess { self.ecx.parse_sess() } - - fn lit(&mut self, kind: token::LitKind, symbol: Symbol, suffix: Option) -> Literal { - Literal { - lit: token::Lit::new(kind, symbol, suffix), - span: server::Context::call_site(self), - } - } } impl server::Types for Rustc<'_, '_> { type FreeFunctions = FreeFunctions; type TokenStream = TokenStream; - type Literal = Literal; type SourceFile = Lrc; type MultiSpan = Vec; type Diagnostic = Diagnostic; @@ -356,6 +381,94 @@ impl server::FreeFunctions for Rustc<'_, '_> { fn track_path(&mut self, path: &str) { self.sess().file_depinfo.borrow_mut().insert(Symbol::intern(path)); } + + fn literal_from_str(&mut self, s: &str) -> Result, ()> { + let name = FileName::proc_macro_source_code(s); + let mut parser = rustc_parse::new_parser_from_source_str(self.sess(), name, s.to_owned()); + + let first_span = parser.token.span.data(); + let minus_present = parser.eat(&token::BinOp(token::Minus)); + + let lit_span = parser.token.span.data(); + let token::Literal(mut lit) = parser.token.kind else { + return Err(()); + }; + + // Check no comment or whitespace surrounding the (possibly negative) + // literal, or more tokens after it. + if (lit_span.hi.0 - first_span.lo.0) as usize != s.len() { + return Err(()); + } + + if minus_present { + // If minus is present, check no comment or whitespace in between it + // and the literal token. + if first_span.hi.0 != lit_span.lo.0 { + return Err(()); + } + + // Check literal is a kind we allow to be negated in a proc macro token. + match lit.kind { + token::LitKind::Bool + | token::LitKind::Byte + | token::LitKind::Char + | token::LitKind::Str + | token::LitKind::StrRaw(_) + | token::LitKind::ByteStr + | token::LitKind::ByteStrRaw(_) + | token::LitKind::Err => return Err(()), + token::LitKind::Integer | token::LitKind::Float => {} + } + + // Synthesize a new symbol that includes the minus sign. + let symbol = Symbol::intern(&s[..1 + lit.symbol.as_str().len()]); + lit = token::Lit::new(lit.kind, symbol, lit.suffix); + } + let token::Lit { kind, symbol, suffix } = lit; + Ok(Literal { + kind: FromInternal::from_internal(kind), + symbol, + suffix, + span: self.call_site, + }) + } + + fn literal_subspan( + &mut self, + literal: Literal, + start: Bound, + end: Bound, + ) -> Option { + let span = literal.span; + let length = span.hi().to_usize() - span.lo().to_usize(); + + let start = match start { + Bound::Included(lo) => lo, + Bound::Excluded(lo) => lo.checked_add(1)?, + Bound::Unbounded => 0, + }; + + let end = match end { + Bound::Included(hi) => hi.checked_add(1)?, + Bound::Excluded(hi) => hi, + Bound::Unbounded => length, + }; + + // Bounds check the values, preventing addition overflow and OOB spans. + if start > u32::MAX as usize + || end > u32::MAX as usize + || (u32::MAX - start as u32) < span.lo().to_u32() + || (u32::MAX - end as u32) < span.lo().to_u32() + || start >= end + || end > length + { + return None; + } + + let new_lo = span.lo() + BytePos::from_usize(start); + let new_hi = span.lo() + BytePos::from_usize(end); + Some(span.with_lo(new_lo).with_hi(new_hi)) + } } impl server::TokenStream for Rustc<'_, '_> { @@ -425,14 +538,14 @@ impl server::TokenStream for Rustc<'_, '_> { } fn from_token_tree( &mut self, - tree: TokenTree, + tree: TokenTree, ) -> Self::TokenStream { tree.to_internal() } fn concat_trees( &mut self, base: Option, - trees: Vec>, + trees: Vec>, ) -> Self::TokenStream { let mut builder = tokenstream::TokenStreamBuilder::new(); if let Some(base) = base { @@ -460,149 +573,11 @@ impl server::TokenStream for Rustc<'_, '_> { fn into_iter( &mut self, stream: Self::TokenStream, - ) -> Vec> { + ) -> Vec> { FromInternal::from_internal((stream, self)) } } -impl server::Literal for Rustc<'_, '_> { - fn from_str(&mut self, s: &str) -> Result { - let name = FileName::proc_macro_source_code(s); - let mut parser = rustc_parse::new_parser_from_source_str(self.sess(), name, s.to_owned()); - - let first_span = parser.token.span.data(); - let minus_present = parser.eat(&token::BinOp(token::Minus)); - - let lit_span = parser.token.span.data(); - let token::Literal(mut lit) = parser.token.kind else { - return Err(()); - }; - - // Check no comment or whitespace surrounding the (possibly negative) - // literal, or more tokens after it. - if (lit_span.hi.0 - first_span.lo.0) as usize != s.len() { - return Err(()); - } - - if minus_present { - // If minus is present, check no comment or whitespace in between it - // and the literal token. - if first_span.hi.0 != lit_span.lo.0 { - return Err(()); - } - - // Check literal is a kind we allow to be negated in a proc macro token. - match lit.kind { - token::LitKind::Bool - | token::LitKind::Byte - | token::LitKind::Char - | token::LitKind::Str - | token::LitKind::StrRaw(_) - | token::LitKind::ByteStr - | token::LitKind::ByteStrRaw(_) - | token::LitKind::Err => return Err(()), - token::LitKind::Integer | token::LitKind::Float => {} - } - - // Synthesize a new symbol that includes the minus sign. - let symbol = Symbol::intern(&s[..1 + lit.symbol.as_str().len()]); - lit = token::Lit::new(lit.kind, symbol, lit.suffix); - } - - Ok(Literal { lit, span: self.call_site }) - } - fn to_string(&mut self, literal: &Self::Literal) -> String { - literal.lit.to_string() - } - fn debug_kind(&mut self, literal: &Self::Literal) -> String { - format!("{:?}", literal.lit.kind) - } - fn symbol(&mut self, literal: &Self::Literal) -> String { - literal.lit.symbol.to_string() - } - fn suffix(&mut self, literal: &Self::Literal) -> Option { - literal.lit.suffix.as_ref().map(Symbol::to_string) - } - fn integer(&mut self, n: &str) -> Self::Literal { - self.lit(token::Integer, Symbol::intern(n), None) - } - fn typed_integer(&mut self, n: &str, kind: &str) -> Self::Literal { - self.lit(token::Integer, Symbol::intern(n), Some(Symbol::intern(kind))) - } - fn float(&mut self, n: &str) -> Self::Literal { - self.lit(token::Float, Symbol::intern(n), None) - } - fn f32(&mut self, n: &str) -> Self::Literal { - self.lit(token::Float, Symbol::intern(n), Some(sym::f32)) - } - fn f64(&mut self, n: &str) -> Self::Literal { - self.lit(token::Float, Symbol::intern(n), Some(sym::f64)) - } - fn string(&mut self, string: &str) -> Self::Literal { - let quoted = format!("{:?}", string); - assert!(quoted.starts_with('"') && quoted.ends_with('"')); - let symbol = "ed[1..quoted.len() - 1]; - self.lit(token::Str, Symbol::intern(symbol), None) - } - fn character(&mut self, ch: char) -> Self::Literal { - let quoted = format!("{:?}", ch); - assert!(quoted.starts_with('\'') && quoted.ends_with('\'')); - let symbol = "ed[1..quoted.len() - 1]; - self.lit(token::Char, Symbol::intern(symbol), None) - } - fn byte_string(&mut self, bytes: &[u8]) -> Self::Literal { - let string = bytes - .iter() - .cloned() - .flat_map(ascii::escape_default) - .map(Into::::into) - .collect::(); - self.lit(token::ByteStr, Symbol::intern(&string), None) - } - fn span(&mut self, literal: &Self::Literal) -> Self::Span { - literal.span - } - fn set_span(&mut self, literal: &mut Self::Literal, span: Self::Span) { - literal.span = span; - } - fn subspan( - &mut self, - literal: &Self::Literal, - start: Bound, - end: Bound, - ) -> Option { - let span = literal.span; - let length = span.hi().to_usize() - span.lo().to_usize(); - - let start = match start { - Bound::Included(lo) => lo, - Bound::Excluded(lo) => lo.checked_add(1)?, - Bound::Unbounded => 0, - }; - - let end = match end { - Bound::Included(hi) => hi.checked_add(1)?, - Bound::Excluded(hi) => hi, - Bound::Unbounded => length, - }; - - // Bounds check the values, preventing addition overflow and OOB spans. - if start > u32::MAX as usize - || end > u32::MAX as usize - || (u32::MAX - start as u32) < span.lo().to_u32() - || (u32::MAX - end as u32) < span.lo().to_u32() - || start >= end - || end > length - { - return None; - } - - let new_lo = span.lo() + BytePos::from_usize(start); - let new_hi = span.lo() + BytePos::from_usize(end); - Some(span.with_lo(new_lo).with_hi(new_hi)) - } -} - impl server::SourceFile for Rustc<'_, '_> { fn eq(&mut self, file1: &Self::SourceFile, file2: &Self::SourceFile) -> bool { Lrc::ptr_eq(file1, file2) diff --git a/library/proc_macro/src/bridge/client.rs b/library/proc_macro/src/bridge/client.rs index eb6e351095b69..911140b44fcd2 100644 --- a/library/proc_macro/src/bridge/client.rs +++ b/library/proc_macro/src/bridge/client.rs @@ -179,7 +179,6 @@ define_handles! { 'owned: FreeFunctions, TokenStream, - Literal, SourceFile, MultiSpan, Diagnostic, @@ -200,25 +199,6 @@ impl Clone for TokenStream { } } -impl Clone for Literal { - fn clone(&self) -> Self { - self.clone() - } -} - -impl fmt::Debug for Literal { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("Literal") - // format the kind without quotes, as in `kind: Float` - .field("kind", &format_args!("{}", &self.debug_kind())) - .field("symbol", &self.symbol()) - // format `Some("...")` on one line even in {:#?} mode - .field("suffix", &format_args!("{:?}", &self.suffix())) - .field("span", &self.span()) - .finish() - } -} - impl Clone for SourceFile { fn clone(&self) -> Self { self.clone() @@ -250,6 +230,11 @@ impl fmt::Debug for Span { pub(crate) struct Symbol(handle::Handle); impl Symbol { + /// Intern a new `Symbol` + pub(crate) fn new(string: &str) -> Self { + Symbol(Bridge::with(|bridge| bridge.symbols.alloc(string))) + } + /// Create a new `Symbol` for an identifier. /// /// Validates and normalizes before converting it to a symbol. diff --git a/library/proc_macro/src/bridge/mod.rs b/library/proc_macro/src/bridge/mod.rs index c6803ea25a5fe..8c1088f3596b2 100644 --- a/library/proc_macro/src/bridge/mod.rs +++ b/library/proc_macro/src/bridge/mod.rs @@ -56,6 +56,8 @@ macro_rules! with_api { fn drop($self: $S::FreeFunctions); fn track_env_var(var: &str, value: Option<&str>); fn track_path(path: &str); + fn literal_from_str(s: &str) -> Result, ()>; + fn literal_subspan(lit: Literal<$S::Span, $S::Symbol>, start: Bound, end: Bound) -> Option<$S::Span>; }, TokenStream { fn drop($self: $S::TokenStream); @@ -65,11 +67,11 @@ macro_rules! with_api { fn from_str(src: &str) -> $S::TokenStream; fn to_string($self: &$S::TokenStream) -> String; fn from_token_tree( - tree: TokenTree<$S::TokenStream, $S::Span, $S::Symbol, $S::Literal>, + tree: TokenTree<$S::TokenStream, $S::Span, $S::Symbol>, ) -> $S::TokenStream; fn concat_trees( base: Option<$S::TokenStream>, - trees: Vec>, + trees: Vec>, ) -> $S::TokenStream; fn concat_streams( base: Option<$S::TokenStream>, @@ -77,31 +79,7 @@ macro_rules! with_api { ) -> $S::TokenStream; fn into_iter( $self: $S::TokenStream - ) -> Vec>; - }, - Literal { - fn drop($self: $S::Literal); - fn clone($self: &$S::Literal) -> $S::Literal; - fn from_str(s: &str) -> Result<$S::Literal, ()>; - fn to_string($self: &$S::Literal) -> String; - fn debug_kind($self: &$S::Literal) -> String; - fn symbol($self: &$S::Literal) -> String; - fn suffix($self: &$S::Literal) -> Option; - fn integer(n: &str) -> $S::Literal; - fn typed_integer(n: &str, kind: &str) -> $S::Literal; - fn float(n: &str) -> $S::Literal; - fn f32(n: &str) -> $S::Literal; - fn f64(n: &str) -> $S::Literal; - fn string(string: &str) -> $S::Literal; - fn character(ch: char) -> $S::Literal; - fn byte_string(bytes: &[u8]) -> $S::Literal; - fn span($self: &$S::Literal) -> $S::Span; - fn set_span($self: &mut $S::Literal, span: $S::Span); - fn subspan( - $self: &$S::Literal, - start: Bound, - end: Bound, - ) -> Option<$S::Span>; + ) -> Vec>; }, SourceFile { fn drop($self: $S::SourceFile); @@ -351,6 +329,7 @@ mark_noop! { String, usize, Delimiter, + LitKind, Level, LineColumn, Spacing, @@ -381,6 +360,33 @@ rpc_encode_decode!( } ); +#[derive(Copy, Clone, Eq, PartialEq, Debug)] +pub enum LitKind { + Byte, + Char, + Integer, + Float, + Str, + StrRaw(u8), + ByteStr, + ByteStrRaw(u8), + Err, +} + +rpc_encode_decode!( + enum LitKind { + Byte, + Char, + Integer, + Float, + Str, + StrRaw(n), + ByteStr, + ByteStrRaw(n), + Err, + } +); + macro_rules! mark_compound { (struct $name:ident <$($T:ident),+> { $($field:ident),* $(,)? }) => { impl<$($T: Mark),+> Mark for $name <$($T),+> { @@ -475,16 +481,26 @@ pub struct Ident { compound_traits!(struct Ident { sym, is_raw, span }); +#[derive(Clone, Eq, PartialEq)] +pub struct Literal { + pub kind: LitKind, + pub symbol: Sy, + pub suffix: Option, + pub span: Sp, +} + +compound_traits!(struct Literal { kind, symbol, suffix, span }); + #[derive(Clone)] -pub enum TokenTree { +pub enum TokenTree { Group(Group), Punct(Punct), Ident(Ident), - Literal(L), + Literal(Literal), } compound_traits!( - enum TokenTree { + enum TokenTree { Group(tt), Punct(tt), Ident(tt), diff --git a/library/proc_macro/src/bridge/rpc.rs b/library/proc_macro/src/bridge/rpc.rs index a94334e07362e..b3ffc66e5621d 100644 --- a/library/proc_macro/src/bridge/rpc.rs +++ b/library/proc_macro/src/bridge/rpc.rs @@ -128,6 +128,7 @@ impl DecodeMut<'_, '_, S> for u8 { } } +rpc_encode_decode!(le u16); rpc_encode_decode!(le u32); rpc_encode_decode!(le usize); diff --git a/library/proc_macro/src/bridge/server.rs b/library/proc_macro/src/bridge/server.rs index 0431354010951..920ba835773bb 100644 --- a/library/proc_macro/src/bridge/server.rs +++ b/library/proc_macro/src/bridge/server.rs @@ -8,7 +8,6 @@ use super::client::HandleStore; pub trait Types { type FreeFunctions: 'static; type TokenStream: 'static + Clone; - type Literal: 'static + Clone; type SourceFile: 'static + Clone; type MultiSpan: 'static; type Diagnostic: 'static; diff --git a/library/proc_macro/src/lib.rs b/library/proc_macro/src/lib.rs index 843842fd28e7a..5513038b71dc3 100644 --- a/library/proc_macro/src/lib.rs +++ b/library/proc_macro/src/lib.rs @@ -212,12 +212,7 @@ pub use quote::{quote, quote_span}; fn tree_to_bridge_tree( tree: TokenTree, -) -> bridge::TokenTree< - bridge::client::TokenStream, - bridge::client::Span, - bridge::client::Symbol, - bridge::client::Literal, -> { +) -> bridge::TokenTree { match tree { TokenTree::Group(tt) => bridge::TokenTree::Group(tt.0), TokenTree::Punct(tt) => bridge::TokenTree::Punct(tt.0), @@ -293,7 +288,6 @@ pub mod token_stream { bridge::client::TokenStream, bridge::client::Span, bridge::client::Symbol, - bridge::client::Literal, >, >, ); @@ -1045,7 +1039,7 @@ impl fmt::Debug for Ident { /// Boolean literals like `true` and `false` do not belong here, they are `Ident`s. #[derive(Clone)] #[stable(feature = "proc_macro_lib2", since = "1.29.0")] -pub struct Literal(bridge::client::Literal); +pub struct Literal(bridge::Literal); macro_rules! suffixed_int_literals { ($($name:ident => $kind:ident,)*) => ($( @@ -1062,7 +1056,12 @@ macro_rules! suffixed_int_literals { /// below. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn $name(n: $kind) -> Literal { - Literal(bridge::client::Literal::typed_integer(&n.to_string(), stringify!($kind))) + Literal(bridge::Literal { + kind: bridge::LitKind::Integer, + symbol: bridge::client::Symbol::new(&n.to_string()), + suffix: Some(bridge::client::Symbol::new(stringify!($kind))), + span: Span::call_site().0, + }) } )*) } @@ -1084,12 +1083,26 @@ macro_rules! unsuffixed_int_literals { /// below. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn $name(n: $kind) -> Literal { - Literal(bridge::client::Literal::integer(&n.to_string())) + Literal(bridge::Literal { + kind: bridge::LitKind::Integer, + symbol: bridge::client::Symbol::new(&n.to_string()), + suffix: None, + span: Span::call_site().0, + }) } )*) } impl Literal { + fn new(kind: bridge::LitKind, value: &str, suffix: Option<&str>) -> Self { + Literal(bridge::Literal { + kind, + symbol: bridge::client::Symbol::new(value), + suffix: suffix.map(bridge::client::Symbol::new), + span: Span::call_site().0, + }) + } + suffixed_int_literals! { u8_suffixed => u8, u16_suffixed => u16, @@ -1141,7 +1154,7 @@ impl Literal { if !repr.contains('.') { repr.push_str(".0"); } - Literal(bridge::client::Literal::float(&repr)) + Literal::new(bridge::LitKind::Float, &repr, None) } /// Creates a new suffixed floating-point literal. @@ -1162,7 +1175,7 @@ impl Literal { if !n.is_finite() { panic!("Invalid float literal {n}"); } - Literal(bridge::client::Literal::f32(&n.to_string())) + Literal::new(bridge::LitKind::Float, &n.to_string(), Some("f32")) } /// Creates a new unsuffixed floating-point literal. @@ -1186,7 +1199,7 @@ impl Literal { if !repr.contains('.') { repr.push_str(".0"); } - Literal(bridge::client::Literal::float(&repr)) + Literal::new(bridge::LitKind::Float, &repr, None) } /// Creates a new suffixed floating-point literal. @@ -1207,37 +1220,49 @@ impl Literal { if !n.is_finite() { panic!("Invalid float literal {n}"); } - Literal(bridge::client::Literal::f64(&n.to_string())) + Literal::new(bridge::LitKind::Float, &n.to_string(), Some("f64")) } /// String literal. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn string(string: &str) -> Literal { - Literal(bridge::client::Literal::string(string)) + let quoted = format!("{:?}", string); + assert!(quoted.starts_with('"') && quoted.ends_with('"')); + let symbol = "ed[1..quoted.len() - 1]; + Literal::new(bridge::LitKind::Str, symbol, None) } /// Character literal. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn character(ch: char) -> Literal { - Literal(bridge::client::Literal::character(ch)) + let quoted = format!("{:?}", ch); + assert!(quoted.starts_with('\'') && quoted.ends_with('\'')); + let symbol = "ed[1..quoted.len() - 1]; + Literal::new(bridge::LitKind::Char, symbol, None) } /// Byte string literal. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn byte_string(bytes: &[u8]) -> Literal { - Literal(bridge::client::Literal::byte_string(bytes)) + let string = bytes + .iter() + .cloned() + .flat_map(std::ascii::escape_default) + .map(Into::::into) + .collect::(); + Literal::new(bridge::LitKind::ByteStr, &string, None) } /// Returns the span encompassing this literal. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn span(&self) -> Span { - Span(self.0.span()) + Span(self.0.span) } /// Configures the span associated for this literal. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn set_span(&mut self, span: Span) { - self.0.set_span(span.0); + self.0.span = span.0; } /// Returns a `Span` that is a subset of `self.span()` containing only the @@ -1253,7 +1278,12 @@ impl Literal { // was 'c' or whether it was '\u{63}'. #[unstable(feature = "proc_macro_span", issue = "54725")] pub fn subspan>(&self, range: R) -> Option { - self.0.subspan(range.start_bound().cloned(), range.end_bound().cloned()).map(Span) + bridge::client::FreeFunctions::literal_subspan( + self.0.clone(), + range.start_bound().cloned(), + range.end_bound().cloned(), + ) + .map(Span) } } @@ -1272,35 +1302,56 @@ impl FromStr for Literal { type Err = LexError; fn from_str(src: &str) -> Result { - match bridge::client::Literal::from_str(src) { + match bridge::client::FreeFunctions::literal_from_str(src) { Ok(literal) => Ok(Literal(literal)), Err(()) => Err(LexError), } } } -// N.B., the bridge only provides `to_string`, implement `fmt::Display` -// based on it (the reverse of the usual relationship between the two). -#[stable(feature = "proc_macro_lib", since = "1.15.0")] -impl ToString for Literal { - fn to_string(&self) -> String { - self.0.to_string() - } -} - /// Prints the literal as a string that should be losslessly convertible /// back into the same literal (except for possible rounding for floating point literals). #[stable(feature = "proc_macro_lib2", since = "1.29.0")] impl fmt::Display for Literal { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(&self.to_string()) + // Based on `literal_to_string` from `pprust/state.rs` + match self.0.kind { + bridge::LitKind::Byte => write!(f, "b'{}'", self.0.symbol)?, + bridge::LitKind::Char => write!(f, "'{}'", self.0.symbol)?, + bridge::LitKind::Str => write!(f, "\"{}\"", self.0.symbol)?, + bridge::LitKind::StrRaw(n) => write!( + f, + "r{delim}\"{string}\"{delim}", + delim = "#".repeat(n as usize), + string = self.0.symbol + )?, + bridge::LitKind::ByteStr => write!(f, "b\"{}\"", self.0.symbol)?, + bridge::LitKind::ByteStrRaw(n) => write!( + f, + "br{delim}\"{string}\"{delim}", + delim = "#".repeat(n as usize), + string = self.0.symbol + )?, + _ => write!(f, "{}", self.0.symbol)?, + } + if let Some(suffix) = self.0.suffix { + write!(f, "{}", suffix)?; + } + Ok(()) } } #[stable(feature = "proc_macro_lib2", since = "1.29.0")] impl fmt::Debug for Literal { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.0.fmt(f) + f.debug_struct("Literal") + // format the kind on one line even in {:#?} mode + .field("kind", &format_args!("{:?}", &self.0.kind)) + .field("symbol", &self.0.symbol) + // format `Some("...")` on one line even in {:#?} mode + .field("suffix", &format_args!("{:?}", &self.0.suffix)) + .field("span", &self.0.span) + .finish() } } From 6a0962f16810aadb6618c31e202c9d8f740575b5 Mon Sep 17 00:00:00 2001 From: Nika Layzell Date: Sat, 3 Jul 2021 11:02:40 -0400 Subject: [PATCH 10/11] add idents to the symbol gallery when converting to internal --- compiler/rustc_expand/src/proc_macro_server.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/compiler/rustc_expand/src/proc_macro_server.rs b/compiler/rustc_expand/src/proc_macro_server.rs index edf0d8812d45e..d7d6a85c5ac8f 100644 --- a/compiler/rustc_expand/src/proc_macro_server.rs +++ b/compiler/rustc_expand/src/proc_macro_server.rs @@ -236,11 +236,12 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec for TokenTree { +impl ToInternal for (TokenTree, &mut Rustc<'_, '_>) { fn to_internal(self) -> TokenStream { use rustc_ast::token::*; - let (ch, joint, span) = match self { + let (tree, rustc) = self; + let (ch, joint, span) = match tree { TokenTree::Punct(Punct { ch, joint, span }) => (ch, joint, span), TokenTree::Group(Group { delimiter, stream, span: DelimSpan { open, close, .. } }) => { return tokenstream::TokenTree::Delimited( @@ -251,6 +252,7 @@ impl ToInternal for TokenTree { .into(); } TokenTree::Ident(self::Ident { sym, is_raw, span }) => { + rustc.sess().symbol_gallery.insert(sym, span); return tokenstream::TokenTree::token(Ident(sym, is_raw), span).into(); } TokenTree::Literal(self::Literal { @@ -540,7 +542,7 @@ impl server::TokenStream for Rustc<'_, '_> { &mut self, tree: TokenTree, ) -> Self::TokenStream { - tree.to_internal() + (tree, self).to_internal() } fn concat_trees( &mut self, @@ -552,7 +554,7 @@ impl server::TokenStream for Rustc<'_, '_> { builder.push(base); } for tree in trees { - builder.push(tree.to_internal()); + builder.push((tree, &mut *self).to_internal()); } builder.build() } From fea1d01e36e5cf7c42a3cd1a5ed3db631b09229a Mon Sep 17 00:00:00 2001 From: Nika Layzell Date: Sun, 15 May 2022 13:46:33 -0400 Subject: [PATCH 11/11] Try to reduce codegen complexity of TokenStream's FromIterator and Extend impls This is an experimental patch to try to reduce the codegen complexity of TokenStream's FromIterator and Extend implementations for downstream crates, by moving the core logic into a helper type. This might help improve build performance of crates which depend on proc_macro as iterators are used less, and the compiler may take less time to do things like attempt specializations or other iterator optimizations. The change intentionally sacrifices some optimization opportunities, such as using the specializations for collecting iterators derived from Vec::into_iter() into Vec. This is one of the simpler potential approaches to reducing the amount of code generated in crates depending on proc_macro, so it seems worth trying before other more-involved changes. --- library/proc_macro/src/lib.rs | 107 +++++++++++++++++++++++++++++----- 1 file changed, 91 insertions(+), 16 deletions(-) diff --git a/library/proc_macro/src/lib.rs b/library/proc_macro/src/lib.rs index 5513038b71dc3..042e70df1fe1e 100644 --- a/library/proc_macro/src/lib.rs +++ b/library/proc_macro/src/lib.rs @@ -229,14 +229,89 @@ impl From for TokenStream { } } +/// Non-generic helper for implementing `FromIterator` and +/// `Extend` with less monomorphization in calling crates. +struct ExtendStreamWithTreesHelper { + trees: Vec< + bridge::TokenTree< + bridge::client::TokenStream, + bridge::client::Span, + bridge::client::Symbol, + >, + >, +} + +impl ExtendStreamWithTreesHelper { + fn new(capacity: usize) -> Self { + ExtendStreamWithTreesHelper { trees: Vec::with_capacity(capacity) } + } + + fn push(&mut self, tree: TokenTree) { + self.trees.push(tree_to_bridge_tree(tree)); + } + + fn build(self) -> TokenStream { + if self.trees.is_empty() { + TokenStream(None) + } else { + TokenStream(Some(bridge::client::TokenStream::concat_trees(None, self.trees))) + } + } + + fn extend(self, stream: &mut TokenStream) { + if self.trees.is_empty() { + return; + } + stream.0 = Some(bridge::client::TokenStream::concat_trees(stream.0.take(), self.trees)) + } +} + +/// Non-generic helper for implementing `FromIterator` and +/// `Extend` with less monomorphization in calling crates. +struct ExtendStreamWithStreamsHelper { + streams: Vec, +} + +impl ExtendStreamWithStreamsHelper { + fn new(capacity: usize) -> Self { + ExtendStreamWithStreamsHelper { streams: Vec::with_capacity(capacity) } + } + + fn push(&mut self, stream: TokenStream) { + if let Some(stream) = stream.0 { + self.streams.push(stream); + } + } + + fn build(mut self) -> TokenStream { + if self.streams.len() <= 1 { + TokenStream(self.streams.pop()) + } else { + TokenStream(Some(bridge::client::TokenStream::concat_streams(None, self.streams))) + } + } + + fn extend(mut self, stream: &mut TokenStream) { + if self.streams.is_empty() { + return; + } + let base = stream.0.take(); + if base.is_none() && self.streams.len() == 1 { + stream.0 = self.streams.pop(); + } else { + stream.0 = Some(bridge::client::TokenStream::concat_streams(base, self.streams)); + } + } +} + /// Collects a number of token trees into a single stream. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] impl iter::FromIterator for TokenStream { fn from_iter>(trees: I) -> Self { - TokenStream(Some(bridge::client::TokenStream::concat_trees( - None, - trees.into_iter().map(tree_to_bridge_tree).collect(), - ))) + let iter = trees.into_iter(); + let mut builder = ExtendStreamWithTreesHelper::new(iter.size_hint().0); + iter.for_each(|tree| builder.push(tree)); + builder.build() } } @@ -245,30 +320,30 @@ impl iter::FromIterator for TokenStream { #[stable(feature = "proc_macro_lib", since = "1.15.0")] impl iter::FromIterator for TokenStream { fn from_iter>(streams: I) -> Self { - TokenStream(Some(bridge::client::TokenStream::concat_streams( - None, - streams.into_iter().filter_map(|stream| stream.0).collect(), - ))) + let iter = streams.into_iter(); + let mut builder = ExtendStreamWithStreamsHelper::new(iter.size_hint().0); + iter.for_each(|stream| builder.push(stream)); + builder.build() } } #[stable(feature = "token_stream_extend", since = "1.30.0")] impl Extend for TokenStream { fn extend>(&mut self, trees: I) { - *self = TokenStream(Some(bridge::client::TokenStream::concat_trees( - self.0.take(), - trees.into_iter().map(|tree| tree_to_bridge_tree(tree)).collect(), - ))); + let iter = trees.into_iter(); + let mut builder = ExtendStreamWithTreesHelper::new(iter.size_hint().0); + iter.for_each(|tree| builder.push(tree)); + builder.extend(self); } } #[stable(feature = "token_stream_extend", since = "1.30.0")] impl Extend for TokenStream { fn extend>(&mut self, streams: I) { - *self = TokenStream(Some(bridge::client::TokenStream::concat_streams( - self.0.take(), - streams.into_iter().filter_map(|stream| stream.0).collect(), - ))); + let iter = streams.into_iter(); + let mut builder = ExtendStreamWithStreamsHelper::new(iter.size_hint().0); + iter.for_each(|stream| builder.push(stream)); + builder.extend(self); } }