diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs index 1e18b1232de76..26e9871fa61a6 100644 --- a/compiler/rustc_ast/src/tokenstream.rs +++ b/compiler/rustc_ast/src/tokenstream.rs @@ -301,7 +301,7 @@ pub struct AttributesData { /// Today's `TokenTree`s can still contain AST via `token::Interpolated` for /// backwards compatibility. #[derive(Clone, Debug, Default, Encodable, Decodable)] -pub struct TokenStream(pub(crate) Lrc>); +pub struct TokenStream(pub Lrc>); /// Similar to `proc_macro::Spacing`, but for tokens. /// diff --git a/compiler/rustc_expand/src/proc_macro.rs b/compiler/rustc_expand/src/proc_macro.rs index c617cd76e3cb0..e666ba6375943 100644 --- a/compiler/rustc_expand/src/proc_macro.rs +++ b/compiler/rustc_expand/src/proc_macro.rs @@ -60,14 +60,18 @@ impl base::BangProcMacro for BangProcMacro { let proc_macro_backtrace = ecx.ecfg.proc_macro_backtrace; let strategy = exec_strategy(ecx); let server = proc_macro_server::Rustc::new(ecx); - self.client.run(&strategy, server, input, proc_macro_backtrace).map_err(|e| { - ecx.sess.emit_err(errors::ProcMacroPanicked { - span, - message: e - .as_str() - .map(|message| errors::ProcMacroPanickedHelp { message: message.into() }), + let input = crate::proc_macro_server::TokenStream2::from_internal(input, true); + self.client + .run(&strategy, server, input, proc_macro_backtrace) + .map_err(|e| { + ecx.sess.emit_err(errors::ProcMacroPanicked { + span, + message: e + .as_str() + .map(|message| errors::ProcMacroPanickedHelp { message: message.into() }), + }) }) - }) + .map(|stream| stream.to_internal()) } } @@ -91,15 +95,18 @@ impl base::AttrProcMacro for AttrProcMacro { let proc_macro_backtrace = ecx.ecfg.proc_macro_backtrace; let strategy = exec_strategy(ecx); let server = proc_macro_server::Rustc::new(ecx); - self.client.run(&strategy, server, annotation, annotated, proc_macro_backtrace).map_err( - |e| { + let annotation = crate::proc_macro_server::TokenStream2::from_internal(annotation, true); + let annotated = crate::proc_macro_server::TokenStream2::from_internal(annotated, true); + self.client + .run(&strategy, server, annotation, annotated, proc_macro_backtrace) + .map_err(|e| { let mut err = ecx.struct_span_err(span, "custom attribute panicked"); if let Some(s) = e.as_str() { err.help(format!("message: {s}")); } err.emit() - }, - ) + }) + .map(|stream| stream.to_internal()) } } @@ -143,6 +150,7 @@ impl MultiItemModifier for DeriveProcMacro { let proc_macro_backtrace = ecx.ecfg.proc_macro_backtrace; let strategy = exec_strategy(ecx); let server = proc_macro_server::Rustc::new(ecx); + let input = crate::proc_macro_server::TokenStream2::from_internal(input, true); match self.client.run(&strategy, server, input, proc_macro_backtrace) { Ok(stream) => stream, Err(e) => { @@ -157,6 +165,7 @@ impl MultiItemModifier for DeriveProcMacro { }; let error_count_before = ecx.sess.parse_sess.span_diagnostic.err_count(); + let stream = stream.to_internal(); let mut parser = rustc_parse::stream_to_parser(&ecx.sess.parse_sess, stream, Some("proc-macro derive")); let mut items = vec![]; diff --git a/compiler/rustc_expand/src/proc_macro_server.rs b/compiler/rustc_expand/src/proc_macro_server.rs index 2dc9b51a37ea0..14ac01e4bb16e 100644 --- a/compiler/rustc_expand/src/proc_macro_server.rs +++ b/compiler/rustc_expand/src/proc_macro_server.rs @@ -87,18 +87,20 @@ impl ToInternal for LitKind { } } -impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec> { - fn from_internal((stream, rustc): (TokenStream, &mut Rustc<'_, '_>)) -> Self { +impl FromInternal<(TokenStream2, &mut Rustc<'_, '_>)> + for Vec> +{ + fn from_internal((stream, rustc): (TokenStream2, &mut Rustc<'_, '_>)) -> Self { use rustc_ast::token::*; // Estimate the capacity as `stream.len()` rounded up to the next power // of two to limit the number of required reallocations. let mut trees = Vec::with_capacity(stream.len().next_power_of_two()); - let mut cursor = stream.trees(); + let mut cursor = stream.0.iter(); while let Some(tree) = cursor.next() { let (Token { kind, span }, joint) = match tree.clone() { - tokenstream::TokenTree::Delimited(span, delim, tts) => { + TokenTree2::Delimited(span, delim, tts) => { let delimiter = pm::Delimiter::from_internal(delim); trees.push(TokenTree::Group(Group { delimiter, @@ -111,7 +113,7 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec (token, spacing == Joint), + TokenTree2::Token(token, spacing) => (token, spacing == Joint), }; // Split the operator into one or more `Punct`s, one per character. @@ -213,7 +215,7 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec)> for Vec { - let stream = TokenStream::from_nonterminal_ast(&nt); + let stream = TokenStream2::from_internal(TokenStream::from_nonterminal_ast(&nt), true); // A hack used to pass AST fragments to attribute and derive // macros as a single nonterminal token instead of a token // stream. Such token needs to be "unwrapped" and not @@ -258,10 +260,10 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec> - for (TokenTree, &mut Rustc<'_, '_>) +impl ToInternal> + for (TokenTree, &mut Rustc<'_, '_>) { - fn to_internal(self) -> SmallVec<[tokenstream::TokenTree; 2]> { + fn to_internal(self) -> SmallVec<[TokenTree2; 2]> { use rustc_ast::token::*; let (tree, rustc) = self; @@ -293,13 +295,13 @@ impl ToInternal> _ => unreachable!(), }; smallvec![if joint { - tokenstream::TokenTree::token_joint(kind, span) + TokenTree2::token_joint(kind, span) } else { - tokenstream::TokenTree::token_alone(kind, span) + TokenTree2::token_alone(kind, span) }] } TokenTree::Group(Group { delimiter, stream, span: DelimSpan { open, close, .. } }) => { - smallvec![tokenstream::TokenTree::Delimited( + smallvec![TokenTree2::Delimited( tokenstream::DelimSpan { open, close }, delimiter.to_internal(), stream.unwrap_or_default(), @@ -307,7 +309,7 @@ impl ToInternal> } TokenTree::Ident(self::Ident { sym, is_raw, span }) => { rustc.sess().symbol_gallery.insert(sym, span); - smallvec![tokenstream::TokenTree::token_alone(Ident(sym, is_raw), span)] + smallvec![TokenTree2::token_alone(Ident(sym, is_raw), span)] } TokenTree::Literal(self::Literal { kind: self::LitKind::Integer, @@ -318,8 +320,8 @@ impl ToInternal> let minus = BinOp(BinOpToken::Minus); let symbol = Symbol::intern(&symbol.as_str()[1..]); let integer = TokenKind::lit(token::Integer, symbol, suffix); - let a = tokenstream::TokenTree::token_alone(minus, span); - let b = tokenstream::TokenTree::token_alone(integer, span); + let a = TokenTree2::token_alone(minus, span); + let b = TokenTree2::token_alone(integer, span); smallvec![a, b] } TokenTree::Literal(self::Literal { @@ -331,12 +333,12 @@ impl ToInternal> let minus = BinOp(BinOpToken::Minus); let symbol = Symbol::intern(&symbol.as_str()[1..]); let float = TokenKind::lit(token::Float, symbol, suffix); - let a = tokenstream::TokenTree::token_alone(minus, span); - let b = tokenstream::TokenTree::token_alone(float, span); + let a = TokenTree2::token_alone(minus, span); + let b = TokenTree2::token_alone(float, span); smallvec![a, b] } TokenTree::Literal(self::Literal { kind, symbol, suffix, span }) => { - smallvec![tokenstream::TokenTree::token_alone( + smallvec![TokenTree2::token_alone( TokenKind::lit(kind.to_internal(), symbol, suffix), span, )] @@ -386,9 +388,150 @@ impl<'a, 'b> Rustc<'a, 'b> { } } +#[derive(Clone, Debug, Default, Encodable, Decodable)] +pub struct TokenStream2(Lrc>); + +#[derive(Debug, Clone, Encodable, Decodable)] +pub enum TokenTree2 { + /// A single token. Should never be `OpenDelim` or `CloseDelim`, because + /// delimiters are implicitly represented by `Delimited`. + Token(rustc_ast::token::Token, rustc_ast::tokenstream::Spacing), + /// A delimited sequence of token trees. + Delimited(rustc_ast::tokenstream::DelimSpan, rustc_ast::token::Delimiter, TokenStream2), +} + +impl FromIterator for TokenStream2 { + fn from_iter>(iter: I) -> Self { + TokenStream2::new(iter.into_iter().collect::>(), false) + } +} + +impl TokenStream2 { + pub fn new(mut streams: Vec, censored: bool) -> TokenStream2 { + if censored { + for i in 0..streams.len() { + if !matches!(streams.get(i + 1), Some(TokenTree2::Token(t, _)) if t.is_op()) { + if let TokenTree2::Token(_, spacing) = &mut streams[i] { + *spacing = tokenstream::Spacing::Alone; + } + } + } + } + TokenStream2(Lrc::new(streams)) + } + + pub fn len(&self) -> usize { + self.0.len() + } + + pub fn from_internal(mut stream: TokenStream, censored: bool) -> Self { + TokenStream2::new( + std::mem::take(Lrc::make_mut(&mut stream.0)) + .into_iter() + .map(|tree| TokenTree2::from_internal(tree, censored)) + .collect(), + censored, + ) + } + + pub fn to_internal(mut self) -> TokenStream { + TokenStream::new( + std::mem::take(Lrc::make_mut(&mut self.0)) + .into_iter() + .map(|tree| tree.to_internal()) + .collect(), + ) + } + + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + pub fn token_alone(kind: token::TokenKind, span: Span) -> TokenStream2 { + TokenStream2::new(vec![TokenTree2::token_alone(kind, span)], true) + } + + fn try_glue_to_last(vec: &mut Vec, tt: &TokenTree2) -> bool { + if let Some(TokenTree2::Token(last_tok, tokenstream::Spacing::Joint)) = vec.last() + && let TokenTree2::Token(tok, spacing) = tt + && let Some(glued_tok) = last_tok.glue(tok) + { + // ...then overwrite the last token tree in `vec` with the + // glued token, and skip the first token tree from `stream`. + *vec.last_mut().unwrap() = TokenTree2::Token(glued_tok, *spacing); + true + } else { + false + } + } + + /// Push `tt` onto the end of the stream, possibly gluing it to the last + /// token. Uses `make_mut` to maximize efficiency. + pub fn push_tree(&mut self, tt: TokenTree2) { + let vec_mut = Lrc::make_mut(&mut self.0); + + if Self::try_glue_to_last(vec_mut, &tt) { + // nothing else to do + } else { + vec_mut.push(tt); + } + } + + /// Push `stream` onto the end of the stream, possibly gluing the first + /// token tree to the last token. (No other token trees will be glued.) + /// Uses `make_mut` to maximize efficiency. + pub fn push_stream(&mut self, stream: TokenStream2) { + let vec_mut = Lrc::make_mut(&mut self.0); + + let stream_iter = stream.0.iter().cloned(); + + if let Some(first) = stream.0.first() && Self::try_glue_to_last(vec_mut, first) { + // Now skip the first token tree from `stream`. + vec_mut.extend(stream_iter.skip(1)); + } else { + // Append all of `stream`. + vec_mut.extend(stream_iter); + } + } +} + +impl TokenTree2 { + pub fn from_internal(tree: tokenstream::TokenTree, censored: bool) -> Self { + match tree { + tokenstream::TokenTree::Token(token, spacing) => TokenTree2::Token(token, spacing), + tokenstream::TokenTree::Delimited(dspan, delim, stream) => { + TokenTree2::Delimited(dspan, delim, TokenStream2::from_internal(stream, censored)) + } + } + } + + pub fn to_internal(self) -> tokenstream::TokenTree { + match self { + TokenTree2::Token(token, spacing) => tokenstream::TokenTree::Token(token, spacing), + TokenTree2::Delimited(dspan, delim, stream) => { + tokenstream::TokenTree::Delimited(dspan, delim, stream.to_internal()) + } + } + } + + pub fn token_alone(kind: token::TokenKind, span: Span) -> TokenTree2 { + TokenTree2::Token( + rustc_ast::token::Token::new(kind, span), + rustc_ast::tokenstream::Spacing::Alone, + ) + } + + pub fn token_joint(kind: token::TokenKind, span: Span) -> TokenTree2 { + TokenTree2::Token( + rustc_ast::token::Token::new(kind, span), + rustc_ast::tokenstream::Spacing::Joint, + ) + } +} + impl server::Types for Rustc<'_, '_> { type FreeFunctions = FreeFunctions; - type TokenStream = TokenStream; + type TokenStream = TokenStream2; type SourceFile = Lrc; type Span = Span; type Symbol = Symbol; @@ -481,16 +624,19 @@ impl server::TokenStream for Rustc<'_, '_> { } fn from_str(&mut self, src: &str) -> Self::TokenStream { - parse_stream_from_source_str( - FileName::proc_macro_source_code(src), - src.to_string(), - self.sess(), - Some(self.call_site), + TokenStream2::from_internal( + parse_stream_from_source_str( + FileName::proc_macro_source_code(src), + src.to_string(), + self.sess(), + Some(self.call_site), + ), + true, ) } fn to_string(&mut self, stream: &Self::TokenStream) -> String { - pprust::tts_to_string(stream) + pprust::tts_to_string(&stream.clone().to_internal()) } fn expand_expr(&mut self, stream: &Self::TokenStream) -> Result { @@ -498,7 +644,7 @@ impl server::TokenStream for Rustc<'_, '_> { let expr: PResult<'_, _> = try { let mut p = rustc_parse::stream_to_parser( self.sess(), - stream.clone(), + stream.clone().to_internal(), Some("proc_macro expand expr"), ); let expr = p.parse_expr()?; @@ -524,17 +670,14 @@ impl server::TokenStream for Rustc<'_, '_> { // be recovered in the general case. match &expr.kind { ast::ExprKind::Lit(token_lit) if token_lit.kind == token::Bool => { - Ok(tokenstream::TokenStream::token_alone( - token::Ident(token_lit.symbol, false), - expr.span, - )) + Ok(TokenStream2::token_alone(token::Ident(token_lit.symbol, false), expr.span)) } ast::ExprKind::Lit(token_lit) => { - Ok(tokenstream::TokenStream::token_alone(token::Literal(*token_lit), expr.span)) + Ok(TokenStream2::token_alone(token::Literal(*token_lit), expr.span)) } ast::ExprKind::IncludedBytes(bytes) => { let lit = token::Lit::new(token::ByteStr, escape_byte_str_symbol(bytes), None); - Ok(tokenstream::TokenStream::token_alone(token::TokenKind::Literal(lit), expr.span)) + Ok(TokenStream2::token_alone(token::TokenKind::Literal(lit), expr.span)) } ast::ExprKind::Unary(ast::UnOp::Neg, e) => match &e.kind { ast::ExprKind::Lit(token_lit) => match token_lit { @@ -542,8 +685,8 @@ impl server::TokenStream for Rustc<'_, '_> { Ok(Self::TokenStream::from_iter([ // FIXME: The span of the `-` token is lost when // parsing, so we cannot faithfully recover it here. - tokenstream::TokenTree::token_alone(token::BinOp(token::Minus), e.span), - tokenstream::TokenTree::token_alone(token::Literal(*token_lit), e.span), + TokenTree2::token_alone(token::BinOp(token::Minus), e.span), + TokenTree2::token_alone(token::Literal(*token_lit), e.span), ])) } _ => Err(()), @@ -558,7 +701,10 @@ impl server::TokenStream for Rustc<'_, '_> { &mut self, tree: TokenTree, ) -> Self::TokenStream { - Self::TokenStream::new((tree, &mut *self).to_internal().into_iter().collect::>()) + Self::TokenStream::new( + (tree, &mut *self).to_internal().into_iter().collect::>(), + false, + ) } fn concat_trees( @@ -566,8 +712,7 @@ impl server::TokenStream for Rustc<'_, '_> { base: Option, trees: Vec>, ) -> Self::TokenStream { - let mut stream = - if let Some(base) = base { base } else { tokenstream::TokenStream::default() }; + let mut stream = if let Some(base) = base { base } else { TokenStream2::default() }; for tree in trees { for tt in (tree, &mut *self).to_internal() { stream.push_tree(tt); @@ -581,8 +726,7 @@ impl server::TokenStream for Rustc<'_, '_> { base: Option, streams: Vec, ) -> Self::TokenStream { - let mut stream = - if let Some(base) = base { base } else { tokenstream::TokenStream::default() }; + let mut stream = if let Some(base) = base { base } else { TokenStream2::default() }; for s in streams { stream.push_stream(s); } diff --git a/compiler/rustc_parse/src/lexer/tokentrees.rs b/compiler/rustc_parse/src/lexer/tokentrees.rs index 07910113dee7f..2332fa8429bdb 100644 --- a/compiler/rustc_parse/src/lexer/tokentrees.rs +++ b/compiler/rustc_parse/src/lexer/tokentrees.rs @@ -59,9 +59,7 @@ impl<'a> TokenTreesReader<'a> { if let Some(glued) = self.token.glue(&next_tok) { self.token = glued; } else { - let this_spacing = - if next_tok.is_op() { Spacing::Joint } else { Spacing::Alone }; - break (this_spacing, next_tok); + break (Spacing::Joint, next_tok); } } else { break (Spacing::Alone, next_tok); diff --git a/tests/ui/proc-macro/auxiliary/issue-76399.rs b/tests/ui/proc-macro/auxiliary/issue-76399.rs new file mode 100644 index 0000000000000..9a729e4d5c426 --- /dev/null +++ b/tests/ui/proc-macro/auxiliary/issue-76399.rs @@ -0,0 +1,13 @@ +// force-host +// no-prefer-dynamic + +#![crate_type = "proc-macro"] + +extern crate proc_macro; +use proc_macro::*; + +#[proc_macro] +pub fn m(_input: TokenStream) -> TokenStream { + eprintln!("{:#?}", TokenStream::from(TokenTree::Punct(Punct::new(':', Spacing::Joint)))); + TokenStream::new() +} diff --git a/tests/ui/proc-macro/issue-75930-derive-cfg.stdout b/tests/ui/proc-macro/issue-75930-derive-cfg.stdout index 83afd0d3eaec3..bf5b01eefc306 100644 --- a/tests/ui/proc-macro/issue-75930-derive-cfg.stdout +++ b/tests/ui/proc-macro/issue-75930-derive-cfg.stdout @@ -1371,7 +1371,7 @@ PRINT-DERIVE INPUT (DEBUG): TokenStream [ }, Punct { ch: '<', - spacing: Joint, + spacing: Alone, span: $DIR/issue-75930-derive-cfg.rs:25:11: 25:12 (#0), }, Ident { diff --git a/tests/ui/proc-macro/issue-76399.rs b/tests/ui/proc-macro/issue-76399.rs new file mode 100644 index 0000000000000..be75ec198247d --- /dev/null +++ b/tests/ui/proc-macro/issue-76399.rs @@ -0,0 +1,8 @@ +// check-pass +// aux-build:issue-76399.rs + +extern crate issue_76399; + +issue_76399::m!(); + +fn main() {} diff --git a/tests/ui/proc-macro/issue-76399.stderr b/tests/ui/proc-macro/issue-76399.stderr new file mode 100644 index 0000000000000..84f8aa3716a26 --- /dev/null +++ b/tests/ui/proc-macro/issue-76399.stderr @@ -0,0 +1,7 @@ +TokenStream [ + Punct { + ch: ':', + spacing: Joint, + span: #5 bytes(70..87), + }, +]