Auto merge of #95928 - nnethercote:rm-TokenTree-Clone, r=petrochenkov

Remove `<mbe::TokenTree as Clone>`

`mbe::TokenTree` doesn't really need to implement `Clone`, and getting rid of that impl leads to some speed-ups.

r? `@petrochenkov`

Author: bors

compiler/rustc_expand/src/mbe.rs

@@ -13,32 +13,19 @@ crate mod transcribe;
 use metavar_expr::MetaVarExpr;
 use rustc_ast::token::{self, NonterminalKind, Token, TokenKind};
 use rustc_ast::tokenstream::DelimSpan;
-use rustc_data_structures::sync::Lrc;
 use rustc_span::symbol::Ident;
 use rustc_span::Span;
 
 /// Contains the sub-token-trees of a "delimited" token tree such as `(a b c)`. The delimiters
 /// might be `NoDelim`, but they are not represented explicitly.
-#[derive(Clone, PartialEq, Encodable, Decodable, Debug)]
+#[derive(PartialEq, Encodable, Decodable, Debug)]
 struct Delimited {
     delim: token::DelimToken,
     /// FIXME: #67062 has details about why this is sub-optimal.
     tts: Vec<TokenTree>,
 }
 
-impl Delimited {
-    /// Returns a `self::TokenTree` with a `Span` corresponding to the opening delimiter.
-    fn open_tt(&self, span: DelimSpan) -> TokenTree {
-        TokenTree::token(token::OpenDelim(self.delim), span.open)
-    }
-
-    /// Returns a `self::TokenTree` with a `Span` corresponding to the closing delimiter.
-    fn close_tt(&self, span: DelimSpan) -> TokenTree {
-        TokenTree::token(token::CloseDelim(self.delim), span.close)
-    }
-}
-
-#[derive(Clone, PartialEq, Encodable, Decodable, Debug)]
+#[derive(PartialEq, Encodable, Decodable, Debug)]
 struct SequenceRepetition {
     /// The sequence of token trees
     tts: Vec<TokenTree>,
@@ -76,13 +63,13 @@ enum KleeneOp {
 
 /// Similar to `tokenstream::TokenTree`, except that `Sequence`, `MetaVar`, `MetaVarDecl`, and
 /// `MetaVarExpr` are "first-class" token trees. Useful for parsing macros.
-#[derive(Debug, Clone, PartialEq, Encodable, Decodable)]
+#[derive(Debug, PartialEq, Encodable, Decodable)]
 enum TokenTree {
     Token(Token),
     /// A delimited sequence, e.g. `($e:expr)` (RHS) or `{ $e }` (LHS).
-    Delimited(DelimSpan, Lrc<Delimited>),
+    Delimited(DelimSpan, Delimited),
     /// A kleene-style repetition sequence, e.g. `$($e:expr)*` (RHS) or `$($e),*` (LHS).
-    Sequence(DelimSpan, Lrc<SequenceRepetition>),
+    Sequence(DelimSpan, SequenceRepetition),
     /// e.g., `$var`.
     MetaVar(Span, Ident),
     /// e.g., `$var:expr`. Only appears on the LHS.

compiler/rustc_expand/src/mbe/macro_parser.rs

@@ -142,10 +142,13 @@ pub(super) fn compute_locs(sess: &ParseSess, matcher: &[TokenTree]) -> Vec<Match
                     locs.push(MatcherLoc::Token { token: token.clone() });
                 }
                 TokenTree::Delimited(span, delimited) => {
+                    let open_token = Token::new(token::OpenDelim(delimited.delim), span.open);
+                    let close_token = Token::new(token::CloseDelim(delimited.delim), span.close);
+
                     locs.push(MatcherLoc::Delimited);
-                    inner(sess, &[delimited.open_tt(*span)], locs, next_metavar, seq_depth);
+                    locs.push(MatcherLoc::Token { token: open_token });
                     inner(sess, &delimited.tts, locs, next_metavar, seq_depth);
-                    inner(sess, &[delimited.close_tt(*span)], locs, next_metavar, seq_depth);
+                    locs.push(MatcherLoc::Token { token: close_token });
                 }
                 TokenTree::Sequence(_, seq) => {
                     // We can't determine `idx_first_after` and construct the final

compiler/rustc_expand/src/mbe/macro_rules.rs

@@ -8,13 +8,12 @@ use crate::mbe::macro_parser::{MatchedSeq, MatchedTokenTree, MatcherLoc};
 use crate::mbe::transcribe::transcribe;
 
 use rustc_ast as ast;
-use rustc_ast::token::{self, NonterminalKind, Token, TokenKind::*};
+use rustc_ast::token::{self, NonterminalKind, Token, TokenKind, TokenKind::*};
 use rustc_ast::tokenstream::{DelimSpan, TokenStream};
 use rustc_ast::{NodeId, DUMMY_NODE_ID};
 use rustc_ast_pretty::pprust;
 use rustc_attr::{self as attr, TransparencyError};
 use rustc_data_structures::fx::FxHashMap;
-use rustc_data_structures::sync::Lrc;
 use rustc_errors::{Applicability, Diagnostic, DiagnosticBuilder};
 use rustc_feature::Features;
 use rustc_lint_defs::builtin::{
@@ -263,14 +262,14 @@ fn generic_extension<'cx, 'tt>(
 
                 // Ignore the delimiters on the RHS.
                 let rhs = match &rhses[i] {
-                    mbe::TokenTree::Delimited(_, delimited) => delimited.tts.to_vec(),
+                    mbe::TokenTree::Delimited(_, delimited) => &delimited.tts,
                     _ => cx.span_bug(sp, "malformed macro rhs"),
                 };
                 let arm_span = rhses[i].span();
 
                 let rhs_spans = rhs.iter().map(|t| t.span()).collect::<Vec<_>>();
                 // rhs has holes ( `$id` and `$(...)` that need filled)
-                let mut tts = match transcribe(cx, &named_matches, rhs, transparency) {
+                let mut tts = match transcribe(cx, &named_matches, &rhs, transparency) {
                     Ok(tts) => tts,
                     Err(mut err) => {
                         err.emit();
@@ -407,7 +406,7 @@ pub fn compile_declarative_macro(
     let argument_gram = vec![
         mbe::TokenTree::Sequence(
             DelimSpan::dummy(),
-            Lrc::new(mbe::SequenceRepetition {
+            mbe::SequenceRepetition {
                 tts: vec![
                     mbe::TokenTree::MetaVarDecl(def.span, lhs_nm, tt_spec),
                     mbe::TokenTree::token(token::FatArrow, def.span),
@@ -419,20 +418,20 @@ pub fn compile_declarative_macro(
                 )),
                 kleene: mbe::KleeneToken::new(mbe::KleeneOp::OneOrMore, def.span),
                 num_captures: 2,
-            }),
+            },
         ),
         // to phase into semicolon-termination instead of semicolon-separation
         mbe::TokenTree::Sequence(
             DelimSpan::dummy(),
-            Lrc::new(mbe::SequenceRepetition {
+            mbe::SequenceRepetition {
                 tts: vec![mbe::TokenTree::token(
                     if macro_rules { token::Semi } else { token::Comma },
                     def.span,
                 )],
                 separator: None,
                 kleene: mbe::KleeneToken::new(mbe::KleeneOp::ZeroOrMore, def.span),
                 num_captures: 0,
-            }),
+            },
         ),
     ];
     // Convert it into `MatcherLoc` form.
@@ -658,18 +657,18 @@ fn check_matcher(
 // that do not try to inject artificial span information. My plan is
 // to try to catch such cases ahead of time and not include them in
 // the precomputed mapping.)
-struct FirstSets {
+struct FirstSets<'tt> {
     // this maps each TokenTree::Sequence `$(tt ...) SEP OP` that is uniquely identified by its
     // span in the original matcher to the First set for the inner sequence `tt ...`.
     //
     // If two sequences have the same span in a matcher, then map that
     // span to None (invalidating the mapping here and forcing the code to
     // use a slow path).
-    first: FxHashMap<Span, Option<TokenSet>>,
+    first: FxHashMap<Span, Option<TokenSet<'tt>>>,
 }
 
-impl FirstSets {
-    fn new(tts: &[mbe::TokenTree]) -> FirstSets {
+impl<'tt> FirstSets<'tt> {
+    fn new(tts: &'tt [mbe::TokenTree]) -> FirstSets<'tt> {
         use mbe::TokenTree;
 
         let mut sets = FirstSets { first: FxHashMap::default() };
@@ -679,19 +678,22 @@ impl FirstSets {
         // walks backward over `tts`, returning the FIRST for `tts`
         // and updating `sets` at the same time for all sequence
         // substructure we find within `tts`.
-        fn build_recur(sets: &mut FirstSets, tts: &[TokenTree]) -> TokenSet {
+        fn build_recur<'tt>(sets: &mut FirstSets<'tt>, tts: &'tt [TokenTree]) -> TokenSet<'tt> {
             let mut first = TokenSet::empty();
             for tt in tts.iter().rev() {
                 match *tt {
                     TokenTree::Token(..)
                     | TokenTree::MetaVar(..)
                     | TokenTree::MetaVarDecl(..)
                     | TokenTree::MetaVarExpr(..) => {
-                        first.replace_with(tt.clone());
+                        first.replace_with(TtHandle::TtRef(tt));
                     }
                     TokenTree::Delimited(span, ref delimited) => {
                         build_recur(sets, &delimited.tts);
-                        first.replace_with(delimited.open_tt(span));
+                        first.replace_with(TtHandle::from_token_kind(
+                            token::OpenDelim(delimited.delim),
+                            span.open,
+                        ));
                     }
                     TokenTree::Sequence(sp, ref seq_rep) => {
                         let subfirst = build_recur(sets, &seq_rep.tts);
@@ -715,7 +717,7 @@ impl FirstSets {
                         // token could be the separator token itself.
 
                         if let (Some(sep), true) = (&seq_rep.separator, subfirst.maybe_empty) {
-                            first.add_one_maybe(TokenTree::Token(sep.clone()));
+                            first.add_one_maybe(TtHandle::from_token(sep.clone()));
                         }
 
                         // Reverse scan: Sequence comes before `first`.
@@ -741,7 +743,7 @@ impl FirstSets {
 
     // walks forward over `tts` until all potential FIRST tokens are
     // identified.
-    fn first(&self, tts: &[mbe::TokenTree]) -> TokenSet {
+    fn first(&self, tts: &'tt [mbe::TokenTree]) -> TokenSet<'tt> {
         use mbe::TokenTree;
 
         let mut first = TokenSet::empty();
@@ -752,11 +754,14 @@ impl FirstSets {
                 | TokenTree::MetaVar(..)
                 | TokenTree::MetaVarDecl(..)
                 | TokenTree::MetaVarExpr(..) => {
-                    first.add_one(tt.clone());
+                    first.add_one(TtHandle::TtRef(tt));
                     return first;
                 }
                 TokenTree::Delimited(span, ref delimited) => {
-                    first.add_one(delimited.open_tt(span));
+                    first.add_one(TtHandle::from_token_kind(
+                        token::OpenDelim(delimited.delim),
+                        span.open,
+                    ));
                     return first;
                 }
                 TokenTree::Sequence(sp, ref seq_rep) => {
@@ -775,7 +780,7 @@ impl FirstSets {
                     // If the sequence contents can be empty, then the first
                     // token could be the separator token itself.
                     if let (Some(sep), true) = (&seq_rep.separator, subfirst.maybe_empty) {
-                        first.add_one_maybe(TokenTree::Token(sep.clone()));
+                        first.add_one_maybe(TtHandle::from_token(sep.clone()));
                     }
 
                     assert!(first.maybe_empty);
@@ -803,6 +808,62 @@ impl FirstSets {
     }
 }
 
+// Most `mbe::TokenTree`s are pre-existing in the matcher, but some are defined
+// implicitly, such as opening/closing delimiters and sequence repetition ops.
+// This type encapsulates both kinds. It implements `Clone` while avoiding the
+// need for `mbe::TokenTree` to implement `Clone`.
+#[derive(Debug)]
+enum TtHandle<'tt> {
+    /// This is used in most cases.
+    TtRef(&'tt mbe::TokenTree),
+
+    /// This is only used for implicit token trees. The `mbe::TokenTree` *must*
+    /// be `mbe::TokenTree::Token`. No other variants are allowed. We store an
+    /// `mbe::TokenTree` rather than a `Token` so that `get()` can return a
+    /// `&mbe::TokenTree`.
+    Token(mbe::TokenTree),
+}
+
+impl<'tt> TtHandle<'tt> {
+    fn from_token(tok: Token) -> Self {
+        TtHandle::Token(mbe::TokenTree::Token(tok))
+    }
+
+    fn from_token_kind(kind: TokenKind, span: Span) -> Self {
+        TtHandle::from_token(Token::new(kind, span))
+    }
+
+    // Get a reference to a token tree.
+    fn get(&'tt self) -> &'tt mbe::TokenTree {
+        match self {
+            TtHandle::TtRef(tt) => tt,
+            TtHandle::Token(token_tt) => &token_tt,
+        }
+    }
+}
+
+impl<'tt> PartialEq for TtHandle<'tt> {
+    fn eq(&self, other: &TtHandle<'tt>) -> bool {
+        self.get() == other.get()
+    }
+}
+
+impl<'tt> Clone for TtHandle<'tt> {
+    fn clone(&self) -> Self {
+        match self {
+            TtHandle::TtRef(tt) => TtHandle::TtRef(tt),
+
+            // This variant *must* contain a `mbe::TokenTree::Token`, and not
+            // any other variant of `mbe::TokenTree`.
+            TtHandle::Token(mbe::TokenTree::Token(tok)) => {
+                TtHandle::Token(mbe::TokenTree::Token(tok.clone()))
+            }
+
+            _ => unreachable!(),
+        }
+    }
+}
+
 // A set of `mbe::TokenTree`s, which may include `TokenTree::Match`s
 // (for macro-by-example syntactic variables). It also carries the
 // `maybe_empty` flag; that is true if and only if the matcher can
@@ -814,28 +875,28 @@ impl FirstSets {
 //
 // (Notably, we must allow for *-op to occur zero times.)
 #[derive(Clone, Debug)]
-struct TokenSet {
-    tokens: Vec<mbe::TokenTree>,
+struct TokenSet<'tt> {
+    tokens: Vec<TtHandle<'tt>>,
     maybe_empty: bool,
 }
 
-impl TokenSet {
+impl<'tt> TokenSet<'tt> {
     // Returns a set for the empty sequence.
     fn empty() -> Self {
         TokenSet { tokens: Vec::new(), maybe_empty: true }
     }
 
     // Returns the set `{ tok }` for the single-token (and thus
     // non-empty) sequence [tok].
-    fn singleton(tok: mbe::TokenTree) -> Self {
-        TokenSet { tokens: vec![tok], maybe_empty: false }
+    fn singleton(tt: TtHandle<'tt>) -> Self {
+        TokenSet { tokens: vec![tt], maybe_empty: false }
     }
 
     // Changes self to be the set `{ tok }`.
     // Since `tok` is always present, marks self as non-empty.
-    fn replace_with(&mut self, tok: mbe::TokenTree) {
+    fn replace_with(&mut self, tt: TtHandle<'tt>) {
         self.tokens.clear();
-        self.tokens.push(tok);
+        self.tokens.push(tt);
         self.maybe_empty = false;
     }
 
@@ -848,17 +909,17 @@ impl TokenSet {
     }
 
     // Adds `tok` to the set for `self`, marking sequence as non-empy.
-    fn add_one(&mut self, tok: mbe::TokenTree) {
-        if !self.tokens.contains(&tok) {
-            self.tokens.push(tok);
+    fn add_one(&mut self, tt: TtHandle<'tt>) {
+        if !self.tokens.contains(&tt) {
+            self.tokens.push(tt);
         }
         self.maybe_empty = false;
     }
 
     // Adds `tok` to the set for `self`. (Leaves `maybe_empty` flag alone.)
-    fn add_one_maybe(&mut self, tok: mbe::TokenTree) {
-        if !self.tokens.contains(&tok) {
-            self.tokens.push(tok);
+    fn add_one_maybe(&mut self, tt: TtHandle<'tt>) {
+        if !self.tokens.contains(&tt) {
+            self.tokens.push(tt);
         }
     }
 
@@ -870,9 +931,9 @@ impl TokenSet {
     // setting of the empty flag of `self`. If `other` is guaranteed
     // non-empty, then `self` is marked non-empty.
     fn add_all(&mut self, other: &Self) {
-        for tok in &other.tokens {
-            if !self.tokens.contains(tok) {
-                self.tokens.push(tok.clone());
+        for tt in &other.tokens {
+            if !self.tokens.contains(tt) {
+                self.tokens.push(tt.clone());
             }
         }
         if !other.maybe_empty {
@@ -892,14 +953,14 @@ impl TokenSet {
 //
 // Requires that `first_sets` is pre-computed for `matcher`;
 // see `FirstSets::new`.
-fn check_matcher_core(
+fn check_matcher_core<'tt>(
     sess: &ParseSess,
     features: &Features,
     def: &ast::Item,
-    first_sets: &FirstSets,
-    matcher: &[mbe::TokenTree],
-    follow: &TokenSet,
-) -> TokenSet {
+    first_sets: &FirstSets<'tt>,
+    matcher: &'tt [mbe::TokenTree],
+    follow: &TokenSet<'tt>,
+) -> TokenSet<'tt> {
     use mbe::TokenTree;
 
     let mut last = TokenSet::empty();
@@ -938,12 +999,15 @@ fn check_matcher_core(
                     // followed by anything against SUFFIX.
                     continue 'each_token;
                 } else {
-                    last.replace_with(token.clone());
+                    last.replace_with(TtHandle::TtRef(token));
                     suffix_first = build_suffix_first();
                 }
             }
             TokenTree::Delimited(span, ref d) => {
-                let my_suffix = TokenSet::singleton(d.close_tt(span));
+                let my_suffix = TokenSet::singleton(TtHandle::from_token_kind(
+                    token::CloseDelim(d.delim),
+                    span.close,
+                ));
                 check_matcher_core(sess, features, def, first_sets, &d.tts, &my_suffix);
                 // don't track non NT tokens
                 last.replace_with_irrelevant();
@@ -967,7 +1031,7 @@ fn check_matcher_core(
                 let mut new;
                 let my_suffix = if let Some(sep) = &seq_rep.separator {
                     new = suffix_first.clone();
-                    new.add_one_maybe(TokenTree::Token(sep.clone()));
+                    new.add_one_maybe(TtHandle::from_token(sep.clone()));
                     &new
                 } else {
                     &suffix_first
@@ -994,9 +1058,11 @@ fn check_matcher_core(
 
         // Now `last` holds the complete set of NT tokens that could
         // end the sequence before SUFFIX. Check that every one works with `suffix`.
-        for token in &last.tokens {
-            if let TokenTree::MetaVarDecl(span, name, Some(kind)) = *token {
+        for tt in &last.tokens {
+            if let &TokenTree::MetaVarDecl(span, name, Some(kind)) = tt.get() {
                 for next_token in &suffix_first.tokens {
+                    let next_token = next_token.get();
+
                     // Check if the old pat is used and the next token is `|`
                     // to warn about incompatibility with Rust 2021.
                     // We only emit this lint if we're parsing the original

compiler/rustc_expand/src/mbe/quoted.rs

@@ -11,8 +11,6 @@ use rustc_span::symbol::{kw, sym, Ident};
 use rustc_span::edition::Edition;
 use rustc_span::{Span, SyntaxContext};
 
-use rustc_data_structures::sync::Lrc;
-
 const VALID_FRAGMENT_NAMES_MSG: &str = "valid fragment specifiers are \
                                         `ident`, `block`, `stmt`, `expr`, `pat`, `ty`, `lifetime`, \
                                         `literal`, `path`, `meta`, `tt`, `item` and `vis`";
@@ -213,12 +211,7 @@ fn parse_tree(
                         if parsing_patterns { count_metavar_decls(&sequence) } else { 0 };
                     TokenTree::Sequence(
                         delim_span,
-                        Lrc::new(SequenceRepetition {
-                            tts: sequence,
-                            separator,
-                            kleene,
-                            num_captures,
-                        }),
+                        SequenceRepetition { tts: sequence, separator, kleene, num_captures },
                     )
                 }
 
@@ -269,10 +262,10 @@ fn parse_tree(
         // descend into the delimited set and further parse it.
         tokenstream::TokenTree::Delimited(span, delim, tts) => TokenTree::Delimited(
             span,
-            Lrc::new(Delimited {
+            Delimited {
                 delim,
                 tts: parse(tts, parsing_patterns, sess, node_id, features, edition),
-            }),
+            },
         ),
     }
 }

compiler/rustc_expand/src/mbe/transcribe.rs

@@ -5,7 +5,6 @@ use rustc_ast::mut_visit::{self, MutVisitor};
 use rustc_ast::token::{self, Token, TokenKind};
 use rustc_ast::tokenstream::{DelimSpan, TokenStream, TokenTree, TreeAndSpacing};
 use rustc_data_structures::fx::FxHashMap;
-use rustc_data_structures::sync::Lrc;
 use rustc_errors::{pluralize, PResult};
 use rustc_errors::{DiagnosticBuilder, ErrorGuaranteed};
 use rustc_span::hygiene::{LocalExpnId, Transparency};
@@ -27,31 +26,35 @@ impl MutVisitor for Marker {
 }
 
 /// An iterator over the token trees in a delimited token tree (`{ ... }`) or a sequence (`$(...)`).
-enum Frame {
-    Delimited { forest: Lrc<mbe::Delimited>, idx: usize, span: DelimSpan },
-    Sequence { forest: Lrc<mbe::SequenceRepetition>, idx: usize, sep: Option<Token> },
+enum Frame<'a> {
+    Delimited {
+        tts: &'a [mbe::TokenTree],
+        delim_token: token::DelimToken,
+        idx: usize,
+        span: DelimSpan,
+    },
+    Sequence {
+        tts: &'a [mbe::TokenTree],
+        idx: usize,
+        sep: Option<Token>,
+    },
 }
 
-impl Frame {
+impl<'a> Frame<'a> {
     /// Construct a new frame around the delimited set of tokens.
-    fn new(tts: Vec<mbe::TokenTree>) -> Frame {
-        let forest = Lrc::new(mbe::Delimited { delim: token::NoDelim, tts });
-        Frame::Delimited { forest, idx: 0, span: DelimSpan::dummy() }
+    fn new(tts: &'a [mbe::TokenTree]) -> Frame<'a> {
+        Frame::Delimited { tts, delim_token: token::NoDelim, idx: 0, span: DelimSpan::dummy() }
     }
 }
 
-impl Iterator for Frame {
-    type Item = mbe::TokenTree;
+impl<'a> Iterator for Frame<'a> {
+    type Item = &'a mbe::TokenTree;
 
-    fn next(&mut self) -> Option<mbe::TokenTree> {
-        match *self {
-            Frame::Delimited { ref forest, ref mut idx, .. } => {
-                let res = forest.tts.get(*idx).cloned();
-                *idx += 1;
-                res
-            }
-            Frame::Sequence { ref forest, ref mut idx, .. } => {
-                let res = forest.tts.get(*idx).cloned();
+    fn next(&mut self) -> Option<&'a mbe::TokenTree> {
+        match self {
+            Frame::Delimited { tts, ref mut idx, .. }
+            | Frame::Sequence { tts, ref mut idx, .. } => {
+                let res = tts.get(*idx);
                 *idx += 1;
                 res
             }
@@ -82,7 +85,7 @@ impl Iterator for Frame {
 pub(super) fn transcribe<'a>(
     cx: &ExtCtxt<'a>,
     interp: &FxHashMap<MacroRulesNormalizedIdent, NamedMatch>,
-    src: Vec<mbe::TokenTree>,
+    src: &[mbe::TokenTree],
     transparency: Transparency,
 ) -> PResult<'a, TokenStream> {
     // Nothing for us to transcribe...
@@ -92,7 +95,7 @@ pub(super) fn transcribe<'a>(
 
     // We descend into the RHS (`src`), expanding things as we go. This stack contains the things
     // we have yet to expand/are still expanding. We start the stack off with the whole RHS.
-    let mut stack: SmallVec<[Frame; 1]> = smallvec![Frame::new(src)];
+    let mut stack: SmallVec<[Frame<'_>; 1]> = smallvec![Frame::new(&src)];
 
     // As we descend in the RHS, we will need to be able to match nested sequences of matchers.
     // `repeats` keeps track of where we are in matching at each level, with the last element being
@@ -146,14 +149,14 @@ pub(super) fn transcribe<'a>(
                 // We are done processing a Delimited. If this is the top-level delimited, we are
                 // done. Otherwise, we unwind the result_stack to append what we have produced to
                 // any previous results.
-                Frame::Delimited { forest, span, .. } => {
+                Frame::Delimited { delim_token, span, .. } => {
                     if result_stack.is_empty() {
                         // No results left to compute! We are back at the top-level.
                         return Ok(TokenStream::new(result));
                     }
 
                     // Step back into the parent Delimited.
-                    let tree = TokenTree::Delimited(span, forest.delim, TokenStream::new(result));
+                    let tree = TokenTree::Delimited(span, delim_token, TokenStream::new(result));
                     result = result_stack.pop().unwrap();
                     result.push(tree.into());
                 }
@@ -167,7 +170,7 @@ pub(super) fn transcribe<'a>(
             // We are descending into a sequence. We first make sure that the matchers in the RHS
             // and the matches in `interp` have the same shape. Otherwise, either the caller or the
             // macro writer has made a mistake.
-            seq @ mbe::TokenTree::Sequence(..) => {
+            seq @ mbe::TokenTree::Sequence(_, delimited) => {
                 match lockstep_iter_size(&seq, interp, &repeats) {
                     LockstepIterSize::Unconstrained => {
                         return Err(cx.struct_span_err(
@@ -214,7 +217,7 @@ pub(super) fn transcribe<'a>(
                             stack.push(Frame::Sequence {
                                 idx: 0,
                                 sep: seq.separator.clone(),
-                                forest: seq,
+                                tts: &delimited.tts,
                             });
                         }
                     }
@@ -272,15 +275,21 @@ pub(super) fn transcribe<'a>(
             // the previous results (from outside the Delimited).
             mbe::TokenTree::Delimited(mut span, delimited) => {
                 mut_visit::visit_delim_span(&mut span, &mut marker);
-                stack.push(Frame::Delimited { forest: delimited, idx: 0, span });
+                stack.push(Frame::Delimited {
+                    tts: &delimited.tts,
+                    delim_token: delimited.delim,
+                    idx: 0,
+                    span,
+                });
                 result_stack.push(mem::take(&mut result));
             }
 
             // Nothing much to do here. Just push the token to the result, being careful to
             // preserve syntax context.
             mbe::TokenTree::Token(token) => {
-                let mut tt = TokenTree::Token(token);
-                mut_visit::visit_tt(&mut tt, &mut marker);
+                let mut token = token.clone();
+                mut_visit::visit_token(&mut token, &mut marker);
+                let tt = TokenTree::Token(token);
                 result.push(tt.into());
             }
 
@@ -516,7 +525,7 @@ fn out_of_bounds_err<'a>(
 
 fn transcribe_metavar_expr<'a>(
     cx: &ExtCtxt<'a>,
-    expr: MetaVarExpr,
+    expr: &MetaVarExpr,
     interp: &FxHashMap<MacroRulesNormalizedIdent, NamedMatch>,
     marker: &mut Marker,
     repeats: &[(usize, usize)],
@@ -528,7 +537,7 @@ fn transcribe_metavar_expr<'a>(
         marker.visit_span(&mut span);
         span
     };
-    match expr {
+    match *expr {
         MetaVarExpr::Count(original_ident, depth_opt) => {
             let matched = matched_from_ident(cx, original_ident, interp)?;
             let count = count_repetitions(cx, depth_opt, matched, &repeats, sp)?;