Skip to content

Commit ca1bcfd

Browse files
committedJun 7, 2019
Auto merge of #61541 - petrochenkov:tsp, r=oli-obk
syntax: Keep token span as a part of `Token` In the world with proc macros and edition hygiene `Token` without a span is not self-contained. In practice this means that tokens and spans are always stored and passed somewhere along with each other. This PR combines them into a single struct by doing the next renaming/replacement: - `Token` -> `TokenKind` - `TokenAndSpan` -> `Token` - `(Token, Span)` -> `Token` Some later commits (fb6e2fe and 1cdee86) remove duplicate spans in `token::Ident` and `token::Lifetime`. Those spans were supposed to be identical to token spans, but could easily go out of sync, as was noticed in #60965 (comment). The `(Token, Span)` -> `Token` change is a soft pre-requisite for this de-duplication since it allows to avoid some larger churn (passing spans to most of functions classifying identifiers).
·
1.88.01.37.0
2 parents c5295ac + 3a31f06 commit ca1bcfd

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

43 files changed

+978
-959
lines changed
 

‎src/doc/unstable-book/src/language-features/plugin.md

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -56,15 +56,15 @@ extern crate syntax_pos;
5656
extern crate rustc;
5757
extern crate rustc_plugin;
5858
59-
use syntax::parse::token;
59+
use syntax::parse::token::{self, Token};
6060
use syntax::tokenstream::TokenTree;
6161
use syntax::ext::base::{ExtCtxt, MacResult, DummyResult, MacEager};
6262
use syntax::ext::build::AstBuilder; // A trait for expr_usize.
6363
use syntax_pos::Span;
6464
use rustc_plugin::Registry;
6565
6666
fn expand_rn(cx: &mut ExtCtxt, sp: Span, args: &[TokenTree])
67-
-> Box<MacResult + 'static> {
67+
-> Box<dyn MacResult + 'static> {
6868
6969
static NUMERALS: &'static [(&'static str, usize)] = &[
7070
("M", 1000), ("CM", 900), ("D", 500), ("CD", 400),
@@ -80,7 +80,7 @@ fn expand_rn(cx: &mut ExtCtxt, sp: Span, args: &[TokenTree])
8080
}
8181
8282
let text = match args[0] {
83-
TokenTree::Token(_, token::Ident(s)) => s.to_string(),
83+
TokenTree::Token(Token { kind: token::Ident(s, _), .. }) => s.to_string(),
8484
_ => {
8585
cx.span_err(sp, "argument should be a single identifier");
8686
return DummyResult::any(sp);

‎src/librustc/hir/lowering.rs

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ use syntax::source_map::CompilerDesugaringKind::IfTemporary;
6767
use syntax::std_inject;
6868
use syntax::symbol::{kw, sym, Symbol};
6969
use syntax::tokenstream::{TokenStream, TokenTree};
70-
use syntax::parse::token::Token;
70+
use syntax::parse::token::{self, Token};
7171
use syntax::visit::{self, Visitor};
7272
use syntax_pos::{DUMMY_SP, edition, Span};
7373

@@ -1328,7 +1328,7 @@ impl<'a> LoweringContext<'a> {
13281328

13291329
fn lower_token_tree(&mut self, tree: TokenTree) -> TokenStream {
13301330
match tree {
1331-
TokenTree::Token(span, token) => self.lower_token(token, span),
1331+
TokenTree::Token(token) => self.lower_token(token),
13321332
TokenTree::Delimited(span, delim, tts) => TokenTree::Delimited(
13331333
span,
13341334
delim,
@@ -1337,13 +1337,13 @@ impl<'a> LoweringContext<'a> {
13371337
}
13381338
}
13391339

1340-
fn lower_token(&mut self, token: Token, span: Span) -> TokenStream {
1341-
match token {
1342-
Token::Interpolated(nt) => {
1343-
let tts = nt.to_tokenstream(&self.sess.parse_sess, span);
1340+
fn lower_token(&mut self, token: Token) -> TokenStream {
1341+
match token.kind {
1342+
token::Interpolated(nt) => {
1343+
let tts = nt.to_tokenstream(&self.sess.parse_sess, token.span);
13441344
self.lower_token_stream(tts)
13451345
}
1346-
other => TokenTree::Token(span, other).into(),
1346+
_ => TokenTree::Token(token).into(),
13471347
}
13481348
}
13491349

0 commit comments

Comments
 (0)
Please sign in to comment.