Skip to content

Remove LazyAttrTokenStream #127478

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 6 commits into from
Closed

Remove LazyAttrTokenStream #127478

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
a47ae57
Use an `@` pattern to shorten some code.
nnethercote Jul 8, 2024
b162013
Use iterator normally in `make_attr_token_stream`.
nnethercote Jul 8, 2024
a88c4d6
Split the stack in `make_attr_token_stream`.
nnethercote Jul 8, 2024
b4feaa1
XXX: move Spacing into FlatToken
nnethercote Jul 8, 2024
ee0c16b
XXX: Remove LazyAttrTokenStream
nnethercote Jul 8, 2024
1854d09
XXX: rm LazyAttrTokenStreamImpl
nnethercote Jul 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 13 additions & 13 deletions compiler/rustc_ast/src/ast.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ pub use UnsafeSource::*;

use crate::ptr::P;
use crate::token::{self, CommentKind, Delimiter};
use crate::tokenstream::{DelimSpan, LazyAttrTokenStream, TokenStream};
use crate::tokenstream::{AttrTokenStream, DelimSpan, TokenStream};
pub use rustc_ast_ir::{Movability, Mutability};
use rustc_data_structures::packed::Pu128;
use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
Expand Down Expand Up @@ -94,7 +94,7 @@ pub struct Path {
/// The segments in the path: the things separated by `::`.
/// Global paths begin with `kw::PathRoot`.
pub segments: ThinVec<PathSegment>,
pub tokens: Option<LazyAttrTokenStream>,
pub tokens: Option<AttrTokenStream>,
}

impl PartialEq<Symbol> for Path {
Expand Down Expand Up @@ -544,7 +544,7 @@ pub struct Block {
/// Distinguishes between `unsafe { ... }` and `{ ... }`.
pub rules: BlockCheckMode,
pub span: Span,
pub tokens: Option<LazyAttrTokenStream>,
pub tokens: Option<AttrTokenStream>,
/// The following *isn't* a parse error, but will cause multiple errors in following stages.
/// ```compile_fail
/// let x = {
Expand All @@ -563,7 +563,7 @@ pub struct Pat {
pub id: NodeId,
pub kind: PatKind,
pub span: Span,
pub tokens: Option<LazyAttrTokenStream>,
pub tokens: Option<AttrTokenStream>,
}

impl Pat {
Expand Down Expand Up @@ -1002,7 +1002,7 @@ impl Stmt {
/// a trailing semicolon.
///
/// This only modifies the parsed AST struct, not the attached
/// `LazyAttrTokenStream`. The parser is responsible for calling
/// `AttrTokenStream`. The parser is responsible for calling
/// `ToAttrTokenStream::add_trailing_semi` when there is actually
/// a semicolon in the tokenstream.
pub fn add_trailing_semicolon(mut self) -> Self {
Expand Down Expand Up @@ -1050,7 +1050,7 @@ pub struct MacCallStmt {
pub mac: P<MacCall>,
pub style: MacStmtStyle,
pub attrs: AttrVec,
pub tokens: Option<LazyAttrTokenStream>,
pub tokens: Option<AttrTokenStream>,
}

#[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug)]
Expand All @@ -1076,7 +1076,7 @@ pub struct Local {
pub span: Span,
pub colon_sp: Option<Span>,
pub attrs: AttrVec,
pub tokens: Option<LazyAttrTokenStream>,
pub tokens: Option<AttrTokenStream>,
}

#[derive(Clone, Encodable, Decodable, Debug)]
Expand Down Expand Up @@ -1175,7 +1175,7 @@ pub struct Expr {
pub kind: ExprKind,
pub span: Span,
pub attrs: AttrVec,
pub tokens: Option<LazyAttrTokenStream>,
pub tokens: Option<AttrTokenStream>,
}

impl Expr {
Expand Down Expand Up @@ -2102,7 +2102,7 @@ pub struct Ty {
pub id: NodeId,
pub kind: TyKind,
pub span: Span,
pub tokens: Option<LazyAttrTokenStream>,
pub tokens: Option<AttrTokenStream>,
}

impl Clone for Ty {
Expand Down Expand Up @@ -2831,7 +2831,7 @@ pub enum AttrKind {
pub struct NormalAttr {
pub item: AttrItem,
// Tokens for the full attribute, e.g. `#[foo]`, `#![bar]`.
pub tokens: Option<LazyAttrTokenStream>,
pub tokens: Option<AttrTokenStream>,
}

impl NormalAttr {
Expand All @@ -2854,7 +2854,7 @@ pub struct AttrItem {
pub path: Path,
pub args: AttrArgs,
// Tokens for the meta item, e.g. just the `foo` within `#[foo]` or `#![foo]`.
pub tokens: Option<LazyAttrTokenStream>,
pub tokens: Option<AttrTokenStream>,
}

/// `TraitRef`s appear in impls.
Expand Down Expand Up @@ -2894,7 +2894,7 @@ impl PolyTraitRef {
pub struct Visibility {
pub kind: VisibilityKind,
pub span: Span,
pub tokens: Option<LazyAttrTokenStream>,
pub tokens: Option<AttrTokenStream>,
}

#[derive(Clone, Encodable, Decodable, Debug)]
Expand Down Expand Up @@ -2987,7 +2987,7 @@ pub struct Item<K = ItemKind> {
///
/// Note that the tokens here do not include the outer attributes, but will
/// include inner attributes.
pub tokens: Option<LazyAttrTokenStream>,
pub tokens: Option<AttrTokenStream>,
}

impl Item {
Expand Down
38 changes: 19 additions & 19 deletions compiler/rustc_ast/src/ast_traits.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

use crate::ptr::P;
use crate::token::Nonterminal;
use crate::tokenstream::LazyAttrTokenStream;
use crate::tokenstream::AttrTokenStream;
use crate::{Arm, Crate, ExprField, FieldDef, GenericParam, Param, PatField, Variant};
use crate::{AssocItem, Expr, ForeignItem, Item, NodeId};
use crate::{AttrItem, AttrKind, Block, Pat, Path, Ty, Visibility};
Expand Down Expand Up @@ -91,18 +91,18 @@ impl<T: AstDeref<Target: HasNodeId>> HasNodeId for T {

/// A trait for AST nodes having (or not having) collected tokens.
pub trait HasTokens {
fn tokens(&self) -> Option<&LazyAttrTokenStream>;
fn tokens_mut(&mut self) -> Option<&mut Option<LazyAttrTokenStream>>;
fn tokens(&self) -> Option<&AttrTokenStream>;
fn tokens_mut(&mut self) -> Option<&mut Option<AttrTokenStream>>;
}

macro_rules! impl_has_tokens {
($($T:ty),+ $(,)?) => {
$(
impl HasTokens for $T {
fn tokens(&self) -> Option<&LazyAttrTokenStream> {
fn tokens(&self) -> Option<&AttrTokenStream> {
self.tokens.as_ref()
}
fn tokens_mut(&mut self) -> Option<&mut Option<LazyAttrTokenStream>> {
fn tokens_mut(&mut self) -> Option<&mut Option<AttrTokenStream>> {
Some(&mut self.tokens)
}
}
Expand All @@ -114,10 +114,10 @@ macro_rules! impl_has_tokens_none {
($($T:ty),+ $(,)?) => {
$(
impl HasTokens for $T {
fn tokens(&self) -> Option<&LazyAttrTokenStream> {
fn tokens(&self) -> Option<&AttrTokenStream> {
None
}
fn tokens_mut(&mut self) -> Option<&mut Option<LazyAttrTokenStream>> {
fn tokens_mut(&mut self) -> Option<&mut Option<AttrTokenStream>> {
None
}
}
Expand All @@ -129,25 +129,25 @@ impl_has_tokens!(AssocItem, AttrItem, Block, Expr, ForeignItem, Item, Pat, Path,
impl_has_tokens_none!(Arm, ExprField, FieldDef, GenericParam, Param, PatField, Variant);

impl<T: AstDeref<Target: HasTokens>> HasTokens for T {
fn tokens(&self) -> Option<&LazyAttrTokenStream> {
fn tokens(&self) -> Option<&AttrTokenStream> {
self.ast_deref().tokens()
}
fn tokens_mut(&mut self) -> Option<&mut Option<LazyAttrTokenStream>> {
fn tokens_mut(&mut self) -> Option<&mut Option<AttrTokenStream>> {
self.ast_deref_mut().tokens_mut()
}
}

impl<T: HasTokens> HasTokens for Option<T> {
fn tokens(&self) -> Option<&LazyAttrTokenStream> {
fn tokens(&self) -> Option<&AttrTokenStream> {
self.as_ref().and_then(|inner| inner.tokens())
}
fn tokens_mut(&mut self) -> Option<&mut Option<LazyAttrTokenStream>> {
fn tokens_mut(&mut self) -> Option<&mut Option<AttrTokenStream>> {
self.as_mut().and_then(|inner| inner.tokens_mut())
}
}

impl HasTokens for StmtKind {
fn tokens(&self) -> Option<&LazyAttrTokenStream> {
fn tokens(&self) -> Option<&AttrTokenStream> {
match self {
StmtKind::Let(local) => local.tokens.as_ref(),
StmtKind::Item(item) => item.tokens(),
Expand All @@ -156,7 +156,7 @@ impl HasTokens for StmtKind {
StmtKind::MacCall(mac) => mac.tokens.as_ref(),
}
}
fn tokens_mut(&mut self) -> Option<&mut Option<LazyAttrTokenStream>> {
fn tokens_mut(&mut self) -> Option<&mut Option<AttrTokenStream>> {
match self {
StmtKind::Let(local) => Some(&mut local.tokens),
StmtKind::Item(item) => item.tokens_mut(),
Expand All @@ -168,24 +168,24 @@ impl HasTokens for StmtKind {
}

impl HasTokens for Stmt {
fn tokens(&self) -> Option<&LazyAttrTokenStream> {
fn tokens(&self) -> Option<&AttrTokenStream> {
self.kind.tokens()
}
fn tokens_mut(&mut self) -> Option<&mut Option<LazyAttrTokenStream>> {
fn tokens_mut(&mut self) -> Option<&mut Option<AttrTokenStream>> {
self.kind.tokens_mut()
}
}

impl HasTokens for Attribute {
fn tokens(&self) -> Option<&LazyAttrTokenStream> {
fn tokens(&self) -> Option<&AttrTokenStream> {
match &self.kind {
AttrKind::Normal(normal) => normal.tokens.as_ref(),
kind @ AttrKind::DocComment(..) => {
panic!("Called tokens on doc comment attr {kind:?}")
}
}
}
fn tokens_mut(&mut self) -> Option<&mut Option<LazyAttrTokenStream>> {
fn tokens_mut(&mut self) -> Option<&mut Option<AttrTokenStream>> {
Some(match &mut self.kind {
AttrKind::Normal(normal) => &mut normal.tokens,
kind @ AttrKind::DocComment(..) => {
Expand All @@ -196,7 +196,7 @@ impl HasTokens for Attribute {
}

impl HasTokens for Nonterminal {
fn tokens(&self) -> Option<&LazyAttrTokenStream> {
fn tokens(&self) -> Option<&AttrTokenStream> {
match self {
Nonterminal::NtItem(item) => item.tokens(),
Nonterminal::NtStmt(stmt) => stmt.tokens(),
Expand All @@ -209,7 +209,7 @@ impl HasTokens for Nonterminal {
Nonterminal::NtBlock(block) => block.tokens(),
}
}
fn tokens_mut(&mut self) -> Option<&mut Option<LazyAttrTokenStream>> {
fn tokens_mut(&mut self) -> Option<&mut Option<AttrTokenStream>> {
match self {
Nonterminal::NtItem(item) => item.tokens_mut(),
Nonterminal::NtStmt(stmt) => stmt.tokens_mut(),
Expand Down
5 changes: 2 additions & 3 deletions compiler/rustc_ast/src/attr/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@ use crate::ast::{MetaItem, MetaItemKind, NestedMetaItem, NormalAttr};
use crate::ast::{Path, PathSegment, DUMMY_NODE_ID};
use crate::ptr::P;
use crate::token::{self, CommentKind, Delimiter, Token};
use crate::tokenstream::{AttrTokenStream, TokenStream};
use crate::tokenstream::{DelimSpan, Spacing, TokenTree};
use crate::tokenstream::{LazyAttrTokenStream, TokenStream};
use crate::util::comments;
use crate::util::literal::escape_string_symbol;
use rustc_index::bit_set::GrowableBitSet;
Expand Down Expand Up @@ -210,7 +210,6 @@ impl Attribute {
.tokens
.as_ref()
.unwrap_or_else(|| panic!("attribute is missing tokens: {self:?}"))
.to_attr_token_stream()
.to_token_trees(),
),
&AttrKind::DocComment(comment_kind, data) => TokenStream::token_alone(
Expand Down Expand Up @@ -586,7 +585,7 @@ pub fn mk_attr(
pub fn mk_attr_from_item(
g: &AttrIdGenerator,
item: AttrItem,
tokens: Option<LazyAttrTokenStream>,
tokens: Option<AttrTokenStream>,
style: AttrStyle,
span: Span,
) -> Attribute {
Expand Down
Loading
Loading