From c201a0603f1de91b43c4ed3ea69fb5c35346a4c1 Mon Sep 17 00:00:00 2001 From: Caleb Cartwright Date: Mon, 20 Dec 2021 17:55:48 -0600 Subject: [PATCH 1/6] refactor: rename syntux mod to parse --- src/formatting.rs | 4 ++-- src/lib.rs | 4 ++-- src/modules.rs | 4 ++-- src/modules/visitor.rs | 4 ++-- src/parse/mod.rs | 2 ++ src/{syntux => parse}/parser.rs | 2 +- src/{syntux => parse}/session.rs | 2 +- src/rewrite.rs | 2 +- src/source_file.rs | 2 +- src/visitor.rs | 2 +- 10 files changed, 15 insertions(+), 13 deletions(-) create mode 100644 src/parse/mod.rs rename src/{syntux => parse}/parser.rs (99%) rename src/{syntux => parse}/session.rs (99%) diff --git a/src/formatting.rs b/src/formatting.rs index b39480a0ef9..67cf1232f66 100644 --- a/src/formatting.rs +++ b/src/formatting.rs @@ -14,8 +14,8 @@ use crate::config::{Config, FileName, Verbosity}; use crate::formatting::generated::is_generated_file; use crate::issues::BadIssueSeeker; use crate::modules::Module; -use crate::syntux::parser::{DirectoryOwnership, Parser, ParserError}; -use crate::syntux::session::ParseSess; +use crate::parse::parser::{DirectoryOwnership, Parser, ParserError}; +use crate::parse::session::ParseSess; use crate::utils::{contains_skip, count_newlines}; use crate::visitor::FmtVisitor; use crate::{modules, source_file, ErrorKind, FormatReport, Input, Session}; diff --git a/src/lib.rs b/src/lib.rs index 792a1080f0e..f59ebad97ce 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -40,8 +40,8 @@ use crate::emitter::Emitter; use crate::formatting::{FormatErrorMap, FormattingError, ReportedErrors, SourceFile}; use crate::issues::Issue; use crate::modules::ModuleResolutionError; +use crate::parse::parser::DirectoryOwnership; use crate::shape::Indent; -use crate::syntux::parser::DirectoryOwnership; use crate::utils::indent_next_line; pub use crate::config::{ @@ -77,6 +77,7 @@ mod missed_spans; pub(crate) mod modules; mod overflow; mod pairs; +mod parse; mod patterns; mod release_channel; mod reorder; @@ -89,7 +90,6 @@ pub(crate) mod source_map; mod spanned; mod stmt; mod string; -mod syntux; #[cfg(test)] mod test; mod types; diff --git a/src/modules.rs b/src/modules.rs index 9d438a80d94..9c964b274e0 100644 --- a/src/modules.rs +++ b/src/modules.rs @@ -12,10 +12,10 @@ use thiserror::Error; use crate::attr::MetaVisitor; use crate::config::FileName; use crate::items::is_mod_decl; -use crate::syntux::parser::{ +use crate::parse::parser::{ Directory, DirectoryOwnership, ModError, ModulePathSuccess, Parser, ParserError, }; -use crate::syntux::session::ParseSess; +use crate::parse::session::ParseSess; use crate::utils::{contains_skip, mk_sp}; mod visitor; diff --git a/src/modules/visitor.rs b/src/modules/visitor.rs index d5acf3f1cbc..7486a4c4ee1 100644 --- a/src/modules/visitor.rs +++ b/src/modules/visitor.rs @@ -3,8 +3,8 @@ use rustc_ast::visit::Visitor; use rustc_span::Symbol; use crate::attr::MetaVisitor; -use crate::syntux::parser::Parser; -use crate::syntux::session::ParseSess; +use crate::parse::parser::Parser; +use crate::parse::session::ParseSess; pub(crate) struct ModItem { pub(crate) item: ast::Item, diff --git a/src/parse/mod.rs b/src/parse/mod.rs new file mode 100644 index 00000000000..bb7d9ca87d4 --- /dev/null +++ b/src/parse/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod parser; +pub(crate) mod session; diff --git a/src/syntux/parser.rs b/src/parse/parser.rs similarity index 99% rename from src/syntux/parser.rs rename to src/parse/parser.rs index 23d065c9cc9..024b38cebb6 100644 --- a/src/syntux/parser.rs +++ b/src/parse/parser.rs @@ -11,7 +11,7 @@ use rustc_parse::{ use rustc_span::{sym, symbol::kw, Span}; use crate::attr::first_attr_value_str_by_name; -use crate::syntux::session::ParseSess; +use crate::parse::session::ParseSess; use crate::Input; pub(crate) type DirectoryOwnership = rustc_expand::module::DirOwnership; diff --git a/src/syntux/session.rs b/src/parse/session.rs similarity index 99% rename from src/syntux/session.rs rename to src/parse/session.rs index dd7c7352686..624fed0d2de 100644 --- a/src/syntux/session.rs +++ b/src/parse/session.rs @@ -222,7 +222,7 @@ impl ParseSess { } } -// Methods that should be restricted within the syntux module. +// Methods that should be restricted within the parse module. impl ParseSess { pub(super) fn emit_diagnostics(&self, diagnostics: Vec) { for diagnostic in diagnostics { diff --git a/src/rewrite.rs b/src/rewrite.rs index c8abe70141b..4a3bd129d16 100644 --- a/src/rewrite.rs +++ b/src/rewrite.rs @@ -7,9 +7,9 @@ use rustc_ast::ptr; use rustc_span::Span; use crate::config::{Config, IndentStyle}; +use crate::parse::session::ParseSess; use crate::shape::Shape; use crate::skip::SkipContext; -use crate::syntux::session::ParseSess; use crate::visitor::SnippetProvider; use crate::FormatReport; diff --git a/src/source_file.rs b/src/source_file.rs index 853336004d8..56d4ab40038 100644 --- a/src/source_file.rs +++ b/src/source_file.rs @@ -4,7 +4,7 @@ use std::path::Path; use crate::config::FileName; use crate::emitter::{self, Emitter}; -use crate::syntux::session::ParseSess; +use crate::parse::session::ParseSess; use crate::NewlineStyle; #[cfg(test)] diff --git a/src/visitor.rs b/src/visitor.rs index 1896a4744fe..0177689958a 100644 --- a/src/visitor.rs +++ b/src/visitor.rs @@ -16,13 +16,13 @@ use crate::items::{ }; use crate::macros::{macro_style, rewrite_macro, rewrite_macro_def, MacroPosition}; use crate::modules::Module; +use crate::parse::session::ParseSess; use crate::rewrite::{Rewrite, RewriteContext}; use crate::shape::{Indent, Shape}; use crate::skip::{is_skip_attr, SkipContext}; use crate::source_map::{LineRangeUtils, SpanUtils}; use crate::spanned::Spanned; use crate::stmt::Stmt; -use crate::syntux::session::ParseSess; use crate::utils::{ self, contains_skip, count_newlines, depr_skip_annotation, format_unsafety, inner_attributes, last_line_width, mk_sp, ptr_vec_to_ref_vec, rewrite_ident, starts_with_newline, stmt_expr, From 4e877c4c1af2d90d105f9cd2660402f4de4b4961 Mon Sep 17 00:00:00 2001 From: Caleb Cartwright Date: Mon, 20 Dec 2021 18:59:01 -0600 Subject: [PATCH 2/6] refactor: move macro arg parsing to parse mod --- src/macros.rs | 216 ++++------------------------------------ src/parse/macros/mod.rs | 216 ++++++++++++++++++++++++++++++++++++++++ src/parse/mod.rs | 1 + 3 files changed, 234 insertions(+), 199 deletions(-) create mode 100644 src/parse/macros/mod.rs diff --git a/src/macros.rs b/src/macros.rs index a52568be9ea..26a68dfdcf2 100644 --- a/src/macros.rs +++ b/src/macros.rs @@ -16,8 +16,6 @@ use rustc_ast::token::{BinOpToken, DelimToken, Token, TokenKind}; use rustc_ast::tokenstream::{Cursor, Spacing, TokenStream, TokenTree}; use rustc_ast::{ast, ptr}; use rustc_ast_pretty::pprust; -use rustc_parse::parser::{ForceCollect, Parser}; -use rustc_parse::{stream_to_parser, MACRO_ARGUMENTS}; use rustc_span::{ symbol::{self, kw}, BytePos, Span, Symbol, DUMMY_SP, @@ -30,6 +28,7 @@ use crate::config::lists::*; use crate::expr::{rewrite_array, rewrite_assign_rhs, RhsAssignKind}; use crate::lists::{itemize_list, write_list, ListFormatting}; use crate::overflow; +use crate::parse::macros::{build_parser, parse_macro_args, ParsedMacroArgs}; use crate::rewrite::{Rewrite, RewriteContext}; use crate::shape::{Indent, Shape}; use crate::source_map::SpanUtils; @@ -60,7 +59,7 @@ pub(crate) enum MacroArg { } impl MacroArg { - fn is_item(&self) -> bool { + pub(crate) fn is_item(&self) -> bool { match self { MacroArg::Item(..) => true, _ => false, @@ -90,61 +89,6 @@ impl Rewrite for MacroArg { } } -fn build_parser<'a>(context: &RewriteContext<'a>, cursor: Cursor) -> Parser<'a> { - stream_to_parser( - context.parse_sess.inner(), - cursor.collect(), - MACRO_ARGUMENTS, - ) -} - -fn parse_macro_arg<'a, 'b: 'a>(parser: &'a mut Parser<'b>) -> Option { - macro_rules! parse_macro_arg { - ($macro_arg:ident, $parser:expr, $f:expr) => { - let mut cloned_parser = (*parser).clone(); - match $parser(&mut cloned_parser) { - Ok(x) => { - if parser.sess.span_diagnostic.has_errors() { - parser.sess.span_diagnostic.reset_err_count(); - } else { - // Parsing succeeded. - *parser = cloned_parser; - return Some(MacroArg::$macro_arg($f(x)?)); - } - } - Err(mut e) => { - e.cancel(); - parser.sess.span_diagnostic.reset_err_count(); - } - } - }; - } - - parse_macro_arg!( - Expr, - |parser: &mut rustc_parse::parser::Parser<'b>| parser.parse_expr(), - |x: ptr::P| Some(x) - ); - parse_macro_arg!( - Ty, - |parser: &mut rustc_parse::parser::Parser<'b>| parser.parse_ty(), - |x: ptr::P| Some(x) - ); - parse_macro_arg!( - Pat, - |parser: &mut rustc_parse::parser::Parser<'b>| parser.parse_pat_no_top_alt(None), - |x: ptr::P| Some(x) - ); - // `parse_item` returns `Option>`. - parse_macro_arg!( - Item, - |parser: &mut rustc_parse::parser::Parser<'b>| parser.parse_item(ForceCollect::No), - |x: Option>| x - ); - - None -} - /// Rewrite macro name without using pretty-printer if possible. fn rewrite_macro_name( context: &RewriteContext<'_>, @@ -232,25 +176,6 @@ pub(crate) fn rewrite_macro( } } -fn check_keyword<'a, 'b: 'a>(parser: &'a mut Parser<'b>) -> Option { - for &keyword in RUST_KW.iter() { - if parser.token.is_keyword(keyword) - && parser.look_ahead(1, |t| { - t.kind == TokenKind::Eof - || t.kind == TokenKind::Comma - || t.kind == TokenKind::CloseDelim(DelimToken::NoDelim) - }) - { - parser.bump(); - return Some(MacroArg::Keyword( - symbol::Ident::with_dummy_span(keyword), - parser.prev_token.span, - )); - } - } - None -} - fn rewrite_macro_inner( mac: &ast::MacCall, extra_ident: Option, @@ -269,8 +194,9 @@ fn rewrite_macro_inner( let original_style = macro_style(mac, context); let macro_name = rewrite_macro_name(context, &mac.path, extra_ident); + let is_forced_bracket = FORCED_BRACKET_MACROS.contains(&¯o_name[..]); - let style = if FORCED_BRACKET_MACROS.contains(&¯o_name[..]) && !is_nested_macro { + let style = if is_forced_bracket && !is_nested_macro { DelimToken::Bracket } else { original_style @@ -294,67 +220,21 @@ fn rewrite_macro_inner( } // Format well-known macros which cannot be parsed as a valid AST. if macro_name == "lazy_static!" && !has_comment { - if let success @ Some(..) = format_lazy_static(context, shape, &ts) { + if let success @ Some(..) = format_lazy_static(context, shape, ts.trees().collect()) { return success; } } - let mut parser = build_parser(context, ts.trees()); - let mut arg_vec = Vec::new(); - let mut vec_with_semi = false; - let mut trailing_comma = false; - - if DelimToken::Brace != style { - loop { - if let Some(arg) = check_keyword(&mut parser) { - arg_vec.push(arg); - } else if let Some(arg) = parse_macro_arg(&mut parser) { - arg_vec.push(arg); - } else { - return return_macro_parse_failure_fallback(context, shape.indent, mac.span()); - } - - match parser.token.kind { - TokenKind::Eof => break, - TokenKind::Comma => (), - TokenKind::Semi => { - // Try to parse `vec![expr; expr]` - if FORCED_BRACKET_MACROS.contains(&¯o_name[..]) { - parser.bump(); - if parser.token.kind != TokenKind::Eof { - match parse_macro_arg(&mut parser) { - Some(arg) => { - arg_vec.push(arg); - parser.bump(); - if parser.token.kind == TokenKind::Eof && arg_vec.len() == 2 { - vec_with_semi = true; - break; - } - } - None => { - return return_macro_parse_failure_fallback( - context, - shape.indent, - mac.span(), - ); - } - } - } - } - return return_macro_parse_failure_fallback(context, shape.indent, mac.span()); - } - _ if arg_vec.last().map_or(false, MacroArg::is_item) => continue, - _ => return return_macro_parse_failure_fallback(context, shape.indent, mac.span()), - } - - parser.bump(); - - if parser.token.kind == TokenKind::Eof { - trailing_comma = true; - break; - } + let ParsedMacroArgs { + args: arg_vec, + vec_with_semi, + trailing_comma, + } = match parse_macro_args(context, ts, style, is_forced_bracket) { + Some(args) => args, + None => { + return return_macro_parse_failure_fallback(context, shape.indent, mac.span()); } - } + }; if !arg_vec.is_empty() && arg_vec.iter().all(MacroArg::is_item) { return rewrite_macro_with_items( @@ -1179,7 +1059,7 @@ pub(crate) fn convert_try_mac( let path = &pprust::path_to_string(&mac.path); if path == "try" || path == "r#try" { let ts = mac.args.inner_tokens(); - let mut parser = build_parser(context, ts.trees()); + let mut parser = build_parser(context, ts); Some(ast::Expr { id: ast::NodeId::root(), // dummy value @@ -1414,10 +1294,10 @@ impl MacroBranch { fn format_lazy_static( context: &RewriteContext<'_>, shape: Shape, - ts: &TokenStream, + ts: TokenStream, ) -> Option { let mut result = String::with_capacity(1024); - let mut parser = build_parser(context, ts.trees()); + let mut parser = build_parser(context, ts); let nested_shape = shape .block_indent(context.config.tab_spaces()) .with_max_width(context.config); @@ -1528,65 +1408,3 @@ fn rewrite_macro_with_items( result.push_str(trailing_semicolon); Some(result) } - -const RUST_KW: [Symbol; 59] = [ - kw::PathRoot, - kw::DollarCrate, - kw::Underscore, - kw::As, - kw::Box, - kw::Break, - kw::Const, - kw::Continue, - kw::Crate, - kw::Else, - kw::Enum, - kw::Extern, - kw::False, - kw::Fn, - kw::For, - kw::If, - kw::Impl, - kw::In, - kw::Let, - kw::Loop, - kw::Match, - kw::Mod, - kw::Move, - kw::Mut, - kw::Pub, - kw::Ref, - kw::Return, - kw::SelfLower, - kw::SelfUpper, - kw::Static, - kw::Struct, - kw::Super, - kw::Trait, - kw::True, - kw::Type, - kw::Unsafe, - kw::Use, - kw::Where, - kw::While, - kw::Abstract, - kw::Become, - kw::Do, - kw::Final, - kw::Macro, - kw::Override, - kw::Priv, - kw::Typeof, - kw::Unsized, - kw::Virtual, - kw::Yield, - kw::Dyn, - kw::Async, - kw::Try, - kw::UnderscoreLifetime, - kw::StaticLifetime, - kw::Auto, - kw::Catch, - kw::Default, - kw::Union, -]; diff --git a/src/parse/macros/mod.rs b/src/parse/macros/mod.rs new file mode 100644 index 00000000000..e7844c9a4dc --- /dev/null +++ b/src/parse/macros/mod.rs @@ -0,0 +1,216 @@ +use rustc_ast::token::{BinOpToken, DelimToken, Token, TokenKind}; +use rustc_ast::tokenstream::{Cursor, Spacing, TokenStream, TokenTree}; +use rustc_ast::{ast, ptr}; +use rustc_parse::parser::{ForceCollect, Parser}; +use rustc_parse::{stream_to_parser, MACRO_ARGUMENTS}; +use rustc_span::{ + symbol::{self, kw}, + BytePos, Span, Symbol, DUMMY_SP, +}; + +use crate::macros::MacroArg; +use crate::rewrite::{Rewrite, RewriteContext}; + +pub(crate) fn build_parser<'a>(context: &RewriteContext<'a>, tokens: TokenStream) -> Parser<'a> { + stream_to_parser(context.parse_sess.inner(), tokens, MACRO_ARGUMENTS) +} + +fn parse_macro_arg<'a, 'b: 'a>(parser: &'a mut Parser<'b>) -> Option { + macro_rules! parse_macro_arg { + ($macro_arg:ident, $parser:expr, $f:expr) => { + let mut cloned_parser = (*parser).clone(); + match $parser(&mut cloned_parser) { + Ok(x) => { + if parser.sess.span_diagnostic.has_errors() { + parser.sess.span_diagnostic.reset_err_count(); + } else { + // Parsing succeeded. + *parser = cloned_parser; + return Some(MacroArg::$macro_arg($f(x)?)); + } + } + Err(mut e) => { + e.cancel(); + parser.sess.span_diagnostic.reset_err_count(); + } + } + }; + } + + parse_macro_arg!( + Expr, + |parser: &mut rustc_parse::parser::Parser<'b>| parser.parse_expr(), + |x: ptr::P| Some(x) + ); + parse_macro_arg!( + Ty, + |parser: &mut rustc_parse::parser::Parser<'b>| parser.parse_ty(), + |x: ptr::P| Some(x) + ); + parse_macro_arg!( + Pat, + |parser: &mut rustc_parse::parser::Parser<'b>| parser.parse_pat_no_top_alt(None), + |x: ptr::P| Some(x) + ); + // `parse_item` returns `Option>`. + parse_macro_arg!( + Item, + |parser: &mut rustc_parse::parser::Parser<'b>| parser.parse_item(ForceCollect::No), + |x: Option>| x + ); + + None +} + +pub(crate) struct ParsedMacroArgs { + pub(crate) vec_with_semi: bool, + pub(crate) trailing_comma: bool, + pub(crate) args: Vec, +} + +fn check_keyword<'a, 'b: 'a>(parser: &'a mut Parser<'b>) -> Option { + for &keyword in RUST_KW.iter() { + if parser.token.is_keyword(keyword) + && parser.look_ahead(1, |t| { + t.kind == TokenKind::Eof + || t.kind == TokenKind::Comma + || t.kind == TokenKind::CloseDelim(DelimToken::NoDelim) + }) + { + parser.bump(); + return Some(MacroArg::Keyword( + symbol::Ident::with_dummy_span(keyword), + parser.prev_token.span, + )); + } + } + None +} + +pub(crate) fn parse_macro_args( + context: &RewriteContext<'_>, + tokens: TokenStream, + style: DelimToken, + forced_bracket: bool, +) -> Option { + let mut parser = build_parser(context, tokens); + let mut args = Vec::new(); + let mut vec_with_semi = false; + let mut trailing_comma = false; + + if DelimToken::Brace != style { + loop { + if let Some(arg) = check_keyword(&mut parser) { + args.push(arg); + } else if let Some(arg) = parse_macro_arg(&mut parser) { + args.push(arg); + } else { + return None; + } + + match parser.token.kind { + TokenKind::Eof => break, + TokenKind::Comma => (), + TokenKind::Semi => { + // Try to parse `vec![expr; expr]` + if forced_bracket { + parser.bump(); + if parser.token.kind != TokenKind::Eof { + match parse_macro_arg(&mut parser) { + Some(arg) => { + args.push(arg); + parser.bump(); + if parser.token.kind == TokenKind::Eof && args.len() == 2 { + vec_with_semi = true; + break; + } + } + None => { + return None; + } + } + } + } + return None; + } + _ if args.last().map_or(false, MacroArg::is_item) => continue, + _ => return None, + } + + parser.bump(); + + if parser.token.kind == TokenKind::Eof { + trailing_comma = true; + break; + } + } + } + + Some(ParsedMacroArgs { + vec_with_semi, + trailing_comma, + args, + }) +} + +const RUST_KW: [Symbol; 59] = [ + kw::PathRoot, + kw::DollarCrate, + kw::Underscore, + kw::As, + kw::Box, + kw::Break, + kw::Const, + kw::Continue, + kw::Crate, + kw::Else, + kw::Enum, + kw::Extern, + kw::False, + kw::Fn, + kw::For, + kw::If, + kw::Impl, + kw::In, + kw::Let, + kw::Loop, + kw::Match, + kw::Mod, + kw::Move, + kw::Mut, + kw::Pub, + kw::Ref, + kw::Return, + kw::SelfLower, + kw::SelfUpper, + kw::Static, + kw::Struct, + kw::Super, + kw::Trait, + kw::True, + kw::Type, + kw::Unsafe, + kw::Use, + kw::Where, + kw::While, + kw::Abstract, + kw::Become, + kw::Do, + kw::Final, + kw::Macro, + kw::Override, + kw::Priv, + kw::Typeof, + kw::Unsized, + kw::Virtual, + kw::Yield, + kw::Dyn, + kw::Async, + kw::Try, + kw::UnderscoreLifetime, + kw::StaticLifetime, + kw::Auto, + kw::Catch, + kw::Default, + kw::Union, +]; diff --git a/src/parse/mod.rs b/src/parse/mod.rs index bb7d9ca87d4..5e88826ea8c 100644 --- a/src/parse/mod.rs +++ b/src/parse/mod.rs @@ -1,2 +1,3 @@ +pub(crate) mod macros; pub(crate) mod parser; pub(crate) mod session; From e6d71b7199324e684b33da2226a3ed27c51ec728 Mon Sep 17 00:00:00 2001 From: Caleb Cartwright Date: Mon, 20 Dec 2021 19:38:00 -0600 Subject: [PATCH 3/6] refactor: move lazy_static parsing to parse mod --- src/macros.rs | 43 ++++------------------------ src/parse/macros/lazy_static.rs | 50 +++++++++++++++++++++++++++++++++ src/parse/macros/mod.rs | 2 ++ 3 files changed, 58 insertions(+), 37 deletions(-) create mode 100644 src/parse/macros/lazy_static.rs diff --git a/src/macros.rs b/src/macros.rs index 26a68dfdcf2..b14b67f2a0a 100644 --- a/src/macros.rs +++ b/src/macros.rs @@ -28,6 +28,7 @@ use crate::config::lists::*; use crate::expr::{rewrite_array, rewrite_assign_rhs, RhsAssignKind}; use crate::lists::{itemize_list, write_list, ListFormatting}; use crate::overflow; +use crate::parse::macros::lazy_static::parse_lazy_static; use crate::parse::macros::{build_parser, parse_macro_args, ParsedMacroArgs}; use crate::rewrite::{Rewrite, RewriteContext}; use crate::shape::{Indent, Shape}; @@ -1297,7 +1298,6 @@ fn format_lazy_static( ts: TokenStream, ) -> Option { let mut result = String::with_capacity(1024); - let mut parser = build_parser(context, ts); let nested_shape = shape .block_indent(context.config.tab_spaces()) .with_max_width(context.config); @@ -1305,42 +1305,11 @@ fn format_lazy_static( result.push_str("lazy_static! {"); result.push_str(&nested_shape.indent.to_string_with_newline(context.config)); - macro_rules! parse_or { - ($method:ident $(,)* $($arg:expr),* $(,)*) => { - match parser.$method($($arg,)*) { - Ok(val) => { - if parser.sess.span_diagnostic.has_errors() { - parser.sess.span_diagnostic.reset_err_count(); - return None; - } else { - val - } - } - Err(mut err) => { - err.cancel(); - parser.sess.span_diagnostic.reset_err_count(); - return None; - } - } - } - } - - while parser.token.kind != TokenKind::Eof { - // Parse a `lazy_static!` item. - let vis = crate::utils::format_visibility( - context, - &parse_or!(parse_visibility, rustc_parse::parser::FollowedByType::No), - ); - parser.eat_keyword(kw::Static); - parser.eat_keyword(kw::Ref); - let id = parse_or!(parse_ident); - parser.eat(&TokenKind::Colon); - let ty = parse_or!(parse_ty); - parser.eat(&TokenKind::Eq); - let expr = parse_or!(parse_expr); - parser.eat(&TokenKind::Semi); - + let parsed_elems = parse_lazy_static(context, ts)?; + let last = parsed_elems.len() - 1; + for (i, (vis, id, ty, expr)) in parsed_elems.iter().enumerate() { // Rewrite as a static item. + let vis = crate::utils::format_visibility(context, vis); let mut stmt = String::with_capacity(128); stmt.push_str(&format!( "{}static ref {}: {} =", @@ -1356,7 +1325,7 @@ fn format_lazy_static( nested_shape.sub_width(1)?, )?); result.push(';'); - if parser.token.kind != TokenKind::Eof { + if i != last { result.push_str(&nested_shape.indent.to_string_with_newline(context.config)); } } diff --git a/src/parse/macros/lazy_static.rs b/src/parse/macros/lazy_static.rs new file mode 100644 index 00000000000..9c8651aa3fa --- /dev/null +++ b/src/parse/macros/lazy_static.rs @@ -0,0 +1,50 @@ +use rustc_ast::ast; +use rustc_ast::ptr::P; +use rustc_ast::token::TokenKind; +use rustc_ast::tokenstream::TokenStream; +use rustc_span::symbol::{self, kw}; + +use crate::rewrite::RewriteContext; + +pub(crate) fn parse_lazy_static( + context: &RewriteContext<'_>, + ts: TokenStream, +) -> Option, P)>> { + let mut result = vec![]; + let mut parser = super::build_parser(context, ts); + macro_rules! parse_or { + ($method:ident $(,)* $($arg:expr),* $(,)*) => { + match parser.$method($($arg,)*) { + Ok(val) => { + if parser.sess.span_diagnostic.has_errors() { + parser.sess.span_diagnostic.reset_err_count(); + return None; + } else { + val + } + } + Err(mut err) => { + err.cancel(); + parser.sess.span_diagnostic.reset_err_count(); + return None; + } + } + } + } + + while parser.token.kind != TokenKind::Eof { + // Parse a `lazy_static!` item. + let vis = parse_or!(parse_visibility, rustc_parse::parser::FollowedByType::No); + parser.eat_keyword(kw::Static); + parser.eat_keyword(kw::Ref); + let id = parse_or!(parse_ident); + parser.eat(&TokenKind::Colon); + let ty = parse_or!(parse_ty); + parser.eat(&TokenKind::Eq); + let expr = parse_or!(parse_expr); + parser.eat(&TokenKind::Semi); + result.push((vis, id, ty, expr)); + } + + Some(result) +} diff --git a/src/parse/macros/mod.rs b/src/parse/macros/mod.rs index e7844c9a4dc..7115302a479 100644 --- a/src/parse/macros/mod.rs +++ b/src/parse/macros/mod.rs @@ -11,6 +11,8 @@ use rustc_span::{ use crate::macros::MacroArg; use crate::rewrite::{Rewrite, RewriteContext}; +pub(crate) mod lazy_static; + pub(crate) fn build_parser<'a>(context: &RewriteContext<'a>, tokens: TokenStream) -> Parser<'a> { stream_to_parser(context.parse_sess.inner(), tokens, MACRO_ARGUMENTS) } From d6e5b0c6da2f0563d3a1cc2b135eaba8b08cea36 Mon Sep 17 00:00:00 2001 From: Caleb Cartwright Date: Mon, 20 Dec 2021 19:44:23 -0600 Subject: [PATCH 4/6] refactor: extract final rustc_parse touchpoint from macros.rs --- src/macros.rs | 5 ++--- src/parse/macros/mod.rs | 10 +++++++++- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/macros.rs b/src/macros.rs index b14b67f2a0a..f29552caf8d 100644 --- a/src/macros.rs +++ b/src/macros.rs @@ -29,7 +29,7 @@ use crate::expr::{rewrite_array, rewrite_assign_rhs, RhsAssignKind}; use crate::lists::{itemize_list, write_list, ListFormatting}; use crate::overflow; use crate::parse::macros::lazy_static::parse_lazy_static; -use crate::parse::macros::{build_parser, parse_macro_args, ParsedMacroArgs}; +use crate::parse::macros::{parse_expr, parse_macro_args, ParsedMacroArgs}; use crate::rewrite::{Rewrite, RewriteContext}; use crate::shape::{Indent, Shape}; use crate::source_map::SpanUtils; @@ -1060,11 +1060,10 @@ pub(crate) fn convert_try_mac( let path = &pprust::path_to_string(&mac.path); if path == "try" || path == "r#try" { let ts = mac.args.inner_tokens(); - let mut parser = build_parser(context, ts); Some(ast::Expr { id: ast::NodeId::root(), // dummy value - kind: ast::ExprKind::Try(parser.parse_expr().ok()?), + kind: ast::ExprKind::Try(parse_expr(context, ts)?), span: mac.span(), // incorrect span, but shouldn't matter too much attrs: ast::AttrVec::new(), tokens: None, diff --git a/src/parse/macros/mod.rs b/src/parse/macros/mod.rs index 7115302a479..414b72f2921 100644 --- a/src/parse/macros/mod.rs +++ b/src/parse/macros/mod.rs @@ -13,7 +13,7 @@ use crate::rewrite::{Rewrite, RewriteContext}; pub(crate) mod lazy_static; -pub(crate) fn build_parser<'a>(context: &RewriteContext<'a>, tokens: TokenStream) -> Parser<'a> { +fn build_parser<'a>(context: &RewriteContext<'a>, tokens: TokenStream) -> Parser<'a> { stream_to_parser(context.parse_sess.inner(), tokens, MACRO_ARGUMENTS) } @@ -155,6 +155,14 @@ pub(crate) fn parse_macro_args( }) } +pub(crate) fn parse_expr( + context: &RewriteContext<'_>, + tokens: TokenStream, +) -> Option> { + let mut parser = build_parser(context, tokens); + parser.parse_expr().ok() +} + const RUST_KW: [Symbol; 59] = [ kw::PathRoot, kw::DollarCrate, From 025769b421a227bd31c71ae60529f1bbc0f1a0cf Mon Sep 17 00:00:00 2001 From: Caleb Cartwright Date: Mon, 20 Dec 2021 20:28:55 -0600 Subject: [PATCH 5/6] refactor: encapsulate cfg_if parsing within parse mod --- src/modules/visitor.rs | 4 +- src/parse/macros/cfg_if.rs | 89 ++++++++++++++++++++++++++++++++++++++ src/parse/macros/mod.rs | 8 +++- src/parse/parser.rs | 80 ---------------------------------- 4 files changed, 98 insertions(+), 83 deletions(-) create mode 100644 src/parse/macros/cfg_if.rs diff --git a/src/modules/visitor.rs b/src/modules/visitor.rs index 7486a4c4ee1..ea67977c17a 100644 --- a/src/modules/visitor.rs +++ b/src/modules/visitor.rs @@ -3,7 +3,7 @@ use rustc_ast::visit::Visitor; use rustc_span::Symbol; use crate::attr::MetaVisitor; -use crate::parse::parser::Parser; +use crate::parse::macros::cfg_if::parse_cfg_if; use crate::parse::session::ParseSess; pub(crate) struct ModItem { @@ -62,7 +62,7 @@ impl<'a, 'ast: 'a> CfgIfVisitor<'a> { } }; - let items = Parser::parse_cfg_if(self.parse_sess, mac)?; + let items = parse_cfg_if(self.parse_sess, mac)?; self.mods .append(&mut items.into_iter().map(|item| ModItem { item }).collect()); diff --git a/src/parse/macros/cfg_if.rs b/src/parse/macros/cfg_if.rs new file mode 100644 index 00000000000..e10fbe64bcd --- /dev/null +++ b/src/parse/macros/cfg_if.rs @@ -0,0 +1,89 @@ +use std::panic::{catch_unwind, AssertUnwindSafe}; + +use rustc_ast::ast; +use rustc_ast::token::{DelimToken, TokenKind}; +use rustc_parse::parser::ForceCollect; +use rustc_span::symbol::kw; + +use crate::parse::macros::build_stream_parser; +use crate::parse::session::ParseSess; + +pub(crate) fn parse_cfg_if<'a>( + sess: &'a ParseSess, + mac: &'a ast::MacCall, +) -> Result, &'static str> { + match catch_unwind(AssertUnwindSafe(|| parse_cfg_if_inner(sess, mac))) { + Ok(Ok(items)) => Ok(items), + Ok(err @ Err(_)) => err, + Err(..) => Err("failed to parse cfg_if!"), + } +} + +fn parse_cfg_if_inner<'a>( + sess: &'a ParseSess, + mac: &'a ast::MacCall, +) -> Result, &'static str> { + let ts = mac.args.inner_tokens(); + let mut parser = build_stream_parser(sess.inner(), ts); + + let mut items = vec![]; + let mut process_if_cfg = true; + + while parser.token.kind != TokenKind::Eof { + if process_if_cfg { + if !parser.eat_keyword(kw::If) { + return Err("Expected `if`"); + } + // Inner attributes are not actually syntactically permitted here, but we don't + // care about inner vs outer attributes in this position. Our purpose with this + // special case parsing of cfg_if macros is to ensure we can correctly resolve + // imported modules that may have a custom `path` defined. + // + // As such, we just need to advance the parser past the attribute and up to + // to the opening brace. + // See also https://github.com/rust-lang/rust/pull/79433 + parser + .parse_attribute(rustc_parse::parser::attr::InnerAttrPolicy::Permitted) + .map_err(|_| "Failed to parse attributes")?; + } + + if !parser.eat(&TokenKind::OpenDelim(DelimToken::Brace)) { + return Err("Expected an opening brace"); + } + + while parser.token != TokenKind::CloseDelim(DelimToken::Brace) + && parser.token.kind != TokenKind::Eof + { + let item = match parser.parse_item(ForceCollect::No) { + Ok(Some(item_ptr)) => item_ptr.into_inner(), + Ok(None) => continue, + Err(mut err) => { + err.cancel(); + parser.sess.span_diagnostic.reset_err_count(); + return Err( + "Expected item inside cfg_if block, but failed to parse it as an item", + ); + } + }; + if let ast::ItemKind::Mod(..) = item.kind { + items.push(item); + } + } + + if !parser.eat(&TokenKind::CloseDelim(DelimToken::Brace)) { + return Err("Expected a closing brace"); + } + + if parser.eat(&TokenKind::Eof) { + break; + } + + if !parser.eat_keyword(kw::Else) { + return Err("Expected `else`"); + } + + process_if_cfg = parser.token.is_keyword(kw::If); + } + + Ok(items) +} diff --git a/src/parse/macros/mod.rs b/src/parse/macros/mod.rs index 414b72f2921..fbd8c3f9095 100644 --- a/src/parse/macros/mod.rs +++ b/src/parse/macros/mod.rs @@ -3,6 +3,7 @@ use rustc_ast::tokenstream::{Cursor, Spacing, TokenStream, TokenTree}; use rustc_ast::{ast, ptr}; use rustc_parse::parser::{ForceCollect, Parser}; use rustc_parse::{stream_to_parser, MACRO_ARGUMENTS}; +use rustc_session::parse::ParseSess; use rustc_span::{ symbol::{self, kw}, BytePos, Span, Symbol, DUMMY_SP, @@ -11,10 +12,15 @@ use rustc_span::{ use crate::macros::MacroArg; use crate::rewrite::{Rewrite, RewriteContext}; +pub(crate) mod cfg_if; pub(crate) mod lazy_static; +fn build_stream_parser<'a>(sess: &'a ParseSess, tokens: TokenStream) -> Parser<'a> { + stream_to_parser(sess, tokens, MACRO_ARGUMENTS) +} + fn build_parser<'a>(context: &RewriteContext<'a>, tokens: TokenStream) -> Parser<'a> { - stream_to_parser(context.parse_sess.inner(), tokens, MACRO_ARGUMENTS) + build_stream_parser(context.parse_sess.inner(), tokens) } fn parse_macro_arg<'a, 'b: 'a>(parser: &'a mut Parser<'b>) -> Option { diff --git a/src/parse/parser.rs b/src/parse/parser.rs index 024b38cebb6..8efd2bf257b 100644 --- a/src/parse/parser.rs +++ b/src/parse/parser.rs @@ -175,84 +175,4 @@ impl<'a> Parser<'a> { Err(_) => Err(ParserError::ParsePanicError), } } - - pub(crate) fn parse_cfg_if( - sess: &'a ParseSess, - mac: &'a ast::MacCall, - ) -> Result, &'static str> { - match catch_unwind(AssertUnwindSafe(|| Parser::parse_cfg_if_inner(sess, mac))) { - Ok(Ok(items)) => Ok(items), - Ok(err @ Err(_)) => err, - Err(..) => Err("failed to parse cfg_if!"), - } - } - - fn parse_cfg_if_inner( - sess: &'a ParseSess, - mac: &'a ast::MacCall, - ) -> Result, &'static str> { - let token_stream = mac.args.inner_tokens(); - let mut parser = rustc_parse::stream_to_parser(sess.inner(), token_stream, Some("")); - - let mut items = vec![]; - let mut process_if_cfg = true; - - while parser.token.kind != TokenKind::Eof { - if process_if_cfg { - if !parser.eat_keyword(kw::If) { - return Err("Expected `if`"); - } - // Inner attributes are not actually syntactically permitted here, but we don't - // care about inner vs outer attributes in this position. Our purpose with this - // special case parsing of cfg_if macros is to ensure we can correctly resolve - // imported modules that may have a custom `path` defined. - // - // As such, we just need to advance the parser past the attribute and up to - // to the opening brace. - // See also https://github.com/rust-lang/rust/pull/79433 - parser - .parse_attribute(rustc_parse::parser::attr::InnerAttrPolicy::Permitted) - .map_err(|_| "Failed to parse attributes")?; - } - - if !parser.eat(&TokenKind::OpenDelim(DelimToken::Brace)) { - return Err("Expected an opening brace"); - } - - while parser.token != TokenKind::CloseDelim(DelimToken::Brace) - && parser.token.kind != TokenKind::Eof - { - let item = match parser.parse_item(ForceCollect::No) { - Ok(Some(item_ptr)) => item_ptr.into_inner(), - Ok(None) => continue, - Err(mut err) => { - err.cancel(); - parser.sess.span_diagnostic.reset_err_count(); - return Err( - "Expected item inside cfg_if block, but failed to parse it as an item", - ); - } - }; - if let ast::ItemKind::Mod(..) = item.kind { - items.push(item); - } - } - - if !parser.eat(&TokenKind::CloseDelim(DelimToken::Brace)) { - return Err("Expected a closing brace"); - } - - if parser.eat(&TokenKind::Eof) { - break; - } - - if !parser.eat_keyword(kw::Else) { - return Err("Expected `else`"); - } - - process_if_cfg = parser.token.is_keyword(kw::If); - } - - Ok(items) - } } From 9ebd501af26405e267d982d7dc21011b322f7627 Mon Sep 17 00:00:00 2001 From: Caleb Cartwright Date: Mon, 20 Dec 2021 20:33:20 -0600 Subject: [PATCH 6/6] chore: cleanup unused imports --- src/parse/macros/mod.rs | 12 +++++------- src/parse/parser.rs | 9 +++------ 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/src/parse/macros/mod.rs b/src/parse/macros/mod.rs index fbd8c3f9095..7e12f82af15 100644 --- a/src/parse/macros/mod.rs +++ b/src/parse/macros/mod.rs @@ -1,16 +1,14 @@ -use rustc_ast::token::{BinOpToken, DelimToken, Token, TokenKind}; -use rustc_ast::tokenstream::{Cursor, Spacing, TokenStream, TokenTree}; +use rustc_ast::token::{DelimToken, TokenKind}; +use rustc_ast::tokenstream::TokenStream; use rustc_ast::{ast, ptr}; use rustc_parse::parser::{ForceCollect, Parser}; use rustc_parse::{stream_to_parser, MACRO_ARGUMENTS}; use rustc_session::parse::ParseSess; -use rustc_span::{ - symbol::{self, kw}, - BytePos, Span, Symbol, DUMMY_SP, -}; +use rustc_span::symbol::{self, kw}; +use rustc_span::Symbol; use crate::macros::MacroArg; -use crate::rewrite::{Rewrite, RewriteContext}; +use crate::rewrite::RewriteContext; pub(crate) mod cfg_if; pub(crate) mod lazy_static; diff --git a/src/parse/parser.rs b/src/parse/parser.rs index 8efd2bf257b..657217633f4 100644 --- a/src/parse/parser.rs +++ b/src/parse/parser.rs @@ -1,14 +1,11 @@ use std::panic::{catch_unwind, AssertUnwindSafe}; use std::path::{Path, PathBuf}; -use rustc_ast::token::{DelimToken, TokenKind}; +use rustc_ast::token::TokenKind; use rustc_ast::{ast, ptr}; use rustc_errors::Diagnostic; -use rustc_parse::{ - new_parser_from_file, - parser::{ForceCollect, Parser as RawParser}, -}; -use rustc_span::{sym, symbol::kw, Span}; +use rustc_parse::{new_parser_from_file, parser::Parser as RawParser}; +use rustc_span::{sym, Span}; use crate::attr::first_attr_value_str_by_name; use crate::parse::session::ParseSess;