From e1ae0fa055bf358cec14d41e7ddd96cd8964eb9d Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Wed, 29 May 2024 16:20:19 +1000 Subject: [PATCH 01/14] `UNICODE_ARRAY` and `ASCII_ARRAY` fixes. - Avoid unnecessary escaping of single quotes within string literals. - Add a missing blank line between two `UNICODE_ARRAY` sections. --- .../rustc_parse/src/lexer/unicode_chars.rs | 75 ++++++++++--------- 1 file changed, 38 insertions(+), 37 deletions(-) diff --git a/compiler/rustc_parse/src/lexer/unicode_chars.rs b/compiler/rustc_parse/src/lexer/unicode_chars.rs index c9470151a7ba..8eb299108d10 100644 --- a/compiler/rustc_parse/src/lexer/unicode_chars.rs +++ b/compiler/rustc_parse/src/lexer/unicode_chars.rs @@ -129,42 +129,42 @@ pub(super) const UNICODE_ARRAY: &[(char, &str, &str)] = &[ ('。', "Ideographic Full Stop", "."), ('︒', "Presentation Form For Vertical Ideographic Full Stop", "."), - ('՝', "Armenian Comma", "\'"), - ('＇', "Fullwidth Apostrophe", "\'"), - ('‘', "Left Single Quotation Mark", "\'"), - ('’', "Right Single Quotation Mark", "\'"), - ('‛', "Single High-Reversed-9 Quotation Mark", "\'"), - ('′', "Prime", "\'"), - ('‵', "Reversed Prime", "\'"), - ('՚', "Armenian Apostrophe", "\'"), - ('׳', "Hebrew Punctuation Geresh", "\'"), - ('`', "Grave Accent", "\'"), - ('`', "Greek Varia", "\'"), - ('｀', "Fullwidth Grave Accent", "\'"), - ('´', "Acute Accent", "\'"), - ('΄', "Greek Tonos", "\'"), - ('´', "Greek Oxia", "\'"), - ('᾽', "Greek Koronis", "\'"), - ('᾿', "Greek Psili", "\'"), - ('῾', "Greek Dasia", "\'"), - ('ʹ', "Modifier Letter Prime", "\'"), - ('ʹ', "Greek Numeral Sign", "\'"), - ('ˈ', "Modifier Letter Vertical Line", "\'"), - ('ˊ', "Modifier Letter Acute Accent", "\'"), - ('ˋ', "Modifier Letter Grave Accent", "\'"), - ('˴', "Modifier Letter Middle Grave Accent", "\'"), - ('ʻ', "Modifier Letter Turned Comma", "\'"), - ('ʽ', "Modifier Letter Reversed Comma", "\'"), - ('ʼ', "Modifier Letter Apostrophe", "\'"), - ('ʾ', "Modifier Letter Right Half Ring", "\'"), - ('ꞌ', "Latin Small Letter Saltillo", "\'"), - ('י', "Hebrew Letter Yod", "\'"), - ('ߴ', "Nko High Tone Apostrophe", "\'"), - ('ߵ', "Nko Low Tone Apostrophe", "\'"), - ('ᑊ', "Canadian Syllabics West-Cree P", "\'"), - ('ᛌ', "Runic Letter Short-Twig-Sol S", "\'"), - ('𖽑', "Miao Sign Aspiration", "\'"), - ('𖽒', "Miao Sign Reformed Voicing", "\'"), + ('՝', "Armenian Comma", "'"), + ('＇', "Fullwidth Apostrophe", "'"), + ('‘', "Left Single Quotation Mark", "'"), + ('’', "Right Single Quotation Mark", "'"), + ('‛', "Single High-Reversed-9 Quotation Mark", "'"), + ('′', "Prime", "'"), + ('‵', "Reversed Prime", "'"), + ('՚', "Armenian Apostrophe", "'"), + ('׳', "Hebrew Punctuation Geresh", "'"), + ('`', "Grave Accent", "'"), + ('`', "Greek Varia", "'"), + ('｀', "Fullwidth Grave Accent", "'"), + ('´', "Acute Accent", "'"), + ('΄', "Greek Tonos", "'"), + ('´', "Greek Oxia", "'"), + ('᾽', "Greek Koronis", "'"), + ('᾿', "Greek Psili", "'"), + ('῾', "Greek Dasia", "'"), + ('ʹ', "Modifier Letter Prime", "'"), + ('ʹ', "Greek Numeral Sign", "'"), + ('ˈ', "Modifier Letter Vertical Line", "'"), + ('ˊ', "Modifier Letter Acute Accent", "'"), + ('ˋ', "Modifier Letter Grave Accent", "'"), + ('˴', "Modifier Letter Middle Grave Accent", "'"), + ('ʻ', "Modifier Letter Turned Comma", "'"), + ('ʽ', "Modifier Letter Reversed Comma", "'"), + ('ʼ', "Modifier Letter Apostrophe", "'"), + ('ʾ', "Modifier Letter Right Half Ring", "'"), + ('ꞌ', "Latin Small Letter Saltillo", "'"), + ('י', "Hebrew Letter Yod", "'"), + ('ߴ', "Nko High Tone Apostrophe", "'"), + ('ߵ', "Nko Low Tone Apostrophe", "'"), + ('ᑊ', "Canadian Syllabics West-Cree P", "'"), + ('ᛌ', "Runic Letter Short-Twig-Sol S", "'"), + ('𖽑', "Miao Sign Aspiration", "'"), + ('𖽒', "Miao Sign Reformed Voicing", "'"), ('᳓', "Vedic Sign Nihshvasa", "\""), ('＂', "Fullwidth Quotation Mark", "\""), @@ -298,6 +298,7 @@ pub(super) const UNICODE_ARRAY: &[(char, &str, &str)] = &[ ('〉', "Right Angle Bracket", ">"), ('》', "Right Double Angle Bracket", ">"), ('＞', "Fullwidth Greater-Than Sign", ">"), + ('⩵', "Two Consecutive Equals Signs", "==") ]; @@ -332,7 +333,7 @@ const ASCII_ARRAY: &[(&str, &str, Option)] = &[ (">", "Greater-Than Sign", Some(token::Gt)), // FIXME: Literals are already lexed by this point, so we can't recover gracefully just by // spitting the correct token out. - ("\'", "Single Quote", None), + ("'", "Single Quote", None), ("\"", "Quotation Mark", None), ]; From d1215da26e7848bd925a9fffecdaa7ea51b360c3 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Fri, 31 May 2024 09:23:35 +1000 Subject: [PATCH 02/14] Don't use the word "parse" for lexing operations. Lexing converts source text into a token stream. Parsing converts a token stream into AST fragments. This commit renames several lexing operations that have "parse" in the name. I think these names have been subtly confusing me for years. This is just a `s/parse/lex/` on function names, with one exception: `parse_stream_from_source_str` becomes `source_str_to_stream`, to make it consistent with the existing `source_file_to_stream`. The commit also moves that function's location in the file to be just above `source_file_to_stream`. The commit also cleans up a few comments along the way. --- .../rustc_expand/src/proc_macro_server.rs | 4 +- compiler/rustc_parse/src/lexer/mod.rs | 4 +- compiler/rustc_parse/src/lexer/tokentrees.rs | 47 +++++++++---------- compiler/rustc_parse/src/lib.rs | 24 +++++----- .../passes/lint/check_code_block_syntax.rs | 11 ++--- 5 files changed, 41 insertions(+), 49 deletions(-) diff --git a/compiler/rustc_expand/src/proc_macro_server.rs b/compiler/rustc_expand/src/proc_macro_server.rs index c2e799abae80..3b167c42e57d 100644 --- a/compiler/rustc_expand/src/proc_macro_server.rs +++ b/compiler/rustc_expand/src/proc_macro_server.rs @@ -13,7 +13,7 @@ use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::sync::Lrc; use rustc_errors::{Diag, ErrorGuaranteed, MultiSpan, PResult}; use rustc_parse::lexer::nfc_normalize; -use rustc_parse::parse_stream_from_source_str; +use rustc_parse::source_str_to_stream; use rustc_session::parse::ParseSess; use rustc_span::def_id::CrateNum; use rustc_span::symbol::{self, sym, Symbol}; @@ -538,7 +538,7 @@ impl server::TokenStream for Rustc<'_, '_> { } fn from_str(&mut self, src: &str) -> Self::TokenStream { - parse_stream_from_source_str( + source_str_to_stream( FileName::proc_macro_source_code(src), src.to_string(), self.psess(), diff --git a/compiler/rustc_parse/src/lexer/mod.rs b/compiler/rustc_parse/src/lexer/mod.rs index d3a6a0339784..43f4963b27ac 100644 --- a/compiler/rustc_parse/src/lexer/mod.rs +++ b/compiler/rustc_parse/src/lexer/mod.rs @@ -42,7 +42,7 @@ pub(crate) struct UnmatchedDelim { pub candidate_span: Option, } -pub(crate) fn parse_token_trees<'psess, 'src>( +pub(crate) fn lex_token_trees<'psess, 'src>( psess: &'psess ParseSess, mut src: &'src str, mut start_pos: BytePos, @@ -66,7 +66,7 @@ pub(crate) fn parse_token_trees<'psess, 'src>( last_lifetime: None, }; let (stream, res, unmatched_delims) = - tokentrees::TokenTreesReader::parse_all_token_trees(string_reader); + tokentrees::TokenTreesReader::lex_all_token_trees(string_reader); match res { Ok(()) if unmatched_delims.is_empty() => Ok(stream), _ => { diff --git a/compiler/rustc_parse/src/lexer/tokentrees.rs b/compiler/rustc_parse/src/lexer/tokentrees.rs index eabe0226b2fb..fcbfa108d066 100644 --- a/compiler/rustc_parse/src/lexer/tokentrees.rs +++ b/compiler/rustc_parse/src/lexer/tokentrees.rs @@ -17,7 +17,7 @@ pub(super) struct TokenTreesReader<'psess, 'src> { } impl<'psess, 'src> TokenTreesReader<'psess, 'src> { - pub(super) fn parse_all_token_trees( + pub(super) fn lex_all_token_trees( string_reader: StringReader<'psess, 'src>, ) -> (TokenStream, Result<(), Vec>>, Vec) { let mut tt_reader = TokenTreesReader { @@ -25,14 +25,13 @@ impl<'psess, 'src> TokenTreesReader<'psess, 'src> { token: Token::dummy(), diag_info: TokenTreeDiagInfo::default(), }; - let (_open_spacing, stream, res) = - tt_reader.parse_token_trees(/* is_delimited */ false); + let (_open_spacing, stream, res) = tt_reader.lex_token_trees(/* is_delimited */ false); (stream, res, tt_reader.diag_info.unmatched_delims) } - // Parse a stream of tokens into a list of `TokenTree`s. The `Spacing` in - // the result is that of the opening delimiter. - fn parse_token_trees( + // Lex into a token stream. The `Spacing` in the result is that of the + // opening delimiter. + fn lex_token_trees( &mut self, is_delimited: bool, ) -> (Spacing, TokenStream, Result<(), Vec>>) { @@ -42,12 +41,10 @@ impl<'psess, 'src> TokenTreesReader<'psess, 'src> { let mut buf = Vec::new(); loop { match self.token.kind { - token::OpenDelim(delim) => { - buf.push(match self.parse_token_tree_open_delim(delim) { - Ok(val) => val, - Err(errs) => return (open_spacing, TokenStream::new(buf), Err(errs)), - }) - } + token::OpenDelim(delim) => buf.push(match self.lex_token_tree_open_delim(delim) { + Ok(val) => val, + Err(errs) => return (open_spacing, TokenStream::new(buf), Err(errs)), + }), token::CloseDelim(delim) => { return ( open_spacing, @@ -95,24 +92,24 @@ impl<'psess, 'src> TokenTreesReader<'psess, 'src> { err } - fn parse_token_tree_open_delim( + fn lex_token_tree_open_delim( &mut self, open_delim: Delimiter, ) -> Result>> { - // The span for beginning of the delimited section + // The span for beginning of the delimited section. let pre_span = self.token.span; self.diag_info.open_braces.push((open_delim, self.token.span)); - // Parse the token trees within the delimiters. + // Lex the token trees within the delimiters. // We stop at any delimiter so we can try to recover if the user // uses an incorrect delimiter. - let (open_spacing, tts, res) = self.parse_token_trees(/* is_delimited */ true); + let (open_spacing, tts, res) = self.lex_token_trees(/* is_delimited */ true); if let Err(errs) = res { return Err(self.unclosed_delim_err(tts, errs)); } - // Expand to cover the entire delimited token tree + // Expand to cover the entire delimited token tree. let delim_span = DelimSpan::from_pair(pre_span, self.token.span); let sm = self.string_reader.psess.source_map(); @@ -150,7 +147,7 @@ impl<'psess, 'src> TokenTreesReader<'psess, 'src> { self.diag_info.last_unclosed_found_span = Some(self.token.span); // This is a conservative error: only report the last unclosed // delimiter. The previous unclosed delimiters could actually be - // closed! The parser just hasn't gotten to them yet. + // closed! The lexer just hasn't gotten to them yet. if let Some(&(_, sp)) = self.diag_info.open_braces.last() { unclosed_delimiter = Some(sp); }; @@ -236,9 +233,9 @@ impl<'psess, 'src> TokenTreesReader<'psess, 'src> { // out instead of complaining about the unclosed delims. let mut parser = crate::stream_to_parser(self.string_reader.psess, tts, None); let mut diff_errs = vec![]; - // Suggest removing a `{` we think appears in an `if`/`while` condition - // We want to suggest removing a `{` only if we think we're in an `if`/`while` condition, but - // we have no way of tracking this in the lexer itself, so we piggyback on the parser + // Suggest removing a `{` we think appears in an `if`/`while` condition. + // We want to suggest removing a `{` only if we think we're in an `if`/`while` condition, + // but we have no way of tracking this in the lexer itself, so we piggyback on the parser. let mut in_cond = false; while parser.token != token::Eof { if let Err(diff_err) = parser.err_vcs_conflict_marker() { @@ -249,14 +246,15 @@ impl<'psess, 'src> TokenTreesReader<'psess, 'src> { parser.token.kind, token::CloseDelim(Delimiter::Brace) | token::FatArrow ) { - // end of the `if`/`while` body, or the end of a `match` guard + // End of the `if`/`while` body, or the end of a `match` guard. in_cond = false; } else if in_cond && parser.token == token::OpenDelim(Delimiter::Brace) { // Store the `&&` and `let` to use their spans later when creating the diagnostic let maybe_andand = parser.look_ahead(1, |t| t.clone()); let maybe_let = parser.look_ahead(2, |t| t.clone()); if maybe_andand == token::OpenDelim(Delimiter::Brace) { - // This might be the beginning of the `if`/`while` body (i.e., the end of the condition) + // This might be the beginning of the `if`/`while` body (i.e., the end of the + // condition). in_cond = false; } else if maybe_andand == token::AndAnd && maybe_let.is_keyword(kw::Let) { let mut err = parser.dcx().struct_span_err( @@ -288,8 +286,7 @@ impl<'psess, 'src> TokenTreesReader<'psess, 'src> { } fn close_delim_err(&mut self, delim: Delimiter) -> PErr<'psess> { - // An unexpected closing delimiter (i.e., there is no - // matching opening delimiter). + // An unexpected closing delimiter (i.e., there is no matching opening delimiter). let token_str = token_to_string(&self.token); let msg = format!("unexpected closing delimiter: `{token_str}`"); let mut err = self.string_reader.psess.dcx.struct_span_err(self.token.span, msg); diff --git a/compiler/rustc_parse/src/lib.rs b/compiler/rustc_parse/src/lib.rs index 322739be3fb3..b4610447be7b 100644 --- a/compiler/rustc_parse/src/lib.rs +++ b/compiler/rustc_parse/src/lib.rs @@ -84,15 +84,6 @@ pub fn parse_crate_attrs_from_source_str( new_parser_from_source_str(psess, name, source).parse_inner_attributes() } -pub fn parse_stream_from_source_str( - name: FileName, - source: String, - psess: &ParseSess, - override_span: Option, -) -> TokenStream { - source_file_to_stream(psess, psess.source_map().new_source_file(name, source), override_span) -} - /// Creates a new parser from a source string. pub fn new_parser_from_source_str(psess: &ParseSess, name: FileName, source: String) -> Parser<'_> { panictry_buffer!(maybe_new_parser_from_source_str(psess, name, source)) @@ -142,6 +133,15 @@ fn maybe_source_file_to_parser( // Base abstractions +pub fn source_str_to_stream( + name: FileName, + source: String, + psess: &ParseSess, + override_span: Option, +) -> TokenStream { + source_file_to_stream(psess, psess.source_map().new_source_file(name, source), override_span) +} + /// Given a `source_file`, produces a sequence of token trees. pub fn source_file_to_stream( psess: &ParseSess, @@ -165,7 +165,7 @@ fn maybe_file_to_stream<'psess>( )); }); - lexer::parse_token_trees(psess, src.as_str(), source_file.start_pos, override_span) + lexer::lex_token_trees(psess, src.as_str(), source_file.start_pos, override_span) } /// Given a stream and the `ParseSess`, produces a parser. @@ -195,13 +195,13 @@ pub fn parse_in<'a, T>( pub fn fake_token_stream_for_item(psess: &ParseSess, item: &ast::Item) -> TokenStream { let source = pprust::item_to_string(item); let filename = FileName::macro_expansion_source_code(&source); - parse_stream_from_source_str(filename, source, psess, Some(item.span)) + source_str_to_stream(filename, source, psess, Some(item.span)) } pub fn fake_token_stream_for_crate(psess: &ParseSess, krate: &ast::Crate) -> TokenStream { let source = pprust::crate_to_string_for_macros(krate); let filename = FileName::macro_expansion_source_code(&source); - parse_stream_from_source_str(filename, source, psess, Some(krate.spans.inner_span)) + source_str_to_stream(filename, source, psess, Some(krate.spans.inner_span)) } pub fn parse_cfg_attr( diff --git a/src/librustdoc/passes/lint/check_code_block_syntax.rs b/src/librustdoc/passes/lint/check_code_block_syntax.rs index 39350f4cbbb6..4f044450df27 100644 --- a/src/librustdoc/passes/lint/check_code_block_syntax.rs +++ b/src/librustdoc/passes/lint/check_code_block_syntax.rs @@ -5,7 +5,7 @@ use rustc_errors::{ translation::{to_fluent_args, Translate}, Applicability, DiagCtxt, DiagInner, LazyFallbackBundle, }; -use rustc_parse::parse_stream_from_source_str; +use rustc_parse::source_str_to_stream; use rustc_resolve::rustdoc::source_span_for_markdown_range; use rustc_session::parse::ParseSess; use rustc_span::hygiene::{AstPass, ExpnData, ExpnKind, LocalExpnId, Transparency}; @@ -51,13 +51,8 @@ fn check_rust_syntax( let span = DUMMY_SP.apply_mark(expn_id.to_expn_id(), Transparency::Transparent); let is_empty = rustc_driver::catch_fatal_errors(|| { - parse_stream_from_source_str( - FileName::Custom(String::from("doctest")), - source, - &psess, - Some(span), - ) - .is_empty() + source_str_to_stream(FileName::Custom(String::from("doctest")), source, &psess, Some(span)) + .is_empty() }) .unwrap_or(false); let buffer = buffer.borrow(); From f6576249abf9d66b936d55a4d15c44932c800fe2 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Fri, 31 May 2024 10:59:16 +1000 Subject: [PATCH 03/14] Inline and remove `error_malformed_cfg_attr_missing`. It has a single call site. This also means `CFG_ATTR_{GRAMMAR_HELP,NOTE_REF}` can be moved into `parse_cfg_attr`, now that it's the only function that uses them. And the commit removes the line break in the URL. --- compiler/rustc_parse/src/lib.rs | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/compiler/rustc_parse/src/lib.rs b/compiler/rustc_parse/src/lib.rs index b4610447be7b..a3de5db73474 100644 --- a/compiler/rustc_parse/src/lib.rs +++ b/compiler/rustc_parse/src/lib.rs @@ -208,6 +208,10 @@ pub fn parse_cfg_attr( attr: &Attribute, psess: &ParseSess, ) -> Option<(MetaItem, Vec<(AttrItem, Span)>)> { + const CFG_ATTR_GRAMMAR_HELP: &str = "#[cfg_attr(condition, attribute, other_attribute, ...)]"; + const CFG_ATTR_NOTE_REF: &str = "for more information, visit \ + "; + match attr.get_normal_item().args { ast::AttrArgs::Delimited(ast::DelimArgs { dspan, delim, ref tokens }) if !tokens.is_empty() => @@ -222,16 +226,12 @@ pub fn parse_cfg_attr( } } } - _ => error_malformed_cfg_attr_missing(attr.span, psess), + _ => { + psess.dcx.emit_err(errors::MalformedCfgAttr { + span: attr.span, + sugg: CFG_ATTR_GRAMMAR_HELP, + }); + } } None } - -const CFG_ATTR_GRAMMAR_HELP: &str = "#[cfg_attr(condition, attribute, other_attribute, ...)]"; -const CFG_ATTR_NOTE_REF: &str = "for more information, visit \ - "; - -fn error_malformed_cfg_attr_missing(span: Span, psess: &ParseSess) { - psess.dcx.emit_err(errors::MalformedCfgAttr { span, sugg: CFG_ATTR_GRAMMAR_HELP }); -} From 769ca3f661e3c9ef3b97007f210f73415fb92796 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Fri, 31 May 2024 11:01:23 +1000 Subject: [PATCH 04/14] Rename `maybe_file_to_stream` as `maybe_source_file_to_stream`. Because it takes an `Lrc`, and for consistency with `source_file_to_stream`. --- compiler/rustc_parse/src/lib.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/compiler/rustc_parse/src/lib.rs b/compiler/rustc_parse/src/lib.rs index a3de5db73474..a818aae27b10 100644 --- a/compiler/rustc_parse/src/lib.rs +++ b/compiler/rustc_parse/src/lib.rs @@ -122,7 +122,7 @@ fn maybe_source_file_to_parser( source_file: Lrc, ) -> Result, Vec>> { let end_pos = source_file.end_position(); - let stream = maybe_file_to_stream(psess, source_file, None)?; + let stream = maybe_source_file_to_stream(psess, source_file, None)?; let mut parser = stream_to_parser(psess, stream, None); if parser.token == token::Eof { parser.token.span = Span::new(end_pos, end_pos, parser.token.span.ctxt(), None); @@ -148,12 +148,12 @@ pub fn source_file_to_stream( source_file: Lrc, override_span: Option, ) -> TokenStream { - panictry_buffer!(maybe_file_to_stream(psess, source_file, override_span)) + panictry_buffer!(maybe_source_file_to_stream(psess, source_file, override_span)) } /// Given a source file, produces a sequence of token trees. Returns any buffered errors from /// parsing the token stream. -fn maybe_file_to_stream<'psess>( +fn maybe_source_file_to_stream<'psess>( psess: &'psess ParseSess, source_file: Lrc, override_span: Option, From 3c321b9ea8706ca0b8e05222b82100bad8da805e Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Fri, 31 May 2024 13:32:54 +1000 Subject: [PATCH 05/14] Remove `stream_to_parser`. It's a zero-value wrapper of `Parser::new`. --- compiler/rustc_builtin_macros/src/cfg_eval.rs | 2 +- compiler/rustc_expand/src/base.rs | 6 +++--- compiler/rustc_expand/src/proc_macro.rs | 5 ++--- compiler/rustc_expand/src/proc_macro_server.rs | 7 ++----- compiler/rustc_parse/src/lexer/tokentrees.rs | 3 ++- compiler/rustc_parse/src/lib.rs | 11 +---------- src/tools/rustfmt/src/parse/macros/mod.rs | 4 ++-- 7 files changed, 13 insertions(+), 25 deletions(-) diff --git a/compiler/rustc_builtin_macros/src/cfg_eval.rs b/compiler/rustc_builtin_macros/src/cfg_eval.rs index 5f63a8ae0a8c..03aff6f96330 100644 --- a/compiler/rustc_builtin_macros/src/cfg_eval.rs +++ b/compiler/rustc_builtin_macros/src/cfg_eval.rs @@ -196,7 +196,7 @@ impl CfgEval<'_, '_> { // Re-parse the tokens, setting the `capture_cfg` flag to save extra information // to the captured `AttrTokenStream` (specifically, we capture // `AttrTokenTree::AttributesData` for all occurrences of `#[cfg]` and `#[cfg_attr]`) - let mut parser = rustc_parse::stream_to_parser(&self.cfg.sess.psess, orig_tokens, None); + let mut parser = Parser::new(&self.cfg.sess.psess, orig_tokens, None); parser.capture_cfg = true; match parse_annotatable_with(&mut parser) { Ok(a) => annotatable = a, diff --git a/compiler/rustc_expand/src/base.rs b/compiler/rustc_expand/src/base.rs index 4539129bd0f7..b3f6a35f3a4b 100644 --- a/compiler/rustc_expand/src/base.rs +++ b/compiler/rustc_expand/src/base.rs @@ -15,7 +15,7 @@ use rustc_data_structures::sync::{self, Lrc}; use rustc_errors::{DiagCtxt, ErrorGuaranteed, PResult}; use rustc_feature::Features; use rustc_lint_defs::{BufferedEarlyLint, RegisteredTools}; -use rustc_parse::{parser, MACRO_ARGUMENTS}; +use rustc_parse::{parser::Parser, MACRO_ARGUMENTS}; use rustc_session::config::CollapseMacroDebuginfo; use rustc_session::{parse::ParseSess, Limit, Session}; use rustc_span::def_id::{CrateNum, DefId, LocalDefId}; @@ -1149,8 +1149,8 @@ impl<'a> ExtCtxt<'a> { pub fn monotonic_expander<'b>(&'b mut self) -> expand::MacroExpander<'b, 'a> { expand::MacroExpander::new(self, true) } - pub fn new_parser_from_tts(&self, stream: TokenStream) -> parser::Parser<'a> { - rustc_parse::stream_to_parser(&self.sess.psess, stream, MACRO_ARGUMENTS) + pub fn new_parser_from_tts(&self, stream: TokenStream) -> Parser<'a> { + Parser::new(&self.sess.psess, stream, MACRO_ARGUMENTS) } pub fn source_map(&self) -> &'a SourceMap { self.sess.psess.source_map() diff --git a/compiler/rustc_expand/src/proc_macro.rs b/compiler/rustc_expand/src/proc_macro.rs index c11369f505f8..96145affe0ae 100644 --- a/compiler/rustc_expand/src/proc_macro.rs +++ b/compiler/rustc_expand/src/proc_macro.rs @@ -6,7 +6,7 @@ use rustc_ast as ast; use rustc_ast::ptr::P; use rustc_ast::tokenstream::TokenStream; use rustc_errors::ErrorGuaranteed; -use rustc_parse::parser::ForceCollect; +use rustc_parse::parser::{ForceCollect, Parser}; use rustc_session::config::ProcMacroExecutionStrategy; use rustc_span::profiling::SpannedEventArgRecorder; use rustc_span::Span; @@ -154,8 +154,7 @@ impl MultiItemModifier for DeriveProcMacro { }; let error_count_before = ecx.dcx().err_count(); - let mut parser = - rustc_parse::stream_to_parser(&ecx.sess.psess, stream, Some("proc-macro derive")); + let mut parser = Parser::new(&ecx.sess.psess, stream, Some("proc-macro derive")); let mut items = vec![]; loop { diff --git a/compiler/rustc_expand/src/proc_macro_server.rs b/compiler/rustc_expand/src/proc_macro_server.rs index 3b167c42e57d..9ae9f7c10c01 100644 --- a/compiler/rustc_expand/src/proc_macro_server.rs +++ b/compiler/rustc_expand/src/proc_macro_server.rs @@ -13,6 +13,7 @@ use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::sync::Lrc; use rustc_errors::{Diag, ErrorGuaranteed, MultiSpan, PResult}; use rustc_parse::lexer::nfc_normalize; +use rustc_parse::parser::Parser; use rustc_parse::source_str_to_stream; use rustc_session::parse::ParseSess; use rustc_span::def_id::CrateNum; @@ -553,11 +554,7 @@ impl server::TokenStream for Rustc<'_, '_> { fn expand_expr(&mut self, stream: &Self::TokenStream) -> Result { // Parse the expression from our tokenstream. let expr: PResult<'_, _> = try { - let mut p = rustc_parse::stream_to_parser( - self.psess(), - stream.clone(), - Some("proc_macro expand expr"), - ); + let mut p = Parser::new(self.psess(), stream.clone(), Some("proc_macro expand expr")); let expr = p.parse_expr()?; if p.token != token::Eof { p.unexpected()?; diff --git a/compiler/rustc_parse/src/lexer/tokentrees.rs b/compiler/rustc_parse/src/lexer/tokentrees.rs index fcbfa108d066..f7645446081a 100644 --- a/compiler/rustc_parse/src/lexer/tokentrees.rs +++ b/compiler/rustc_parse/src/lexer/tokentrees.rs @@ -2,6 +2,7 @@ use super::diagnostics::report_suspicious_mismatch_block; use super::diagnostics::same_indentation_level; use super::diagnostics::TokenTreeDiagInfo; use super::{StringReader, UnmatchedDelim}; +use crate::Parser; use rustc_ast::token::{self, Delimiter, Token}; use rustc_ast::tokenstream::{DelimSpacing, DelimSpan, Spacing, TokenStream, TokenTree}; use rustc_ast_pretty::pprust::token_to_string; @@ -231,7 +232,7 @@ impl<'psess, 'src> TokenTreesReader<'psess, 'src> { ) -> Vec> { // If there are unclosed delims, see if there are diff markers and if so, point them // out instead of complaining about the unclosed delims. - let mut parser = crate::stream_to_parser(self.string_reader.psess, tts, None); + let mut parser = Parser::new(self.string_reader.psess, tts, None); let mut diff_errs = vec![]; // Suggest removing a `{` we think appears in an `if`/`while` condition. // We want to suggest removing a `{` only if we think we're in an `if`/`while` condition, diff --git a/compiler/rustc_parse/src/lib.rs b/compiler/rustc_parse/src/lib.rs index a818aae27b10..5bbbfe1430c3 100644 --- a/compiler/rustc_parse/src/lib.rs +++ b/compiler/rustc_parse/src/lib.rs @@ -123,7 +123,7 @@ fn maybe_source_file_to_parser( ) -> Result, Vec>> { let end_pos = source_file.end_position(); let stream = maybe_source_file_to_stream(psess, source_file, None)?; - let mut parser = stream_to_parser(psess, stream, None); + let mut parser = Parser::new(psess, stream, None); if parser.token == token::Eof { parser.token.span = Span::new(end_pos, end_pos, parser.token.span.ctxt(), None); } @@ -168,15 +168,6 @@ fn maybe_source_file_to_stream<'psess>( lexer::lex_token_trees(psess, src.as_str(), source_file.start_pos, override_span) } -/// Given a stream and the `ParseSess`, produces a parser. -pub fn stream_to_parser<'a>( - psess: &'a ParseSess, - stream: TokenStream, - subparser_name: Option<&'static str>, -) -> Parser<'a> { - Parser::new(psess, stream, subparser_name) -} - /// Runs the given subparser `f` on the tokens of the given `attr`'s item. pub fn parse_in<'a, T>( psess: &'a ParseSess, diff --git a/src/tools/rustfmt/src/parse/macros/mod.rs b/src/tools/rustfmt/src/parse/macros/mod.rs index 3cf133c647cc..cbcc0b2d6364 100644 --- a/src/tools/rustfmt/src/parse/macros/mod.rs +++ b/src/tools/rustfmt/src/parse/macros/mod.rs @@ -2,7 +2,7 @@ use rustc_ast::token::{Delimiter, NonterminalKind, TokenKind}; use rustc_ast::tokenstream::TokenStream; use rustc_ast::{ast, ptr}; use rustc_parse::parser::{ForceCollect, Parser, Recovery}; -use rustc_parse::{stream_to_parser, MACRO_ARGUMENTS}; +use rustc_parse::MACRO_ARGUMENTS; use rustc_session::parse::ParseSess; use rustc_span::symbol::{self, kw}; use rustc_span::Symbol; @@ -15,7 +15,7 @@ pub(crate) mod cfg_if; pub(crate) mod lazy_static; fn build_stream_parser<'a>(psess: &'a ParseSess, tokens: TokenStream) -> Parser<'a> { - stream_to_parser(psess, tokens, MACRO_ARGUMENTS).recovery(Recovery::Forbidden) + Parser::new(psess, tokens, MACRO_ARGUMENTS).recovery(Recovery::Forbidden) } fn build_parser<'a>(context: &RewriteContext<'a>, tokens: TokenStream) -> Parser<'a> { From af13b4892786a3f77d440016409414eeadd7d37e Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Fri, 31 May 2024 13:36:18 +1000 Subject: [PATCH 06/14] Improve `panictry_buffer!`. - Convert it from a macro to a function, which is nicer. - Rename it as `unwrap_or_emit_fatal`, which is clearer. - Fix the comment. In particular, `panictry!` no longer exists. - Remove the unnecessary `use` declaration. --- compiler/rustc_parse/src/lib.rs | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/compiler/rustc_parse/src/lib.rs b/compiler/rustc_parse/src/lib.rs index 5bbbfe1430c3..298d02d2e58f 100644 --- a/compiler/rustc_parse/src/lib.rs +++ b/compiler/rustc_parse/src/lib.rs @@ -39,20 +39,17 @@ rustc_fluent_macro::fluent_messages! { "../messages.ftl" } // uses a HOF to parse anything, and includes file and // `source_str`. -/// A variant of 'panictry!' that works on a `Vec` instead of a single `Diag`. -macro_rules! panictry_buffer { - ($e:expr) => {{ - use std::result::Result::{Err, Ok}; - match $e { - Ok(e) => e, - Err(errs) => { - for e in errs { - e.emit(); - } - FatalError.raise() +// Unwrap the result if `Ok`, otherwise emit the diagnostics and abort. +fn unwrap_or_emit_fatal(expr: Result>>) -> T { + match expr { + Ok(expr) => expr, + Err(errs) => { + for err in errs { + err.emit(); } + FatalError.raise() } - }}; + } } pub fn parse_crate_from_file<'a>(input: &Path, psess: &'a ParseSess) -> PResult<'a, ast::Crate> { @@ -86,7 +83,7 @@ pub fn parse_crate_attrs_from_source_str( /// Creates a new parser from a source string. pub fn new_parser_from_source_str(psess: &ParseSess, name: FileName, source: String) -> Parser<'_> { - panictry_buffer!(maybe_new_parser_from_source_str(psess, name, source)) + unwrap_or_emit_fatal(maybe_new_parser_from_source_str(psess, name, source)) } /// Creates a new parser from a source string. Returns any buffered errors from lexing the initial @@ -112,7 +109,7 @@ pub fn new_parser_from_file<'a>(psess: &'a ParseSess, path: &Path, sp: Option, override_span: Option, ) -> TokenStream { - panictry_buffer!(maybe_source_file_to_stream(psess, source_file, override_span)) + unwrap_or_emit_fatal(maybe_source_file_to_stream(psess, source_file, override_span)) } /// Given a source file, produces a sequence of token trees. Returns any buffered errors from From 29e6e2859be6ea69a486bdc155df9311455f55dc Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Fri, 31 May 2024 13:38:51 +1000 Subject: [PATCH 07/14] Remove low-value comments. The first one is out-of-date -- there are no longer functions expr, item, stmt. And I don't know what a "HOF" is. The second one doesn't really tell you anything. --- compiler/rustc_parse/src/lib.rs | 7 ------- 1 file changed, 7 deletions(-) diff --git a/compiler/rustc_parse/src/lib.rs b/compiler/rustc_parse/src/lib.rs index 298d02d2e58f..2c5a0abc6aca 100644 --- a/compiler/rustc_parse/src/lib.rs +++ b/compiler/rustc_parse/src/lib.rs @@ -34,11 +34,6 @@ mod errors; rustc_fluent_macro::fluent_messages! { "../messages.ftl" } -// A bunch of utility functions of the form `parse__from_` -// where includes crate, expr, item, stmt, tts, and one that -// uses a HOF to parse anything, and includes file and -// `source_str`. - // Unwrap the result if `Ok`, otherwise emit the diagnostics and abort. fn unwrap_or_emit_fatal(expr: Result>>) -> T { match expr { @@ -128,8 +123,6 @@ fn maybe_source_file_to_parser( Ok(parser) } -// Base abstractions - pub fn source_str_to_stream( name: FileName, source: String, From 191b76ef3116736416c28d39d35102c78cb33a07 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Fri, 31 May 2024 13:39:24 +1000 Subject: [PATCH 08/14] Rename `maybe_source_file_to_parser` as `maybe_new_parser_from_source_file`. For consistency with `new_parser_from_{file,source_str}` and `maybe_new_parser_from_source_str`. --- compiler/rustc_parse/src/lib.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/compiler/rustc_parse/src/lib.rs b/compiler/rustc_parse/src/lib.rs index 2c5a0abc6aca..1a0535684e8e 100644 --- a/compiler/rustc_parse/src/lib.rs +++ b/compiler/rustc_parse/src/lib.rs @@ -89,7 +89,7 @@ pub fn maybe_new_parser_from_source_str( name: FileName, source: String, ) -> Result, Vec>> { - maybe_source_file_to_parser(psess, psess.source_map().new_source_file(name, source)) + maybe_new_parser_from_source_file(psess, psess.source_map().new_source_file(name, source)) } /// Creates a new parser, aborting if the file doesn't exist. If a span is given, that is used on @@ -104,12 +104,12 @@ pub fn new_parser_from_file<'a>(psess: &'a ParseSess, path: &Path, sp: Option, ) -> Result, Vec>> { From 8964106e44c39cced6cc039bf512a69513a2bbe5 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Fri, 31 May 2024 13:43:38 +1000 Subject: [PATCH 09/14] Use `source_str_to_stream` in a test file. It does exactly what is required. --- compiler/rustc_parse/src/parser/tests.rs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/compiler/rustc_parse/src/parser/tests.rs b/compiler/rustc_parse/src/parser/tests.rs index a31e350541ab..b72cfd63d00d 100644 --- a/compiler/rustc_parse/src/parser/tests.rs +++ b/compiler/rustc_parse/src/parser/tests.rs @@ -1,5 +1,5 @@ use crate::parser::ForceCollect; -use crate::{new_parser_from_source_str, parser::Parser, source_file_to_stream}; +use crate::{new_parser_from_source_str, parser::Parser, source_str_to_stream}; use ast::token::IdentIsRaw; use rustc_ast::ptr::P; use rustc_ast::token::{self, Delimiter, Token}; @@ -82,11 +82,7 @@ where /// Maps a string to tts, using a made-up filename. pub(crate) fn string_to_stream(source_str: String) -> TokenStream { let psess = psess(); - source_file_to_stream( - &psess, - psess.source_map().new_source_file(PathBuf::from("bogofile").into(), source_str), - None, - ) + source_str_to_stream(PathBuf::from("bogofile").into(), source_str, &psess, None) } /// Parses a string, returns a crate. From 25972aec67e900aeefe4014deecf5ef853a4bab4 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Fri, 31 May 2024 13:56:44 +1000 Subject: [PATCH 10/14] Inline and remove `parse_crate{,_attrs}_from_{file,source_str}`. All four functions are simple and have a single call site. This requires making `Parser::parse_inner_attributes` public, which is no big deal. --- compiler/rustc_driver_impl/src/lib.rs | 10 +++++---- compiler/rustc_interface/src/passes.rs | 15 ++++++++----- compiler/rustc_parse/src/lib.rs | 29 ------------------------- compiler/rustc_parse/src/parser/attr.rs | 2 +- 4 files changed, 16 insertions(+), 40 deletions(-) diff --git a/compiler/rustc_driver_impl/src/lib.rs b/compiler/rustc_driver_impl/src/lib.rs index 627fd74c8d77..236880420fc2 100644 --- a/compiler/rustc_driver_impl/src/lib.rs +++ b/compiler/rustc_driver_impl/src/lib.rs @@ -32,6 +32,7 @@ use rustc_interface::{interface, Queries}; use rustc_lint::unerased_lint_store; use rustc_metadata::creader::MetadataLoader; use rustc_metadata::locator; +use rustc_parse::{new_parser_from_file, new_parser_from_source_str}; use rustc_session::config::{nightly_options, CG_OPTIONS, Z_OPTIONS}; use rustc_session::config::{ErrorOutputType, Input, OutFileName, OutputType}; use rustc_session::getopts::{self, Matches}; @@ -1264,12 +1265,13 @@ pub fn handle_options(early_dcx: &EarlyDiagCtxt, args: &[String]) -> Option(sess: &'a Session) -> PResult<'a, ast::AttrVec> { - match &sess.io.input { - Input::File(ifile) => rustc_parse::parse_crate_attrs_from_file(ifile, &sess.psess), + let mut parser = match &sess.io.input { + Input::File(file) => new_parser_from_file(&sess.psess, file, None), Input::Str { name, input } => { - rustc_parse::parse_crate_attrs_from_source_str(name.clone(), input.clone(), &sess.psess) + new_parser_from_source_str(&sess.psess, name.clone(), input.clone()) } - } + }; + parser.parse_inner_attributes() } /// Runs a closure and catches unwinds triggered by fatal errors. diff --git a/compiler/rustc_interface/src/passes.rs b/compiler/rustc_interface/src/passes.rs index b678582766d5..b3ed66220977 100644 --- a/compiler/rustc_interface/src/passes.rs +++ b/compiler/rustc_interface/src/passes.rs @@ -19,7 +19,7 @@ use rustc_middle::arena::Arena; use rustc_middle::dep_graph::DepGraph; use rustc_middle::ty::{self, GlobalCtxt, RegisteredTools, TyCtxt}; use rustc_middle::util::Providers; -use rustc_parse::{parse_crate_from_file, parse_crate_from_source_str, validate_attr}; +use rustc_parse::{new_parser_from_file, new_parser_from_source_str, validate_attr}; use rustc_passes::{abi_test, hir_stats, layout_test}; use rustc_resolve::Resolver; use rustc_session::code_stats::VTableSizeInfo; @@ -42,11 +42,14 @@ use std::{env, fs, iter}; use tracing::{info, instrument}; pub fn parse<'a>(sess: &'a Session) -> PResult<'a, ast::Crate> { - let krate = sess.time("parse_crate", || match &sess.io.input { - Input::File(file) => parse_crate_from_file(file, &sess.psess), - Input::Str { input, name } => { - parse_crate_from_source_str(name.clone(), input.clone(), &sess.psess) - } + let krate = sess.time("parse_crate", || { + let mut parser = match &sess.io.input { + Input::File(file) => new_parser_from_file(&sess.psess, file, None), + Input::Str { input, name } => { + new_parser_from_source_str(&sess.psess, name.clone(), input.clone()) + } + }; + parser.parse_crate_mod() })?; if sess.opts.unstable_opts.input_stats { diff --git a/compiler/rustc_parse/src/lib.rs b/compiler/rustc_parse/src/lib.rs index 1a0535684e8e..b804e77e7a4e 100644 --- a/compiler/rustc_parse/src/lib.rs +++ b/compiler/rustc_parse/src/lib.rs @@ -47,35 +47,6 @@ fn unwrap_or_emit_fatal(expr: Result>>) -> T { } } -pub fn parse_crate_from_file<'a>(input: &Path, psess: &'a ParseSess) -> PResult<'a, ast::Crate> { - let mut parser = new_parser_from_file(psess, input, None); - parser.parse_crate_mod() -} - -pub fn parse_crate_attrs_from_file<'a>( - input: &Path, - psess: &'a ParseSess, -) -> PResult<'a, ast::AttrVec> { - let mut parser = new_parser_from_file(psess, input, None); - parser.parse_inner_attributes() -} - -pub fn parse_crate_from_source_str( - name: FileName, - source: String, - psess: &ParseSess, -) -> PResult<'_, ast::Crate> { - new_parser_from_source_str(psess, name, source).parse_crate_mod() -} - -pub fn parse_crate_attrs_from_source_str( - name: FileName, - source: String, - psess: &ParseSess, -) -> PResult<'_, ast::AttrVec> { - new_parser_from_source_str(psess, name, source).parse_inner_attributes() -} - /// Creates a new parser from a source string. pub fn new_parser_from_source_str(psess: &ParseSess, name: FileName, source: String) -> Parser<'_> { unwrap_or_emit_fatal(maybe_new_parser_from_source_str(psess, name, source)) diff --git a/compiler/rustc_parse/src/parser/attr.rs b/compiler/rustc_parse/src/parser/attr.rs index a57eb70c7053..4acc610d8c40 100644 --- a/compiler/rustc_parse/src/parser/attr.rs +++ b/compiler/rustc_parse/src/parser/attr.rs @@ -265,7 +265,7 @@ impl<'a> Parser<'a> { /// terminated by a semicolon. /// /// Matches `inner_attrs*`. - pub(crate) fn parse_inner_attributes(&mut self) -> PResult<'a, ast::AttrVec> { + pub fn parse_inner_attributes(&mut self) -> PResult<'a, ast::AttrVec> { let mut attrs = ast::AttrVec::new(); loop { let start_pos: u32 = self.num_bump_calls.try_into().unwrap(); From ab192a0c97cd0d03d24e402640576474cb62c972 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Fri, 31 May 2024 14:33:56 +1000 Subject: [PATCH 11/14] Reorder `source_str_to_stream` arguments. It's the only one of these functions where `psess` isn't the first argument. --- compiler/rustc_expand/src/proc_macro_server.rs | 2 +- compiler/rustc_parse/src/lib.rs | 6 +++--- compiler/rustc_parse/src/parser/tests.rs | 2 +- src/librustdoc/passes/lint/check_code_block_syntax.rs | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/compiler/rustc_expand/src/proc_macro_server.rs b/compiler/rustc_expand/src/proc_macro_server.rs index 9ae9f7c10c01..330a6c347702 100644 --- a/compiler/rustc_expand/src/proc_macro_server.rs +++ b/compiler/rustc_expand/src/proc_macro_server.rs @@ -540,9 +540,9 @@ impl server::TokenStream for Rustc<'_, '_> { fn from_str(&mut self, src: &str) -> Self::TokenStream { source_str_to_stream( + self.psess(), FileName::proc_macro_source_code(src), src.to_string(), - self.psess(), Some(self.call_site), ) } diff --git a/compiler/rustc_parse/src/lib.rs b/compiler/rustc_parse/src/lib.rs index b804e77e7a4e..5ba6db3b2620 100644 --- a/compiler/rustc_parse/src/lib.rs +++ b/compiler/rustc_parse/src/lib.rs @@ -95,9 +95,9 @@ fn maybe_new_parser_from_source_file( } pub fn source_str_to_stream( + psess: &ParseSess, name: FileName, source: String, - psess: &ParseSess, override_span: Option, ) -> TokenStream { source_file_to_stream(psess, psess.source_map().new_source_file(name, source), override_span) @@ -147,13 +147,13 @@ pub fn parse_in<'a, T>( pub fn fake_token_stream_for_item(psess: &ParseSess, item: &ast::Item) -> TokenStream { let source = pprust::item_to_string(item); let filename = FileName::macro_expansion_source_code(&source); - source_str_to_stream(filename, source, psess, Some(item.span)) + source_str_to_stream(psess, filename, source, Some(item.span)) } pub fn fake_token_stream_for_crate(psess: &ParseSess, krate: &ast::Crate) -> TokenStream { let source = pprust::crate_to_string_for_macros(krate); let filename = FileName::macro_expansion_source_code(&source); - source_str_to_stream(filename, source, psess, Some(krate.spans.inner_span)) + source_str_to_stream(psess, filename, source, Some(krate.spans.inner_span)) } pub fn parse_cfg_attr( diff --git a/compiler/rustc_parse/src/parser/tests.rs b/compiler/rustc_parse/src/parser/tests.rs index b72cfd63d00d..2900d44089ea 100644 --- a/compiler/rustc_parse/src/parser/tests.rs +++ b/compiler/rustc_parse/src/parser/tests.rs @@ -82,7 +82,7 @@ where /// Maps a string to tts, using a made-up filename. pub(crate) fn string_to_stream(source_str: String) -> TokenStream { let psess = psess(); - source_str_to_stream(PathBuf::from("bogofile").into(), source_str, &psess, None) + source_str_to_stream(&psess, PathBuf::from("bogofile").into(), source_str, None) } /// Parses a string, returns a crate. diff --git a/src/librustdoc/passes/lint/check_code_block_syntax.rs b/src/librustdoc/passes/lint/check_code_block_syntax.rs index 4f044450df27..409546734cb1 100644 --- a/src/librustdoc/passes/lint/check_code_block_syntax.rs +++ b/src/librustdoc/passes/lint/check_code_block_syntax.rs @@ -51,7 +51,7 @@ fn check_rust_syntax( let span = DUMMY_SP.apply_mark(expn_id.to_expn_id(), Transparency::Transparent); let is_empty = rustc_driver::catch_fatal_errors(|| { - source_str_to_stream(FileName::Custom(String::from("doctest")), source, &psess, Some(span)) + source_str_to_stream(&psess, FileName::Custom(String::from("doctest")), source, Some(span)) .is_empty() }) .unwrap_or(false); From 264dbe4d8145c076557a95a93183b8c1d6e1f564 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Fri, 31 May 2024 14:38:34 +1000 Subject: [PATCH 12/14] Inline and remove `source_file_to_stream`. It has a single call site. --- compiler/rustc_parse/src/lib.rs | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/compiler/rustc_parse/src/lib.rs b/compiler/rustc_parse/src/lib.rs index 5ba6db3b2620..8a061ce8f1b5 100644 --- a/compiler/rustc_parse/src/lib.rs +++ b/compiler/rustc_parse/src/lib.rs @@ -100,15 +100,7 @@ pub fn source_str_to_stream( source: String, override_span: Option, ) -> TokenStream { - source_file_to_stream(psess, psess.source_map().new_source_file(name, source), override_span) -} - -/// Given a `source_file`, produces a sequence of token trees. -pub fn source_file_to_stream( - psess: &ParseSess, - source_file: Lrc, - override_span: Option, -) -> TokenStream { + let source_file = psess.source_map().new_source_file(name, source); unwrap_or_emit_fatal(maybe_source_file_to_stream(psess, source_file, override_span)) } From b9037339cbc636349b02088486effa21f9ab16aa Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Fri, 31 May 2024 15:43:18 +1000 Subject: [PATCH 13/14] Make top-level `rustc_parse` functions fallible. Currently we have an awkward mix of fallible and infallible functions: ``` new_parser_from_source_str maybe_new_parser_from_source_str new_parser_from_file (maybe_new_parser_from_file) // missing (new_parser_from_source_file) // missing maybe_new_parser_from_source_file source_str_to_stream maybe_source_file_to_stream ``` We could add the two missing functions, but instead this commit removes of all the infallible ones and renames the fallible ones leaving us with these which are all fallible: ``` new_parser_from_source_str new_parser_from_file new_parser_from_source_file source_str_to_stream source_file_to_stream ``` This requires making `unwrap_or_emit_fatal` public so callers of formerly infallible functions can still work. This does make some of the call sites slightly more verbose, but I think it's worth it for the simpler API. Also, there are two `catch_unwind` calls and one `catch_fatal_errors` call in this diff that become removable thanks this change. (I will do that in a follow-up PR.) --- .../rustc_builtin_macros/src/cmdline_attrs.rs | 5 +- .../rustc_builtin_macros/src/source_util.rs | 4 +- compiler/rustc_driver_impl/src/lib.rs | 6 +-- compiler/rustc_expand/src/module.rs | 5 +- .../rustc_expand/src/proc_macro_server.rs | 9 ++-- compiler/rustc_interface/src/interface.rs | 6 +-- compiler/rustc_interface/src/passes.rs | 8 +-- compiler/rustc_parse/src/lib.rs | 54 ++++++++++--------- compiler/rustc_parse/src/parser/tests.rs | 22 ++++++-- src/librustdoc/clean/render_macro_matchers.rs | 2 +- src/librustdoc/doctest.rs | 23 ++++---- .../passes/lint/check_code_block_syntax.rs | 11 ++-- .../src/doc/needless_doctest_main.rs | 4 +- src/tools/rustfmt/src/parse/parser.rs | 10 ++-- .../ui-fulldeps/mod_dir_path_canonicalized.rs | 4 +- tests/ui-fulldeps/pprust-expr-roundtrip.rs | 7 +-- 16 files changed, 105 insertions(+), 75 deletions(-) diff --git a/compiler/rustc_builtin_macros/src/cmdline_attrs.rs b/compiler/rustc_builtin_macros/src/cmdline_attrs.rs index ada82e45712d..e9b63b4abebb 100644 --- a/compiler/rustc_builtin_macros/src/cmdline_attrs.rs +++ b/compiler/rustc_builtin_macros/src/cmdline_attrs.rs @@ -4,16 +4,17 @@ use crate::errors; use rustc_ast::attr::mk_attr; use rustc_ast::token; use rustc_ast::{self as ast, AttrItem, AttrStyle}; +use rustc_parse::{new_parser_from_source_str, unwrap_or_emit_fatal}; use rustc_session::parse::ParseSess; use rustc_span::FileName; pub fn inject(krate: &mut ast::Crate, psess: &ParseSess, attrs: &[String]) { for raw_attr in attrs { - let mut parser = rustc_parse::new_parser_from_source_str( + let mut parser = unwrap_or_emit_fatal(new_parser_from_source_str( psess, FileName::cli_crate_attr_source_code(raw_attr), raw_attr.clone(), - ); + )); let start_span = parser.token.span; let AttrItem { path, args, tokens: _ } = match parser.parse_attr_item(false) { diff --git a/compiler/rustc_builtin_macros/src/source_util.rs b/compiler/rustc_builtin_macros/src/source_util.rs index 29e991525a9b..dc1d82df0c39 100644 --- a/compiler/rustc_builtin_macros/src/source_util.rs +++ b/compiler/rustc_builtin_macros/src/source_util.rs @@ -12,8 +12,8 @@ use rustc_expand::base::{ }; use rustc_expand::module::DirOwnership; use rustc_lint_defs::BuiltinLintDiag; -use rustc_parse::new_parser_from_file; use rustc_parse::parser::{ForceCollect, Parser}; +use rustc_parse::{new_parser_from_file, unwrap_or_emit_fatal}; use rustc_session::lint::builtin::INCOMPLETE_INCLUDE; use rustc_span::source_map::SourceMap; use rustc_span::symbol::Symbol; @@ -126,7 +126,7 @@ pub(crate) fn expand_include<'cx>( return ExpandResult::Ready(DummyResult::any(sp, guar)); } }; - let p = new_parser_from_file(cx.psess(), &file, Some(sp)); + let p = unwrap_or_emit_fatal(new_parser_from_file(cx.psess(), &file, Some(sp))); // If in the included file we have e.g., `mod bar;`, // then the path of `bar.rs` should be relative to the directory of `file`. diff --git a/compiler/rustc_driver_impl/src/lib.rs b/compiler/rustc_driver_impl/src/lib.rs index 236880420fc2..93a65290602a 100644 --- a/compiler/rustc_driver_impl/src/lib.rs +++ b/compiler/rustc_driver_impl/src/lib.rs @@ -32,7 +32,7 @@ use rustc_interface::{interface, Queries}; use rustc_lint::unerased_lint_store; use rustc_metadata::creader::MetadataLoader; use rustc_metadata::locator; -use rustc_parse::{new_parser_from_file, new_parser_from_source_str}; +use rustc_parse::{new_parser_from_file, new_parser_from_source_str, unwrap_or_emit_fatal}; use rustc_session::config::{nightly_options, CG_OPTIONS, Z_OPTIONS}; use rustc_session::config::{ErrorOutputType, Input, OutFileName, OutputType}; use rustc_session::getopts::{self, Matches}; @@ -1265,12 +1265,12 @@ pub fn handle_options(early_dcx: &EarlyDiagCtxt, args: &[String]) -> Option(sess: &'a Session) -> PResult<'a, ast::AttrVec> { - let mut parser = match &sess.io.input { + let mut parser = unwrap_or_emit_fatal(match &sess.io.input { Input::File(file) => new_parser_from_file(&sess.psess, file, None), Input::Str { name, input } => { new_parser_from_source_str(&sess.psess, name.clone(), input.clone()) } - }; + }); parser.parse_inner_attributes() } diff --git a/compiler/rustc_expand/src/module.rs b/compiler/rustc_expand/src/module.rs index c8983619e701..506bd445be35 100644 --- a/compiler/rustc_expand/src/module.rs +++ b/compiler/rustc_expand/src/module.rs @@ -5,8 +5,8 @@ use crate::errors::{ use rustc_ast::ptr::P; use rustc_ast::{token, AttrVec, Attribute, Inline, Item, ModSpans}; use rustc_errors::{Diag, ErrorGuaranteed}; -use rustc_parse::new_parser_from_file; use rustc_parse::validate_attr; +use rustc_parse::{new_parser_from_file, unwrap_or_emit_fatal}; use rustc_session::parse::ParseSess; use rustc_session::Session; use rustc_span::symbol::{sym, Ident}; @@ -66,7 +66,8 @@ pub(crate) fn parse_external_mod( } // Actually parse the external file as a module. - let mut parser = new_parser_from_file(&sess.psess, &mp.file_path, Some(span)); + let mut parser = + unwrap_or_emit_fatal(new_parser_from_file(&sess.psess, &mp.file_path, Some(span))); let (inner_attrs, items, inner_span) = parser.parse_mod(&token::Eof).map_err(|err| ModError::ParserError(err))?; attrs.extend(inner_attrs); diff --git a/compiler/rustc_expand/src/proc_macro_server.rs b/compiler/rustc_expand/src/proc_macro_server.rs index 330a6c347702..93f8682090d7 100644 --- a/compiler/rustc_expand/src/proc_macro_server.rs +++ b/compiler/rustc_expand/src/proc_macro_server.rs @@ -14,7 +14,7 @@ use rustc_data_structures::sync::Lrc; use rustc_errors::{Diag, ErrorGuaranteed, MultiSpan, PResult}; use rustc_parse::lexer::nfc_normalize; use rustc_parse::parser::Parser; -use rustc_parse::source_str_to_stream; +use rustc_parse::{new_parser_from_source_str, source_str_to_stream, unwrap_or_emit_fatal}; use rustc_session::parse::ParseSess; use rustc_span::def_id::CrateNum; use rustc_span::symbol::{self, sym, Symbol}; @@ -467,7 +467,8 @@ impl server::FreeFunctions for Rustc<'_, '_> { fn literal_from_str(&mut self, s: &str) -> Result, ()> { let name = FileName::proc_macro_source_code(s); - let mut parser = rustc_parse::new_parser_from_source_str(self.psess(), name, s.to_owned()); + let mut parser = + unwrap_or_emit_fatal(new_parser_from_source_str(self.psess(), name, s.to_owned())); let first_span = parser.token.span.data(); let minus_present = parser.eat(&token::BinOp(token::Minus)); @@ -539,12 +540,12 @@ impl server::TokenStream for Rustc<'_, '_> { } fn from_str(&mut self, src: &str) -> Self::TokenStream { - source_str_to_stream( + unwrap_or_emit_fatal(source_str_to_stream( self.psess(), FileName::proc_macro_source_code(src), src.to_string(), Some(self.call_site), - ) + )) } fn to_string(&mut self, stream: &Self::TokenStream) -> String { diff --git a/compiler/rustc_interface/src/interface.rs b/compiler/rustc_interface/src/interface.rs index 5254a6576f90..c95a10f4e8d2 100644 --- a/compiler/rustc_interface/src/interface.rs +++ b/compiler/rustc_interface/src/interface.rs @@ -14,7 +14,7 @@ use rustc_lint::LintStore; use rustc_middle::ty; use rustc_middle::ty::CurrentGcx; use rustc_middle::util::Providers; -use rustc_parse::maybe_new_parser_from_source_str; +use rustc_parse::new_parser_from_source_str; use rustc_query_impl::QueryCtxt; use rustc_query_system::query::print_query_stack; use rustc_session::config::{self, Cfg, CheckCfg, ExpectedValues, Input, OutFileName}; @@ -67,7 +67,7 @@ pub(crate) fn parse_cfg(dcx: &DiagCtxt, cfgs: Vec) -> Cfg { }; } - match maybe_new_parser_from_source_str(&psess, filename, s.to_string()) { + match new_parser_from_source_str(&psess, filename, s.to_string()) { Ok(mut parser) => match parser.parse_meta_item() { Ok(meta_item) if parser.token == token::Eof => { if meta_item.path.segments.len() != 1 { @@ -166,7 +166,7 @@ pub(crate) fn parse_check_cfg(dcx: &DiagCtxt, specs: Vec) -> CheckCfg { error!("expected `cfg(name, values(\"value1\", \"value2\", ... \"valueN\"))`") }; - let mut parser = match maybe_new_parser_from_source_str(&psess, filename, s.to_string()) { + let mut parser = match new_parser_from_source_str(&psess, filename, s.to_string()) { Ok(parser) => parser, Err(errs) => { errs.into_iter().for_each(|err| err.cancel()); diff --git a/compiler/rustc_interface/src/passes.rs b/compiler/rustc_interface/src/passes.rs index b3ed66220977..a342577cffb8 100644 --- a/compiler/rustc_interface/src/passes.rs +++ b/compiler/rustc_interface/src/passes.rs @@ -19,7 +19,9 @@ use rustc_middle::arena::Arena; use rustc_middle::dep_graph::DepGraph; use rustc_middle::ty::{self, GlobalCtxt, RegisteredTools, TyCtxt}; use rustc_middle::util::Providers; -use rustc_parse::{new_parser_from_file, new_parser_from_source_str, validate_attr}; +use rustc_parse::{ + new_parser_from_file, new_parser_from_source_str, unwrap_or_emit_fatal, validate_attr, +}; use rustc_passes::{abi_test, hir_stats, layout_test}; use rustc_resolve::Resolver; use rustc_session::code_stats::VTableSizeInfo; @@ -43,12 +45,12 @@ use tracing::{info, instrument}; pub fn parse<'a>(sess: &'a Session) -> PResult<'a, ast::Crate> { let krate = sess.time("parse_crate", || { - let mut parser = match &sess.io.input { + let mut parser = unwrap_or_emit_fatal(match &sess.io.input { Input::File(file) => new_parser_from_file(&sess.psess, file, None), Input::Str { input, name } => { new_parser_from_source_str(&sess.psess, name.clone(), input.clone()) } - }; + }); parser.parse_crate_mod() })?; diff --git a/compiler/rustc_parse/src/lib.rs b/compiler/rustc_parse/src/lib.rs index 8a061ce8f1b5..b316327a262d 100644 --- a/compiler/rustc_parse/src/lib.rs +++ b/compiler/rustc_parse/src/lib.rs @@ -35,7 +35,7 @@ mod errors; rustc_fluent_macro::fluent_messages! { "../messages.ftl" } // Unwrap the result if `Ok`, otherwise emit the diagnostics and abort. -fn unwrap_or_emit_fatal(expr: Result>>) -> T { +pub fn unwrap_or_emit_fatal(expr: Result>>) -> T { match expr { Ok(expr) => expr, Err(errs) => { @@ -47,25 +47,28 @@ fn unwrap_or_emit_fatal(expr: Result>>) -> T { } } -/// Creates a new parser from a source string. -pub fn new_parser_from_source_str(psess: &ParseSess, name: FileName, source: String) -> Parser<'_> { - unwrap_or_emit_fatal(maybe_new_parser_from_source_str(psess, name, source)) -} - -/// Creates a new parser from a source string. Returns any buffered errors from lexing the initial -/// token stream; these must be consumed via `emit`, `cancel`, etc., otherwise a panic will occur -/// when they are dropped. -pub fn maybe_new_parser_from_source_str( +/// Creates a new parser from a source string. On failure, the errors must be consumed via +/// `unwrap_or_emit_fatal`, `emit`, `cancel`, etc., otherwise a panic will occur when they are +/// dropped. +pub fn new_parser_from_source_str( psess: &ParseSess, name: FileName, source: String, ) -> Result, Vec>> { - maybe_new_parser_from_source_file(psess, psess.source_map().new_source_file(name, source)) + let source_file = psess.source_map().new_source_file(name, source); + new_parser_from_source_file(psess, source_file) } -/// Creates a new parser, aborting if the file doesn't exist. If a span is given, that is used on -/// an error as the source of the problem. -pub fn new_parser_from_file<'a>(psess: &'a ParseSess, path: &Path, sp: Option) -> Parser<'a> { +/// Creates a new parser from a filename. On failure, the errors must be consumed via +/// `unwrap_or_emit_fatal`, `emit`, `cancel`, etc., otherwise a panic will occur when they are +/// dropped. +/// +/// If a span is given, that is used on an error as the source of the problem. +pub fn new_parser_from_file<'a>( + psess: &'a ParseSess, + path: &Path, + sp: Option, +) -> Result, Vec>> { let source_file = psess.source_map().load_file(path).unwrap_or_else(|e| { let msg = format!("couldn't read {}: {}", path.display(), e); let mut err = psess.dcx.struct_fatal(msg); @@ -74,23 +77,21 @@ pub fn new_parser_from_file<'a>(psess: &'a ParseSess, path: &Path, sp: Option, ) -> Result, Vec>> { let end_pos = source_file.end_position(); - let stream = maybe_source_file_to_stream(psess, source_file, None)?; + let stream = source_file_to_stream(psess, source_file, None)?; let mut parser = Parser::new(psess, stream, None); if parser.token == token::Eof { parser.token.span = Span::new(end_pos, end_pos, parser.token.span.ctxt(), None); } - Ok(parser) } @@ -99,14 +100,14 @@ pub fn source_str_to_stream( name: FileName, source: String, override_span: Option, -) -> TokenStream { +) -> Result>> { let source_file = psess.source_map().new_source_file(name, source); - unwrap_or_emit_fatal(maybe_source_file_to_stream(psess, source_file, override_span)) + source_file_to_stream(psess, source_file, override_span) } /// Given a source file, produces a sequence of token trees. Returns any buffered errors from /// parsing the token stream. -fn maybe_source_file_to_stream<'psess>( +fn source_file_to_stream<'psess>( psess: &'psess ParseSess, source_file: Lrc, override_span: Option, @@ -139,13 +140,18 @@ pub fn parse_in<'a, T>( pub fn fake_token_stream_for_item(psess: &ParseSess, item: &ast::Item) -> TokenStream { let source = pprust::item_to_string(item); let filename = FileName::macro_expansion_source_code(&source); - source_str_to_stream(psess, filename, source, Some(item.span)) + unwrap_or_emit_fatal(source_str_to_stream(psess, filename, source, Some(item.span))) } pub fn fake_token_stream_for_crate(psess: &ParseSess, krate: &ast::Crate) -> TokenStream { let source = pprust::crate_to_string_for_macros(krate); let filename = FileName::macro_expansion_source_code(&source); - source_str_to_stream(psess, filename, source, Some(krate.spans.inner_span)) + unwrap_or_emit_fatal(source_str_to_stream( + psess, + filename, + source, + Some(krate.spans.inner_span), + )) } pub fn parse_cfg_attr( diff --git a/compiler/rustc_parse/src/parser/tests.rs b/compiler/rustc_parse/src/parser/tests.rs index 2900d44089ea..79a6cf1b541e 100644 --- a/compiler/rustc_parse/src/parser/tests.rs +++ b/compiler/rustc_parse/src/parser/tests.rs @@ -1,5 +1,7 @@ use crate::parser::ForceCollect; -use crate::{new_parser_from_source_str, parser::Parser, source_str_to_stream}; +use crate::{ + new_parser_from_source_str, parser::Parser, source_str_to_stream, unwrap_or_emit_fatal, +}; use ast::token::IdentIsRaw; use rustc_ast::ptr::P; use rustc_ast::token::{self, Delimiter, Token}; @@ -29,7 +31,11 @@ fn psess() -> ParseSess { /// Map string to parser (via tts). fn string_to_parser(psess: &ParseSess, source_str: String) -> Parser<'_> { - new_parser_from_source_str(psess, PathBuf::from("bogofile").into(), source_str) + unwrap_or_emit_fatal(new_parser_from_source_str( + psess, + PathBuf::from("bogofile").into(), + source_str, + )) } fn create_test_handler() -> (DiagCtxt, Lrc, Arc>>) { @@ -82,7 +88,12 @@ where /// Maps a string to tts, using a made-up filename. pub(crate) fn string_to_stream(source_str: String) -> TokenStream { let psess = psess(); - source_str_to_stream(&psess, PathBuf::from("bogofile").into(), source_str, None) + unwrap_or_emit_fatal(source_str_to_stream( + &psess, + PathBuf::from("bogofile").into(), + source_str, + None, + )) } /// Parses a string, returns a crate. @@ -1064,7 +1075,8 @@ fn parse_item_from_source_str( source: String, psess: &ParseSess, ) -> PResult<'_, Option>> { - new_parser_from_source_str(psess, name, source).parse_item(ForceCollect::No) + unwrap_or_emit_fatal(new_parser_from_source_str(psess, name, source)) + .parse_item(ForceCollect::No) } // Produces a `rustc_span::span`. @@ -1345,7 +1357,7 @@ fn ttdelim_span() { source: String, psess: &ParseSess, ) -> PResult<'_, P> { - new_parser_from_source_str(psess, name, source).parse_expr() + unwrap_or_emit_fatal(new_parser_from_source_str(psess, name, source)).parse_expr() } create_default_session_globals_then(|| { diff --git a/src/librustdoc/clean/render_macro_matchers.rs b/src/librustdoc/clean/render_macro_matchers.rs index 5ac1b742c386..995919f73f83 100644 --- a/src/librustdoc/clean/render_macro_matchers.rs +++ b/src/librustdoc/clean/render_macro_matchers.rs @@ -65,7 +65,7 @@ fn snippet_equal_to_token(tcx: TyCtxt<'_>, matcher: &TokenTree) -> Option parser, Err(errs) => { errs.into_iter().for_each(|err| err.cancel()); diff --git a/src/librustdoc/doctest.rs b/src/librustdoc/doctest.rs index a536a7139219..0d4bad6921db 100644 --- a/src/librustdoc/doctest.rs +++ b/src/librustdoc/doctest.rs @@ -9,7 +9,7 @@ use rustc_interface::interface; use rustc_middle::hir::map::Map; use rustc_middle::hir::nested_filter; use rustc_middle::ty::TyCtxt; -use rustc_parse::maybe_new_parser_from_source_str; +use rustc_parse::new_parser_from_source_str; use rustc_parse::parser::attr::InnerAttrPolicy; use rustc_resolve::rustdoc::span_of_fragments; use rustc_session::config::{self, CrateType, ErrorOutputType}; @@ -638,7 +638,7 @@ pub(crate) fn make_test( let mut found_extern_crate = crate_name.is_none(); let mut found_macro = false; - let mut parser = match maybe_new_parser_from_source_str(&psess, filename, source) { + let mut parser = match new_parser_from_source_str(&psess, filename, source) { Ok(p) => p, Err(errs) => { errs.into_iter().for_each(|err| err.cancel()); @@ -818,16 +818,15 @@ fn check_if_attr_is_complete(source: &str, edition: Edition) -> bool { let dcx = DiagCtxt::new(Box::new(emitter)).disable_warnings(); let psess = ParseSess::with_dcx(dcx, sm); - let mut parser = - match maybe_new_parser_from_source_str(&psess, filename, source.to_owned()) { - Ok(p) => p, - Err(errs) => { - errs.into_iter().for_each(|err| err.cancel()); - // If there is an unclosed delimiter, an error will be returned by the - // tokentrees. - return false; - } - }; + let mut parser = match new_parser_from_source_str(&psess, filename, source.to_owned()) { + Ok(p) => p, + Err(errs) => { + errs.into_iter().for_each(|err| err.cancel()); + // If there is an unclosed delimiter, an error will be returned by the + // tokentrees. + return false; + } + }; // If a parsing error happened, it's very likely that the attribute is incomplete. if let Err(e) = parser.parse_attribute(InnerAttrPolicy::Permitted) { e.cancel(); diff --git a/src/librustdoc/passes/lint/check_code_block_syntax.rs b/src/librustdoc/passes/lint/check_code_block_syntax.rs index 409546734cb1..c185442fd55d 100644 --- a/src/librustdoc/passes/lint/check_code_block_syntax.rs +++ b/src/librustdoc/passes/lint/check_code_block_syntax.rs @@ -5,7 +5,7 @@ use rustc_errors::{ translation::{to_fluent_args, Translate}, Applicability, DiagCtxt, DiagInner, LazyFallbackBundle, }; -use rustc_parse::source_str_to_stream; +use rustc_parse::{source_str_to_stream, unwrap_or_emit_fatal}; use rustc_resolve::rustdoc::source_span_for_markdown_range; use rustc_session::parse::ParseSess; use rustc_span::hygiene::{AstPass, ExpnData, ExpnKind, LocalExpnId, Transparency}; @@ -51,8 +51,13 @@ fn check_rust_syntax( let span = DUMMY_SP.apply_mark(expn_id.to_expn_id(), Transparency::Transparent); let is_empty = rustc_driver::catch_fatal_errors(|| { - source_str_to_stream(&psess, FileName::Custom(String::from("doctest")), source, Some(span)) - .is_empty() + unwrap_or_emit_fatal(source_str_to_stream( + &psess, + FileName::Custom(String::from("doctest")), + source, + Some(span), + )) + .is_empty() }) .unwrap_or(false); let buffer = buffer.borrow(); diff --git a/src/tools/clippy/clippy_lints/src/doc/needless_doctest_main.rs b/src/tools/clippy/clippy_lints/src/doc/needless_doctest_main.rs index 651f2ebaee6f..c3e3c0431e6b 100644 --- a/src/tools/clippy/clippy_lints/src/doc/needless_doctest_main.rs +++ b/src/tools/clippy/clippy_lints/src/doc/needless_doctest_main.rs @@ -8,7 +8,7 @@ use rustc_data_structures::sync::Lrc; use rustc_errors::emitter::HumanEmitter; use rustc_errors::{Diag, DiagCtxt}; use rustc_lint::LateContext; -use rustc_parse::maybe_new_parser_from_source_str; +use rustc_parse::new_parser_from_source_str; use rustc_parse::parser::ForceCollect; use rustc_session::parse::ParseSess; use rustc_span::edition::Edition; @@ -50,7 +50,7 @@ pub fn check( let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); let psess = ParseSess::with_dcx(dcx, sm); - let mut parser = match maybe_new_parser_from_source_str(&psess, filename, code) { + let mut parser = match new_parser_from_source_str(&psess, filename, code) { Ok(p) => p, Err(errs) => { errs.into_iter().for_each(Diag::cancel); diff --git a/src/tools/rustfmt/src/parse/parser.rs b/src/tools/rustfmt/src/parse/parser.rs index 5dcdca1d9538..7adf339b453a 100644 --- a/src/tools/rustfmt/src/parse/parser.rs +++ b/src/tools/rustfmt/src/parse/parser.rs @@ -4,7 +4,8 @@ use std::path::{Path, PathBuf}; use rustc_ast::token::TokenKind; use rustc_ast::{ast, attr, ptr}; use rustc_errors::Diag; -use rustc_parse::{new_parser_from_file, parser::Parser as RawParser}; +use rustc_parse::parser::Parser as RawParser; +use rustc_parse::{new_parser_from_file, unwrap_or_emit_fatal}; use rustc_span::{sym, Span}; use thin_vec::ThinVec; @@ -68,10 +69,10 @@ impl<'a> ParserBuilder<'a> { ) -> Result, Option>>> { match input { Input::File(ref file) => catch_unwind(AssertUnwindSafe(move || { - new_parser_from_file(psess, file, None) + unwrap_or_emit_fatal(new_parser_from_file(psess, file, None)) })) .map_err(|_| None), - Input::Text(text) => rustc_parse::maybe_new_parser_from_source_str( + Input::Text(text) => rustc_parse::new_parser_from_source_str( psess, rustc_span::FileName::Custom("stdin".to_owned()), text, @@ -111,7 +112,8 @@ impl<'a> Parser<'a> { span: Span, ) -> Result<(ast::AttrVec, ThinVec>, Span), ParserError> { let result = catch_unwind(AssertUnwindSafe(|| { - let mut parser = new_parser_from_file(psess.inner(), path, Some(span)); + let mut parser = + unwrap_or_emit_fatal(new_parser_from_file(psess.inner(), path, Some(span))); match parser.parse_mod(&TokenKind::Eof) { Ok((a, i, spans)) => Some((a, i, spans.inner_span)), Err(e) => { diff --git a/tests/ui-fulldeps/mod_dir_path_canonicalized.rs b/tests/ui-fulldeps/mod_dir_path_canonicalized.rs index ef70e64ed7e9..99cb5fc5aa1c 100644 --- a/tests/ui-fulldeps/mod_dir_path_canonicalized.rs +++ b/tests/ui-fulldeps/mod_dir_path_canonicalized.rs @@ -16,7 +16,7 @@ extern crate rustc_span; #[allow(unused_extern_crates)] extern crate rustc_driver; -use rustc_parse::new_parser_from_file; +use rustc_parse::{new_parser_from_file, unwrap_or_emit_fatal}; use rustc_session::parse::ParseSess; use std::path::Path; @@ -34,6 +34,6 @@ fn parse() { let path = Path::new(file!()); let path = path.canonicalize().unwrap(); - let mut parser = new_parser_from_file(&psess, &path, None); + let mut parser = unwrap_or_emit_fatal(new_parser_from_file(&psess, &path, None)); let _ = parser.parse_crate_mod(); } diff --git a/tests/ui-fulldeps/pprust-expr-roundtrip.rs b/tests/ui-fulldeps/pprust-expr-roundtrip.rs index 2b1fec943871..762ad0b79ecc 100644 --- a/tests/ui-fulldeps/pprust-expr-roundtrip.rs +++ b/tests/ui-fulldeps/pprust-expr-roundtrip.rs @@ -36,7 +36,7 @@ use rustc_ast::mut_visit::{visit_clobber, MutVisitor}; use rustc_ast::ptr::P; use rustc_ast::*; use rustc_ast_pretty::pprust; -use rustc_parse::new_parser_from_source_str; +use rustc_parse::{new_parser_from_source_str, unwrap_or_emit_fatal}; use rustc_session::parse::ParseSess; use rustc_span::source_map::Spanned; use rustc_span::symbol::Ident; @@ -46,8 +46,9 @@ use thin_vec::{thin_vec, ThinVec}; fn parse_expr(psess: &ParseSess, src: &str) -> Option> { let src_as_string = src.to_string(); - let mut p = - new_parser_from_source_str(psess, FileName::Custom(src_as_string.clone()), src_as_string); + let mut p = unwrap_or_emit_fatal( + new_parser_from_source_str(psess, FileName::Custom(src_as_string.clone()), src_as_string) + ); p.parse_expr().map_err(|e| e.cancel()).ok() } From 2d4e7df14422ec46422e79ddae5af3ed78632bef Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Mon, 3 Jun 2024 14:32:03 +1000 Subject: [PATCH 14/14] rustfmt: Remove an unnecessary `catch_unwind` use. The `Input::File` and `Input::Text` cases should be very similar. However, currently the `Input::File` case uses `catch_unwind` because, until recently (#125815) there was a fallible version of `new_parser_from_source_str` but only an infallible version of `new_parser_from_file`. This difference wasn't fundamental, just an overlooked gap in the API of `rustc_parse`. Both of those operations are now fallible, so the `Input::File` and `Input::Text` cases can made more similar, with no need for `catch_unwind`. This also lets us simplify an `Option>` to `Vec`. --- src/tools/rustfmt/src/parse/parser.rs | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/src/tools/rustfmt/src/parse/parser.rs b/src/tools/rustfmt/src/parse/parser.rs index 7adf339b453a..6051241309d1 100644 --- a/src/tools/rustfmt/src/parse/parser.rs +++ b/src/tools/rustfmt/src/parse/parser.rs @@ -5,7 +5,7 @@ use rustc_ast::token::TokenKind; use rustc_ast::{ast, attr, ptr}; use rustc_errors::Diag; use rustc_parse::parser::Parser as RawParser; -use rustc_parse::{new_parser_from_file, unwrap_or_emit_fatal}; +use rustc_parse::{new_parser_from_file, new_parser_from_source_str, unwrap_or_emit_fatal}; use rustc_span::{sym, Span}; use thin_vec::ThinVec; @@ -51,12 +51,9 @@ impl<'a> ParserBuilder<'a> { let parser = match Self::parser(psess.inner(), input) { Ok(p) => p, - Err(db) => { - if let Some(diagnostics) = db { - psess.emit_diagnostics(diagnostics); - return Err(ParserError::ParserCreationError); - } - return Err(ParserError::ParsePanicError); + Err(diagnostics) => { + psess.emit_diagnostics(diagnostics); + return Err(ParserError::ParserCreationError); } }; @@ -66,18 +63,14 @@ impl<'a> ParserBuilder<'a> { fn parser( psess: &'a rustc_session::parse::ParseSess, input: Input, - ) -> Result, Option>>> { + ) -> Result, Vec>> { match input { - Input::File(ref file) => catch_unwind(AssertUnwindSafe(move || { - unwrap_or_emit_fatal(new_parser_from_file(psess, file, None)) - })) - .map_err(|_| None), - Input::Text(text) => rustc_parse::new_parser_from_source_str( + Input::File(ref file) => new_parser_from_file(psess, file, None), + Input::Text(text) => new_parser_from_source_str( psess, rustc_span::FileName::Custom("stdin".to_owned()), text, - ) - .map_err(Some), + ), } } }