diff --git a/compiler/rustc_parse/src/lexer/mod.rs b/compiler/rustc_parse/src/lexer/mod.rs index e3bd6a9a3270f..f5f081efc4956 100644 --- a/compiler/rustc_parse/src/lexer/mod.rs +++ b/compiler/rustc_parse/src/lexer/mod.rs @@ -44,19 +44,44 @@ pub(crate) struct UnmatchedDelim { pub candidate_span: Option, } +/// Which tokens should be stripped before lexing the tokens. +pub(crate) enum StripTokens { + /// Strip both shebang and frontmatter. + ShebangAndFrontmatter, + /// Strip the shebang but not frontmatter. + /// + /// That means that char sequences looking like frontmatter are simply + /// interpreted as regular Rust lexemes. + Shebang, + /// Strip nothing. + /// + /// In other words, char sequences looking like a shebang or frontmatter + /// are simply interpreted as regular Rust lexemes. + Nothing, +} + pub(crate) fn lex_token_trees<'psess, 'src>( psess: &'psess ParseSess, mut src: &'src str, mut start_pos: BytePos, override_span: Option, - frontmatter_allowed: FrontmatterAllowed, + strip_tokens: StripTokens, ) -> Result>> { - // Skip `#!`, if present. - if let Some(shebang_len) = rustc_lexer::strip_shebang(src) { - src = &src[shebang_len..]; - start_pos = start_pos + BytePos::from_usize(shebang_len); + match strip_tokens { + StripTokens::Shebang | StripTokens::ShebangAndFrontmatter => { + if let Some(shebang_len) = rustc_lexer::strip_shebang(src) { + src = &src[shebang_len..]; + start_pos = start_pos + BytePos::from_usize(shebang_len); + } + } + StripTokens::Nothing => {} } + let frontmatter_allowed = match strip_tokens { + StripTokens::ShebangAndFrontmatter => FrontmatterAllowed::Yes, + StripTokens::Shebang | StripTokens::Nothing => FrontmatterAllowed::No, + }; + let cursor = Cursor::new(src, frontmatter_allowed); let mut lexer = Lexer { psess, diff --git a/compiler/rustc_parse/src/lib.rs b/compiler/rustc_parse/src/lib.rs index b790966acfd92..d8792d7af4c36 100644 --- a/compiler/rustc_parse/src/lib.rs +++ b/compiler/rustc_parse/src/lib.rs @@ -21,7 +21,6 @@ use rustc_ast::tokenstream::{DelimSpan, TokenStream}; use rustc_ast::{AttrItem, Attribute, MetaItemInner, token}; use rustc_ast_pretty::pprust; use rustc_errors::{Diag, EmissionGuarantee, FatalError, PResult, pluralize}; -use rustc_lexer::FrontmatterAllowed; use rustc_session::parse::ParseSess; use rustc_span::source_map::SourceMap; use rustc_span::{FileName, SourceFile, Span}; @@ -34,6 +33,8 @@ pub mod parser; use parser::Parser; use rustc_ast::token::Delimiter; +use crate::lexer::StripTokens; + pub mod lexer; mod errors; @@ -62,10 +63,10 @@ pub fn new_parser_from_source_str( source: String, ) -> Result, Vec>> { let source_file = psess.source_map().new_source_file(name, source); - new_parser_from_source_file(psess, source_file, FrontmatterAllowed::Yes) + new_parser_from_source_file(psess, source_file, StripTokens::ShebangAndFrontmatter) } -/// Creates a new parser from a simple (no frontmatter) source string. +/// Creates a new parser from a simple (no shebang, no frontmatter) source string. /// /// On failure, the errors must be consumed via `unwrap_or_emit_fatal`, `emit`, `cancel`, /// etc., otherwise a panic will occur when they are dropped. @@ -75,7 +76,7 @@ pub fn new_parser_from_simple_source_str( source: String, ) -> Result, Vec>> { let source_file = psess.source_map().new_source_file(name, source); - new_parser_from_source_file(psess, source_file, FrontmatterAllowed::No) + new_parser_from_source_file(psess, source_file, StripTokens::Nothing) } /// Creates a new parser from a filename. On failure, the errors must be consumed via @@ -109,7 +110,7 @@ pub fn new_parser_from_file<'a>( } err.emit(); }); - new_parser_from_source_file(psess, source_file, FrontmatterAllowed::Yes) + new_parser_from_source_file(psess, source_file, StripTokens::ShebangAndFrontmatter) } pub fn utf8_error( @@ -160,10 +161,10 @@ pub fn utf8_error( fn new_parser_from_source_file( psess: &ParseSess, source_file: Arc, - frontmatter_allowed: FrontmatterAllowed, + strip_tokens: StripTokens, ) -> Result, Vec>> { let end_pos = source_file.end_position(); - let stream = source_file_to_stream(psess, source_file, None, frontmatter_allowed)?; + let stream = source_file_to_stream(psess, source_file, None, strip_tokens)?; let mut parser = Parser::new(psess, stream, None); if parser.token == token::Eof { parser.token.span = Span::new(end_pos, end_pos, parser.token.span.ctxt(), None); @@ -179,8 +180,8 @@ pub fn source_str_to_stream( ) -> Result>> { let source_file = psess.source_map().new_source_file(name, source); // used mainly for `proc_macro` and the likes, not for our parsing purposes, so don't parse - // frontmatters as frontmatters. - source_file_to_stream(psess, source_file, override_span, FrontmatterAllowed::No) + // frontmatters as frontmatters, but for compatibility reason still strip the shebang + source_file_to_stream(psess, source_file, override_span, StripTokens::Shebang) } /// Given a source file, produces a sequence of token trees. Returns any buffered errors from @@ -189,7 +190,7 @@ fn source_file_to_stream<'psess>( psess: &'psess ParseSess, source_file: Arc, override_span: Option, - frontmatter_allowed: FrontmatterAllowed, + strip_tokens: StripTokens, ) -> Result>> { let src = source_file.src.as_ref().unwrap_or_else(|| { psess.dcx().bug(format!( @@ -198,13 +199,7 @@ fn source_file_to_stream<'psess>( )); }); - lexer::lex_token_trees( - psess, - src.as_str(), - source_file.start_pos, - override_span, - frontmatter_allowed, - ) + lexer::lex_token_trees(psess, src.as_str(), source_file.start_pos, override_span, strip_tokens) } /// Runs the given subparser `f` on the tokens of the given `attr`'s item. diff --git a/tests/run-make/multiline-args-value/cfg.stderr b/tests/run-make/multiline-args-value/cfg-frontmatter.stderr similarity index 100% rename from tests/run-make/multiline-args-value/cfg.stderr rename to tests/run-make/multiline-args-value/cfg-frontmatter.stderr diff --git a/tests/run-make/multiline-args-value/cfg-shebang.stderr b/tests/run-make/multiline-args-value/cfg-shebang.stderr new file mode 100644 index 0000000000000..09baf3b51239c --- /dev/null +++ b/tests/run-make/multiline-args-value/cfg-shebang.stderr @@ -0,0 +1,3 @@ +error: invalid `--cfg` argument: `#!/usr/bin/shebang + key` (expected `key` or `key="value"`) + diff --git a/tests/run-make/multiline-args-value/check-cfg.stderr b/tests/run-make/multiline-args-value/check-cfg-frontmatter.stderr similarity index 100% rename from tests/run-make/multiline-args-value/check-cfg.stderr rename to tests/run-make/multiline-args-value/check-cfg-frontmatter.stderr diff --git a/tests/run-make/multiline-args-value/check-cfg-shebang.stderr b/tests/run-make/multiline-args-value/check-cfg-shebang.stderr new file mode 100644 index 0000000000000..5bf18dc2b14c4 --- /dev/null +++ b/tests/run-make/multiline-args-value/check-cfg-shebang.stderr @@ -0,0 +1,6 @@ +error: invalid `--check-cfg` argument: `#!/usr/bin/shebang + cfg(key)` + | + = note: expected `cfg(name, values("value1", "value2", ... "valueN"))` + = note: visit for more details + diff --git a/tests/run-make/multiline-args-value/rmake.rs b/tests/run-make/multiline-args-value/rmake.rs index 825cbd158c0a3..3964cbbc1e605 100644 --- a/tests/run-make/multiline-args-value/rmake.rs +++ b/tests/run-make/multiline-args-value/rmake.rs @@ -1,15 +1,14 @@ use run_make_support::{cwd, diff, rustc}; -fn test_and_compare(flag: &str, val: &str) { +fn test_and_compare(test_name: &str, flag: &str, val: &str) { let mut cmd = rustc(); - let output = - cmd.input("").arg("--crate-type=lib").arg(&format!("--{flag}")).arg(val).run_fail(); + let output = cmd.input("").arg("--crate-type=lib").arg(flag).arg(val).run_fail(); assert_eq!(output.stdout_utf8(), ""); diff() - .expected_file(format!("{flag}.stderr")) - .actual_text("output", output.stderr_utf8()) + .expected_file(format!("{test_name}.stderr")) + .actual_text("stderr", output.stderr_utf8()) .run(); } @@ -17,7 +16,8 @@ fn main() { // Verify that frontmatter isn't allowed in `--cfg` arguments. // https://github.com/rust-lang/rust/issues/146130 test_and_compare( - "cfg", + "cfg-frontmatter", + "--cfg", r#"--- --- key"#, @@ -26,9 +26,26 @@ key"#, // Verify that frontmatter isn't allowed in `--check-cfg` arguments. // https://github.com/rust-lang/rust/issues/146130 test_and_compare( - "check-cfg", + "check-cfg-frontmatter", + "--check-cfg", r#"--- --- +cfg(key)"#, + ); + + // Verify that shebang isn't allowed in `--cfg` arguments. + test_and_compare( + "cfg-shebang", + "--cfg", + r#"#!/usr/bin/shebang +key"#, + ); + + // Verify that shebang isn't allowed in `--check-cfg` arguments. + test_and_compare( + "check-cfg-shebang", + "--check-cfg", + r#"#!/usr/bin/shebang cfg(key)"#, ); }