Rollup merge of rust-lang#118680 - djkoloski:shell_argfiles, r=compiler-errors

Add support for shell argfiles

Closes rust-lang/compiler-team#684

Author: GuillaumeGomez

Cargo.lock

@@ -3738,6 +3738,7 @@ dependencies = [
  "rustc_trait_selection",
  "rustc_ty_utils",
  "serde_json",
+ "shlex",
  "time",
  "tracing",
  "windows",

compiler/rustc_driver_impl/Cargo.toml

@@ -50,6 +50,7 @@ rustc_target = { path = "../rustc_target" }
 rustc_trait_selection = { path = "../rustc_trait_selection" }
 rustc_ty_utils = { path = "../rustc_ty_utils" }
 serde_json = "1.0.59"
+shlex = "1.0"
 time = { version = "0.3", default-features = false, features = ["alloc", "formatting"] }
 tracing = { version = "0.1.35" }
 # tidy-alphabetical-end

compiler/rustc_driver_impl/src/args.rs

@@ -5,39 +5,113 @@ use std::io;
 
 use rustc_session::EarlyDiagCtxt;
 
-fn arg_expand(arg: String) -> Result<Vec<String>, Error> {
-    if let Some(path) = arg.strip_prefix('@') {
-        let file = match fs::read_to_string(path) {
-            Ok(file) => file,
-            Err(ref err) if err.kind() == io::ErrorKind::InvalidData => {
-                return Err(Error::Utf8Error(Some(path.to_string())));
+/// Expands argfiles in command line arguments.
+#[derive(Default)]
+struct Expander {
+    shell_argfiles: bool,
+    next_is_unstable_option: bool,
+    expanded: Vec<String>,
+}
+
+impl Expander {
+    /// Handles the next argument. If the argument is an argfile, it is expanded
+    /// inline.
+    fn arg(&mut self, arg: &str) -> Result<(), Error> {
+        if let Some(argfile) = arg.strip_prefix('@') {
+            match argfile.split_once(':') {
+                Some(("shell", path)) if self.shell_argfiles => {
+                    shlex::split(&Self::read_file(path)?)
+                        .ok_or_else(|| Error::ShellParseError(path.to_string()))?
+                        .into_iter()
+                        .for_each(|arg| self.push(arg));
+                }
+                _ => {
+                    let contents = Self::read_file(argfile)?;
+                    contents.lines().for_each(|arg| self.push(arg.to_string()));
+                }
+            }
+        } else {
+            self.push(arg.to_string());
+        }
+
+        Ok(())
+    }
+
+    /// Adds a command line argument verbatim with no argfile expansion.
+    fn push(&mut self, arg: String) {
+        // Unfortunately, we have to do some eager argparsing to handle unstable
+        // options which change the behavior of argfile arguments.
+        //
+        // Normally, all of the argfile arguments (e.g. `@args.txt`) are
+        // expanded into our arguments list *and then* the whole list of
+        // arguments are passed on to be parsed. However, argfile parsing
+        // options like `-Zshell_argfiles` need to change the behavior of that
+        // argument expansion. So we have to do a little parsing on our own here
+        // instead of leaning on the existing logic.
+        //
+        // All we care about are unstable options, so we parse those out and
+        // look for any that affect how we expand argfiles. This argument
+        // inspection is very conservative; we only change behavior when we see
+        // exactly the options we're looking for and everything gets passed
+        // through.
+
+        if self.next_is_unstable_option {
+            self.inspect_unstable_option(&arg);
+            self.next_is_unstable_option = false;
+        } else if let Some(unstable_option) = arg.strip_prefix("-Z") {
+            if unstable_option.is_empty() {
+                self.next_is_unstable_option = true;
+            } else {
+                self.inspect_unstable_option(unstable_option);
+            }
+        }
+
+        self.expanded.push(arg);
+    }
+
+    /// Consumes the `Expander`, returning the expanded arguments.
+    fn finish(self) -> Vec<String> {
+        self.expanded
+    }
+
+    /// Parses any relevant unstable flags specified on the command line.
+    fn inspect_unstable_option(&mut self, option: &str) {
+        match option {
+            "shell-argfiles" => self.shell_argfiles = true,
+            _ => (),
+        }
+    }
+
+    /// Reads the contents of a file as UTF-8.
+    fn read_file(path: &str) -> Result<String, Error> {
+        fs::read_to_string(path).map_err(|e| {
+            if e.kind() == io::ErrorKind::InvalidData {
+                Error::Utf8Error(Some(path.to_string()))
+            } else {
+                Error::IOError(path.to_string(), e)
             }
-            Err(err) => return Err(Error::IOError(path.to_string(), err)),
-        };
-        Ok(file.lines().map(ToString::to_string).collect())
-    } else {
-        Ok(vec![arg])
+        })
     }
 }
 
 /// **Note:** This function doesn't interpret argument 0 in any special way.
 /// If this function is intended to be used with command line arguments,
 /// `argv[0]` must be removed prior to calling it manually.
 pub fn arg_expand_all(early_dcx: &EarlyDiagCtxt, at_args: &[String]) -> Vec<String> {
-    let mut args = Vec::new();
+    let mut expander = Expander::default();
     for arg in at_args {
-        match arg_expand(arg.clone()) {
-            Ok(arg) => args.extend(arg),
-            Err(err) => early_dcx.early_fatal(format!("Failed to load argument file: {err}")),
+        if let Err(err) = expander.arg(arg) {
+            early_dcx.early_fatal(format!("Failed to load argument file: {err}"));
         }
     }
-    args
+    expander.finish()
 }
 
 #[derive(Debug)]
 pub enum Error {
     Utf8Error(Option<String>),
     IOError(String, io::Error),
+    ShellParseError(String),
 }
 
 impl fmt::Display for Error {
@@ -46,6 +120,7 @@ impl fmt::Display for Error {
             Error::Utf8Error(None) => write!(fmt, "Utf8 error"),
             Error::Utf8Error(Some(path)) => write!(fmt, "Utf8 error in {path}"),
             Error::IOError(path, err) => write!(fmt, "IO Error: {path}: {err}"),
+            Error::ShellParseError(path) => write!(fmt, "Invalid shell-style arguments in {path}"),
         }
     }
 }

compiler/rustc_interface/src/tests.rs

@@ -698,6 +698,7 @@ fn test_unstable_options_tracking_hash() {
     untracked!(query_dep_graph, true);
     untracked!(self_profile, SwitchWithOptPath::Enabled(None));
     untracked!(self_profile_events, Some(vec![String::new()]));
+    untracked!(shell_argfiles, true);
     untracked!(span_debug, true);
     untracked!(span_free_formats, true);
     untracked!(temps_dir, Some(String::from("abc")));

compiler/rustc_session/src/options.rs

@@ -1841,6 +1841,8 @@ written to standard error output)"),
                      query-blocked, incr-cache-load, incr-result-hashing, query-keys, function-args, args, llvm, artifact-sizes"),
     share_generics: Option<bool> = (None, parse_opt_bool, [TRACKED],
         "make the current crate share its generic instantiations"),
+    shell_argfiles: bool = (false, parse_bool, [UNTRACKED],
+        "allow argument files to be specified with POSIX \"shell-style\" argument quoting"),
     show_span: Option<String> = (None, parse_opt_string, [TRACKED],
         "show spans for compiler debugging (expr|pat|ty)"),
     simulate_remapped_rust_src_base: Option<PathBuf> = (None, parse_opt_pathbuf, [TRACKED],

src/doc/unstable-book/src/compiler-flags/shell-argfiles.md

@@ -0,0 +1,11 @@
+# `shell-argfiles`
+
+--------------------
+
+The `-Zshell-argfiles` compiler flag allows argfiles to be parsed using POSIX
+"shell-style" quoting. When enabled, the compiler will use `shlex` to parse the
+arguments from argfiles specified with `@shell:<path>`.
+
+Because this feature controls the parsing of input arguments, the
+`-Zshell-argfiles` flag must be present before the argument specifying the
+shell-style arguemnt file.

src/tools/tidy/src/deps.rs

@@ -325,6 +325,7 @@ const PERMITTED_RUSTC_DEPENDENCIES: &[&str] = &[
     "sha1",
     "sha2",
     "sharded-slab",
+    "shlex",
     "smallvec",
     "snap",
     "stable_deref_trait",

src/tools/tidy/src/ui_tests.rs

@@ -11,7 +11,7 @@ use std::path::{Path, PathBuf};
 const ENTRY_LIMIT: usize = 900;
 // FIXME: The following limits should be reduced eventually.
 const ISSUES_ENTRY_LIMIT: usize = 1849;
-const ROOT_ENTRY_LIMIT: usize = 867;
+const ROOT_ENTRY_LIMIT: usize = 868;
 
 const EXPECTED_TEST_FILE_EXTENSIONS: &[&str] = &[
     "rs",     // test source files
@@ -35,6 +35,10 @@ const EXTENSION_EXCEPTION_PATHS: &[&str] = &[
     "tests/ui/macros/syntax-extension-source-utils-files/includeme.fragment", // more include
     "tests/ui/proc-macro/auxiliary/included-file.txt", // more include
     "tests/ui/invalid/foo.natvis.xml", // sample debugger visualizer
+    "tests/ui/shell-argfiles/shell-argfiles.args", // passing args via a file
+    "tests/ui/shell-argfiles/shell-argfiles-badquotes.args", // passing args via a file
+    "tests/ui/shell-argfiles/shell-argfiles-via-argfile-shell.args", // passing args via a file
+    "tests/ui/shell-argfiles/shell-argfiles-via-argfile.args", // passing args via a file
 ];
 
 fn check_entries(tests_path: &Path, bad: &mut bool) {

tests/ui/shell-argfiles/shell-argfiles-badquotes-windows.rs

@@ -0,0 +1,11 @@
+// Check to see if we can get parameters from an @argsfile file
+//
+// Path replacement in .stderr files (i.e. `$DIR`) doesn't handle mixed path
+// separators. This test uses backslash as the path separator for the command
+// line arguments and is only run on windows.
+//
+// only-windows
+// compile-flags: --cfg cmdline_set -Z shell-argfiles @shell:{{src-base}}\shell-argfiles\shell-argfiles-badquotes.args
+
+fn main() {
+}

tests/ui/shell-argfiles/shell-argfiles-badquotes-windows.stderr

@@ -0,0 +1,2 @@
+error: Failed to load argument file: Invalid shell-style arguments in $DIR/shell-argfiles-badquotes.args
+

tests/ui/shell-argfiles/shell-argfiles-badquotes.args

@@ -0,0 +1 @@
+"--cfg" "unquoted_set

tests/ui/shell-argfiles/shell-argfiles-badquotes.rs

@@ -0,0 +1,12 @@
+// Check to see if we can get parameters from an @argsfile file
+//
+// Path replacement in .stderr files (i.e. `$DIR`) doesn't handle mixed path
+// separators. We have a duplicated version of this test that uses backslash as
+// the path separator for the command line arguments that is only run on
+// windows.
+//
+// ignore-windows
+// compile-flags: --cfg cmdline_set -Z shell-argfiles @shell:{{src-base}}/shell-argfiles/shell-argfiles-badquotes.args
+
+fn main() {
+}

tests/ui/shell-argfiles/shell-argfiles-badquotes.stderr

@@ -0,0 +1,2 @@
+error: Failed to load argument file: Invalid shell-style arguments in $DIR/shell-argfiles-badquotes.args
+

tests/ui/shell-argfiles/shell-argfiles-via-argfile-shell.args

@@ -0,0 +1 @@
+"--cfg" "shell_args_set"

tests/ui/shell-argfiles/shell-argfiles-via-argfile.args

@@ -0,0 +1 @@
+-Zshell-argfiles

tests/ui/shell-argfiles/shell-argfiles-via-argfile.rs

@@ -0,0 +1,10 @@
+// Check to see if we can get parameters from an @argsfile file
+//
+// build-pass
+// compile-flags: @{{src-base}}/shell-argfiles/shell-argfiles-via-argfile.args @shell:{{src-base}}/shell-argfiles/shell-argfiles-via-argfile-shell.args
+
+#[cfg(not(shell_args_set))]
+compile_error!("shell_args_set not set");
+
+fn main() {
+}

tests/ui/shell-argfiles/shell-argfiles.args

@@ -0,0 +1,3 @@
+--cfg unquoted_set
+'--cfg' 'single_quoted_set'
+"--cfg" "double_quoted_set"

tests/ui/shell-argfiles/shell-argfiles.rs

@@ -0,0 +1,19 @@
+// Check to see if we can get parameters from an @argsfile file
+//
+// build-pass
+// compile-flags: --cfg cmdline_set -Z shell-argfiles @shell:{{src-base}}/shell-argfiles/shell-argfiles.args
+
+#[cfg(not(cmdline_set))]
+compile_error!("cmdline_set not set");
+
+#[cfg(not(unquoted_set))]
+compile_error!("unquoted_set not set");
+
+#[cfg(not(single_quoted_set))]
+compile_error!("single_quoted_set not set");
+
+#[cfg(not(double_quoted_set))]
+compile_error!("double_quoted_set not set");
+
+fn main() {
+}