Auto merge of rust-lang#118069 - onur-ozkan:bypass_bootstrap_lock, r=Mark-Simulacrum

allow bypassing the build directory lock

As bootstrap locks its entire build directory, parallel bootstrapping for anything becomes impossible. This change enables developers to bypass the locking mechanism (with `--bypass-bootstrap-lock` flag) when it is unnecessary for their specific use case.

more context: https://rust-lang.zulipchat.com/#narrow/stream/326414-t-infra.2Fbootstrap/topic/Build.20.28miri.3F.29.20sysroots.20in.20parallel

cc `@saethlin`

Author: bors

src/bootstrap/src/bin/main.rs

@@ -26,35 +26,40 @@ fn main() {
     let mut build_lock;
     #[cfg(all(any(unix, windows), not(target_os = "solaris")))]
     let _build_lock_guard;
-    #[cfg(all(any(unix, windows), not(target_os = "solaris")))]
-    // Display PID of process holding the lock
-    // PID will be stored in a lock file
-    {
-        let path = config.out.join("lock");
-        let pid = match fs::read_to_string(&path) {
-            Ok(contents) => contents,
-            Err(_) => String::new(),
-        };
-
-        build_lock =
-            fd_lock::RwLock::new(t!(fs::OpenOptions::new().write(true).create(true).open(&path)));
-        _build_lock_guard = match build_lock.try_write() {
-            Ok(mut lock) => {
-                t!(lock.write(&process::id().to_string().as_ref()));
-                lock
-            }
-            err => {
-                drop(err);
-                println!("WARNING: build directory locked by process {pid}, waiting for lock");
-                let mut lock = t!(build_lock.write());
-                t!(lock.write(&process::id().to_string().as_ref()));
-                lock
-            }
-        };
-    }
 
-    #[cfg(any(not(any(unix, windows)), target_os = "solaris"))]
-    println!("WARNING: file locking not supported for target, not locking build directory");
+    if !config.bypass_bootstrap_lock {
+        // Display PID of process holding the lock
+        // PID will be stored in a lock file
+        #[cfg(all(any(unix, windows), not(target_os = "solaris")))]
+        {
+            let path = config.out.join("lock");
+            let pid = match fs::read_to_string(&path) {
+                Ok(contents) => contents,
+                Err(_) => String::new(),
+            };
+
+            build_lock = fd_lock::RwLock::new(t!(fs::OpenOptions::new()
+                .write(true)
+                .create(true)
+                .open(&path)));
+            _build_lock_guard = match build_lock.try_write() {
+                Ok(mut lock) => {
+                    t!(lock.write(&process::id().to_string().as_ref()));
+                    lock
+                }
+                err => {
+                    drop(err);
+                    println!("WARNING: build directory locked by process {pid}, waiting for lock");
+                    let mut lock = t!(build_lock.write());
+                    t!(lock.write(&process::id().to_string().as_ref()));
+                    lock
+                }
+            };
+        }
+
+        #[cfg(any(not(any(unix, windows)), target_os = "solaris"))]
+        println!("WARNING: file locking not supported for target, not locking build directory");
+    }
 
     // check_version warnings are not printed during setup
     let changelog_suggestion =

src/bootstrap/src/core/config/config.rs

@@ -117,6 +117,7 @@ impl Display for DebuginfoLevel {
 pub struct Config {
     pub changelog_seen: Option<usize>, // FIXME: Deprecated field. Remove it at 2024.
     pub change_id: Option<usize>,
+    pub bypass_bootstrap_lock: bool,
     pub ccache: Option<String>,
     /// Call Build::ninja() instead of this.
     pub ninja_in_file: bool,
@@ -1059,6 +1060,7 @@ define_config! {
 impl Config {
     pub fn default_opts() -> Config {
         let mut config = Config::default();
+        config.bypass_bootstrap_lock = false;
         config.llvm_optimize = true;
         config.ninja_in_file = true;
         config.llvm_static_stdcpp = false;
@@ -1137,6 +1139,7 @@ impl Config {
         config.llvm_profile_use = flags.llvm_profile_use;
         config.llvm_profile_generate = flags.llvm_profile_generate;
         config.enable_bolt_settings = flags.enable_bolt_settings;
+        config.bypass_bootstrap_lock = flags.bypass_bootstrap_lock;
 
         // Infer the rest of the configuration.
 

src/bootstrap/src/core/config/flags.rs

@@ -133,6 +133,13 @@ pub struct Flags {
     /// whether to use color in cargo and rustc output
     pub color: Color,
 
+    #[arg(global(true), long)]
+    /// Bootstrap uses this value to decide whether it should bypass locking the build process.
+    /// This is rarely needed (e.g., compiling the std library for different targets in parallel).
+    ///
+    /// Unless you know exactly what you are doing, you probably don't need this.
+    pub bypass_bootstrap_lock: bool,
+
     /// whether rebuilding llvm should be skipped, overriding `skip-rebuld` in config.toml
     #[arg(global(true), long, value_name = "VALUE")]
     pub llvm_skip_rebuild: Option<bool>,

src/etc/completions/x.py.fish

@@ -46,11 +46,12 @@ _x.py() {
 '--include-default-paths[include default paths in addition to the provided ones]' \
 '--dry-run[dry run; don'\''t build anything]' \
 '--json-output[use message-format=json]' \
+'--bypass-bootstrap-lock[Bootstrap uses this value to decide whether it should bypass locking the build process. This is rarely needed (e.g., compiling the std library for different targets in parallel)]' \
 '--llvm-profile-generate[generate PGO profile with llvm built for rustc]' \
 '--enable-bolt-settings[Enable BOLT link flags]' \
 '--skip-stage0-validation[Skip stage0 compiler validation]' \
-'-h[Print help]' \
-'--help[Print help]' \
+'-h[Print help (see more with '\''--help'\'')]' \
+'--help[Print help (see more with '\''--help'\'')]' \
 '::paths -- paths for the subcommand:_files' \
 '::free_args -- arguments passed to subcommands:' \
 ":: :_x.py_commands" \
@@ -95,6 +96,7 @@ _arguments "${_arguments_options[@]}" \
 '--include-default-paths[include default paths in addition to the provided ones]' \
 '--dry-run[dry run; don'\''t build anything]' \
 '--json-output[use message-format=json]' \
+'--bypass-bootstrap-lock[Bootstrap uses this value to decide whether it should bypass locking the build process. This is rarely needed (e.g., compiling the std library for different targets in parallel)]' \
 '--llvm-profile-generate[generate PGO profile with llvm built for rustc]' \
 '--enable-bolt-settings[Enable BOLT link flags]' \
 '--skip-stage0-validation[Skip stage0 compiler validation]' \
@@ -137,6 +139,7 @@ _arguments "${_arguments_options[@]}" \
 '--include-default-paths[include default paths in addition to the provided ones]' \
 '--dry-run[dry run; don'\''t build anything]' \
 '--json-output[use message-format=json]' \
+'--bypass-bootstrap-lock[Bootstrap uses this value to decide whether it should bypass locking the build process. This is rarely needed (e.g., compiling the std library for different targets in parallel)]' \
 '--llvm-profile-generate[generate PGO profile with llvm built for rustc]' \
 '--enable-bolt-settings[Enable BOLT link flags]' \
 '--skip-stage0-validation[Skip stage0 compiler validation]' \
@@ -183,6 +186,7 @@ _arguments "${_arguments_options[@]}" \
 '--include-default-paths[include default paths in addition to the provided ones]' \
 '--dry-run[dry run; don'\''t build anything]' \
 '--json-output[use message-format=json]' \
+'--bypass-bootstrap-lock[Bootstrap uses this value to decide whether it should bypass locking the build process. This is rarely needed (e.g., compiling the std library for different targets in parallel)]' \
 '--llvm-profile-generate[generate PGO profile with llvm built for rustc]' \
 '--enable-bolt-settings[Enable BOLT link flags]' \
 '--skip-stage0-validation[Skip stage0 compiler validation]' \
@@ -224,6 +228,7 @@ _arguments "${_arguments_options[@]}" \
 '--include-default-paths[include default paths in addition to the provided ones]' \
 '--dry-run[dry run; don'\''t build anything]' \
 '--json-output[use message-format=json]' \
+'--bypass-bootstrap-lock[Bootstrap uses this value to decide whether it should bypass locking the build process. This is rarely needed (e.g., compiling the std library for different targets in parallel)]' \
 '--llvm-profile-generate[generate PGO profile with llvm built for rustc]' \
 '--enable-bolt-settings[Enable BOLT link flags]' \
 '--skip-stage0-validation[Skip stage0 compiler validation]' \
@@ -266,6 +271,7 @@ _arguments "${_arguments_options[@]}" \
 '--include-default-paths[include default paths in addition to the provided ones]' \
 '--dry-run[dry run; don'\''t build anything]' \
 '--json-output[use message-format=json]' \
+'--bypass-bootstrap-lock[Bootstrap uses this value to decide whether it should bypass locking the build process. This is rarely needed (e.g., compiling the std library for different targets in parallel)]' \
 '--llvm-profile-generate[generate PGO profile with llvm built for rustc]' \
 '--enable-bolt-settings[Enable BOLT link flags]' \
 '--skip-stage0-validation[Skip stage0 compiler validation]' \
@@ -309,6 +315,7 @@ _arguments "${_arguments_options[@]}" \
 '--include-default-paths[include default paths in addition to the provided ones]' \
 '--dry-run[dry run; don'\''t build anything]' \
 '--json-output[use message-format=json]' \
+'--bypass-bootstrap-lock[Bootstrap uses this value to decide whether it should bypass locking the build process. This is rarely needed (e.g., compiling the std library for different targets in parallel)]' \
 '--llvm-profile-generate[generate PGO profile with llvm built for rustc]' \
 '--enable-bolt-settings[Enable BOLT link flags]' \
 '--skip-stage0-validation[Skip stage0 compiler validation]' \
@@ -363,6 +370,7 @@ _arguments "${_arguments_options[@]}" \
 '--include-default-paths[include default paths in addition to the provided ones]' \
 '--dry-run[dry run; don'\''t build anything]' \
 '--json-output[use message-format=json]' \
+'--bypass-bootstrap-lock[Bootstrap uses this value to decide whether it should bypass locking the build process. This is rarely needed (e.g., compiling the std library for different targets in parallel)]' \
 '--llvm-profile-generate[generate PGO profile with llvm built for rustc]' \
 '--enable-bolt-settings[Enable BOLT link flags]' \
 '--skip-stage0-validation[Skip stage0 compiler validation]' \
@@ -405,11 +413,12 @@ _arguments "${_arguments_options[@]}" \
 '--include-default-paths[include default paths in addition to the provided ones]' \
 '--dry-run[dry run; don'\''t build anything]' \
 '--json-output[use message-format=json]' \
+'--bypass-bootstrap-lock[Bootstrap uses this value to decide whether it should bypass locking the build process. This is rarely needed (e.g., compiling the std library for different targets in parallel)]' \
 '--llvm-profile-generate[generate PGO profile with llvm built for rustc]' \
 '--enable-bolt-settings[Enable BOLT link flags]' \
 '--skip-stage0-validation[Skip stage0 compiler validation]' \
-'-h[Print help]' \
-'--help[Print help]' \
+'-h[Print help (see more with '\''--help'\'')]' \
+'--help[Print help (see more with '\''--help'\'')]' \
 '*::paths -- paths for the subcommand:_files' \
 && ret=0
 ;;
@@ -447,11 +456,12 @@ _arguments "${_arguments_options[@]}" \
 '--include-default-paths[include default paths in addition to the provided ones]' \
 '--dry-run[dry run; don'\''t build anything]' \
 '--json-output[use message-format=json]' \
+'--bypass-bootstrap-lock[Bootstrap uses this value to decide whether it should bypass locking the build process. This is rarely needed (e.g., compiling the std library for different targets in parallel)]' \
 '--llvm-profile-generate[generate PGO profile with llvm built for rustc]' \
 '--enable-bolt-settings[Enable BOLT link flags]' \
 '--skip-stage0-validation[Skip stage0 compiler validation]' \
-'-h[Print help]' \
-'--help[Print help]' \
+'-h[Print help (see more with '\''--help'\'')]' \
+'--help[Print help (see more with '\''--help'\'')]' \
 '*::paths -- paths for the subcommand:_files' \
 && ret=0
 ;;
@@ -488,11 +498,12 @@ _arguments "${_arguments_options[@]}" \
 '--include-default-paths[include default paths in addition to the provided ones]' \
 '--dry-run[dry run; don'\''t build anything]' \
 '--json-output[use message-format=json]' \
+'--bypass-bootstrap-lock[Bootstrap uses this value to decide whether it should bypass locking the build process. This is rarely needed (e.g., compiling the std library for different targets in parallel)]' \
 '--llvm-profile-generate[generate PGO profile with llvm built for rustc]' \
 '--enable-bolt-settings[Enable BOLT link flags]' \
 '--skip-stage0-validation[Skip stage0 compiler validation]' \
-'-h[Print help]' \
-'--help[Print help]' \
+'-h[Print help (see more with '\''--help'\'')]' \
+'--help[Print help (see more with '\''--help'\'')]' \
 '*::paths -- paths for the subcommand:_files' \
 && ret=0
 ;;
@@ -529,11 +540,12 @@ _arguments "${_arguments_options[@]}" \
 '--include-default-paths[include default paths in addition to the provided ones]' \
 '--dry-run[dry run; don'\''t build anything]' \
 '--json-output[use message-format=json]' \
+'--bypass-bootstrap-lock[Bootstrap uses this value to decide whether it should bypass locking the build process. This is rarely needed (e.g., compiling the std library for different targets in parallel)]' \
 '--llvm-profile-generate[generate PGO profile with llvm built for rustc]' \
 '--enable-bolt-settings[Enable BOLT link flags]' \
 '--skip-stage0-validation[Skip stage0 compiler validation]' \
-'-h[Print help]' \
-'--help[Print help]' \
+'-h[Print help (see more with '\''--help'\'')]' \
+'--help[Print help (see more with '\''--help'\'')]' \
 '*::paths -- paths for the subcommand:_files' \
 && ret=0
 ;;
@@ -571,6 +583,7 @@ _arguments "${_arguments_options[@]}" \
 '--include-default-paths[include default paths in addition to the provided ones]' \
 '--dry-run[dry run; don'\''t build anything]' \
 '--json-output[use message-format=json]' \
+'--bypass-bootstrap-lock[Bootstrap uses this value to decide whether it should bypass locking the build process. This is rarely needed (e.g., compiling the std library for different targets in parallel)]' \
 '--llvm-profile-generate[generate PGO profile with llvm built for rustc]' \
 '--enable-bolt-settings[Enable BOLT link flags]' \
 '--skip-stage0-validation[Skip stage0 compiler validation]' \
@@ -612,6 +625,7 @@ _arguments "${_arguments_options[@]}" \
 '--include-default-paths[include default paths in addition to the provided ones]' \
 '--dry-run[dry run; don'\''t build anything]' \
 '--json-output[use message-format=json]' \
+'--bypass-bootstrap-lock[Bootstrap uses this value to decide whether it should bypass locking the build process. This is rarely needed (e.g., compiling the std library for different targets in parallel)]' \
 '--llvm-profile-generate[generate PGO profile with llvm built for rustc]' \
 '--enable-bolt-settings[Enable BOLT link flags]' \
 '--skip-stage0-validation[Skip stage0 compiler validation]' \
@@ -655,6 +669,7 @@ _arguments "${_arguments_options[@]}" \
 '--include-default-paths[include default paths in addition to the provided ones]' \
 '--dry-run[dry run; don'\''t build anything]' \
 '--json-output[use message-format=json]' \
+'--bypass-bootstrap-lock[Bootstrap uses this value to decide whether it should bypass locking the build process. This is rarely needed (e.g., compiling the std library for different targets in parallel)]' \
 '--llvm-profile-generate[generate PGO profile with llvm built for rustc]' \
 '--enable-bolt-settings[Enable BOLT link flags]' \
 '--skip-stage0-validation[Skip stage0 compiler validation]' \