Only check std in cross-compilation instead of building it

Author: Kobzol

src/bootstrap/src/core/build_steps/check.rs

@@ -1,7 +1,7 @@
 //! Implementation of compiling the compiler and standard library, in "check"-based modes.
 
 use std::fs;
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 
 use crate::core::build_steps::compile::{
     add_to_sysroot, run_cargo, rustc_cargo, rustc_cargo_env, std_cargo, std_crates_for_run_make,
@@ -36,7 +36,7 @@ impl Std {
 }
 
 impl Step for Std {
-    type Output = ();
+    type Output = BuildStamp;
     const DEFAULT: bool = true;
 
     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
@@ -70,7 +70,7 @@ impl Step for Std {
         });
     }
 
-    fn run(self, builder: &Builder<'_>) {
+    fn run(self, builder: &Builder<'_>) -> Self::Output {
         let build_compiler = self.build_compiler;
         let target = self.target;
 
@@ -101,14 +101,23 @@ impl Step for Std {
             target,
         );
 
-        let stamp = build_stamp::libstd_stamp(builder, build_compiler, target).with_prefix("check");
-        run_cargo(builder, cargo, builder.config.free_args.clone(), &stamp, vec![], true, false);
+        let check_stamp =
+            build_stamp::libstd_stamp(builder, build_compiler, target).with_prefix("check");
+        run_cargo(
+            builder,
+            cargo,
+            builder.config.free_args.clone(),
+            &check_stamp,
+            vec![],
+            true,
+            false,
+        );
 
         drop(_guard);
 
         // don't check test dependencies if we haven't built libtest
         if !self.crates.iter().any(|krate| krate == "test") {
-            return;
+            return check_stamp;
         }
 
         // Then run cargo again, once we've put the rmeta files for the library
@@ -145,6 +154,7 @@ impl Step for Std {
             target,
         );
         run_cargo(builder, cargo, builder.config.free_args.clone(), &stamp, vec![], true, false);
+        check_stamp
     }
 
     fn metadata(&self) -> Option<StepMetadata> {
@@ -156,12 +166,28 @@ impl Step for Std {
 /// Contains directories with .rmeta files generated by checking rustc for a specific
 /// target.
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
-struct RustcRmetaSysroot {
+struct RmetaSysroot {
     host_dir: PathBuf,
     target_dir: PathBuf,
 }
 
-impl RustcRmetaSysroot {
+impl RmetaSysroot {
+    /// Copy rmeta artifacts from the given `stamp` into a sysroot located at `directory`.
+    fn from_stamp(
+        builder: &Builder<'_>,
+        stamp: BuildStamp,
+        target: TargetSelection,
+        directory: &Path,
+    ) -> Self {
+        let host_dir = directory.join("host");
+        let target_dir = directory.join(target);
+        let _ = fs::remove_dir_all(directory);
+        t!(fs::create_dir_all(directory));
+        add_to_sysroot(builder, &target_dir, &host_dir, &stamp);
+
+        Self { host_dir, target_dir }
+    }
+
     /// Configure the given cargo invocation so that the compiled crate will be able to use
     /// rustc .rmeta artifacts that were previously generated.
     fn configure_cargo(&self, cargo: &mut Cargo) {
@@ -180,42 +206,90 @@ impl RustcRmetaSysroot {
 /// "pollute" it (that is especially problematic for the external stage0 rustc).
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
 struct PrepareRustcRmetaSysroot {
-    build_compiler: Compiler,
+    build_compiler: CompilerForCheck,
     target: TargetSelection,
 }
 
+impl PrepareRustcRmetaSysroot {
+    fn new(build_compiler: CompilerForCheck, target: TargetSelection) -> Self {
+        Self { build_compiler, target }
+    }
+}
+
 impl Step for PrepareRustcRmetaSysroot {
-    type Output = RustcRmetaSysroot;
+    type Output = RmetaSysroot;
 
     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
         run.never()
     }
 
     fn run(self, builder: &Builder<'_>) -> Self::Output {
         // Check rustc
-        let stamp =
-            builder.ensure(Rustc::from_build_compiler(self.build_compiler, self.target, vec![]));
+        let stamp = builder.ensure(Rustc::from_build_compiler(
+            self.build_compiler.clone(),
+            self.target,
+            vec![],
+        ));
+
+        let build_compiler = self.build_compiler.build_compiler();
+
+        // Copy the generated rmeta artifacts to a separate directory
+        let dir = builder
+            .out
+            .join(build_compiler.host)
+            .join(format!("stage{}-rustc-rmeta-artifacts", build_compiler.stage + 1));
+        RmetaSysroot::from_stamp(builder, stamp, self.target, &dir)
+    }
+}
+
+/// Checks std using the given `build_compiler` for the given `target`, and produces
+/// a sysroot in the build directory that stores the generated .rmeta files.
+///
+/// This step exists so that we can store the generated .rmeta artifacts into a separate
+/// directory, instead of copying them into the sysroot of `build_compiler`, which would
+/// "pollute" it (that is especially problematic for the external stage0 rustc).
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+struct PrepareStdRmetaSysroot {
+    build_compiler: Compiler,
+    target: TargetSelection,
+}
+
+impl PrepareStdRmetaSysroot {
+    fn new(build_compiler: Compiler, target: TargetSelection) -> Self {
+        Self { build_compiler, target }
+    }
+}
+
+impl Step for PrepareStdRmetaSysroot {
+    type Output = RmetaSysroot;
+
+    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+        run.never()
+    }
+
+    fn run(self, builder: &Builder<'_>) -> Self::Output {
+        // Check std
+        let stamp = builder.ensure(Std {
+            build_compiler: self.build_compiler,
+            target: self.target,
+            crates: vec![],
+        });
 
         // Copy the generated rmeta artifacts to a separate directory
         let dir = builder
             .out
             .join(self.build_compiler.host)
-            .join(format!("stage{}-rustc-check-artifacts", self.build_compiler.stage + 1));
-        let host_dir = dir.join("host");
-        let target_dir = dir.join(self.target);
-        let _ = fs::remove_dir_all(&dir);
-        t!(fs::create_dir_all(&dir));
-        add_to_sysroot(builder, &target_dir, &host_dir, &stamp);
+            .join(format!("stage{}-std-rmeta-artifacts", self.build_compiler.stage));
 
-        RustcRmetaSysroot { host_dir, target_dir }
+        RmetaSysroot::from_stamp(builder, stamp, self.target, &dir)
     }
 }
 
 /// Checks rustc using `build_compiler`.
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
 pub struct Rustc {
     /// Compiler that will check this rustc.
-    pub build_compiler: Compiler,
+    pub build_compiler: CompilerForCheck,
     pub target: TargetSelection,
     /// Whether to build only a subset of crates.
     ///
@@ -227,13 +301,12 @@ pub struct Rustc {
 
 impl Rustc {
     pub fn new(builder: &Builder<'_>, target: TargetSelection, crates: Vec<String>) -> Self {
-        let build_compiler =
-            prepare_compiler_for_check(builder, target, Mode::Rustc).build_compiler;
+        let build_compiler = prepare_compiler_for_check(builder, target, Mode::Rustc);
         Self::from_build_compiler(build_compiler, target, crates)
     }
 
     fn from_build_compiler(
-        build_compiler: Compiler,
+        build_compiler: CompilerForCheck,
         target: TargetSelection,
         crates: Vec<String>,
     ) -> Self {
@@ -263,7 +336,7 @@ impl Step for Rustc {
     ///
     /// If we check a stage 2 compiler, we will have to first build a stage 1 compiler to check it.
     fn run(self, builder: &Builder<'_>) -> Self::Output {
-        let build_compiler = self.build_compiler;
+        let build_compiler = self.build_compiler.build_compiler;
         let target = self.target;
 
         let mut cargo = builder::Cargo::new(
@@ -276,6 +349,7 @@ impl Step for Rustc {
         );
 
         rustc_cargo(builder, &mut cargo, target, &build_compiler, &self.crates);
+        self.build_compiler.configure_cargo(&mut cargo);
 
         // Explicitly pass -p for all compiler crates -- this will force cargo
         // to also check the tests/benches/examples for these crates, rather
@@ -288,7 +362,7 @@ impl Step for Rustc {
             Kind::Check,
             format_args!("compiler artifacts{}", crate_description(&self.crates)),
             Mode::Rustc,
-            self.build_compiler,
+            self.build_compiler.build_compiler(),
             target,
         );
 
@@ -301,7 +375,8 @@ impl Step for Rustc {
     }
 
     fn metadata(&self) -> Option<StepMetadata> {
-        let metadata = StepMetadata::check("rustc", self.target).built_by(self.build_compiler);
+        let metadata = StepMetadata::check("rustc", self.target)
+            .built_by(self.build_compiler.build_compiler());
         let metadata = if self.crates.is_empty() {
             metadata
         } else {
@@ -322,7 +397,8 @@ impl Step for Rustc {
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
 pub struct CompilerForCheck {
     build_compiler: Compiler,
-    rustc_rmeta_sysroot: Option<RustcRmetaSysroot>,
+    rustc_rmeta_sysroot: Option<RmetaSysroot>,
+    std_rmeta_sysroot: Option<RmetaSysroot>,
 }
 
 impl CompilerForCheck {
@@ -336,6 +412,30 @@ impl CompilerForCheck {
         if let Some(sysroot) = &self.rustc_rmeta_sysroot {
             sysroot.configure_cargo(cargo);
         }
+        if let Some(sysroot) = &self.std_rmeta_sysroot {
+            sysroot.configure_cargo(cargo);
+        }
+    }
+}
+
+/// Prepare the standard library for checking something (that requires stdlib) using
+/// `build_compiler`.
+fn prepare_std(
+    builder: &Builder<'_>,
+    build_compiler: Compiler,
+    target: TargetSelection,
+) -> Option<RmetaSysroot> {
+    // We need to build the host stdlib even if we only check, to compile build scripts and proc
+    // macros
+    builder.std(build_compiler, builder.host_target);
+
+    // If we're cross-compiling, we generate the rmeta files for the given target
+    // This check has to be here, because if we generate both .so and .rmeta files, rustc will fail,
+    // as it will have multiple candidates for linking.
+    if builder.host_target != target {
+        Some(builder.ensure(PrepareStdRmetaSysroot::new(build_compiler, target)))
+    } else {
+        None
     }
 }
 
@@ -347,9 +447,13 @@ pub fn prepare_compiler_for_check(
 ) -> CompilerForCheck {
     let host = builder.host_target;
 
-    let mut rmeta_sysroot = None;
+    let mut rustc_rmeta_sysroot = None;
+    let mut std_rmeta_sysroot = None;
     let build_compiler = match mode {
         Mode::ToolBootstrap => builder.compiler(0, host),
+        // We could also only check std here and use `prepare_std`, but `ToolTarget` is currently
+        // only used for running in-tree Clippy on bootstrap tools, so it does not seem worth it to
+        // optimize it. Therefore, here we build std for the target, instead of just checking it.
         Mode::ToolTarget => get_tool_target_compiler(builder, ToolTargetBuildMode::Build(target)),
         Mode::ToolStd => {
             if builder.config.compile_time_deps {
@@ -360,14 +464,7 @@ pub fn prepare_compiler_for_check(
             } else {
                 // These tools require the local standard library to be checked
                 let build_compiler = builder.compiler(builder.top_stage, host);
-
-                // We need to build the host stdlib to check the tool itself.
-                // We need to build the target stdlib so that the tool can link to it.
-                builder.std(build_compiler, host);
-                // We could only check this library in theory, but `check::Std` doesn't copy rmetas
-                // into `build_compiler`'s sysroot to avoid clashes with `.rlibs`, so we build it
-                // instead.
-                builder.std(build_compiler, target);
+                std_rmeta_sysroot = prepare_std(builder, build_compiler, target);
                 build_compiler
             }
         }
@@ -376,11 +473,14 @@ pub fn prepare_compiler_for_check(
             // return the build compiler that was used to check rustc.
             // We do not need to check examples/tests/etc. of Rustc for rustc_private, so we pass
             // an empty set of crates, which will avoid using `cargo -p`.
-            let check = Rustc::new(builder, target, vec![]);
-            let build_compiler = check.build_compiler;
-            builder.ensure(check);
-            rmeta_sysroot =
-                Some(builder.ensure(PrepareRustcRmetaSysroot { build_compiler, target }));
+            let compiler_for_rustc = prepare_compiler_for_check(builder, target, Mode::Rustc);
+            rustc_rmeta_sysroot = Some(
+                builder.ensure(PrepareRustcRmetaSysroot::new(compiler_for_rustc.clone(), target)),
+            );
+            let build_compiler = compiler_for_rustc.build_compiler();
+
+            // To check a rustc_private tool, we also need to check std that it will link to
+            std_rmeta_sysroot = prepare_std(builder, build_compiler, target);
             build_compiler
         }
         Mode::Rustc => {
@@ -394,15 +494,8 @@ pub fn prepare_compiler_for_check(
             let stage = if host == target { builder.top_stage - 1 } else { builder.top_stage };
             let build_compiler = builder.compiler(stage, host);
 
-            // Build host std for compiling build scripts
-            builder.std(build_compiler, build_compiler.host);
-
-            // Build target std so that the checked rustc can link to it during the check
-            // FIXME: maybe we can a way to only do a check of std here?
-            // But for that we would have to copy the stdlib rmetas to the sysroot of the build
-            // compiler, which conflicts with std rlibs, if we also build std.
-            builder.std(build_compiler, target);
-
+            // To check rustc, we need to check std that it will link to
+            std_rmeta_sysroot = prepare_std(builder, build_compiler, target);
             build_compiler
         }
         Mode::Std => {
@@ -412,7 +505,7 @@ pub fn prepare_compiler_for_check(
             builder.compiler(builder.top_stage, host)
         }
     };
-    CompilerForCheck { build_compiler, rustc_rmeta_sysroot: rmeta_sysroot }
+    CompilerForCheck { build_compiler, rustc_rmeta_sysroot, std_rmeta_sysroot }
 }
 
 /// Check the Cranelift codegen backend.