Auto merge of #42109 - Keruspe:master, r=alexcrichton

rustbuild: don't create a source tarball when installing

This splits Install out of Dist as it is not a full dist anymore, and creates the source tarball only for the Dist command.
This will allow splitting install in a few rules if we want as it's done for other phases.

Author: bors

README.md

@@ -35,15 +35,15 @@ Read ["Installing Rust"] from [The Book].
 3. Build and install:
 
     ```sh
-    $ ./x.py build && sudo ./x.py dist --install
+    $ ./x.py build && sudo ./x.py install
     ```
 
     > ***Note:*** Install locations can be adjusted by copying the config file
     > from `./src/bootstrap/config.toml.example` to `./config.toml`, and
     > adjusting the `prefix` option under `[install]`. Various other options are
     > also supported, and are documented in the config file.
 
-    When complete, `sudo ./x.py dist --install` will place several programs into
+    When complete, `sudo ./x.py install` will place several programs into
     `/usr/local/bin`: `rustc`, the Rust compiler, and `rustdoc`, the
     API-documentation tool. This install does not include [Cargo],
     Rust's package manager, which you may also want to build.
@@ -96,7 +96,7 @@ build.
 4. Navigate to Rust's source code (or clone it), then build it:
 
    ```sh
-   $ ./x.py build && ./x.py dist --install
+   $ ./x.py build && ./x.py install
    ```
 
 #### MSVC

src/bootstrap/config.toml.example

@@ -314,3 +314,9 @@
 # Note that this address should not contain a trailing slash as file names will
 # be appended to it.
 #upload-addr = "https://example.com/folder"
+
+# Whether to build a plain source tarball to upload
+# We disable that on Windows not to override the one already uploaded on S3
+# as the one built on Windows will contain backslashes in paths causing problems
+# on linux
+#src-tarball = true

src/bootstrap/dist.rs

@@ -30,11 +30,11 @@ use {Build, Compiler, Mode};
 use channel;
 use util::{cp_r, libdir, is_dylib, cp_filtered, copy, exe};
 
-fn pkgname(build: &Build, component: &str) -> String {
+pub fn pkgname(build: &Build, component: &str) -> String {
     if component == "cargo" {
         format!("{}-{}", component, build.cargo_package_vers())
     } else if component == "rls" {
-        format!("{}-{}", component, build.package_vers(&build.release_num("rls")))
+        format!("{}-{}", component, build.rls_package_vers())
     } else {
         assert!(component.starts_with("rust"));
         format!("{}-{}", component, build.rust_package_vers())
@@ -489,38 +489,7 @@ pub fn analysis(build: &Build, compiler: &Compiler, target: &str) {
     t!(fs::remove_dir_all(&image));
 }
 
-const CARGO_VENDOR_VERSION: &'static str = "0.1.4";
-
-/// Creates the `rust-src` installer component and the plain source tarball
-pub fn rust_src(build: &Build) {
-    if !build.config.rust_dist_src {
-        return
-    }
-
-    println!("Dist src");
-
-    // Make sure that the root folder of tarball has the correct name
-    let plain_name = format!("rustc-{}-src", build.rust_package_vers());
-    let plain_dst_src = tmpdir(build).join(&plain_name);
-    let _ = fs::remove_dir_all(&plain_dst_src);
-    t!(fs::create_dir_all(&plain_dst_src));
-
-    // This is the set of root paths which will become part of the source package
-    let src_files = [
-        "COPYRIGHT",
-        "LICENSE-APACHE",
-        "LICENSE-MIT",
-        "CONTRIBUTING.md",
-        "README.md",
-        "RELEASES.md",
-        "configure",
-        "x.py",
-    ];
-    let src_dirs = [
-        "man",
-        "src",
-    ];
-
+fn copy_src_dirs(build: &Build, src_dirs: &[&str], dst_dir: &Path) {
     let filter_fn = move |path: &Path| {
         let spath = match path.to_str() {
             Some(path) => path,
@@ -549,60 +518,16 @@ pub fn rust_src(build: &Build) {
     };
 
     // Copy the directories using our filter
-    for item in &src_dirs {
-        let dst = &plain_dst_src.join(item);
-        t!(fs::create_dir(dst));
+    for item in src_dirs {
+        let dst = &dst_dir.join(item);
+        t!(fs::create_dir_all(dst));
         cp_filtered(&build.src.join(item), dst, &filter_fn);
     }
-    // Copy the files normally
-    for item in &src_files {
-        copy(&build.src.join(item), &plain_dst_src.join(item));
-    }
-
-    // If we're building from git sources, we need to vendor a complete distribution.
-    if build.src_is_git {
-        // Get cargo-vendor installed, if it isn't already.
-        let mut has_cargo_vendor = false;
-        let mut cmd = Command::new(&build.cargo);
-        for line in output(cmd.arg("install").arg("--list")).lines() {
-            has_cargo_vendor |= line.starts_with("cargo-vendor ");
-        }
-        if !has_cargo_vendor {
-            let mut cmd = Command::new(&build.cargo);
-            cmd.arg("install")
-               .arg("--force")
-               .arg("--debug")
-               .arg("--vers").arg(CARGO_VENDOR_VERSION)
-               .arg("cargo-vendor")
-               .env("RUSTC", &build.rustc);
-            build.run(&mut cmd);
-        }
-
-        // Vendor all Cargo dependencies
-        let mut cmd = Command::new(&build.cargo);
-        cmd.arg("vendor")
-           .current_dir(&plain_dst_src.join("src"));
-        build.run(&mut cmd);
-    }
-
-    // Create the version file
-    write_file(&plain_dst_src.join("version"), build.rust_version().as_bytes());
-
-    // Create plain source tarball
-    let mut tarball = rust_src_location(build);
-    tarball.set_extension(""); // strip .gz
-    tarball.set_extension(""); // strip .tar
-    if let Some(dir) = tarball.parent() {
-        t!(fs::create_dir_all(dir));
-    }
-    let mut cmd = rust_installer(build);
-    cmd.arg("tarball")
-       .arg("--input").arg(&plain_name)
-       .arg("--output").arg(&tarball)
-       .arg("--work-dir=.")
-       .current_dir(tmpdir(build));
-    build.run(&mut cmd);
+}
 
+/// Creates the `rust-src` installer component
+pub fn rust_src(build: &Build) {
+    println!("Dist src");
 
     let name = pkgname(build, "rust-src");
     let image = tmpdir(build).join(format!("{}-image", name));
@@ -636,11 +561,7 @@ pub fn rust_src(build: &Build) {
         "src/rustc/libc_shim",
     ];
 
-    for item in &std_src_dirs {
-        let dst = &dst_src.join(item);
-        t!(fs::create_dir_all(dst));
-        cp_r(&plain_dst_src.join(item), dst);
-    }
+    copy_src_dirs(build, &std_src_dirs[..], &dst_src);
 
     // Create source tarball in rust-installer format
     let mut cmd = rust_installer(build);
@@ -657,7 +578,86 @@ pub fn rust_src(build: &Build) {
     build.run(&mut cmd);
 
     t!(fs::remove_dir_all(&image));
-    t!(fs::remove_dir_all(&plain_dst_src));
+}
+
+const CARGO_VENDOR_VERSION: &'static str = "0.1.4";
+
+/// Creates the plain source tarball
+pub fn plain_source_tarball(build: &Build) {
+    println!("Create plain source tarball");
+
+    // Make sure that the root folder of tarball has the correct name
+    let plain_name = format!("{}-src", pkgname(build, "rustc"));
+    let plain_dst_src = tmpdir(build).join(&plain_name);
+    let _ = fs::remove_dir_all(&plain_dst_src);
+    t!(fs::create_dir_all(&plain_dst_src));
+
+    // This is the set of root paths which will become part of the source package
+    let src_files = [
+        "COPYRIGHT",
+        "LICENSE-APACHE",
+        "LICENSE-MIT",
+        "CONTRIBUTING.md",
+        "README.md",
+        "RELEASES.md",
+        "configure",
+        "x.py",
+    ];
+    let src_dirs = [
+        "man",
+        "src",
+    ];
+
+    copy_src_dirs(build, &src_dirs[..], &plain_dst_src);
+
+    // Copy the files normally
+    for item in &src_files {
+        copy(&build.src.join(item), &plain_dst_src.join(item));
+    }
+
+    // Create the version file
+    write_file(&plain_dst_src.join("version"), build.rust_version().as_bytes());
+
+    // If we're building from git sources, we need to vendor a complete distribution.
+    if build.src_is_git {
+        // Get cargo-vendor installed, if it isn't already.
+        let mut has_cargo_vendor = false;
+        let mut cmd = Command::new(&build.cargo);
+        for line in output(cmd.arg("install").arg("--list")).lines() {
+            has_cargo_vendor |= line.starts_with("cargo-vendor ");
+        }
+        if !has_cargo_vendor {
+            let mut cmd = Command::new(&build.cargo);
+            cmd.arg("install")
+               .arg("--force")
+               .arg("--debug")
+               .arg("--vers").arg(CARGO_VENDOR_VERSION)
+               .arg("cargo-vendor")
+               .env("RUSTC", &build.rustc);
+            build.run(&mut cmd);
+        }
+
+        // Vendor all Cargo dependencies
+        let mut cmd = Command::new(&build.cargo);
+        cmd.arg("vendor")
+           .current_dir(&plain_dst_src.join("src"));
+        build.run(&mut cmd);
+    }
+
+    // Create plain source tarball
+    let mut tarball = rust_src_location(build);
+    tarball.set_extension(""); // strip .gz
+    tarball.set_extension(""); // strip .tar
+    if let Some(dir) = tarball.parent() {
+        t!(fs::create_dir_all(dir));
+    }
+    let mut cmd = rust_installer(build);
+    cmd.arg("tarball")
+       .arg("--input").arg(&plain_name)
+       .arg("--output").arg(&tarball)
+       .arg("--work-dir=.")
+       .current_dir(tmpdir(build));
+    build.run(&mut cmd);
 }
 
 fn install(src: &Path, dstdir: &Path, perms: u32) {

src/bootstrap/flags.rs

@@ -69,7 +69,9 @@ pub enum Subcommand {
     Clean,
     Dist {
         paths: Vec<PathBuf>,
-        install: bool,
+    },
+    Install {
+        paths: Vec<PathBuf>,
     },
 }
 
@@ -85,7 +87,8 @@ Subcommands:
     bench       Build and run some benchmarks
     doc         Build documentation
     clean       Clean out build directories
-    dist        Build and/or install distribution artifacts
+    dist        Build distribution artifacts
+    install     Install distribution artifacts
 
 To learn more about a subcommand, run `./x.py <subcommand> -h`");
 
@@ -125,7 +128,8 @@ To learn more about a subcommand, run `./x.py <subcommand> -h`");
                                         || (s == "bench")
                                         || (s == "doc")
                                         || (s == "clean")
-                                        || (s == "dist"));
+                                        || (s == "dist")
+                                        || (s == "install"));
         let subcommand = match possible_subcommands.first() {
             Some(s) => s,
             None => {
@@ -139,7 +143,6 @@ To learn more about a subcommand, run `./x.py <subcommand> -h`");
         match subcommand.as_str() {
             "test"  => { opts.optmulti("", "test-args", "extra arguments", "ARGS"); },
             "bench" => { opts.optmulti("", "test-args", "extra arguments", "ARGS"); },
-            "dist"  => { opts.optflag("", "install", "run installer as well"); },
             _ => { },
         };
 
@@ -281,7 +284,11 @@ Arguments:
             "dist" => {
                 Subcommand::Dist {
                     paths: paths,
-                    install: matches.opt_present("install"),
+                }
+            }
+            "install" => {
+                Subcommand::Install {
+                    paths: paths,
                 }
             }
             _ => {

src/bootstrap/install.rs

@@ -19,7 +19,7 @@ use std::path::{Path, PathBuf, Component};
 use std::process::Command;
 
 use Build;
-use dist::{sanitize_sh, tmpdir};
+use dist::{pkgname, sanitize_sh, tmpdir};
 
 pub struct Installer<'a> {
     build: &'a Build,
@@ -29,6 +29,13 @@ pub struct Installer<'a> {
     bindir: PathBuf,
     libdir: PathBuf,
     mandir: PathBuf,
+    empty_dir: PathBuf,
+}
+
+impl<'a> Drop for Installer<'a> {
+    fn drop(&mut self) {
+        t!(fs::remove_dir_all(&self.empty_dir));
+    }
 }
 
 impl<'a> Installer<'a> {
@@ -61,6 +68,10 @@ impl<'a> Installer<'a> {
         let libdir = add_destdir(&libdir, &destdir);
         let mandir = add_destdir(&mandir, &destdir);
 
+        let empty_dir = build.out.join("tmp/empty_dir");
+
+        t!(fs::create_dir_all(&empty_dir));
+
         Installer {
             build,
             prefix,
@@ -69,52 +80,49 @@ impl<'a> Installer<'a> {
             bindir,
             libdir,
             mandir,
+            empty_dir,
         }
     }
 
-    /// Installs everything.
-    pub fn install(&self, stage: u32, host: &str) {
-        let empty_dir = self.build.out.join("tmp/empty_dir");
-        t!(fs::create_dir_all(&empty_dir));
-
-        if self.build.config.docs {
-            self.install_sh("docs", "rust-docs", &self.build.rust_package_vers(),
-                            stage, Some(host), &empty_dir);
-        }
+    pub fn install_docs(&self, stage: u32, host: &str) {
+        self.install_sh("docs", "rust-docs", stage, Some(host));
+    }
 
+    pub fn install_std(&self, stage: u32) {
         for target in self.build.config.target.iter() {
-            self.install_sh("std", "rust-std", &self.build.rust_package_vers(),
-                            stage, Some(target), &empty_dir);
+            self.install_sh("std", "rust-std", stage, Some(target));
         }
+    }
 
-        if self.build.config.extended {
-            self.install_sh("cargo", "cargo", &self.build.cargo_package_vers(),
-                            stage, Some(host), &empty_dir);
-            self.install_sh("rls", "rls", &self.build.rls_package_vers(),
-                            stage, Some(host), &empty_dir);
-            self.install_sh("analysis", "rust-analysis", &self.build.rust_package_vers(),
-                            stage, Some(host), &empty_dir);
-            self.install_sh("src", "rust-src", &self.build.rust_package_vers(),
-                            stage, None, &empty_dir);
-        }
+    pub fn install_cargo(&self, stage: u32, host: &str) {
+        self.install_sh("cargo", "cargo", stage, Some(host));
+    }
 
-        self.install_sh("rustc", "rustc", &self.build.rust_package_vers(),
-                        stage, Some(host), &empty_dir);
+    pub fn install_rls(&self, stage: u32, host: &str) {
+        self.install_sh("rls", "rls", stage, Some(host));
+    }
+
+    pub fn install_analysis(&self, stage: u32, host: &str) {
+        self.install_sh("analysis", "rust-analysis", stage, Some(host));
+    }
 
-        t!(fs::remove_dir_all(&empty_dir));
+    pub fn install_src(&self, stage: u32) {
+        self.install_sh("src", "rust-src", stage, None);
+    }
+    pub fn install_rustc(&self, stage: u32, host: &str) {
+        self.install_sh("rustc", "rustc", stage, Some(host));
     }
 
-    fn install_sh(&self, package: &str, name: &str, version: &str,
-                  stage: u32, host: Option<&str>,  empty_dir: &Path) {
+    fn install_sh(&self, package: &str, name: &str, stage: u32, host: Option<&str>) {
         println!("Install {} stage{} ({:?})", package, stage, host);
         let package_name = if let Some(host) = host {
-            format!("{}-{}-{}", name, version, host)
+            format!("{}-{}", pkgname(self.build, name), host)
         } else {
-            format!("{}-{}", name, version)
+            pkgname(self.build, name)
         };
 
         let mut cmd = Command::new("sh");
-        cmd.current_dir(empty_dir)
+        cmd.current_dir(&self.empty_dir)
            .arg(sanitize_sh(&tmpdir(self.build).join(&package_name).join("install.sh")))
            .arg(format!("--prefix={}", sanitize_sh(&self.prefix)))
            .arg(format!("--sysconfdir={}", sanitize_sh(&self.sysconfdir)))

src/bootstrap/mk/Makefile.in

@@ -69,7 +69,7 @@ distcheck:
 	$(Q)$(BOOTSTRAP) dist $(BOOTSTRAP_ARGS)
 	$(Q)$(BOOTSTRAP) test distcheck $(BOOTSTRAP_ARGS)
 install:
-	$(Q)$(BOOTSTRAP) dist --install $(BOOTSTRAP_ARGS)
+	$(Q)$(BOOTSTRAP) install $(BOOTSTRAP_ARGS)
 tidy:
 	$(Q)$(BOOTSTRAP) test src/tools/tidy $(BOOTSTRAP_ARGS)
 prepare:

src/bootstrap/step.rs

@@ -492,6 +492,7 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules {
          .host(true)
          .run(move |s| check::docs(build, &s.compiler()));
     rules.test("check-distcheck", "distcheck")
+         .dep(|s| s.name("dist-plain-source-tarball"))
          .dep(|s| s.name("dist-src"))
          .run(move |_| check::distcheck(build));
 
@@ -734,6 +735,13 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules {
                  dist::mingw(build, s.target)
              }
          });
+    rules.dist("dist-plain-source-tarball", "src")
+         .default(build.config.rust_dist_src)
+         .host(true)
+         .only_build(true)
+         .only_host_build(true)
+         .dep(move |s| tool_rust_installer(build, s))
+         .run(move |_| dist::plain_source_tarball(build));
     rules.dist("dist-src", "src")
          .default(true)
          .host(true)
@@ -759,9 +767,6 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules {
          .dep(|s| s.name("tool-rls"))
          .dep(move |s| tool_rust_installer(build, s))
          .run(move |s| dist::rls(build, s.stage, s.target));
-    rules.dist("install", "path/to/nowhere")
-         .dep(|s| s.name("default:dist"))
-         .run(move |s| install::Installer::new(build).install(s.stage, s.target));
     rules.dist("dist-cargo", "cargo")
          .host(true)
          .only_host_build(true)
@@ -789,6 +794,47 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules {
          .dep(move |s| s.name("tool-build-manifest").target(&build.config.build).stage(0))
          .run(move |_| dist::hash_and_sign(build));
 
+    rules.install("install-docs", "src/doc")
+         .default(build.config.docs)
+         .only_host_build(true)
+         .dep(|s| s.name("dist-docs"))
+         .run(move |s| install::Installer::new(build).install_docs(s.stage, s.target));
+    rules.install("install-std", "src/libstd")
+         .default(true)
+         .only_host_build(true)
+         .dep(|s| s.name("dist-std"))
+         .run(move |s| install::Installer::new(build).install_std(s.stage));
+    rules.install("install-cargo", "cargo")
+         .default(build.config.extended)
+         .host(true)
+         .only_host_build(true)
+         .dep(|s| s.name("dist-cargo"))
+         .run(move |s| install::Installer::new(build).install_cargo(s.stage, s.target));
+    rules.install("install-rls", "rls")
+         .default(build.config.extended)
+         .host(true)
+         .only_host_build(true)
+         .dep(|s| s.name("dist-rls"))
+         .run(move |s| install::Installer::new(build).install_rls(s.stage, s.target));
+    rules.install("install-analysis", "analysis")
+         .default(build.config.extended)
+         .only_host_build(true)
+         .dep(|s| s.name("dist-analysis"))
+         .run(move |s| install::Installer::new(build).install_analysis(s.stage, s.target));
+    rules.install("install-src", "src")
+         .default(build.config.extended)
+         .host(true)
+         .only_build(true)
+         .only_host_build(true)
+         .dep(|s| s.name("dist-src"))
+         .run(move |s| install::Installer::new(build).install_src(s.stage));
+    rules.install("install-rustc", "src/librustc")
+         .default(true)
+         .host(true)
+         .only_host_build(true)
+         .dep(|s| s.name("dist-rustc"))
+         .run(move |s| install::Installer::new(build).install_rustc(s.stage, s.target));
+
     rules.verify();
     return rules;
 
@@ -902,6 +948,7 @@ enum Kind {
     Bench,
     Dist,
     Doc,
+    Install,
 }
 
 impl<'a> Rule<'a> {
@@ -1033,6 +1080,12 @@ impl<'a> Rules<'a> {
         self.rule(name, path, Kind::Dist)
     }
 
+    /// Same as `build`, but for `Kind::Install`.
+    fn install<'b>(&'b mut self, name: &'a str, path: &'a str)
+                -> RuleBuilder<'a, 'b> {
+        self.rule(name, path, Kind::Install)
+    }
+
     fn rule<'b>(&'b mut self,
                 name: &'a str,
                 path: &'a str,
@@ -1073,6 +1126,7 @@ invalid rule dependency graph detected, was a rule added and maybe typo'd?
             "test" => Kind::Test,
             "bench" => Kind::Bench,
             "dist" => Kind::Dist,
+            "install" => Kind::Install,
             _ => return None,
         };
         let rules = self.rules.values().filter(|r| r.kind == kind);
@@ -1122,13 +1176,8 @@ invalid rule dependency graph detected, was a rule added and maybe typo'd?
             Subcommand::Doc { ref paths } => (Kind::Doc, &paths[..]),
             Subcommand::Test { ref paths, test_args: _ } => (Kind::Test, &paths[..]),
             Subcommand::Bench { ref paths, test_args: _ } => (Kind::Bench, &paths[..]),
-            Subcommand::Dist { ref paths, install } => {
-                if install {
-                    return vec![self.sbuild.name("install")]
-                } else {
-                    (Kind::Dist, &paths[..])
-                }
-            }
+            Subcommand::Dist { ref paths } => (Kind::Dist, &paths[..]),
+            Subcommand::Install { ref paths } => (Kind::Install, &paths[..]),
             Subcommand::Clean => panic!(),
         };
 
@@ -1347,10 +1396,6 @@ mod tests {
     use config::Config;
     use flags::Flags;
 
-    macro_rules! a {
-        ($($a:expr),*) => (vec![$($a.to_string()),*])
-    }
-
     fn build(args: &[&str],
              extra_host: &[&str],
              extra_target: &[&str]) -> Build {