Auto merge of #42682 - alexcrichton:jobserver, r=michaelwoerister

Integrate jobserver support to parallel codegen

This commit integrates the `jobserver` crate into the compiler. The crate was
previously integrated in to Cargo as part of rust-lang/cargo#4110. The purpose
here is to two-fold:

* Primarily the compiler can cooperate with Cargo on parallelism. When you run
  `cargo build -j4` then this'll make sure that the entire build process between
  Cargo/rustc won't use more than 4 cores, whereas today you'd get 4 rustc
  instances which may all try to spawn lots of threads.

* Secondarily rustc/Cargo can now integrate with a foreign GNU `make` jobserver.
  This means that if you call cargo/rustc from `make` or another
  jobserver-compatible implementation it'll use foreign parallelism settings
  instead of creating new ones locally.

As the number of parallel codegen instances in the compiler continues to grow
over time with the advent of incremental compilation it's expected that this'll
become more of a problem, so this is intended to nip concurrent concerns in the
bud by having all the tools to cooperate!

Note that while rustc has support for itself creating a jobserver it's far more
likely that rustc will always use the jobserver configured by Cargo. Cargo today
will now set a jobserver unconditionally for rustc to use.

Author: bors

src/Cargo.lock

@@ -9,6 +9,9 @@ path = "lib.rs"
 test = false
 bench = false
 
+[dev-dependencies]
+rand = { path = "../librand" }
+
 [[test]]
 name = "coretests"
 path = "../libcore/tests/lib.rs"

src/libcore/Cargo.toml

@@ -12,6 +12,7 @@ crate-type = ["dylib"]
 arena = { path = "../libarena" }
 fmt_macros = { path = "../libfmt_macros" }
 graphviz = { path = "../libgraphviz" }
+jobserver = "0.1"
 log = "0.3"
 owning_ref = "0.3.3"
 rustc_back = { path = "../librustc_back" }

src/librustc/Cargo.toml

@@ -61,6 +61,7 @@ extern crate rustc_errors as errors;
 #[macro_use] extern crate syntax;
 extern crate syntax_pos;
 #[macro_use] #[no_link] extern crate rustc_bitflags;
+extern crate jobserver;
 
 extern crate serialize as rustc_serialize; // used by deriving
 

src/librustc/lib.rs

@@ -38,14 +38,16 @@ use syntax_pos::{Span, MultiSpan};
 use rustc_back::{LinkerFlavor, PanicStrategy};
 use rustc_back::target::Target;
 use rustc_data_structures::flock;
+use jobserver::Client;
 
-use std::path::{Path, PathBuf};
 use std::cell::{self, Cell, RefCell};
 use std::collections::HashMap;
 use std::env;
+use std::fmt;
 use std::io::Write;
+use std::path::{Path, PathBuf};
 use std::rc::Rc;
-use std::fmt;
+use std::sync::{Once, ONCE_INIT};
 use std::time::Duration;
 
 mod code_stats;
@@ -134,6 +136,10 @@ pub struct Session {
     pub print_fuel_crate: Option<String>,
     /// Always set to zero and incremented so that we can print fuel expended by a crate.
     pub print_fuel: Cell<u64>,
+
+    /// Loaded up early on in the initialization of this `Session` to avoid
+    /// false positives about a job server in our environment.
+    pub jobserver_from_env: Option<Client>,
 }
 
 pub struct PerfStats {
@@ -697,6 +703,24 @@ pub fn build_session_(sopts: config::Options,
         print_fuel_crate: print_fuel_crate,
         print_fuel: print_fuel,
         out_of_fuel: Cell::new(false),
+
+        // Note that this is unsafe because it may misinterpret file descriptors
+        // on Unix as jobserver file descriptors. We hopefully execute this near
+        // the beginning of the process though to ensure we don't get false
+        // positives, or in other words we try to execute this before we open
+        // any file descriptors ourselves.
+        //
+        // Also note that we stick this in a global because there could be
+        // multiple `Session` instances in this process, and the jobserver is
+        // per-process.
+        jobserver_from_env: unsafe {
+            static mut GLOBAL_JOBSERVER: *mut Option<Client> = 0 as *mut _;
+            static INIT: Once = ONCE_INIT;
+            INIT.call_once(|| {
+                GLOBAL_JOBSERVER = Box::into_raw(Box::new(Client::from_env()));
+            });
+            (*GLOBAL_JOBSERVER).clone()
+        },
     };
 
     sess

src/librustc/session/mod.rs

@@ -10,7 +10,9 @@ crate-type = ["dylib"]
 test = false
 
 [dependencies]
+crossbeam = "0.2"
 flate2 = "0.2"
+jobserver = "0.1.5"
 log = "0.3"
 owning_ref = "0.3.3"
 rustc = { path = "../librustc" }

src/librustc_trans/Cargo.toml

@@ -329,34 +329,38 @@ pub fn filename_for_input(sess: &Session,
 }
 
 pub fn each_linked_rlib(sess: &Session,
-                        f: &mut FnMut(CrateNum, &Path)) {
+                        f: &mut FnMut(CrateNum, &Path)) -> Result<(), String> {
     let crates = sess.cstore.used_crates(LinkagePreference::RequireStatic).into_iter();
     let fmts = sess.dependency_formats.borrow();
     let fmts = fmts.get(&config::CrateTypeExecutable)
                    .or_else(|| fmts.get(&config::CrateTypeStaticlib))
                    .or_else(|| fmts.get(&config::CrateTypeCdylib))
                    .or_else(|| fmts.get(&config::CrateTypeProcMacro));
-    let fmts = fmts.unwrap_or_else(|| {
-        bug!("could not find formats for rlibs");
-    });
+    let fmts = match fmts {
+        Some(f) => f,
+        None => return Err(format!("could not find formats for rlibs"))
+    };
     for (cnum, path) in crates {
-        match fmts[cnum.as_usize() - 1] {
-            Linkage::NotLinked | Linkage::IncludedFromDylib => continue,
-            _ => {}
+        match fmts.get(cnum.as_usize() - 1) {
+            Some(&Linkage::NotLinked) |
+            Some(&Linkage::IncludedFromDylib) => continue,
+            Some(_) => {}
+            None => return Err(format!("could not find formats for rlibs"))
         }
         let name = sess.cstore.crate_name(cnum).clone();
         let path = match path {
             LibSource::Some(p) => p,
             LibSource::MetadataOnly => {
-                sess.fatal(&format!("could not find rlib for: `{}`, found rmeta (metadata) file",
-                                    name));
+                return Err(format!("could not find rlib for: `{}`, found rmeta (metadata) file",
+                                   name))
             }
             LibSource::None => {
-                sess.fatal(&format!("could not find rlib for: `{}`", name));
+                return Err(format!("could not find rlib for: `{}`", name))
             }
         };
         f(cnum, &path);
     }
+    Ok(())
 }
 
 fn out_filename(sess: &Session,
@@ -669,7 +673,7 @@ fn link_staticlib(sess: &Session, objects: &[PathBuf], out_filename: &Path,
     let mut ab = link_rlib(sess, None, objects, out_filename, tempdir);
     let mut all_native_libs = vec![];
 
-    each_linked_rlib(sess, &mut |cnum, path| {
+    let res = each_linked_rlib(sess, &mut |cnum, path| {
         let name = sess.cstore.crate_name(cnum);
         let native_libs = sess.cstore.native_libraries(cnum);
 
@@ -694,6 +698,9 @@ fn link_staticlib(sess: &Session, objects: &[PathBuf], out_filename: &Path,
 
         all_native_libs.extend(sess.cstore.native_libraries(cnum));
     });
+    if let Err(e) = res {
+        sess.fatal(&e);
+    }
 
     ab.update_symbols();
     ab.build();

src/librustc_trans/back/link.rs

@@ -10,15 +10,16 @@
 
 use back::link;
 use back::write;
-use back::symbol_export::{self, ExportedSymbols};
-use rustc::session::{self, config};
+use back::symbol_export;
+use rustc::session::config;
+use errors::FatalError;
 use llvm;
 use llvm::archive_ro::ArchiveRO;
 use llvm::{ModuleRef, TargetMachineRef, True, False};
 use rustc::util::common::time;
 use rustc::util::common::path2cstr;
 use rustc::hir::def_id::LOCAL_CRATE;
-use back::write::{ModuleConfig, with_llvm_pmb};
+use back::write::{ModuleConfig, with_llvm_pmb, CodegenContext};
 
 use libc;
 use flate2::read::ZlibDecoder;
@@ -39,30 +40,31 @@ pub fn crate_type_allows_lto(crate_type: config::CrateType) -> bool {
     }
 }
 
-pub fn run(sess: &session::Session,
+pub fn run(cgcx: &CodegenContext,
            llmod: ModuleRef,
            tm: TargetMachineRef,
-           exported_symbols: &ExportedSymbols,
            config: &ModuleConfig,
-           temp_no_opt_bc_filename: &Path) {
-    if sess.opts.cg.prefer_dynamic {
-        sess.struct_err("cannot prefer dynamic linking when performing LTO")
+           temp_no_opt_bc_filename: &Path) -> Result<(), FatalError> {
+    let handler = cgcx.handler;
+    if cgcx.opts.cg.prefer_dynamic {
+        handler.struct_err("cannot prefer dynamic linking when performing LTO")
             .note("only 'staticlib', 'bin', and 'cdylib' outputs are \
                    supported with LTO")
             .emit();
-        sess.abort_if_errors();
+        return Err(FatalError)
     }
 
     // Make sure we actually can run LTO
-    for crate_type in sess.crate_types.borrow().iter() {
+    for crate_type in cgcx.crate_types.iter() {
         if !crate_type_allows_lto(*crate_type) {
-            sess.fatal("lto can only be run for executables, cdylibs and \
-                            static library outputs");
+            let e = handler.fatal("lto can only be run for executables, cdylibs and \
+                                   static library outputs");
+            return Err(e)
         }
     }
 
     let export_threshold =
-        symbol_export::crates_export_threshold(&sess.crate_types.borrow());
+        symbol_export::crates_export_threshold(&cgcx.crate_types);
 
     let symbol_filter = &|&(ref name, level): &(String, _)| {
         if symbol_export::is_below_threshold(level, export_threshold) {
@@ -74,7 +76,7 @@ pub fn run(sess: &session::Session,
         }
     };
 
-    let mut symbol_white_list: Vec<CString> = exported_symbols
+    let mut symbol_white_list: Vec<CString> = cgcx.exported_symbols
         .exported_symbols(LOCAL_CRATE)
         .iter()
         .filter_map(symbol_filter)
@@ -83,16 +85,11 @@ pub fn run(sess: &session::Session,
     // For each of our upstream dependencies, find the corresponding rlib and
     // load the bitcode from the archive. Then merge it into the current LLVM
     // module that we've got.
-    link::each_linked_rlib(sess, &mut |cnum, path| {
-        // `#![no_builtins]` crates don't participate in LTO.
-        if sess.cstore.is_no_builtins(cnum) {
-            return;
-        }
-
+    for &(cnum, ref path) in cgcx.each_linked_rlib_for_lto.iter() {
         symbol_white_list.extend(
-            exported_symbols.exported_symbols(cnum)
-                            .iter()
-                            .filter_map(symbol_filter));
+            cgcx.exported_symbols.exported_symbols(cnum)
+                                 .iter()
+                                 .filter_map(symbol_filter));
 
         let archive = ArchiveRO::open(&path).expect("wanted an rlib");
         let bytecodes = archive.iter().filter_map(|child| {
@@ -102,7 +99,7 @@ pub fn run(sess: &session::Session,
             let bc_encoded = data.data();
 
             let bc_decoded = if is_versioned_bytecode_format(bc_encoded) {
-                time(sess.time_passes(), &format!("decode {}", name), || {
+                time(cgcx.time_passes, &format!("decode {}", name), || {
                     // Read the version
                     let version = extract_bytecode_format_version(bc_encoded);
 
@@ -117,44 +114,49 @@ pub fn run(sess: &session::Session,
                         let res = ZlibDecoder::new(compressed_data)
                             .read_to_end(&mut inflated);
                         if res.is_err() {
-                            sess.fatal(&format!("failed to decompress bc of `{}`",
-                                               name))
+                            let msg = format!("failed to decompress bc of `{}`",
+                                              name);
+                            Err(handler.fatal(&msg))
+                        } else {
+                            Ok(inflated)
                         }
-                        inflated
                     } else {
-                        sess.fatal(&format!("Unsupported bytecode format version {}",
-                                           version))
+                        Err(handler.fatal(&format!("Unsupported bytecode format version {}",
+                                                   version)))
                     }
-                })
+                })?
             } else {
-                time(sess.time_passes(), &format!("decode {}", name), || {
+                time(cgcx.time_passes, &format!("decode {}", name), || {
                     // the object must be in the old, pre-versioning format, so
                     // simply inflate everything and let LLVM decide if it can
                     // make sense of it
                     let mut inflated = Vec::new();
                     let res = ZlibDecoder::new(bc_encoded)
                         .read_to_end(&mut inflated);
                     if res.is_err() {
-                        sess.fatal(&format!("failed to decompress bc of `{}`",
-                                           name))
+                        let msg = format!("failed to decompress bc of `{}`",
+                                          name);
+                        Err(handler.fatal(&msg))
+                    } else {
+                        Ok(inflated)
                     }
-                    inflated
-                })
+                })?
             };
 
             let ptr = bc_decoded.as_ptr();
             debug!("linking {}", name);
-            time(sess.time_passes(), &format!("ll link {}", name), || unsafe {
-                if !llvm::LLVMRustLinkInExternalBitcode(llmod,
-                                                        ptr as *const libc::c_char,
-                                                        bc_decoded.len() as libc::size_t) {
-                    write::llvm_err(sess.diagnostic(),
-                                    format!("failed to load bc of `{}`",
-                                            name));
+            time(cgcx.time_passes, &format!("ll link {}", name), || unsafe {
+                if llvm::LLVMRustLinkInExternalBitcode(llmod,
+                                                       ptr as *const libc::c_char,
+                                                       bc_decoded.len() as libc::size_t) {
+                    Ok(())
+                } else {
+                    let msg = format!("failed to load bc of `{}`", name);
+                    Err(write::llvm_err(handler, msg))
                 }
-            });
+            })?;
         }
-    });
+    }
 
     // Internalize everything but the exported symbols of the current module
     let arr: Vec<*const libc::c_char> = symbol_white_list.iter()
@@ -167,13 +169,13 @@ pub fn run(sess: &session::Session,
                                          arr.len() as libc::size_t);
     }
 
-    if sess.no_landing_pads() {
+    if cgcx.no_landing_pads {
         unsafe {
             llvm::LLVMRustMarkAllFunctionsNounwind(llmod);
         }
     }
 
-    if sess.opts.cg.save_temps {
+    if cgcx.opts.cg.save_temps {
         let cstr = path2cstr(temp_no_opt_bc_filename);
         unsafe {
             llvm::LLVMWriteBitcodeToFile(llmod, cstr.as_ptr());
@@ -203,12 +205,13 @@ pub fn run(sess: &session::Session,
         assert!(!pass.is_null());
         llvm::LLVMRustAddPass(pm, pass);
 
-        time(sess.time_passes(), "LTO passes", ||
+        time(cgcx.time_passes, "LTO passes", ||
              llvm::LLVMRunPassManager(pm, llmod));
 
         llvm::LLVMDisposePassManager(pm);
     }
     debug!("lto done");
+    Ok(())
 }
 
 fn is_versioned_bytecode_format(bc: &[u8]) -> bool {

src/librustc_trans/back/lto.rs

@@ -40,6 +40,7 @@ use rustc::dep_graph::WorkProduct;
 use syntax_pos::symbol::Symbol;
 
 extern crate flate2;
+extern crate crossbeam;
 extern crate libc;
 extern crate owning_ref;
 #[macro_use] extern crate rustc;
@@ -52,6 +53,7 @@ extern crate rustc_const_math;
 #[macro_use]
 #[no_link]
 extern crate rustc_bitflags;
+extern crate jobserver;
 
 #[macro_use] extern crate log;
 #[macro_use] extern crate syntax;

src/librustc_trans/back/write.rs

@@ -10,7 +10,6 @@
 
 #![no_std]
 
-extern crate rand;
 extern crate serialize as rustc_serialize;
 
 #[derive(RustcEncodable)]  //~ ERROR this trait cannot be derived

src/librustc_trans/lib.rs

@@ -7,17 +7,16 @@
 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
-//
+
 // WONTFIX(#20184) Needs landing pads (not present in stage1) or the compiler hangs.
 // ignore-stage1
 // compile-flags: -C codegen-units=2
-// error-pattern: build without -C codegen-units for more exact errors
 // ignore-emscripten
 
 #![feature(asm)]
 
 fn main() {
     unsafe {
-        asm!("nowayisthisavalidinstruction");
+        asm!("nowayisthisavalidinstruction"); //~ ERROR instruction
     }
 }

src/test/compile-fail-fulldeps/derive-no-std-not-supported.rs

@@ -0,0 +1,11 @@
+// Copyright 2017 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+pub trait Foo {}

src/test/compile-fail/asm-src-loc-codegen-units.rs

@@ -0,0 +1,11 @@
+// Copyright 2017 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+pub fn foo() {}

src/test/compile-fail/auxiliary/issue-36881-aux.rs

@@ -0,0 +1,11 @@
+// Copyright 2017 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+pub fn foo() {}

src/test/compile-fail/auxiliary/lint_unused_extern_crate2.rs

@@ -0,0 +1,9 @@
+// Copyright 2017 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.

src/test/compile-fail/auxiliary/lint_unused_extern_crate3.rs

@@ -8,9 +8,9 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-#![feature(rand)]
+// aux-build:issue-36881-aux.rs
 
 fn main() {
-    extern crate rand;
-    use rand::Rng; //~ ERROR unresolved import
+    extern crate issue_36881_aux;
+    use issue_36881_aux::Foo; //~ ERROR unresolved import
 }

src/test/compile-fail/auxiliary/lint_unused_extern_crate4.rs

@@ -9,34 +9,34 @@
 // except according to those terms.
 
 // aux-build:lint_unused_extern_crate.rs
+// aux-build:lint_unused_extern_crate2.rs
+// aux-build:lint_unused_extern_crate3.rs
+// aux-build:lint_unused_extern_crate4.rs
 
 #![deny(unused_extern_crates)]
 #![allow(unused_variables)]
 #![allow(deprecated)]
-#![feature(alloc)]
-#![feature(libc)]
-#![feature(rand)]
 
-extern crate libc; //~ ERROR: unused extern crate
+extern crate lint_unused_extern_crate4; //~ ERROR: unused extern crate
 
-extern crate alloc as collecs; // no error, it is used
+extern crate lint_unused_extern_crate3; // no error, it is used
 
-extern crate rand; // no error, the use marks it as used
-                   // even if imported objects aren't used
+extern crate lint_unused_extern_crate2; // no error, the use marks it as used
+                                        // even if imported objects aren't used
 
 extern crate lint_unused_extern_crate as other; // no error, the use * marks it as used
 
 #[allow(unused_imports)]
-use rand::isaac::IsaacRng;
+use lint_unused_extern_crate2::foo as bar;
 
 use other::*;
 
 mod foo {
-    // Test that this is unused even though an earler `extern crate rand` is used.
-    extern crate rand; //~ ERROR unused extern crate
+    // Test that this is unused even though an earler `extern crate` is used.
+    extern crate lint_unused_extern_crate2; //~ ERROR unused extern crate
 }
 
 fn main() {
-    let x: collecs::vec::Vec<usize> = Vec::new();
+    lint_unused_extern_crate3::foo();
     let y = foo();
 }

src/test/compile-fail/issue-36881.rs

@@ -10,11 +10,13 @@
 
 // no-prefer-dynamic
 
-#![feature(allocator, core_intrinsics)]
+#![feature(allocator, core_intrinsics, panic_unwind)]
 #![allocator]
 #![crate_type = "rlib"]
 #![no_std]
 
+extern crate unwind;
+
 pub static mut HITS: usize = 0;
 
 type size_t = usize;