diff --git a/src/Cargo.lock b/src/Cargo.lock index 7620fe8ddb3c3..5e7909ff43552 100644 --- a/src/Cargo.lock +++ b/src/Cargo.lock @@ -2585,6 +2585,11 @@ dependencies = [ [[package]] name = "tidy" version = "0.1.0" +dependencies = [ + "serde 1.0.27 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.27 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)", +] [[package]] name = "time" diff --git a/src/bootstrap/test.rs b/src/bootstrap/test.rs index c0998c1e42c97..48490493525f9 100644 --- a/src/bootstrap/test.rs +++ b/src/bootstrap/test.rs @@ -528,6 +528,7 @@ impl Step for Tidy { println!("tidy check ({})", host); let mut cmd = builder.tool_cmd(Tool::Tidy); cmd.arg(build.src.join("src")); + cmd.arg(&build.initial_cargo); if !build.config.vendor { cmd.arg("--no-vendor"); } diff --git a/src/tools/tidy/Cargo.toml b/src/tools/tidy/Cargo.toml index 664aecfcbdb9d..f7b491823f838 100644 --- a/src/tools/tidy/Cargo.toml +++ b/src/tools/tidy/Cargo.toml @@ -2,3 +2,8 @@ name = "tidy" version = "0.1.0" authors = ["Alex Crichton "] + +[dependencies] +serde = "1.0.8" +serde_derive = "1.0.8" +serde_json = "1.0.2" diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs index 4dedf6bfe779b..f40c7a72a452e 100644 --- a/src/tools/tidy/src/deps.rs +++ b/src/tools/tidy/src/deps.rs @@ -10,9 +10,13 @@ //! Check license of third-party deps by inspecting src/vendor +use std::collections::{BTreeSet, HashSet}; use std::fs::File; use std::io::Read; use std::path::Path; +use std::process::Command; + +use serde_json; static LICENSES: &'static [&'static str] = &[ "MIT/Apache-2.0", @@ -24,52 +28,182 @@ static LICENSES: &'static [&'static str] = &[ "Unlicense/MIT", ]; -// These are exceptions to Rust's permissive licensing policy, and -// should be considered bugs. Exceptions are only allowed in Rust -// tooling. It is _crucial_ that no exception crates be dependencies -// of the Rust runtime (std / test). +/// These are exceptions to Rust's permissive licensing policy, and +/// should be considered bugs. Exceptions are only allowed in Rust +/// tooling. It is _crucial_ that no exception crates be dependencies +/// of the Rust runtime (std / test). static EXCEPTIONS: &'static [&'static str] = &[ - "mdbook", // MPL2, mdbook - "openssl", // BSD+advertising clause, cargo, mdbook - "pest", // MPL2, mdbook via handlebars - "thread-id", // Apache-2.0, mdbook - "toml-query", // MPL-2.0, mdbook - "is-match", // MPL-2.0, mdbook - "cssparser", // MPL-2.0, rustdoc - "smallvec", // MPL-2.0, rustdoc + "mdbook", // MPL2, mdbook + "openssl", // BSD+advertising clause, cargo, mdbook + "pest", // MPL2, mdbook via handlebars + "thread-id", // Apache-2.0, mdbook + "toml-query", // MPL-2.0, mdbook + "is-match", // MPL-2.0, mdbook + "cssparser", // MPL-2.0, rustdoc + "smallvec", // MPL-2.0, rustdoc "fuchsia-zircon-sys", // BSD-3-Clause, rustdoc, rustc, cargo - "fuchsia-zircon", // BSD-3-Clause, rustdoc, rustc, cargo (jobserver & tempdir) - "cssparser-macros", // MPL-2.0, rustdoc - "selectors", // MPL-2.0, rustdoc - "clippy_lints", // MPL-2.0 rls + "fuchsia-zircon", // BSD-3-Clause, rustdoc, rustc, cargo (jobserver & tempdir) + "cssparser-macros", // MPL-2.0, rustdoc + "selectors", // MPL-2.0, rustdoc + "clippy_lints", // MPL-2.0 rls +]; + +/// Which crates to check against the whitelist? +static WHITELIST_CRATES: &'static [CrateVersion] = &[ + CrateVersion("rustc", "0.0.0"), + CrateVersion("rustc_trans", "0.0.0"), ]; +/// Whitelist of crates rustc is allowed to depend on. Avoid adding to the list if possible. +static WHITELIST: &'static [Crate] = &[ + Crate("ar"), + Crate("backtrace"), + Crate("backtrace-sys"), + Crate("bitflags"), + Crate("byteorder"), + Crate("cc"), + Crate("cfg-if"), + Crate("cmake"), + Crate("ena"), + Crate("filetime"), + Crate("flate2"), + Crate("fuchsia-zircon"), + Crate("fuchsia-zircon-sys"), + Crate("jobserver"), + Crate("kernel32-sys"), + Crate("lazy_static"), + Crate("libc"), + Crate("log"), + Crate("log_settings"), + Crate("miniz-sys"), + Crate("num_cpus"), + Crate("owning_ref"), + Crate("parking_lot"), + Crate("parking_lot_core"), + Crate("rand"), + Crate("redox_syscall"), + Crate("rustc-demangle"), + Crate("smallvec"), + Crate("stable_deref_trait"), + Crate("tempdir"), + Crate("unicode-width"), + Crate("winapi"), + Crate("winapi-build"), + Crate("winapi-i686-pc-windows-gnu"), + Crate("winapi-x86_64-pc-windows-gnu"), +]; + +// Some types for Serde to deserialize the output of `cargo metadata` to... + +#[derive(Deserialize)] +struct Output { + resolve: Resolve, +} + +#[derive(Deserialize)] +struct Resolve { + nodes: Vec, +} + +#[derive(Deserialize)] +struct ResolveNode { + id: String, + dependencies: Vec, +} + +/// A unique identifier for a crate +#[derive(Copy, Clone, PartialOrd, Ord, PartialEq, Eq, Debug, Hash)] +struct Crate<'a>(&'a str); // (name,) + +#[derive(Copy, Clone, PartialOrd, Ord, PartialEq, Eq, Debug, Hash)] +struct CrateVersion<'a>(&'a str, &'a str); // (name, version) + +impl<'a> Crate<'a> { + pub fn id_str(&self) -> String { + format!("{} ", self.0) + } +} + +impl<'a> CrateVersion<'a> { + /// Returns the struct and whether or not the dep is in-tree + pub fn from_str(s: &'a str) -> (Self, bool) { + let mut parts = s.split(" "); + let name = parts.next().unwrap(); + let version = parts.next().unwrap(); + let path = parts.next().unwrap(); + + let is_path_dep = path.starts_with("(path+"); + + (CrateVersion(name, version), is_path_dep) + } + + pub fn id_str(&self) -> String { + format!("{} {}", self.0, self.1) + } +} + +impl<'a> From> for Crate<'a> { + fn from(cv: CrateVersion<'a>) -> Crate<'a> { + Crate(cv.0) + } +} + +/// Checks the dependency at the given path. Changes `bad` to `true` if a check failed. +/// +/// Specifically, this checks that the license is correct. pub fn check(path: &Path, bad: &mut bool) { + // Check licences let path = path.join("vendor"); assert!(path.exists(), "vendor directory missing"); let mut saw_dir = false; - 'next_path: for dir in t!(path.read_dir()) { + for dir in t!(path.read_dir()) { saw_dir = true; let dir = t!(dir); // skip our exceptions - for exception in EXCEPTIONS { - if dir.path() + if EXCEPTIONS.iter().any(|exception| { + dir.path() .to_str() .unwrap() - .contains(&format!("src/vendor/{}", exception)) { - continue 'next_path; - } + .contains(&format!("src/vendor/{}", exception)) + }) { + continue; } let toml = dir.path().join("Cargo.toml"); - if !check_license(&toml) { - *bad = true; - } + *bad = *bad || !check_license(&toml); } assert!(saw_dir, "no vendored source"); } +/// Checks the dependency of WHITELIST_CRATES at the given path. Changes `bad` to `true` if a check +/// failed. +/// +/// Specifically, this checks that the dependencies are on the WHITELIST. +pub fn check_whitelist(path: &Path, cargo: &Path, bad: &mut bool) { + // Get dependencies from cargo metadata + let resolve = get_deps(path, cargo); + + // Get the whitelist into a convenient form + let whitelist: HashSet<_> = WHITELIST.iter().cloned().collect(); + + // Check dependencies + let mut visited = BTreeSet::new(); + let mut unapproved = BTreeSet::new(); + for &krate in WHITELIST_CRATES.iter() { + let mut bad = check_crate_whitelist(&whitelist, &resolve, &mut visited, krate, false); + unapproved.append(&mut bad); + } + + if unapproved.len() > 0 { + println!("Dependencies not on the whitelist:"); + for dep in unapproved { + println!("* {}", dep.id_str()); + } + *bad = true; + } +} + fn check_license(path: &Path) -> bool { if !path.exists() { panic!("{} does not exist", path.display()); @@ -102,9 +236,71 @@ fn extract_license(line: &str) -> String { let first_quote = line.find('"'); let last_quote = line.rfind('"'); if let (Some(f), Some(l)) = (first_quote, last_quote) { - let license = &line[f + 1 .. l]; + let license = &line[f + 1..l]; license.into() } else { "bad-license-parse".into() } } + +/// Get the dependencies of the crate at the given path using `cargo metadata`. +fn get_deps(path: &Path, cargo: &Path) -> Resolve { + // Run `cargo metadata` to get the set of dependencies + let output = Command::new(cargo) + .arg("metadata") + .arg("--format-version") + .arg("1") + .arg("--manifest-path") + .arg(path.join("Cargo.toml")) + .output() + .expect("Unable to run `cargo metadata`") + .stdout; + let output = String::from_utf8_lossy(&output); + let output: Output = serde_json::from_str(&output).unwrap(); + + output.resolve +} + +/// Checks the dependencies of the given crate from the given cargo metadata to see if they are on +/// the whitelist. Returns a list of illegal dependencies. +fn check_crate_whitelist<'a, 'b>( + whitelist: &'a HashSet, + resolve: &'a Resolve, + visited: &'b mut BTreeSet>, + krate: CrateVersion<'a>, + must_be_on_whitelist: bool, +) -> BTreeSet> { + // Will contain bad deps + let mut unapproved = BTreeSet::new(); + + // Check if we have already visited this crate + if visited.contains(&krate) { + return unapproved; + } + + visited.insert(krate); + + // If this path is in-tree, we don't require it to be on the whitelist + if must_be_on_whitelist { + // If this dependency is not on the WHITELIST, add to bad set + if !whitelist.contains(&krate.into()) { + unapproved.insert(krate.into()); + } + } + + // Do a DFS in the crate graph (it's a DAG, so we know we have no cycles!) + let to_check = resolve + .nodes + .iter() + .find(|n| n.id.starts_with(&krate.id_str())) + .expect("crate does not exist"); + + for dep in to_check.dependencies.iter() { + let (krate, is_path_dep) = CrateVersion::from_str(dep); + + let mut bad = check_crate_whitelist(whitelist, resolve, visited, krate, !is_path_dep); + unapproved.append(&mut bad); + } + + unapproved +} diff --git a/src/tools/tidy/src/lib.rs b/src/tools/tidy/src/lib.rs index 5134c86991261..c927ff19b279b 100644 --- a/src/tools/tidy/src/lib.rs +++ b/src/tools/tidy/src/lib.rs @@ -15,6 +15,11 @@ #![deny(warnings)] +extern crate serde; +extern crate serde_json; +#[macro_use] +extern crate serde_derive; + use std::fs; use std::path::Path; diff --git a/src/tools/tidy/src/main.rs b/src/tools/tidy/src/main.rs index f6640c902bcb4..afa3ebd198319 100644 --- a/src/tools/tidy/src/main.rs +++ b/src/tools/tidy/src/main.rs @@ -24,9 +24,12 @@ use std::path::PathBuf; use std::env; fn main() { - let path = env::args_os().skip(1).next().expect("need an argument"); + let path = env::args_os().skip(1).next().expect("need path to src"); let path = PathBuf::from(path); + let cargo = env::args_os().skip(2).next().expect("need path to cargo"); + let cargo = PathBuf::from(cargo); + let args: Vec = env::args().skip(1).collect(); let mut bad = false; @@ -41,6 +44,7 @@ fn main() { if !args.iter().any(|s| *s == "--no-vendor") { deps::check(&path, &mut bad); } + deps::check_whitelist(&path, &cargo, &mut bad); if bad { eprintln!("some tidy checks failed");