diff --git a/src/bin/cargo/commands/mod.rs b/src/bin/cargo/commands/mod.rs index 860c78de674..16e02774db3 100644 --- a/src/bin/cargo/commands/mod.rs +++ b/src/bin/cargo/commands/mod.rs @@ -30,6 +30,7 @@ pub fn builtin() -> Vec { test::cli(), uninstall::cli(), update::cli(), + vendor::cli(), verify_project::cli(), version::cli(), yank::cli(), @@ -66,6 +67,7 @@ pub fn builtin_exec(cmd: &str) -> Option) -> Cli "test" => test::exec, "uninstall" => uninstall::exec, "update" => update::exec, + "vendor" => vendor::exec, "verify-project" => verify_project::exec, "version" => version::exec, "yank" => yank::exec, @@ -102,6 +104,7 @@ pub mod search; pub mod test; pub mod uninstall; pub mod update; +pub mod vendor; pub mod verify_project; pub mod version; pub mod yank; diff --git a/src/bin/cargo/commands/vendor.rs b/src/bin/cargo/commands/vendor.rs new file mode 100644 index 00000000000..509a5c2b2b1 --- /dev/null +++ b/src/bin/cargo/commands/vendor.rs @@ -0,0 +1,123 @@ +use crate::command_prelude::*; +use cargo::ops; +use std::path::PathBuf; + +pub fn cli() -> App { + subcommand("vendor") + .about("Vendor all dependencies for a project locally") + .arg(opt("quiet", "No output printed to stdout").short("q")) + .arg_manifest_path() + .arg(Arg::with_name("path").help("Where to vendor crates (`vendor` by default)")) + .arg( + Arg::with_name("no-delete") + .long("no-delete") + .help("Don't delete older crates in the vendor directory"), + ) + .arg( + Arg::with_name("tomls") + .short("s") + .long("sync") + .help("Additional `Cargo.toml` to sync and vendor") + .value_name("TOML") + .multiple(true), + ) + .arg( + Arg::with_name("respect-source-config") + .long("respect-source-config") + .help("Respect `[source]` config in `.cargo/config`") + .multiple(true), + ) + .arg( + Arg::with_name("no-merge-sources") + .long("no-merge-sources") + .hidden(true), + ) + .arg( + Arg::with_name("relative-path") + .long("relative-path") + .hidden(true), + ) + .arg( + Arg::with_name("only-git-deps") + .long("only-git-deps") + .hidden(true), + ) + .arg( + Arg::with_name("explicit-version") + .short("-x") + .long("explicit-version") + .hidden(true), + ) + .arg( + Arg::with_name("disallow-duplicates") + .long("disallow-duplicates") + .hidden(true), + ) + .after_help( + "\ +This cargo subcommand will vendor all crates.io and git dependencies for a +project into the specified directory at ``. After this command completes +the vendor directory specified by `` will contain all remote sources from +dependencies specified. Additional manifests beyond the default one can be +specified with the `-s` option. + +The `cargo vendor` command will also print out the configuration necessary +to use the vendored sources, which you will need to add to `.cargo/config`. +", + ) +} + +pub fn exec(config: &mut Config, args: &ArgMatches<'_>) -> CliResult { + // We're doing the vendoring operation outselves, so we don't actually want + // to respect any of the `source` configuration in Cargo itself. That's + // intended for other consumers of Cargo, but we want to go straight to the + // source, e.g. crates.io, to fetch crates. + if !args.is_present("respect-source-config") { + config.values_mut()?.remove("source"); + } + + // When we moved `cargo vendor` into Cargo itself we didn't stabilize a few + // flags, so try to provide a helpful error message in that case to enusre + // that users currently using the flag aren't tripped up. + let crates_io_cargo_vendor_flag = if args.is_present("no-merge-sources") { + Some("--no-merge-sources") + } else if args.is_present("relative-path") { + Some("--relative-path") + } else if args.is_present("only-git-deps") { + Some("--only-git-deps") + } else if args.is_present("explicit-version") { + Some("--explicit-version") + } else if args.is_present("disallow-duplicates") { + Some("--disallow-duplicates") + } else { + None + }; + if let Some(flag) = crates_io_cargo_vendor_flag { + return Err(failure::format_err!("\ +the crates.io `cargo vendor` command has now been merged into Cargo itself +and does not support the flag `{}` currently; to continue using the flag you +can execute `cargo-vendor vendor ...`, and if you would like to see this flag +supported in Cargo itself please feel free to file an issue at +https://github.com/rust-lang/cargo/issues/new +", flag).into()); + } + + let ws = args.workspace(config)?; + let path = args + .value_of_os("path") + .map(|val| PathBuf::from(val.to_os_string())) + .unwrap_or(PathBuf::from("vendor")); + ops::vendor( + &ws, + &ops::VendorOptions { + no_delete: args.is_present("no-delete"), + destination: &path, + extra: args + .values_of_os("tomls") + .unwrap_or_default() + .map(|s| PathBuf::from(s.to_os_string())) + .collect(), + }, + )?; + Ok(()) +} diff --git a/src/cargo/lib.rs b/src/cargo/lib.rs index 16e67a45836..b91905d6c95 100644 --- a/src/cargo/lib.rs +++ b/src/cargo/lib.rs @@ -26,6 +26,7 @@ #![allow(clippy::unneeded_field_pattern)] use std::fmt; +use std::io; use failure::Error; use log::debug; @@ -147,23 +148,38 @@ fn handle_cause(cargo_err: &Error, shell: &mut Shell) -> bool { drop(writeln!(shell.err(), " {}", error)); } + fn print_stderror_causes(error: &dyn std::error::Error, shell: &mut Shell) { + let mut cur = std::error::Error::source(error); + while let Some(err) = cur { + print(&err.to_string(), shell); + cur = std::error::Error::source(err); + } + } + let verbose = shell.verbosity(); - if verbose == Verbose { - // The first error has already been printed to the shell. - // Print all remaining errors. - for err in cargo_err.iter_causes() { - print(&err.to_string(), shell); + // The first error has already been printed to the shell. + for err in cargo_err.iter_causes() { + + // If we're not in verbose mode then print remaining errors until one + // marked as `Internal` appears. + if verbose != Verbose && err.downcast_ref::().is_some() { + return false; } - } else { - // The first error has already been printed to the shell. - // Print remaining errors until one marked as `Internal` appears. - for err in cargo_err.iter_causes() { - if err.downcast_ref::().is_some() { - return false; - } - print(&err.to_string(), shell); + print(&err.to_string(), shell); + + // Using the `failure` crate currently means that when using + // `iter_causes` we're only iterating over the `failure` causes, but + // this doesn't include the causes from the standard library `Error` + // trait. We don't have a great way of getting an `&dyn Error` from a + // `&dyn Fail`, so we currently just special case a few errors that are + // known to maybe have causes and we try to print them here. + // + // Note that this isn't an exhaustive match since causes for + // `std::error::Error` aren't the most common thing in the world. + if let Some(io) = err.downcast_ref::() { + print_stderror_causes(io, shell); } } diff --git a/src/cargo/ops/mod.rs b/src/cargo/ops/mod.rs index 997ead71fa6..9e40062b246 100644 --- a/src/cargo/ops/mod.rs +++ b/src/cargo/ops/mod.rs @@ -26,6 +26,7 @@ pub use self::resolve::{ add_overrides, get_resolved_packages, resolve_with_previous, resolve_ws, resolve_ws_precisely, resolve_ws_with_method, }; +pub use self::vendor::{vendor, VendorOptions}; mod cargo_clean; mod cargo_compile; @@ -46,3 +47,4 @@ mod fix; mod lockfile; mod registry; mod resolve; +mod vendor; diff --git a/src/cargo/ops/vendor.rs b/src/cargo/ops/vendor.rs new file mode 100644 index 00000000000..fccee96997e --- /dev/null +++ b/src/cargo/ops/vendor.rs @@ -0,0 +1,327 @@ +use crate::core::shell::Verbosity; +use crate::core::{GitReference, Workspace}; +use crate::ops; +use crate::sources::path::PathSource; +use crate::util::Sha256; +use crate::util::{paths, CargoResult, CargoResultExt, Config}; +use failure::bail; +use serde::Serialize; +use std::collections::HashSet; +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::fs::{self, File}; +use std::io::Write; +use std::path::{Path, PathBuf}; + +pub struct VendorOptions<'a> { + pub no_delete: bool, + pub destination: &'a Path, + pub extra: Vec, +} + +pub fn vendor(ws: &Workspace<'_>, opts: &VendorOptions<'_>) -> CargoResult<()> { + let mut extra_workspaces = Vec::new(); + for extra in opts.extra.iter() { + let extra = ws.config().cwd().join(extra); + let ws = Workspace::new(&extra, ws.config())?; + extra_workspaces.push(ws); + } + let workspaces = extra_workspaces.iter().chain(Some(ws)).collect::>(); + let vendor_config = + sync(ws.config(), &workspaces, opts).chain_err(|| format!("failed to sync"))?; + + let shell = ws.config().shell(); + if shell.verbosity() != Verbosity::Quiet { + eprint!("To use vendored sources, add this to your .cargo/config for this project:\n\n"); + print!("{}", &toml::to_string(&vendor_config).unwrap()); + } + + Ok(()) +} + +#[derive(Serialize)] +struct VendorConfig { + source: BTreeMap, +} + +#[derive(Serialize)] +#[serde(rename_all = "lowercase", untagged)] +enum VendorSource { + Directory { + directory: PathBuf, + }, + Registry { + registry: Option, + #[serde(rename = "replace-with")] + replace_with: String, + }, + Git { + git: String, + branch: Option, + tag: Option, + rev: Option, + #[serde(rename = "replace-with")] + replace_with: String, + }, +} + +fn sync( + config: &Config, + workspaces: &[&Workspace<'_>], + opts: &VendorOptions<'_>, +) -> CargoResult { + let canonical_destination = opts.destination.canonicalize(); + let canonical_destination = canonical_destination + .as_ref() + .map(|p| &**p) + .unwrap_or(opts.destination); + + fs::create_dir_all(&canonical_destination)?; + let mut to_remove = HashSet::new(); + if !opts.no_delete { + for entry in canonical_destination.read_dir()? { + let entry = entry?; + to_remove.insert(entry.path()); + } + } + + // First up attempt to work around rust-lang/cargo#5956. Apparently build + // artifacts sprout up in Cargo's global cache for whatever reason, although + // it's unsure what tool is causing these issues at this time. For now we + // apply a heavy-hammer approach which is to delete Cargo's unpacked version + // of each crate to start off with. After we do this we'll re-resolve and + // redownload again, which should trigger Cargo to re-extract all the + // crates. + // + // Note that errors are largely ignored here as this is a best-effort + // attempt. If anything fails here we basically just move on to the next + // crate to work with. + for ws in workspaces { + let (packages, resolve) = + ops::resolve_ws(&ws).chain_err(|| "failed to load pkg lockfile")?; + + packages + .get_many(resolve.iter()) + .chain_err(|| "failed to download packages")?; + + for pkg in resolve.iter() { + // Don't delete actual source code! + if pkg.source_id().is_path() { + if let Ok(path) = pkg.source_id().url().to_file_path() { + if let Ok(path) = path.canonicalize() { + to_remove.remove(&path); + } + } + continue; + } + if pkg.source_id().is_git() { + continue; + } + if let Ok(pkg) = packages.get_one(pkg) { + drop(fs::remove_dir_all(pkg.manifest_path().parent().unwrap())); + } + } + } + + let mut checksums = HashMap::new(); + let mut ids = BTreeMap::new(); + + // Next up let's actually download all crates and start storing internal + // tables about them. + for ws in workspaces { + let (packages, resolve) = + ops::resolve_ws(&ws).chain_err(|| "failed to load pkg lockfile")?; + + packages + .get_many(resolve.iter()) + .chain_err(|| "failed to download packages")?; + + for pkg in resolve.iter() { + // No need to vendor path crates since they're already in the + // repository + if pkg.source_id().is_path() { + continue; + } + ids.insert( + pkg.clone(), + packages + .get_one(pkg) + .chain_err(|| "failed to fetch package")? + .clone(), + ); + + checksums.insert(pkg.clone(), resolve.checksums().get(&pkg).cloned()); + } + } + + let mut versions = HashMap::new(); + for id in ids.keys() { + let map = versions.entry(id.name()).or_insert_with(BTreeMap::default); + if let Some(prev) = map.get(&id.version()) { + bail!( + "found duplicate version of package `{} v{}` \ + vendored from two sources:\n\ + \n\ + \tsource 1: {}\n\ + \tsource 2: {}", + id.name(), + id.version(), + prev, + id.source_id() + ); + } + map.insert(id.version(), id.source_id()); + } + + let mut sources = BTreeSet::new(); + for (id, pkg) in ids.iter() { + // Next up, copy it to the vendor directory + let src = pkg + .manifest_path() + .parent() + .expect("manifest_path should point to a file"); + let max_version = *versions[&id.name()].iter().rev().next().unwrap().0; + let dir_has_version_suffix = id.version() != max_version; + let dst_name = if dir_has_version_suffix { + // Eg vendor/futures-0.1.13 + format!("{}-{}", id.name(), id.version()) + } else { + // Eg vendor/futures + id.name().to_string() + }; + + sources.insert(id.source_id()); + let dst = canonical_destination.join(&dst_name); + to_remove.remove(&dst); + let cksum = dst.join(".cargo-checksum.json"); + if dir_has_version_suffix && cksum.exists() { + // Always re-copy directory without version suffix in case the version changed + continue; + } + + config.shell().status( + "Vendoring", + &format!("{} ({}) to {}", id, src.to_string_lossy(), dst.display()), + )?; + + let _ = fs::remove_dir_all(&dst); + let pathsource = PathSource::new(&src, id.source_id(), config); + let paths = pathsource.list_files(&pkg)?; + let mut map = BTreeMap::new(); + cp_sources(&src, &paths, &dst, &mut map) + .chain_err(|| format!("failed to copy over vendored sources for: {}", id))?; + + // Finally, emit the metadata about this package + let json = serde_json::json!({ + "package": checksums.get(id), + "files": map, + }); + + File::create(&cksum)?.write_all(json.to_string().as_bytes())?; + } + + for path in to_remove { + if path.is_dir() { + paths::remove_dir_all(&path)?; + } else { + paths::remove_file(&path)?; + } + } + + // add our vendored source + let mut config = BTreeMap::new(); + + let merged_source_name = "vendored-sources"; + config.insert( + merged_source_name.to_string(), + VendorSource::Directory { + directory: canonical_destination.to_path_buf(), + }, + ); + + // replace original sources with vendor + for source_id in sources { + let name = if source_id.is_default_registry() { + "crates-io".to_string() + } else { + source_id.url().to_string() + }; + + let source = if source_id.is_default_registry() { + VendorSource::Registry { + registry: None, + replace_with: merged_source_name.to_string(), + } + } else if source_id.is_git() { + let mut branch = None; + let mut tag = None; + let mut rev = None; + if let Some(reference) = source_id.git_reference() { + match *reference { + GitReference::Branch(ref b) => branch = Some(b.clone()), + GitReference::Tag(ref t) => tag = Some(t.clone()), + GitReference::Rev(ref r) => rev = Some(r.clone()), + } + } + VendorSource::Git { + git: source_id.url().to_string(), + branch, + tag, + rev, + replace_with: merged_source_name.to_string(), + } + } else { + panic!("Invalid source ID: {}", source_id) + }; + config.insert(name, source); + } + + Ok(VendorConfig { source: config }) +} + +fn cp_sources( + src: &Path, + paths: &[PathBuf], + dst: &Path, + cksums: &mut BTreeMap, +) -> CargoResult<()> { + for p in paths { + let relative = p.strip_prefix(&src).unwrap(); + + match relative.to_str() { + // Skip git config files as they're not relevant to builds most of + // the time and if we respect them (e.g. in git) then it'll + // probably mess with the checksums when a vendor dir is checked + // into someone else's source control + Some(".gitattributes") | Some(".gitignore") | Some(".git") => continue, + + // Temporary Cargo files + Some(".cargo-ok") => continue, + + // Skip patch-style orig/rej files. Published crates on crates.io + // have `Cargo.toml.orig` which we don't want to use here and + // otherwise these are rarely used as part of the build process. + Some(filename) => { + if filename.ends_with(".orig") || filename.ends_with(".rej") { + continue; + } + } + _ => {} + }; + + // Join pathname components individually to make sure that the joined + // path uses the correct directory separators everywhere, since + // `relative` may use Unix-style and `dst` may require Windows-style + // backslashes. + let dst = relative + .iter() + .fold(dst.to_owned(), |acc, component| acc.join(&component)); + + fs::create_dir_all(dst.parent().unwrap())?; + + fs::copy(&p, &dst) + .chain_err(|| format!("failed to copy `{}` to `{}`", p.display(), dst.display()))?; + let cksum = Sha256::new().update_path(dst)?.finish_hex(); + cksums.insert(relative.to_str().unwrap().replace("\\", "/"), cksum); + } + Ok(()) +} diff --git a/src/cargo/sources/directory.rs b/src/cargo/sources/directory.rs index aab31383c72..06f668f4db0 100644 --- a/src/cargo/sources/directory.rs +++ b/src/cargo/sources/directory.rs @@ -1,7 +1,5 @@ use std::collections::HashMap; use std::fmt::{self, Debug, Formatter}; -use std::fs::File; -use std::io::Read; use std::path::{Path, PathBuf}; use serde::Deserialize; @@ -170,23 +168,12 @@ impl<'cfg> Source for DirectorySource<'cfg> { None => failure::bail!("failed to find entry for `{}` in directory source", id), }; - let mut buf = [0; 16 * 1024]; for (file, cksum) in cksum.files.iter() { - let mut h = Sha256::new(); let file = pkg.root().join(file); - - (|| -> CargoResult<()> { - let mut f = File::open(&file)?; - loop { - match f.read(&mut buf)? { - 0 => return Ok(()), - n => h.update(&buf[..n]), - } - } - })() - .chain_err(|| format!("failed to calculate checksum of: {}", file.display()))?; - - let actual = hex::encode(h.finish()); + let actual = Sha256::new() + .update_path(&file) + .chain_err(|| format!("failed to calculate checksum of: {}", file.display()))? + .finish_hex(); if &*actual != cksum { failure::bail!( "\ diff --git a/src/cargo/sources/registry/local.rs b/src/cargo/sources/registry/local.rs index d2f6dc936e2..35c8db48361 100644 --- a/src/cargo/sources/registry/local.rs +++ b/src/cargo/sources/registry/local.rs @@ -1,9 +1,8 @@ use crate::core::{InternedString, PackageId}; use crate::sources::registry::{MaybeLock, RegistryConfig, RegistryData}; -use crate::util::errors::{CargoResult, CargoResultExt}; +use crate::util::errors::CargoResult; use crate::util::paths; use crate::util::{Config, Filesystem, Sha256}; -use hex; use std::fs::File; use std::io::prelude::*; use std::io::SeekFrom; @@ -99,18 +98,8 @@ impl<'cfg> RegistryData for LocalRegistry<'cfg> { // We don't actually need to download anything per-se, we just need to // verify the checksum matches the .crate file itself. - let mut state = Sha256::new(); - let mut buf = [0; 64 * 1024]; - loop { - let n = crate_file - .read(&mut buf) - .chain_err(|| format!("failed to read `{}`", path.display()))?; - if n == 0 { - break; - } - state.update(&buf[..n]); - } - if hex::encode(state.finish()) != checksum { + let actual = Sha256::new().update_file(&crate_file)?.finish_hex(); + if actual != checksum { failure::bail!("failed to verify the checksum of `{}`", pkg) } diff --git a/src/cargo/sources/registry/remote.rs b/src/cargo/sources/registry/remote.rs index 1abae42e909..c44964432e2 100644 --- a/src/cargo/sources/registry/remote.rs +++ b/src/cargo/sources/registry/remote.rs @@ -281,9 +281,8 @@ impl<'cfg> RegistryData for RemoteRegistry<'cfg> { data: &[u8], ) -> CargoResult { // Verify what we just downloaded - let mut state = Sha256::new(); - state.update(data); - if hex::encode(state.finish()) != checksum { + let actual = Sha256::new().update(data).finish_hex(); + if actual != checksum { failure::bail!("failed to verify the checksum of `{}`", pkg) } diff --git a/src/cargo/util/config.rs b/src/cargo/util/config.rs index 54ee81704e8..9409277f6d3 100644 --- a/src/cargo/util/config.rs +++ b/src/cargo/util/config.rs @@ -294,6 +294,13 @@ impl Config { self.values.try_borrow_with(|| self.load_values()) } + pub fn values_mut(&mut self) -> CargoResult<&mut HashMap> { + match self.values.borrow_mut() { + Some(map) => Ok(map), + None => failure::bail!("config values not loaded yet"), + } + } + // Note: this is used by RLS, not Cargo. pub fn set_values(&self, values: HashMap) -> CargoResult<()> { if self.values.borrow().is_some() { diff --git a/src/cargo/util/sha256.rs b/src/cargo/util/sha256.rs index 32d04ae935e..a8f762c15e0 100644 --- a/src/cargo/util/sha256.rs +++ b/src/cargo/util/sha256.rs @@ -1,6 +1,9 @@ use self::crypto_hash::{Algorithm, Hasher}; +use std::fs::File; use crypto_hash; -use std::io::Write; +use std::io::{self, Write, Read}; +use std::path::Path; +use crate::util::{CargoResult, CargoResultExt}; pub struct Sha256(Hasher); @@ -10,8 +13,28 @@ impl Sha256 { Sha256(hasher) } - pub fn update(&mut self, bytes: &[u8]) { + pub fn update(&mut self, bytes: &[u8]) -> &mut Sha256 { let _ = self.0.write_all(bytes); + self + } + + pub fn update_file(&mut self, mut file: &File) -> io::Result<&mut Sha256> { + let mut buf = [0; 64 * 1024]; + loop { + let n = file.read(&mut buf)?; + if n == 0 { + break Ok(self); + } + self.update(&buf[..n]); + } + } + + pub fn update_path>(&mut self, path: P) -> CargoResult<&mut Sha256> { + let path = path.as_ref(); + let file = File::open(path)?; + self.update_file(&file) + .chain_err(|| format!("failed to read `{}`", path.display()))?; + Ok(self) } pub fn finish(&mut self) -> [u8; 32] { @@ -20,6 +43,10 @@ impl Sha256 { ret.copy_from_slice(&data[..]); ret } + + pub fn finish_hex(&mut self) -> String { + hex::encode(self.finish()) + } } impl Default for Sha256 { diff --git a/src/doc/man/cargo-vendor.adoc b/src/doc/man/cargo-vendor.adoc new file mode 100644 index 00000000000..7f52ec815d8 --- /dev/null +++ b/src/doc/man/cargo-vendor.adoc @@ -0,0 +1,75 @@ += cargo-vendor(1) +:idprefix: cargo_vendor_ +:doctype: manpage + +== NAME + +cargo-vendor - Vendor all dependencies locally + +== SYNOPSIS + +`cargo vendor [_OPTIONS_] [_PATH_]` + +== DESCRIPTION + +This cargo subcommand will vendor all crates.io and git dependencies for a +project into the specified directory at ``. After this command completes +the vendor directory specified by `` will contain all remote sources from +dependencies specified. Additional manifests beyond the default one can be +specified with the `-s` option. + +The `cargo vendor` command will also print out the configuration necessary +to use the vendored sources, which you will need to add to `.cargo/config`. + +== OPTIONS + +=== Owner Options + +*-s* _MANIFEST_:: +*--sync* _MANIFEST_:: + Specify extra `Cargo.toml` manifests to workspaces which should also be + vendored and synced to the output. + +*--no-delete*:: + Don't delete the "vendor" directory when vendoring, but rather keep all + existing contents of the vendor directory + +*--respect-source-config*:: + Instead of ignoring `[source]` configuration by default in `.cargo/config` + read it and use it when downloading crates from crates.io, for example + +=== Manifest Options + +include::options-manifest-path.adoc[] + +=== Display Options + +include::options-display.adoc[] + +=== Common Options + +include::options-common.adoc[] + +include::options-locked.adoc[] + +include::section-environment.adoc[] + +include::section-exit-status.adoc[] + +== EXAMPLES + +. Vendor all dependencies into a local "vendor" folder + + cargo vendor + +. Vendor all dependencies into a local "third-part/vendor" folder + + cargo vendor third-party/vendor + +. Vendor the current workspace as well as another to "vendor" + + cargo vendor -s ../path/to/Cargo.toml + +== SEE ALSO +man:cargo[1] + diff --git a/src/doc/man/generated/cargo-metadata.html b/src/doc/man/generated/cargo-metadata.html index 25665e31977..007f7ff81f9 100644 --- a/src/doc/man/generated/cargo-metadata.html +++ b/src/doc/man/generated/cargo-metadata.html @@ -35,7 +35,7 @@

OUTPUT FORMAT

-
{
+
{
     /* Array of all packages in the workspace.
        It also includes all feature-enabled dependencies unless --no-deps is used.
     */
diff --git a/src/doc/man/generated/cargo-vendor.html b/src/doc/man/generated/cargo-vendor.html
new file mode 100644
index 00000000000..87ead76249b
--- /dev/null
+++ b/src/doc/man/generated/cargo-vendor.html
@@ -0,0 +1,207 @@
+

NAME

+
+

cargo-vendor - Vendor all dependencies locally

+
+
+

SYNOPSIS

+
+
+

cargo vendor [OPTIONS] [PATH]

+
+
+
+
+

DESCRIPTION

+
+
+

This cargo subcommand will vendor all crates.io and git dependencies for a +project into the specified directory at <path>. After this command completes +the vendor directory specified by <path> will contain all remote sources from +dependencies specified. Additional manifests beyond the default one can be +specified with the -s option.

+
+
+

The cargo vendor command will also print out the configuration necessary +to use the vendored sources, which you will need to add to .cargo/config.

+
+
+
+
+

OPTIONS

+
+
+

Owner Options

+
+
+
-s MANIFEST
+
--sync MANIFEST
+
+

Specify extra Cargo.toml manifests to workspaces which should also be +vendored and synced to the output.

+
+
--no-delete
+
+

Don’t delete the "vendor" directory when vendoring, but rather keep all +existing contents of the vendor directory

+
+
--respect-source-config
+
+

Instead of ignoring [source] configuration by default in .cargo/config +read it and use it when downloading crates from crates.io, for example

+
+
+
+
+
+

Manifest Options

+
+
+
--manifest-path PATH
+
+

Path to the Cargo.toml file. By default, Cargo searches in the current +directory or any parent directory for the Cargo.toml file.

+
+
+
+
+
+

Display Options

+
+
+
-v
+
--verbose
+
+

Use verbose output. May be specified twice for "very verbose" output which +includes extra output such as dependency warnings and build script output. +May also be specified with the term.verbose +config value.

+
+
-q
+
--quiet
+
+

No output printed to stdout.

+
+
--color WHEN
+
+

Control when colored output is used. Valid values:

+
+
    +
  • +

    auto (default): Automatically detect if color support is available on the +terminal.

    +
  • +
  • +

    always: Always display colors.

    +
  • +
  • +

    never: Never display colors.

    +
  • +
+
+
+

May also be specified with the term.color +config value.

+
+
+
+
+
+
+

Common Options

+
+
+
-h
+
--help
+
+

Prints help information.

+
+
-Z FLAG…​
+
+

Unstable (nightly-only) flags to Cargo. Run cargo -Z help for +details.

+
+
--frozen
+
--locked
+
+

Either of these flags requires that the Cargo.lock file is +up-to-date. If the lock file is missing, or it needs to be updated, Cargo will +exit with an error. The --frozen flag also prevents Cargo from +attempting to access the network to determine if it is out-of-date.

+
+

These may be used in environments where you want to assert that the +Cargo.lock file is up-to-date (such as a CI build) or want to avoid network +access.

+
+
+
+
+
+
+
+
+

ENVIRONMENT

+
+
+

See the reference for +details on environment variables that Cargo reads.

+
+
+
+
+

Exit Status

+
+
+
+
0
+
+

Cargo succeeded.

+
+
101
+
+

Cargo failed to complete.

+
+
+
+
+
+
+

EXAMPLES

+
+
+
    +
  1. +

    Vendor all dependencies into a local "vendor" folder

    +
    +
    +
    cargo vendor
    +
    +
    +
  2. +
  3. +

    Vendor all dependencies into a local "third-part/vendor" folder

    +
    +
    +
    cargo vendor third-party/vendor
    +
    +
    +
  4. +
  5. +

    Vendor the current workspace as well as another to "vendor"

    +
    +
    +
    cargo vendor -s ../path/to/Cargo.toml
    +
    +
    +
  6. +
+
+
+
+
+

SEE ALSO

+
+ +
+
\ No newline at end of file diff --git a/src/doc/src/SUMMARY.md b/src/doc/src/SUMMARY.md index 3a69c0a312d..4943d251268 100644 --- a/src/doc/src/SUMMARY.md +++ b/src/doc/src/SUMMARY.md @@ -49,6 +49,7 @@ * [metadata](commands/cargo-metadata.md) * [pkgid](commands/cargo-pkgid.md) * [update](commands/cargo-update.md) + * [vendor](commands/cargo-vendor.md) * [verify-project](commands/cargo-verify-project.md) * [Package Commands](commands/package-commands.md) * [init](commands/cargo-init.md) diff --git a/src/doc/src/commands/cargo-vendor.md b/src/doc/src/commands/cargo-vendor.md new file mode 100644 index 00000000000..1e0f333fe6d --- /dev/null +++ b/src/doc/src/commands/cargo-vendor.md @@ -0,0 +1,4 @@ +# cargo vendor +{{#include command-common.html}} +{{#include ../../man/generated/cargo-vendor.html}} + diff --git a/src/doc/src/reference/source-replacement.md b/src/doc/src/reference/source-replacement.md index d7e23fa9982..802f41d6277 100644 --- a/src/doc/src/reference/source-replacement.md +++ b/src/doc/src/reference/source-replacement.md @@ -107,12 +107,8 @@ the crates that are present). A "directory source" is similar to a local registry source where it contains a number of crates available on the local filesystem, suitable for vendoring -dependencies. Also like local registries, directory sources can primarily be -managed by an external subcommand, [`cargo-vendor`][cargo-vendor], -[available on crates.io][cargo-vendor] and can be -installed with `cargo install cargo-vendor`. - -[cargo-vendor]: https://crates.io/crates/cargo-vendor +dependencies. Directory sources are primarily managed the `cargo vendor` +subcommand. Directory sources are distinct from local registries though in that they contain the unpacked version of `*.crate` files, making it more suitable in some diff --git a/src/etc/man/cargo-vendor.1 b/src/etc/man/cargo-vendor.1 new file mode 100644 index 00000000000..3937cc9495e --- /dev/null +++ b/src/etc/man/cargo-vendor.1 @@ -0,0 +1,223 @@ +'\" t +.\" Title: cargo-vendor +.\" Author: [see the "AUTHOR(S)" section] +.\" Generator: Asciidoctor 2.0.8 +.\" Date: 2019-04-29 +.\" Manual: \ \& +.\" Source: \ \& +.\" Language: English +.\" +.TH "CARGO\-VENDOR" "1" "2019-04-29" "\ \&" "\ \&" +.ie \n(.g .ds Aq \(aq +.el .ds Aq ' +.ss \n[.ss] 0 +.nh +.ad l +.de URL +\fI\\$2\fP <\\$1>\\$3 +.. +.als MTO URL +.if \n[.g] \{\ +. mso www.tmac +. am URL +. ad l +. . +. am MTO +. ad l +. . +. LINKSTYLE blue R < > +.\} +.SH "NAME" +cargo\-vendor \- Vendor all dependencies locally +.SH "SYNOPSIS" +.sp +\fBcargo vendor [\fIOPTIONS\fP] [\fIPATH\fP]\fP +.SH "DESCRIPTION" +.sp +This cargo subcommand will vendor all crates.io and git dependencies for a +project into the specified directory at \fB\fP. After this command completes +the vendor directory specified by \fB\fP will contain all remote sources from +dependencies specified. Additional manifests beyond the default one can be +specified with the \fB\-s\fP option. +.sp +The \fBcargo vendor\fP command will also print out the configuration necessary +to use the vendored sources, which you will need to add to \fB.cargo/config\fP. +.SH "OPTIONS" +.SS "Owner Options" +.sp +\fB\-s\fP \fIMANIFEST\fP, \fB\-\-sync\fP \fIMANIFEST\fP +.RS 4 +Specify extra \fBCargo.toml\fP manifests to workspaces which should also be +vendored and synced to the output. +.RE +.sp +\fB\-\-no\-delete\fP +.RS 4 +Don\(cqt delete the "vendor" directory when vendoring, but rather keep all +existing contents of the vendor directory +.RE +.sp +\fB\-\-respect\-source\-config\fP +.RS 4 +Instead of ignoring \fB[source]\fP configuration by default in \fB.cargo/config\fP +read it and use it when downloading crates from crates.io, for example +.RE +.SS "Manifest Options" +.sp +\fB\-\-manifest\-path\fP \fIPATH\fP +.RS 4 +Path to the \fBCargo.toml\fP file. By default, Cargo searches in the current +directory or any parent directory for the \fBCargo.toml\fP file. +.RE +.SS "Display Options" +.sp +\fB\-v\fP, \fB\-\-verbose\fP +.RS 4 +Use verbose output. May be specified twice for "very verbose" output which +includes extra output such as dependency warnings and build script output. +May also be specified with the \fBterm.verbose\fP +.URL "https://doc.rust\-lang.org/cargo/reference/config.html" "config value" "." +.RE +.sp +\fB\-q\fP, \fB\-\-quiet\fP +.RS 4 +No output printed to stdout. +.RE +.sp +\fB\-\-color\fP \fIWHEN\fP +.RS 4 +Control when colored output is used. Valid values: +.sp +.RS 4 +.ie n \{\ +\h'-04'\(bu\h'+03'\c +.\} +.el \{\ +. sp -1 +. IP \(bu 2.3 +.\} +\fBauto\fP (default): Automatically detect if color support is available on the +terminal. +.RE +.sp +.RS 4 +.ie n \{\ +\h'-04'\(bu\h'+03'\c +.\} +.el \{\ +. sp -1 +. IP \(bu 2.3 +.\} +\fBalways\fP: Always display colors. +.RE +.sp +.RS 4 +.ie n \{\ +\h'-04'\(bu\h'+03'\c +.\} +.el \{\ +. sp -1 +. IP \(bu 2.3 +.\} +\fBnever\fP: Never display colors. +.RE +.sp +May also be specified with the \fBterm.color\fP +.URL "https://doc.rust\-lang.org/cargo/reference/config.html" "config value" "." +.RE +.SS "Common Options" +.sp +\fB\-h\fP, \fB\-\-help\fP +.RS 4 +Prints help information. +.RE +.sp +\fB\-Z\fP \fIFLAG\fP... +.RS 4 +Unstable (nightly\-only) flags to Cargo. Run \fBcargo \-Z help\fP for +details. +.RE +.sp +\fB\-\-frozen\fP, \fB\-\-locked\fP +.RS 4 +Either of these flags requires that the \fBCargo.lock\fP file is +up\-to\-date. If the lock file is missing, or it needs to be updated, Cargo will +exit with an error. The \fB\-\-frozen\fP flag also prevents Cargo from +attempting to access the network to determine if it is out\-of\-date. +.sp +These may be used in environments where you want to assert that the +\fBCargo.lock\fP file is up\-to\-date (such as a CI build) or want to avoid network +access. +.RE +.SH "ENVIRONMENT" +.sp +See \c +.URL "https://doc.rust\-lang.org/cargo/reference/environment\-variables.html" "the reference" " " +for +details on environment variables that Cargo reads. +.SH "EXIT STATUS" +.sp +0 +.RS 4 +Cargo succeeded. +.RE +.sp +101 +.RS 4 +Cargo failed to complete. +.RE +.SH "EXAMPLES" +.sp +.RS 4 +.ie n \{\ +\h'-04' 1.\h'+01'\c +.\} +.el \{\ +. sp -1 +. IP " 1." 4.2 +.\} +Vendor all dependencies into a local "vendor" folder +.sp +.if n .RS 4 +.nf +cargo vendor +.fi +.if n .RE +.RE +.sp +.RS 4 +.ie n \{\ +\h'-04' 2.\h'+01'\c +.\} +.el \{\ +. sp -1 +. IP " 2." 4.2 +.\} +Vendor all dependencies into a local "third\-part/vendor" folder +.sp +.if n .RS 4 +.nf +cargo vendor third\-party/vendor +.fi +.if n .RE +.RE +.sp +.RS 4 +.ie n \{\ +\h'-04' 3.\h'+01'\c +.\} +.el \{\ +. sp -1 +. IP " 3." 4.2 +.\} +Vendor the current workspace as well as another to "vendor" +.sp +.if n .RS 4 +.nf +cargo vendor \-s ../path/to/Cargo.toml +.fi +.if n .RE +.RE +.SH "SEE ALSO" +.sp +\fBcargo\fP(1) \ No newline at end of file diff --git a/src/etc/man/cargo.1 b/src/etc/man/cargo.1 index f715eea6f2f..757510051b2 100644 --- a/src/etc/man/cargo.1 +++ b/src/etc/man/cargo.1 @@ -487,4 +487,4 @@ See \c for issues. .SH "SEE ALSO" .sp -\fBrustc\fP(1), \fBrustdoc\fP(1) \ No newline at end of file +\fBrustc\fP(1), \fBrustdoc\fP(1) diff --git a/tests/testsuite/main.rs b/tests/testsuite/main.rs index 5dcf126ec82..031e5c9cebc 100644 --- a/tests/testsuite/main.rs +++ b/tests/testsuite/main.rs @@ -88,6 +88,7 @@ mod small_fd_limits; mod test; mod tool_paths; mod update; +mod vendor; mod verify_project; mod version; mod warn_on_failure; diff --git a/tests/testsuite/support/registry.rs b/tests/testsuite/support/registry.rs index 023ece91320..709f748ce6a 100644 --- a/tests/testsuite/support/registry.rs +++ b/tests/testsuite/support/registry.rs @@ -538,9 +538,7 @@ impl Package { } pub fn cksum(s: &[u8]) -> String { - let mut sha = Sha256::new(); - sha.update(s); - hex::encode(&sha.finish()) + Sha256::new().update(s).finish_hex() } impl Dependency { diff --git a/tests/testsuite/vendor.rs b/tests/testsuite/vendor.rs new file mode 100644 index 00000000000..aa5e9347007 --- /dev/null +++ b/tests/testsuite/vendor.rs @@ -0,0 +1,508 @@ +use crate::support::git; +use crate::support::registry::Package; +use crate::support::{basic_lib_manifest, project, Project}; + +#[test] +fn vendor_simple() { + let p = project() + .file( + "Cargo.toml", + r#" + [package] + name = "foo" + version = "0.1.0" + + [dependencies] + log = "0.3.5" + "#, + ) + .file("src/lib.rs", "") + .build(); + + Package::new("log", "0.3.5").publish(); + + p.cargo("vendor --respect-source-config").run(); + let lock = p.read_file("vendor/log/Cargo.toml"); + assert!(lock.contains("version = \"0.3.5\"")); + + add_vendor_config(&p); + p.cargo("build").run(); +} + +fn add_vendor_config(p: &Project) { + p.change_file( + ".cargo/config", + r#" + [source.crates-io] + replace-with = 'vendor' + + [source.vendor] + directory = 'vendor' + "#, + ); +} + +#[test] +fn two_versions() { + let p = project() + .file( + "Cargo.toml", + r#" + [package] + name = "foo" + version = "0.1.0" + + [dependencies] + bitflags = "0.8.0" + bar = { path = "bar" } + "#, + ) + .file("src/lib.rs", "") + .file( + "bar/Cargo.toml", + r#" + [package] + name = "bar" + version = "0.1.0" + + [dependencies] + bitflags = "0.7.0" + "#, + ) + .file("bar/src/lib.rs", "") + .build(); + + Package::new("bitflags", "0.7.0").publish(); + Package::new("bitflags", "0.8.0").publish(); + + p.cargo("vendor --respect-source-config").run(); + + let lock = p.read_file("vendor/bitflags/Cargo.toml"); + assert!(lock.contains("version = \"0.8.0\"")); + let lock = p.read_file("vendor/bitflags-0.7.0/Cargo.toml"); + assert!(lock.contains("version = \"0.7.0\"")); + + add_vendor_config(&p); + p.cargo("build").run(); +} + +#[test] +fn help() { + let p = project().build(); + p.cargo("vendor -h").run(); +} + +#[test] +fn update_versions() { + let p = project() + .file( + "Cargo.toml", + r#" + [package] + name = "foo" + version = "0.1.0" + + [dependencies] + bitflags = "0.7.0" + "#, + ) + .file("src/lib.rs", "") + .build(); + + Package::new("bitflags", "0.7.0").publish(); + Package::new("bitflags", "0.8.0").publish(); + + p.cargo("vendor --respect-source-config").run(); + + let lock = p.read_file("vendor/bitflags/Cargo.toml"); + assert!(lock.contains("version = \"0.7.0\"")); + + p.change_file( + "Cargo.toml", + r#" + [package] + name = "foo" + version = "0.1.0" + + [dependencies] + bitflags = "0.8.0" + "#, + ); + p.cargo("vendor --respect-source-config").run(); + + let lock = p.read_file("vendor/bitflags/Cargo.toml"); + assert!(lock.contains("version = \"0.8.0\"")); +} + +#[test] +fn two_lockfiles() { + let p = project() + .no_manifest() + .file( + "foo/Cargo.toml", + r#" + [package] + name = "foo" + version = "0.1.0" + + [dependencies] + bitflags = "=0.7.0" + "#, + ) + .file("foo/src/lib.rs", "") + .file( + "bar/Cargo.toml", + r#" + [package] + name = "bar" + version = "0.1.0" + + [dependencies] + bitflags = "=0.8.0" + "#, + ) + .file("bar/src/lib.rs", "") + .build(); + + Package::new("bitflags", "0.7.0").publish(); + Package::new("bitflags", "0.8.0").publish(); + + p.cargo("vendor --respect-source-config -s bar/Cargo.toml --manifest-path foo/Cargo.toml") + .run(); + + let lock = p.read_file("vendor/bitflags/Cargo.toml"); + assert!(lock.contains("version = \"0.8.0\"")); + let lock = p.read_file("vendor/bitflags-0.7.0/Cargo.toml"); + assert!(lock.contains("version = \"0.7.0\"")); + + add_vendor_config(&p); + p.cargo("build").cwd("foo").run(); + p.cargo("build").cwd("bar").run(); +} + +#[test] +fn delete_old_crates() { + let p = project() + .file( + "Cargo.toml", + r#" + [package] + name = "foo" + version = "0.1.0" + + [dependencies] + bitflags = "=0.7.0" + "#, + ) + .file("src/lib.rs", "") + .build(); + + Package::new("bitflags", "0.7.0").publish(); + Package::new("log", "0.3.5").publish(); + + p.cargo("vendor --respect-source-config").run(); + p.read_file("vendor/bitflags/Cargo.toml"); + + p.change_file( + "Cargo.toml", + r#" + [package] + name = "foo" + version = "0.1.0" + + [dependencies] + log = "=0.3.5" + "#, + ); + + p.cargo("vendor --respect-source-config").run(); + let lock = p.read_file("vendor/log/Cargo.toml"); + assert!(lock.contains("version = \"0.3.5\"")); + assert!(!p.root().join("vendor/bitflags/Cargo.toml").exists()); +} + +#[test] +fn ignore_files() { + let p = project() + .file( + "Cargo.toml", + r#" + [package] + name = "foo" + version = "0.1.0" + + [dependencies] + url = "1.4.1" + "#, + ) + .file("src/lib.rs", "") + .build(); + + Package::new("url", "1.4.1") + .file("src/lib.rs", "") + .file("foo.orig", "") + .file(".gitignore", "") + .file(".gitattributes", "") + .file("foo.rej", "") + .publish(); + + p.cargo("vendor --respect-source-config").run(); + let csum = p.read_file("vendor/url/.cargo-checksum.json"); + assert!(!csum.contains("foo.orig")); + assert!(!csum.contains(".gitignore")); + assert!(!csum.contains(".gitattributes")); + assert!(!csum.contains(".cargo-ok")); + assert!(!csum.contains("foo.rej")); +} + +#[test] +fn included_files_only() { + let git = git::new("a", |p| { + p.file("Cargo.toml", &basic_lib_manifest("a")) + .file("src/lib.rs", "") + .file(".gitignore", "a") + .file("a/b.md", "") + }) + .unwrap(); + + let p = project() + .file( + "Cargo.toml", + &format!( + r#" + [package] + name = "foo" + version = "0.1.0" + + [dependencies] + a = {{ git = '{}' }} + "#, + git.url() + ), + ) + .file("src/lib.rs", "") + .build(); + + p.cargo("vendor --respect-source-config").run(); + let csum = p.read_file("vendor/a/.cargo-checksum.json"); + assert!(!csum.contains("a/b.md")); +} + +#[test] +fn dependent_crates_in_crates() { + let git = git::new("a", |p| { + p.file( + "Cargo.toml", + r#" + [package] + name = "a" + version = "0.1.0" + + [dependencies] + b = { path = 'b' } + "#, + ) + .file("src/lib.rs", "") + .file("b/Cargo.toml", &basic_lib_manifest("b")) + .file("b/src/lib.rs", "") + }) + .unwrap(); + let p = project() + .file( + "Cargo.toml", + &format!( + r#" + [package] + name = "foo" + version = "0.1.0" + + [dependencies] + a = {{ git = '{}' }} + "#, + git.url() + ), + ) + .file("src/lib.rs", "") + .build(); + + p.cargo("vendor --respect-source-config").run(); + p.read_file("vendor/a/.cargo-checksum.json"); + p.read_file("vendor/b/.cargo-checksum.json"); +} + +#[test] +fn vendoring_git_crates() { + let git = git::new("git", |p| { + p.file("Cargo.toml", &basic_lib_manifest("serde_derive")) + .file("src/lib.rs", "") + .file("src/wut.rs", "") + }) + .unwrap(); + + let p = project() + .file( + "Cargo.toml", + &format!( + r#" + [package] + name = "foo" + version = "0.1.0" + + [dependencies.serde] + version = "0.5.0" + + [dependencies.serde_derive] + version = "0.5.0" + + [patch.crates-io] + serde_derive = {{ git = '{}' }} + "#, + git.url() + ), + ) + .file("src/lib.rs", "") + .build(); + Package::new("serde", "0.5.0") + .dep("serde_derive", "0.5") + .publish(); + Package::new("serde_derive", "0.5.0").publish(); + + p.cargo("vendor --respect-source-config").run(); + p.read_file("vendor/serde_derive/src/wut.rs"); + + add_vendor_config(&p); + p.cargo("build").run(); +} + +#[test] +fn git_simple() { + let git = git::new("git", |p| { + p.file("Cargo.toml", &basic_lib_manifest("a")) + .file("src/lib.rs", "") + }) + .unwrap(); + + let p = project() + .file( + "Cargo.toml", + &format!( + r#" + [package] + name = "foo" + version = "0.1.0" + + [dependencies] + a = {{ git = '{}' }} + "#, + git.url() + ), + ) + .file("src/lib.rs", "") + .build(); + + p.cargo("vendor --respect-source-config").run(); + let csum = p.read_file("vendor/a/.cargo-checksum.json"); + assert!(csum.contains("\"package\":null")); +} + +#[test] +fn git_duplicate() { + let git = git::new("a", |p| { + p.file( + "Cargo.toml", + r#" + [package] + name = "a" + version = "0.1.0" + + [dependencies] + b = { path = 'b' } + "#, + ) + .file("src/lib.rs", "") + .file("b/Cargo.toml", &basic_lib_manifest("b")) + .file("b/src/lib.rs", "") + }) + .unwrap(); + + let p = project() + .file( + "Cargo.toml", + &format!( + r#" + [package] + name = "foo" + version = "0.1.0" + + [dependencies] + a = {{ git = '{}' }} + b = '0.5.0' + + "#, + git.url() + ), + ) + .file("src/lib.rs", "") + .build(); + Package::new("b", "0.5.0").publish(); + + p.cargo("vendor --respect-source-config") + .with_stderr( + "\ +[UPDATING] [..] +[UPDATING] [..] +[DOWNLOADING] [..] +[DOWNLOADED] [..] +error: failed to sync + +Caused by: + found duplicate version of package `b v0.5.0` vendored from two sources: + +source 1: [..] +source 2: [..] +", + ) + .with_status(101) + .run(); +} + +#[test] +fn depend_on_vendor_dir_not_deleted() { + let p = project() + .file( + "Cargo.toml", + r#" + [package] + name = "foo" + version = "0.1.0" + + [dependencies] + libc = "0.2.30" + "#, + ) + .file("src/lib.rs", "") + .build(); + + Package::new("libc", "0.2.30").publish(); + + p.cargo("vendor --respect-source-config").run(); + assert!(p.root().join("vendor/libc").is_dir()); + + p.change_file( + "Cargo.toml", + r#" + [package] + name = "foo" + version = "0.1.0" + + [dependencies] + libc = "0.2.30" + + [patch.crates-io] + libc = { path = 'vendor/libc' } + "#, + ); + + p.cargo("vendor --respect-source-config").run(); + assert!(p.root().join("vendor/libc").is_dir()); +}