diff --git a/Cargo.lock b/Cargo.lock index df26dcd6291..bda1d52b03a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3,13 +3,14 @@ name = "cargo" version = "0.0.1-pre" dependencies = [ "curl 0.0.1 (git+https://github.com/alexcrichton/curl-rust?ref=bundle#36b015de91daf6310227cec04ef30acf5929dfb6)", - "docopt 0.6.4 (git+https://github.com/docopt/docopt.rs#4544a9f422b115c2ffef4ee9baf27ceb07c34602)", + "docopt 0.6.5 (git+https://github.com/docopt/docopt.rs#1c9a63e0362848570f7c503f8891770ebf2d1eab)", "flate2 0.0.1 (git+https://github.com/alexcrichton/flate2-rs#68971ae77a523c7ec3f19b4bcd195f76291ea390)", "git2 0.0.1 (git+https://github.com/alexcrichton/git2-rs#7d7fba10893590793ae88c8fc6ab2aeffcb8f10b)", "glob 0.0.1 (git+https://github.com/rust-lang/glob#469a6bc1a0fc289ab220170e691cffbc01dcf1e6)", "hamcrest 0.1.0 (git+https://github.com/carllerche/hamcrest-rust.git#7d46e76514ac606530dfb0e93270fffcf64ca76d)", - "semver 0.1.0 (git+https://github.com/rust-lang/semver#9bb8265ea6cf01eddfa7dc5ec9334883443e9fc7)", - "tar 0.0.1 (git+https://github.com/alexcrichton/tar-rs#943d7c0173c7fa5e8c58978add0180569c68d7f7)", + "registry 0.0.1-pre", + "semver 0.1.0 (git+https://github.com/rust-lang/semver#0eee1b33e90a62ed03a123b94c8e06cdbaf5b662)", + "tar 0.0.1 (git+https://github.com/alexcrichton/tar-rs#c477f1ca1b6dde36ebf1f4a739033fe485895722)", "toml 0.1.0 (git+https://github.com/alexcrichton/toml-rs#8a3ba4c65cfa22a3d924293a1fb3a70bfac5e66c)", "url 0.1.0 (git+https://github.com/servo/rust-url#7f1991d847fb8cf8648def2ff121bae90b89131f)", ] @@ -31,8 +32,8 @@ source = "git+https://github.com/alexcrichton/curl-rust?ref=bundle#36b015de91daf [[package]] name = "docopt" -version = "0.6.4" -source = "git+https://github.com/docopt/docopt.rs#4544a9f422b115c2ffef4ee9baf27ceb07c34602" +version = "0.6.5" +source = "git+https://github.com/docopt/docopt.rs#1c9a63e0362848570f7c503f8891770ebf2d1eab" [[package]] name = "encoding" @@ -88,15 +89,22 @@ name = "openssl-static-sys" version = "0.0.1" source = "git+https://github.com/alexcrichton/openssl-static-sys#d218fa63aabefb3ac56a44985e2df8a2dc932450" +[[package]] +name = "registry" +version = "0.0.1-pre" +dependencies = [ + "curl 0.0.1 (git+https://github.com/alexcrichton/curl-rust?ref=bundle#36b015de91daf6310227cec04ef30acf5929dfb6)", +] + [[package]] name = "semver" version = "0.1.0" -source = "git+https://github.com/rust-lang/semver#9bb8265ea6cf01eddfa7dc5ec9334883443e9fc7" +source = "git+https://github.com/rust-lang/semver#0eee1b33e90a62ed03a123b94c8e06cdbaf5b662" [[package]] name = "tar" version = "0.0.1" -source = "git+https://github.com/alexcrichton/tar-rs#943d7c0173c7fa5e8c58978add0180569c68d7f7" +source = "git+https://github.com/alexcrichton/tar-rs#c477f1ca1b6dde36ebf1f4a739033fe485895722" [[package]] name = "toml" diff --git a/Cargo.toml b/Cargo.toml index a397a6a6696..a1561014267 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -39,6 +39,9 @@ git = "https://github.com/alexcrichton/git2-rs" [dependencies.glob] git = "https://github.com/rust-lang/glob" +[dependencies.registry] +path = "src/registry" + [[bin]] name = "cargo" test = false diff --git a/src/bin/cargo.rs b/src/bin/cargo.rs index 238f7775a6b..f243689d34d 100644 --- a/src/bin/cargo.rs +++ b/src/bin/cargo.rs @@ -68,15 +68,17 @@ macro_rules! each_subcommand( ($macro:ident) => ({ $macro!(locate_project) $macro!(login) $macro!(new) + $macro!(owner) $macro!(package) $macro!(pkgid) + $macro!(publish) $macro!(read_manifest) $macro!(run) $macro!(test) $macro!(update) - $macro!(upload) $macro!(verify_project) $macro!(version) + $macro!(yank) }) ) /** diff --git a/src/bin/git_checkout.rs b/src/bin/git_checkout.rs index a903f28d722..a42cbbd03fa 100644 --- a/src/bin/git_checkout.rs +++ b/src/bin/git_checkout.rs @@ -30,7 +30,7 @@ pub fn execute(options: Options, shell: &mut MultiShell) -> CliResult }) .map_err(|e| CliError::from_boxed(e, 1))); - let source_id = SourceId::for_git(&url, reference.as_slice(), None); + let source_id = SourceId::for_git(&url, reference.as_slice()); let mut config = try!(Config::new(shell, None, None).map_err(|e| { CliError::from_boxed(e, 1) diff --git a/src/bin/login.rs b/src/bin/login.rs index e12905ecee9..ad4c4bbc48b 100644 --- a/src/bin/login.rs +++ b/src/bin/login.rs @@ -40,7 +40,7 @@ pub fn execute(options: Options, shell: &mut MultiShell) -> CliResult }; let token = token.as_slice().trim().to_string(); - try!(ops::upload_login(shell, token).map_err(|e| { + try!(ops::registry_login(shell, token).map_err(|e| { CliError::from_boxed(e, 101) })); Ok(None) diff --git a/src/bin/owner.rs b/src/bin/owner.rs new file mode 100644 index 00000000000..efa1d9ba7db --- /dev/null +++ b/src/bin/owner.rs @@ -0,0 +1,49 @@ +use cargo::ops; +use cargo::core::MultiShell; +use cargo::util::{CliResult, CliError}; +use cargo::util::important_paths::find_root_manifest_for_cwd; + +#[deriving(Decodable)] +struct Options { + arg_crate: Option, + flag_token: Option, + flag_add: Option>, + flag_remove: Option>, + flag_index: Option, + flag_verbose: bool, +} + +pub const USAGE: &'static str = " +Manage the owners of a crate on the registry + +Usage: + cargo owner [options] [] + +Options: + -h, --help Print this message + -a, --add LOGIN Login of a user to add as an owner + -r, --remove LOGIN Login of a user to remove as an owner + --index INDEX Registry index to modify owners for + --token TOKEN API token to use when authenticating + -v, --verbose Use verbose output + +This command will modify the owners for a package on the specified registry (or +default). Note that owners of a package can upload new versions, yank old +versions, and also modify the set of owners, so take caution! +"; + +pub fn execute(options: Options, shell: &mut MultiShell) -> CliResult> { + shell.set_verbose(options.flag_verbose); + let root = try!(find_root_manifest_for_cwd(None)); + try!(ops::modify_owners(&root, shell, + options.arg_crate, + options.flag_token, + options.flag_index, + options.flag_add, + options.flag_remove).map_err(|e| { + CliError::from_boxed(e, 101) + })); + Ok(None) +} + + diff --git a/src/bin/package.rs b/src/bin/package.rs index 704b4ed96d4..108d6b4500f 100644 --- a/src/bin/package.rs +++ b/src/bin/package.rs @@ -7,6 +7,7 @@ use cargo::util::important_paths::find_root_manifest_for_cwd; struct Options { flag_verbose: bool, flag_manifest_path: Option, + flag_no_verify: bool, } pub const USAGE: &'static str = " @@ -18,19 +19,15 @@ Usage: Options: -h, --help Print this message --manifest-path PATH Path to the manifest to compile + --no-verify Don't verify the contents by building them -v, --verbose Use verbose output "; pub fn execute(options: Options, shell: &mut MultiShell) -> CliResult> { shell.set_verbose(options.flag_verbose); - let Options { - flag_manifest_path, - .. - } = options; - - let root = try!(find_root_manifest_for_cwd(flag_manifest_path.clone())); - ops::package(&root, shell).map(|_| None).map_err(|err| { + let root = try!(find_root_manifest_for_cwd(options.flag_manifest_path)); + ops::package(&root, shell, !options.flag_no_verify).map(|_| None).map_err(|err| { CliError::from_boxed(err, 101) }) } diff --git a/src/bin/upload.rs b/src/bin/publish.rs similarity index 81% rename from src/bin/upload.rs rename to src/bin/publish.rs index b66a995f75a..54bc57d3271 100644 --- a/src/bin/upload.rs +++ b/src/bin/publish.rs @@ -9,18 +9,20 @@ struct Options { flag_token: Option, flag_manifest_path: Option, flag_verbose: bool, + flag_no_verify: bool, } pub const USAGE: &'static str = " Upload a package to the registry Usage: - cargo upload [options] + cargo publish [options] Options: -h, --help Print this message --host HOST Host to upload the package to --token TOKEN Token to use when uploading + --no-verify Don't verify package tarball before publish --manifest-path PATH Path to the manifest to compile -v, --verbose Use verbose output @@ -32,11 +34,12 @@ pub fn execute(options: Options, shell: &mut MultiShell) -> CliResult flag_token: token, flag_host: host, flag_manifest_path, + flag_no_verify: no_verify, .. } = options; let root = try!(find_root_manifest_for_cwd(flag_manifest_path.clone())); - ops::upload(&root, shell, token, host).map(|_| None).map_err(|err| { + ops::publish(&root, shell, token, host, !no_verify).map(|_| None).map_err(|err| { CliError::from_boxed(err, 101) }) } diff --git a/src/bin/yank.rs b/src/bin/yank.rs new file mode 100644 index 00000000000..f3dc4fd6525 --- /dev/null +++ b/src/bin/yank.rs @@ -0,0 +1,54 @@ +use cargo::ops; +use cargo::core::MultiShell; +use cargo::util::{CliResult, CliError}; +use cargo::util::important_paths::find_root_manifest_for_cwd; + +#[deriving(Decodable)] +struct Options { + arg_crate: Option, + flag_token: Option, + flag_vers: Option, + flag_index: Option, + flag_verbose: bool, + flag_undo: bool, +} + +pub static USAGE: &'static str = " +Remove a pushed crate from the index + +Usage: + cargo yank [options] [] + +Options: + -h, --help Print this message + --vers VERSION The version to yank or un-yank + --undo Undo a yank, putting a version back into the index + --index INDEX Registry index to yank from + --token TOKEN API token to use when authenticating + -v, --verbose Use verbose output + +The yank command removes a previously pushed crate's version from the server's +index. This command does not delete any data, and the crate will still be +available for download via the registry's download link. + +Note that existing crates locked to a yanked version will still be able to +download the yanked version to use it. Cargo will, however, not allow any new +crates to be locked to any yanked version. +"; + +pub fn execute(options: Options, shell: &mut MultiShell) -> CliResult> { + shell.set_verbose(options.flag_verbose); + let root = try!(find_root_manifest_for_cwd(None)); + try!(ops::yank(&root, shell, + options.arg_crate, + options.flag_vers, + options.flag_token, + options.flag_index, + options.flag_undo).map_err(|e| { + CliError::from_boxed(e, 101) + })); + Ok(None) +} + + + diff --git a/src/cargo/core/dependency.rs b/src/cargo/core/dependency.rs index 79485a5d5f3..f877cbadb37 100644 --- a/src/cargo/core/dependency.rs +++ b/src/cargo/core/dependency.rs @@ -1,5 +1,6 @@ -use core::{SourceId,Summary}; use semver::VersionReq; + +use core::{SourceId, Summary, PackageId}; use util::CargoResult; /// Informations about a dependency requested by a Cargo manifest. @@ -8,6 +9,7 @@ pub struct Dependency { name: String, source_id: SourceId, req: VersionReq, + specified_req: Option, transitive: bool, only_match_name: bool, @@ -31,14 +33,15 @@ impl Dependency { pub fn parse(name: &str, version: Option<&str>, source_id: &SourceId) -> CargoResult { - let version = match version { + let version_req = match version { Some(v) => try!(VersionReq::parse(v)), None => VersionReq::any() }; Ok(Dependency { only_match_name: false, - req: version, + req: version_req, + specified_req: version.map(|s| s.to_string()), .. Dependency::new_override(name, source_id) }) } @@ -53,6 +56,7 @@ impl Dependency { optional: false, features: Vec::new(), default_features: true, + specified_req: None, } } @@ -61,6 +65,10 @@ impl Dependency { &self.req } + pub fn get_specified_req(&self) -> Option<&str> { + self.specified_req.as_ref().map(|s| s.as_slice()) + } + pub fn get_name(&self) -> &str { self.name.as_slice() } @@ -93,6 +101,26 @@ impl Dependency { self } + /// Set the source id for this dependency + pub fn source_id(mut self, id: SourceId) -> Dependency { + self.source_id = id; + self + } + + /// Set the version requirement for this dependency + pub fn version_req(mut self, req: VersionReq) -> Dependency { + self.req = req; + self + } + + /// Lock this dependency to depending on the specified package id + pub fn lock_to(self, id: &PackageId) -> Dependency { + assert_eq!(self.source_id, *id.get_source_id()); + assert!(self.req.matches(id.get_version())); + self.version_req(VersionReq::exact(id.get_version())) + .source_id(id.get_source_id().clone()) + } + /// Returns false if the dependency is only used to build the local package. pub fn is_transitive(&self) -> bool { self.transitive } pub fn is_optional(&self) -> bool { self.optional } @@ -103,12 +131,14 @@ impl Dependency { /// Returns true if the package (`sum`) can fulfill this dependency request. pub fn matches(&self, sum: &Summary) -> bool { - debug!("matches; self={}; summary={}", self, sum); - debug!(" a={}; b={}", self.source_id, sum.get_source_id()); + self.matches_id(sum.get_package_id()) + } - self.name.as_slice() == sum.get_name() && - (self.only_match_name || (self.req.matches(sum.get_version()) && - &self.source_id == sum.get_source_id())) + /// Returns true if the package (`id`) can fulfill this dependency request. + pub fn matches_id(&self, id: &PackageId) -> bool { + self.name.as_slice() == id.get_name() && + (self.only_match_name || (self.req.matches(id.get_version()) && + &self.source_id == id.get_source_id())) } } diff --git a/src/cargo/core/manifest.rs b/src/cargo/core/manifest.rs index e274dbbc260..4ef140b0366 100644 --- a/src/cargo/core/manifest.rs +++ b/src/cargo/core/manifest.rs @@ -4,12 +4,7 @@ use std::fmt::{mod, Show, Formatter}; use semver::Version; use serialize::{Encoder,Encodable}; -use core::source::SourceId; -use core::{ - Dependency, - PackageId, - Summary -}; +use core::{Dependency, PackageId, Summary}; use core::package_id::Metadata; use core::dependency::SerializedDependency; use util::{CargoResult, human}; @@ -18,31 +13,49 @@ use util::{CargoResult, human}; #[deriving(PartialEq,Clone)] pub struct Manifest { summary: Summary, - authors: Vec, targets: Vec, target_dir: Path, doc_dir: Path, - sources: Vec, build: Vec, warnings: Vec, exclude: Vec, + metadata: ManifestMetadata, } impl Show for Manifest { fn fmt(&self, f: &mut Formatter) -> fmt::Result { - write!(f, "Manifest({}, authors={}, targets={}, target_dir={}, \ + write!(f, "Manifest({}, targets={}, target_dir={}, \ build={})", - self.summary, self.authors, self.targets, - self.target_dir.display(), self.build) + self.summary, self.targets, self.target_dir.display(), + self.build) } } +/// General metadata about a package which is just blindly uploaded to the +/// registry. +/// +/// Note that many of these fields can contain invalid values such as the +/// homepage, repository, documentation, or license. These fields are not +/// validated by cargo itself, but rather it is up to the registry when uploaded +/// to validate these fields. Cargo will itself accept any valid TOML +/// specification for these values. +#[deriving(PartialEq, Clone)] +pub struct ManifestMetadata { + pub authors: Vec, + pub keywords: Vec, + pub license: Option, + pub description: Option, // not markdown + pub readme: Option, // file, not contents + pub homepage: Option, // url + pub repository: Option, // url + pub documentation: Option, // url +} + #[deriving(PartialEq,Clone,Encodable)] pub struct SerializedManifest { name: String, version: String, dependencies: Vec, - authors: Vec, targets: Vec, target_dir: String, doc_dir: String, @@ -57,7 +70,6 @@ impl> Encodable for Manifest { dependencies: self.summary.get_dependencies().iter().map(|d| { SerializedDependency::from_dependency(d) }).collect(), - authors: self.authors.clone(), targets: self.targets.clone(), target_dir: self.target_dir.display().to_string(), doc_dir: self.doc_dir.display().to_string(), @@ -351,18 +363,18 @@ impl Show for Target { impl Manifest { pub fn new(summary: Summary, targets: Vec, - target_dir: Path, doc_dir: Path, sources: Vec, - build: Vec, exclude: Vec) -> Manifest { + target_dir: Path, doc_dir: Path, + build: Vec, exclude: Vec, + metadata: ManifestMetadata) -> Manifest { Manifest { summary: summary, - authors: Vec::new(), targets: targets, target_dir: target_dir, doc_dir: doc_dir, - sources: sources, build: build, warnings: Vec::new(), exclude: exclude, + metadata: metadata, } } @@ -382,10 +394,6 @@ impl Manifest { self.get_summary().get_package_id().get_version() } - pub fn get_authors(&self) -> &[String] { - self.authors.as_slice() - } - pub fn get_dependencies(&self) -> &[Dependency] { self.get_summary().get_dependencies() } @@ -402,11 +410,6 @@ impl Manifest { &self.doc_dir } - /// Returns a list of all the potential `SourceId`s of the dependencies. - pub fn get_source_ids(&self) -> &[SourceId] { - self.sources.as_slice() - } - pub fn get_build(&self) -> &[String] { self.build.as_slice() } @@ -422,6 +425,12 @@ impl Manifest { pub fn get_exclude(&self) -> &[String] { self.exclude.as_slice() } + + pub fn get_metadata(&self) -> &ManifestMetadata { &self.metadata } + + pub fn set_summary(&mut self, summary: Summary) { + self.summary = summary; + } } impl Target { diff --git a/src/cargo/core/mod.rs b/src/cargo/core/mod.rs index 50d86caa4d7..f7ffee33701 100644 --- a/src/cargo/core/mod.rs +++ b/src/cargo/core/mod.rs @@ -6,8 +6,7 @@ pub use self::package_id_spec::PackageIdSpec; pub use self::registry::Registry; pub use self::resolver::Resolve; pub use self::shell::{Shell, MultiShell, ShellConfig}; -pub use self::source::{PathKind, RegistryKind}; -pub use self::source::{Source, SourceId, SourceMap, SourceSet, GitKind}; +pub use self::source::{Source, SourceId, SourceMap, SourceSet}; pub use self::summary::Summary; pub mod source; diff --git a/src/cargo/core/package.rs b/src/cargo/core/package.rs index f0e65e7d889..fa84b9c3bf8 100644 --- a/src/cargo/core/package.rs +++ b/src/cargo/core/package.rs @@ -34,7 +34,6 @@ struct SerializedPackage { name: String, version: String, dependencies: Vec, - authors: Vec, targets: Vec, manifest_path: String, } @@ -51,7 +50,6 @@ impl> Encodable for Package { dependencies: summary.get_dependencies().iter().map(|d| { SerializedDependency::from_dependency(d) }).collect(), - authors: manifest.get_authors().to_vec(), targets: manifest.get_targets().to_vec(), manifest_path: self.manifest_path.display().to_string() }.encode(s) @@ -112,12 +110,6 @@ impl Package { pub fn get_absolute_target_dir(&self) -> Path { self.get_root().join(self.get_target_dir()) } - - pub fn get_source_ids(&self) -> Vec { - let mut ret = vec!(self.source_id.clone()); - ret.push_all(self.manifest.get_source_ids()); - ret - } } impl Show for Package { diff --git a/src/cargo/core/package_id.rs b/src/cargo/core/package_id.rs index 1442a781563..f9ee93b4030 100644 --- a/src/cargo/core/package_id.rs +++ b/src/cargo/core/package_id.rs @@ -159,13 +159,13 @@ impl Show for PackageId { #[cfg(test)] mod tests { use super::{PackageId, CENTRAL_REPO}; - use core::source::{RegistryKind, SourceId}; + use core::source::SourceId; use util::ToUrl; #[test] fn invalid_version_handled_nicely() { let loc = CENTRAL_REPO.to_url().unwrap(); - let repo = SourceId::new(RegistryKind, loc); + let repo = SourceId::for_registry(&loc); assert!(PackageId::new("foo", "1.0", &repo).is_err()); assert!(PackageId::new("foo", "1", &repo).is_err()); diff --git a/src/cargo/core/registry.rs b/src/cargo/core/registry.rs index 007cf4b7625..ad32d1448ba 100644 --- a/src/cargo/core/registry.rs +++ b/src/cargo/core/registry.rs @@ -1,4 +1,4 @@ -use std::collections::HashSet; +use std::collections::hashmap::{HashSet, HashMap, Occupied, Vacant}; use core::{Source, SourceId, SourceMap, Summary, Dependency, PackageId, Package}; use util::{CargoResult, ChainError, Config, human, profile}; @@ -21,18 +21,62 @@ impl Registry for Vec { } } +/// This structure represents a registry of known packages. It internally +/// contains a number of `Box` instances which are used to load a +/// `Package` from. +/// +/// The resolution phase of Cargo uses this to drive knowledge about new +/// packages as well as querying for lists of new packages. It is here that +/// sources are updated and (e.g. network operations) as well as overrides are +/// handled. +/// +/// The general idea behind this registry is that it is centered around the +/// `SourceMap` structure contained within which is a mapping of a `SourceId` to +/// a `Source`. Each `Source` in the map has been updated (using network +/// operations if necessary) and is ready to be queried for packages. pub struct PackageRegistry<'a> { sources: SourceMap<'a>, + config: &'a mut Config<'a>, + + // A list of sources which are considered "overrides" which take precedent + // when querying for packages. overrides: Vec, - config: &'a mut Config<'a> + + // Note that each SourceId does not take into account its `precise` field + // when hashing or testing for equality. When adding a new `SourceId`, we + // want to avoid duplicates in the `SourceMap` (to prevent re-updating the + // same git repo twice for example), but we also want to ensure that the + // loaded source is always updated. + // + // Sources with a `precise` field normally don't need to be updated because + // their contents are already on disk, but sources without a `precise` field + // almost always need to be updated. If we have a cached `Source` for a + // precise `SourceId`, then when we add a new `SourceId` that is not precise + // we want to ensure that the underlying source is updated. + // + // This is basically a long-winded way of saying that we want to know + // precisely what the keys of `sources` are, so this is a mapping of key to + // what exactly the key is. + source_ids: HashMap, + + locked: HashMap)>>>, +} + +#[deriving(PartialEq, Eq)] +enum Kind { + Override, + Locked, + Normal, } impl<'a> PackageRegistry<'a> { pub fn new<'a>(config: &'a mut Config<'a>) -> PackageRegistry<'a> { PackageRegistry { sources: SourceMap::new(), + source_ids: HashMap::new(), overrides: vec!(), - config: config + config: config, + locked: HashMap::new(), } } @@ -64,27 +108,60 @@ impl<'a> PackageRegistry<'a> { } fn ensure_loaded(&mut self, namespace: &SourceId) -> CargoResult<()> { - if self.sources.contains(namespace) { return Ok(()); } + match self.source_ids.find(namespace) { + // We've previously loaded this source, and we've already locked it, + // so we're not allowed to change it even if `namespace` has a + // slightly different precise version listed. + Some(&(_, Locked)) => return Ok(()), + + // If the previous source was not a precise source, then we can be + // sure that it's already been updated if we've already loaded it. + Some(&(ref previous, _)) if previous.get_precise().is_none() => { + return Ok(()) + } + + // If the previous source has the same precise version as we do, + // then we're done, otherwise we need to need to move forward + // updating this source. + Some(&(ref previous, _)) => { + if previous.get_precise() == namespace.get_precise() { + return Ok(()) + } + } + None => {} + } - try!(self.load(namespace, false)); + try!(self.load(namespace, Normal)); Ok(()) } - pub fn add_sources(&mut self, ids: Vec) -> CargoResult<()> { - for id in dedup(ids).iter() { - try!(self.load(id, false)); + pub fn add_sources(&mut self, ids: &[SourceId]) -> CargoResult<()> { + for id in ids.iter() { + try!(self.load(id, Locked)); } Ok(()) } pub fn add_overrides(&mut self, ids: Vec) -> CargoResult<()> { for id in ids.iter() { - try!(self.load(id, true)); + try!(self.load(id, Override)); } Ok(()) } - fn load(&mut self, source_id: &SourceId, is_override: bool) -> CargoResult<()> { + pub fn register_lock(&mut self, id: PackageId, deps: Vec) { + let sub_map = match self.locked.entry(id.get_source_id().clone()) { + Occupied(e) => e.into_mut(), + Vacant(e) => e.set(HashMap::new()), + }; + let sub_vec = match sub_map.entry(id.get_name().to_string()) { + Occupied(e) => e.into_mut(), + Vacant(e) => e.set(Vec::new()), + }; + sub_vec.push((id, deps)); + } + + fn load(&mut self, source_id: &SourceId, kind: Kind) -> CargoResult<()> { (|| { let mut source = source_id.load(self.config); @@ -93,12 +170,13 @@ impl<'a> PackageRegistry<'a> { try!(source.update()); drop(p); - if is_override { + if kind == Override { self.overrides.push(source_id.clone()); } // Save off the source self.sources.insert(source_id, source); + self.source_ids.insert(source_id.clone(), (source_id.clone(), kind)); Ok(()) }).chain_error(|| human(format!("Unable to update {}", source_id))) @@ -117,34 +195,86 @@ impl<'a> PackageRegistry<'a> { } Ok(ret) } -} - -fn dedup(ids: Vec) -> Vec { - let mut seen = vec!(); - for id in ids.into_iter() { - if seen.contains(&id) { continue; } - seen.push(id); + // This function is used to transform a summary to another locked summary if + // possible. This is where the the concept of a lockfile comes into play. + // + // If a summary points at a package id which was previously locked, then we + // override the summary's id itself as well as all dependencies to be + // rewritten to the locked versions. This will transform the summary's + // source to a precise source (listed in the locked version) as well as + // transforming all of the dependencies from range requirements on imprecise + // sources to exact requirements on precise sources. + // + // If a summary does not point at a package id which was previously locked, + // we still want to avoid updating as many dependencies as possible to keep + // the graph stable. In this case we map all of the summary's dependencies + // to be rewritten to a locked version wherever possible. If we're unable to + // map a dependency though, we just pass it on through. + fn lock(&self, summary: Summary) -> Summary { + let pair = self.locked.find(summary.get_source_id()).and_then(|map| { + map.find_equiv(&summary.get_name()) + }).and_then(|vec| { + vec.iter().find(|&&(ref id, _)| id == summary.get_package_id()) + }); + + // Lock the summary's id if possible + let summary = match pair { + Some(&(ref precise, _)) => summary.override_id(precise.clone()), + None => summary, + }; + summary.map_dependencies(|dep| { + match pair { + // If this summary has a locked version, then we need to lock + // this dependency. If this dependency doesn't have a locked + // version, then it was likely an optional dependency which + // wasn't included and we just pass it through anyway. + Some(&(_, ref deps)) => { + match deps.iter().find(|d| d.get_name() == dep.get_name()) { + Some(lock) => dep.lock_to(lock), + None => dep, + } + } + + // If this summary did not have a locked version, then we query + // all known locked packages to see if they match this + // dependency. If anything does then we lock it to that and move + // on. + None => { + let v = self.locked.find(dep.get_source_id()).and_then(|map| { + map.find_equiv(&dep.get_name()) + }).and_then(|vec| { + vec.iter().find(|&&(ref id, _)| dep.matches_id(id)) + }); + match v { + Some(&(ref id, _)) => dep.lock_to(id), + None => dep + } + } + } + }) } - - seen } impl<'a> Registry for PackageRegistry<'a> { fn query(&mut self, dep: &Dependency) -> CargoResult> { let overrides = try!(self.query_overrides(dep)); - if overrides.len() == 0 { + let ret = if overrides.len() == 0 { // Ensure the requested source_id is loaded try!(self.ensure_loaded(dep.get_source_id())); let mut ret = Vec::new(); for src in self.sources.sources_mut() { ret.extend(try!(src.query(dep)).into_iter()); } - Ok(ret) + ret } else { - Ok(overrides) - } + overrides + }; + + // post-process all returned summaries to ensure that we lock all + // relevant summaries to the right versions and sources + Ok(ret.into_iter().map(|summary| self.lock(summary)).collect()) } } diff --git a/src/cargo/core/source.rs b/src/cargo/core/source.rs index 3740ca3eaf2..9d3edf68ad5 100644 --- a/src/cargo/core/source.rs +++ b/src/cargo/core/source.rs @@ -46,13 +46,13 @@ pub trait Source: Registry { } #[deriving(Encodable, Decodable, Show, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub enum SourceKind { +enum SourceKind { /// GitKind() represents a git repository GitKind(String), /// represents a local path PathKind, /// represents the central registry - RegistryKind + RegistryKind, } type Error = Box; @@ -72,7 +72,7 @@ struct SourceIdInner { } impl SourceId { - pub fn new(kind: SourceKind, url: Url) -> SourceId { + fn new(kind: SourceKind, url: Url) -> SourceId { SourceId { inner: Arc::new(SourceIdInner { kind: kind, @@ -109,11 +109,13 @@ impl SourceId { } url.query = None; let precise = mem::replace(&mut url.fragment, None); - SourceId::for_git(&url, reference.as_slice(), precise) + SourceId::for_git(&url, reference.as_slice()) + .with_precise(precise) }, "registry" => { let url = url.to_url().unwrap(); - SourceId::for_registry(&url) + SourceId::new(RegistryKind, url) + .with_precise(Some("locked".to_string())) } "path" => SourceId::for_path(&Path::new(url.slice_from(5))).unwrap(), _ => fail!("Unsupported serialized SourceId") @@ -143,9 +145,8 @@ impl SourceId { format!("git+{}{}{}", url, ref_str, precise_str) }, - SourceIdInner { kind: RegistryKind, .. } => { - // TODO: Central registry vs. alternates - "registry+https://crates.io/".to_string() + SourceIdInner { kind: RegistryKind, ref url, .. } => { + format!("registry+{}", url) } } } @@ -156,13 +157,8 @@ impl SourceId { Ok(SourceId::new(PathKind, url)) } - pub fn for_git(url: &Url, reference: &str, precise: Option) -> SourceId { - let mut id = SourceId::new(GitKind(reference.to_string()), url.clone()); - if precise.is_some() { - id = id.with_precise(precise.unwrap()); - } - - id + pub fn for_git(url: &Url, reference: &str) -> SourceId { + SourceId::new(GitKind(reference.to_string()), url.clone()) } pub fn for_registry(url: &Url) -> SourceId { @@ -177,17 +173,9 @@ impl SourceId { Ok(SourceId::for_registry(&try!(RegistrySource::url()))) } - pub fn get_url(&self) -> &Url { - &self.inner.url - } - - pub fn get_kind(&self) -> &SourceKind { - &self.inner.kind - } - - pub fn is_path(&self) -> bool { - self.inner.kind == PathKind - } + pub fn get_url(&self) -> &Url { &self.inner.url } + pub fn is_path(&self) -> bool { self.inner.kind == PathKind } + pub fn is_registry(&self) -> bool { self.inner.kind == RegistryKind } pub fn is_git(&self) -> bool { match self.inner.kind { @@ -208,7 +196,9 @@ impl SourceId { }; box PathSource::new(&path, self) as Box }, - RegistryKind => box RegistrySource::new(self, config) as Box, + RegistryKind => { + box RegistrySource::new(self, config) as Box + } } } @@ -216,10 +206,17 @@ impl SourceId { self.inner.precise.as_ref().map(|s| s.as_slice()) } - pub fn with_precise(&self, v: String) -> SourceId { + pub fn git_reference(&self) -> Option<&str> { + match self.inner.kind { + GitKind(ref s) => Some(s.as_slice()), + _ => None, + } + } + + pub fn with_precise(&self, v: Option) -> SourceId { SourceId { inner: Arc::new(SourceIdInner { - precise: Some(v), + precise: v, .. (*self.inner).clone() }), } diff --git a/src/cargo/core/summary.rs b/src/cargo/core/summary.rs index 04abb7eb835..12737fc6d96 100644 --- a/src/cargo/core/summary.rs +++ b/src/cargo/core/summary.rs @@ -1,11 +1,13 @@ use std::collections::HashMap; +use std::mem; use semver::Version; use core::{Dependency, PackageId, SourceId}; use util::{CargoResult, human}; -/// Subset of a `Manifest`. Contains only the most important informations about a package. +/// Subset of a `Manifest`. Contains only the most important informations about +/// a package. /// /// Summaries are cloned, and should not be mutated after creation #[deriving(Show,Clone)] @@ -89,6 +91,17 @@ impl Summary { pub fn get_features(&self) -> &HashMap> { &self.features } + + pub fn override_id(mut self, id: PackageId) -> Summary { + self.package_id = id; + self + } + + pub fn map_dependencies(mut self, f: |Dependency| -> Dependency) -> Summary { + let deps = mem::replace(&mut self.dependencies, Vec::new()); + self.dependencies = deps.into_iter().map(f).collect(); + self + } } impl PartialEq for Summary { diff --git a/src/cargo/lib.rs b/src/cargo/lib.rs index 15298150c4a..17a84c7b323 100644 --- a/src/cargo/lib.rs +++ b/src/cargo/lib.rs @@ -24,6 +24,8 @@ extern crate toml; extern crate url; #[cfg(test)] extern crate hamcrest; +extern crate registry; + use std::os; use std::io::stdio::{stdout_raw, stderr_raw}; use std::io::{mod, stdout, stderr}; diff --git a/src/cargo/ops/cargo_compile.rs b/src/cargo/ops/cargo_compile.rs index 438947a4bb4..79ae34f559c 100644 --- a/src/cargo/ops/cargo_compile.rs +++ b/src/cargo/ops/cargo_compile.rs @@ -26,7 +26,7 @@ use std::os; use std::collections::HashMap; use core::registry::PackageRegistry; -use core::{MultiShell, Source, SourceId, PackageSet, Target, PackageId}; +use core::{MultiShell, Source, SourceId, PackageSet, Package, Target, PackageId}; use core::resolver; use ops; use sources::{PathSource}; @@ -51,20 +51,8 @@ pub struct CompileOptions<'a> { pub fn compile(manifest_path: &Path, options: &mut CompileOptions) -> CargoResult { - let CompileOptions { env, ref mut shell, jobs, target, spec, - dev_deps, features, no_default_features } = *options; - let target = target.map(|s| s.to_string()); - let features = features.iter().flat_map(|s| { - s.as_slice().split(' ') - }).map(|s| s.to_string()).collect::>(); - log!(4, "compile; manifest-path={}", manifest_path.display()); - if spec.is_some() && (no_default_features || features.len() > 0) { - return Err(human("features cannot be modified when the main package \ - is not being built")) - } - let mut source = try!(PathSource::for_path(&manifest_path.dir_path())); try!(source.update()); @@ -73,12 +61,28 @@ pub fn compile(manifest_path: &Path, debug!("loaded package; package={}", package); for key in package.get_manifest().get_warnings().iter() { - try!(shell.warn(key)) + try!(options.shell.warn(key)) + } + compile_pkg(&package, options) +} + +pub fn compile_pkg(package: &Package, options: &mut CompileOptions) + -> CargoResult { + let CompileOptions { env, ref mut shell, jobs, target, spec, + dev_deps, features, no_default_features } = *options; + let target = target.map(|s| s.to_string()); + let features = features.iter().flat_map(|s| { + s.as_slice().split(' ') + }).map(|s| s.to_string()).collect::>(); + + if spec.is_some() && (no_default_features || features.len() > 0) { + return Err(human("features cannot be modified when the main package \ + is not being built")) } let user_configs = try!(config::all_configs(os::getcwd())); let override_ids = try!(source_ids_from_config(&user_configs, - manifest_path.dir_path())); + package.get_root())); let (packages, resolve_with_overrides, sources) = { let mut config = try!(Config::new(*shell, jobs, target.clone())); @@ -86,7 +90,7 @@ pub fn compile(manifest_path: &Path, // First, resolve the package's *listed* dependencies, as well as // downloading and updating all remotes and such. - try!(ops::resolve_and_fetch(&mut registry, &package)); + let resolve = try!(ops::resolve_pkg(&mut registry, package)); // Second, resolve with precisely what we're doing. Filter out // transitive dependencies if necessary, specify features, handle @@ -97,8 +101,8 @@ pub fn compile(manifest_path: &Path, let method = resolver::ResolveRequired(dev_deps, features.as_slice(), !no_default_features); let resolved_with_overrides = - try!(resolver::resolve(package.get_summary(), method, - &mut registry)); + try!(ops::resolve_with_previous(&mut registry, package, method, + Some(&resolve), None)); let req: Vec = resolved_with_overrides.iter().map(|r| { r.clone() @@ -117,7 +121,7 @@ pub fn compile(manifest_path: &Path, let pkgid = try!(resolve_with_overrides.query(spec)); packages.iter().find(|p| p.get_package_id() == pkgid).unwrap() } - None => &package, + None => package, }; let targets = to_build.get_targets().iter().filter(|target| { diff --git a/src/cargo/ops/cargo_fetch.rs b/src/cargo/ops/cargo_fetch.rs index 3f04af2951c..4c4cb0ebffe 100644 --- a/src/cargo/ops/cargo_fetch.rs +++ b/src/cargo/ops/cargo_fetch.rs @@ -1,13 +1,9 @@ -use std::collections::{HashSet, HashMap}; - -use core::{MultiShell, Package, PackageId, Summary}; +use core::MultiShell; use core::registry::PackageRegistry; -use core::resolver::{mod, Resolve}; use core::source::Source; use ops; use sources::PathSource; use util::{CargoResult, Config}; -use util::profile; /// Executes `cargo fetch`. pub fn fetch(manifest_path: &Path, @@ -18,94 +14,6 @@ pub fn fetch(manifest_path: &Path, let mut config = try!(Config::new(shell, None, None)); let mut registry = PackageRegistry::new(&mut config); - try!(resolve_and_fetch(&mut registry, &package)); + try!(ops::resolve_pkg(&mut registry, &package)); Ok(()) } - -/// Finds all the packages required to compile the specified `Package`, -/// and loads them in the `PackageRegistry`. -/// -/// Also write the `Cargo.lock` file with the results. -pub fn resolve_and_fetch(registry: &mut PackageRegistry, package: &Package) - -> CargoResult { - let _p = profile::start("resolve and fetch..."); - - let lockfile = package.get_manifest_path().dir_path().join("Cargo.lock"); - let source_id = package.get_package_id().get_source_id(); - let previous_resolve = try!(ops::load_lockfile(&lockfile, source_id)); - match previous_resolve { - Some(ref r) => try!(add_lockfile_sources(registry, package, r)), - None => try!(registry.add_sources(package.get_source_ids())), - } - - let mut resolved = try!(resolver::resolve(package.get_summary(), - resolver::ResolveEverything, - registry)); - match previous_resolve { - Some(ref prev) => resolved.copy_metadata(prev), - None => {} - } - try!(ops::write_resolve(package, &resolved)); - Ok(resolved) -} - -/// When a lockfile is present, we want to keep as many dependencies at their -/// original revision as possible. We need to account, however, for -/// modifications to the manifest in terms of modifying, adding, or deleting -/// dependencies. -/// -/// This method will add any appropriate sources from the lockfile into the -/// registry, and add all other sources from the root package to the registry. -/// Any dependency which has not been modified has its source added to the -/// registry (to retain the precise field if possible). Any dependency which -/// *has* changed has its source id listed in the manifest added and all of its -/// transitive dependencies are blacklisted to not be added from the lockfile. -/// -/// TODO: this won't work too well for registry-based packages, but we don't -/// have many of those anyway so we should be ok for now. -fn add_lockfile_sources(registry: &mut PackageRegistry, - root: &Package, - resolve: &Resolve) -> CargoResult<()> { - let deps = resolve.deps(root.get_package_id()).into_iter().flat_map(|deps| { - deps.map(|d| (d.get_name(), d)) - }).collect::>(); - - let mut sources = vec![root.get_package_id().get_source_id().clone()]; - let mut to_avoid = HashSet::new(); - let mut to_add = HashSet::new(); - for dep in root.get_dependencies().iter() { - match deps.find(&dep.get_name()) { - Some(&lockfile_dep) => { - let summary = Summary::new(lockfile_dep.clone(), Vec::new(), - HashMap::new()).unwrap(); - if dep.matches(&summary) { - fill_with_deps(resolve, lockfile_dep, &mut to_add); - } else { - fill_with_deps(resolve, lockfile_dep, &mut to_avoid); - sources.push(dep.get_source_id().clone()); - } - } - None => sources.push(dep.get_source_id().clone()), - } - } - - // Only afterward once we know the entire blacklist are the lockfile - // sources added. - for addition in to_add.iter() { - if !to_avoid.contains(addition) { - sources.push(addition.get_source_id().clone()); - } - } - - return registry.add_sources(sources); - - fn fill_with_deps<'a>(resolve: &'a Resolve, dep: &'a PackageId, - set: &mut HashSet<&'a PackageId>) { - if !set.insert(dep) { return } - for mut deps in resolve.deps(dep).into_iter() { - for dep in deps { - fill_with_deps(resolve, dep, set); - } - } - } -} diff --git a/src/cargo/ops/cargo_generate_lockfile.rs b/src/cargo/ops/cargo_generate_lockfile.rs index 43be2d6e820..059e9064c16 100644 --- a/src/cargo/ops/cargo_generate_lockfile.rs +++ b/src/cargo/ops/cargo_generate_lockfile.rs @@ -1,16 +1,12 @@ use std::collections::HashSet; -use std::io::File; -use serialize::{Encodable, Decodable}; -use toml::{mod, Encoder}; - -use core::registry::PackageRegistry; -use core::{MultiShell, Source, Resolve, resolver, Package, SourceId}; use core::PackageId; +use core::registry::PackageRegistry; +use core::{MultiShell, Source, Resolve, resolver}; +use ops; use sources::{PathSource}; use util::config::{Config}; use util::{CargoResult, human}; -use util::toml as cargo_toml; pub struct UpdateOptions<'a> { pub shell: &'a mut MultiShell<'a>, @@ -25,17 +21,12 @@ pub fn generate_lockfile(manifest_path: &Path, let mut source = try!(PathSource::for_path(&manifest_path.dir_path())); try!(source.update()); let package = try!(source.get_root_package()); - let source_ids = package.get_source_ids(); let mut config = try!(Config::new(shell, None, None)); - - let resolve = { - let mut registry = PackageRegistry::new(&mut config); - try!(registry.add_sources(source_ids)); - try!(resolver::resolve(package.get_summary(), - resolver::ResolveEverything, - &mut registry)) - }; - try!(write_resolve(&package, &resolve)); + let mut registry = PackageRegistry::new(&mut config); + let resolve = try!(ops::resolve_with_previous(&mut registry, &package, + resolver::ResolveEverything, + None, None)); + try!(ops::write_pkg_lockfile(&package, &resolve)); Ok(()) } @@ -45,9 +36,7 @@ pub fn update_lockfile(manifest_path: &Path, try!(source.update()); let package = try!(source.get_root_package()); - let lockfile = package.get_root().join("Cargo.lock"); - let source_id = package.get_package_id().get_source_id(); - let previous_resolve = match try!(load_lockfile(&lockfile, source_id)) { + let previous_resolve = match try!(ops::load_pkg_lockfile(&package)) { Some(resolve) => resolve, None => return Err(human("A Cargo.lock must exist before it is updated")) }; @@ -59,46 +48,42 @@ pub fn update_lockfile(manifest_path: &Path, let mut config = try!(Config::new(opts.shell, None, None)); let mut registry = PackageRegistry::new(&mut config); + let mut to_avoid = HashSet::new(); - let mut sources = Vec::new(); match opts.to_update { Some(name) => { - let mut to_avoid = HashSet::new(); let dep = try!(previous_resolve.query(name)); if opts.aggressive { fill_with_deps(&previous_resolve, dep, &mut to_avoid, &mut HashSet::new()); } else { - to_avoid.insert(dep.get_source_id()); + to_avoid.insert(dep); match opts.precise { Some(precise) => { - sources.push(dep.get_source_id().clone() - .with_precise(precise.to_string())); + let precise = dep.get_source_id().clone() + .with_precise(Some(precise.to_string())); + try!(registry.add_sources(&[precise])); } None => {} } } - sources.extend(previous_resolve.iter() - .map(|p| p.get_source_id()) - .filter(|s| !to_avoid.contains(s)) - .map(|s| s.clone())); } - None => sources.extend(package.get_source_ids().into_iter()), + None => to_avoid.extend(previous_resolve.iter()), } - try!(registry.add_sources(sources)); - let mut resolve = try!(resolver::resolve(package.get_summary(), - resolver::ResolveEverything, - &mut registry)); - resolve.copy_metadata(&previous_resolve); - try!(write_resolve(&package, &resolve)); + let resolve = try!(ops::resolve_with_previous(&mut registry, + &package, + resolver::ResolveEverything, + Some(&previous_resolve), + Some(&to_avoid))); + try!(ops::write_pkg_lockfile(&package, &resolve)); return Ok(()); fn fill_with_deps<'a>(resolve: &'a Resolve, dep: &'a PackageId, - set: &mut HashSet<&'a SourceId>, + set: &mut HashSet<&'a PackageId>, visited: &mut HashSet<&'a PackageId>) { if !visited.insert(dep) { return } - set.insert(dep.get_source_id()); + set.insert(dep); match resolve.deps(dep) { Some(mut deps) => { for dep in deps { @@ -109,82 +94,3 @@ pub fn update_lockfile(manifest_path: &Path, } } } - -pub fn load_lockfile(path: &Path, sid: &SourceId) -> CargoResult> { - // If there is no lockfile, return none. - let mut f = match File::open(path) { - Ok(f) => f, - Err(_) => return Ok(None) - }; - - let s = try!(f.read_to_string()); - - let table = toml::Table(try!(cargo_toml::parse(s.as_slice(), path))); - let mut d = toml::Decoder::new(table); - let v: resolver::EncodableResolve = Decodable::decode(&mut d).unwrap(); - Ok(Some(try!(v.to_resolve(sid)))) -} - -pub fn write_resolve(pkg: &Package, resolve: &Resolve) -> CargoResult<()> { - let loc = pkg.get_root().join("Cargo.lock"); - - let mut e = Encoder::new(); - resolve.encode(&mut e).unwrap(); - - let mut out = String::new(); - - // Note that we do not use e.toml.to_string() as we want to control the - // exact format the toml is in to ensure pretty diffs between updates to the - // lockfile. - let root = e.toml.find(&"root".to_string()).unwrap(); - - out.push_str("[root]\n"); - emit_package(root.as_table().unwrap(), &mut out); - - let deps = e.toml.find(&"package".to_string()).unwrap().as_slice().unwrap(); - for dep in deps.iter() { - let dep = dep.as_table().unwrap(); - - out.push_str("[[package]]\n"); - emit_package(dep, &mut out); - } - - match e.toml.find(&"metadata".to_string()) { - Some(metadata) => { - out.push_str("[metadata]\n"); - out.push_str(metadata.to_string().as_slice()); - } - None => {} - } - - try!(File::create(&loc).write_str(out.as_slice())); - Ok(()) -} - -fn emit_package(dep: &toml::TomlTable, out: &mut String) { - out.push_str(format!("name = {}\n", lookup(dep, "name")).as_slice()); - out.push_str(format!("version = {}\n", lookup(dep, "version")).as_slice()); - - if dep.contains_key(&"source".to_string()) { - out.push_str(format!("source = {}\n", lookup(dep, "source")).as_slice()); - } - - if let Some(ref s) = dep.find(&"dependencies".to_string()) { - let slice = s.as_slice().unwrap(); - - if !slice.is_empty() { - out.push_str("dependencies = [\n"); - - for child in s.as_slice().unwrap().iter() { - out.push_str(format!(" {},\n", child).as_slice()); - } - - out.push_str("]\n"); - } - out.push_str("\n"); - } -} - -fn lookup<'a>(table: &'a toml::TomlTable, key: &str) -> &'a toml::Value { - table.find(&key.to_string()).expect(format!("Didn't find {}", key).as_slice()) -} diff --git a/src/cargo/ops/cargo_package.rs b/src/cargo/ops/cargo_package.rs index 40b3ddcf74f..99444fdabb4 100644 --- a/src/cargo/ops/cargo_package.rs +++ b/src/cargo/ops/cargo_package.rs @@ -1,17 +1,31 @@ -use std::io::File; +use std::io::{fs, File}; use std::io::fs::PathExtensions; use std::path; use tar::Archive; use flate2::{GzBuilder, BestCompression}; +use flate2::reader::GzDecoder; -use core::source::Source; +use core::source::{Source, SourceId}; use core::{Package, MultiShell}; use sources::PathSource; use util::{CargoResult, human, internal, ChainError, Require}; +use ops; + +struct Bomb { path: Option } + +impl Drop for Bomb { + fn drop(&mut self) { + match self.path.as_ref() { + Some(path) => { let _ = fs::unlink(path); } + None => {} + } + } +} pub fn package(manifest_path: &Path, - shell: &mut MultiShell) -> CargoResult { + shell: &mut MultiShell, + verify: bool) -> CargoResult { let mut src = try!(PathSource::for_path(&manifest_path.dir_path())); try!(src.update()); let pkg = try!(src.get_root_package()); @@ -20,11 +34,18 @@ pub fn package(manifest_path: &Path, let dst = pkg.get_manifest_path().dir_path().join(filename); if dst.exists() { return Ok(dst) } + let mut bomb = Bomb { path: Some(dst.clone()) }; + try!(shell.status("Packaging", pkg.get_package_id().to_string())); try!(tar(&pkg, &src, shell, &dst).chain_error(|| { human("failed to prepare local package for uploading") })); - Ok(dst) + if verify { + try!(run_verify(&pkg, shell, &dst).chain_error(|| { + human("failed to verify package tarball") + })) + } + Ok(bomb.path.take().unwrap()) } fn tar(pkg: &Package, src: &PathSource, shell: &mut MultiShell, @@ -62,3 +83,46 @@ fn tar(pkg: &Package, src: &PathSource, shell: &mut MultiShell, try!(ar.finish()); Ok(()) } + +fn run_verify(pkg: &Package, shell: &mut MultiShell, tar: &Path) + -> CargoResult<()> { + try!(shell.status("Verifying", pkg)); + + let f = try!(GzDecoder::new(try!(File::open(tar)))); + let dst = pkg.get_root().join("target/package"); + if dst.exists() { + try!(fs::rmdir_recursive(&dst)); + } + let mut archive = Archive::new(f); + try!(archive.unpack(&dst)); + let manifest_path = dst.join(format!("{}-{}/Cargo.toml", pkg.get_name(), + pkg.get_version())); + + // When packages are uploaded to the registry, all path dependencies are + // implicitly converted to registry-based dependencies, so we rewrite those + // dependencies here. + let registry = try!(SourceId::for_central()); + let new_summary = pkg.get_summary().clone().map_dependencies(|d| { + if !d.get_source_id().is_path() { return d } + d.source_id(registry.clone()) + }); + let mut new_manifest = pkg.get_manifest().clone(); + new_manifest.set_summary(new_summary); + let new_pkg = Package::new(new_manifest, + &manifest_path, + pkg.get_package_id().get_source_id()); + + // Now that we've rewritten all our path dependencies, compile it! + try!(ops::compile_pkg(&new_pkg, &mut ops::CompileOptions { + env: "compile", + shell: shell, + jobs: None, + target: None, + dev_deps: false, + features: [], + no_default_features: false, + spec: None, + })); + + Ok(()) +} diff --git a/src/cargo/ops/cargo_read_manifest.rs b/src/cargo/ops/cargo_read_manifest.rs index 1b73726829c..bdaa89b8b10 100644 --- a/src/cargo/ops/cargo_read_manifest.rs +++ b/src/cargo/ops/cargo_read_manifest.rs @@ -102,10 +102,14 @@ fn read_nested_packages(path: &Path, source_id: &SourceId, let (pkg, nested) = try!(read_package(&manifest, source_id)); let mut ret = vec![pkg]; - for p in nested.iter() { - ret.extend(try!(read_nested_packages(&path.join(p), - source_id, - visited)).into_iter()); + // Registry sources are not allowed to have `path=` dependencies because + // they're all translated to actual registry dependencies. + if !source_id.is_registry() { + for p in nested.iter() { + ret.extend(try!(read_nested_packages(&path.join(p), + source_id, + visited)).into_iter()); + } } Ok(ret) diff --git a/src/cargo/ops/cargo_rustc/fingerprint.rs b/src/cargo/ops/cargo_rustc/fingerprint.rs index 4027d3a2fbf..195e75caeb6 100644 --- a/src/cargo/ops/cargo_rustc/fingerprint.rs +++ b/src/cargo/ops/cargo_rustc/fingerprint.rs @@ -3,7 +3,7 @@ use std::hash::{Hash, Hasher}; use std::hash::sip::SipHasher; use std::io::{fs, File, USER_RWX, BufferedReader}; -use core::{Package, Target, PathKind}; +use core::{Package, Target}; use util; use util::{CargoResult, Fresh, Dirty, Freshness, internal, Require, profile}; @@ -54,10 +54,7 @@ pub fn prepare_target(cx: &mut Context, pkg: &Package, target: &Target, // constant (which is the responsibility of the source) let use_pkg = { let doc = target.get_profile().is_doc(); - let path = match *pkg.get_summary().get_source_id().get_kind() { - PathKind => true, - _ => false, - }; + let path = pkg.get_summary().get_source_id().is_path(); doc || !path }; diff --git a/src/cargo/ops/cargo_upload.rs b/src/cargo/ops/cargo_upload.rs deleted file mode 100644 index d1bd7fbd4d0..00000000000 --- a/src/cargo/ops/cargo_upload.rs +++ /dev/null @@ -1,208 +0,0 @@ -use std::collections::HashMap; -use std::io::File; -use std::os; -use std::str; -use serialize::json::{mod, ToJson}; - -use curl::http; -use git2; - -use core::source::Source; -use core::{Package, MultiShell, SourceId, RegistryKind}; -use ops; -use sources::{PathSource, RegistrySource}; -use util::config; -use util::{CargoResult, human, internal, ChainError, Require, ToUrl}; -use util::config::{Config, Table}; - -pub struct UploadConfig { - pub host: Option, - pub token: Option, -} - -pub fn upload(manifest_path: &Path, - shell: &mut MultiShell, - token: Option, - host: Option) -> CargoResult<()> { - let mut src = try!(PathSource::for_path(&manifest_path.dir_path())); - try!(src.update()); - let pkg = try!(src.get_root_package()); - - // Parse all configuration options - let UploadConfig { token: token_config, .. } = try!(upload_configuration()); - let token = try!(token.or(token_config).require(|| { - human("no upload token found, please run `cargo login`") - })); - let host = host.unwrap_or(try!(RegistrySource::url()).to_string()); - let host = try!(host.as_slice().to_url().map_err(human)); - let upload = { - let sid = SourceId::new(RegistryKind, host.clone()); - let mut config = try!(Config::new(shell, None, None)); - let mut src = RegistrySource::new(&sid, &mut config); - try!(src.update().chain_error(|| { - human(format!("Failed to update registry {}", host)) - })); - (try!(src.config())).upload - }; - - // First, prepare a tarball - let tarball = try!(ops::package(manifest_path, shell)); - let tarball = try!(File::open(&tarball)); - - // Upload said tarball to the specified destination - try!(shell.status("Uploading", pkg.get_package_id().to_string())); - try!(transmit(&pkg, tarball, token.as_slice(), - upload.as_slice()).chain_error(|| { - human(format!("failed to upload package to registry: {}", upload)) - })); - - Ok(()) -} - -fn transmit(pkg: &Package, mut tarball: File, - token: &str, host: &str) -> CargoResult<()> { - let stat = try!(tarball.stat()); - let url = try!(host.to_url().map_err(human)); - let registry_src = SourceId::for_registry(&url); - - let mut handle = try!(http_handle()); - let mut req = handle.put(host, &mut tarball) - .content_length(stat.size as uint) - .content_type("application/x-tar") - .header("Content-Encoding", "x-gzip") - .header("X-Cargo-Auth", token) - .header("X-Cargo-Pkg-Name", pkg.get_name()) - .header("X-Cargo-Pkg-Version", - pkg.get_version().to_string().as_slice()); - - let mut dep_header = String::new(); - for (i, dep) in pkg.get_dependencies().iter().enumerate() { - if !dep.is_transitive() { continue } - if dep.get_source_id() != ®istry_src { - return Err(human(format!("All dependencies must come from the \ - same registry.\nDependency `{}` comes \ - from {} instead", dep.get_name(), - dep.get_source_id()))) - } - - // See Registry::parse_registry_dependency for format - let opt = if dep.is_optional() {"-"} else {""}; - let default = if dep.uses_default_features() {""} else {"*"}; - let features = dep.get_features().connect(","); - let header = format!("{}{}{}|{}|{}", opt, default, dep.get_name(), - features, dep.get_version_req()); - if i > 0 { dep_header.push_str(";"); } - dep_header.push_str(header.as_slice()); - } - req = req.header("X-Cargo-Pkg-Dep", dep_header.as_slice()); - - let feature_header = pkg.get_summary().get_features().to_json().to_string(); - req = req.header("X-Cargo-Pkg-Feature", feature_header.as_slice()); - - let response = try!(req.exec()); - - if response.get_code() == 0 { return Ok(()) } // file upload url - if response.get_code() != 200 { - return Err(internal(format!("failed to get a 200 response: {}", - response))) - } - - let body = try!(str::from_utf8(response.get_body()).require(|| { - internal("failed to get a utf-8 response") - })); - - #[deriving(Decodable)] - struct Response { ok: bool } - #[deriving(Decodable)] - struct BadResponse { error: String } - let json = try!(json::decode::(body)); - if json.ok { return Ok(()) } - - let json = try!(json::decode::(body)); - Err(human(format!("failed to upload `{}`: {}", pkg, json.error))) -} - -pub fn upload_configuration() -> CargoResult { - let configs = try!(config::all_configs(os::getcwd())); - let registry = match configs.find_equiv(&"registry") { - None => return Ok(UploadConfig { host: None, token: None }), - Some(registry) => try!(registry.table().chain_error(|| { - internal("invalid configuration for the key `registry`") - })), - }; - let host = match registry.find_equiv(&"host") { - None => None, - Some(host) => { - Some(try!(host.string().chain_error(|| { - internal("invalid configuration for key `host`") - })).ref0().to_string()) - } - }; - let token = match registry.find_equiv(&"token") { - None => None, - Some(token) => { - Some(try!(token.string().chain_error(|| { - internal("invalid configuration for key `token`") - })).ref0().to_string()) - } - }; - Ok(UploadConfig { host: host, token: token }) -} - -/// Create a new HTTP handle with appropriate global configuration for cargo. -pub fn http_handle() -> CargoResult { - Ok(match try!(http_proxy()) { - Some(proxy) => http::handle().proxy(proxy), - None => http::handle(), - }) -} - -/// Find a globally configured HTTP proxy if one is available. -/// -/// Favor cargo's `http.proxy`, then git's `http.proxy`, then finally a -/// HTTP_PROXY env var. -pub fn http_proxy() -> CargoResult> { - let configs = try!(config::all_configs(os::getcwd())); - match configs.find_equiv(&"http") { - Some(http) => { - let http = try!(http.table().chain_error(|| { - internal("invalid configuration for the key `http`") - })); - match http.find_equiv(&"proxy") { - Some(proxy) => { - return Ok(Some(try!(proxy.string().chain_error(|| { - internal("invalid configuration for key `http.proxy`") - })).ref0().to_string())) - } - None => {}, - } - } - None => {} - } - match git2::Config::open_default() { - Ok(cfg) => { - match cfg.get_str("http.proxy") { - Ok(s) => return Ok(Some(s.to_string())), - Err(..) => {} - } - } - Err(..) => {} - } - Ok(os::getenv("HTTP_PROXY")) -} - -pub fn upload_login(shell: &mut MultiShell, token: String) -> CargoResult<()> { - let config = try!(Config::new(shell, None, None)); - let UploadConfig { host, token: _ } = try!(upload_configuration()); - let mut map = HashMap::new(); - let p = os::getcwd(); - match host { - Some(host) => { - map.insert("host".to_string(), config::String(host, p.clone())); - } - None => {} - } - map.insert("token".to_string(), config::String(token, p)); - - config::set_config(&config, config::Global, "registry", config::Table(map)) -} diff --git a/src/cargo/ops/lockfile.rs b/src/cargo/ops/lockfile.rs new file mode 100644 index 00000000000..441c11d46db --- /dev/null +++ b/src/cargo/ops/lockfile.rs @@ -0,0 +1,96 @@ +use std::io::File; + +use serialize::{Encodable, Decodable}; +use toml::{mod, Encoder}; + +use core::{Resolve, resolver, Package, SourceId}; +use util::CargoResult; +use util::toml as cargo_toml; + +pub fn load_pkg_lockfile(pkg: &Package) -> CargoResult> { + let lockfile = pkg.get_manifest_path().dir_path().join("Cargo.lock"); + let source_id = pkg.get_package_id().get_source_id(); + load_lockfile(&lockfile, source_id) +} + +pub fn load_lockfile(path: &Path, sid: &SourceId) -> CargoResult> { + // If there is no lockfile, return none. + let mut f = match File::open(path) { + Ok(f) => f, + Err(_) => return Ok(None) + }; + + let s = try!(f.read_to_string()); + + let table = toml::Table(try!(cargo_toml::parse(s.as_slice(), path))); + let mut d = toml::Decoder::new(table); + let v: resolver::EncodableResolve = Decodable::decode(&mut d).unwrap(); + Ok(Some(try!(v.to_resolve(sid)))) +} + +pub fn write_pkg_lockfile(pkg: &Package, resolve: &Resolve) -> CargoResult<()> { + let loc = pkg.get_root().join("Cargo.lock"); + write_lockfile(&loc, resolve) +} + +pub fn write_lockfile(dst: &Path, resolve: &Resolve) -> CargoResult<()> { + let mut e = Encoder::new(); + resolve.encode(&mut e).unwrap(); + + let mut out = String::new(); + + // Note that we do not use e.toml.to_string() as we want to control the + // exact format the toml is in to ensure pretty diffs between updates to the + // lockfile. + let root = e.toml.find(&"root".to_string()).unwrap(); + + out.push_str("[root]\n"); + emit_package(root.as_table().unwrap(), &mut out); + + let deps = e.toml.find(&"package".to_string()).unwrap().as_slice().unwrap(); + for dep in deps.iter() { + let dep = dep.as_table().unwrap(); + + out.push_str("[[package]]\n"); + emit_package(dep, &mut out); + } + + match e.toml.find(&"metadata".to_string()) { + Some(metadata) => { + out.push_str("[metadata]\n"); + out.push_str(metadata.to_string().as_slice()); + } + None => {} + } + + try!(File::create(dst).write_str(out.as_slice())); + Ok(()) +} + +fn emit_package(dep: &toml::TomlTable, out: &mut String) { + out.push_str(format!("name = {}\n", lookup(dep, "name")).as_slice()); + out.push_str(format!("version = {}\n", lookup(dep, "version")).as_slice()); + + if dep.contains_key(&"source".to_string()) { + out.push_str(format!("source = {}\n", lookup(dep, "source")).as_slice()); + } + + if let Some(ref s) = dep.find(&"dependencies".to_string()) { + let slice = s.as_slice().unwrap(); + + if !slice.is_empty() { + out.push_str("dependencies = [\n"); + + for child in s.as_slice().unwrap().iter() { + out.push_str(format!(" {},\n", child).as_slice()); + } + + out.push_str("]\n"); + } + out.push_str("\n"); + } +} + +fn lookup<'a>(table: &'a toml::TomlTable, key: &str) -> &'a toml::Value { + table.find(&key.to_string()).expect(format!("Didn't find {}", key).as_slice()) +} diff --git a/src/cargo/ops/mod.rs b/src/cargo/ops/mod.rs index ddf5c2b9f8c..5176ce75f29 100644 --- a/src/cargo/ops/mod.rs +++ b/src/cargo/ops/mod.rs @@ -1,5 +1,5 @@ pub use self::cargo_clean::{clean, CleanOptions}; -pub use self::cargo_compile::{compile, CompileOptions}; +pub use self::cargo_compile::{compile, compile_pkg, CompileOptions}; pub use self::cargo_read_manifest::{read_manifest,read_package,read_packages}; pub use self::cargo_rustc::{compile_targets, Compilation, Layout, Kind}; pub use self::cargo_rustc::{KindTarget, KindPlugin, Context, LayoutProxy}; @@ -8,26 +8,32 @@ pub use self::cargo_rustc::{PlatformPlugin, PlatformPluginAndTarget}; pub use self::cargo_run::run; pub use self::cargo_new::{new, NewOptions}; pub use self::cargo_doc::{doc, DocOptions}; -pub use self::cargo_generate_lockfile::{generate_lockfile, write_resolve}; -pub use self::cargo_generate_lockfile::{update_lockfile, load_lockfile}; +pub use self::cargo_generate_lockfile::{generate_lockfile}; +pub use self::cargo_generate_lockfile::{update_lockfile}; pub use self::cargo_generate_lockfile::UpdateOptions; +pub use self::lockfile::{load_lockfile, load_pkg_lockfile}; +pub use self::lockfile::{write_lockfile, write_pkg_lockfile}; pub use self::cargo_test::{run_tests, run_benches, TestOptions}; pub use self::cargo_package::package; -pub use self::cargo_upload::{upload, upload_configuration, UploadConfig}; -pub use self::cargo_upload::{upload_login, http_proxy, http_handle}; -pub use self::cargo_fetch::{fetch, resolve_and_fetch}; +pub use self::registry::{publish, registry_configuration, RegistryConfig}; +pub use self::registry::{registry_login, http_proxy, http_handle}; +pub use self::registry::{modify_owners, yank}; +pub use self::cargo_fetch::{fetch}; pub use self::cargo_pkgid::pkgid; +pub use self::resolve::{resolve_pkg, resolve_with_previous}; mod cargo_clean; mod cargo_compile; -mod cargo_read_manifest; -mod cargo_rustc; -mod cargo_run; -mod cargo_new; mod cargo_doc; +mod cargo_fetch; mod cargo_generate_lockfile; -mod cargo_test; +mod cargo_new; mod cargo_package; -mod cargo_upload; -mod cargo_fetch; mod cargo_pkgid; +mod cargo_read_manifest; +mod cargo_run; +mod cargo_rustc; +mod cargo_test; +mod lockfile; +mod registry; +mod resolve; diff --git a/src/cargo/ops/registry.rs b/src/cargo/ops/registry.rs new file mode 100644 index 00000000000..fbef2c62917 --- /dev/null +++ b/src/cargo/ops/registry.rs @@ -0,0 +1,302 @@ +use std::collections::HashMap; +use std::io::File; +use std::os; + +use curl::http; +use git2; +use registry::{Registry, NewCrate, NewCrateDependency}; + +use core::source::Source; +use core::{Package, MultiShell, SourceId}; +use core::manifest::ManifestMetadata; +use ops; +use sources::{PathSource, RegistrySource}; +use util::config; +use util::{CargoResult, human, internal, ChainError, Require, ToUrl}; +use util::config::{Config, Table}; + +pub struct RegistryConfig { + pub index: Option, + pub token: Option, +} + +pub fn publish(manifest_path: &Path, + shell: &mut MultiShell, + token: Option, + index: Option, + verify: bool) -> CargoResult<()> { + let mut src = try!(PathSource::for_path(&manifest_path.dir_path())); + try!(src.update()); + let pkg = try!(src.get_root_package()); + + let (mut registry, reg_id) = try!(registry(shell, token, index)); + try!(verify_dependencies(&pkg, ®_id)); + + // Prepare a tarball + let tarball = try!(ops::package(manifest_path, shell, verify)); + + // Upload said tarball to the specified destination + try!(shell.status("Uploading", pkg.get_package_id().to_string())); + try!(transmit(&pkg, &tarball, &mut registry)); + + Ok(()) +} + +fn verify_dependencies(pkg: &Package, registry_src: &SourceId) + -> CargoResult<()> { + for dep in pkg.get_dependencies().iter() { + if dep.get_source_id().is_path() { + if dep.get_specified_req().is_none() { + return Err(human(format!("all path dependencies must have \ + a version specified when being \ + uploaded to the registry.\n\ + dependency `{}` does not specify \ + a version", dep.get_name()))) + } + } else if dep.get_source_id() != registry_src { + return Err(human(format!("all dependencies must come from the \ + same registry.\ndependency `{}` comes \ + from {} instead", dep.get_name(), + dep.get_source_id()))) + } + } + Ok(()) +} + +fn transmit(pkg: &Package, tarball: &Path, registry: &mut Registry) + -> CargoResult<()> { + let deps = pkg.get_dependencies().iter().map(|dep| { + NewCrateDependency { + optional: dep.is_optional(), + default_features: dep.uses_default_features(), + name: dep.get_name().to_string(), + features: dep.get_features().to_vec(), + version_req: dep.get_version_req().to_string(), + target: None, // FIXME: fill this out + } + }).collect::>(); + let manifest = pkg.get_manifest(); + let ManifestMetadata { + ref authors, ref description, ref homepage, ref documentation, + ref keywords, ref readme, ref repository, ref license, + } = *manifest.get_metadata(); + let readme = match *readme { + Some(ref readme) => { + let path = pkg.get_root().join(readme.as_slice()); + Some(try!(File::open(&path).read_to_string().chain_error(|| { + human("failed to read the specified README") + }))) + } + None => None, + }; + registry.publish(&NewCrate { + name: pkg.get_name().to_string(), + vers: pkg.get_version().to_string(), + deps: deps, + features: pkg.get_summary().get_features().clone(), + authors: authors.clone(), + description: description.clone(), + homepage: homepage.clone(), + documentation: documentation.clone(), + keywords: keywords.clone(), + readme: readme, + repository: repository.clone(), + license: license.clone(), + }, tarball).map_err(|e| { + human(e.to_string()) + }) +} + +pub fn registry_configuration() -> CargoResult { + let configs = try!(config::all_configs(os::getcwd())); + let registry = match configs.find_equiv(&"registry") { + None => return Ok(RegistryConfig { index: None, token: None }), + Some(registry) => try!(registry.table().chain_error(|| { + internal("invalid configuration for the key `registry`") + })), + }; + let index = match registry.find_equiv(&"index") { + None => None, + Some(index) => { + Some(try!(index.string().chain_error(|| { + internal("invalid configuration for key `index`") + })).ref0().to_string()) + } + }; + let token = match registry.find_equiv(&"token") { + None => None, + Some(token) => { + Some(try!(token.string().chain_error(|| { + internal("invalid configuration for key `token`") + })).ref0().to_string()) + } + }; + Ok(RegistryConfig { index: index, token: token }) +} + +pub fn registry(shell: &mut MultiShell, + token: Option, + index: Option) -> CargoResult<(Registry, SourceId)> { + // Parse all configuration options + let RegistryConfig { + token: token_config, + index: index_config, + } = try!(registry_configuration()); + let token = try!(token.or(token_config).require(|| { + human("no upload token found, please run `cargo login`") + })); + let index = index.or(index_config).unwrap_or(RegistrySource::default_url()); + let index = try!(index.as_slice().to_url().map_err(human)); + let sid = SourceId::for_registry(&index); + let api_host = { + let mut config = try!(Config::new(shell, None, None)); + let mut src = RegistrySource::new(&sid, &mut config); + try!(src.update().chain_error(|| { + human(format!("Failed to update registry {}", index)) + })); + (try!(src.config())).api + }; + let handle = try!(http_handle()); + Ok((Registry::new_handle(api_host, token, handle), sid)) +} + +/// Create a new HTTP handle with appropriate global configuration for cargo. +pub fn http_handle() -> CargoResult { + Ok(match try!(http_proxy()) { + Some(proxy) => http::handle().proxy(proxy), + None => http::handle(), + }) +} + +/// Find a globally configured HTTP proxy if one is available. +/// +/// Favor cargo's `http.proxy`, then git's `http.proxy`, then finally a +/// HTTP_PROXY env var. +pub fn http_proxy() -> CargoResult> { + let configs = try!(config::all_configs(os::getcwd())); + match configs.find_equiv(&"http") { + Some(http) => { + let http = try!(http.table().chain_error(|| { + internal("invalid configuration for the key `http`") + })); + match http.find_equiv(&"proxy") { + Some(proxy) => { + return Ok(Some(try!(proxy.string().chain_error(|| { + internal("invalid configuration for key `http.proxy`") + })).ref0().to_string())) + } + None => {}, + } + } + None => {} + } + match git2::Config::open_default() { + Ok(cfg) => { + match cfg.get_str("http.proxy") { + Ok(s) => return Ok(Some(s.to_string())), + Err(..) => {} + } + } + Err(..) => {} + } + Ok(os::getenv("HTTP_PROXY")) +} + +pub fn registry_login(shell: &mut MultiShell, token: String) -> CargoResult<()> { + let config = try!(Config::new(shell, None, None)); + let RegistryConfig { index, token: _ } = try!(registry_configuration()); + let mut map = HashMap::new(); + let p = os::getcwd(); + match index { + Some(index) => { + map.insert("index".to_string(), config::String(index, p.clone())); + } + None => {} + } + map.insert("token".to_string(), config::String(token, p)); + + config::set_config(&config, config::Global, "registry", config::Table(map)) +} + +pub fn modify_owners(manifest_path: &Path, + shell: &mut MultiShell, + krate: Option, + token: Option, + index: Option, + to_add: Option>, + to_remove: Option>) -> CargoResult<()> { + let name = match krate { + Some(name) => name, + None => { + let mut src = try!(PathSource::for_path(&manifest_path.dir_path())); + try!(src.update()); + let pkg = try!(src.get_root_package()); + pkg.get_name().to_string() + } + }; + + let (mut registry, _) = try!(registry(shell, token, index)); + + match to_add { + Some(v) => { + let v = v.iter().map(|s| s.as_slice()).collect::>(); + try!(shell.status("Owner", format!("adding `{:#}` to `{}`", v, name))); + try!(registry.add_owners(name.as_slice(), v.as_slice()).map_err(|e| { + human(format!("failed to add owners: {}", e)) + })); + } + None => {} + } + + match to_remove { + Some(v) => { + let v = v.iter().map(|s| s.as_slice()).collect::>(); + try!(shell.status("Owner", format!("removing `{:#}` from `{}`", + v, name))); + try!(registry.remove_owners(name.as_slice(), v.as_slice()).map_err(|e| { + human(format!("failed to add owners: {}", e)) + })); + } + None => {} + } + + Ok(()) +} + +pub fn yank(manifest_path: &Path, + shell: &mut MultiShell, + krate: Option, + version: Option, + token: Option, + index: Option, + undo: bool) -> CargoResult<()> { + let name = match krate { + Some(name) => name, + None => { + let mut src = try!(PathSource::for_path(&manifest_path.dir_path())); + try!(src.update()); + let pkg = try!(src.get_root_package()); + pkg.get_name().to_string() + } + }; + let version = match version { + Some(v) => v, + None => return Err(human("a version must be specified to yank")) + }; + + let (mut registry, _) = try!(registry(shell, token, index)); + + if undo { + try!(shell.status("Unyank", format!("{}:{}", name, version))); + try!(registry.unyank(name.as_slice(), version.as_slice()).map_err(|e| { + human(format!("failed to undo a yank: {}", e)) + })); + } else { + try!(shell.status("Yank", format!("{}:{}", name, version))); + try!(registry.yank(name.as_slice(), version.as_slice()).map_err(|e| { + human(format!("failed to yank: {}", e)) + })); + } + + Ok(()) +} diff --git a/src/cargo/ops/resolve.rs b/src/cargo/ops/resolve.rs new file mode 100644 index 00000000000..62b6e9f3f35 --- /dev/null +++ b/src/cargo/ops/resolve.rs @@ -0,0 +1,102 @@ +use std::collections::{HashMap, HashSet}; + +use core::{Package, PackageId}; +use core::registry::PackageRegistry; +use core::resolver::{mod, Resolve}; +use ops; +use util::CargoResult; + +/// Resolve all dependencies for the specified `package` using the previous +/// lockfile as a guide if present. +/// +/// This function will also generate a write the result of resolution as a new +/// lockfile. +pub fn resolve_pkg(registry: &mut PackageRegistry, package: &Package) + -> CargoResult { + let prev = try!(ops::load_pkg_lockfile(package)); + let resolve = try!(resolve_with_previous(registry, package, + resolver::ResolveEverything, + prev.as_ref(), None)); + try!(ops::write_pkg_lockfile(package, &resolve)); + Ok(resolve) +} + +/// Resolve all dependencies for a package using an optional previous instance +/// of resolve to guide the resolution process. +/// +/// This also takes an optional hash set, `to_avoid`, which is a list of package +/// ids that should be avoided when consulting the previous instance of resolve +/// (often used in pairings with updates). +/// +/// The previous resolve normally comes from a lockfile. This function does not +/// read or write lockfiles from the filesystem. +pub fn resolve_with_previous<'a>(registry: &mut PackageRegistry, + package: &Package, + method: resolver::ResolveMethod, + previous: Option<&'a Resolve>, + to_avoid: Option<&HashSet<&'a PackageId>>) + -> CargoResult { + let root = package.get_package_id().get_source_id().clone(); + try!(registry.add_sources(&[root])); + + let summary = package.get_summary().clone(); + let summary = match previous { + Some(r) => { + // In the case where a previous instance of resolve is available, we + // want to lock as many packages as possible to the previous version + // without disturbing the graph structure. To this end we perform + // two actions here: + // + // 1. We inform the package registry of all locked packages. This + // involves informing it of both the locked package's id as well + // as the versions of all locked dependencies. The registry will + // then takes this information into account when it is queried. + // + // 2. The specified package's summary will have its dependencies + // modified to their precise variants. This will instruct the + // first step of the resolution process to not query for ranges + // but rather precise dependency versions. + // + // This process must handle altered dependencies, however, as + // it's possible for a manifest to change over time to have + // dependencies added, removed, or modified to different version + // ranges. To deal with this, we only actually lock a dependency + // to the previously resolved version if the dependency listed + // still matches the locked version. + for node in r.iter().filter(|p| keep(p, to_avoid)) { + let deps = r.deps(node).into_iter().flat_map(|i| i) + .filter(|p| keep(p, to_avoid)) + .map(|p| p.clone()).collect(); + registry.register_lock(node.clone(), deps); + } + + let map = r.deps(r.root()).into_iter().flat_map(|i| i).filter(|p| { + keep(p, to_avoid) + }).map(|d| { + (d.get_name(), d) + }).collect::>(); + summary.map_dependencies(|d| { + match map.find_equiv(&d.get_name()) { + Some(&lock) if d.matches_id(lock) => d.lock_to(lock), + _ => d, + } + }) + } + None => summary, + }; + + let mut resolved = try!(resolver::resolve(&summary, method, registry)); + match previous { + Some(r) => resolved.copy_metadata(r), + None => {} + } + return Ok(resolved); + + fn keep<'a>(p: &&'a PackageId, to_avoid: Option<&HashSet<&'a PackageId>>) + -> bool { + match to_avoid { + Some(set) => !set.contains(p), + None => true, + } + } +} diff --git a/src/cargo/sources/git/source.rs b/src/cargo/sources/git/source.rs index 8938bac5599..870bcaa3e80 100644 --- a/src/cargo/sources/git/source.rs +++ b/src/cargo/sources/git/source.rs @@ -4,7 +4,7 @@ use std::hash::sip::SipHasher; use std::mem; use url::{mod, Url}; -use core::source::{Source, SourceId, GitKind}; +use core::source::{Source, SourceId}; use core::{Package, PackageId, Summary, Registry, Dependency}; use util::{CargoResult, Config, to_hex}; use sources::PathSource; @@ -28,9 +28,9 @@ impl<'a, 'b> GitSource<'a, 'b> { config: &'a mut Config<'b>) -> GitSource<'a, 'b> { assert!(source_id.is_git(), "id is not git, id={}", source_id); - let reference = match *source_id.get_kind() { - GitKind(ref reference) => reference, - _ => fail!("Not a git source; id={}", source_id) + let reference = match source_id.git_reference() { + Some(reference) => reference, + None => fail!("Not a git source; id={}", source_id), }; let remote = GitRemote::new(source_id.get_url()); @@ -176,7 +176,7 @@ impl<'a, 'b> Source for GitSource<'a, 'b> { try!(repo.copy_to(actual_rev.clone(), &self.checkout_path)); - let source_id = self.source_id.with_precise(actual_rev.to_string()); + let source_id = self.source_id.with_precise(Some(actual_rev.to_string())); let path_source = PathSource::new(&self.checkout_path, &source_id); self.path_source = Some(path_source); diff --git a/src/cargo/sources/registry.rs b/src/cargo/sources/registry.rs index 7571cf6454d..f989559c3c1 100644 --- a/src/cargo/sources/registry.rs +++ b/src/cargo/sources/registry.rs @@ -1,4 +1,164 @@ -use std::io::{mod, fs, File, MemReader}; +//! A `Source` for registry-based packages. +//! +//! # What's a Registry? +//! +//! Registries are central locations where packages can be uploaded to, +//! discovered, and searched for. The purpose of a registry is to have a +//! location that serves as permanent storage for versions of a crate over time. +//! +//! Compared to git sources, a registry provides many packages as well as many +//! versions simultaneously. Git sources can also have commits deleted through +//! rebasings where registries cannot have their versions deleted. +//! +//! # The Index of a Registry +//! +//! One of the major difficulties with a registry is that hosting so many +//! packages may quickly run into performance problems when dealing with +//! dependency graphs. It's infeasible for cargo to download the entire contents +//! of the registry just to resolve one package's dependencies, for example. As +//! a result, cargo needs some efficient method of querying what packages are +//! available on a registry, what versions are available, and what the +//! dependencies for each version is. +//! +//! One method of doing so would be having the registry expose an HTTP endpoint +//! which can be queried with a list of packages and a response of their +//! dependencies and versions is returned. This is somewhat inefficient however +//! as we may have to hit the endpoint many times and we may have already +//! queried for much of the data locally already (for other packages, for +//! example). This also involves inventing a transport format between the +//! registry and Cargo itself, so this route was not taken. +//! +//! Instead, Cargo communicates with registries through a git repository +//! referred to as the Index. The Index of a registry is essentially an easily +//! query-able version of the registry's database for a list of versions of a +//! package as well as a list of dependencies for each version. +//! +//! Using git to host this index provides a number of benefits: +//! +//! * The entire index can be stored efficiently locally on disk. This means +//! that all queries of a registry can happen locally and don't need to touch +//! the network. +//! +//! * Updates of the index are quite efficient. Using git buys incremental +//! updates, compressed transmission, etc for free. The index must be updated +//! each time we need fresh information from a registry, but this is one +//! update of a git repository that probably hasn't changed a whole lot so +//! it shouldn't be too expensive. +//! +//! Additionally, each modification to the index is just appending a line at +//! the end of a file (the exact format is described later). This means that +//! the commits for an index are quite small and easily applied/compressable. +//! +//! ## The format of the Index +//! +//! The index is a store for the list of versions for all packages known, so its +//! format on disk is optimized slightly to ensure that `ls registry` doesn't +//! produce a list of all packages ever known. The index also wants to ensure +//! that there's not a million files which may actually end up hitting +//! filesystem limits at some point. To this end, a few decisions were made +//! about the format of the registry: +//! +//! 1. Each crate will have one file corresponding to it. Each version for a +//! crate will just be a line in this file. +//! 2. There will be two tiers of directories for crate names, under which +//! crates corresponding to those tiers will be located. +//! +//! As an example, this is an example hierarchy of an index: +//! +//! ```notrust +//! . +//! ├── 3 +//! │   └── u +//! │   └── url +//! ├── bz +//! │   └── ip +//! │   └── bzip2 +//! ├── config.json +//! ├── en +//! │   └── co +//! │   └── encoding +//! └── li +//!    ├── bg +//!    │   └── libgit2 +//!    └── nk +//!    └── link-config +//! ``` +//! +//! The root of the index contains a `config.json` file with a few entries +//! corresponding to the registry (see `RegistryConfig` below). +//! +//! Otherwise, there are three numbered directories (1, 2, 3) for crates with +//! names 1, 2, and 3 characters in length. The 1/2 directories simply have the +//! crate files underneath them, while the 3 directory is sharded by the first +//! letter of the crate name. +//! +//! Otherwise the top-level directory contains many two-letter directory names, +//! each of which has many sub-folders with two letters. At the end of all these +//! are the actual crate files themselves. +//! +//! The purpose of this layou tis to hopefully cut down on `ls` sizes as well as +//! efficient lookup based on the crate name itself. +//! +//! ## Crate files +//! +//! Each file in the index is the history of one crate over time. Each line in +//! the file corresponds to one version of a crate, stored in JSON format (see +//! the `RegistryPackage` structure below). +//! +//! As new versions are published, new lines are appended to this file. The only +//! modifications to this file that should happen over time are yanks of a +//! particular version. +//! +//! # Downloading Packages +//! +//! The purpose of the Index was to provide an efficient method to resolve the +//! dependency graph for a package. So far we only required one network +//! interaction to update the registry's repository (yay!). After resolution has +//! been performed, however we need to download the contents of packages so we +//! can read the full manifest and build the source code. +//! +//! To accomplish this, this source's `download` method will make an HTTP +//! request per-package requested to download tarballs into a local cache. These +//! tarballs will then be unpacked into a destination folder. +//! +//! Note that because versions uploaded to the registry are frozen forever that +//! the HTTP download and unpacking can all be skipped if the version has +//! already been downloaded and unpacked. This caching allows us to only +//! download a package when absolutely necessary. +//! +//! # Filesystem Hierarchy +//! +//! Overall, the `$HOME/.cargo` looks like this when talking about the registry: +//! +//! ```notrust +//! # A folder under which all registry metadata is hosted (similar to +//! # $HOME/.cargo/git) +//! $HOME/.cargo/registry/ +//! +//! # For each registry that cargo knows about (keyed by hostname + hash) +//! # there is a folder which is the checked out version of the index for +//! # the registry in this location. Note that this is done so cargo can +//! # support multiple registries simultaneously +//! index/ +//! registry1-/ +//! registry2-/ +//! ... +//! +//! # This folder is a cache for all downloaded tarballs from a registry. +//! # Once downloaded and verified, a tarball never changes. +//! cache/ +//! registry1-/-.tar.gz +//! ... +//! +//! # Location in which all tarballs are unpacked. Each tarball is known to +//! # be frozen after downloading, so transitively this folder is also +//! # frozen once its unpacked (it's never unpacked again) +//! src/ +//! registry1-/-/... +//! ... +//! ``` + +use std::io::{mod, fs, File}; use std::io::fs::PathExtensions; use std::collections::HashMap; @@ -28,21 +188,40 @@ pub struct RegistrySource<'a, 'b:'a> { handle: Option, sources: Vec, hashes: HashMap<(String, String), String>, // (name, vers) => cksum + cache: HashMap>, + updated: bool, } #[deriving(Decodable)] pub struct RegistryConfig { + /// Download endpoint for all crates. This will be appended with + /// `///download` and then will be hit with an HTTP GET + /// request to download the tarball for a crate. pub dl: String, - pub upload: String, + + /// API endpoint for the registry. This is what's actually hit to perform + /// operations like yanks, owner modifications, publish new crates, etc. + pub api: String, } #[deriving(Decodable)] struct RegistryPackage { name: String, vers: String, - deps: Vec, + deps: Vec, features: HashMap>, cksum: String, + yanked: Option, +} + +#[deriving(Decodable)] +struct RegistryDependency { + name: String, + req: String, + features: Vec, + optional: bool, + default_features: bool, + target: Option, } impl<'a, 'b> RegistrySource<'a, 'b> { @@ -60,6 +239,8 @@ impl<'a, 'b> RegistrySource<'a, 'b> { handle: None, sources: Vec::new(), hashes: HashMap::new(), + cache: HashMap::new(), + updated: false, } } @@ -68,11 +249,16 @@ impl<'a, 'b> RegistrySource<'a, 'b> { /// This is the main cargo registry by default, but it can be overridden in /// a .cargo/config pub fn url() -> CargoResult { - let config = try!(ops::upload_configuration()); - let url = config.host.unwrap_or(CENTRAL.to_string()); + let config = try!(ops::registry_configuration()); + let url = config.index.unwrap_or(CENTRAL.to_string()); url.as_slice().to_url().map_err(human) } + /// Get the default url for the registry + pub fn default_url() -> String { + CENTRAL.to_string() + } + /// Decode the configuration stored within the registry. /// /// This requires that the index has been at least checked out. @@ -109,8 +295,9 @@ impl<'a, 'b> RegistrySource<'a, 'b> { /// No action is taken if the package is already downloaded. fn download_package(&mut self, pkg: &PackageId, url: &Url) -> CargoResult { - let dst = self.cache_path.join(url.path().unwrap().last().unwrap() - .as_slice()); + // TODO: should discover from the S3 redirect + let filename = format!("{}-{}.tar.gz", pkg.get_name(), pkg.get_version()); + let dst = self.cache_path.join(filename); if dst.exists() { return Ok(dst) } try!(self.config.shell().status("Downloading", pkg)); @@ -123,7 +310,7 @@ impl<'a, 'b> RegistrySource<'a, 'b> { } }; // TODO: don't download into memory (curl-rust doesn't expose it) - let resp = try!(handle.get(url.to_string()).exec()); + let resp = try!(handle.get(url.to_string()).follow_redirects(true).exec()); if resp.get_code() != 200 && resp.get_code() != 0 { return Err(internal(format!("Failed to get 200 reponse from {}\n{}", url, resp))) @@ -161,109 +348,85 @@ impl<'a, 'b> RegistrySource<'a, 'b> { try!(fs::mkdir_recursive(&dst.dir_path(), io::USER_DIR)); let f = try!(File::open(&tarball)); - let mut gz = try!(GzDecoder::new(f)); - // TODO: don't read into memory (Archive requires Seek) - let mem = try!(gz.read_to_end()); - let tar = Archive::new(MemReader::new(mem)); - for file in try!(tar.files()) { - let mut file = try!(file); - let dst = dst.dir_path().join(file.filename_bytes()); - try!(fs::mkdir_recursive(&dst.dir_path(), io::USER_DIR)); - let mut dst = try!(File::create(&dst)); - try!(io::util::copy(&mut file, &mut dst)); - } + let gz = try!(GzDecoder::new(f)); + let mut tar = Archive::new(gz); + try!(tar.unpack(&dst.dir_path())); try!(File::create(&dst.join(".cargo-ok"))); Ok(dst) } + /// Parse the on-disk metadata for the package provided + fn summaries(&mut self, name: &str) -> CargoResult<&Vec<(Summary, bool)>> { + if self.cache.contains_key_equiv(&name) { + return Ok(self.cache.find_equiv(&name).unwrap()); + } + // see module comment for why this is structured the way it is + let path = self.checkout_path.clone(); + let path = match name.len() { + 1 => path.join("1").join(name), + 2 => path.join("2").join(name), + 3 => path.join("3").join(name.slice_to(1)).join(name), + _ => path.join(name.slice(0, 2)) + .join(name.slice(2, 4)) + .join(name), + }; + let summaries = match File::open(&path) { + Ok(mut f) => { + let contents = try!(f.read_to_string()); + let ret: CargoResult>; + ret = contents.as_slice().lines().filter(|l| l.trim().len() > 0) + .map(|l| self.parse_registry_package(l)) + .collect(); + try!(ret.chain_error(|| { + internal(format!("Failed to parse registry's information \ + for: {}", name)) + })) + } + Err(..) => Vec::new(), + }; + self.cache.insert(name.to_string(), summaries); + Ok(self.cache.find_equiv(&name).unwrap()) + } + /// Parse a line from the registry's index file into a Summary for a /// package. - fn parse_registry_package(&mut self, line: &str) -> CargoResult { - let pkg = try!(json::decode::(line)); - let pkgid = try!(PackageId::new(pkg.name.as_slice(), - pkg.vers.as_slice(), + /// + /// The returned boolean is whether or not the summary has been yanked. + fn parse_registry_package(&mut self, line: &str) + -> CargoResult<(Summary, bool)> { + let RegistryPackage { + name, vers, cksum, deps, features, yanked + } = try!(json::decode::(line)); + let pkgid = try!(PackageId::new(name.as_slice(), + vers.as_slice(), &self.source_id)); - let deps: CargoResult> = pkg.deps.iter().map(|dep| { - self.parse_registry_dependency(dep.as_slice()) + let deps: CargoResult> = deps.into_iter().map(|dep| { + self.parse_registry_dependency(dep) }).collect(); let deps = try!(deps); - let RegistryPackage { name, vers, cksum, .. } = pkg; self.hashes.insert((name, vers), cksum); - Summary::new(pkgid, deps, pkg.features) + Ok((try!(Summary::new(pkgid, deps, features)), yanked.unwrap_or(false))) } - /// Parse a dependency listed in the registry into a `Dependency`. - /// - /// Currently the format for dependencies is: - /// - /// ```notrust - /// dep := ['-'] ['*'] name '|' [ name ',' ] * '|' version_req - /// ``` - /// - /// The '-' indicates that this is an optional dependency, and the '*' - /// indicates that the dependency does *not* use the default features - /// provided. The comma-separate list of names in brackets are the enabled - /// features for the dependency, and the final element is the version - /// requirement of the dependency. - fn parse_registry_dependency(&self, dep: &str) -> CargoResult { - let mut parts = dep.as_slice().splitn(2, '|'); - let name = parts.next().unwrap(); - let features = try!(parts.next().require(|| { - human(format!("malformed dependency in registry: {}", dep)) - })); - let vers = try!(parts.next().require(|| { - human(format!("malformed dependency in registry: {}", dep)) - })); - let (name, optional) = if name.starts_with("-") { - (name.slice_from(1), true) - } else { - (name, false) - }; - let (name, default_features) = if name.starts_with("*") { - (name.slice_from(1), false) - } else { - (name, true) - }; - let features = features.split(',').filter(|s| !s.is_empty()) - .map(|s| s.to_string()).collect(); - let dep = try!(Dependency::parse(name, Some(vers), &self.source_id)); + /// Converts an encoded dependency in the registry to a cargo dependency + fn parse_registry_dependency(&self, dep: RegistryDependency) + -> CargoResult { + let RegistryDependency { + name, req, features, optional, default_features, target + } = dep; + + let dep = try!(Dependency::parse(name.as_slice(), Some(req.as_slice()), + &self.source_id)); + drop(target); // FIXME: pass this in Ok(dep.optional(optional) .default_features(default_features) .features(features)) } -} -impl<'a, 'b> Registry for RegistrySource<'a, 'b> { - fn query(&mut self, dep: &Dependency) -> CargoResult> { - let name = dep.get_name(); - let path = self.checkout_path.clone(); - let path = match name.len() { - 1 => path.join("1").join(name), - 2 => path.join("2").join(name), - 3 => path.join("3").join(name.slice_to(1)).join(name), - _ => path.join(name.slice(0, 2)) - .join(name.slice(2, 4)) - .join(name), - }; - let contents = match File::open(&path) { - Ok(mut f) => try!(f.read_to_string()), - Err(..) => return Ok(Vec::new()), - }; - - let ret: CargoResult>; - ret = contents.as_slice().lines().filter(|l| l.trim().len() > 0) - .map(|l| self.parse_registry_package(l)) - .collect(); - let mut summaries = try!(ret.chain_error(|| { - internal(format!("Failed to parse registry's information for: {}", - dep.get_name())) - })); - summaries.query(dep) - } -} + /// Actually perform network operations to update the registry + fn do_update(&mut self) -> CargoResult<()> { + if self.updated { return Ok(()) } -impl<'a, 'b> Source for RegistrySource<'a, 'b> { - fn update(&mut self) -> CargoResult<()> { try!(self.config.shell().status("Updating", format!("registry `{}`", self.source_id.get_url()))); let repo = try!(self.open()); @@ -281,6 +444,41 @@ impl<'a, 'b> Source for RegistrySource<'a, 'b> { log!(5, "[{}] updating to rev {}", self.source_id, oid); let object = try!(repo.find_object(oid, None)); try!(repo.reset(&object, git2::Hard, None, None)); + self.updated = true; + self.cache.clear(); + Ok(()) + } +} + +impl<'a, 'b> Registry for RegistrySource<'a, 'b> { + fn query(&mut self, dep: &Dependency) -> CargoResult> { + // If this is a precise dependency, then it came from a lockfile and in + // theory the registry is known to contain this version. If, however, we + // come back with no summaries, then our registry may need to be + // updated, so we fall back to performing a lazy update. + if dep.get_source_id().get_precise().is_some() && + try!(self.summaries(dep.get_name())).len() == 0 { + try!(self.do_update()); + } + + let summaries = try!(self.summaries(dep.get_name())); + let mut summaries = summaries.iter().filter(|&&(_, yanked)| { + dep.get_source_id().get_precise().is_some() || !yanked + }).map(|&(ref s, _)| s.clone()).collect::>(); + summaries.query(dep) + } +} + +impl<'a, 'b> Source for RegistrySource<'a, 'b> { + fn update(&mut self) -> CargoResult<()> { + // If we have an imprecise version then we don't know what we're going + // to look for, so we always atempt to perform an update here. + // + // If we have a precise version, then we'll update lazily during the + // querying phase. + if self.source_id.get_precise().is_none() { + try!(self.do_update()); + } Ok(()) } @@ -291,11 +489,9 @@ impl<'a, 'b> Source for RegistrySource<'a, 'b> { if self.source_id != *package.get_source_id() { continue } let mut url = url.clone(); - url.path_mut().unwrap().push("pkg".to_string()); url.path_mut().unwrap().push(package.get_name().to_string()); - url.path_mut().unwrap().push(format!("{}-{}.tar.gz", - package.get_name(), - package.get_version())); + url.path_mut().unwrap().push(package.get_version().to_string()); + url.path_mut().unwrap().push("download".to_string()); let path = try!(self.download_package(package, &url).chain_error(|| { internal(format!("Failed to download package `{}` from {}", package, url)) diff --git a/src/cargo/util/toml.rs b/src/cargo/util/toml.rs index 8fe9829bf0e..27082011d64 100644 --- a/src/cargo/util/toml.rs +++ b/src/cargo/util/toml.rs @@ -9,8 +9,8 @@ use toml; use semver; use serialize::{Decodable, Decoder}; -use core::{SourceId, GitKind}; -use core::manifest::{LibKind, Lib, Dylib, Profile}; +use core::SourceId; +use core::manifest::{LibKind, Lib, Dylib, Profile, ManifestMetadata}; use core::{Summary, Manifest, Target, Dependency, PackageId}; use core::package_id::Metadata; use util::{CargoResult, Require, human, ToUrl, ToSemver}; @@ -249,9 +249,18 @@ impl ManyOrOne { pub struct TomlProject { name: String, version: TomlVersion, - pub authors: Vec, + authors: Vec, build: Option, exclude: Option>, + + // package metadata + description: Option, + homepage: Option, + documentation: Option, + readme: Option, + keywords: Option>, + license: Option, + repository: Option, } #[deriving(Decodable)] @@ -284,7 +293,6 @@ impl TomlProject { struct Context<'a> { deps: &'a mut Vec, source_id: &'a SourceId, - source_ids: &'a mut Vec, nested_paths: &'a mut Vec } @@ -362,7 +370,6 @@ fn inferred_bench_targets(layout: &Layout) -> Vec { impl TomlManifest { pub fn to_manifest(&self, source_id: &SourceId, layout: &Layout) -> CargoResult<(Manifest, Vec)> { - let mut sources = vec!(); let mut nested_paths = vec!(); let project = self.project.as_ref().or_else(|| self.package.as_ref()); @@ -453,7 +460,6 @@ impl TomlManifest { let mut cx = Context { deps: &mut deps, source_id: source_id, - source_ids: &mut sources, nested_paths: &mut nested_paths }; @@ -472,13 +478,23 @@ impl TomlManifest { let summary = try!(Summary::new(pkgid, deps, self.features.clone() .unwrap_or(HashMap::new()))); + let metadata = ManifestMetadata { + description: project.description.clone(), + homepage: project.homepage.clone(), + documentation: project.documentation.clone(), + readme: project.readme.clone(), + authors: project.authors.clone(), + license: project.license.clone(), + repository: project.repository.clone(), + keywords: project.keywords.clone().unwrap_or(Vec::new()), + }; let mut manifest = Manifest::new(summary, targets, layout.root.join("target"), layout.root.join("doc"), - sources, build, - exclude); + exclude, + metadata); if used_deprecated_lib { manifest.add_warning(format!("the [[lib]] section has been \ deprecated in favor of [lib]")); @@ -510,14 +526,10 @@ fn process_dependencies<'a>(cx: &mut Context<'a>, dev: bool, let new_source_id = match details.git { Some(ref git) => { - let kind = GitKind(reference.clone()); let loc = try!(git.as_slice().to_url().map_err(|e| { human(e) })); - let source_id = SourceId::new(kind, loc); - // TODO: Don't do this for path - cx.source_ids.push(source_id.clone()); - Some(source_id) + Some(SourceId::for_git(&loc, reference.as_slice())) } None => { details.path.as_ref().map(|path| { diff --git a/src/doc/config.md b/src/doc/config.md index d0669cba1fd..fa6a69e57d5 100644 --- a/src/doc/config.md +++ b/src/doc/config.md @@ -68,7 +68,7 @@ linker = ".." # Configuration keys related to the registry [registry] -host = "..." # URL of the registry (defaults to the central repository) +index = "..." # URL of the registry index (defaults to the central repository) token = "..." # Access token (found on the central repo's website) [http] diff --git a/src/doc/manifest.md b/src/doc/manifest.md index 9d923661692..60905b86c96 100644 --- a/src/doc/manifest.md +++ b/src/doc/manifest.md @@ -64,6 +64,39 @@ the VCS's ignore settings (`.gitignore` for git for example). exclude = ["build/**/*.o", "doc/**/*.html"] ``` +## Package metadata + +There are a number of optional metadata fields also accepted under the +`[package]` section: + +```toml +[package] +# ... + +# A short blurb about the package. This is not rendered in any format when +# uploaded to registries. +description = "..." + +# These URLs point to more information about the repository +documentation = "..." +homepage = "..." +repository = "..." + +# This points to a file in the repository (relative to this Cargo.toml). The +# contents of this file are stored and indexed in the registry. +readme = "..." + +# This is a small list of keywords used to categorize and search for this +# package. +keywords = ["...", "..."] + +# This is a string description of the license for this package. Currently +# the registry will validate the license provided against a whitelist of known +# licenses. +license = "..." +``` + + # The `[dependencies.*]` Sections You list dependencies using `[dependencies.]`. For example, if you diff --git a/src/registry/Cargo.toml b/src/registry/Cargo.toml new file mode 100644 index 00000000000..695caa0f270 --- /dev/null +++ b/src/registry/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "registry" +version = "0.0.1-pre" +authors = ["Alex Crichton "] + +[lib] +name = "registry" +path = "lib.rs" + +[dependencies.curl] +git = "https://github.com/alexcrichton/curl-rust" +branch = "bundle" diff --git a/src/registry/lib.rs b/src/registry/lib.rs new file mode 100644 index 00000000000..2c5938c6b6c --- /dev/null +++ b/src/registry/lib.rs @@ -0,0 +1,197 @@ +extern crate curl; +extern crate serialize; + +use std::fmt; +use std::io::{mod, fs, MemReader, MemWriter, File}; +use std::collections::HashMap; +use std::io::util::ChainedReader; +use std::result; + +use curl::http; +use serialize::json; + +pub struct Registry { + host: String, + token: String, + handle: http::Handle, +} + +pub type Result = result::Result; + +pub enum Error { + CurlError(curl::ErrCode), + NotOkResponse(http::Response), + NonUtf8Body, + ApiErrors(Vec), + Unauthorized, + IoError(io::IoError), +} + +#[deriving(Encodable)] +pub struct NewCrate { + pub name: String, + pub vers: String, + pub deps: Vec, + pub features: HashMap>, + pub authors: Vec, + pub description: Option, + pub documentation: Option, + pub homepage: Option, + pub readme: Option, + pub keywords: Vec, + pub license: Option, + pub repository: Option, +} + +#[deriving(Encodable)] +pub struct NewCrateDependency { + pub optional: bool, + pub default_features: bool, + pub name: String, + pub features: Vec, + pub version_req: String, + pub target: Option, +} + +#[deriving(Decodable)] struct R { ok: bool } +#[deriving(Decodable)] struct ApiErrorList { errors: Vec } +#[deriving(Decodable)] struct ApiError { detail: String } +#[deriving(Encodable)] struct OwnersReq<'a> { users: &'a [&'a str] } + +impl Registry { + pub fn new(host: String, token: String) -> Registry { + Registry::new_handle(host, token, http::Handle::new()) + } + + pub fn new_handle(host: String, token: String, + handle: http::Handle) -> Registry { + Registry { + host: host, + token: token, + handle: handle, + } + } + + pub fn add_owners(&mut self, krate: &str, owners: &[&str]) -> Result<()> { + let body = json::encode(&OwnersReq { users: owners }); + let body = try!(self.put(format!("/crates/{}/owners", krate), + body.as_bytes())); + assert!(json::decode::(body.as_slice()).unwrap().ok); + Ok(()) + } + + pub fn remove_owners(&mut self, krate: &str, owners: &[&str]) -> Result<()> { + let body = json::encode(&OwnersReq { users: owners }); + let body = try!(self.delete(format!("/crates/{}/owners", krate), + Some(body.as_bytes()))); + assert!(json::decode::(body.as_slice()).unwrap().ok); + Ok(()) + } + + pub fn publish(&mut self, krate: &NewCrate, tarball: &Path) -> Result<()> { + let json = json::encode(krate); + // Prepare the body. The format of the upload request is: + // + // + // (metadata for the package) + // + // + let stat = try!(fs::stat(tarball).map_err(IoError)); + let header = { + let mut w = MemWriter::new(); + w.write_le_u32(json.len() as u32).unwrap(); + w.write_str(json.as_slice()).unwrap(); + w.write_le_u32(stat.size as u32).unwrap(); + MemReader::new(w.unwrap()) + }; + let tarball = try!(File::open(tarball).map_err(IoError)); + let size = stat.size as uint + header.get_ref().len(); + let mut body = ChainedReader::new(vec![box header as Box, + box tarball as Box].into_iter()); + + let url = format!("{}/api/v1/crates/new", self.host); + let response = handle(self.handle.put(url, &mut body) + .content_length(size) + .header("Authorization", + self.token.as_slice()) + .header("Accept", "application/json") + .exec()); + let _body = try!(response); + Ok(()) + } + + pub fn yank(&mut self, krate: &str, version: &str) -> Result<()> { + let body = try!(self.delete(format!("/crates/{}/{}/yank", krate, version), + None)); + assert!(json::decode::(body.as_slice()).unwrap().ok); + Ok(()) + } + + pub fn unyank(&mut self, krate: &str, version: &str) -> Result<()> { + let body = try!(self.put(format!("/crates/{}/{}/unyank", krate, version), + [])); + assert!(json::decode::(body.as_slice()).unwrap().ok); + Ok(()) + } + + fn put(&mut self, path: String, b: &[u8]) -> Result { + handle(self.handle.put(format!("{}/api/v1{}", self.host, path), b) + .header("Authorization", self.token.as_slice()) + .header("Accept", "application/json") + .content_type("application/json") + .exec()) + } + + fn delete(&mut self, path: String, b: Option<&[u8]>) -> Result { + let mut req = self.handle.delete(format!("{}/api/v1{}", self.host, path)) + .header("Authorization", self.token.as_slice()) + .header("Accept", "application/json") + .content_type("application/json"); + match b { + Some(b) => req = req.body(b), + None => {} + } + handle(req.exec()) + } +} + +fn handle(response: result::Result) + -> Result { + let response = try!(response.map_err(CurlError)); + match response.get_code() { + 0 => {} // file upload url sometimes + 200 => {} + 403 => return Err(Unauthorized), + _ => return Err(NotOkResponse(response)) + } + + let body = match String::from_utf8(response.move_body()) { + Ok(body) => body, + Err(..) => return Err(NonUtf8Body), + }; + match json::decode::(body.as_slice()) { + Ok(errors) => { + return Err(ApiErrors(errors.errors.into_iter().map(|s| s.detail) + .collect())) + } + Err(..) => {} + } + Ok(body) +} + +impl fmt::Show for Error { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match *self { + NonUtf8Body => write!(f, "reponse body was not utf-8"), + CurlError(ref err) => write!(f, "http error: {}", err), + NotOkResponse(ref resp) => { + write!(f, "failed to get a 200 OK response: {}", resp) + } + ApiErrors(ref errs) => { + write!(f, "api errors: {}", errs.connect(", ")) + } + Unauthorized => write!(f, "unauthorized API access"), + IoError(ref e) => write!(f, "io error: {}", e), + } + } +} diff --git a/tests/fixtures/bar-0.0.1.tar.gz b/tests/fixtures/bar-0.0.1.tar.gz deleted file mode 100644 index 55b2f5a6c6f..00000000000 Binary files a/tests/fixtures/bar-0.0.1.tar.gz and /dev/null differ diff --git a/tests/fixtures/foo-0.0.1.tar.gz b/tests/fixtures/foo-0.0.1.tar.gz deleted file mode 100644 index bf1fc952707..00000000000 Binary files a/tests/fixtures/foo-0.0.1.tar.gz and /dev/null differ diff --git a/tests/fixtures/hello.rs b/tests/fixtures/hello.rs deleted file mode 100644 index e7a694f5aad..00000000000 --- a/tests/fixtures/hello.rs +++ /dev/null @@ -1,3 +0,0 @@ -fn main() { - println!("WINNING"); -} diff --git a/tests/fixtures/notyet-0.0.1.tar.gz b/tests/fixtures/notyet-0.0.1.tar.gz deleted file mode 100644 index 5b8ae93676d..00000000000 Binary files a/tests/fixtures/notyet-0.0.1.tar.gz and /dev/null differ diff --git a/tests/resolve.rs b/tests/resolve.rs index 30c8c04fca8..44fae3425be 100644 --- a/tests/resolve.rs +++ b/tests/resolve.rs @@ -7,7 +7,7 @@ use std::collections::HashMap; use hamcrest::{assert_that, equal_to, contains}; -use cargo::core::source::{SourceId, RegistryKind, GitKind}; +use cargo::core::source::SourceId; use cargo::core::{Dependency, PackageId, Summary, Registry}; use cargo::util::{CargoResult, ToUrl}; use cargo::core::resolver::{mod, ResolveEverything}; @@ -29,7 +29,7 @@ trait ToDep { impl ToDep for &'static str { fn to_dep(self) -> Dependency { let url = "http://example.com".to_url().unwrap(); - let source_id = SourceId::new(RegistryKind, url); + let source_id = SourceId::for_registry(&url); Dependency::parse(self, Some("1.0.0"), &source_id).unwrap() } } @@ -71,7 +71,7 @@ macro_rules! pkg( fn registry_loc() -> SourceId { let remote = "http://example.com".to_url().unwrap(); - SourceId::new(RegistryKind, remote) + SourceId::for_registry(&remote) } fn pkg(name: &str) -> Summary { @@ -84,8 +84,7 @@ fn pkg_id(name: &str) -> PackageId { fn pkg_id_loc(name: &str, loc: &str) -> PackageId { let remote = loc.to_url(); - let source_id = SourceId::new(GitKind("master".to_string()), - remote.unwrap()); + let source_id = SourceId::for_git(&remote.unwrap(), "master"); PackageId::new(name, "1.0.0", &source_id).unwrap() } @@ -97,13 +96,13 @@ fn pkg_loc(name: &str, loc: &str) -> Summary { fn dep(name: &str) -> Dependency { dep_req(name, "1.0.0") } fn dep_req(name: &str, req: &str) -> Dependency { let url = "http://example.com".to_url().unwrap(); - let source_id = SourceId::new(RegistryKind, url); + let source_id = SourceId::for_registry(&url); Dependency::parse(name, Some(req), &source_id).unwrap() } fn dep_loc(name: &str, location: &str) -> Dependency { let url = location.to_url().unwrap(); - let source_id = SourceId::new(GitKind("master".to_string()), url); + let source_id = SourceId::for_git(&url, "master"); Dependency::parse(name, Some("1.0.0"), &source_id).unwrap() } diff --git a/tests/support/mod.rs b/tests/support/mod.rs index e22f0b9eeff..cb8f1644370 100644 --- a/tests/support/mod.rs +++ b/tests/support/mod.rs @@ -15,6 +15,7 @@ use support::paths::PathExt; pub mod paths; pub mod git; +pub mod registry; /* * @@ -514,3 +515,4 @@ pub static DOCTEST: &'static str = " Doc-tests"; pub static PACKAGING: &'static str = " Packaging"; pub static DOWNLOADING: &'static str = " Downloading"; pub static UPLOADING: &'static str = " Uploading"; +pub static VERIFYING: &'static str = " Verifying"; diff --git a/tests/support/registry.rs b/tests/support/registry.rs new file mode 100644 index 00000000000..bac46784df7 --- /dev/null +++ b/tests/support/registry.rs @@ -0,0 +1,132 @@ +use std::io::{mod, fs, File}; + +use flate2::Default; +use flate2::writer::GzEncoder; +use git2; +use serialize::hex::ToHex; +use tar::Archive; +use url::Url; + +use support::{ResultTest, project}; +use support::paths; +use support::git::repo; +use cargo::util::Sha256; + +pub fn registry_path() -> Path { paths::root().join("registry") } +pub fn registry() -> Url { Url::from_file_path(®istry_path()).unwrap() } +pub fn dl_path() -> Path { paths::root().join("dl") } +pub fn dl_url() -> Url { Url::from_file_path(&dl_path()).unwrap() } + +pub fn init() { + let config = paths::root().join(".cargo/config"); + fs::mkdir_recursive(&config.dir_path(), io::USER_DIR).assert(); + File::create(&config).write_str(format!(r#" + [registry] + index = "{reg}" + token = "api-token" + "#, reg = registry()).as_slice()).assert(); + + // Init a new registry + repo(®istry_path()) + .file("config.json", format!(r#" + {{"dl":"{}","api":""}} + "#, dl_url()).as_slice()) + .build(); +} + +pub fn mock_archive(name: &str, version: &str, deps: &[(&str, &str)]) { + let mut manifest = format!(r#" + [package] + name = "{}" + version = "{}" + authors = [] + "#, name, version); + for &(dep, req) in deps.iter() { + manifest.push_str(format!(r#" + [dependencies.{}] + version = "{}" + "#, dep, req).as_slice()); + } + let p = project(name) + .file("Cargo.toml", manifest.as_slice()) + .file("src/lib.rs", ""); + p.build(); + + let dst = mock_archive_dst(name, version); + fs::mkdir_recursive(&dst.dir_path(), io::USER_DIR).assert(); + let f = File::create(&dst).unwrap(); + let a = Archive::new(GzEncoder::new(f, Default)); + a.append(format!("{}-{}/Cargo.toml", name, version).as_slice(), + &mut File::open(&p.root().join("Cargo.toml")).unwrap()).unwrap(); + a.append(format!("{}-{}/src/lib.rs", name, version).as_slice(), + &mut File::open(&p.root().join("src/lib.rs")).unwrap()).unwrap(); + a.finish().unwrap(); +} + +pub fn mock_archive_dst(name: &str, version: &str) -> Path { + dl_path().join(name).join(version).join("download") +} + +pub fn mock_pkg(name: &str, version: &str, deps: &[(&str, &str)]) { + mock_pkg_yank(name, version, deps, false) +} + +pub fn mock_pkg_yank(name: &str, version: &str, deps: &[(&str, &str)], + yanked: bool) { + mock_archive(name, version, deps); + let c = File::open(&mock_archive_dst(name, version)).read_to_end().unwrap(); + let line = pkg(name, version, deps, cksum(c.as_slice()).as_slice(), yanked); + + let file = match name.len() { + 1 => format!("1/{}", name), + 2 => format!("2/{}", name), + 3 => format!("3/{}/{}", name.slice_to(1), name), + _ => format!("{}/{}/{}", name.slice(0, 2), name.slice(2, 4), name), + }; + publish(file.as_slice(), line.as_slice()); +} + +pub fn publish(file: &str, line: &str) { + let repo = git2::Repository::open(®istry_path()).unwrap(); + let mut index = repo.index().unwrap(); + { + let dst = registry_path().join(file); + let prev = File::open(&dst).read_to_string().unwrap_or(String::new()); + fs::mkdir_recursive(&dst.dir_path(), io::USER_DIR).unwrap(); + File::create(&dst).write_str((prev + line + "\n").as_slice()).unwrap(); + } + index.add_path(&Path::new(file)).unwrap(); + index.write().unwrap(); + let id = index.write_tree().unwrap(); + let tree = repo.find_tree(id).unwrap(); + let sig = repo.signature().unwrap(); + let parent = repo.refname_to_id("refs/heads/master").unwrap(); + let parent = repo.find_commit(parent).unwrap(); + repo.commit(Some("HEAD"), &sig, &sig, + "Another commit", &tree, + [&parent]).unwrap(); +} + +pub fn pkg(name: &str, vers: &str, deps: &[(&str, &str)], cksum: &str, + yanked: bool) -> String { + let deps = deps.iter().map(|&(a, b)| dep(a, b)).collect::>(); + format!("{{\"name\":\"{}\",\"vers\":\"{}\",\ + \"deps\":{},\"cksum\":\"{}\",\"features\":{{}},\ + \"yanked\":{}}}", + name, vers, deps, cksum, yanked) +} + +pub fn dep(name: &str, req: &str) -> String { + format!("{{\"name\":\"{}\",\ + \"req\":\"{}\",\ + \"features\":[],\ + \"default_features\":false,\ + \"target\":null,\ + \"optional\":false}}", name, req) +} + +pub fn cksum(s: &[u8]) -> String { + let mut sha = Sha256::new(); + sha.update(s); + sha.finish().to_hex() +} diff --git a/tests/test_cargo_compile_git_deps.rs b/tests/test_cargo_compile_git_deps.rs index 8b4da434cd2..82d166a86cb 100644 --- a/tests/test_cargo_compile_git_deps.rs +++ b/tests/test_cargo_compile_git_deps.rs @@ -696,16 +696,19 @@ test!(update_with_shared_deps { timer::sleep(Duration::milliseconds(1000)); // By default, not transitive updates + println!("dep1 update"); assert_that(p.process(cargo_dir().join("cargo")).arg("update").arg("dep1"), execs().with_stdout("")); // Specifying a precise rev to the old rev shouldn't actually update // anything because we already have the rev in the db. + println!("bar precise update"); assert_that(p.process(cargo_dir().join("cargo")).arg("update").arg("bar") .arg("--precise").arg(old_head.to_string()), execs().with_stdout("")); // Updating aggressively should, however, update the repo. + println!("dep1 aggressive update"); assert_that(p.process(cargo_dir().join("cargo")).arg("update").arg("dep1") .arg("--aggressive"), execs().with_stdout(format!("{} git repository `{}`", @@ -713,6 +716,7 @@ test!(update_with_shared_deps { git_project.url()))); // Make sure we still only compile one version of the git repo + println!("build"); assert_that(p.process(cargo_dir().join("cargo")).arg("build"), execs().with_stdout(format!("\ {compiling} bar v0.5.0 ({git}#[..]) @@ -1427,3 +1431,80 @@ test!(update_one_dep_in_repo_with_many_deps { Updating git repository `{}` ", foo.url()))); }) + +test!(switch_deps_does_not_update_transitive { + let transitive = git_repo("transitive", |project| { + project.file("Cargo.toml", r#" + [package] + name = "transitive" + version = "0.5.0" + authors = ["wycats@example.com"] + "#) + .file("src/lib.rs", "") + }).assert(); + let dep1 = git_repo("dep1", |project| { + project.file("Cargo.toml", format!(r#" + [package] + name = "dep" + version = "0.5.0" + authors = ["wycats@example.com"] + + [dependencies.transitive] + git = '{}' + "#, transitive.url()).as_slice()) + .file("src/lib.rs", "") + }).assert(); + let dep2 = git_repo("dep2", |project| { + project.file("Cargo.toml", format!(r#" + [package] + name = "dep" + version = "0.5.0" + authors = ["wycats@example.com"] + + [dependencies.transitive] + git = '{}' + "#, transitive.url()).as_slice()) + .file("src/lib.rs", "") + }).assert(); + + let p = project("project") + .file("Cargo.toml", format!(r#" + [project] + name = "project" + version = "0.5.0" + authors = [] + [dependencies.dep] + git = '{}' + "#, dep1.url()).as_slice()) + .file("src/main.rs", "fn main() {}"); + + p.build(); + assert_that(p.process(cargo_dir().join("cargo")).arg("build"), + execs().with_status(0) + .with_stdout(format!("\ +Updating git repository `{}` +Updating git repository `{}` +{compiling} transitive [..] +{compiling} dep [..] +{compiling} project [..] +", dep1.url(), transitive.url(), compiling = COMPILING))); + + // Update the dependency to point to the second repository, but this + // shouldn't update the transitive dependency which is the same. + File::create(&p.root().join("Cargo.toml")).write_str(format!(r#" + [project] + name = "project" + version = "0.5.0" + authors = [] + [dependencies.dep] + git = '{}' + "#, dep2.url()).as_slice()).unwrap(); + + assert_that(p.process(cargo_dir().join("cargo")).arg("build"), + execs().with_status(0) + .with_stdout(format!("\ +Updating git repository `{}` +{compiling} dep [..] +{compiling} project [..] +", dep2.url(), compiling = COMPILING))); +}) diff --git a/tests/test_cargo_package.rs b/tests/test_cargo_package.rs index de6a28049fb..cc5776c9d33 100644 --- a/tests/test_cargo_package.rs +++ b/tests/test_cargo_package.rs @@ -4,7 +4,7 @@ use tar::Archive; use flate2::reader::GzDecoder; use support::{project, execs, cargo_dir, ResultTest}; -use support::{PACKAGING}; +use support::{PACKAGING, VERIFYING, COMPILING}; use hamcrest::{assert_that, existing_file}; fn setup() { @@ -27,8 +27,12 @@ test!(simple { assert_that(p.cargo_process("package"), execs().with_status(0).with_stdout(format!("\ {packaging} foo v0.0.1 ({dir}) +{verifying} foo v0.0.1 ({dir}) +{compiling} foo v0.0.1 ({dir}[..]) ", packaging = PACKAGING, + verifying = VERIFYING, + compiling = COMPILING, dir = p.url()).as_slice())); assert_that(&p.root().join("foo-0.0.1.tar.gz"), existing_file()); assert_that(p.process(cargo_dir().join("cargo")).arg("package"), diff --git a/tests/test_cargo_upload.rs b/tests/test_cargo_publish.rs similarity index 61% rename from tests/test_cargo_upload.rs rename to tests/test_cargo_publish.rs index 7ad46c7cb3c..bf291d6fbf8 100644 --- a/tests/test_cargo_upload.rs +++ b/tests/test_cargo_publish.rs @@ -21,14 +21,15 @@ fn setup() { fs::mkdir_recursive(&config.dir_path(), io::USER_DIR).assert(); File::create(&config).write_str(format!(r#" [registry] - host = "{reg}" + index = "{reg}" token = "api-token" "#, reg = registry()).as_slice()).assert(); + fs::mkdir_recursive(&upload_path().join("api/v1/crates"), io::USER_DIR).assert(); repo(®istry_path()) .file("config.json", format!(r#"{{ - "dl": "", - "upload": "{}" + "dl": "{0}", + "api": "{0}" }}"#, upload())) .build(); } @@ -43,7 +44,7 @@ test!(simple { "#) .file("src/main.rs", "fn main() {}"); - assert_that(p.cargo_process("upload"), + assert_that(p.cargo_process("publish").arg("--no-verify"), execs().with_status(0).with_stdout(format!("\ {updating} registry `{reg}` {packaging} foo v0.0.1 ({dir}) @@ -55,7 +56,13 @@ test!(simple { dir = p.url(), reg = registry()).as_slice())); - let mut rdr = GzDecoder::new(File::open(&upload_path()).unwrap()).unwrap(); + let mut f = File::open(&upload_path().join("api/v1/crates/new")).unwrap(); + // Skip the metadata payload and the size of the tarball + let sz = f.read_le_u32().unwrap(); + f.seek(sz as i64 + 4, io::SeekCur).unwrap(); + + // Verify the tarball + let mut rdr = GzDecoder::new(f).unwrap(); assert_eq!(rdr.header().filename(), Some(b"foo-0.0.1.tar.gz")); let inner = MemReader::new(rdr.read_to_end().unwrap()); let ar = Archive::new(inner); @@ -81,12 +88,37 @@ test!(git_deps { "#) .file("src/main.rs", "fn main() {}"); - assert_that(p.cargo_process("upload").arg("-v"), + assert_that(p.cargo_process("publish").arg("-v").arg("--no-verify"), execs().with_status(101).with_stderr("\ -failed to upload package to registry: [..] +all dependencies must come from the same registry +dependency `foo` comes from git://path/to/nowhere instead +")); +}) + +test!(path_dependency_no_version { + let p = project("foo") + .file("Cargo.toml", r#" + [project] + name = "foo" + version = "0.0.1" + authors = [] + + [dependencies.bar] + path = "bar" + "#) + .file("src/main.rs", "fn main() {}") + .file("bar/Cargo.toml", r#" + [package] + name = "bar" + version = "0.0.1" + authors = [] + "#) + .file("bar/src/lib.rs", ""); -Caused by: - All dependencies must come from the same registry. -Dependency `foo` comes from git://path/to/nowhere instead + assert_that(p.cargo_process("publish"), + execs().with_status(101).with_stderr("\ +all path dependencies must have a version specified when being uploaded \ +to the registry +dependency `bar` does not specify a version ")); }) diff --git a/tests/test_cargo_registry.rs b/tests/test_cargo_registry.rs index 2c598c15360..596764b26a3 100644 --- a/tests/test_cargo_registry.rs +++ b/tests/test_cargo_registry.rs @@ -1,71 +1,14 @@ -use std::io::{mod, fs, File}; -use url::Url; -use git2; -use serialize::hex::ToHex; +use std::io::{fs, File}; -use support::{ResultTest, project, execs, cargo_dir}; -use support::{UPDATING, DOWNLOADING, COMPILING}; -use support::paths; -use support::git::repo; -use cargo::util::Sha256; +use support::{project, execs, cargo_dir}; +use support::{UPDATING, DOWNLOADING, COMPILING, PACKAGING, VERIFYING}; +use support::paths::{mod, PathExt}; +use support::registry as r; use hamcrest::assert_that; -fn registry_path() -> Path { paths::root().join("registry") } -fn registry() -> Url { Url::from_file_path(®istry_path()).unwrap() } -fn dl_path() -> Path { paths::root().join("dl") } -fn dl_url() -> Url { Url::from_file_path(&dl_path()).unwrap() } - -fn cksum(s: &[u8]) -> String { - let mut sha = Sha256::new(); - sha.update(s); - sha.finish().to_hex() -} - fn setup() { - let config = paths::root().join(".cargo/config"); - fs::mkdir_recursive(&config.dir_path(), io::USER_DIR).assert(); - File::create(&config).write_str(format!(r#" - [registry] - host = "{reg}" - token = "api-token" - "#, reg = registry()).as_slice()).assert(); - - // Prepare the "to download" artifacts - let foo = include_bin!("fixtures/foo-0.0.1.tar.gz"); - let bar = include_bin!("fixtures/bar-0.0.1.tar.gz"); - let notyet = include_bin!("fixtures/notyet-0.0.1.tar.gz"); - let foo_cksum = dl("pkg/foo/foo-0.0.1.tar.gz", foo); - let bar_cksum = dl("pkg/bar/bar-0.0.1.tar.gz", bar); - dl("pkg/bad-cksum/bad-cksum-0.0.1.tar.gz", foo); - let notyet = dl("pkg/notyet/notyet-0.0.1.tar.gz", notyet); - - // Init a new registry - repo(®istry_path()) - .file("config.json", format!(r#" - {{"dl":"{}","upload":""}} - "#, dl_url()).as_slice()) - .file("3/f/foo", pkg("foo", "0.0.1", [], &foo_cksum)) - .file("3/b/bar", pkg("bar", "0.0.1", ["foo||>=0.0.0"], &bar_cksum)) - .file("ba/d-/bad-cksum", pkg("bad-cksum", "0.0.1", [], &bar_cksum)) - .nocommit_file("no/ty/notyet", pkg("notyet", "0.0.1", [], ¬yet)) - .build(); - - fn pkg(name: &str, vers: &str, deps: &[&str], cksum: &String) -> String { - let deps: Vec = deps.iter().map(|s| { - format!("\"{}\"", s) - }).collect(); - let deps = deps.connect(","); - - format!(r#"{{"name":"{}","vers":"{}","deps":[{}],"cksum":"{}","features":{{}}}}"#, - name, vers, deps, cksum) - } - fn dl(path: &str, contents: &[u8]) -> String { - let dst = dl_path().join(path); - fs::mkdir_recursive(&dst.dir_path(), io::USER_DIR).assert(); - File::create(&dst).write(contents).unwrap(); - cksum(contents) - } + r::init(); } test!(simple { @@ -77,33 +20,35 @@ test!(simple { authors = [] [dependencies] - foo = ">= 0.0.0" + bar = ">= 0.0.0" "#) .file("src/main.rs", "fn main() {}"); + r::mock_pkg("bar", "0.0.1", []); + assert_that(p.cargo_process("build"), execs().with_status(0).with_stdout(format!("\ {updating} registry `{reg}` -{downloading} foo v0.0.1 (the package registry) -{compiling} foo v0.0.1 (the package registry) +{downloading} bar v0.0.1 (the package registry) +{compiling} bar v0.0.1 (the package registry) {compiling} foo v0.0.1 ({dir}) ", updating = UPDATING, downloading = DOWNLOADING, compiling = COMPILING, dir = p.url(), - reg = registry()).as_slice())); + reg = r::registry()).as_slice())); // Don't download a second time assert_that(p.cargo_process("build"), execs().with_status(0).with_stdout(format!("\ {updating} registry `{reg}` -[..] foo v0.0.1 (the package registry) +[..] bar v0.0.1 (the package registry) [..] foo v0.0.1 ({dir}) ", updating = UPDATING, dir = p.url(), - reg = registry()).as_slice())); + reg = r::registry()).as_slice())); }) test!(deps { @@ -119,12 +64,15 @@ test!(deps { "#) .file("src/main.rs", "fn main() {}"); + r::mock_pkg("baz", "0.0.1", []); + r::mock_pkg("bar", "0.0.1", [("baz", "*")]); + assert_that(p.cargo_process("build"), execs().with_status(0).with_stdout(format!("\ {updating} registry `{reg}` {downloading} [..] v0.0.1 (the package registry) {downloading} [..] v0.0.1 (the package registry) -{compiling} foo v0.0.1 (the package registry) +{compiling} baz v0.0.1 (the package registry) {compiling} bar v0.0.1 (the package registry) {compiling} foo v0.0.1 ({dir}) ", @@ -132,7 +80,7 @@ test!(deps { downloading = DOWNLOADING, compiling = COMPILING, dir = p.url(), - reg = registry()).as_slice())); + reg = r::registry()).as_slice())); }) test!(nonexistent { @@ -169,6 +117,9 @@ test!(bad_cksum { "#) .file("src/main.rs", "fn main() {}"); + r::mock_pkg("bad-cksum", "0.0.1", []); + File::create(&r::mock_archive_dst("bad-cksum", "0.0.1")).unwrap(); + assert_that(p.cargo_process("build").arg("-v"), execs().with_status(101).with_stderr("\ Unable to get packages from source @@ -201,18 +152,7 @@ location searched: the package registry version required: >= 0.0.0 ")); - // Add the package and commit - let repo = git2::Repository::open(®istry_path()).unwrap(); - let mut index = repo.index().unwrap(); - index.add_path(&Path::new("no/ty/notyet")).unwrap(); - let id = index.write_tree().unwrap(); - let tree = repo.find_tree(id).unwrap(); - let sig = repo.signature().unwrap(); - let parent = repo.refname_to_id("refs/heads/master").unwrap(); - let parent = repo.find_commit(parent).unwrap(); - repo.commit(Some("HEAD"), &sig, &sig, - "Another commit", &tree, - [&parent]).unwrap(); + r::mock_pkg("notyet", "0.0.1", []); assert_that(p.process(cargo_dir().join("cargo")).arg("build"), execs().with_status(0).with_stdout(format!("\ @@ -225,5 +165,278 @@ version required: >= 0.0.0 downloading = DOWNLOADING, compiling = COMPILING, dir = p.url(), - reg = registry()).as_slice())); + reg = r::registry()).as_slice())); +}) + +test!(package_with_path_deps { + let p = project("foo") + .file("Cargo.toml", r#" + [project] + name = "foo" + version = "0.0.1" + authors = [] + + [dependencies.notyet] + version = "0.0.1" + path = "notyet" + "#) + .file("src/main.rs", "fn main() {}") + .file("notyet/Cargo.toml", r#" + [package] + name = "notyet" + version = "0.0.1" + authors = [] + "#) + .file("notyet/src/lib.rs", ""); + p.build(); + + assert_that(p.process(cargo_dir().join("cargo")).arg("package").arg("-v"), + execs().with_status(101).with_stderr("\ +failed to verify package tarball + +Caused by: + no package named `notyet` found (required by `foo`) +location searched: the package registry +version required: ^0.0.1 +")); + + r::mock_pkg("notyet", "0.0.1", []); + + assert_that(p.process(cargo_dir().join("cargo")).arg("package"), + execs().with_status(0).with_stdout(format!("\ +{packaging} foo v0.0.1 ({dir}) +{verifying} foo v0.0.1 ({dir}) +{updating} registry `[..]` +{downloading} notyet v0.0.1 (the package registry) +{compiling} notyet v0.0.1 (the package registry) +{compiling} foo v0.0.1 ({dir}) +", + packaging = PACKAGING, + verifying = VERIFYING, + updating = UPDATING, + downloading = DOWNLOADING, + compiling = COMPILING, + dir = p.url(), +))); +}) + +test!(lockfile_locks { + let p = project("foo") + .file("Cargo.toml", r#" + [project] + name = "foo" + version = "0.0.1" + authors = [] + + [dependencies] + bar = "*" + "#) + .file("src/main.rs", "fn main() {}"); + p.build(); + + r::mock_pkg("bar", "0.0.1", []); + + assert_that(p.process(cargo_dir().join("cargo")).arg("build"), + execs().with_status(0).with_stdout(format!("\ +{updating} registry `[..]` +{downloading} bar v0.0.1 (the package registry) +{compiling} bar v0.0.1 (the package registry) +{compiling} foo v0.0.1 ({dir}) +", updating = UPDATING, downloading = DOWNLOADING, compiling = COMPILING, + dir = p.url()).as_slice())); + + p.root().move_into_the_past().unwrap(); + r::mock_pkg("bar", "0.0.2", []); + + assert_that(p.process(cargo_dir().join("cargo")).arg("build"), + execs().with_status(0).with_stdout("")); +}) + +test!(lockfile_locks_transitively { + let p = project("foo") + .file("Cargo.toml", r#" + [project] + name = "foo" + version = "0.0.1" + authors = [] + + [dependencies] + bar = "*" + "#) + .file("src/main.rs", "fn main() {}"); + p.build(); + + r::mock_pkg("baz", "0.0.1", []); + r::mock_pkg("bar", "0.0.1", [("baz", "*")]); + + assert_that(p.process(cargo_dir().join("cargo")).arg("build"), + execs().with_status(0).with_stdout(format!("\ +{updating} registry `[..]` +{downloading} [..] v0.0.1 (the package registry) +{downloading} [..] v0.0.1 (the package registry) +{compiling} baz v0.0.1 (the package registry) +{compiling} bar v0.0.1 (the package registry) +{compiling} foo v0.0.1 ({dir}) +", updating = UPDATING, downloading = DOWNLOADING, compiling = COMPILING, + dir = p.url()).as_slice())); + + p.root().move_into_the_past().unwrap(); + r::mock_pkg("baz", "0.0.2", []); + r::mock_pkg("bar", "0.0.2", [("baz", "*")]); + + assert_that(p.process(cargo_dir().join("cargo")).arg("build"), + execs().with_status(0).with_stdout("")); +}) + +test!(yanks_are_not_used { + let p = project("foo") + .file("Cargo.toml", r#" + [project] + name = "foo" + version = "0.0.1" + authors = [] + + [dependencies] + bar = "*" + "#) + .file("src/main.rs", "fn main() {}"); + p.build(); + + r::mock_pkg("baz", "0.0.1", []); + r::mock_pkg_yank("baz", "0.0.2", [], true); + r::mock_pkg("bar", "0.0.1", [("baz", "*")]); + r::mock_pkg_yank("bar", "0.0.2", [("baz", "*")], true); + + assert_that(p.process(cargo_dir().join("cargo")).arg("build"), + execs().with_status(0).with_stdout(format!("\ +{updating} registry `[..]` +{downloading} [..] v0.0.1 (the package registry) +{downloading} [..] v0.0.1 (the package registry) +{compiling} baz v0.0.1 (the package registry) +{compiling} bar v0.0.1 (the package registry) +{compiling} foo v0.0.1 ({dir}) +", updating = UPDATING, downloading = DOWNLOADING, compiling = COMPILING, + dir = p.url()).as_slice())); +}) + +test!(relying_on_a_yank_is_bad { + let p = project("foo") + .file("Cargo.toml", r#" + [project] + name = "foo" + version = "0.0.1" + authors = [] + + [dependencies] + bar = "*" + "#) + .file("src/main.rs", "fn main() {}"); + p.build(); + + r::mock_pkg("baz", "0.0.1", []); + r::mock_pkg_yank("baz", "0.0.2", [], true); + r::mock_pkg("bar", "0.0.1", [("baz", "=0.0.2")]); + + assert_that(p.process(cargo_dir().join("cargo")).arg("build"), + execs().with_status(101).with_stderr("\ +no package named `baz` found (required by `bar`) +location searched: the package registry +version required: = 0.0.2 +")); +}) + +test!(yanks_in_lockfiles_are_ok { + let p = project("foo") + .file("Cargo.toml", r#" + [project] + name = "foo" + version = "0.0.1" + authors = [] + + [dependencies] + bar = "*" + "#) + .file("src/main.rs", "fn main() {}"); + p.build(); + + r::mock_pkg("bar", "0.0.1", []); + + assert_that(p.process(cargo_dir().join("cargo")).arg("build"), + execs().with_status(0)); + + fs::rmdir_recursive(&r::registry_path().join("3")).unwrap(); + + r::mock_pkg_yank("bar", "0.0.1", [], true); + + assert_that(p.process(cargo_dir().join("cargo")).arg("build"), + execs().with_status(0).with_stdout("")); + + assert_that(p.process(cargo_dir().join("cargo")).arg("update"), + execs().with_status(101).with_stderr("\ +no package named `bar` found (required by `foo`) +location searched: the package registry +version required: * +")); +}) + +test!(update_with_lockfile_if_packages_missing { + let p = project("foo") + .file("Cargo.toml", r#" + [project] + name = "foo" + version = "0.0.1" + authors = [] + + [dependencies] + bar = "*" + "#) + .file("src/main.rs", "fn main() {}"); + p.build(); + + r::mock_pkg("bar", "0.0.1", []); + assert_that(p.process(cargo_dir().join("cargo")).arg("build"), + execs().with_status(0)); + p.root().move_into_the_past().unwrap(); + + fs::rmdir_recursive(&paths::home().join(".cargo/registry")).unwrap(); + assert_that(p.process(cargo_dir().join("cargo")).arg("build"), + execs().with_status(0).with_stdout(format!("\ +{updating} registry `[..]` +{downloading} bar v0.0.1 (the package registry) +", updating = UPDATING, downloading = DOWNLOADING).as_slice())); +}) + +test!(update_lockfile { + let p = project("foo") + .file("Cargo.toml", r#" + [project] + name = "foo" + version = "0.0.1" + authors = [] + + [dependencies] + bar = "*" + "#) + .file("src/main.rs", "fn main() {}"); + p.build(); + + r::mock_pkg("bar", "0.0.1", []); + assert_that(p.process(cargo_dir().join("cargo")).arg("build"), + execs().with_status(0)); + + r::mock_pkg("bar", "0.0.2", []); + fs::rmdir_recursive(&paths::home().join(".cargo/registry")).unwrap(); + assert_that(p.process(cargo_dir().join("cargo")).arg("update") + .arg("-p").arg("bar"), + execs().with_status(0).with_stdout(format!("\ +{updating} registry `[..]` +", updating = UPDATING).as_slice())); + + assert_that(p.process(cargo_dir().join("cargo")).arg("build"), + execs().with_status(0).with_stdout(format!("\ +{downloading} [..] v0.0.2 (the package registry) +{compiling} bar v0.0.2 (the package registry) +{compiling} foo v0.0.1 ({dir}) +", downloading = DOWNLOADING, compiling = COMPILING, + dir = p.url()).as_slice())); }) diff --git a/tests/tests.rs b/tests/tests.rs index 488a4f66ff4..6807fae6ded 100644 --- a/tests/tests.rs +++ b/tests/tests.rs @@ -46,5 +46,5 @@ mod test_cargo_profiles; mod test_cargo_package; mod test_cargo_build_auth; mod test_cargo_registry; -mod test_cargo_upload; +mod test_cargo_publish; mod test_cargo_fetch;