diff --git a/crates/cargo-test-support/src/publish.rs b/crates/cargo-test-support/src/publish.rs index e66f1902762..31ebb3f1695 100644 --- a/crates/cargo-test-support/src/publish.rs +++ b/crates/cargo-test-support/src/publish.rs @@ -30,12 +30,16 @@ pub fn validate_upload(expected_json: &str, expected_crate_name: &str, expected_ /// Checks the result of a crate publish, along with the contents of the files. pub fn validate_upload_with_contents( + version: &str, expected_json: &str, expected_crate_name: &str, expected_files: &[&str], expected_contents: &[(&str, &str)], ) { - let new_path = registry::api_path().join("api/v1/crates/new"); + let new_path = registry::api_path() + .join("api") + .join(version) + .join("crates/new"); _validate_upload( &new_path, expected_json, diff --git a/crates/cargo-test-support/src/registry.rs b/crates/cargo-test-support/src/registry.rs index e2e3b03024d..a01c9907237 100644 --- a/crates/cargo-test-support/src/registry.rs +++ b/crates/cargo-test-support/src/registry.rs @@ -4,10 +4,13 @@ use cargo::sources::CRATES_IO_INDEX; use cargo::util::Sha256; use flate2::write::GzEncoder; use flate2::Compression; -use std::collections::HashMap; +use std::collections::BTreeMap; +use std::fmt::Write as _; use std::fs::{self, File}; -use std::io::prelude::*; +use std::io::{BufRead, BufReader, Write}; +use std::net::TcpListener; use std::path::{Path, PathBuf}; +use std::thread; use tar::{Builder, Header}; use url::Url; @@ -70,6 +73,183 @@ pub fn generate_alt_dl_url(name: &str) -> String { format!("{}/{{crate}}/{{version}}/{{crate}}-{{version}}.crate", base) } +/// A builder for initializing registries. +pub struct RegistryBuilder { + /// If `true`, adds source replacement for crates.io to a registry on the filesystem. + replace_crates_io: bool, + /// If `true`, configures a registry named "alternative". + alternative: bool, + /// If set, sets the API url for the "alternative" registry. + /// This defaults to a directory on the filesystem. + alt_api_url: Option, + /// If `true`, configures `.cargo/credentials` with some tokens. + add_tokens: bool, +} + +impl RegistryBuilder { + pub fn new() -> RegistryBuilder { + RegistryBuilder { + replace_crates_io: true, + alternative: false, + alt_api_url: None, + add_tokens: true, + } + } + + /// Sets whether or not to replace crates.io with a registry on the filesystem. + /// Default is `true`. + pub fn replace_crates_io(&mut self, replace: bool) -> &mut Self { + self.replace_crates_io = replace; + self + } + + /// Sets whether or not to initialize an alternative registry named "alternative". + /// Default is `false`. + pub fn alternative(&mut self, alt: bool) -> &mut Self { + self.alternative = alt; + self + } + + /// Sets the API url for the "alternative" registry. + /// Defaults to a path on the filesystem ([`alt_api_path`]). + pub fn alternative_api_url(&mut self, url: &str) -> &mut Self { + self.alternative = true; + self.alt_api_url = Some(url.to_string()); + self + } + + /// Sets whether or not to initialize `.cargo/credentials` with some tokens. + /// Defaults to `true`. + pub fn add_tokens(&mut self, add: bool) -> &mut Self { + self.add_tokens = add; + self + } + + /// Initializes the registries. + pub fn build(&self) { + let config_path = paths::home().join(".cargo/config"); + if config_path.exists() { + panic!( + "{} already exists, the registry may only be initialized once, \ + and must be done before the config file is created", + config_path.display() + ); + } + t!(fs::create_dir_all(config_path.parent().unwrap())); + let mut config = String::new(); + if self.replace_crates_io { + write!( + &mut config, + " + [source.crates-io] + replace-with = 'dummy-registry' + + [source.dummy-registry] + registry = '{}' + ", + registry_url() + ) + .unwrap(); + } + if self.alternative { + write!( + config, + " + [registries.alternative] + index = '{}' + ", + alt_registry_url() + ) + .unwrap(); + } + t!(fs::write(&config_path, config)); + + if self.add_tokens { + let credentials = paths::home().join(".cargo/credentials"); + t!(fs::write( + &credentials, + r#" + [registry] + token = "api-token" + + [registries.alternative] + token = "api-token" + "# + )); + } + + if self.replace_crates_io { + init_registry( + registry_path(), + dl_url().into_string(), + api_url(), + api_path(), + ); + } + + if self.alternative { + init_registry( + alt_registry_path(), + alt_dl_url(), + self.alt_api_url + .as_ref() + .map_or_else(alt_api_url, |url| Url::parse(&url).expect("valid url")), + alt_api_path(), + ); + } + } + + /// Initializes the registries, and sets up an HTTP server for the + /// "alternative" registry. + /// + /// The given callback takes a `Vec` of headers when a request comes in. + /// The first entry should be the HTTP command, such as + /// `PUT /api/v1/crates/new HTTP/1.1`. + /// + /// The callback should return the HTTP code for the response, and the + /// response body. + /// + /// This method returns a `JoinHandle` which you should call + /// `.join().unwrap()` on before exiting the test. + pub fn build_api_server<'a>( + &mut self, + handler: &'static (dyn (Fn(Vec) -> (u32, &'a dyn AsRef<[u8]>)) + Sync), + ) -> thread::JoinHandle<()> { + let server = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = server.local_addr().unwrap(); + let api_url = format!("http://{}", addr); + + self.replace_crates_io(false) + .alternative_api_url(&api_url) + .build(); + + let t = thread::spawn(move || { + let mut conn = BufReader::new(server.accept().unwrap().0); + let headers: Vec<_> = (&mut conn) + .lines() + .map(|s| s.unwrap()) + .take_while(|s| s.len() > 2) + .map(|s| s.trim().to_string()) + .collect(); + let (code, response) = handler(headers); + let response = response.as_ref(); + let stream = conn.get_mut(); + write!( + stream, + "HTTP/1.1 {}\r\n\ + Content-Length: {}\r\n\ + \r\n", + code, + response.len() + ) + .unwrap(); + stream.write_all(response).unwrap(); + }); + + t + } +} + /// A builder for creating a new package in a registry. /// /// This uses "source replacement" using an automatically generated @@ -140,7 +320,7 @@ pub struct Package { files: Vec<(String, String)>, extra_files: Vec<(String, String)>, yanked: bool, - features: HashMap>, + features: BTreeMap>, local: bool, alternative: bool, invalid_json: bool, @@ -148,6 +328,7 @@ pub struct Package { links: Option, rust_version: Option, cargo_features: Vec, + v: Option, } #[derive(Clone)] @@ -162,70 +343,28 @@ pub struct Dependency { optional: bool, } +/// Initializes the on-disk registry and sets up the config so that crates.io +/// is replaced with the one on disk. pub fn init() { let config = paths::home().join(".cargo/config"); - t!(fs::create_dir_all(config.parent().unwrap())); if config.exists() { return; } - t!(fs::write( - &config, - format!( - r#" - [source.crates-io] - registry = 'https://wut' - replace-with = 'dummy-registry' - - [source.dummy-registry] - registry = '{reg}' - - [registries.alternative] - index = '{alt}' - "#, - reg = registry_url(), - alt = alt_registry_url() - ) - )); - let credentials = paths::home().join(".cargo/credentials"); - t!(fs::write( - &credentials, - r#" - [registry] - token = "api-token" - - [registries.alternative] - token = "api-token" - "# - )); + RegistryBuilder::new().build(); +} - // Initialize a new registry. - init_registry( - registry_path(), - dl_url().into_string(), - api_url(), - api_path(), - ); - - // Initialize an alternative registry. - init_registry( - alt_registry_path(), - alt_dl_url(), - alt_api_url(), - alt_api_path(), - ); +/// Variant of `init` that initializes the "alternative" registry. +pub fn alt_init() { + RegistryBuilder::new().alternative(true).build(); } +/// Creates a new on-disk registry. pub fn init_registry(registry_path: PathBuf, dl_url: String, api_url: Url, api_path: PathBuf) { // Initialize a new registry. repo(®istry_path) .file( "config.json", - &format!( - r#" - {{"dl":"{}","api":"{}"}} - "#, - dl_url, api_url - ), + &format!(r#"{{"dl":"{}","api":"{}"}}"#, dl_url, api_url), ) .build(); fs::create_dir_all(api_path.join("api/v1/crates")).unwrap(); @@ -243,7 +382,7 @@ impl Package { files: Vec::new(), extra_files: Vec::new(), yanked: false, - features: HashMap::new(), + features: BTreeMap::new(), local: false, alternative: false, invalid_json: false, @@ -251,6 +390,7 @@ impl Package { links: None, rust_version: None, cargo_features: Vec::new(), + v: None, } } @@ -390,6 +530,14 @@ impl Package { self } + /// Sets the index schema version for this package. + /// + /// See [`cargo::sources::registry::RegistryPackage`] for more information. + pub fn schema_version(&mut self, version: u32) -> &mut Package { + self.v = Some(version); + self + } + /// Creates the package and place it in the registry. /// /// This does not actually use Cargo's publishing system, but instead @@ -435,16 +583,24 @@ impl Package { } else { serde_json::json!(self.name) }; - let line = serde_json::json!({ + let (features, features2) = cargo::ops::features_for_publish(Some(&self.features)); + let mut json = serde_json::json!({ "name": name, "vers": self.vers, "deps": deps, "cksum": cksum, - "features": self.features, + "features": features, "yanked": self.yanked, "links": self.links, - }) - .to_string(); + }); + if let Some(f2) = &features2 { + json["features2"] = serde_json::json!(f2); + json["v"] = serde_json::json!(2); + } + if let Some(v) = self.v { + json["v"] = serde_json::json!(v); + } + let line = json.to_string(); let file = match self.name.len() { 1 => format!("1/{}", self.name), diff --git a/crates/crates-io/lib.rs b/crates/crates-io/lib.rs index 6f6769eaf04..006cff1cc2e 100644 --- a/crates/crates-io/lib.rs +++ b/crates/crates-io/lib.rs @@ -2,12 +2,13 @@ #![allow(clippy::identity_op)] // used for vertical alignment use std::collections::BTreeMap; +use std::fmt; use std::fs::File; use std::io::prelude::*; use std::io::{Cursor, SeekFrom}; use std::time::Instant; -use anyhow::{bail, Context, Result}; +use anyhow::{bail, format_err, Context, Result}; use curl::easy::{Easy, List}; use percent_encoding::{percent_encode, NON_ALPHANUMERIC}; use serde::{Deserialize, Serialize}; @@ -36,12 +37,16 @@ pub struct Crate { pub max_version: String, } +pub type NewFeatureMap = BTreeMap>; + #[derive(Serialize)] pub struct NewCrate { pub name: String, pub vers: String, pub deps: Vec, - pub features: BTreeMap>, + pub features: NewFeatureMap, + #[serde(skip_serializing_if = "Option::is_none")] + pub features2: Option, pub authors: Vec, pub description: Option, pub documentation: Option, @@ -55,6 +60,8 @@ pub struct NewCrate { pub repository: Option, pub badges: BTreeMap>, pub links: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub v: Option, } #[derive(Serialize)] @@ -100,7 +107,7 @@ struct OwnerResponse { struct ApiErrorList { errors: Vec, } -#[derive(Deserialize)] +#[derive(Debug, Deserialize)] struct ApiError { detail: String, } @@ -121,6 +128,70 @@ struct Crates { crates: Vec, meta: TotalCrates, } + +#[derive(Debug)] +pub enum ResponseError { + Curl(curl::Error), + Api { + code: u32, + errors: Vec, + }, + Code { + code: u32, + headers: Vec, + body: String, + }, + Other(anyhow::Error), +} + +impl std::error::Error for ResponseError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + ResponseError::Curl(..) => None, + ResponseError::Api { .. } => None, + ResponseError::Code { .. } => None, + ResponseError::Other(e) => Some(e.as_ref()), + } + } +} + +impl fmt::Display for ResponseError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + ResponseError::Curl(e) => write!(f, "{}", e), + ResponseError::Api { code, errors } => write!( + f, + "api errors (status {} {}): {}", + code, + reason(*code), + errors.join(", ") + ), + ResponseError::Code { + code, + headers, + body, + } => write!( + f, + "failed to get a 200 OK response, got {}\n\ + headers:\n\ + \t{}\n\ + body:\n\ + {}", + code, + headers.join("\n\t"), + body + ), + ResponseError::Other(..) => write!(f, "invalid response from server"), + } + } +} + +impl From for ResponseError { + fn from(error: curl::Error) -> Self { + ResponseError::Curl(error) + } +} + impl Registry { /// Creates a new `Registry`. /// @@ -200,21 +271,45 @@ impl Registry { let size = tarball_len as usize + header.len(); let mut body = Cursor::new(header).chain(tarball); - let url = format!("{}/api/v1/crates/new", self.host); + self.set_url(krate.v.unwrap_or(1), "/crates/new")?; let token = match self.token.as_ref() { Some(s) => s, None => bail!("no upload token found, please run `cargo login`"), }; self.handle.put(true)?; - self.handle.url(&url)?; self.handle.in_filesize(size as u64)?; let mut headers = List::new(); headers.append("Accept: application/json")?; headers.append(&format!("Authorization: {}", token))?; self.handle.http_headers(headers)?; - let body = self.handle(&mut |buf| body.read(buf).unwrap_or(0))?; + let started = Instant::now(); + let body = self + .handle(&mut |buf| body.read(buf).unwrap_or(0)) + .map_err(|e| match e { + ResponseError::Code { code, .. } + if code == 503 + && started.elapsed().as_secs() >= 29 + && self.host_is_crates_io() => + { + format_err!( + "Request timed out after 30 seconds. If you're trying to \ + upload a crate it may be too large. If the crate is under \ + 10MB in size, you can email help@crates.io for assistance.\n\ + Total size was {}.", + tarball_len + ) + } + ResponseError::Code { code, .. } if code == 404 && krate.features2.is_some() => { + format_err!( + "This package uses new feature syntax that is not supported by \ + the registry at {}.", + self.host, + ) + } + _ => e.into(), + })?; let response = if body.is_empty() { "{}".parse()? @@ -289,8 +384,16 @@ impl Registry { self.req(path, b, Auth::Authorized) } + fn set_url(&mut self, version: u32, path: &str) -> Result<()> { + assert!(path.starts_with('/')); + let sep = if self.host.ends_with('/') { "" } else { "/" }; + self.handle + .url(&format!("{}{}api/v{}{}", self.host, sep, version, path))?; + Ok(()) + } + fn req(&mut self, path: &str, body: Option<&[u8]>, authorized: Auth) -> Result { - self.handle.url(&format!("{}/api/v1{}", self.host, path))?; + self.set_url(1, path)?; let mut headers = List::new(); headers.append("Accept: application/json")?; headers.append("Content-Type: application/json")?; @@ -308,15 +411,18 @@ impl Registry { self.handle.upload(true)?; self.handle.in_filesize(body.len() as u64)?; self.handle(&mut |buf| body.read(buf).unwrap_or(0)) + .map_err(|e| e.into()) } - None => self.handle(&mut |_| 0), + None => self.handle(&mut |_| 0).map_err(|e| e.into()), } } - fn handle(&mut self, read: &mut dyn FnMut(&mut [u8]) -> usize) -> Result { + fn handle( + &mut self, + read: &mut dyn FnMut(&mut [u8]) -> usize, + ) -> std::result::Result { let mut headers = Vec::new(); let mut body = Vec::new(); - let started; { let mut handle = self.handle.transfer(); handle.read_function(|buf| Ok(read(buf)))?; @@ -325,50 +431,36 @@ impl Registry { Ok(data.len()) })?; handle.header_function(|data| { - headers.push(String::from_utf8_lossy(data).into_owned()); + // Headers contain trailing \r\n, trim them to make it easier + // to work with. + let s = String::from_utf8_lossy(data).trim().to_string(); + headers.push(s); true })?; - started = Instant::now(); handle.perform()?; } let body = match String::from_utf8(body) { Ok(body) => body, - Err(..) => bail!("response body was not valid utf-8"), + Err(..) => { + return Err(ResponseError::Other(format_err!( + "response body was not valid utf-8" + ))) + } }; let errors = serde_json::from_str::(&body) .ok() .map(|s| s.errors.into_iter().map(|s| s.detail).collect::>()); match (self.handle.response_code()?, errors) { - (0, None) | (200, None) => {} - (503, None) if started.elapsed().as_secs() >= 29 && self.host_is_crates_io() => bail!( - "Request timed out after 30 seconds. If you're trying to \ - upload a crate it may be too large. If the crate is under \ - 10MB in size, you can email help@crates.io for assistance." - ), - (code, Some(errors)) => { - let reason = reason(code); - bail!( - "api errors (status {} {}): {}", - code, - reason, - errors.join(", ") - ) - } - (code, None) => bail!( - "failed to get a 200 OK response, got {}\n\ - headers:\n\ - \t{}\n\ - body:\n\ - {}", + (0, None) | (200, None) => Ok(body), + (code, Some(errors)) => Err(ResponseError::Api { code, errors }), + (code, None) => Err(ResponseError::Code { code, - headers.join("\n\t"), + headers, body, - ), + }), } - - Ok(body) } } diff --git a/src/cargo/ops/mod.rs b/src/cargo/ops/mod.rs index 4853993e4a5..a499341cbbd 100644 --- a/src/cargo/ops/mod.rs +++ b/src/cargo/ops/mod.rs @@ -19,11 +19,11 @@ pub use self::cargo_test::{run_benches, run_tests, TestOptions}; pub use self::cargo_uninstall::uninstall; pub use self::fix::{fix, fix_maybe_exec_rustc, FixOptions}; pub use self::lockfile::{load_pkg_lockfile, resolve_to_string, write_pkg_lockfile}; -pub use self::registry::HttpTimeout; -pub use self::registry::{configure_http_handle, http_handle, http_handle_and_timeout}; -pub use self::registry::{modify_owners, yank, OwnersOptions, PublishOpts}; -pub use self::registry::{needs_custom_http_transport, registry_login, registry_logout, search}; -pub use self::registry::{publish, registry_configuration, RegistryConfig}; +pub use self::registry::{ + configure_http_handle, features_for_publish, http_handle, http_handle_and_timeout, + modify_owners, needs_custom_http_transport, publish, registry_configuration, registry_login, + registry_logout, search, yank, HttpTimeout, OwnersOptions, PublishOpts, RegistryConfig, +}; pub use self::resolve::{ add_overrides, get_resolved_packages, resolve_with_previous, resolve_ws, resolve_ws_with_opts, }; diff --git a/src/cargo/ops/registry.rs b/src/cargo/ops/registry.rs index afb1adbf9d2..4cc7a6f2c84 100644 --- a/src/cargo/ops/registry.rs +++ b/src/cargo/ops/registry.rs @@ -8,7 +8,7 @@ use std::time::Duration; use std::{cmp, env}; use anyhow::{bail, format_err}; -use crates_io::{self, NewCrate, NewCrateDependency, Registry}; +use crates_io::{self, NewCrate, NewCrateDependency, NewFeatureMap, Registry}; use curl::easy::{Easy, InfoType, SslOpt, SslVersion}; use log::{log, Level}; use percent_encoding::{percent_encode, NON_ALPHANUMERIC}; @@ -273,25 +273,16 @@ fn transmit( return Ok(()); } - let string_features = match manifest.original().features() { - Some(features) => features - .iter() - .map(|(feat, values)| { - ( - feat.to_string(), - values.iter().map(|fv| fv.to_string()).collect(), - ) - }) - .collect::>>(), - None => BTreeMap::new(), - }; + let (features, features2) = features_for_publish(manifest.original().features()); + let v = if features2.is_some() { Some(2) } else { None }; let publish = registry.publish( &NewCrate { name: pkg.name().to_string(), vers: pkg.version().to_string(), deps, - features: string_features, + features, + features2, authors: authors.clone(), description: description.clone(), homepage: homepage.clone(), @@ -305,6 +296,7 @@ fn transmit( license_file: license_file.clone(), badges: badges.clone(), links: links.clone(), + v, }, tarball, ); @@ -346,6 +338,44 @@ fn transmit( } } +/// Extracts the features for publishing, since publishing needs new feature +/// syntax to be separated. +pub fn features_for_publish>( + features: Option<&BTreeMap>>, +) -> (NewFeatureMap, Option) { + let features = match features { + Some(features) => features, + None => return (NewFeatureMap::new(), None), + }; + let mut features: NewFeatureMap = features + .iter() + .map(|(feat, values)| { + ( + feat.as_ref().to_string(), + values.iter().map(|v| v.as_ref().to_string()).collect(), + ) + }) + .collect(); + let mut features2 = NewFeatureMap::new(); + for (feat, values) in features.iter_mut() { + if values + .iter() + .any(|value| value.starts_with("dep:") || value.contains("?/")) + { + // This leaves the old feature with an empty list, otherwise 1.11 + // and older will fail due to missing features, even with + // Cargo.lock. + let new_values = values.drain(..).collect(); + features2.insert(feat.clone(), new_values); + } + } + if features2.is_empty() { + (features, None) + } else { + (features, Some(features2)) + } +} + /// Returns the index and token from the config file for the given registry. /// /// `registry` is typically the registry specified on the command-line. If diff --git a/src/cargo/sources/registry/index.rs b/src/cargo/sources/registry/index.rs index f7690f3d652..b676ee2f6e6 100644 --- a/src/cargo/sources/registry/index.rs +++ b/src/cargo/sources/registry/index.rs @@ -68,10 +68,11 @@ use crate::core::dependency::Dependency; use crate::core::{PackageId, SourceId, Summary}; -use crate::sources::registry::{RegistryData, RegistryPackage}; +use crate::sources::registry::{RegistryData, RegistryPackage, INDEX_SCHEMA_VERSION}; use crate::util::interning::InternedString; use crate::util::paths; use crate::util::{internal, CargoResult, Config, Filesystem, ToSemver}; +use anyhow::bail; use log::info; use semver::{Version, VersionReq}; use std::collections::{HashMap, HashSet}; @@ -164,10 +165,28 @@ fn overflow_hyphen() { ) } +/// Manager for handling the on-disk index. +/// +/// Note that local and remote registries store the index differently. Local +/// is a simple on-disk tree of files of the raw index. Remote registries are +/// stored as a raw git repository. The different means of access are handled +/// via the [`RegistryData`] trait abstraction. +/// +/// This transparently handles caching of the index in a more efficient format. pub struct RegistryIndex<'cfg> { source_id: SourceId, + /// Root directory of the index for the registry. path: Filesystem, + /// Cache of summary data. + /// + /// This is keyed off the package name. The [`Summaries`] value handles + /// loading the summary data. It keeps an optimized on-disk representation + /// of the JSON files, which is created in an as-needed fashion. If it + /// hasn't been cached already, it uses [`RegistryData::load`] to access + /// to JSON files from the index, and the creates the optimized on-disk + /// summary cache. summaries_cache: HashMap, + /// [`Config`] reference for convenience. config: &'cfg Config, } @@ -641,19 +660,19 @@ impl<'a> SummariesCache<'a> { .split_first() .ok_or_else(|| anyhow::format_err!("malformed cache"))?; if *first_byte != CURRENT_CACHE_VERSION { - anyhow::bail!("looks like a different Cargo's cache, bailing out"); + bail!("looks like a different Cargo's cache, bailing out"); } let mut iter = split(rest, 0); if let Some(update) = iter.next() { if update != last_index_update.as_bytes() { - anyhow::bail!( + bail!( "cache out of date: current index ({}) != cache ({})", last_index_update, str::from_utf8(update)?, ) } } else { - anyhow::bail!("malformed file"); + bail!("malformed file"); } let mut ret = SummariesCache::default(); while let Some(version) = iter.next() { @@ -728,16 +747,27 @@ impl IndexSummary { vers, cksum, deps, - features, + mut features, + features2, yanked, links, + v, } = serde_json::from_slice(line)?; + let v = v.unwrap_or(1); + if v > INDEX_SCHEMA_VERSION { + bail!("unsupported schema version {} ({} {})", v, name, vers); + } log::trace!("json parsed registry {}/{}", name, vers); let pkgid = PackageId::new(name, &vers, source_id)?; let deps = deps .into_iter() .map(|dep| dep.into_dep(source_id)) .collect::>>()?; + if let Some(features2) = features2 { + for (name, values) in features2 { + features.entry(name).or_default().extend(values); + } + } let mut summary = Summary::new(config, pkgid, deps, &features, links)?; summary.set_checksum(cksum); Ok(IndexSummary { diff --git a/src/cargo/sources/registry/local.rs b/src/cargo/sources/registry/local.rs index d35345eb86c..7276c688b58 100644 --- a/src/cargo/sources/registry/local.rs +++ b/src/cargo/sources/registry/local.rs @@ -9,6 +9,9 @@ use std::io::prelude::*; use std::io::SeekFrom; use std::path::Path; +/// A local registry is a registry that lives on the filesystem as a set of +/// `.crate` files with an `index` directory in the same format as a remote +/// registry. pub struct LocalRegistry<'cfg> { index_path: Filesystem, root: Filesystem, diff --git a/src/cargo/sources/registry/mod.rs b/src/cargo/sources/registry/mod.rs index 159b0952900..de198a40f46 100644 --- a/src/cargo/sources/registry/mod.rs +++ b/src/cargo/sources/registry/mod.rs @@ -85,7 +85,7 @@ //! ``` //! //! The root of the index contains a `config.json` file with a few entries -//! corresponding to the registry (see `RegistryConfig` below). +//! corresponding to the registry (see [`RegistryConfig`] below). //! //! Otherwise, there are three numbered directories (1, 2, 3) for crates with //! names 1, 2, and 3 characters in length. The 1/2 directories simply have the @@ -188,17 +188,44 @@ const CRATE_TEMPLATE: &str = "{crate}"; const VERSION_TEMPLATE: &str = "{version}"; const PREFIX_TEMPLATE: &str = "{prefix}"; const LOWER_PREFIX_TEMPLATE: &str = "{lowerprefix}"; - +pub const INDEX_SCHEMA_VERSION: u32 = 2; + +/// A "source" for a [local](local::LocalRegistry) or +/// [remote](remote::RemoteRegistry) registry. +/// +/// This contains common functionality that is shared between the two registry +/// kinds, with the registry-specific logic implemented as part of the +/// [`RegistryData`] trait referenced via the `ops` field. pub struct RegistrySource<'cfg> { source_id: SourceId, + /// The path where crate files are extracted (`$CARGO_HOME/registry/src/$REG-HASH`). src_path: Filesystem, + /// Local reference to [`Config`] for convenience. config: &'cfg Config, + /// Whether or not the index has been updated. + /// + /// This is used as an optimization to avoid updating if not needed, such + /// as `Cargo.lock` already exists and the index already contains the + /// locked entries. Or, to avoid updating multiple times. + /// + /// Only remote registries really need to update. Local registries only + /// check that the index exists. updated: bool, + /// Abstraction for interfacing to the different registry kinds. ops: Box, + /// Interface for managing the on-disk index. index: index::RegistryIndex<'cfg>, + /// A set of packages that should be allowed to be used, even if they are + /// yanked. + /// + /// This is populated from the entries in `Cargo.lock` to ensure that + /// `cargo update -p somepkg` won't unlock yanked entries in `Cargo.lock`. + /// Otherwise, the resolver would think that those entries no longer + /// exist, and it would trigger updates to unrelated packages. yanked_whitelist: HashSet, } +/// The `config.json` file stored in the index. #[derive(Deserialize)] pub struct RegistryConfig { /// Download endpoint for all crates. @@ -232,6 +259,13 @@ pub struct RegistryPackage<'a> { #[serde(borrow)] deps: Vec>, features: BTreeMap>, + /// This field contains features with new, extended syntax. Specifically, + /// namespaced features (`dep:`) and weak dependencies (`pkg?/feat`). + /// + /// This is separated from `features` because versions older than 1.19 + /// will fail to load due to not being able to parse the new syntax, even + /// with a `Cargo.lock` file. + features2: Option>>, cksum: String, /// If `true`, Cargo will skip this version when resolving. /// @@ -243,6 +277,25 @@ pub struct RegistryPackage<'a> { /// Added early 2018 (see ), /// can be `None` if published before then. links: Option, + /// The schema version for this entry. + /// + /// If this is None, it defaults to version 1. Version 2 means the + /// `features2` field is present. Entries with unknown versions are + /// ignored. + /// + /// This provides a method to safely introduce changes to index entries + /// and allow older versions of cargo to ignore newer entries it doesn't + /// understand. This is honored as of 1.51, so unfortunately older + /// versions will ignore it, and potentially misinterpret version 2 and + /// newer entries. + /// + /// The intent is that versions older than 1.51 will work with a + /// pre-existing `Cargo.lock`, but they may not correctly process `cargo + /// update` or build a lock from scratch. In that case, cargo may + /// incorrectly select a new package that uses a new index format. A + /// workaround is to downgrade any packages that are incompatible with the + /// `--precise` flag of `cargo update`. + v: Option, } #[test] @@ -278,18 +331,7 @@ fn escaped_char_in_json() { .unwrap(); } -#[derive(Deserialize)] -#[serde(field_identifier, rename_all = "lowercase")] -enum Field { - Name, - Vers, - Deps, - Features, - Cksum, - Yanked, - Links, -} - +/// A dependency as encoded in the index JSON. #[derive(Deserialize)] struct RegistryDependency<'a> { name: InternedString, @@ -369,30 +411,108 @@ impl<'a> RegistryDependency<'a> { } } +/// An abstract interface to handle both a [local](local::LocalRegistry) and +/// [remote](remote::RemoteRegistry) registry. +/// +/// This allows [`RegistrySource`] to abstractly handle both registry kinds. pub trait RegistryData { + /// Performs initialization for the registry. + /// + /// This should be safe to call multiple times, the implementation is + /// expected to not do any work if it is already prepared. fn prepare(&self) -> CargoResult<()>; + + /// Returns the path to the index. + /// + /// Note that different registries store the index in different formats + /// (remote=git, local=files). fn index_path(&self) -> &Filesystem; + + /// Loads the JSON for a specific named package from the index. + /// + /// * `root` is the root path to the index. + /// * `path` is the relative path to the package to load (like `ca/rg/cargo`). + /// * `data` is a callback that will receive the raw bytes of the index JSON file. fn load( &self, root: &Path, path: &Path, data: &mut dyn FnMut(&[u8]) -> CargoResult<()>, ) -> CargoResult<()>; + + /// Loads the `config.json` file and returns it. + /// + /// Local registries don't have a config, and return `None`. fn config(&mut self) -> CargoResult>; + + /// Updates the index. + /// + /// For a remote registry, this updates the index over the network. Local + /// registries only check that the index exists. fn update_index(&mut self) -> CargoResult<()>; + + /// Prepare to start downloading a `.crate` file. + /// + /// Despite the name, this doesn't actually download anything. If the + /// `.crate` is already downloaded, then it returns [`MaybeLock::Ready`]. + /// If it hasn't been downloaded, then it returns [`MaybeLock::Download`] + /// which contains the URL to download. The [`crate::core::package::Download`] + /// system handles the actual download process. After downloading, it + /// calls [`finish_download`] to save the downloaded file. + /// + /// `checksum` is currently only used by local registries to verify the + /// file contents (because local registries never actually download + /// anything). Remote registries will validate the checksum in + /// `finish_download`. For already downloaded `.crate` files, it does not + /// validate the checksum, assuming the filesystem does not suffer from + /// corruption or manipulation. fn download(&mut self, pkg: PackageId, checksum: &str) -> CargoResult; + + /// Finish a download by saving a `.crate` file to disk. + /// + /// After [`crate::core::package::Download`] has finished a download, + /// it will call this to save the `.crate` file. This is only relevant + /// for remote registries. This should validate the checksum and save + /// the given data to the on-disk cache. + /// + /// Returns a [`File`] handle to the `.crate` file, positioned at the start. fn finish_download(&mut self, pkg: PackageId, checksum: &str, data: &[u8]) -> CargoResult; + /// Returns whether or not the `.crate` file is already downloaded. fn is_crate_downloaded(&self, _pkg: PackageId) -> bool { true } + + /// Validates that the global package cache lock is held. + /// + /// Given the [`Filesystem`], this will make sure that the package cache + /// lock is held. If not, it will panic. See + /// [`Config::acquire_package_cache_lock`] for acquiring the global lock. + /// + /// Returns the [`Path`] to the [`Filesystem`]. fn assert_index_locked<'a>(&self, path: &'a Filesystem) -> &'a Path; + + /// Returns the current "version" of the index. + /// + /// For local registries, this returns `None` because there is no + /// versioning. For remote registries, this returns the SHA hash of the + /// git index on disk (or None if the index hasn't been downloaded yet). + /// + /// This is used by index caching to check if the cache is out of date. fn current_version(&self) -> Option; } +/// The status of [`RegistryData::download`] which indicates if a `.crate` +/// file has already been downloaded, or if not then the URL to download. pub enum MaybeLock { + /// The `.crate` file is already downloaded. [`File`] is a handle to the + /// opened `.crate` file on the filesystem. Ready(File), + /// The `.crate` file is not downloaded, here's the URL to download it from. + /// + /// `descriptor` is just a text string to display to the user of what is + /// being downloaded. Download { url: String, descriptor: String }, } diff --git a/src/cargo/sources/registry/remote.rs b/src/cargo/sources/registry/remote.rs index 2e44d9ae3ea..d3f9eb9c03c 100644 --- a/src/cargo/sources/registry/remote.rs +++ b/src/cargo/sources/registry/remote.rs @@ -29,8 +29,12 @@ fn make_dep_prefix(name: &str) -> String { } } +/// A remote registry is a registry that lives at a remote URL (such as +/// crates.io). The git index is cloned locally, and `.crate` files are +/// downloaded as needed and cached locally. pub struct RemoteRegistry<'cfg> { index_path: Filesystem, + /// Path to the cache of `.crate` files (`$CARGO_HOME/registry/path/$REG-HASH`). cache_path: Filesystem, source_id: SourceId, index_git_ref: GitReference, diff --git a/src/cargo/util/toml/mod.rs b/src/cargo/util/toml/mod.rs index 24f362ce75f..840ecbe1b98 100644 --- a/src/cargo/util/toml/mod.rs +++ b/src/cargo/util/toml/mod.rs @@ -876,7 +876,7 @@ struct Context<'a, 'b> { } impl TomlManifest { - /// Prepares the manfiest for publishing. + /// Prepares the manifest for publishing. // - Path and git components of dependency specifications are removed. // - License path is updated to point within the package. pub fn prepare_for_publish( diff --git a/tests/testsuite/alt_registry.rs b/tests/testsuite/alt_registry.rs index 748d5dcaaa5..336e978ffd8 100644 --- a/tests/testsuite/alt_registry.rs +++ b/tests/testsuite/alt_registry.rs @@ -8,6 +8,7 @@ use std::fs; #[cargo_test] fn depend_on_alt_registry() { + registry::alt_init(); let p = project() .file( "Cargo.toml", @@ -57,6 +58,7 @@ fn depend_on_alt_registry() { #[cargo_test] fn depend_on_alt_registry_depends_on_same_registry_no_index() { + registry::alt_init(); let p = project() .file( "Cargo.toml", @@ -99,6 +101,7 @@ fn depend_on_alt_registry_depends_on_same_registry_no_index() { #[cargo_test] fn depend_on_alt_registry_depends_on_same_registry() { + registry::alt_init(); let p = project() .file( "Cargo.toml", @@ -141,6 +144,7 @@ fn depend_on_alt_registry_depends_on_same_registry() { #[cargo_test] fn depend_on_alt_registry_depends_on_crates_io() { + registry::alt_init(); let p = project() .file( "Cargo.toml", @@ -185,7 +189,7 @@ fn depend_on_alt_registry_depends_on_crates_io() { #[cargo_test] fn registry_and_path_dep_works() { - registry::init(); + registry::alt_init(); let p = project() .file( @@ -219,7 +223,7 @@ fn registry_and_path_dep_works() { #[cargo_test] fn registry_incompatible_with_git() { - registry::init(); + registry::alt_init(); let p = project() .file( @@ -249,6 +253,7 @@ fn registry_incompatible_with_git() { #[cargo_test] fn cannot_publish_to_crates_io_with_registry_dependency() { + registry::alt_init(); let fakeio_path = paths::root().join("fake.io"); let fakeio_url = fakeio_path.into_url().unwrap(); let p = project() @@ -307,6 +312,7 @@ fn cannot_publish_to_crates_io_with_registry_dependency() { #[cargo_test] fn publish_with_registry_dependency() { + registry::alt_init(); let p = project() .file( "Cargo.toml", @@ -370,6 +376,7 @@ fn publish_with_registry_dependency() { #[cargo_test] fn alt_registry_and_crates_io_deps() { + registry::alt_init(); let p = project() .file( "Cargo.toml", @@ -415,7 +422,7 @@ fn alt_registry_and_crates_io_deps() { #[cargo_test] fn block_publish_due_to_no_token() { - registry::init(); + registry::alt_init(); let p = project().file("src/lib.rs", "").build(); fs::remove_file(paths::home().join(".cargo/credentials")).unwrap(); @@ -432,6 +439,7 @@ fn block_publish_due_to_no_token() { #[cargo_test] fn publish_to_alt_registry() { + registry::alt_init(); let p = project().file("src/main.rs", "fn main() {}").build(); // Setup the registry by publishing a package @@ -472,6 +480,7 @@ fn publish_to_alt_registry() { #[cargo_test] fn publish_with_crates_io_dep() { + registry::alt_init(); let p = project() .file( "Cargo.toml", @@ -537,7 +546,7 @@ fn publish_with_crates_io_dep() { #[cargo_test] fn passwords_in_registries_index_url_forbidden() { - registry::init(); + registry::alt_init(); let config = paths::home().join(".cargo/config"); @@ -567,6 +576,7 @@ Caused by: #[cargo_test] fn patch_alt_reg() { + registry::alt_init(); Package::new("bar", "0.1.0").publish(); let p = project() .file( @@ -656,6 +666,7 @@ Caused by: #[cargo_test] fn no_api() { + registry::alt_init(); Package::new("bar", "0.0.1").alternative(true).publish(); // Configure without `api`. let repo = git2::Repository::open(registry::alt_registry_path()).unwrap(); @@ -739,6 +750,7 @@ fn no_api() { #[cargo_test] fn alt_reg_metadata() { // Check for "registry" entries in `cargo metadata` with alternative registries. + registry::alt_init(); let p = project() .file( "Cargo.toml", @@ -1033,6 +1045,7 @@ fn alt_reg_metadata() { fn unknown_registry() { // A known registry refers to an unknown registry. // foo -> bar(crates.io) -> baz(alt) + registry::alt_init(); let p = project() .file( "Cargo.toml", @@ -1188,6 +1201,7 @@ fn unknown_registry() { #[cargo_test] fn registries_index_relative_url() { + registry::alt_init(); let config = paths::root().join(".cargo/config"); fs::create_dir_all(config.parent().unwrap()).unwrap(); fs::write( @@ -1237,6 +1251,7 @@ fn registries_index_relative_url() { #[cargo_test] fn registries_index_relative_path_not_allowed() { + registry::alt_init(); let config = paths::root().join(".cargo/config"); fs::create_dir_all(config.parent().unwrap()).unwrap(); fs::write( diff --git a/tests/testsuite/credential_process.rs b/tests/testsuite/credential_process.rs index 8360ae4c627..69d839aa85e 100644 --- a/tests/testsuite/credential_process.rs +++ b/tests/testsuite/credential_process.rs @@ -1,12 +1,8 @@ //! Tests for credential-process. -use cargo_test_support::paths::CargoPathExt; use cargo_test_support::{basic_manifest, cargo_process, paths, project, registry, Project}; use std::fs; -use std::io::{BufRead, BufReader, Write}; -use std::net::TcpListener; use std::thread; -use url::Url; fn toml_bin(proj: &Project, name: &str) -> String { proj.bin(name).display().to_string().replace('\\', "\\\\") @@ -14,9 +10,10 @@ fn toml_bin(proj: &Project, name: &str) -> String { #[cargo_test] fn gated() { - registry::init(); - - paths::home().join(".cargo/credentials").rm_rf(); + registry::RegistryBuilder::new() + .alternative(true) + .add_tokens(false) + .build(); let p = project() .file( @@ -64,8 +61,10 @@ fn gated() { #[cargo_test] fn warn_both_token_and_process() { // Specifying both credential-process and a token in config should issue a warning. - registry::init(); - paths::home().join(".cargo/credentials").rm_rf(); + registry::RegistryBuilder::new() + .alternative(true) + .add_tokens(false) + .build(); let p = project() .file( ".cargo/config", @@ -138,38 +137,16 @@ Only one of these values may be set, remove one or the other to proceed. /// Returns a thread handle for the API server, the test should join it when /// finished. Also returns the simple `foo` project to test against. fn get_token_test() -> (Project, thread::JoinHandle<()>) { - let server = TcpListener::bind("127.0.0.1:0").unwrap(); - let addr = server.local_addr().unwrap(); - let api_url = format!("http://{}", addr); - - registry::init_registry( - registry::alt_registry_path(), - registry::alt_dl_url(), - Url::parse(&api_url).unwrap(), - registry::alt_api_path(), - ); - // API server that checks that the token is included correctly. - let t = thread::spawn(move || { - let mut conn = BufReader::new(server.accept().unwrap().0); - let headers: Vec<_> = (&mut conn) - .lines() - .map(|s| s.unwrap()) - .take_while(|s| s.len() > 2) - .map(|s| s.trim().to_string()) - .collect(); - assert!(headers - .iter() - .any(|header| header == "Authorization: sekrit")); - conn.get_mut() - .write_all( - b"HTTP/1.1 200\r\n\ - Content-Length: 33\r\n\ - \r\n\ - {\"ok\": true, \"msg\": \"completed!\"}\r\n", - ) - .unwrap(); - }); + let server = registry::RegistryBuilder::new() + .add_tokens(false) + .build_api_server(&|headers| { + assert!(headers + .iter() + .any(|header| header == "Authorization: sekrit")); + + (200, &r#"{"ok": true, "msg": "completed!"}"#) + }); // The credential process to use. let cred_proj = project() @@ -206,7 +183,7 @@ fn get_token_test() -> (Project, thread::JoinHandle<()>) { ) .file("src/lib.rs", "") .build(); - (p, t) + (p, server) } #[cargo_test] @@ -231,10 +208,7 @@ fn publish() { #[cargo_test] fn basic_unsupported() { // Non-action commands don't support login/logout. - registry::init(); - // If both `credential-process` and `token` are specified, it will ignore - // `credential-process`, so remove the default tokens. - paths::home().join(".cargo/credentials").rm_rf(); + registry::RegistryBuilder::new().add_tokens(false).build(); cargo::util::paths::append( &paths::home().join(".cargo/config"), br#" @@ -327,10 +301,7 @@ fn login() { #[cargo_test] fn logout() { - registry::init(); - // If both `credential-process` and `token` are specified, it will ignore - // `credential-process`, so remove the default tokens. - paths::home().join(".cargo/credentials").rm_rf(); + registry::RegistryBuilder::new().add_tokens(false).build(); // The credential process to use. let cred_proj = project() .at("cred_proj") @@ -418,9 +389,7 @@ fn owner() { #[cargo_test] fn libexec_path() { // cargo: prefixed names use the sysroot - registry::init(); - - paths::home().join(".cargo/credentials").rm_rf(); + registry::RegistryBuilder::new().add_tokens(false).build(); cargo::util::paths::append( &paths::home().join(".cargo/config"), br#" @@ -448,8 +417,10 @@ Caused by: #[cargo_test] fn invalid_token_output() { // Error when credential process does not output the expected format for a token. - registry::init(); - paths::home().join(".cargo/credentials").rm_rf(); + registry::RegistryBuilder::new() + .alternative(true) + .add_tokens(false) + .build(); let cred_proj = project() .at("cred_proj") .file("Cargo.toml", &basic_manifest("test-cred", "1.0.0")) diff --git a/tests/testsuite/features_namespaced.rs b/tests/testsuite/features_namespaced.rs index 55078f1c28e..000d034b57e 100644 --- a/tests/testsuite/features_namespaced.rs +++ b/tests/testsuite/features_namespaced.rs @@ -1,7 +1,9 @@ //! Tests for namespaced features. -use cargo_test_support::registry::{Dependency, Package}; -use cargo_test_support::{project, publish}; +use cargo_test_support::paths::CargoPathExt; +use cargo_test_support::registry::{self, Dependency, Package}; +use cargo_test_support::{paths, process, project, publish, rustc_host}; +use std::fs; #[cargo_test] fn gated() { @@ -1037,6 +1039,7 @@ fn publish_no_implicit() { .run(); publish::validate_upload_with_contents( + "v1", r#" { "authors": [], @@ -1106,3 +1109,410 @@ feat = ["opt-dep1"] )], ); } + +#[cargo_test] +fn publish() { + // Publish uploads `features2` in JSON. + Package::new("bar", "1.0.0").publish(); + let p = project() + .file( + "Cargo.toml", + r#" + [package] + name = "foo" + version = "0.1.0" + description = "foo" + license = "MIT" + homepage = "https://example.com/" + + [dependencies] + bar = { version = "1.0", optional = true } + + [features] + feat1 = [] + feat2 = ["dep:bar"] + feat3 = ["feat2"] + "#, + ) + .file("src/lib.rs", "") + .build(); + + registry::api_path().join("api/v2/crates").mkdir_p(); + + p.cargo("publish --token sekrit -Z namespaced-features") + .masquerade_as_nightly_cargo() + .with_stderr( + "\ +[UPDATING] [..] +[PACKAGING] foo v0.1.0 [..] +[VERIFYING] foo v0.1.0 [..] +[COMPILING] foo v0.1.0 [..] +[FINISHED] [..] +[UPLOADING] foo v0.1.0 [..] +", + ) + .run(); + + publish::validate_upload_with_contents( + "v2", + r#" + { + "authors": [], + "badges": {}, + "categories": [], + "deps": [ + { + "default_features": true, + "features": [], + "kind": "normal", + "name": "bar", + "optional": true, + "registry": "https://github.com/rust-lang/crates.io-index", + "target": null, + "version_req": "^1.0" + } + ], + "description": "foo", + "documentation": null, + "features": { + "feat1": [], + "feat2": [], + "feat3": ["feat2"] + }, + "features2": { + "feat2": ["dep:bar"] + }, + "homepage": "https://example.com/", + "keywords": [], + "license": "MIT", + "license_file": null, + "links": null, + "name": "foo", + "readme": null, + "readme_file": null, + "repository": null, + "vers": "0.1.0", + "v": 2 + } + "#, + "foo-0.1.0.crate", + &["Cargo.toml", "Cargo.toml.orig", "src/lib.rs"], + &[( + "Cargo.toml", + r#"[..] +[package] +name = "foo" +version = "0.1.0" +description = "foo" +homepage = "https://example.com/" +license = "MIT" +[dependencies.bar] +version = "1.0" +optional = true + +[features] +feat1 = [] +feat2 = ["dep:bar"] +feat3 = ["feat2"] +"#, + )], + ); +} + +#[cargo_test] +fn old_registry_publish_error() { + // What happens if a registry does not support the v2 api. + let server = registry::RegistryBuilder::new().build_api_server(&|headers| { + assert_eq!(headers[0], "PUT /api/v2/crates/new HTTP/1.1"); + (404, &"") + }); + + Package::new("bar", "1.0.0").alternative(true).publish(); + let p = project() + .file( + "Cargo.toml", + r#" + [package] + name = "foo" + version = "0.1.0" + description = "foo" + license = "MIT" + homepage = "https://example.com/" + + [dependencies] + bar = { version = "1.0", optional = true, registry = "alternative" } + + [features] + feat = ["dep:bar"] + "#, + ) + .file("src/lib.rs", "") + .build(); + + p.cargo("publish --registry alternative -Z namespaced-features") + .masquerade_as_nightly_cargo() + .with_status(101) + .with_stderr("\ +[UPDATING] [..] +[PACKAGING] foo v0.1.0 [..] +[VERIFYING] foo v0.1.0 [..] +[COMPILING] foo v0.1.0 [..] +[FINISHED] [..] +[UPLOADING] foo v0.1.0 [..] +[ERROR] This package uses new feature syntax that is not supported by the registry at http://127.0.0.1:[..] +") + .run(); + + server.join().unwrap(); +} + +// This is a test for exercising the behavior of older versions of cargo. You +// will need rustup installed. This will iterate over the installed +// toolchains, and run some tests over each one, producing a report at the +// end. +// +// This is ignored because it is intended to be run on a developer system with +// a bunch of toolchains installed. As of this writing, I have tested 1.0 to +// 1.51. Run this with: +// +// cargo test --test testsuite -- old_cargos --nocapture --ignored +#[ignore] +#[cargo_test] +fn old_cargos() { + if std::process::Command::new("rustup").output().is_err() { + eprintln!("old_cargos ignored, rustup not installed"); + return; + } + Package::new("new-baz-dep", "1.0.0").publish(); + + Package::new("baz", "1.0.0").publish(); + Package::new("baz", "1.0.1") + .add_dep(Dependency::new("new-baz-dep", "1.0").optional(true)) + .feature("new-feat", &["dep:new-baz-dep"]) + .publish(); + + Package::new("bar", "1.0.0") + .add_dep(Dependency::new("baz", "1.0").optional(true)) + .feature("feat", &["baz"]) + .publish(); + let bar_cksum = Package::new("bar", "1.0.1") + .add_dep(Dependency::new("baz", "1.0").optional(true)) + .feature("feat", &["dep:baz"]) + .publish(); + Package::new("bar", "1.0.2") + .add_dep(Dependency::new("baz", "1.0").enable_features(&["new-feat"])) + .publish(); + + let p = project() + .file( + "Cargo.toml", + r#" + [package] + name = "foo" + version = "0.1.0" + + [dependencies] + bar = "1.0" + "#, + ) + .file("src/lib.rs", "") + .build(); + + // Collect a sorted list of all installed toolchains for this host. + let host = rustc_host(); + // I tend to have lots of toolchains installed, but I don't want to test + // all of them (like dated nightlies, or toolchains for non-host targets). + let valid_names = &[ + format!("stable-{}", host), + format!("beta-{}", host), + format!("nightly-{}", host), + ]; + let output = cargo::util::process("rustup") + .args(&["toolchain", "list"]) + .exec_with_output() + .expect("rustup should be installed"); + let stdout = std::str::from_utf8(&output.stdout).unwrap(); + let mut toolchains: Vec<_> = stdout + .lines() + .map(|line| { + // Some lines say things like (default), just get the version. + line.split_whitespace().next().expect("non-empty line") + }) + .filter(|line| { + line.ends_with(&host) + && (line.starts_with("1.") || valid_names.iter().any(|name| name == line)) + }) + .map(|line| { + let output = cargo::util::process("rustc") + .args(&[format!("+{}", line).as_str(), "-V"]) + .exec_with_output() + .expect("rustc installed"); + let version = std::str::from_utf8(&output.stdout).unwrap(); + let parts: Vec<_> = version.split_whitespace().collect(); + assert_eq!(parts[0], "rustc"); + assert!(parts[1].starts_with("1.")); + + ( + semver::Version::parse(parts[1]).expect("valid version"), + line, + ) + }) + .collect(); + + toolchains.sort_by(|a, b| a.0.cmp(&b.0)); + + let config_path = paths::home().join(".cargo/config"); + let lock_path = p.root().join("Cargo.lock"); + + // Results collected for printing a final report. + let mut results: Vec> = Vec::new(); + + for (version, toolchain) in toolchains { + let mut toolchain_result = vec![toolchain.to_string()]; + if version < semver::Version::new(1, 12, 0) { + fs::write( + &config_path, + format!( + r#" + [registry] + index = "{}" + "#, + registry::registry_url() + ), + ) + .unwrap(); + } else { + fs::write( + &config_path, + format!( + " + [source.crates-io] + registry = 'https://wut' # only needed by 1.12 + replace-with = 'dummy-registry' + + [source.dummy-registry] + registry = '{}' + ", + registry::registry_url() + ), + ) + .unwrap(); + } + + let run_cargo = || -> String { + match process("cargo") + .args(&[format!("+{}", toolchain).as_str(), "build"]) + .cwd(p.root()) + .exec_with_output() + { + Ok(_output) => { + eprintln!("{} ok", toolchain); + let output = process("cargo") + .args(&[format!("+{}", toolchain).as_str(), "pkgid", "bar"]) + .cwd(p.root()) + .exec_with_output() + .expect("pkgid should succeed"); + let stdout = std::str::from_utf8(&output.stdout).unwrap(); + let version = stdout + .trim() + .rsplitn(2, ':') + .next() + .expect("version after colon"); + format!("success bar={}", version) + } + Err(e) => { + eprintln!("{} err {}", toolchain, e); + "failed".to_string() + } + } + }; + + lock_path.rm_rf(); + p.build_dir().rm_rf(); + + toolchain_result.push(run_cargo()); + if version < semver::Version::new(1, 12, 0) { + p.change_file( + "Cargo.lock", + &format!( + r#" + [root] + name = "foo" + version = "0.1.0" + dependencies = [ + "bar 1.0.1 (registry+{url})", + ] + + [[package]] + name = "bar" + version = "1.0.1" + source = "registry+{url}" + "#, + url = registry::registry_url() + ), + ); + } else { + p.change_file( + "Cargo.lock", + &format!( + r#" + [root] + name = "foo" + version = "0.1.0" + dependencies = [ + "bar 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + ] + + [[package]] + name = "bar" + version = "1.0.1" + source = "registry+https://github.com/rust-lang/crates.io-index" + + [metadata] + "checksum bar 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "{}" + "#, + bar_cksum + ), + ); + } + toolchain_result.push(run_cargo()); + results.push(toolchain_result); + } + + // Generate a report. + let headers = vec!["Version", "Unlocked", "Locked"]; + let init: Vec = headers.iter().map(|h| h.len()).collect(); + let col_widths = results.iter().fold(init, |acc, row| { + acc.iter().zip(row).map(|(a, b)| *a.max(&b.len())).collect() + }); + // Print headers + let spaced: Vec<_> = headers + .iter() + .enumerate() + .map(|(i, h)| { + format!( + " {}{}", + h, + " ".repeat(col_widths[i].saturating_sub(h.len())) + ) + }) + .collect(); + eprintln!("{}", spaced.join(" |")); + let lines: Vec<_> = col_widths.iter().map(|w| "-".repeat(*w + 2)).collect(); + eprintln!("{}", lines.join("|")); + // Print columns. + for row in results { + let rs: Vec<_> = row + .iter() + .enumerate() + .map(|(i, c)| { + format!( + " {}{} ", + c, + " ".repeat(col_widths[i].saturating_sub(c.len())) + ) + }) + .collect(); + eprintln!("{}", rs.join("|")); + } +} diff --git a/tests/testsuite/install_upgrade.rs b/tests/testsuite/install_upgrade.rs index 7d3c8beac89..4fa8c154c40 100644 --- a/tests/testsuite/install_upgrade.rs +++ b/tests/testsuite/install_upgrade.rs @@ -9,7 +9,7 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use cargo_test_support::install::{cargo_home, exe}; use cargo_test_support::paths::CargoPathExt; -use cargo_test_support::registry::Package; +use cargo_test_support::registry::{self, Package}; use cargo_test_support::{ basic_manifest, cargo_process, cross_compile, execs, git, process, project, Execs, }; @@ -549,6 +549,7 @@ fn upgrade_git() { fn switch_sources() { // Installing what appears to be the same thing, but from different // sources should reinstall. + registry::alt_init(); pkg("foo", "1.0.0"); Package::new("foo", "1.0.0") .file("src/main.rs", r#"fn main() { println!("alt"); }"#) diff --git a/tests/testsuite/login.rs b/tests/testsuite/login.rs index c226c07a57c..835461af8de 100644 --- a/tests/testsuite/login.rs +++ b/tests/testsuite/login.rs @@ -145,7 +145,7 @@ fn new_credentials_is_used_instead_old() { #[cargo_test] fn registry_credentials() { - registry::init(); + registry::alt_init(); let config = paths::home().join(".cargo/config"); let mut f = OpenOptions::new().append(true).open(config).unwrap(); diff --git a/tests/testsuite/logout.rs b/tests/testsuite/logout.rs index 041d3fb5976..606a06c8484 100644 --- a/tests/testsuite/logout.rs +++ b/tests/testsuite/logout.rs @@ -78,5 +78,6 @@ fn default_registry() { #[cargo_test] fn other_registry() { + registry::alt_init(); simple_logout_test(Some("alternative"), "--registry alternative"); } diff --git a/tests/testsuite/package.rs b/tests/testsuite/package.rs index 2dac71825e9..cbe998e82fb 100644 --- a/tests/testsuite/package.rs +++ b/tests/testsuite/package.rs @@ -818,6 +818,7 @@ to proceed despite this and include the uncommitted changes, pass the `--allow-d #[cargo_test] fn generated_manifest() { + registry::alt_init(); Package::new("abc", "1.0.0").publish(); Package::new("def", "1.0.0").alternative(true).publish(); Package::new("ghi", "1.0.0").publish(); diff --git a/tests/testsuite/patch.rs b/tests/testsuite/patch.rs index 8e2a3b1a305..0890b1e8565 100644 --- a/tests/testsuite/patch.rs +++ b/tests/testsuite/patch.rs @@ -2,7 +2,7 @@ use cargo_test_support::git; use cargo_test_support::paths; -use cargo_test_support::registry::Package; +use cargo_test_support::registry::{self, Package}; use cargo_test_support::{basic_manifest, project}; use std::fs; @@ -1543,6 +1543,7 @@ fn update_unused_new_version() { #[cargo_test] fn too_many_matches() { // The patch locations has multiple versions that match. + registry::alt_init(); Package::new("bar", "0.1.0").publish(); Package::new("bar", "0.1.0").alternative(true).publish(); Package::new("bar", "0.1.1").alternative(true).publish(); @@ -1866,6 +1867,7 @@ fn patched_dep_new_version() { fn patch_update_doesnt_update_other_sources() { // Very extreme edge case, make sure a patch update doesn't update other // sources. + registry::alt_init(); Package::new("bar", "0.1.0").publish(); Package::new("bar", "0.1.0").alternative(true).publish(); @@ -1931,6 +1933,7 @@ fn patch_update_doesnt_update_other_sources() { #[cargo_test] fn can_update_with_alt_reg() { // A patch to an alt reg can update. + registry::alt_init(); Package::new("bar", "0.1.0").publish(); Package::new("bar", "0.1.0").alternative(true).publish(); Package::new("bar", "0.1.1").alternative(true).publish(); diff --git a/tests/testsuite/publish.rs b/tests/testsuite/publish.rs index 02bb318e7f8..556cf01ccf2 100644 --- a/tests/testsuite/publish.rs +++ b/tests/testsuite/publish.rs @@ -677,7 +677,7 @@ The registry `alternative` is not listed in the `publish` value in Cargo.toml. #[cargo_test] fn publish_allowed_registry() { - registry::init(); + registry::alt_init(); let p = project().build(); @@ -717,7 +717,7 @@ fn publish_allowed_registry() { #[cargo_test] fn publish_implicitly_to_only_allowed_registry() { - registry::init(); + registry::alt_init(); let p = project().build(); @@ -1175,6 +1175,7 @@ fn publish_git_with_version() { p.cargo("publish --no-verify --token sekrit").run(); publish::validate_upload_with_contents( + "v1", r#" { "authors": [], @@ -1272,6 +1273,7 @@ fn publish_dev_dep_no_version() { .run(); publish::validate_upload_with_contents( + "v1", r#" { "authors": [], @@ -1457,3 +1459,172 @@ Caused by: )) .run(); } + +#[cargo_test] +fn api_error_json() { + // Registry returns an API error. + let t = registry::RegistryBuilder::new().build_api_server(&|_headers| { + (403, &r#"{"errors": [{"detail": "you must be logged in"}]}"#) + }); + + let p = project() + .file( + "Cargo.toml", + r#" + [project] + name = "foo" + version = "0.0.1" + authors = [] + license = "MIT" + description = "foo" + documentation = "foo" + homepage = "foo" + repository = "foo" + "#, + ) + .file("src/lib.rs", "") + .build(); + + p.cargo("publish --no-verify --registry alternative") + .with_status(101) + .with_stderr( + "\ +[UPDATING] [..] +[PACKAGING] foo v0.0.1 [..] +[UPLOADING] foo v0.0.1 [..] +[ERROR] api errors (status 403 Forbidden): you must be logged in +", + ) + .run(); + + t.join().unwrap(); +} + +#[cargo_test] +fn api_error_code() { + // Registry returns an error code without a JSON message. + let t = registry::RegistryBuilder::new().build_api_server(&|_headers| (400, &"go away")); + + let p = project() + .file( + "Cargo.toml", + r#" + [project] + name = "foo" + version = "0.0.1" + authors = [] + license = "MIT" + description = "foo" + documentation = "foo" + homepage = "foo" + repository = "foo" + "#, + ) + .file("src/lib.rs", "") + .build(); + + p.cargo("publish --no-verify --registry alternative") + .with_status(101) + .with_stderr( + "\ +[UPDATING] [..] +[PACKAGING] foo v0.0.1 [..] +[UPLOADING] foo v0.0.1 [..] +[ERROR] failed to get a 200 OK response, got 400 +headers: +HTTP/1.1 400 +Content-Length: 7 + +body: +go away +", + ) + .run(); + + t.join().unwrap(); +} + +#[cargo_test] +fn api_curl_error() { + // Registry has a network error. + let t = registry::RegistryBuilder::new().build_api_server(&|_headers| panic!("broke!")); + + let p = project() + .file( + "Cargo.toml", + r#" + [project] + name = "foo" + version = "0.0.1" + authors = [] + license = "MIT" + description = "foo" + documentation = "foo" + homepage = "foo" + repository = "foo" + "#, + ) + .file("src/lib.rs", "") + .build(); + + // This doesn't check for the exact text of the error in the remote + // possibility that cargo is linked with a weird version of libcurl, or + // curl changes the text of the message. Currently the message 52 + // (CURLE_GOT_NOTHING) is: + // Server returned nothing (no headers, no data) (Empty reply from server) + p.cargo("publish --no-verify --registry alternative") + .with_status(101) + .with_stderr( + "\ +[UPDATING] [..] +[PACKAGING] foo v0.0.1 [..] +[UPLOADING] foo v0.0.1 [..] +[ERROR] [52] [..] +", + ) + .run(); + + let e = t.join().unwrap_err(); + assert_eq!(*e.downcast::<&str>().unwrap(), "broke!"); +} + +#[cargo_test] +fn api_other_error() { + // Registry returns an invalid response. + let t = registry::RegistryBuilder::new().build_api_server(&|_headers| (200, b"\xff")); + + let p = project() + .file( + "Cargo.toml", + r#" + [project] + name = "foo" + version = "0.0.1" + authors = [] + license = "MIT" + description = "foo" + documentation = "foo" + homepage = "foo" + repository = "foo" + "#, + ) + .file("src/lib.rs", "") + .build(); + + p.cargo("publish --no-verify --registry alternative") + .with_status(101) + .with_stderr( + "\ +[UPDATING] [..] +[PACKAGING] foo v0.0.1 [..] +[UPLOADING] foo v0.0.1 [..] +[ERROR] invalid response from server + +Caused by: + response body was not valid utf-8 +", + ) + .run(); + + t.join().unwrap(); +} diff --git a/tests/testsuite/registry.rs b/tests/testsuite/registry.rs index acba4a2c413..441e965744a 100644 --- a/tests/testsuite/registry.rs +++ b/tests/testsuite/registry.rs @@ -2170,3 +2170,33 @@ fn package_lock_inside_package_is_overwritten() { assert_eq!(ok.metadata().unwrap().len(), 2); } + +#[cargo_test] +fn ignores_unknown_index_version() { + // If the version field is not understood, it is ignored. + Package::new("bar", "1.0.0").publish(); + Package::new("bar", "1.0.1").schema_version(9999).publish(); + + let p = project() + .file( + "Cargo.toml", + r#" + [package] + name = "foo" + version = "0.1.0" + + [dependencies] + bar = "1.0" + "#, + ) + .file("src/lib.rs", "") + .build(); + + p.cargo("tree") + .with_stdout( + "foo v0.1.0 [..]\n\ + └── bar v1.0.0\n\ + ", + ) + .run(); +} diff --git a/tests/testsuite/rename_deps.rs b/tests/testsuite/rename_deps.rs index 16d15994b2e..fc3d11d2737 100644 --- a/tests/testsuite/rename_deps.rs +++ b/tests/testsuite/rename_deps.rs @@ -2,7 +2,7 @@ use cargo_test_support::git; use cargo_test_support::paths; -use cargo_test_support::registry::Package; +use cargo_test_support::registry::{self, Package}; use cargo_test_support::{basic_manifest, project}; #[cargo_test] @@ -66,6 +66,7 @@ fn rename_with_different_names() { #[cargo_test] fn lots_of_names() { + registry::alt_init(); Package::new("foo", "0.1.0") .file("src/lib.rs", "pub fn foo1() {}") .publish(); diff --git a/tests/testsuite/rustdoc_extern_html.rs b/tests/testsuite/rustdoc_extern_html.rs index b5b970320dd..820cf7bd644 100644 --- a/tests/testsuite/rustdoc_extern_html.rs +++ b/tests/testsuite/rustdoc_extern_html.rs @@ -1,6 +1,6 @@ //! Tests for the -Zrustdoc-map feature. -use cargo_test_support::registry::Package; +use cargo_test_support::registry::{self, Package}; use cargo_test_support::{is_nightly, paths, project, Project}; fn basic_project() -> Project { @@ -215,6 +215,7 @@ fn alt_registry() { // --extern-html-root-url is unstable return; } + registry::alt_init(); Package::new("bar", "1.0.0") .alternative(true) .file( diff --git a/tests/testsuite/vendor.rs b/tests/testsuite/vendor.rs index 5d081d79902..e55fe414267 100644 --- a/tests/testsuite/vendor.rs +++ b/tests/testsuite/vendor.rs @@ -7,7 +7,7 @@ use std::fs; use cargo_test_support::git; -use cargo_test_support::registry::Package; +use cargo_test_support::registry::{self, Package}; use cargo_test_support::{basic_lib_manifest, paths, project, Project}; #[cargo_test] @@ -594,6 +594,7 @@ fn ignore_hidden() { #[cargo_test] fn config_instructions_works() { // Check that the config instructions work for all dependency kinds. + registry::alt_init(); Package::new("dep", "0.1.0").publish(); Package::new("altdep", "0.1.0").alternative(true).publish(); let git_project = git::new("gitdep", |project| { diff --git a/tests/testsuite/weak_dep_features.rs b/tests/testsuite/weak_dep_features.rs index 255e5628e24..0dd34705521 100644 --- a/tests/testsuite/weak_dep_features.rs +++ b/tests/testsuite/weak_dep_features.rs @@ -1,7 +1,8 @@ //! Tests for weak-dep-features. -use cargo_test_support::project; -use cargo_test_support::registry::{Dependency, Package}; +use cargo_test_support::paths::CargoPathExt; +use cargo_test_support::registry::{self, Dependency, Package}; +use cargo_test_support::{project, publish}; use std::fmt::Write; // Helper to create lib.rs files that check features. @@ -565,3 +566,109 @@ bar v1.0.0 ) .run(); } + +#[cargo_test] +fn publish() { + // Publish uploads `features2` in JSON. + Package::new("bar", "1.0.0").feature("feat", &[]).publish(); + let p = project() + .file( + "Cargo.toml", + r#" + [package] + name = "foo" + version = "0.1.0" + description = "foo" + license = "MIT" + homepage = "https://example.com/" + + [dependencies] + bar = { version = "1.0", optional = true } + + [features] + feat1 = [] + feat2 = ["bar?/feat"] + "#, + ) + .file("src/lib.rs", "") + .build(); + + registry::api_path().join("api/v2/crates").mkdir_p(); + + p.cargo("publish --token sekrit -Z weak-dep-features") + .masquerade_as_nightly_cargo() + .with_stderr( + "\ +[UPDATING] [..] +[PACKAGING] foo v0.1.0 [..] +[VERIFYING] foo v0.1.0 [..] +[COMPILING] foo v0.1.0 [..] +[FINISHED] [..] +[UPLOADING] foo v0.1.0 [..] +", + ) + .run(); + + publish::validate_upload_with_contents( + "v2", + r#" + { + "authors": [], + "badges": {}, + "categories": [], + "deps": [ + { + "default_features": true, + "features": [], + "kind": "normal", + "name": "bar", + "optional": true, + "registry": "https://github.com/rust-lang/crates.io-index", + "target": null, + "version_req": "^1.0" + } + ], + "description": "foo", + "documentation": null, + "features": { + "feat1": [], + "feat2": [] + }, + "features2": { + "feat2": ["bar?/feat"] + }, + "homepage": "https://example.com/", + "keywords": [], + "license": "MIT", + "license_file": null, + "links": null, + "name": "foo", + "readme": null, + "readme_file": null, + "repository": null, + "vers": "0.1.0", + "v": 2 + } + "#, + "foo-0.1.0.crate", + &["Cargo.toml", "Cargo.toml.orig", "src/lib.rs"], + &[( + "Cargo.toml", + r#"[..] +[package] +name = "foo" +version = "0.1.0" +description = "foo" +homepage = "https://example.com/" +license = "MIT" +[dependencies.bar] +version = "1.0" +optional = true + +[features] +feat1 = [] +feat2 = ["bar?/feat"] +"#, + )], + ); +}