Load registry config to determine which API to use

Cargo.toml

@@ -13,13 +13,15 @@ log = "0.4"
 regex = "1"
 structopt = "0.3"
 crates-index-diff = "7"
-reqwest = { version = "0.10.6", features = ["blocking"] } # TODO: Remove blocking when async is ready
-semver = "0.9"
+reqwest = { version = "0.10.6", features = ["blocking", "json"] } # TODO: Remove blocking when async is ready
+semver = { version = "0.9", features = ["serde"] }
 slug = "=0.1.1"
 env_logger = "0.7"
 r2d2 = "0.8"
 r2d2_postgres = "0.14"
-url = "1.4"
+# iron needs url@1, but it reexports it as iron::url, so we can start using
+# url@2 for other usecases
+url = { version = "2.1.1", features = ["serde"] }
 badge = { path = "src/web/badge" }
 failure = "0.1.3"
 comrak = { version = "0.3", default-features = false }
@@ -39,6 +41,7 @@ rustwide = "0.7.1"
 mime_guess = "2"
 dotenv = "0.15"
 zstd = "0.5"
+git2 = { version = "0.13.6", default-features = false }
 
 # Data serialization and deserialization
 serde = { version = "1.0", features = ["derive"] }

src/db/add_package.rs

@@ -32,7 +32,7 @@ pub(crate) fn add_package_into_database(
     default_target: &str,
     source_files: Option<Value>,
     doc_targets: Vec<String>,
-    cratesio_data: &RegistryCrateData,
+    registry_data: &RegistryCrateData,
     has_docs: bool,
     has_examples: bool,
     compression_algorithms: std::collections::HashSet<CompressionAlgorithm>,
@@ -87,10 +87,10 @@ pub(crate) fn add_package_into_database(
         &[
             &crate_id,
             &metadata_pkg.version,
-            &cratesio_data.release_time.naive_utc(),
+            &registry_data.release_time.naive_utc(),
             &serde_json::to_value(&dependencies)?,
             &metadata_pkg.package_name(),
-            &cratesio_data.yanked,
+            &registry_data.yanked,
             &res.successful,
             &has_docs,
             &false, // TODO: Add test status somehow
@@ -103,7 +103,7 @@ pub(crate) fn add_package_into_database(
             &serde_json::to_value(&metadata_pkg.authors)?,
             &serde_json::to_value(&metadata_pkg.keywords)?,
             &has_examples,
-            &cratesio_data.downloads,
+            &registry_data.downloads,
             &source_files,
             &serde_json::to_value(&doc_targets)?,
             &is_library,
@@ -117,7 +117,7 @@ pub(crate) fn add_package_into_database(
 
     add_keywords_into_database(&conn, &metadata_pkg, release_id)?;
     add_authors_into_database(&conn, &metadata_pkg, release_id)?;
-    add_owners_into_database(&conn, &cratesio_data.owners, crate_id)?;
+    add_owners_into_database(&conn, &registry_data.owners, crate_id)?;
     add_compression_into_database(&conn, compression_algorithms.into_iter(), release_id)?;
 
     // Update the crates table with the new release

src/docbuilder/mod.rs

@@ -11,6 +11,7 @@ pub(crate) use self::rustwide_builder::BuildResult;
 pub use self::rustwide_builder::RustwideBuilder;
 
 use crate::error::Result;
+use crate::index::Index;
 use crate::DocBuilderOptions;
 use log::debug;
 use std::collections::BTreeSet;
@@ -22,14 +23,17 @@ use std::path::PathBuf;
 /// chroot based documentation builder
 pub struct DocBuilder {
     options: DocBuilderOptions,
+    index: Index,
     cache: BTreeSet<String>,
     db_cache: BTreeSet<String>,
 }
 
 impl DocBuilder {
     pub fn new(options: DocBuilderOptions) -> DocBuilder {
+        let index = Index::new(&options.registry_index_path).expect("valid index");
         DocBuilder {
             options,
+            index,
             cache: BTreeSet::new(),
             db_cache: BTreeSet::new(),
         }

src/docbuilder/queue.rs

@@ -4,16 +4,15 @@ use super::{DocBuilder, RustwideBuilder};
 use crate::db::connect_db;
 use crate::error::Result;
 use crate::utils::{add_crate_to_queue, get_crate_priority};
-use crates_index_diff::{ChangeKind, Index};
+use crates_index_diff::ChangeKind;
 use log::{debug, error};
 
 impl DocBuilder {
     /// Updates registry index repository and adds new crates into build queue.
     /// Returns the number of crates added
     pub fn get_new_crates(&mut self) -> Result<usize> {
         let conn = connect_db()?;
-        let index = Index::from_path_or_cloned(&self.options.registry_index_path)?;
-        let (mut changes, oid) = index.peek_changes()?;
+        let (mut changes, oid) = self.index.diff().peek_changes()?;
         let mut crates_added = 0;
 
         // I believe this will fix ordering of queue if we get more than one crate from changes
@@ -59,7 +58,7 @@ impl DocBuilder {
             }
         }
 
-        index.set_last_seen_reference(oid)?;
+        self.index.diff().set_last_seen_reference(oid)?;
 
         Ok(crates_added)
     }

src/docbuilder/rustwide_builder.rs

@@ -5,7 +5,6 @@ use crate::db::file::add_path_into_database;
 use crate::db::{add_build_into_database, add_package_into_database, connect_db};
 use crate::docbuilder::{crates::crates_from_path, Limits};
 use crate::error::Result;
-use crate::index::api::RegistryCrateData;
 use crate::storage::CompressionAlgorithms;
 use crate::utils::{copy_doc_dir, parse_rustc_version, CargoMetadata};
 use failure::ResultExt;
@@ -397,7 +396,7 @@ impl RustwideBuilder {
                     &res.target,
                     files_list,
                     successful_targets,
-                    &RegistryCrateData::get_from_network(res.cargo_metadata.root())?,
+                    &doc_builder.index.api().get_crate_data(name, version),
                     has_docs,
                     has_examples,
                     algs,