Capture more metadata about crates and stop using json for dependencies

crates/metadata/build.rs

@@ -4,6 +4,7 @@ fn main() {
         "cargo:rustc-env=DOCS_RS_METADATA_HOST_TARGET={}",
         std::env::var("TARGET").unwrap(),
     );
+
     // This only needs to be rerun if the TARGET changed, in which case cargo reruns it anyway.
     // See https://doc.rust-lang.org/cargo/reference/build-scripts.html#cargorerun-if-env-changedname
     println!("cargo:rerun-if-changed=build.rs");

crates/metadata/lib.rs

@@ -51,6 +51,7 @@ use toml::Value;
 ///
 /// [`TARGET`]: https://doc.rust-lang.org/cargo/reference/environment-variables.html#environment-variables-cargo-sets-for-build-scripts
 pub const HOST_TARGET: &str = env!("DOCS_RS_METADATA_HOST_TARGET");
+
 /// The targets that are built if no `targets` section is specified.
 ///
 /// Currently, this is guaranteed to have only [tier one] targets.
@@ -88,17 +89,17 @@ pub enum MetadataError {
 /// name = "test"
 ///
 /// [package.metadata.docs.rs]
-/// features = [ "feature1", "feature2" ]
+/// features = ["feature1", "feature2"]
 /// all-features = true
 /// no-default-features = true
 /// default-target = "x86_64-unknown-linux-gnu"
-/// targets = [ "x86_64-apple-darwin", "x86_64-pc-windows-msvc" ]
-/// rustc-args = [ "--example-rustc-arg" ]
-/// rustdoc-args = [ "--example-rustdoc-arg" ]
+/// targets = ["x86_64-apple-darwin", "x86_64-pc-windows-msvc"]
+/// rustc-args = ["--example-rustc-arg"]
+/// rustdoc-args = ["--example-rustdoc-arg"]
 /// ```
 ///
 /// You can define one or more fields in your `Cargo.toml`.
-#[derive(Default, Deserialize)]
+#[derive(Default, Clone, PartialEq, Deserialize)]
 #[serde(rename_all = "kebab-case")]
 pub struct Metadata {
     /// List of features to pass on to `cargo`.
@@ -113,7 +114,7 @@ pub struct Metadata {
     /// Whether to pass `--no-default-features` to `cargo`.
     //
     /// By default, Docs.rs will build default features.
-    /// Set `no-default-fatures` to `true` if you want to build only certain features.
+    /// Set `no-default-features` to `true` if you want to build only certain features.
     #[serde(default)]
     no_default_features: bool,
 
@@ -249,7 +250,7 @@ impl Metadata {
     pub fn environment_variables(&self) -> HashMap<&'static str, String> {
         let joined = |v: &Option<Vec<_>>| v.as_ref().map(|args| args.join(" ")).unwrap_or_default();
 
-        let mut map = HashMap::new();
+        let mut map = HashMap::with_capacity(3);
         map.insert("RUSTFLAGS", joined(&self.rustc_args));
         map.insert("RUSTDOCFLAGS", self.rustdoc_args.join(" "));
         // For docs.rs detection from build scripts:

src/db/add_package.rs

@@ -9,7 +9,7 @@ use crate::{
     error::Result,
     index::api::{CrateData, CrateOwner, ReleaseData},
     storage::CompressionAlgorithm,
-    utils::MetadataPackage,
+    utils::{Dependency, MetadataPackage},
 };
 use log::{debug, info};
 use postgres::Client;
@@ -115,9 +115,10 @@ pub(crate) fn add_package_into_database(
 
     let release_id: i32 = rows[0].get(0);
 
-    add_keywords_into_database(conn, &metadata_pkg, release_id)?;
-    add_authors_into_database(conn, &metadata_pkg, release_id)?;
+    add_keywords_into_database(conn, metadata_pkg, release_id)?;
+    add_authors_into_database(conn, metadata_pkg, release_id)?;
     add_compression_into_database(conn, compression_algorithms.into_iter(), release_id)?;
+    add_dependencies_into_database(conn, metadata_pkg.dependencies.iter(), release_id)?;
 
     // Update the crates table with the new release
     conn.execute(
@@ -195,7 +196,7 @@ fn convert_dependencies(pkg: &MetadataPackage) -> Vec<(String, String, String)>
         .iter()
         .map(|dependency| {
             let name = dependency.name.clone();
-            let version = dependency.req.clone();
+            let version = dependency.version_requirement.clone();
             let kind = dependency
                 .kind
                 .clone()
@@ -444,3 +445,45 @@ where
     }
     Ok(())
 }
+
+fn add_dependencies_into_database<'a, I>(
+    conn: &mut Client,
+    dependencies: I,
+    release_id: i32,
+) -> Result<()>
+where
+    I: Iterator<Item = &'a Dependency> + 'a,
+{
+    let statement = conn.prepare(
+        "
+        INSERT INTO dependencies (
+            release_id,
+            name,
+            version_requirement,
+            optional,
+            default_features,
+            features,
+            target
+        )
+        VALUES ($1, $2, $3, $4, $5, $6, $7)
+        ON CONFLICT DO NOTHING;
+        ",
+    )?;
+
+    for dependency in dependencies {
+        conn.query(
+            &statement,
+            &[
+                &release_id,
+                &dependency.name,
+                &dependency.version_requirement,
+                &dependency.optional,
+                &dependency.default_features,
+                &dependency.features,
+                &dependency.target,
+            ],
+        )?;
+    }
+
+    Ok(())
+}

src/db/migrate.rs

@@ -398,6 +398,52 @@ pub fn migrate(version: Option<Version>, conn: &mut Client) -> CratesfyiResult<(
             // downgrade query
             "DROP TABLE doc_coverage;"
         ),
+        migration!(
+            context,
+            // version
+            17,
+            // description
+            "Create a new table for dependencies",
+            // upgrade query
+            "
+            -- Create the new table to hold dependencies
+            CREATE TABLE dependencies (
+                release_id INT UNIQUE REFERENCES releases(id),
+                name TEXT NOT NULL,
+                optional BOOL,
+                version_requirement TEXT,
+                default_features BOOL,
+                features TEXT[],
+                target TEXT,
+                -- The version of the dependency schema, everything pre-existing in the
+                -- database is given 'v0' while new additions get 'v1'
+                dependency_version TEXT NOT NULL
+            );
+
+            -- Migrate over old dependencies into the new table database
+            INSERT INTO dependencies
+            SELECT
+                dependency[0] AS name,
+                dependency[1] AS version_requirement,
+                'v0' as dependency_version
+            FROM releases
+            CROSS JOIN LATERAL json_array_elements(releases.dependencies) AS dependency;
+
+            -- Remove the old dependency column from the releases table
+            ALTER TABLE releases DROP COLUMN dependencies;
+            ",
+            // downgrade query
+            "
+            -- Add the dependency column back
+            ALTER TABLE releases ADD COLUMN dependencies;
+
+            -- Collapse the dependencies table into json and re-insert it into releases
+            -- TODO
+
+            -- Drop the dependencies table
+            DROP TABLE dependencies;
+            "
+        ),
     ];
 
     for migration in migrations {

src/test/fakes.rs

@@ -5,7 +5,10 @@ use crate::storage::Storage;
 use crate::utils::{Dependency, MetadataPackage, Target};
 use chrono::{DateTime, Utc};
 use failure::Error;
-use std::sync::Arc;
+use std::{collections::HashMap, sync::Arc};
+
+const DEFAULT_CONTENT: &[u8] =
+    b"<html><head></head><body>default content for test/fakes</body></html>";
 
 #[must_use = "FakeRelease does nothing until you call .create()"]
 pub(crate) struct FakeRelease<'a> {
@@ -27,9 +30,6 @@ pub(crate) struct FakeRelease<'a> {
     readme: Option<&'a str>,
 }
 
-const DEFAULT_CONTENT: &[u8] =
-    b"<html><head></head><body>default content for test/fakes</body></html>";
-
 impl<'a> FakeRelease<'a> {
     pub(super) fn new(db: &'a TestDatabase, storage: Arc<Storage>) -> Self {
         FakeRelease {
@@ -40,19 +40,29 @@ impl<'a> FakeRelease<'a> {
                 name: "fake-package".into(),
                 version: "1.0.0".into(),
                 license: Some("MIT".into()),
+                license_file: None,
                 repository: Some("https://git.example.com".into()),
                 homepage: Some("https://www.example.com".into()),
                 description: Some("Fake package".into()),
                 documentation: Some("https://docs.example.com".into()),
                 dependencies: vec![Dependency {
                     name: "fake-dependency".into(),
-                    req: "^1.0.0".into(),
+                    version_requirement: "^1.0.0".into(),
                     kind: None,
+                    rename: None,
+                    optional: false,
+                    default_features: true,
+                    features: Vec::new(),
+                    target: None,
                 }],
                 targets: vec![Target::dummy_lib("fake_package".into(), None)],
                 readme: None,
                 keywords: vec!["fake".into(), "package".into()],
                 authors: vec!["Fake Person <[email protected]>".into()],
+                features: HashMap::new(),
+                categories: Vec::new(),
+                edition: "2018".to_owned(),
+                metadata: None,
             },
             build_result: BuildResult {
                 rustc_version: "rustc 2.0.0-nightly (000000000 1970-01-01)".into(),