From f32d1a99697c5fb0e814b12c3cf3f9e7735c7899 Mon Sep 17 00:00:00 2001 From: Chase Wilson Date: Sun, 3 May 2020 22:30:17 -0500 Subject: [PATCH 1/4] Used ON CONFLICT DO UPDATE and formatted a few queries --- src/db/add_package.rs | 178 ++++++++++---------------- src/utils/github_updater.rs | 26 ++-- src/utils/release_activity_updater.rs | 12 +- src/web/builds.rs | 30 ++--- src/web/crate_details.rs | 56 ++++---- 5 files changed, 129 insertions(+), 173 deletions(-) diff --git a/src/db/add_package.rs b/src/db/add_package.rs index 1d74f41d4..73e2b1ff6 100644 --- a/src/db/add_package.rs +++ b/src/db/add_package.rs @@ -44,116 +44,76 @@ pub(crate) fn add_package_into_database( let readme = get_readme(metadata_pkg, source_dir).unwrap_or(None); let is_library = metadata_pkg.is_library(); - let release_id: i32 = { - let rows = conn.query( - "SELECT id FROM releases WHERE crate_id = $1 AND version = $2", - &[&crate_id, &metadata_pkg.version.to_string()], - )?; + let rows = conn.query( + "INSERT INTO releases ( + crate_id, version, release_time, + dependencies, target_name, yanked, build_status, + rustdoc_status, test_status, license, repository_url, + homepage_url, description, description_long, readme, + authors, keywords, have_examples, downloads, files, + doc_targets, is_library, doc_rustc_version, + documentation_url, default_target + ) + VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, + $10, $11, $12, $13, $14, $15, $16, $17, $18, + $19, $20, $21, $22, $23, $24, $25 + ) + ON CONFLICT (crate_id, version) DO UPDATE + SET release_time = $3, + dependencies = $4, + target_name = $5, + yanked = $6, + build_status = $7, + rustdoc_status = $8, + test_status = $9, + license = $10, + repository_url = $11, + homepage_url = $12, + description = $13, + description_long = $14, + readme = $15, + authors = $16, + keywords = $17, + have_examples = $18, + downloads = $19, + files = $20, + doc_targets = $21, + is_library = $22, + doc_rustc_version = $23, + documentation_url = $24, + default_target = $25 + RETURNING id", + &[ + &crate_id, + &metadata_pkg.version, + &cratesio_data.release_time, + &dependencies.to_json(), + &metadata_pkg.package_name(), + &cratesio_data.yanked, + &res.successful, + &has_docs, + &false, // TODO: Add test status somehow + &metadata_pkg.license, + &metadata_pkg.repository, + &metadata_pkg.homepage, + &metadata_pkg.description, + &rustdoc, + &readme, + &metadata_pkg.authors.to_json(), + &metadata_pkg.keywords.to_json(), + &has_examples, + &cratesio_data.downloads, + &source_files, + &doc_targets.to_json(), + &is_library, + &res.rustc_version, + &metadata_pkg.documentation, + &default_target, + ], + )?; - if rows.is_empty() { - let rows = conn.query( - "INSERT INTO releases ( - crate_id, version, release_time, - dependencies, target_name, yanked, build_status, - rustdoc_status, test_status, license, repository_url, - homepage_url, description, description_long, readme, - authors, keywords, have_examples, downloads, files, - doc_targets, is_library, doc_rustc_version, - documentation_url, default_target - ) - VALUES ( $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, - $11, $12, $13, $14, $15, $16, $17, $18, $19, - $20, $21, $22, $23, $24, $25 - ) - RETURNING id", - &[ - &crate_id, - &metadata_pkg.version, - &cratesio_data.release_time, - &dependencies.to_json(), - &metadata_pkg.package_name(), - &cratesio_data.yanked, - &res.successful, - &has_docs, - &false, // TODO: Add test status somehow - &metadata_pkg.license, - &metadata_pkg.repository, - &metadata_pkg.homepage, - &metadata_pkg.description, - &rustdoc, - &readme, - &metadata_pkg.authors.to_json(), - &metadata_pkg.keywords.to_json(), - &has_examples, - &cratesio_data.downloads, - &source_files, - &doc_targets.to_json(), - &is_library, - &res.rustc_version, - &metadata_pkg.documentation, - &default_target, - ], - )?; - // return id - rows.get(0).get(0) - } else { - conn.query( - "UPDATE releases - SET release_time = $3, - dependencies = $4, - target_name = $5, - yanked = $6, - build_status = $7, - rustdoc_status = $8, - test_status = $9, - license = $10, - repository_url = $11, - homepage_url = $12, - description = $13, - description_long = $14, - readme = $15, - authors = $16, - keywords = $17, - have_examples = $18, - downloads = $19, - files = $20, - doc_targets = $21, - is_library = $22, - doc_rustc_version = $23, - documentation_url = $24, - default_target = $25 - WHERE crate_id = $1 AND version = $2", - &[ - &crate_id, - &metadata_pkg.version.to_string(), - &cratesio_data.release_time, - &dependencies.to_json(), - &metadata_pkg.package_name(), - &cratesio_data.yanked, - &res.successful, - &has_docs, - &false, // TODO: Add test status somehow - &metadata_pkg.license, - &metadata_pkg.repository, - &metadata_pkg.homepage, - &metadata_pkg.description, - &rustdoc, - &readme, - &metadata_pkg.authors.to_json(), - &metadata_pkg.keywords.to_json(), - &has_examples, - &cratesio_data.downloads, - &source_files, - &doc_targets.to_json(), - &is_library, - &res.rustc_version, - &metadata_pkg.documentation, - &default_target, - ], - )?; - rows.get(0).get(0) - } - }; + let release_id: i32 = rows.get(0).get(0); add_keywords_into_database(&conn, &metadata_pkg, release_id)?; add_authors_into_database(&conn, &metadata_pkg, release_id)?; diff --git a/src/utils/github_updater.rs b/src/utils/github_updater.rs index eafc9e328..1f898510d 100644 --- a/src/utils/github_updater.rs +++ b/src/utils/github_updater.rs @@ -21,15 +21,15 @@ pub fn github_updater() -> Result<()> { // always the same across all versions of a crate for row in &conn.query( "SELECT DISTINCT ON (crates.name) - crates.name, - crates.id, - releases.repository_url - FROM crates - INNER JOIN releases ON releases.crate_id = crates.id - WHERE releases.repository_url ~ '^https*://github.com' AND - (crates.github_last_update < NOW() - INTERVAL '1 day' OR - crates.github_last_update IS NULL) - ORDER BY crates.name, releases.release_time DESC", + crates.name, + crates.id, + releases.repository_url + FROM crates + INNER JOIN releases ON releases.crate_id = crates.id + WHERE releases.repository_url ~ '^https*://github.com' AND + (crates.github_last_update < NOW() - INTERVAL '1 day' OR + crates.github_last_update IS NULL) + ORDER BY crates.name, releases.release_time DESC", &[], )? { let crate_name: String = row.get(0); @@ -42,10 +42,10 @@ pub fn github_updater() -> Result<()> { .and_then(|fields| { conn.execute( "UPDATE crates - SET github_description = $1, - github_stars = $2, github_forks = $3, - github_issues = $4, github_last_commit = $5, - github_last_update = NOW() WHERE id = $6", + SET github_description = $1, + github_stars = $2, github_forks = $3, + github_issues = $4, github_last_commit = $5, + github_last_update = NOW() WHERE id = $6", &[ &fields.description, &(fields.stars as i32), diff --git a/src/utils/release_activity_updater.rs b/src/utils/release_activity_updater.rs index 1d1b9ac84..21d157976 100644 --- a/src/utils/release_activity_updater.rs +++ b/src/utils/release_activity_updater.rs @@ -61,15 +61,11 @@ pub fn update_release_activity() -> Result<()> { }; conn.query( - "INSERT INTO config (name, value) VALUES ('release_activity', $1)", + "INSERT INTO config (name, value) VALUES ('release_activity', $1) + ON CONFLICT (name) + SET value = $1 WHERE name = 'release_activity'", &[&map], - ) - .or_else(|_| { - conn.query( - "UPDATE config SET value = $1 WHERE name = 'release_activity'", - &[&map], - ) - })?; + )?; Ok(()) } diff --git a/src/web/builds.rs b/src/web/builds.rs index 3f42deddc..5e657fe79 100644 --- a/src/web/builds.rs +++ b/src/web/builds.rs @@ -74,21 +74,21 @@ pub fn build_list_handler(req: &mut Request) -> IronResult { // FIXME: getting builds.output may cause performance issues when release have tons of builds for row in &ctry!(conn.query( "SELECT crates.name, - releases.version, - releases.description, - releases.rustdoc_status, - releases.target_name, - builds.id, - builds.rustc_version, - builds.cratesfyi_version, - builds.build_status, - builds.build_time, - builds.output - FROM builds - INNER JOIN releases ON releases.id = builds.rid - INNER JOIN crates ON releases.crate_id = crates.id - WHERE crates.name = $1 AND releases.version = $2 - ORDER BY id DESC", + releases.version, + releases.description, + releases.rustdoc_status, + releases.target_name, + builds.id, + builds.rustc_version, + builds.cratesfyi_version, + builds.build_status, + builds.build_time, + builds.output + FROM builds + INNER JOIN releases ON releases.id = builds.rid + INNER JOIN crates ON releases.crate_id = crates.id + WHERE crates.name = $1 AND releases.version = $2 + ORDER BY id DESC", &[&name, &version] )) { let id: i32 = row.get(5); diff --git a/src/web/crate_details.rs b/src/web/crate_details.rs index 602c493d3..e80e7c6e4 100644 --- a/src/web/crate_details.rs +++ b/src/web/crate_details.rs @@ -113,34 +113,34 @@ impl CrateDetails { pub fn new(conn: &Connection, name: &str, version: &str) -> Option { // get all stuff, I love you rustfmt let query = "SELECT crates.id, - releases.id, - crates.name, - releases.version, - releases.description, - releases.authors, - releases.dependencies, - releases.readme, - releases.description_long, - releases.release_time, - releases.build_status, - releases.rustdoc_status, - releases.repository_url, - releases.homepage_url, - releases.keywords, - releases.have_examples, - releases.target_name, - crates.versions, - crates.github_stars, - crates.github_forks, - crates.github_issues, - releases.is_library, - releases.doc_targets, - releases.license, - releases.documentation_url, - releases.default_target - FROM releases - INNER JOIN crates ON releases.crate_id = crates.id - WHERE crates.name = $1 AND releases.version = $2;"; + releases.id, + crates.name, + releases.version, + releases.description, + releases.authors, + releases.dependencies, + releases.readme, + releases.description_long, + releases.release_time, + releases.build_status, + releases.rustdoc_status, + releases.repository_url, + releases.homepage_url, + releases.keywords, + releases.have_examples, + releases.target_name, + crates.versions, + crates.github_stars, + crates.github_forks, + crates.github_issues, + releases.is_library, + releases.doc_targets, + releases.license, + releases.documentation_url, + releases.default_target + FROM releases + INNER JOIN crates ON releases.crate_id = crates.id + WHERE crates.name = $1 AND releases.version = $2;"; let rows = conn.query(query, &[&name, &version]).unwrap(); From 144ba541daecae0129710328c7ea10bbdd2db153 Mon Sep 17 00:00:00 2001 From: Chase Wilson Date: Sun, 3 May 2020 22:46:18 -0500 Subject: [PATCH 2/4] Forgot the DO UPDATE part --- src/utils/release_activity_updater.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils/release_activity_updater.rs b/src/utils/release_activity_updater.rs index 21d157976..693d7f7be 100644 --- a/src/utils/release_activity_updater.rs +++ b/src/utils/release_activity_updater.rs @@ -62,8 +62,8 @@ pub fn update_release_activity() -> Result<()> { conn.query( "INSERT INTO config (name, value) VALUES ('release_activity', $1) - ON CONFLICT (name) - SET value = $1 WHERE name = 'release_activity'", + ON CONFLICT (name) DO UPDATE + SET value = $1 WHERE config.name = 'release_activity'", &[&map], )?; From 995076e932dc498e4cba27614b91e2b188d32853 Mon Sep 17 00:00:00 2001 From: Chase Wilson Date: Wed, 6 May 2020 07:45:06 -0500 Subject: [PATCH 3/4] Update src/utils/github_updater.rs Co-authored-by: Joshua Nelson --- src/utils/github_updater.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/utils/github_updater.rs b/src/utils/github_updater.rs index 1f898510d..3e8b80c3f 100644 --- a/src/utils/github_updater.rs +++ b/src/utils/github_updater.rs @@ -45,7 +45,8 @@ pub fn github_updater() -> Result<()> { SET github_description = $1, github_stars = $2, github_forks = $3, github_issues = $4, github_last_commit = $5, - github_last_update = NOW() WHERE id = $6", + github_last_update = NOW() + WHERE id = $6", &[ &fields.description, &(fields.stars as i32), From 51abeedd78c1d2f3cb61d7f5fb76806b3d4eb3b0 Mon Sep 17 00:00:00 2001 From: Chase Wilson Date: Wed, 6 May 2020 07:45:53 -0500 Subject: [PATCH 4/4] Changed github regex --- src/utils/github_updater.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/github_updater.rs b/src/utils/github_updater.rs index 3e8b80c3f..8cd96fd14 100644 --- a/src/utils/github_updater.rs +++ b/src/utils/github_updater.rs @@ -26,7 +26,7 @@ pub fn github_updater() -> Result<()> { releases.repository_url FROM crates INNER JOIN releases ON releases.crate_id = crates.id - WHERE releases.repository_url ~ '^https*://github.com' AND + WHERE releases.repository_url ~ '^https?://github.com' AND (crates.github_last_update < NOW() - INTERVAL '1 day' OR crates.github_last_update IS NULL) ORDER BY crates.name, releases.release_time DESC",