Add bench_runtime_local command to collector and a simple hashmap runtime benchmark

Author: Kobzol

Cargo.lock

@@ -1,6 +1,6 @@
 [workspace]
 members = ["collector", "collector/benchlib", "site", "database", "intern"]
-exclude = ["collector/compile-benchmarks", "rust/src"]
+exclude = ["collector/compile-benchmarks", "collector/runtime-benchmarks", "rust/src"]
 
 [profile.release.package.site]
 debug = 1

Cargo.toml

@@ -33,6 +33,8 @@ filetime = "0.2.14"
 walkdir = "2"
 flate2 = { version = "1.0.22", features = ["rust_backend"] }
 rayon = "1.5.2"
+cargo_metadata = "0.15.0"
+benchlib = { path = "benchlib" }
 
 [target.'cfg(windows)'.dependencies]
 miow = "0.3"

collector/Cargo.toml

@@ -0,0 +1,2 @@
+[workspace]
+members = ["hashmap"]

collector/runtime-benchmarks/Cargo.lock

@@ -0,0 +1,11 @@
+[package]
+name = "hashmap-bench"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+benchlib = { path = "../../benchlib" }
+hashbrown = "0.12.3"
+fxhash = "0.2.1"

collector/runtime-benchmarks/Cargo.toml

@@ -0,0 +1,17 @@
+use benchlib;
+use benchlib::benchmark::BenchmarkSuite;
+
+fn main() {
+    let mut suite = BenchmarkSuite::new();
+
+    suite.register("hashmap-insert-10k", || {
+        let mut map =
+            hashbrown::HashMap::with_capacity_and_hasher(10000, fxhash::FxBuildHasher::default());
+        move || {
+            for index in 0..10000 {
+                map.insert(index, index);
+            }
+        }
+    });
+    suite.run().unwrap();
+}

collector/runtime-benchmarks/hashmap/Cargo.toml

@@ -305,11 +305,20 @@ impl Benchmark {
     }
 }
 
-pub fn compile_time_benchmark_dir() -> PathBuf {
+/// Directory containing compile-time benchmarks.
+/// We measure how long does it take to compile these crates.
+pub fn compile_benchmark_dir() -> PathBuf {
     PathBuf::from("collector/compile-benchmarks")
 }
 
-pub fn get_benchmarks(
+/// Directory containing runtime benchmarks.
+/// We measure how long does it take to execute these crates, which is a proxy of the quality
+/// of code generated by rustc.
+pub fn runtime_benchmark_dir() -> PathBuf {
+    PathBuf::from("collector/runtime-benchmarks")
+}
+
+pub fn get_compile_benchmarks(
     benchmark_dir: &Path,
     include: Option<&str>,
     exclude: Option<&str>,

collector/runtime-benchmarks/hashmap/src/main.rs

@@ -7,7 +7,9 @@ use collector::api::next_artifact::NextArtifact;
 use collector::benchmark::category::Category;
 use collector::benchmark::profile::Profile;
 use collector::benchmark::scenario::Scenario;
-use collector::benchmark::{compile_time_benchmark_dir, get_benchmarks, Benchmark, BenchmarkName};
+use collector::benchmark::{
+    compile_benchmark_dir, get_compile_benchmarks, runtime_benchmark_dir, Benchmark, BenchmarkName,
+};
 use collector::utils;
 use database::{ArtifactId, Commit, CommitType, Pool};
 use rayon::iter::{IndexedParallelIterator, IntoParallelRefIterator, ParallelIterator};
@@ -24,6 +26,7 @@ use tokio::runtime::Runtime;
 
 use collector::execute::bencher::BenchProcessor;
 use collector::execute::profiler::{ProfileProcessor, Profiler};
+use collector::runtime::bench_runtime;
 use collector::toolchain::{get_local_toolchain, Compiler, Sysroot};
 
 fn n_normal_benchmarks_remaining(n: usize) -> String {
@@ -557,6 +560,13 @@ struct BenchRustcOption {
 #[derive(Debug, clap::Subcommand)]
 #[clap(rename_all = "snake_case")]
 enum Commands {
+    /// Benchmarks the performance of programs generated by a local rustc
+    BenchRuntimeLocal {
+        rustc: String,
+        /// Identifier to associate benchmark results with
+        #[clap(long)]
+        id: Option<String>,
+    },
     /// Benchmarks a local rustc
     BenchLocal {
         #[clap(flatten)]
@@ -667,7 +677,8 @@ fn main_result() -> anyhow::Result<i32> {
 
     let args = Cli::parse();
 
-    let benchmark_dir = compile_time_benchmark_dir();
+    let compile_benchmark_dir = compile_benchmark_dir();
+    let runtime_benchmark_dir = runtime_benchmark_dir();
 
     let mut builder = tokio::runtime::Builder::new_multi_thread();
     // We want to minimize noise from the runtime
@@ -681,6 +692,10 @@ fn main_result() -> anyhow::Result<i32> {
     let target_triple = format!("{}-unknown-linux-gnu", std::env::consts::ARCH);
 
     match args.command {
+        Commands::BenchRuntimeLocal { rustc, id } => {
+            bench_runtime(&rustc, id.as_deref(), runtime_benchmark_dir)?;
+            Ok(0)
+        }
         Commands::BenchLocal {
             local,
             db,
@@ -693,7 +708,7 @@ fn main_result() -> anyhow::Result<i32> {
 
             let pool = database::Pool::open(&db.db);
 
-            let (rustc, rustdoc, cargo, id) = get_local_toolchain(
+            let toolchain = get_local_toolchain(
                 &profiles,
                 &local.rustc,
                 local.rustdoc.as_deref(),
@@ -702,8 +717,8 @@ fn main_result() -> anyhow::Result<i32> {
                 "",
             )?;
 
-            let mut benchmarks = get_benchmarks(
-                &benchmark_dir,
+            let mut benchmarks = get_compile_benchmarks(
+                &compile_benchmark_dir,
                 local.include.as_deref(),
                 local.exclude.as_deref(),
             )?;
@@ -712,17 +727,11 @@ fn main_result() -> anyhow::Result<i32> {
             let res = bench(
                 &mut rt,
                 pool,
-                &ArtifactId::Tag(id),
+                &ArtifactId::Tag(toolchain.id.clone()),
                 &profiles,
                 &scenarios,
                 bench_rustc.bench_rustc,
-                Compiler {
-                    rustc: &rustc,
-                    rustdoc: rustdoc.as_deref(),
-                    cargo: &cargo,
-                    triple: &target_triple,
-                    is_nightly: true,
-                },
+                Compiler::from_toolchain(&toolchain, &target_triple),
                 &benchmarks,
                 Some(iterations),
                 self_profile.self_profile,
@@ -755,7 +764,13 @@ fn main_result() -> anyhow::Result<i32> {
 
             match next {
                 NextArtifact::Release(tag) => {
-                    bench_published_artifact(tag, pool, &mut rt, &target_triple, &benchmark_dir)?;
+                    bench_published_artifact(
+                        tag,
+                        pool,
+                        &mut rt,
+                        &target_triple,
+                        &compile_benchmark_dir,
+                    )?;
                 }
                 NextArtifact::Commit {
                     commit,
@@ -766,8 +781,11 @@ fn main_result() -> anyhow::Result<i32> {
                     let sysroot = Sysroot::install(commit.sha.to_string(), &target_triple)
                         .with_context(|| format!("failed to install sysroot for {:?}", commit))?;
 
-                    let mut benchmarks =
-                        get_benchmarks(&benchmark_dir, include.as_deref(), exclude.as_deref())?;
+                    let mut benchmarks = get_compile_benchmarks(
+                        &compile_benchmark_dir,
+                        include.as_deref(),
+                        exclude.as_deref(),
+                    )?;
                     benchmarks.retain(|b| b.category().is_primary_or_secondary());
 
                     let res = bench(
@@ -794,7 +812,13 @@ fn main_result() -> anyhow::Result<i32> {
 
         Commands::BenchPublished { toolchain, db } => {
             let pool = database::Pool::open(&db.db);
-            bench_published_artifact(toolchain, pool, &mut rt, &target_triple, &benchmark_dir)?;
+            bench_published_artifact(
+                toolchain,
+                pool,
+                &mut rt,
+                &target_triple,
+                &compile_benchmark_dir,
+            )?;
             Ok(0)
         }
 
@@ -816,8 +840,8 @@ fn main_result() -> anyhow::Result<i32> {
             let profiles = &local.profiles.0;
             let scenarios = &local.scenarios.0;
 
-            let mut benchmarks = get_benchmarks(
-                &benchmark_dir,
+            let mut benchmarks = get_compile_benchmarks(
+                &compile_benchmark_dir,
                 local.include.as_deref(),
                 local.exclude.as_deref(),
             )?;
@@ -833,32 +857,26 @@ fn main_result() -> anyhow::Result<i32> {
 
             let mut get_toolchain_and_profile =
                 |rustc: &str, suffix: &str| -> anyhow::Result<String> {
-                    let (rustc, rustdoc, cargo, id) = get_local_toolchain(
+                    let toolchain = get_local_toolchain(
                         &profiles,
                         &rustc,
                         local.rustdoc.as_deref(),
                         local.cargo.as_deref(),
                         local.id.as_deref(),
                         suffix,
                     )?;
-                    let compiler = Compiler {
-                        rustc: &rustc,
-                        rustdoc: rustdoc.as_deref(),
-                        cargo: &cargo,
-                        triple: &target_triple,
-                        is_nightly: true,
-                    };
+                    let compiler = Compiler::from_toolchain(&toolchain, &target_triple);
                     profile(
                         compiler,
-                        &id,
+                        &toolchain.id,
                         profiler,
                         &out_dir,
                         &benchmarks,
                         &profiles,
                         &scenarios,
                         &mut errors,
                     );
-                    Ok(id)
+                    Ok(toolchain.id)
                 };
 
             if let Some(rustc2) = rustc2 {
@@ -923,7 +941,7 @@ fn main_result() -> anyhow::Result<i32> {
             Ok(0)
         }
         Commands::Download(cmd) => {
-            let target_dir = get_downloaded_crate_target(&benchmark_dir, &cmd);
+            let target_dir = get_downloaded_crate_target(&compile_benchmark_dir, &cmd);
             check_target_dir(&target_dir, cmd.force)?;
 
             match cmd.command {
@@ -985,7 +1003,7 @@ fn bench_published_artifact(
     let cargo = which("cargo")?;
 
     // Exclude benchmarks that don't work with a stable compiler.
-    let mut benchmarks = get_benchmarks(&benchmark_dir, None, None)?;
+    let mut benchmarks = get_compile_benchmarks(&benchmark_dir, None, None)?;
     benchmarks.retain(|b| b.category().is_stable());
 
     let res = bench(

collector/src/benchmark/mod.rs

@@ -8,6 +8,7 @@ use std::process::{self, Command};
 pub mod api;
 pub mod benchmark;
 pub mod execute;
+pub mod runtime;
 pub mod toolchain;
 pub mod utils;
 

collector/src/bin/collector.rs

@@ -0,0 +1,85 @@
+use crate::benchmark::profile::Profile;
+use crate::toolchain::{get_local_toolchain, LocalToolchain};
+use benchlib::messages::BenchmarkResult;
+use cargo_metadata::Message;
+use std::path::{Path, PathBuf};
+use std::process::Command;
+
+#[derive(Debug)]
+struct BenchmarkBinary {
+    path: PathBuf,
+}
+
+pub fn bench_runtime(rustc: &str, id: Option<&str>, benchmark_dir: PathBuf) -> anyhow::Result<()> {
+    let toolchain = get_local_toolchain(&[Profile::Opt], rustc, None, None, id, "")?;
+    let output = compile_runtime_benchmarks(&toolchain, &benchmark_dir)?;
+    let binaries = gather_binaries(&output)?;
+
+    for binary in binaries {
+        let name = binary.path.file_name().and_then(|s| s.to_str()).unwrap();
+
+        let result = Command::new(&binary.path).arg("benchmark").output()?;
+        if !result.status.success() {
+            anyhow::bail!(
+                "Failed to run runtime benchmark {name}\n{}\n{}",
+                String::from_utf8_lossy(&result.stdout),
+                String::from_utf8_lossy(&result.stderr)
+            );
+        } else {
+            log::info!("Successfully ran runtime benchmark {name}",);
+
+            let data: Vec<BenchmarkResult> = serde_json::from_slice(&result.stdout)?;
+            // TODO: do something with the result
+            println!("{name}: {:?}", data);
+        }
+    }
+
+    Ok(())
+}
+
+/// Compiles all runtime benchmarks and returns the stdout output of Cargo.
+fn compile_runtime_benchmarks(toolchain: &LocalToolchain, dir: &Path) -> anyhow::Result<Vec<u8>> {
+    let result = Command::new(&toolchain.cargo)
+        .env("RUSTC", &toolchain.rustc)
+        .arg("build")
+        .arg("--release")
+        .arg("--message-format")
+        .arg("json")
+        .current_dir(dir)
+        .output()?;
+
+    if !result.status.success() {
+        anyhow::bail!(
+            "Failed to compile runtime benchmarks\n{}\n{}",
+            String::from_utf8_lossy(&result.stdout),
+            String::from_utf8_lossy(&result.stderr)
+        );
+    } else {
+        log::info!("Successfully compiled runtime benchmarks");
+        return Ok(result.stdout);
+    }
+}
+
+/// Parse Cargo JSON output and find all compiled binaries.
+fn gather_binaries(cargo_stdout: &[u8]) -> anyhow::Result<Vec<BenchmarkBinary>> {
+    let mut binaries = vec![];
+
+    for message in Message::parse_stream(cargo_stdout) {
+        let message = message?;
+        match message {
+            Message::CompilerArtifact(artifact) => {
+                if let Some(ref executable) = artifact.executable {
+                    if artifact.target.kind.iter().any(|k| k == "bin") {
+                        let path = executable.as_std_path().to_path_buf();
+                        binaries.push(BenchmarkBinary { path });
+                    }
+                }
+            }
+            _ => {}
+        }
+    }
+
+    log::debug!("Found binaries: {:?}", binaries);
+
+    Ok(binaries)
+}

collector/src/lib.rs

@@ -233,6 +233,23 @@ impl<'a> Compiler<'a> {
             is_nightly: true,
         }
     }
+    pub fn from_toolchain(toolchain: &'a LocalToolchain, target_triple: &'a str) -> Compiler<'a> {
+        Compiler {
+            rustc: &toolchain.rustc,
+            rustdoc: toolchain.rustdoc.as_deref(),
+            cargo: &toolchain.cargo,
+            triple: target_triple,
+            is_nightly: true,
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct LocalToolchain {
+    pub rustc: PathBuf,
+    pub rustdoc: Option<PathBuf>,
+    pub cargo: PathBuf,
+    pub id: String,
 }
 
 /// Get a toolchain from the input.
@@ -248,7 +265,7 @@ pub fn get_local_toolchain(
     cargo: Option<&Path>,
     id: Option<&str>,
     id_suffix: &str,
-) -> anyhow::Result<(PathBuf, Option<PathBuf>, PathBuf, String)> {
+) -> anyhow::Result<LocalToolchain> {
     // `+`-prefixed rustc is an indicator to fetch the rustc of the toolchain
     // specified. This follows the similar pattern used by rustup's binaries
     // (e.g., `rustc +stage1`).
@@ -371,5 +388,10 @@ pub fn get_local_toolchain(
         cargo
     };
 
-    Ok((rustc, rustdoc, cargo, id))
+    Ok(LocalToolchain {
+        rustc,
+        rustdoc,
+        cargo,
+        id,
+    })
 }