Give bench_local the option to run stable benchmarks

collector/README.md

@@ -123,7 +123,9 @@ The following options alter the behaviour of the `bench_local` subcommand.
 - `--exclude <EXCLUDE>`: this is used to run a subset of the benchmarks. The
   argument is a comma-separated list of benchmark prefixes. When this option is
   specified, a benchmark is excluded from the run if its name matches one of
-  the given prefixes.
+  the given prefixes. You can also specify `primary`, `secondary` and `stable`
+  to filter on the category of benchmarks. `rust-timer` does not run `stable`
+  benchmarks on PRs.
 - `--exclude-suffix <EXCLUDE>`: this is used to run a subset of the benchmarks. The
   argument is a comma-separated list of benchmark suffixes. When this option is
   specified, a benchmark is excluded from the run if its name matches one of
@@ -134,7 +136,9 @@ The following options alter the behaviour of the `bench_local` subcommand.
 - `--include <INCLUDE>`: the inverse of `--exclude`. The argument is a
   comma-separated list of benchmark prefixes. When this option is specified, a
   benchmark is included in the run only if its name matches one of the given
-  prefixes.
+  prefixes. You can also specify `primary`, `secondary` and `stable` to filter
+  on the category of benchmarks. `rust-timer` only runs `--include primary,secondary`
+  on PRs.
 - `--profiles <PROFILES>`: the profiles to be benchmarked. The possible choices
   are one or more (comma-separated) of `Check`, `Debug`, `Doc`, `Opt`, and
   `All`. The default is `Check,Debug,Opt`.

collector/src/bin/collector.rs

@@ -899,13 +899,12 @@ fn main_result() -> anyhow::Result<i32> {
                 target_triple,
             )?;
 
-            let mut benchmarks = get_compile_benchmarks(
+            let benchmarks = get_compile_benchmarks(
                 &compile_benchmark_dir,
                 local.include.as_deref(),
                 local.exclude.as_deref(),
                 local.exclude_suffix.as_deref(),
             )?;
-            benchmarks.retain(|b| b.category().is_primary_or_secondary());
 
             let artifact_id = ArtifactId::Tag(toolchain.id.clone());
             let mut conn = rt.block_on(pool.connection());

collector/src/compile/benchmark/mod.rs

@@ -468,28 +468,57 @@ pub fn get_compile_benchmarks(
     let mut excludes = to_hashmap(exclude);
     let mut exclude_suffixes = to_hashmap(exclude_suffix);
 
+    let mut include_primaries = false;
+    let mut include_secondaries = false;
+    let mut include_stables = false;
+    if let Some(includes) = includes.as_mut() {
+        include_primaries = includes.remove(&"primary").is_some();
+        include_secondaries = includes.remove(&"secondary").is_some();
+        include_stables = includes.remove(&"stable").is_some();
+    }
+    let mut exclude_primaries = false;
+    let mut exclude_secondaries = false;
+    let mut exclude_stables = false;
+    if let Some(excludes) = excludes.as_mut() {
+        exclude_primaries = excludes.remove(&"primary").is_some();
+        exclude_secondaries = excludes.remove(&"secondary").is_some();
+        exclude_stables = excludes.remove(&"stable").is_some();
+    }
+
     for (path, name) in paths {
         let mut skip = false;
+        let b = Benchmark::new(name, path)?;
 
         let name_matches_prefix = |prefixes: &mut HashMap<&str, usize>| {
-            substring_matches(prefixes, |prefix| name.starts_with(prefix))
+            substring_matches(prefixes, |prefix| b.name.0.starts_with(prefix))
         };
 
+        if include.is_none() && exclude.is_none() && b.category() == Category::Stable {
+            // Common case: don't run the stable benchmarks.
+            continue;
+        }
+
         if let Some(includes) = includes.as_mut() {
-            skip |= !name_matches_prefix(includes);
+            skip |= !(name_matches_prefix(includes)
+                || (include_primaries && b.category() == Category::Primary)
+                || (include_secondaries && b.category() == Category::Secondary)
+                || (include_stables && b.category() == Category::Stable));
         }
         if let Some(excludes) = excludes.as_mut() {
-            skip |= name_matches_prefix(excludes);
+            skip |= name_matches_prefix(excludes)
+                || (exclude_primaries && b.category() == Category::Primary)
+                || (exclude_secondaries && b.category() == Category::Secondary)
+                || (exclude_stables && b.category() == Category::Stable);
         }
         if let Some(exclude_suffixes) = exclude_suffixes.as_mut() {
-            skip |= substring_matches(exclude_suffixes, |suffix| name.ends_with(suffix));
+            skip |= substring_matches(exclude_suffixes, |suffix| b.name.0.ends_with(suffix));
         }
         if skip {
             continue;
         }
 
-        debug!("benchmark `{}`- registered", name);
-        benchmarks.push(Benchmark::new(name, path)?);
+        debug!("benchmark `{}`- registered", b.name);
+        benchmarks.push(b);
     }
 
     // All prefixes/suffixes must be used at least once. This is to catch typos.