feat(criterion): add benchmark filtering for instrumentation mode

Author: GuillaumeLagrange

Cargo.lock

@@ -1,10 +1,12 @@
 use assert_cmd::assert::OutputAssertExt;
-use predicates::str::contains;
+use predicates::{prelude::PredicateBooleanExt, str::contains};
 
 mod helpers;
 use helpers::*;
 
 const DIR: &str = "tests/simple-criterion.in";
+const FIB_BENCH_NAME: &str = "fib 20";
+const BUBBLE_SORT_BENCH_NAME: &str = "bubble sort";
 
 #[test]
 fn test_criterion_run_without_build() {
@@ -66,8 +68,8 @@ fn test_criterion_build_and_run_single() {
         .args(["--bench", "another_criterion_example"])
         .assert()
         .success()
-        .stderr(contains("Finished running 1 benchmark suite(s)"))
-        .stderr(contains("another_criterion_example"));
+        .stdout(contains("another_criterion_example"))
+        .stderr(contains("Finished running 1 benchmark suite(s)"));
     teardown(dir);
 }
 
@@ -80,20 +82,43 @@ fn test_criterion_build_and_run_filtered_by_name() {
         .arg("fib 20")
         .assert()
         .success()
+        .stdout(contains(FIB_BENCH_NAME))
+        .stdout(contains(BUBBLE_SORT_BENCH_NAME).not())
+        .stderr(contains("Finished running 2 benchmark suite(s)"));
+    cargo_codspeed(&dir)
+        .arg("run")
+        .arg("bu.*le")
+        .assert()
+        .success()
+        .stdout(contains(FIB_BENCH_NAME).not())
+        .stdout(contains(BUBBLE_SORT_BENCH_NAME))
         .stderr(contains("Finished running 2 benchmark suite(s)"));
     teardown(dir);
 }
 
 #[test]
-fn test_criterion_build_and_run_filtered_by_partial_name() {
+fn test_criterion_build_and_run_filtered_by_name_single() {
     let dir = setup(DIR, Project::Simple);
     cargo_codspeed(&dir).arg("build").assert().success();
     cargo_codspeed(&dir)
         .arg("run")
-        .arg("bubble")
+        .arg("bu.*le")
+        .args(["--bench", "criterion_example"])
         .assert()
         .success()
-        .stderr(contains("Finished running 2 benchmark suite(s)"));
+        .stdout(contains(BUBBLE_SORT_BENCH_NAME).not()) // We are filtering with a name that is not in the selected benchmark
+        .stdout(contains(FIB_BENCH_NAME).not())
+        .stderr(contains("Finished running 1 benchmark suite(s)"));
+
+    cargo_codspeed(&dir)
+        .arg("run")
+        .arg("fib")
+        .args(["--bench", "criterion_example"])
+        .assert()
+        .success()
+        .stdout(contains(FIB_BENCH_NAME))
+        .stdout(contains(BUBBLE_SORT_BENCH_NAME).not())
+        .stderr(contains("Finished running 1 benchmark suite(s)"));
     teardown(dir);
 }
 

crates/cargo-codspeed/tests/simple-criterion.rs

@@ -20,6 +20,8 @@ keywords = ["codspeed", "benchmark", "criterion"]
 criterion = { package = "codspeed-criterion-compat-walltime", path = "./criterion_fork", version = "=4.0.0", default-features = false }
 codspeed = { path = "../codspeed", version = "=4.0.0" }
 colored = "2.1.0"
+clap = { version = "4", default-features = false, features = ["std"] }
+regex = { version = "1.5", default-features = false, features = ["std"] }
 
 futures = { version = "0.3", default-features = false, optional = true }
 smol = { version = "2.0", default-features = false, optional = true }

crates/criterion_compat/Cargo.toml

@@ -6,13 +6,15 @@ use criterion::{
     profiler::Profiler,
     PlottingBackend,
 };
+use regex::Regex;
 
-use crate::{Bencher, BenchmarkGroup, BenchmarkId};
+use crate::{Bencher, BenchmarkFilter, BenchmarkGroup, BenchmarkId};
 
 pub struct Criterion<M: Measurement = WallTime> {
     pub codspeed: Option<Rc<RefCell<CodSpeed>>>,
     pub current_file: String,
     pub macro_group: String,
+    pub filter: BenchmarkFilter,
     phantom: PhantomData<*const M>,
 }
 
@@ -23,19 +25,57 @@ impl Criterion {
             "Harness: codspeed-criterion-compat v{}",
             env!("CARGO_PKG_VERSION"),
         );
+
+        // Parse CLI arguments to extract filter
+        let filter = Self::parse_filter();
+
         Criterion {
             codspeed: Some(Rc::new(RefCell::new(CodSpeed::new()))),
             current_file: String::new(),
             macro_group: String::new(),
+            filter,
             phantom: PhantomData,
         }
     }
 
+    fn parse_filter() -> BenchmarkFilter {
+        use clap::{Arg, Command};
+
+        let matches = Command::new("Criterion Benchmark")
+            .arg(
+                Arg::new("FILTER")
+                    .help("Skip benchmarks whose names do not contain FILTER.")
+                    .index(1),
+            )
+            .arg(
+                Arg::new("exact")
+                    .long("exact")
+                    .num_args(0)
+                    .help("Run benchmarks that exactly match the provided filter"),
+            )
+            .get_matches();
+
+        if let Some(filter) = matches.get_one::<String>("FILTER") {
+            if matches.get_flag("exact") {
+                BenchmarkFilter::Exact(filter.to_owned())
+            } else {
+                let regex = Regex::new(filter).unwrap_or_else(|err| {
+                    eprintln!("Unable to parse '{filter}' as a regular expression: {err}");
+                    std::process::exit(1);
+                });
+                BenchmarkFilter::Regex(regex)
+            }
+        } else {
+            BenchmarkFilter::AcceptAll
+        }
+    }
+
     pub fn with_patched_measurement<M: Measurement>(&mut self, _: Criterion<M>) -> Criterion<M> {
         Criterion {
             codspeed: self.codspeed.clone(),
             current_file: self.current_file.clone(),
             macro_group: self.macro_group.clone(),
+            filter: self.filter.clone(),
             phantom: PhantomData,
         }
     }
@@ -92,6 +132,7 @@ impl Default for Criterion {
             codspeed: None,
             current_file: String::new(),
             macro_group: String::new(),
+            filter: BenchmarkFilter::AcceptAll,
             phantom: PhantomData,
         }
     }
@@ -104,6 +145,7 @@ impl<M: Measurement> Criterion<M> {
             codspeed: self.codspeed,
             current_file: self.current_file,
             macro_group: self.macro_group,
+            filter: self.filter,
             phantom: PhantomData::<*const M2>,
         }
     }

crates/criterion_compat/src/compat/criterion.rs

@@ -0,0 +1,26 @@
+use regex::Regex;
+
+/// Benchmark filtering support - re-exported from criterion fork.
+#[derive(Clone, Debug)]
+pub enum BenchmarkFilter {
+    /// Run all benchmarks.
+    AcceptAll,
+    /// Run benchmarks matching this regex.
+    Regex(Regex),
+    /// Run the benchmark matching this string exactly.
+    Exact(String),
+    /// Do not run any benchmarks.
+    RejectAll,
+}
+
+impl BenchmarkFilter {
+    /// Returns true if a string matches this filter.
+    pub fn is_match(&self, id: &str) -> bool {
+        match self {
+            Self::AcceptAll => true,
+            Self::Regex(r) => r.is_match(id),
+            Self::Exact(e) => e == id,
+            Self::RejectAll => false,
+        }
+    }
+}

crates/criterion_compat/src/compat/filter.rs

@@ -5,7 +5,7 @@ use codspeed::{codspeed::CodSpeed, utils::get_git_relative_path};
 use criterion::measurement::WallTime;
 use criterion::{measurement::Measurement, PlotConfiguration, SamplingMode, Throughput};
 
-use crate::{Bencher, Criterion};
+use crate::{Bencher, BenchmarkFilter, Criterion};
 
 /// Deprecated: using the default measurement will be removed in the next major version.
 /// Defaulting to WallTime differs from the original BenchmarkGroup implementation but avoids creating a breaking change
@@ -14,6 +14,7 @@ pub struct BenchmarkGroup<'a, M: Measurement = WallTime> {
     current_file: String,
     macro_group: String,
     group_name: String,
+    filter: BenchmarkFilter,
     _marker: PhantomData<&'a M>,
 }
 
@@ -29,6 +30,7 @@ impl<'a, M: Measurement> BenchmarkGroup<'a, M> {
             current_file: criterion.current_file.clone(),
             macro_group: criterion.macro_group.clone(),
             group_name,
+            filter: criterion.filter.clone(),
             _marker: PhantomData,
         }
     }
@@ -73,6 +75,12 @@ impl<'a, M: Measurement> BenchmarkGroup<'a, M> {
         if let Some(parameter) = id.parameter {
             uri = format!("{uri}[{parameter}]");
         }
+
+        // Apply filter - skip benchmark if it doesn't match
+        if !self.filter.is_match(&uri) {
+            return;
+        }
+
         let mut codspeed = self.codspeed.borrow_mut();
         let mut b = Bencher::new(&mut codspeed, uri);
         f(&mut b, input);

crates/criterion_compat/src/compat/group.rs

@@ -1,8 +1,10 @@
 mod bencher;
 mod criterion;
+mod filter;
 mod group;
 mod macros;
 
 pub use self::bencher::*;
 pub use self::criterion::*;
+pub use self::filter::*;
 pub use self::group::*;