fix: Handle rustflags from .cargo/config.toml

.github/workflows/ci.yml

@@ -107,7 +107,10 @@ jobs:
 
       - run: cargo install --path crates/cargo-codspeed --locked
 
-      - run: cargo codspeed build -p ${{ matrix.package }}
+      - run: |
+          # Remove the cargo config else it forces instrumentation mode
+          rm -f .cargo/config.toml
+          cargo codspeed build -p ${{ matrix.package }}
 
       - name: Run the benchmarks
         uses: CodSpeedHQ/action@main

crates/cargo-codspeed/Cargo.toml

@@ -1,6 +1,7 @@
 [package]
 name = "cargo-codspeed"
 version = "3.0.2"
+rust-version = "1.74" # MSRV
 edition = "2021"
 description = "Cargo extension to build & run your codspeed benchmarks"
 authors = ["Arthur Pastel <[email protected]>"]

crates/cargo-codspeed/src/build.rs

@@ -105,30 +105,62 @@ impl BuildOptions<'_> {
         Ok(built_benches)
     }
 
+    /// Adds debug flags and codspeed compilation
+    ///
+    /// If the user has set `RUSTFLAGS`, it will append the flags to it.
+    /// Else, and if the cargo version allows it, it will set the cargo config through
+    /// `--config 'build.rustflags=[ ... ]'`
+    ///
+    /// # Why we do this
+    /// As tracked in [https://github.com/rust-lang/cargo/issues/5376], setting `RUSTFLAGS`
+    /// completely overrides rustflags from cargo config
+    /// We use the cargo built-in config mechanism to set the flags if the user has not set
+    /// `RUSTFLAGS`.
+    fn add_rust_flags(&self, cargo: &mut Command, measurement_mode: MeasurementMode) {
+        let mut flags = vec![
+            // Add debug info (equivalent to -g)
+            "-Cdebuginfo=2".to_owned(),
+            // Prevent debug info stripping
+            // https://doc.rust-lang.org/cargo/reference/profiles.html#release
+            // According to cargo docs, for release profile which we default to:
+            // `strip = "none"` and `debug = false`.
+            // In practice, if we set debug info through RUSTFLAGS, cargo still strips them, most
+            // likely because debug = false in the release profile.
+            // We also need to disable stripping through rust flags.
+            "-Cstrip=none".to_owned(),
+        ];
+
+        // Add the codspeed cfg flag if instrumentation mode is enabled
+        if measurement_mode == MeasurementMode::Instrumentation {
+            flags.push("--cfg=codspeed".to_owned());
+        }
+
+        match std::env::var("RUSTFLAGS") {
+            Result::Ok(existing_rustflags) => {
+                // Expand already existing RUSTFLAGS env var
+                let flags_str = flags.join(" ");
+                cargo.env("RUSTFLAGS", format!("{existing_rustflags} {flags_str}"));
+            }
+            Err(_) => {
+                // Use --config to set rustflags
+                // Our rust integration has an msrv of 1.74, --config is available since 1.63
+                // https://doc.rust-lang.org/nightly/cargo/CHANGELOG.html#cargo-163-2022-08-11
+                let config_value = format!(
+                    "build.rustflags=[{}]",
+                    flags.into_iter().map(|f| format!("\"{f}\"")).join(",")
+                );
+                cargo.arg("--config").arg(config_value);
+            }
+        }
+    }
+
     /// Generates a subcommand to build the benchmarks by invoking cargo and forwarding the filters
     /// This command explicitly ignores the `self.benches`: all benches are built
     fn build_command(&self, measurement_mode: MeasurementMode) -> Command {
         let mut cargo = Command::new("cargo");
         cargo.args(["build", "--benches"]);
 
-        let mut rust_flags = std::env::var("RUSTFLAGS").unwrap_or_else(|_| "".into());
-        // Add debug info (equivalent to -g)
-        rust_flags.push_str(" -C debuginfo=2");
-
-        // Prevent debug info stripping
-        // https://doc.rust-lang.org/cargo/reference/profiles.html#release
-        // According to cargo docs, for release profile which we default to:
-        // `strip = "none"` and `debug = false`.
-        // In practice, if we set debug info through RUSTFLAGS, cargo still strips them, most
-        // likely because debug = false in the release profile.
-        // We also need to disable stripping through rust flags.
-        rust_flags.push_str(" -C strip=none");
-
-        // Add the codspeed cfg flag if instrumentation mode is enabled
-        if measurement_mode == MeasurementMode::Instrumentation {
-            rust_flags.push_str(" --cfg codspeed");
-        }
-        cargo.env("RUSTFLAGS", rust_flags);
+        self.add_rust_flags(&mut cargo, measurement_mode);
 
         if let Some(features) = self.features {
             cargo.arg("--features").arg(features.join(","));

crates/cargo-codspeed/tests/.gitignore

@@ -1,2 +1,2 @@
+*/target
 */Cargo.lock
-*/target/

crates/cargo-codspeed/tests/cargo_config.in/.cargo/config.toml

@@ -0,0 +1,2 @@
+[build]
+rustflags = ["--cfg", "custom_feature_flag"]

crates/cargo-codspeed/tests/cargo_config.in/Cargo.toml

@@ -0,0 +1,14 @@
+[package]
+name = "cargo_config_test"
+version = "0.1.0"
+edition = "2021"
+publish = false
+
+[dependencies]
+codspeed-criterion-compat = { path = "../../../criterion_compat" }
+
+[workspace]
+
+[[bench]]
+name = "cargo_config_bench"
+harness = false

crates/cargo-codspeed/tests/cargo_config.in/benches/cargo_config_bench.rs

@@ -0,0 +1,17 @@
+use codspeed_criterion_compat::{criterion_group, criterion_main, Criterion};
+
+fn bench_failing_without_custom_flag(c: &mut Criterion) {
+    c.bench_function("custom_flag_disabled", |b| {
+        b.iter(|| {
+            // This will cause a compilation error if custom_feature_flag is not set
+            #[cfg(not(custom_feature_flag))]
+            compile_error!(
+                "custom_feature_flag is not enabled - .cargo/config.toml rustflags not applied"
+            );
+        })
+    });
+}
+
+criterion_group!(benches, bench_failing_without_custom_flag);
+
+criterion_main!(benches);

crates/cargo-codspeed/tests/cargo_config.in/build.rs

@@ -0,0 +1,3 @@
+fn main() {
+    println!("cargo::rustc-check-cfg=cfg(custom_feature_flag)");
+}

crates/cargo-codspeed/tests/cargo_config.rs

@@ -0,0 +1,35 @@
+use std::process::Command;
+
+mod helpers;
+use helpers::*;
+
+#[test]
+fn test_cargo_config_rustflags() {
+    let tmp_dir = setup("tests/cargo_config.in", Project::Simple);
+
+    // Test that cargo bench works with the custom flag
+    let output = Command::new("cargo")
+        .arg("bench")
+        .arg("--no-run")
+        .current_dir(&tmp_dir)
+        .output()
+        .expect("Failed to execute cargo bench");
+
+    assert!(
+        output.status.success(),
+        "cargo codspeed bench should succeed with .cargo/config.toml rustflags",
+    );
+
+    // Test that cargo codspeed build also works with the custom flag
+    let output = cargo_codspeed(&tmp_dir)
+        .arg("build")
+        .output()
+        .expect("Failed to execute cargo codspeed build");
+
+    assert!(
+        output.status.success(),
+        "cargo codspeed build should succeed with .cargo/config.toml rustflags",
+    );
+
+    teardown(tmp_dir);
+}