From 3efac22583e22bf381f399d1eeb3f8238396f9b1 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 18 Jun 2025 07:58:47 +0000 Subject: [PATCH 1/7] Restrict benchmarks to release builds only Co-authored-by: simongdavies <1397489+simongdavies@users.noreply.github.com> --- .github/workflows/Benchmarks.yml | 2 +- .github/workflows/dep_rust.yml | 2 +- Justfile | 12 +++++++----- docs/benchmarking-hyperlight.md | 2 +- src/hyperlight_host/benches/benchmarks.rs | 8 ++++++++ 5 files changed, 18 insertions(+), 8 deletions(-) diff --git a/.github/workflows/Benchmarks.yml b/.github/workflows/Benchmarks.yml index 64df29bb2..b0749e881 100644 --- a/.github/workflows/Benchmarks.yml +++ b/.github/workflows/Benchmarks.yml @@ -57,7 +57,7 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Run Benchmarks - run: just bench-ci main release ${{ matrix.hypervisor == 'mshv3' && 'mshv3' || ''}} + run: just bench-ci main ${{ matrix.hypervisor == 'mshv3' && 'mshv3' || ''}} - uses: actions/upload-artifact@v4 with: diff --git a/.github/workflows/dep_rust.yml b/.github/workflows/dep_rust.yml index 389f24faa..f14b1cf66 100644 --- a/.github/workflows/dep_rust.yml +++ b/.github/workflows/dep_rust.yml @@ -151,5 +151,5 @@ jobs: - name: Run benchmarks run: | - just bench-ci main ${{ matrix.config }} ${{ matrix.hypervisor == 'mshv3' && 'mshv3' || ''}} + just bench-ci main ${{ matrix.hypervisor == 'mshv3' && 'mshv3' || ''}} if: ${{ matrix.config == 'release' }} diff --git a/Justfile b/Justfile index 66b06a587..0cce34d20 100644 --- a/Justfile +++ b/Justfile @@ -163,7 +163,7 @@ fmt-apply: cargo +nightly fmt --manifest-path src/hyperlight_guest_capi/Cargo.toml clippy target=default-target: (witguest-wit) - cargo clippy --all-targets --all-features --profile={{ if target == "debug" { "dev" } else { target } }} -- -D warnings + cargo clippy {{ if target == "debug" { "--lib --bins --tests --examples" } else { "--all-targets" } }} --all-features --profile={{ if target == "debug" { "dev" } else { target } }} -- -D warnings clippy-guests target=default-target: (witguest-wit) cd src/tests/rust_guests/simpleguest && cargo clippy --profile={{ if target == "debug" { "dev" } else { target } }} -- -D warnings @@ -229,11 +229,13 @@ bench-download os hypervisor cpu tag="": tar -zxvf target/benchmarks_{{ os }}_{{ hypervisor }}_{{ cpu }}.tar.gz -C target/criterion/ --strip-components=1 # Warning: compares to and then OVERWRITES the given baseline -bench-ci baseline target=default-target features="": - cargo bench --profile={{ if target == "debug" { "dev" } else { target } }} {{ if features =="" {''} else { "--features " + features } }} -- --verbose --save-baseline {{ baseline }} +# Benchmarks only run with release builds for performance consistency +bench-ci baseline features="": + cargo bench --profile=release {{ if features =="" {''} else { "--features " + features } }} -- --verbose --save-baseline {{ baseline }} -bench target=default-target features="": - cargo bench --profile={{ if target == "debug" { "dev" } else { target } }} {{ if features =="" {''} else { "--features " + features } }} -- --verbose +# Benchmarks only run with release builds for performance consistency +bench features="": + cargo bench --profile=release {{ if features =="" {''} else { "--features " + features } }} -- --verbose ############### ### FUZZING ### diff --git a/docs/benchmarking-hyperlight.md b/docs/benchmarking-hyperlight.md index 3ae56d55c..399a32fc1 100644 --- a/docs/benchmarking-hyperlight.md +++ b/docs/benchmarking-hyperlight.md @@ -72,4 +72,4 @@ Found 1 outliers among 100 measurements (1.00%) ## Running benchmarks locally -Use `just bench [debug/release]` parameter to run benchmarks. Comparing local benchmarks results to github-saved benchmarks doesn't make much sense, since you'd be using different hardware, but you can use `just bench-download os hypervisor [tag] ` to download and extract the GitHub release benchmarks to the correct place folder. You can then run `just bench-ci main` to compare to (and overwrite) the previous release benchmarks. Note that `main` is the name of the baselines stored in GitHub. +Use `just bench` to run benchmarks. Benchmarks only run with release builds for performance consistency. Comparing local benchmarks results to github-saved benchmarks doesn't make much sense, since you'd be using different hardware, but you can use `just bench-download os hypervisor [tag] ` to download and extract the GitHub release benchmarks to the correct place folder. You can then run `just bench-ci main` to compare to (and overwrite) the previous release benchmarks. Note that `main` is the name of the baselines stored in GitHub. diff --git a/src/hyperlight_host/benches/benchmarks.rs b/src/hyperlight_host/benches/benchmarks.rs index c9160ff52..77312c8fc 100644 --- a/src/hyperlight_host/benches/benchmarks.rs +++ b/src/hyperlight_host/benches/benchmarks.rs @@ -14,6 +14,14 @@ See the License for the specific language governing permissions and limitations under the License. */ +// Benchmarks are only meaningful and should only run with release builds. +// Debug builds have different performance characteristics and would not provide +// useful benchmarking data for performance regression testing. +#[cfg(debug_assertions)] +compile_error!( + "Benchmarks must be run with release builds only. Use `cargo bench --release` or `just bench`." +); + use criterion::{Criterion, criterion_group, criterion_main}; use hyperlight_host::GuestBinary; use hyperlight_host::sandbox::{ From c8cb4ef139db180dc5cf95b4e02e133720038c90 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 18 Jun 2025 17:11:10 +0000 Subject: [PATCH 2/7] Replace debug assertions check with optimization level check for benchmarks Co-authored-by: ludfjig <4257730+ludfjig@users.noreply.github.com> --- src/hyperlight_host/benches/benchmarks.rs | 8 ++++---- src/hyperlight_host/build.rs | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/src/hyperlight_host/benches/benchmarks.rs b/src/hyperlight_host/benches/benchmarks.rs index 77312c8fc..f0c67a462 100644 --- a/src/hyperlight_host/benches/benchmarks.rs +++ b/src/hyperlight_host/benches/benchmarks.rs @@ -14,12 +14,12 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Benchmarks are only meaningful and should only run with release builds. -// Debug builds have different performance characteristics and would not provide +// Benchmarks are only meaningful and should only run with optimized builds. +// Unoptimized builds have different performance characteristics and would not provide // useful benchmarking data for performance regression testing. -#[cfg(debug_assertions)] +#[cfg(unoptimized_build)] compile_error!( - "Benchmarks must be run with release builds only. Use `cargo bench --release` or `just bench`." + "Benchmarks must be run with optimized builds only. Use `cargo bench --release` or `just bench`." ); use criterion::{Criterion, criterion_group, criterion_main}; diff --git a/src/hyperlight_host/build.rs b/src/hyperlight_host/build.rs index ca2fd8bf2..54927ecba 100644 --- a/src/hyperlight_host/build.rs +++ b/src/hyperlight_host/build.rs @@ -85,6 +85,24 @@ fn main() -> Result<()> { ); } + // Set a cfg flag based on optimization level for benchmarks + // Benchmarks should only run with optimized builds (opt-level 1+) + println!("cargo:rustc-check-cfg=cfg(unoptimized_build)"); + println!("cargo:rustc-check-cfg=cfg(optimized_build)"); + + if let Ok(opt_level) = std::env::var("OPT_LEVEL") { + if opt_level == "0" { + // Unoptimized build - benchmarks should not run + println!("cargo:rustc-cfg=unoptimized_build"); + } else { + // Optimized build - benchmarks can run + println!("cargo:rustc-cfg=optimized_build"); + } + } else { + // Fallback: if we can't determine opt level, assume unoptimized to be safe + println!("cargo:rustc-cfg=unoptimized_build"); + } + // Makes #[cfg(kvm)] == #[cfg(all(feature = "kvm", target_os = "linux"))] // and #[cfg(mshv)] == #[cfg(all(any(feature = "mshv2", feature = "mshv3"), target_os = "linux"))]. // Essentially the kvm and mshv features are ignored on windows as long as you use #[cfg(kvm)] and not #[cfg(feature = "kvm")]. From 2bb51a600e4924d48e8d789321c2510c0b12dbed Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 18 Jun 2025 17:14:50 +0000 Subject: [PATCH 3/7] Fix formatting in build.rs --- src/hyperlight_host/build.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hyperlight_host/build.rs b/src/hyperlight_host/build.rs index 54927ecba..5a0001e02 100644 --- a/src/hyperlight_host/build.rs +++ b/src/hyperlight_host/build.rs @@ -89,7 +89,7 @@ fn main() -> Result<()> { // Benchmarks should only run with optimized builds (opt-level 1+) println!("cargo:rustc-check-cfg=cfg(unoptimized_build)"); println!("cargo:rustc-check-cfg=cfg(optimized_build)"); - + if let Ok(opt_level) = std::env::var("OPT_LEVEL") { if opt_level == "0" { // Unoptimized build - benchmarks should not run From 16302eae0e445600425b13c6b2121b5a096c5c76 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 19 Jun 2025 12:48:17 +0000 Subject: [PATCH 4/7] Revert clippy command changes in Justfile as requested Co-authored-by: simongdavies <1397489+simongdavies@users.noreply.github.com> --- Justfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Justfile b/Justfile index 0cce34d20..c9482aaa8 100644 --- a/Justfile +++ b/Justfile @@ -163,7 +163,7 @@ fmt-apply: cargo +nightly fmt --manifest-path src/hyperlight_guest_capi/Cargo.toml clippy target=default-target: (witguest-wit) - cargo clippy {{ if target == "debug" { "--lib --bins --tests --examples" } else { "--all-targets" } }} --all-features --profile={{ if target == "debug" { "dev" } else { target } }} -- -D warnings + cargo clippy --all-targets --all-features --profile={{ if target == "debug" { "dev" } else { target } }} -- -D warnings clippy-guests target=default-target: (witguest-wit) cd src/tests/rust_guests/simpleguest && cargo clippy --profile={{ if target == "debug" { "dev" } else { target } }} -- -D warnings From 52c205f673ac874e50abe563340a94aa9aa43ed3 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 3 Jul 2025 07:48:28 +0000 Subject: [PATCH 5/7] Fix clippy debug error by conditionally compiling benchmarks Co-authored-by: simongdavies <1397489+simongdavies@users.noreply.github.com> --- src/hyperlight_host/benches/benchmarks.rs | 25 +++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/src/hyperlight_host/benches/benchmarks.rs b/src/hyperlight_host/benches/benchmarks.rs index f0c67a462..10b5b8806 100644 --- a/src/hyperlight_host/benches/benchmarks.rs +++ b/src/hyperlight_host/benches/benchmarks.rs @@ -17,29 +17,34 @@ limitations under the License. // Benchmarks are only meaningful and should only run with optimized builds. // Unoptimized builds have different performance characteristics and would not provide // useful benchmarking data for performance regression testing. -#[cfg(unoptimized_build)] -compile_error!( - "Benchmarks must be run with optimized builds only. Use `cargo bench --release` or `just bench`." -); +#[cfg(optimized_build)] use criterion::{Criterion, criterion_group, criterion_main}; +#[cfg(optimized_build)] use hyperlight_host::GuestBinary; +#[cfg(optimized_build)] use hyperlight_host::sandbox::{ Callable, MultiUseSandbox, SandboxConfiguration, UninitializedSandbox, }; +#[cfg(optimized_build)] use hyperlight_host::sandbox_state::sandbox::EvolvableSandbox; +#[cfg(optimized_build)] use hyperlight_host::sandbox_state::transition::Noop; +#[cfg(optimized_build)] use hyperlight_testing::simple_guest_as_string; +#[cfg(optimized_build)] fn create_uninit_sandbox() -> UninitializedSandbox { let path = simple_guest_as_string().unwrap(); UninitializedSandbox::new(GuestBinary::FilePath(path), None).unwrap() } +#[cfg(optimized_build)] fn create_multiuse_sandbox() -> MultiUseSandbox { create_uninit_sandbox().evolve(Noop::default()).unwrap() } +#[cfg(optimized_build)] fn guest_call_benchmark(c: &mut Criterion) { let mut group = c.benchmark_group("guest_functions"); @@ -87,6 +92,7 @@ fn guest_call_benchmark(c: &mut Criterion) { group.finish(); } +#[cfg(optimized_build)] fn guest_call_benchmark_large_param(c: &mut Criterion) { let mut group = c.benchmark_group("guest_functions_with_large_parameters"); #[cfg(target_os = "windows")] @@ -122,6 +128,7 @@ fn guest_call_benchmark_large_param(c: &mut Criterion) { group.finish(); } +#[cfg(optimized_build)] fn sandbox_benchmark(c: &mut Criterion) { let mut group = c.benchmark_group("sandboxes"); @@ -161,9 +168,19 @@ fn sandbox_benchmark(c: &mut Criterion) { group.finish(); } +#[cfg(optimized_build)] criterion_group! { name = benches; config = Criterion::default(); targets = guest_call_benchmark, sandbox_benchmark, guest_call_benchmark_large_param } + +#[cfg(optimized_build)] criterion_main!(benches); + +// For unoptimized builds, provide a minimal main function that explains benchmarks are not available +#[cfg(unoptimized_build)] +fn main() { + eprintln!("Benchmarks are only available in optimized builds. Use `cargo bench --release` or `just bench`."); + std::process::exit(1); +} From 45df7bdf5743f57a00da81380d272481e079d8b3 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 3 Jul 2025 16:28:44 +0000 Subject: [PATCH 6/7] Use #![cfg(optimized_build)] at top of benchmark file for cleaner code Co-authored-by: jsturtevant <648372+jsturtevant@users.noreply.github.com> --- src/hyperlight_host/benches/benchmarks.rs | 21 +-------------------- 1 file changed, 1 insertion(+), 20 deletions(-) diff --git a/src/hyperlight_host/benches/benchmarks.rs b/src/hyperlight_host/benches/benchmarks.rs index 10b5b8806..b323268f6 100644 --- a/src/hyperlight_host/benches/benchmarks.rs +++ b/src/hyperlight_host/benches/benchmarks.rs @@ -17,34 +17,26 @@ limitations under the License. // Benchmarks are only meaningful and should only run with optimized builds. // Unoptimized builds have different performance characteristics and would not provide // useful benchmarking data for performance regression testing. +#![cfg(optimized_build)] -#[cfg(optimized_build)] use criterion::{Criterion, criterion_group, criterion_main}; -#[cfg(optimized_build)] use hyperlight_host::GuestBinary; -#[cfg(optimized_build)] use hyperlight_host::sandbox::{ Callable, MultiUseSandbox, SandboxConfiguration, UninitializedSandbox, }; -#[cfg(optimized_build)] use hyperlight_host::sandbox_state::sandbox::EvolvableSandbox; -#[cfg(optimized_build)] use hyperlight_host::sandbox_state::transition::Noop; -#[cfg(optimized_build)] use hyperlight_testing::simple_guest_as_string; -#[cfg(optimized_build)] fn create_uninit_sandbox() -> UninitializedSandbox { let path = simple_guest_as_string().unwrap(); UninitializedSandbox::new(GuestBinary::FilePath(path), None).unwrap() } -#[cfg(optimized_build)] fn create_multiuse_sandbox() -> MultiUseSandbox { create_uninit_sandbox().evolve(Noop::default()).unwrap() } -#[cfg(optimized_build)] fn guest_call_benchmark(c: &mut Criterion) { let mut group = c.benchmark_group("guest_functions"); @@ -92,7 +84,6 @@ fn guest_call_benchmark(c: &mut Criterion) { group.finish(); } -#[cfg(optimized_build)] fn guest_call_benchmark_large_param(c: &mut Criterion) { let mut group = c.benchmark_group("guest_functions_with_large_parameters"); #[cfg(target_os = "windows")] @@ -128,7 +119,6 @@ fn guest_call_benchmark_large_param(c: &mut Criterion) { group.finish(); } -#[cfg(optimized_build)] fn sandbox_benchmark(c: &mut Criterion) { let mut group = c.benchmark_group("sandboxes"); @@ -168,19 +158,10 @@ fn sandbox_benchmark(c: &mut Criterion) { group.finish(); } -#[cfg(optimized_build)] criterion_group! { name = benches; config = Criterion::default(); targets = guest_call_benchmark, sandbox_benchmark, guest_call_benchmark_large_param } -#[cfg(optimized_build)] criterion_main!(benches); - -// For unoptimized builds, provide a minimal main function that explains benchmarks are not available -#[cfg(unoptimized_build)] -fn main() { - eprintln!("Benchmarks are only available in optimized builds. Use `cargo bench --release` or `just bench`."); - std::process::exit(1); -} From 44e925c89cd3f0f95090bdc6f1f39f5dfb0d7096 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 3 Jul 2025 18:08:55 +0000 Subject: [PATCH 7/7] Fix clippy error by providing fallback main function for benchmark in debug mode Co-authored-by: jsturtevant <648372+jsturtevant@users.noreply.github.com> --- src/hyperlight_host/benches/benchmarks.rs | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/src/hyperlight_host/benches/benchmarks.rs b/src/hyperlight_host/benches/benchmarks.rs index b323268f6..db788d545 100644 --- a/src/hyperlight_host/benches/benchmarks.rs +++ b/src/hyperlight_host/benches/benchmarks.rs @@ -17,26 +17,34 @@ limitations under the License. // Benchmarks are only meaningful and should only run with optimized builds. // Unoptimized builds have different performance characteristics and would not provide // useful benchmarking data for performance regression testing. -#![cfg(optimized_build)] +#[cfg(optimized_build)] use criterion::{Criterion, criterion_group, criterion_main}; +#[cfg(optimized_build)] use hyperlight_host::GuestBinary; +#[cfg(optimized_build)] use hyperlight_host::sandbox::{ Callable, MultiUseSandbox, SandboxConfiguration, UninitializedSandbox, }; +#[cfg(optimized_build)] use hyperlight_host::sandbox_state::sandbox::EvolvableSandbox; +#[cfg(optimized_build)] use hyperlight_host::sandbox_state::transition::Noop; +#[cfg(optimized_build)] use hyperlight_testing::simple_guest_as_string; +#[cfg(optimized_build)] fn create_uninit_sandbox() -> UninitializedSandbox { let path = simple_guest_as_string().unwrap(); UninitializedSandbox::new(GuestBinary::FilePath(path), None).unwrap() } +#[cfg(optimized_build)] fn create_multiuse_sandbox() -> MultiUseSandbox { create_uninit_sandbox().evolve(Noop::default()).unwrap() } +#[cfg(optimized_build)] fn guest_call_benchmark(c: &mut Criterion) { let mut group = c.benchmark_group("guest_functions"); @@ -84,6 +92,7 @@ fn guest_call_benchmark(c: &mut Criterion) { group.finish(); } +#[cfg(optimized_build)] fn guest_call_benchmark_large_param(c: &mut Criterion) { let mut group = c.benchmark_group("guest_functions_with_large_parameters"); #[cfg(target_os = "windows")] @@ -119,6 +128,7 @@ fn guest_call_benchmark_large_param(c: &mut Criterion) { group.finish(); } +#[cfg(optimized_build)] fn sandbox_benchmark(c: &mut Criterion) { let mut group = c.benchmark_group("sandboxes"); @@ -158,10 +168,21 @@ fn sandbox_benchmark(c: &mut Criterion) { group.finish(); } +#[cfg(optimized_build)] criterion_group! { name = benches; config = Criterion::default(); targets = guest_call_benchmark, sandbox_benchmark, guest_call_benchmark_large_param } +#[cfg(optimized_build)] criterion_main!(benches); + +// Provide a fallback main function for unoptimized builds +// This prevents compilation errors while providing a clear message +#[cfg(unoptimized_build)] +fn main() { + panic!( + "Benchmarks must be run with optimized builds only. Use `cargo bench --release` or `just bench`." + ); +}