From cd4c18b72dac11c857dd16b2d530bc2ca9407d93 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 29 Jul 2025 23:03:00 +0000 Subject: [PATCH 1/5] Initial plan Signed-off-by: Simon Davies From a7258395de45a8237c8f8aa04a62ea0b4d84f86e Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 29 Jul 2025 23:25:32 +0000 Subject: [PATCH 2/5] Enforce release builds for benchmarks in Justfile and source code Co-authored-by: jsturtevant <648372+jsturtevant@users.noreply.github.com> Signed-off-by: Simon Davies --- Justfile | 14 +++++++++----- docs/benchmarking-hyperlight.md | 4 +++- src/hyperlight_host/benches/benchmarks.rs | 6 ++++++ 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/Justfile b/Justfile index 9ae49e3d2..80f115fac 100644 --- a/Justfile +++ b/Justfile @@ -326,11 +326,15 @@ bench-download os hypervisor cpu tag="": tar -zxvf target/benchmarks_{{ os }}_{{ hypervisor }}_{{ cpu }}.tar.gz -C target/criterion/ --strip-components=1 # Warning: compares to and then OVERWRITES the given baseline -bench-ci baseline target=default-target features="": - cargo bench --profile={{ if target == "debug" { "dev" } else { target } }} {{ if features =="" {''} else { "--features " + features } }} -- --verbose --save-baseline {{ baseline }} - -bench target=default-target features="": - cargo bench --profile={{ if target == "debug" { "dev" } else { target } }} {{ if features =="" {''} else { "--features " + features } }} -- --verbose +bench-ci baseline target="release" features="": + @# Benchmarks should only be run with release builds for meaningful results + @if [ "{{ target }}" != "release" ]; then echo "Error: Benchmarks can only be run with release builds. Use 'just bench-ci release' instead."; exit 1; fi + cargo bench --profile=release {{ if features =="" {''} else { "--features " + features } }} -- --verbose --save-baseline {{ baseline }} + +bench target="release" features="": + @# Benchmarks should only be run with release builds for meaningful results + @if [ "{{ target }}" != "release" ]; then echo "Error: Benchmarks can only be run with release builds. Use 'just bench release' instead."; exit 1; fi + cargo bench --profile=release {{ if features =="" {''} else { "--features " + features } }} -- --verbose ############### ### FUZZING ### diff --git a/docs/benchmarking-hyperlight.md b/docs/benchmarking-hyperlight.md index 3ae56d55c..86b79a30a 100644 --- a/docs/benchmarking-hyperlight.md +++ b/docs/benchmarking-hyperlight.md @@ -72,4 +72,6 @@ Found 1 outliers among 100 measurements (1.00%) ## Running benchmarks locally -Use `just bench [debug/release]` parameter to run benchmarks. Comparing local benchmarks results to github-saved benchmarks doesn't make much sense, since you'd be using different hardware, but you can use `just bench-download os hypervisor [tag] ` to download and extract the GitHub release benchmarks to the correct place folder. You can then run `just bench-ci main` to compare to (and overwrite) the previous release benchmarks. Note that `main` is the name of the baselines stored in GitHub. +Use `just bench` to run benchmarks with release builds (the only supported configuration). Comparing local benchmark results to github-saved benchmarks doesn't make much sense, since you'd be using different hardware, but you can use `just bench-download os hypervisor [tag] ` to download and extract the GitHub release benchmarks to the correct place folder. You can then run `just bench-ci main` to compare to (and overwrite) the previous release benchmarks. Note that `main` is the name of the baselines stored in GitHub. + +**Important**: Benchmarks can only be run with release builds to ensure meaningful performance measurements. Debug builds are not supported for benchmarking. diff --git a/src/hyperlight_host/benches/benchmarks.rs b/src/hyperlight_host/benches/benchmarks.rs index 4896d9e14..ac33f1900 100644 --- a/src/hyperlight_host/benches/benchmarks.rs +++ b/src/hyperlight_host/benches/benchmarks.rs @@ -14,6 +14,12 @@ See the License for the specific language governing permissions and limitations under the License. */ +// Ensure benchmarks only run with release builds for meaningful performance measurements +#[cfg(debug_assertions)] +compile_error!( + "Benchmarks can only be run with release builds. Use '--release' flag when running benchmarks." +); + use criterion::{Criterion, criterion_group, criterion_main}; use hyperlight_host::GuestBinary; use hyperlight_host::sandbox::{MultiUseSandbox, SandboxConfiguration, UninitializedSandbox}; From d1e7d3ce1c7f913b9f0fc5a07518fba5d2ae094f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 29 Jul 2025 23:47:08 +0000 Subject: [PATCH 3/5] Remove compile-time error for benchmarks to allow profiling with debug symbols Co-authored-by: ludfjig <4257730+ludfjig@users.noreply.github.com> Signed-off-by: Simon Davies --- src/hyperlight_host/benches/benchmarks.rs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/hyperlight_host/benches/benchmarks.rs b/src/hyperlight_host/benches/benchmarks.rs index ac33f1900..4896d9e14 100644 --- a/src/hyperlight_host/benches/benchmarks.rs +++ b/src/hyperlight_host/benches/benchmarks.rs @@ -14,12 +14,6 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Ensure benchmarks only run with release builds for meaningful performance measurements -#[cfg(debug_assertions)] -compile_error!( - "Benchmarks can only be run with release builds. Use '--release' flag when running benchmarks." -); - use criterion::{Criterion, criterion_group, criterion_main}; use hyperlight_host::GuestBinary; use hyperlight_host::sandbox::{MultiUseSandbox, SandboxConfiguration, UninitializedSandbox}; From e898d54ef49e950588664c5189242c781f238e8c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 30 Jul 2025 10:26:43 +0000 Subject: [PATCH 4/5] Remove unnecessary target parameter from bench commands - Remove target parameter from bench and bench-ci commands since they only accept release builds - Update GitHub workflows to remove target parameter usage - Simplify command signatures based on @syntactically's feedback Co-authored-by: syntactically <168595099+syntactically@users.noreply.github.com> Signed-off-by: Simon Davies --- .github/workflows/Benchmarks.yml | 2 +- .github/workflows/dep_rust.yml | 2 +- Justfile | 10 ++++------ 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/Benchmarks.yml b/.github/workflows/Benchmarks.yml index 3be8c549f..88b628e1e 100644 --- a/.github/workflows/Benchmarks.yml +++ b/.github/workflows/Benchmarks.yml @@ -57,7 +57,7 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Run Benchmarks - run: just bench-ci main release ${{ matrix.hypervisor == 'mshv' && 'mshv2' || ''}} + run: just bench-ci main ${{ matrix.hypervisor == 'mshv' && 'mshv2' || ''}} - uses: actions/upload-artifact@v4 with: diff --git a/.github/workflows/dep_rust.yml b/.github/workflows/dep_rust.yml index c266c25cc..b93042c50 100644 --- a/.github/workflows/dep_rust.yml +++ b/.github/workflows/dep_rust.yml @@ -186,5 +186,5 @@ jobs: - name: Run benchmarks run: | - just bench-ci main ${{ matrix.config }} ${{ matrix.hypervisor == 'mshv' && 'mshv2' || ''}} + just bench-ci main ${{ matrix.hypervisor == 'mshv' && 'mshv2' || ''}} if: ${{ matrix.config == 'release' }} diff --git a/Justfile b/Justfile index 80f115fac..c364937e7 100644 --- a/Justfile +++ b/Justfile @@ -326,14 +326,12 @@ bench-download os hypervisor cpu tag="": tar -zxvf target/benchmarks_{{ os }}_{{ hypervisor }}_{{ cpu }}.tar.gz -C target/criterion/ --strip-components=1 # Warning: compares to and then OVERWRITES the given baseline -bench-ci baseline target="release" features="": - @# Benchmarks should only be run with release builds for meaningful results - @if [ "{{ target }}" != "release" ]; then echo "Error: Benchmarks can only be run with release builds. Use 'just bench-ci release' instead."; exit 1; fi +bench-ci baseline features="": + @# Benchmarks are always run with release builds for meaningful results cargo bench --profile=release {{ if features =="" {''} else { "--features " + features } }} -- --verbose --save-baseline {{ baseline }} -bench target="release" features="": - @# Benchmarks should only be run with release builds for meaningful results - @if [ "{{ target }}" != "release" ]; then echo "Error: Benchmarks can only be run with release builds. Use 'just bench release' instead."; exit 1; fi +bench features="": + @# Benchmarks are always run with release builds for meaningful results cargo bench --profile=release {{ if features =="" {''} else { "--features " + features } }} -- --verbose ############### From f570c1f701bdca4f97b3671bc586084ccacde029 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 30 Jul 2025 19:41:38 +0000 Subject: [PATCH 5/5] Update documentation to reflect that debug builds are now supported for profiling Co-authored-by: ludfjig <4257730+ludfjig@users.noreply.github.com> Signed-off-by: Simon Davies --- docs/benchmarking-hyperlight.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/benchmarking-hyperlight.md b/docs/benchmarking-hyperlight.md index 86b79a30a..5ac2662f4 100644 --- a/docs/benchmarking-hyperlight.md +++ b/docs/benchmarking-hyperlight.md @@ -74,4 +74,4 @@ Found 1 outliers among 100 measurements (1.00%) Use `just bench` to run benchmarks with release builds (the only supported configuration). Comparing local benchmark results to github-saved benchmarks doesn't make much sense, since you'd be using different hardware, but you can use `just bench-download os hypervisor [tag] ` to download and extract the GitHub release benchmarks to the correct place folder. You can then run `just bench-ci main` to compare to (and overwrite) the previous release benchmarks. Note that `main` is the name of the baselines stored in GitHub. -**Important**: Benchmarks can only be run with release builds to ensure meaningful performance measurements. Debug builds are not supported for benchmarking. +**Important**: The `just bench` command uses release builds by default to ensure meaningful performance measurements. For profiling purposes, you can compile benchmarks with debug symbols by running `cargo bench` directly.