Skip to content

Commit 0f39bec

Browse files
lipchevlipchev
authored
Merge pull request #923 from lipchev/benchmark-workflows
Adding benchmark workflows
2 parents c64a627 + 558c6e2 commit 0f39bec

File tree

8 files changed

+359
-11
lines changed

8 files changed

+359
-11
lines changed

‎.github/actions/run-benchmarks/action.yaml

+43
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
name: 'Run benchmark'
2+
inputs:
3+
framework:
4+
description: 'The runtime version to use (e.g. net5.0)'
5+
required: false
6+
default: 'net5.0'
7+
runtimes:
8+
description: 'The runtime version to use (e.g. netcoreapp31, net5.0)'
9+
required: false
10+
default: 'net5.0'
11+
output-folder:
12+
description: 'The output folder for the benchmark (a results folder is created inside)'
13+
required: false
14+
default: 'Artifacts/Benchmark'
15+
exporters:
16+
description: 'The exporter(s) used for this run (GitHub/StackOverflow/RPlot/CSV/JSON/HTML/XML)'
17+
required: false
18+
default: fulljson rplot
19+
filter:
20+
description: 'An optional class filter to apply'
21+
required: false
22+
default: '*'
23+
categories:
24+
description: 'An optional categories filter to apply'
25+
required: false
26+
default: ''
27+
execution-options:
28+
description: 'Any additional parameters passed to the benchmark'
29+
required: false
30+
default: ''
31+
runs:
32+
using: "composite"
33+
steps:
34+
- name: Generating benchmark results
35+
run: >
36+
dotnet run --project UnitsNet.Benchmark -c Release
37+
--framework ${{ inputs.framework }}
38+
--runtimes ${{ inputs.runtimes }}
39+
--artifacts '${{ inputs.output-folder }}'
40+
--exporters ${{ inputs.exporters }}
41+
--filter '${{ inputs.filter }}'
42+
${{ inputs.categories }} ${{ inputs.execution-options }}
43+
shell: bash

‎.github/workflows/continious-benchmarking.yml

+101
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
name: UnitsNet Benchmarks (auto)
2+
on:
3+
push:
4+
branches: [master]
5+
paths:
6+
- "UnitsNet/*"
7+
- "UnitsNet.Benchmark/*"
8+
- ".github/workflows/**"
9+
- ".github/actions/**"
10+
11+
env:
12+
FRAMEWORK: net5.0
13+
EXECUTION_OPTIONS: --iterationTime 500 --disableLogFile # see https://benchmarkdotnet.org/articles/guides/console-args.html
14+
BENCHMARK_PAGES_BRANCH: gh-pages
15+
BENCHMARK_DATA_FOLDER: benchmarks
16+
17+
jobs:
18+
benchmark:
19+
runs-on: windows-latest # required by the older frameworks
20+
strategy:
21+
# max-parallel: 1 # is it better to avoid running in parallel?
22+
matrix:
23+
runtime: ["netcoreapp50", "netcoreapp21", "net472"]
24+
steps:
25+
- run: echo Starting benchmarks for ${{ matrix.runtime }}
26+
27+
# checkout the current branch
28+
- uses: actions/checkout@v2
29+
30+
# we need all frameworks (even if only running one target at a time)
31+
- uses: actions/setup-dotnet@v1
32+
with:
33+
dotnet-version: '2.1.x'
34+
35+
- uses: actions/setup-dotnet@v1
36+
with:
37+
dotnet-version: '3.1.x'
38+
39+
- uses: actions/setup-dotnet@v1
40+
with:
41+
dotnet-version: '5.0.x'
42+
43+
# executing the benchmark for the current framework, placing the result in a corresponding sub-folder
44+
- uses: ./.github/actions/run-benchmarks
45+
with:
46+
framework: ${{ env.FRAMEWORK }}
47+
runtimes: ${{ matrix.runtime }}
48+
output-folder: Artifacts/Benchmark/${{ matrix.runtime }}
49+
execution-options: ${{ env.EXECUTION_OPTIONS }}
50+
51+
# saving the current artifact (downloadable until the expiration date of this action)
52+
- name: Store benchmark artifact
53+
uses: actions/upload-artifact@v2
54+
with:
55+
name: UnitsNet Benchmarks (${{ matrix.runtime }})
56+
path: Artifacts/Benchmark/${{ matrix.runtime }}/results/
57+
58+
publish-results:
59+
needs: [benchmark]
60+
runs-on: ubuntu-latest
61+
strategy:
62+
max-parallel: 1 # cannot commit on the same branch in parallel
63+
matrix:
64+
runtime: ["netcoreapp50", "netcoreapp21", "net472"]
65+
steps:
66+
- name: Initializing git folder ️
67+
uses: actions/[email protected]
68+
69+
- name: Download Artifacts # The benchmark results are downloaded into a 'runtime' folder.
70+
uses: actions/download-artifact@v1
71+
with:
72+
name: UnitsNet Benchmarks (${{ matrix.runtime }})
73+
path: ${{ matrix.runtime }}
74+
75+
# publishing the current results to the benchmark-pages branch (overriding the previous result)
76+
- name: Saving benchmark results to ${{ env.BENCHMARK_PAGES_BRANCH }}/${{ env.BENCHMARK_DATA_FOLDER }}/${{ matrix.runtime }}/results
77+
uses: JamesIves/[email protected]
78+
with:
79+
folder: ${{ matrix.runtime }}
80+
branch: ${{ env.BENCHMARK_PAGES_BRANCH }}
81+
target-folder: ${{ env.BENCHMARK_DATA_FOLDER }}/${{ matrix.runtime }}/results
82+
commit-message: Automatic benchmark generation for ${{ github.sha }}
83+
84+
# appending to the running benchmark data on the benchmark-pages branch
85+
- name: Updating benchmark charts
86+
uses: starburst997/[email protected]
87+
with:
88+
name: UnitsNet Benchmarks (${{ matrix.runtime }})
89+
tool: 'benchmarkdotnet'
90+
output-file-path: ${{ matrix.runtime }}/UnitsNet.Benchmark.UnitsNetBenchmarks-report-full.json
91+
gh-pages-branch: ${{ env.BENCHMARK_PAGES_BRANCH }}
92+
benchmark-data-dir-path: ${{ env.BENCHMARK_DATA_FOLDER }}/${{ matrix.runtime }}
93+
github-token: ${{ secrets.GITHUB_TOKEN }}
94+
auto-push: true
95+
# Show alert with commit comment on detecting possible performance regression
96+
alert-threshold: '200%'
97+
comment-always: true
98+
comment-on-alert: true
99+
fail-on-alert: false
100+
alert-comment-cc-users: '@lipchev'
101+

‎.github/workflows/run-benchmarks.yml

+142
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
name: Run UnitsNet Benchmarks
2+
on:
3+
workflow_dispatch:
4+
inputs:
5+
benchmark-name:
6+
description: 'A name given for this benchmark'
7+
required: false
8+
default: 'UnitsNet Benchmarks'
9+
runtimes:
10+
description: 'The runtime version to use (e.g. net472 net48 netcoreapp21 netcoreapp31 netcoreapp50)'
11+
default: net472 netcoreapp21 netcoreapp50
12+
required: true
13+
exporters:
14+
description: 'The exporter(s) used for this run (GitHub/StackOverflow/RPlot/CSV/JSON/HTML/XML)'
15+
required: false
16+
default: fulljson rplot
17+
filter:
18+
description: 'An optional class filter to apply'
19+
required: false
20+
default: '*'
21+
categories:
22+
description: 'An optional categories filter to apply'
23+
required: false
24+
default: ''
25+
execution-options:
26+
description: 'Any additional parameters passed to the benchmark'
27+
required: false
28+
default: --disableLogFile
29+
comparison-baseline:
30+
description: 'Compare against a previous result (expecting a link to *-report-full.json)'
31+
required: true
32+
default: 'https://angularsen.github.io/UnitsNet/benchmarks/netcoreapp50/results/UnitsNet.Benchmark.UnitsNetBenchmarks-report-full.json'
33+
comparison-threshold:
34+
description: 'The (comparison) threshold for Statistical Test. Examples: 5%, 10ms, 100ns, 1s'
35+
required: false
36+
default: '1%'
37+
comparison-top:
38+
description: 'Filter the comparison to the top/bottom N results'
39+
required: false
40+
default: 10
41+
framework:
42+
description: 'The dotnet-version version to use (e.g. net5.0)'
43+
default: 'net5.0'
44+
required: true
45+
jobs:
46+
benchmark:
47+
env:
48+
OUTPUT_FOLDER: Artifacts/Benchmarks/${{ github.event.inputs.benchmark-name }}
49+
runs-on: windows-latest
50+
steps:
51+
# checkout the current branch
52+
- uses: actions/checkout@v2
53+
54+
# we need all frameworks (even if only running one target at a time)
55+
- uses: actions/setup-dotnet@v1
56+
with:
57+
dotnet-version: '2.1.x'
58+
59+
- uses: actions/setup-dotnet@v1
60+
with:
61+
dotnet-version: '3.1.x'
62+
63+
- uses: actions/setup-dotnet@v1
64+
with:
65+
dotnet-version: '5.0.x'
66+
67+
# executing the benchmark for the current framework(s), placing the result in the output-folder
68+
- uses: ./.github/actions/run-benchmarks
69+
with:
70+
framework: ${{ github.event.inputs.framework }}
71+
runtimes: ${{ github.event.inputs.runtimes }}
72+
output-folder: ${{ env.OUTPUT_FOLDER }}
73+
filter: ${{ github.event.inputs.filter }}
74+
categories: ${{ github.event.inputs.categories }}
75+
execution-options: ${{ github.event.inputs.execution-options }}
76+
77+
# saving the current artifact (downloadable until the expiration date of this action)
78+
- name: Store benchmark result
79+
uses: actions/upload-artifact@v2
80+
with:
81+
name: ${{ github.event.inputs.benchmark-name }} (${{ github.event.inputs.runtimes }})
82+
path: ${{ env.OUTPUT_FOLDER }}/results
83+
if-no-files-found: error
84+
85+
compare-results:
86+
if: ${{ endsWith(github.event.inputs.comparison-baseline, '-report-full.json') }}
87+
env:
88+
OUPUT_NAME: Baseline vs ${{ github.event.inputs.benchmark-name }} (${{ github.event.inputs.runtimes }})
89+
needs: [benchmark]
90+
runs-on: ubuntu-latest
91+
steps:
92+
# The baseline results are downloaded into a 'baseline' folder.
93+
- name: Download Baseline
94+
uses: carlosperate/[email protected]
95+
with:
96+
file-url: ${{ github.event.inputs.comparison-baseline }}
97+
location: baseline
98+
99+
# The benchmark results are downloaded into a 'runtime' folder.
100+
- name: Download Artifacts
101+
uses: actions/download-artifact@v1
102+
with:
103+
name: ${{ github.event.inputs.benchmark-name }} (${{ github.event.inputs.runtimes }})
104+
path: results
105+
106+
# The benchmark comparer is currently taken from dotnet/performance/src/tools/ResultsComparer (hoping that a 'tool' would eventually be made available)
107+
- name: Download ResultsComparer
108+
uses: actions/checkout@v2
109+
with:
110+
repository: dotnet/performance
111+
path: comparer
112+
113+
- uses: actions/setup-dotnet@v1
114+
with:
115+
dotnet-version: '3.1.x'
116+
117+
- run: mkdir -p artifacts
118+
119+
# Executing the comparer, placing the result in a 'comparison' folder (as well as creating a summary from the output)
120+
- name: Running the ResultsComparer
121+
env:
122+
PERFLAB_TARGET_FRAMEWORKS: netcoreapp3.1
123+
run: >
124+
dotnet run --project 'comparer/src/tools/ResultsComparer' -c Release
125+
--framework netcoreapp31
126+
--base baseline --diff results
127+
--csv "artifacts/${{ env.OUPUT_NAME }}.csv"
128+
--xml "artifacts/${{ env.OUPUT_NAME }}.xml"
129+
--threshold ${{ github.event.inputs.comparison-threshold }}
130+
--top ${{ github.event.inputs.comparison-top }}
131+
> "artifacts/${{ env.OUPUT_NAME }}.md"
132+
133+
- name: Summary
134+
run: cat "artifacts/${{ env.OUPUT_NAME }}.md"
135+
136+
# saving the current artifacts (downloadable until the expiration date of this action)
137+
- name: Store comparison result
138+
uses: actions/upload-artifact@v2
139+
with:
140+
name: ${{ env.OUPUT_NAME }}
141+
path: artifacts
142+

‎UnitsNet.Benchmark/Program.cs

+18-3
Original file line numberDiff line numberDiff line change
@@ -11,62 +11,77 @@ public class UnitsNetBenchmarks
1111
private IQuantity lengthIQuantity = Length.FromMeters(3.0);
1212

1313
[Benchmark]
14+
[BenchmarkCategory("Construction")]
1415
public Length Constructor() => new Length(3.0, LengthUnit.Meter);
1516

1617
[Benchmark]
18+
[BenchmarkCategory("Construction")]
1719
public Length Constructor_SI() => new Length(3.0, UnitSystem.SI);
1820

1921
[Benchmark]
22+
[BenchmarkCategory("Construction")]
2023
public Length FromMethod() => Length.FromMeters(3.0);
2124

2225
[Benchmark]
26+
[BenchmarkCategory("Transformation")]
2327
public double ToProperty() => length.Centimeters;
2428

2529
[Benchmark]
30+
[BenchmarkCategory("Transformation, Value")]
2631
public double As() => length.As(LengthUnit.Centimeter);
2732

2833
[Benchmark]
34+
[BenchmarkCategory("Transformation, Value")]
2935
public double As_SI() => length.As(UnitSystem.SI);
3036

3137
[Benchmark]
38+
[BenchmarkCategory("Transformation, Quantity")]
3239
public Length ToUnit() => length.ToUnit(LengthUnit.Centimeter);
3340

3441
[Benchmark]
42+
[BenchmarkCategory("Transformation, Quantity")]
3543
public Length ToUnit_SI() => length.ToUnit(UnitSystem.SI);
3644

3745
[Benchmark]
46+
[BenchmarkCategory("ToString")]
3847
public string ToStringTest() => length.ToString();
3948

4049
[Benchmark]
50+
[BenchmarkCategory("Parsing")]
4151
public Length Parse() => Length.Parse("3.0 m");
4252

4353
[Benchmark]
54+
[BenchmarkCategory("Parsing")]
4455
public bool TryParseValid() => Length.TryParse("3.0 m", out var l);
4556

4657
[Benchmark]
58+
[BenchmarkCategory("Parsing")]
4759
public bool TryParseInvalid() => Length.TryParse("3.0 zoom", out var l);
4860

4961
[Benchmark]
62+
[BenchmarkCategory("Construction")]
5063
public IQuantity QuantityFrom() => Quantity.From(3.0, LengthUnit.Meter);
5164

5265
[Benchmark]
66+
[BenchmarkCategory("Transformation, Value")]
5367
public double IQuantity_As() => lengthIQuantity.As(LengthUnit.Centimeter);
5468

5569
[Benchmark]
70+
[BenchmarkCategory("Transformation, Value")]
5671
public double IQuantity_As_SI() => lengthIQuantity.As(UnitSystem.SI);
5772

5873
[Benchmark]
74+
[BenchmarkCategory("Transformation, Quantity")]
5975
public IQuantity IQuantity_ToUnit() => lengthIQuantity.ToUnit(LengthUnit.Centimeter);
6076

6177
[Benchmark]
78+
[BenchmarkCategory("ToString")]
6279
public string IQuantity_ToStringTest() => lengthIQuantity.ToString();
6380
}
6481

6582
class Program
6683
{
6784
static void Main(string[] args)
68-
{
69-
var summary = BenchmarkRunner.Run<UnitsNetBenchmarks>();
70-
}
85+
=> BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args);
7186
}
7287
}

‎UnitsNet.Benchmark/Scripts/json-export-all-runtimes.bat

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
@echo off
2+
SET scriptdir=%~dp0
3+
SET projectdir="%scriptdir%..\.."
4+
SET exportdir="%projectdir%\Artifacts\Benchmark"
5+
:: this fails on the build server (also tested with the nightly benchmark.net package: 0.12.1.1533): possibly related to https://github.com/dotnet/BenchmarkDotNet/issues/1487
6+
dotnet run --project "%projectdir%/UnitsNet.Benchmark" -c Release ^
7+
--framework net5.0 ^
8+
--runtimes net472 net48 netcoreapp2.1 netcoreapp3.1 netcoreapp50 ^
9+
--artifacts=%exportdir% ^
10+
--exporters json ^
11+
--filter * ^
12+
--iterationTime 250 ^
13+
--statisticalTest 0.001ms ^
14+
--join %1 %2 %3
15+
16+
:: this runs fine, however there is currently no way of displaying multiple-lines-per-chart: see https://github.com/rhysd/github-action-benchmark/issues/18
17+
:: dotnet run --project "%scriptdir%/UnitsNet.Benchmark" -c Release -f net5.0 --runtimes netcoreapp31 netcoreapp50 --filter ** --artifacts="%scriptdir%/Artifacts/Benchmark" --exporters json

0 commit comments

Comments
 (0)