diff --git a/.github/workflows/Benchmark.yml b/.github/workflows/Benchmark.yml deleted file mode 100644 index 31ee4b3bc..000000000 --- a/.github/workflows/Benchmark.yml +++ /dev/null @@ -1,38 +0,0 @@ -name: Run benchmarks -on: - pull_request: - # Only trigger the benchmark job when you add `run benchmark` label to the PR - types: [labeled, opened, synchronize, reopened] -concurrency: - # Skip intermediate builds: always. - # Cancel intermediate builds: only if it is a pull request build. - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} -jobs: - Benchmark: - runs-on: ubuntu-latest - permissions: - pull-requests: write - actions: write # needed to allow julia-actions/cache to proactively delete old caches that it has created - contents: read - if: contains(github.event.pull_request.labels.*.name, 'benchmark') - steps: - - uses: actions/checkout@v4 - - uses: julia-actions/setup-julia@latest - - name: Cache artifacts - uses: actions/cache@v4 - env: - cache-name: cache-artifacts - with: - path: ~/.julia/artifacts - key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }} - restore-keys: | - ${{ runner.os }}-test-${{ env.cache-name }}- - ${{ runner.os }}-test- - ${{ runner.os }}- - - name: Install dependencies - run: julia -e 'using Pkg; pkg"add JSON PkgBenchmark BenchmarkCI@0.1"' - - name: Run benchmarks - run: julia benchmark/run_benchmarks.jl - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 000000000..6376c147b --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,13 @@ +name: Benchmark this PR +on: + pull_request_target: + branches: [ main ] +permissions: + pull-requests: write # needed to post comments +jobs: + bench: + runs-on: ubuntu-latest + steps: + - uses: MilesCranmer/AirspeedVelocity.jl@action-v1 + with: + julia-version: '1' \ No newline at end of file diff --git a/benchmark/Project.toml b/benchmark/Project.toml index ac0ec4a01..2e069fc1b 100644 --- a/benchmark/Project.toml +++ b/benchmark/Project.toml @@ -1,5 +1,6 @@ [deps] BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" +ControlSystemsBase = "aaaaaaaa-a6ca-5380-bf3e-84a91bcd477e" DAQP = "c47d62df-3981-49c8-9651-128b1cd08617" FiniteDiff = "6a86dc24-6348-571c-b903-95158fe2bd41" Ipopt = "b6b21f68-93f8-5de0-b562-5493be1d77c9" diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl index b888697c3..345e49a5e 100644 --- a/benchmark/benchmarks.jl +++ b/benchmark/benchmarks.jl @@ -1,25 +1,141 @@ using BenchmarkTools -using Random +using ModelPredictiveControl, ControlSystemsBase, LinearAlgebra + +Ts = 400.0 +sys = [ tf(1.90,[1800.0,1]) tf(1.90,[1800.0,1]) tf(1.90,[1800.0,1]); + tf(-0.74,[800.0,1]) tf(0.74,[800.0,1]) tf(-0.74,[800.0,1]) ] const SUITE = BenchmarkGroup() -SUITE["utf8"] = BenchmarkGroup(["string", "unicode"]) -teststr = String(join(rand(MersenneTwister(1), 'a':'d', 10^4))) -SUITE["utf8"]["replace"] = @benchmarkable replace($teststr, "a" => "b") -SUITE["utf8"]["join"] = @benchmarkable join($teststr, $teststr) -SUITE["utf8"]["plots"] = BenchmarkGroup() - -SUITE["trigonometry"] = BenchmarkGroup(["math", "triangles"]) -SUITE["trigonometry"]["circular"] = BenchmarkGroup() -for f in (sin, cos, tan) - for x in (0.0, pi) - SUITE["trigonometry"]["circular"][string(f), x] = @benchmarkable ($f)($x) - end +## ================================================================================== +## ================== SimModel benchmarks =========================================== +## ================================================================================== +linmodel = setop!(LinModel(sys, Ts, i_d=[3]), uop=[10, 50], yop=[50, 30], dop=[5]) +function f!(ẋ, x, u, d, p) + mul!(ẋ, p.A, x) + mul!(ẋ, p.Bu, u, 1, 1) + mul!(ẋ, p.Bd, d, 1, 1) + return nothing +end +function h!(y, x, d, p) + mul!(y, p.C, x) + mul!(y, p.Dd, d, 1, 1) + return nothing end +nonlinmodel = NonLinModel(f!, h!, Ts, 2, 4, 2, 1, p=linmodel, solver=nothing) +nonlinmodel = setop!(nonlinmodel, uop=[10, 50], yop=[50, 30], dop=[5]) +u, d, y = [10, 50], [5], [50, 30] + +SUITE["SimModel"]["allocation"] = BenchmarkGroup(["allocation"]) +SUITE["SimModel"]["allocation"]["LinModel_updatestate!"] = @benchmarkable( + updatestate!($linmodel, $u, $d), + samples=1 +) +SUITE["SimModel"]["allocation"]["LinModel_evaloutput"] = @benchmarkable( + evaloutput($linmodel, $d), + samples=1 +) +SUITE["SimModel"]["allocation"]["NonLinModel_updatestate!"] = @benchmarkable( + updatestate!($nonlinmodel, $u, $d), + samples=1 +) +SUITE["SimModel"]["allocation"]["NonLinModel_evaloutput"] = @benchmarkable( + evaloutput($nonlinmodel, $d), + samples=1 +) + +SUITE["SimModel"]["allocation"]["NonLinModel_linearize!"] = @benchmarkable( + linearize!($linmodel, $nonlinmodel), + samples=1 +) + +## ================================================================================== +## ================== StateEstimator benchmarks ===================================== +## ================================================================================== +skf = SteadyKalmanFilter(linmodel) +SUITE["StateEstimator"]["allocation"] = BenchmarkGroup(["allocation"]) +SUITE["StateEstimator"]["allocation"]["SteadyKalmanFilter_preparestate!"] = @benchmarkable( + preparestate!($skf, $y, $d), + samples=1 +) +SUITE["StateEstimator"]["allocation"]["SteadyKalmanFilter_updatestate!"] = @benchmarkable( + updatestate!($skf, $u, $y, $d), + setup=preparestate!($skf, $y, $d), + samples=1 +) +SUITE["StateEstimator"]["allocation"]["SteadyKalmanFilter_evaloutput"] = @benchmarkable( + evaloutput($skf, $d), + setup=preparestate!($skf, $y, $d), + samples=1 +) + +kf = KalmanFilter(linmodel, nint_u=[1, 1], direct=false) +SUITE["StateEstimator"]["allocation"]["KalmanFilter_preparestate!"] = @benchmarkable( + preparestate!($kf, $y, $d), + samples=1 +) +SUITE["StateEstimator"]["allocation"]["KalmanFilter_updatestate!"] = @benchmarkable( + updatestate!($kf, $u, $y, $d), + setup=preparestate!($kf, $y, $d), + samples=1 +) + +lo = Luenberger(linmodel, nint_u=[1, 1]) +#SUITE["StateEstimator"]["allocation"]["Luenberger_preparestate!"] = @benchmarkable( +# preparestate!($lo, $y, $d), +# samples=1 +#) +SUITE["StateEstimator"]["allocation"]["Luenberger_updatestate!"] = @benchmarkable( + updatestate!($lo, $u, $y, $d), + setup=preparestate!($lo, $y, $d), + samples=1 +) + +im = InternalModel(nonlinmodel) +SUITE["StateEstimator"]["allocation"]["InternalModel_preparestate!"] = @benchmarkable( + preparestate!($im, $y, $d), + samples=1 +) +SUITE["StateEstimator"]["allocation"]["InternalModel_updatestate!"] = @benchmarkable( + updatestate!($im, $u, $y, $d), + setup=preparestate!($im, $y, $d), + samples=1 +) + +ukf = UnscentedKalmanFilter(nonlinmodel) +SUITE["StateEstimator"]["allocation"]["UnscentedKalmanFilter_preparestate!"] = @benchmarkable( + preparestate!($ukf, $y, $d), + samples=1 +) +SUITE["StateEstimator"]["allocation"]["UnscentedKalmanFilter_updatestate!"] = @benchmarkable( + updatestate!($ukf, $u, $y, $d), + setup=preparestate!($ukf, $y, $d), + samples=1 +) +SUITE["StateEstimator"]["allocation"]["UnscentedKalmanFilter_evaloutput"] = @benchmarkable( + evaloutput($ukf, $d), + setup=preparestate!($ukf, $y, $d), + samples=1 +) + +ekf = ExtendedKalmanFilter(linmodel, nint_u=[1, 1], direct=false) +SUITE["StateEstimator"]["allocation"]["ExtendedKalmanFilter_preparestate!"] = @benchmarkable( + preparestate!($ekf, $y, $d), + samples=1 +) +SUITE["StateEstimator"]["allocation"]["ExtendedKalmanFilter_updatestate!"] = @benchmarkable( + updatestate!($ekf, $u, $y, $d), + setup=preparestate!($ekf, $y, $d), + samples=1 +) -SUITE["trigonometry"]["hyperbolic"] = BenchmarkGroup() -for f in (sin, cos, tan) - for x in (0.0, pi) - SUITE["trigonometry"]["hyperbolic"][string(f), x] = @benchmarkable ($f)($x) - end -end \ No newline at end of file +## ================================================================================== +## ================== PredictiveController benchmarks =============================== +## ================================================================================== +empc = ExplicitMPC(linmodel, Mwt=[1, 1], Nwt=[0.1, 0.1], Lwt=[0.1, 0.1]) +SUITE["PredictiveController"]["allocation"] = BenchmarkGroup(["allocation"]) +SUITE["PredictiveController"]["allocation"]["ExplicitMPC_moveinput!"] = @benchmarkable( + moveinput!($empc, $y, $d), + setup=preparestate!($empc, $y, $d), + samples=1 +) \ No newline at end of file diff --git a/test/1_test_sim_model.jl b/test/1_test_sim_model.jl index d054a3a02..f4a8907b4 100644 --- a/test/1_test_sim_model.jl +++ b/test/1_test_sim_model.jl @@ -108,11 +108,9 @@ end u, d = [10, 50], Float64[] @test updatestate!(linmodel1, u) ≈ zeros(2) @test updatestate!(linmodel1, u, d) ≈ zeros(2) - @test_skip @allocations(updatestate!(linmodel1, u)) == 0 @test linmodel1.x0 ≈ zeros(2) @test evaloutput(linmodel1) ≈ linmodel1() ≈ [50,30] @test evaloutput(linmodel1, Float64[]) ≈ linmodel1(Float64[]) ≈ [50,30] - @test_skip @allocations(evaloutput(linmodel1)) == 0 x = initstate!(linmodel1, [10, 60]) @test evaloutput(linmodel1) ≈ [50 + 19.0, 30 + 7.4] @test preparestate!(linmodel1) ≈ x # new method @@ -283,11 +281,9 @@ end u, d = zeros(2), Float64[] @test updatestate!(nonlinmodel, u) ≈ zeros(2) @test updatestate!(nonlinmodel, u, d) ≈ zeros(2) - @test_skip @allocations(updatestate!(nonlinmodel, u)) == 0 @test nonlinmodel.x0 ≈ zeros(2) @test evaloutput(nonlinmodel) ≈ nonlinmodel() ≈ zeros(2) @test evaloutput(nonlinmodel, d) ≈ nonlinmodel(Float64[]) ≈ zeros(2) - @test_skip @allocations(evaloutput(nonlinmodel)) == 0 x = initstate!(nonlinmodel, [0, 10]) # do nothing for NonLinModel @test evaloutput(nonlinmodel) ≈ [0, 0] @@ -375,19 +371,6 @@ end h2!(y, x, _, _) = (y .= x) nonlinmodel4 = NonLinModel(f2!,h2!,Ts,1,1,1,0,solver=nothing,jacobian=AutoFiniteDiff()) @test_nowarn linearize(nonlinmodel4, x=[1], u=[2]) - - function f3!(xnext, x, u, d, _) - xnext .= x.*u .+ x.*d - end - function h3!(y, x, d, _) - y .= x.*d - end - nonlinmodel4 = NonLinModel(f3!, h3!, Ts, 1, 1, 1, 1, solver=nothing) - linmodel4 = linearize(nonlinmodel4; x, u, d) - # return nothing (see this issue : https://github.com/JuliaLang/julia/issues/51112): - linearize2!(linmodel, model) = (linearize!(linmodel, model); nothing) - linearize2!(linmodel4, nonlinmodel4) - @test_skip @allocations(linearize2!(linmodel4, nonlinmodel4)) == 0 end @testitem "NonLinModel real time simulations" setup=[SetupMPCtests] begin diff --git a/test/2_test_state_estim.jl b/test/2_test_state_estim.jl index 24134f355..8431028ae 100644 --- a/test/2_test_state_estim.jl +++ b/test/2_test_state_estim.jl @@ -71,12 +71,9 @@ end preparestate!(kalmanfilter1, y) @test updatestate!(kalmanfilter1, u, y, d) ≈ zeros(4) @test kalmanfilter1.x̂0 ≈ zeros(4) - @test_skip @allocations(preparestate!(kalmanfilter1, y)) == 0 - @test_skip @allocations(updatestate!(kalmanfilter1, u, y)) == 0 preparestate!(kalmanfilter1, y) @test evaloutput(kalmanfilter1) ≈ kalmanfilter1() ≈ [50, 30] @test evaloutput(kalmanfilter1, d) ≈ kalmanfilter1(d) ≈ [50, 30] - @test_skip @allocations(evaloutput(kalmanfilter1, d)) == 0 @test initstate!(kalmanfilter1, [10, 50], [50, 30+1]) ≈ [zeros(3); [1]] linmodel2 = LinModel(append(tf(1, [1, 0]), tf(2, [10, 1])), 1.0) kalmanfilter2 = SteadyKalmanFilter(linmodel2, nint_u=[1, 1], direct=false) @@ -205,12 +202,9 @@ end preparestate!(kalmanfilter1, y) @test updatestate!(kalmanfilter1, u, y, d) ≈ zeros(4) @test kalmanfilter1.x̂0 ≈ zeros(4) - @test_skip @allocations(preparestate!(kalmanfilter1, y)) == 0 - @test_skip @allocations(updatestate!(kalmanfilter1, u, y)) == 0 preparestate!(kalmanfilter1, y) @test evaloutput(kalmanfilter1) ≈ kalmanfilter1() ≈ [50, 30] @test evaloutput(kalmanfilter1, d) ≈ kalmanfilter1(d) ≈ [50, 30] - @test_skip @allocations(evaloutput(kalmanfilter1, d)) == 0 @test initstate!(kalmanfilter1, [10, 50], [50, 30+1]) ≈ [zeros(3); [1]] setstate!(kalmanfilter1, [1,2,3,4], diagm(.1:.1:.4)) @test kalmanfilter1.x̂0 ≈ [1,2,3,4] @@ -326,12 +320,9 @@ end preparestate!(lo1, y) @test updatestate!(lo1, u, y, d) ≈ zeros(4) @test lo1.x̂0 ≈ zeros(4) - @test_skip @allocations(preparestate!(lo1, y)) == 0 - @test_skip @allocations(updatestate!(lo1, u, y)) == 0 preparestate!(lo1, y) @test evaloutput(lo1) ≈ lo1() ≈ [50, 30] @test evaloutput(lo1, d) ≈ lo1(d) ≈ [50, 30] - @test_skip @allocations(evaloutput(lo1, d)) == 0 @test initstate!(lo1, [10, 50], [50, 30+1]) ≈ [zeros(3); [1]] setstate!(lo1, [1,2,3,4]) @test lo1.x̂0 ≈ [1,2,3,4] @@ -457,11 +448,8 @@ end @test updatestate!(internalmodel1, u, y, d) ≈ zeros(2) @test internalmodel1.x̂d ≈ internalmodel1.x̂0 ≈ zeros(2) @test internalmodel1.x̂s ≈ ones(2) - @test_skip @allocations(preparestate!(internalmodel1, y)) == 0 - @test_skip @allocations(updatestate!(internalmodel1, u, y)) == 0 preparestate!(internalmodel1, y) @test evaloutput(internalmodel1, d) ≈ [51,31] - @test_skip @allocations(evaloutput(internalmodel1, d)) == 0 @test initstate!(internalmodel1, [10, 50], [50, 30]) ≈ zeros(2) @test internalmodel1.x̂s ≈ zeros(2) setstate!(internalmodel1, [1,2]) @@ -593,12 +581,9 @@ end preparestate!(ukf1, y) @test updatestate!(ukf1, u, y, d) ≈ zeros(4) atol=1e-9 @test ukf1.x̂0 ≈ zeros(4) atol=1e-9 - @test_skip @allocations(preparestate!(ukf1, y)) == 0 - @test_skip @allocations(updatestate!(ukf1, u, y)) == 0 preparestate!(ukf1, y) @test evaloutput(ukf1) ≈ ukf1() ≈ [50, 30] @test evaloutput(ukf1, d) ≈ ukf1(d) ≈ [50, 30] - @test_skip @allocations(evaloutput(ukf1, d)) == 0 @test initstate!(ukf1, [10, 50], [50, 30+1]) ≈ zeros(4) atol=1e-9 setstate!(ukf1, [1,2,3,4], diagm(.1:.1:.4)) @test ukf1.x̂0 ≈ [1,2,3,4] @@ -752,12 +737,9 @@ end preparestate!(ekf1, y) @test updatestate!(ekf1, u, y, d) ≈ zeros(4) atol=1e-9 @test ekf1.x̂0 ≈ zeros(4) atol=1e-9 - @test_skip @allocations(preparestate!(ekf1, y)) == 0 - @test_skip @allocations(updatestate!(ekf1, u, y)) == 0 preparestate!(ekf1, y) @test evaloutput(ekf1) ≈ ekf1() ≈ [50, 30] @test evaloutput(ekf1, d) ≈ ekf1(d) ≈ [50, 30] - @test_skip @allocations(evaloutput(ekf1, d)) == 0 @test initstate!(ekf1, [10, 50], [50, 30+1]) ≈ zeros(4); setstate!(ekf1, [1,2,3,4], diagm(.1:.1:.4)) @test ekf1.x̂0 ≈ [1,2,3,4] diff --git a/test/3_test_predictive_control.jl b/test/3_test_predictive_control.jl index 85185a8aa..c0644f20f 100644 --- a/test/3_test_predictive_control.jl +++ b/test/3_test_predictive_control.jl @@ -496,7 +496,6 @@ end @test u ≈ [1] atol=1e-2 u = mpc1(r) @test u ≈ [1] atol=1e-2 - @test_skip @allocations(moveinput!(mpc1, r)) == 0 info = getinfo(mpc1) @test info[:u] ≈ u @test info[:Ŷ][end] ≈ r[1] atol=1e-2