How to compare local build with NuGet version

[Bykiev]

Hi, I've tried to create a benchmark to measure local build version with published NuGet version of the same libraries and it's seems to be not working and always using local build and memory report is the same for both tests. I'm using an in-process toolchain to avoid antivirus problems. How to solve it?

[gumbarros]

For context, we obtained even the same memory usage:
ncalc/ncalc#142 (comment)

[gumbarros]

This was our benchmark file, we don't if this is the correct way to compare local build and NuGet version:

using BenchmarkDotNet.Attributes;
using BenchmarkDotNet.Configs;
using BenchmarkDotNet.Environments;
using BenchmarkDotNet.Jobs;
using BenchmarkDotNet.Order;
using BenchmarkDotNet.Toolchains.InProcess.Emit;
using NCalc.Domain;


namespace NCalc.Benchmarks
{        
    [Config(typeof(Config))]
    [RankColumn]
    [MemoryDiagnoser]
    [Orderer(SummaryOrderPolicy.FastestToSlowest)]
    public class NCalcBenchmark
    {
        private class Config : ManualConfig
        {
            public Config()
            {
                var currentVersionJob = Job.ShortRun
                   .WithToolchain(InProcessEmitToolchain.Instance);
                var actualNugetVersionJob = Job.ShortRun.WithNuGet("NCalcSync")
                    .WithToolchain(InProcessEmitToolchain.Instance);

                AddJob(currentVersionJob.WithRuntime(ClrRuntime.Net462));
                AddJob(currentVersionJob.WithRuntime(CoreRuntime.Core80));

                AddJob(actualNugetVersionJob.WithRuntime(ClrRuntime.Net462));
                AddJob(actualNugetVersionJob.WithRuntime(CoreRuntime.Core80));
            }
        }

        [Benchmark]
        public object? SimpleExpression()
        {
            string expression = "1 - ( 3 + 2.5 ) * 4 - 1 / 2 + 1 - ( 3 + 2.5 ) * 4 - 1 / 2 + 1 - ( 3 + 2.5 ) * 4 - 1 / 2";

            Expression.CacheEnabled = false;
            var expr = new Expression(expression);
            
            return expr.Evaluate();
        }

        [Benchmark]
        public object? EvaluateCustomFunction()
        {
            Expression.CacheEnabled = false;
            var expr = new Expression("SecretOperation(3, 6)");
            expr.EvaluateFunction += delegate (string name, FunctionArgs args)
            {
                if (name == "SecretOperation")
                    args.Result = (int)args.Parameters[0].Evaluate() + (int)args.Parameters[1].Evaluate();
            };

            return expr.Evaluate();
        }

        [Benchmark]
        public object? EvaluateParameters()
        {
            Expression.CacheEnabled = false;
            var expr = new Expression("Round(Pow([Pi], 2) + Pow([Pi2], 2) + [X], 2)");

            expr.Parameters["Pi2"] = new Expression("Pi * [Pi]");
            expr.Parameters["X"] = 10;

            expr.EvaluateParameter += delegate (string name, ParameterArgs args)
            {
                if (name == "Pi")
                    args.Result = 3.14;
            };

           return expr.Evaluate();
        }
    }
}

[timcassell]

[Edit] Woops, not a duplicate. But WithNuGet does not support in process toolchains.

[timcassell]

If you want to compare multiple versions of the same library, you will have to use out of process toolchains.

[timcassell]

Or another option with in-process toolchain is to use extern alias. https://stackoverflow.com/a/70963102