Skip to content

Commit 83f9bac

Browse files
KrzysztofCwalinaKrzysztofCwalina
authored
Added Microsoft.ML.Benchmarks Project (#62)
* Benchmark * Changed to .NET Core app * Added Accuracy Reporting * fixed build * Feedback from Gleb * Added batch prediction tests * Resolved conflicts the sln file * Renamed the new file to match type name * Removed duplicated method
1 parent 52ea962 commit 83f9bac

File tree

4 files changed

+226
-0
lines changed

4 files changed

+226
-0
lines changed

‎Microsoft.ML.sln‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Microsoft.ML.Parquet", "Mic
101101
pkg\Microsoft.ML.Parquet\Microsoft.ML.Parquet.nupkgproj = pkg\Microsoft.ML.Parquet\Microsoft.ML.Parquet.nupkgproj
102102
EndProjectSection
103103
EndProject
104+
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.Benchmarks", "test\Microsoft.ML.Benchmarks\Microsoft.ML.Benchmarks.csproj", "{7A9DB75F-2CA5-4184-9EF5-1F17EB39483F}"
105+
EndProject
104106
Global
105107
GlobalSection(SolutionConfigurationPlatforms) = preSolution
106108
Debug|Any CPU = Debug|Any CPU
@@ -195,6 +197,10 @@ Global
195197
{55C8122D-79EA-48AB-85D0-EB551FC1C427}.Debug|Any CPU.Build.0 = Debug|Any CPU
196198
{55C8122D-79EA-48AB-85D0-EB551FC1C427}.Release|Any CPU.ActiveCfg = Release|Any CPU
197199
{55C8122D-79EA-48AB-85D0-EB551FC1C427}.Release|Any CPU.Build.0 = Release|Any CPU
200+
{7A9DB75F-2CA5-4184-9EF5-1F17EB39483F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
201+
{7A9DB75F-2CA5-4184-9EF5-1F17EB39483F}.Debug|Any CPU.Build.0 = Debug|Any CPU
202+
{7A9DB75F-2CA5-4184-9EF5-1F17EB39483F}.Release|Any CPU.ActiveCfg = Release|Any CPU
203+
{7A9DB75F-2CA5-4184-9EF5-1F17EB39483F}.Release|Any CPU.Build.0 = Release|Any CPU
198204
EndGlobalSection
199205
GlobalSection(SolutionProperties) = preSolution
200206
HideSolutionNode = FALSE
@@ -228,6 +234,7 @@ Global
228234
{2DEFC784-F2B5-44EA-ABBB-0DCF3E689DAC} = {E20AF96D-3F66-4065-8A89-BEE479D74536}
229235
{DEC8F776-49F7-4D87-836C-FE4DC057D08C} = {D3D38B03-B557-484D-8348-8BADEE4DF592}
230236
{6C95FC87-F5F2-4EEF-BB97-567F2F5DD141} = {D3D38B03-B557-484D-8348-8BADEE4DF592}
237+
{7A9DB75F-2CA5-4184-9EF5-1F17EB39483F} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
231238
EndGlobalSection
232239
GlobalSection(ExtensibilityGlobals) = postSolution
233240
SolutionGuid = {41165AF1-35BB-4832-A189-73060F82B01D}

‎test/Microsoft.ML.Benchmarks/Microsoft.ML.Benchmarks.csproj‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
<Project Sdk="Microsoft.NET.Sdk" ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
2+
<PropertyGroup>
3+
<OutputType>Exe</OutputType>
4+
<LangVersion>7.2</LangVersion>
5+
<StartupObject>Microsoft.ML.Benchmarks.Program</StartupObject>
6+
<TargetFramework>netcoreapp2.0</TargetFramework>
7+
</PropertyGroup>
8+
<ItemGroup>
9+
<Compile Remove="BenchmarkDotNet.Artifacts\**" />
10+
<EmbeddedResource Remove="BenchmarkDotNet.Artifacts\**" />
11+
<None Remove="BenchmarkDotNet.Artifacts\**" />
12+
</ItemGroup>
13+
<ItemGroup>
14+
<PackageReference Include="BenchmarkDotNet" Version="0.10.14" />
15+
</ItemGroup>
16+
<ItemGroup>
17+
<ProjectReference Include="..\..\src\Microsoft.ML.StandardLearners\Microsoft.ML.StandardLearners.csproj" />
18+
<ProjectReference Include="..\..\src\Microsoft.ML\Microsoft.ML.csproj" />
19+
</ItemGroup>
20+
<ItemGroup>
21+
<NativeAssemblyReference Include="CpuMathNative" />
22+
</ItemGroup>
23+
</Project>

‎test/Microsoft.ML.Benchmarks/Program.cs‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
// See the LICENSE file in the project root for more information.
4+
5+
using BenchmarkDotNet.Configs;
6+
using BenchmarkDotNet.Diagnosers;
7+
using BenchmarkDotNet.Jobs;
8+
using BenchmarkDotNet.Running;
9+
using BenchmarkDotNet.Columns;
10+
using BenchmarkDotNet.Reports;
11+
using BenchmarkDotNet.Toolchains.CsProj;
12+
using BenchmarkDotNet.Toolchains.InProcess;
13+
using System;
14+
using System.IO;
15+
using Microsoft.ML.Models;
16+
using Microsoft.ML.Runtime.Api;
17+
using Microsoft.ML.Trainers;
18+
using Microsoft.ML.Transforms;
19+
using Microsoft.ML.Benchmarks;
20+
21+
namespace Microsoft.ML.Benchmarks
22+
{
23+
class Program
24+
{
25+
/// <summary>
26+
/// execute dotnet run -c Release and choose the benchmarks you want to run
27+
/// </summary>
28+
/// <param name="args"></param>
29+
static void Main(string[] args)
30+
{
31+
BenchmarkSwitcher
32+
.FromAssembly(typeof(Program).Assembly)
33+
.Run(null, CreateClrVsCoreConfig());
34+
}
35+
36+
private static IConfig CreateClrVsCoreConfig()
37+
{
38+
var config = DefaultConfig.Instance.With(
39+
Job.ShortRun.
40+
With(InProcessToolchain.Instance)).
41+
With(new ClassificationMetricsColumn("AccuracyMacro", "Macro-average accuracy of the model")).
42+
With(MemoryDiagnoser.Default);
43+
return config;
44+
}
45+
46+
internal static string GetDataPath(string name)
47+
=> Path.GetFullPath(Path.Combine(_dataRoot, name));
48+
49+
static readonly string _dataRoot;
50+
static Program()
51+
{
52+
var currentAssemblyLocation = new FileInfo(typeof(Program).Assembly.Location);
53+
var rootDir = currentAssemblyLocation.Directory.Parent.Parent.Parent.Parent.FullName;
54+
_dataRoot = Path.Combine(rootDir, "test", "data");
55+
}
56+
}
57+
58+
public class ClassificationMetricsColumn : IColumn
59+
{
60+
string _metricName;
61+
string _legend;
62+
63+
public ClassificationMetricsColumn(string metricName, string legend)
64+
{
65+
_metricName = metricName;
66+
_legend = legend;
67+
}
68+
69+
public string ColumnName => _metricName;
70+
public string Id => _metricName;
71+
public string Legend => _legend;
72+
public bool IsNumeric => true;
73+
public bool IsDefault(Summary summary, Benchmark benchmark) => true;
74+
public bool IsAvailable(Summary summary) => true;
75+
public bool AlwaysShow => true;
76+
public ColumnCategory Category => ColumnCategory.Custom;
77+
public int PriorityInCategory => 1;
78+
public UnitType UnitType => UnitType.Dimensionless;
79+
80+
public string GetValue(Summary summary, Benchmark benchmark, ISummaryStyle style)
81+
{
82+
var property = typeof(ClassificationMetrics).GetProperty(_metricName);
83+
return property.GetValue(StochasticDualCoordinateAscentClassifierBench.s_metrics).ToString();
84+
}
85+
public string GetValue(Summary summary, Benchmark benchmark) => GetValue(summary, benchmark, null);
86+
87+
public override string ToString() => ColumnName;
88+
}
89+
}

‎test/Microsoft.ML.Benchmarks/StochasticDualCoordinateAscentClassifierBench.cs‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
// See the LICENSE file in the project root for more information.
4+
5+
using BenchmarkDotNet.Attributes;
6+
using BenchmarkDotNet.Running;
7+
using Microsoft.ML.Models;
8+
using Microsoft.ML.Runtime.Api;
9+
using Microsoft.ML.Trainers;
10+
using Microsoft.ML.Transforms;
11+
using System;
12+
using System.Collections.Generic;
13+
using System.Linq;
14+
15+
namespace Microsoft.ML.Benchmarks
16+
{
17+
public class StochasticDualCoordinateAscentClassifierBench
18+
{
19+
internal static ClassificationMetrics s_metrics;
20+
private static PredictionModel<IrisData, IrisPrediction> s_trainedModel;
21+
private static string s_dataPath;
22+
private static IrisData[][] s_batches;
23+
private static readonly int[] s_batchSizes = new int[] { 1, 2, 5 };
24+
private readonly Random r = new Random(0);
25+
private readonly static IrisData s_example = new IrisData()
26+
{
27+
SepalLength = 3.3f,
28+
SepalWidth = 1.6f,
29+
PetalLength = 0.2f,
30+
PetalWidth = 5.1f,
31+
};
32+
33+
[Benchmark]
34+
public PredictionModel<IrisData, IrisPrediction> TrainIris() => TrainCore();
35+
36+
[Benchmark]
37+
public float[] PredictIris() => s_trainedModel.Predict(s_example).PredictedLabels;
38+
39+
[Benchmark]
40+
public IEnumerable<IrisPrediction> PredictIrisBatchOf1() => s_trainedModel.Predict(s_batches[0]);
41+
[Benchmark]
42+
public IEnumerable<IrisPrediction> PredictIrisBatchOf2() => s_trainedModel.Predict(s_batches[1]);
43+
[Benchmark]
44+
public IEnumerable<IrisPrediction> PredictIrisBatchOf5() => s_trainedModel.Predict(s_batches[2]);
45+
46+
[GlobalSetup]
47+
public void Setup()
48+
{
49+
s_dataPath = Program.GetDataPath("iris.txt");
50+
s_trainedModel = TrainCore();
51+
IrisPrediction prediction = s_trainedModel.Predict(s_example);
52+
53+
var testData = new TextLoader<IrisData>(s_dataPath, useHeader: true, separator: "tab");
54+
var evaluator = new ClassificationEvaluator();
55+
s_metrics = evaluator.Evaluate(s_trainedModel, testData);
56+
57+
s_batches = new IrisData[s_batchSizes.Length][];
58+
for (int i = 0; i < s_batches.Length; i++)
59+
{
60+
var batch = new IrisData[s_batchSizes[i]];
61+
s_batches[i] = batch;
62+
for (int bi = 0; bi < batch.Length; bi++)
63+
{
64+
batch[bi] = s_example;
65+
}
66+
}
67+
}
68+
69+
private static PredictionModel<IrisData, IrisPrediction> TrainCore()
70+
{
71+
var pipeline = new LearningPipeline();
72+
73+
pipeline.Add(new TextLoader<IrisData>(s_dataPath, useHeader: true, separator: "tab"));
74+
pipeline.Add(new ColumnConcatenator(outputColumn: "Features",
75+
"SepalLength", "SepalWidth", "PetalLength", "PetalWidth"));
76+
77+
pipeline.Add(new StochasticDualCoordinateAscentClassifier());
78+
79+
PredictionModel<IrisData, IrisPrediction> model = pipeline.Train<IrisData, IrisPrediction>();
80+
return model;
81+
}
82+
83+
public class IrisData
84+
{
85+
[Column("0")]
86+
public float Label;
87+
88+
[Column("1")]
89+
public float SepalLength;
90+
91+
[Column("2")]
92+
public float SepalWidth;
93+
94+
[Column("3")]
95+
public float PetalLength;
96+
97+
[Column("4")]
98+
public float PetalWidth;
99+
}
100+
101+
public class IrisPrediction
102+
{
103+
[ColumnName("Score")]
104+
public float[] PredictedLabels;
105+
}
106+
}
107+
}

0 commit comments

Comments
 (0)