diff --git a/Microsoft.ML.sln b/Microsoft.ML.sln
index 3e905b7328..c3815d87f3 100644
--- a/Microsoft.ML.sln
+++ b/Microsoft.ML.sln
@@ -101,6 +101,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Microsoft.ML.Parquet", "Mic
pkg\Microsoft.ML.Parquet\Microsoft.ML.Parquet.nupkgproj = pkg\Microsoft.ML.Parquet\Microsoft.ML.Parquet.nupkgproj
EndProjectSection
EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.Benchmarks", "test\Microsoft.ML.Benchmarks\Microsoft.ML.Benchmarks.csproj", "{7A9DB75F-2CA5-4184-9EF5-1F17EB39483F}"
+EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@@ -195,6 +197,10 @@ Global
{55C8122D-79EA-48AB-85D0-EB551FC1C427}.Debug|Any CPU.Build.0 = Debug|Any CPU
{55C8122D-79EA-48AB-85D0-EB551FC1C427}.Release|Any CPU.ActiveCfg = Release|Any CPU
{55C8122D-79EA-48AB-85D0-EB551FC1C427}.Release|Any CPU.Build.0 = Release|Any CPU
+ {7A9DB75F-2CA5-4184-9EF5-1F17EB39483F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {7A9DB75F-2CA5-4184-9EF5-1F17EB39483F}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {7A9DB75F-2CA5-4184-9EF5-1F17EB39483F}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {7A9DB75F-2CA5-4184-9EF5-1F17EB39483F}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
@@ -228,6 +234,7 @@ Global
{2DEFC784-F2B5-44EA-ABBB-0DCF3E689DAC} = {E20AF96D-3F66-4065-8A89-BEE479D74536}
{DEC8F776-49F7-4D87-836C-FE4DC057D08C} = {D3D38B03-B557-484D-8348-8BADEE4DF592}
{6C95FC87-F5F2-4EEF-BB97-567F2F5DD141} = {D3D38B03-B557-484D-8348-8BADEE4DF592}
+ {7A9DB75F-2CA5-4184-9EF5-1F17EB39483F} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {41165AF1-35BB-4832-A189-73060F82B01D}
diff --git a/test/Microsoft.ML.Benchmarks/Microsoft.ML.Benchmarks.csproj b/test/Microsoft.ML.Benchmarks/Microsoft.ML.Benchmarks.csproj
new file mode 100644
index 0000000000..735b6e1dd8
--- /dev/null
+++ b/test/Microsoft.ML.Benchmarks/Microsoft.ML.Benchmarks.csproj
@@ -0,0 +1,23 @@
+
+
+ Exe
+ 7.2
+ Microsoft.ML.Benchmarks.Program
+ netcoreapp2.0
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/test/Microsoft.ML.Benchmarks/Program.cs b/test/Microsoft.ML.Benchmarks/Program.cs
new file mode 100644
index 0000000000..9203fde247
--- /dev/null
+++ b/test/Microsoft.ML.Benchmarks/Program.cs
@@ -0,0 +1,89 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using BenchmarkDotNet.Configs;
+using BenchmarkDotNet.Diagnosers;
+using BenchmarkDotNet.Jobs;
+using BenchmarkDotNet.Running;
+using BenchmarkDotNet.Columns;
+using BenchmarkDotNet.Reports;
+using BenchmarkDotNet.Toolchains.CsProj;
+using BenchmarkDotNet.Toolchains.InProcess;
+using System;
+using System.IO;
+using Microsoft.ML.Models;
+using Microsoft.ML.Runtime.Api;
+using Microsoft.ML.Trainers;
+using Microsoft.ML.Transforms;
+using Microsoft.ML.Benchmarks;
+
+namespace Microsoft.ML.Benchmarks
+{
+ class Program
+ {
+ ///
+ /// execute dotnet run -c Release and choose the benchmarks you want to run
+ ///
+ ///
+ static void Main(string[] args)
+ {
+ BenchmarkSwitcher
+ .FromAssembly(typeof(Program).Assembly)
+ .Run(null, CreateClrVsCoreConfig());
+ }
+
+ private static IConfig CreateClrVsCoreConfig()
+ {
+ var config = DefaultConfig.Instance.With(
+ Job.ShortRun.
+ With(InProcessToolchain.Instance)).
+ With(new ClassificationMetricsColumn("AccuracyMacro", "Macro-average accuracy of the model")).
+ With(MemoryDiagnoser.Default);
+ return config;
+ }
+
+ internal static string GetDataPath(string name)
+ => Path.GetFullPath(Path.Combine(_dataRoot, name));
+
+ static readonly string _dataRoot;
+ static Program()
+ {
+ var currentAssemblyLocation = new FileInfo(typeof(Program).Assembly.Location);
+ var rootDir = currentAssemblyLocation.Directory.Parent.Parent.Parent.Parent.FullName;
+ _dataRoot = Path.Combine(rootDir, "test", "data");
+ }
+ }
+
+ public class ClassificationMetricsColumn : IColumn
+ {
+ string _metricName;
+ string _legend;
+
+ public ClassificationMetricsColumn(string metricName, string legend)
+ {
+ _metricName = metricName;
+ _legend = legend;
+ }
+
+ public string ColumnName => _metricName;
+ public string Id => _metricName;
+ public string Legend => _legend;
+ public bool IsNumeric => true;
+ public bool IsDefault(Summary summary, Benchmark benchmark) => true;
+ public bool IsAvailable(Summary summary) => true;
+ public bool AlwaysShow => true;
+ public ColumnCategory Category => ColumnCategory.Custom;
+ public int PriorityInCategory => 1;
+ public UnitType UnitType => UnitType.Dimensionless;
+
+ public string GetValue(Summary summary, Benchmark benchmark, ISummaryStyle style)
+ {
+ var property = typeof(ClassificationMetrics).GetProperty(_metricName);
+ return property.GetValue(StochasticDualCoordinateAscentClassifierBench.s_metrics).ToString();
+ }
+ public string GetValue(Summary summary, Benchmark benchmark) => GetValue(summary, benchmark, null);
+
+ public override string ToString() => ColumnName;
+ }
+}
diff --git a/test/Microsoft.ML.Benchmarks/StochasticDualCoordinateAscentClassifierBench.cs b/test/Microsoft.ML.Benchmarks/StochasticDualCoordinateAscentClassifierBench.cs
new file mode 100644
index 0000000000..e0583f58b7
--- /dev/null
+++ b/test/Microsoft.ML.Benchmarks/StochasticDualCoordinateAscentClassifierBench.cs
@@ -0,0 +1,107 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Running;
+using Microsoft.ML.Models;
+using Microsoft.ML.Runtime.Api;
+using Microsoft.ML.Trainers;
+using Microsoft.ML.Transforms;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace Microsoft.ML.Benchmarks
+{
+ public class StochasticDualCoordinateAscentClassifierBench
+ {
+ internal static ClassificationMetrics s_metrics;
+ private static PredictionModel s_trainedModel;
+ private static string s_dataPath;
+ private static IrisData[][] s_batches;
+ private static readonly int[] s_batchSizes = new int[] { 1, 2, 5 };
+ private readonly Random r = new Random(0);
+ private readonly static IrisData s_example = new IrisData()
+ {
+ SepalLength = 3.3f,
+ SepalWidth = 1.6f,
+ PetalLength = 0.2f,
+ PetalWidth = 5.1f,
+ };
+
+ [Benchmark]
+ public PredictionModel TrainIris() => TrainCore();
+
+ [Benchmark]
+ public float[] PredictIris() => s_trainedModel.Predict(s_example).PredictedLabels;
+
+ [Benchmark]
+ public IEnumerable PredictIrisBatchOf1() => s_trainedModel.Predict(s_batches[0]);
+ [Benchmark]
+ public IEnumerable PredictIrisBatchOf2() => s_trainedModel.Predict(s_batches[1]);
+ [Benchmark]
+ public IEnumerable PredictIrisBatchOf5() => s_trainedModel.Predict(s_batches[2]);
+
+ [GlobalSetup]
+ public void Setup()
+ {
+ s_dataPath = Program.GetDataPath("iris.txt");
+ s_trainedModel = TrainCore();
+ IrisPrediction prediction = s_trainedModel.Predict(s_example);
+
+ var testData = new TextLoader(s_dataPath, useHeader: true, separator: "tab");
+ var evaluator = new ClassificationEvaluator();
+ s_metrics = evaluator.Evaluate(s_trainedModel, testData);
+
+ s_batches = new IrisData[s_batchSizes.Length][];
+ for (int i = 0; i < s_batches.Length; i++)
+ {
+ var batch = new IrisData[s_batchSizes[i]];
+ s_batches[i] = batch;
+ for (int bi = 0; bi < batch.Length; bi++)
+ {
+ batch[bi] = s_example;
+ }
+ }
+ }
+
+ private static PredictionModel TrainCore()
+ {
+ var pipeline = new LearningPipeline();
+
+ pipeline.Add(new TextLoader(s_dataPath, useHeader: true, separator: "tab"));
+ pipeline.Add(new ColumnConcatenator(outputColumn: "Features",
+ "SepalLength", "SepalWidth", "PetalLength", "PetalWidth"));
+
+ pipeline.Add(new StochasticDualCoordinateAscentClassifier());
+
+ PredictionModel model = pipeline.Train();
+ return model;
+ }
+
+ public class IrisData
+ {
+ [Column("0")]
+ public float Label;
+
+ [Column("1")]
+ public float SepalLength;
+
+ [Column("2")]
+ public float SepalWidth;
+
+ [Column("3")]
+ public float PetalLength;
+
+ [Column("4")]
+ public float PetalWidth;
+ }
+
+ public class IrisPrediction
+ {
+ [ColumnName("Score")]
+ public float[] PredictedLabels;
+ }
+ }
+}