Splitting OLS to a separate package called AdditionalLearners

Microsoft.ML.sln

@@ -99,6 +99,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.CodeAnalyzer.T
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.ImageAnalytics", "src\Microsoft.ML.ImageAnalytics\Microsoft.ML.ImageAnalytics.csproj", "{00E38F77-1E61-4CDF-8F97-1417D4E85053}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.HalLearners", "src\Microsoft.ML.HalLearners\Microsoft.ML.HalLearners.csproj", "{A7222F41-1CF0-47D9-B80C-B4D77B027A61}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -339,6 +341,14 @@ Global
 		{00E38F77-1E61-4CDF-8F97-1417D4E85053}.Release|Any CPU.Build.0 = Release|Any CPU
 		{00E38F77-1E61-4CDF-8F97-1417D4E85053}.Release-Intrinsics|Any CPU.ActiveCfg = Release|Any CPU
 		{00E38F77-1E61-4CDF-8F97-1417D4E85053}.Release-Intrinsics|Any CPU.Build.0 = Release|Any CPU
+		{A7222F41-1CF0-47D9-B80C-B4D77B027A61}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{A7222F41-1CF0-47D9-B80C-B4D77B027A61}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{A7222F41-1CF0-47D9-B80C-B4D77B027A61}.Debug-Intrinsics|Any CPU.ActiveCfg = Debug|Any CPU
+		{A7222F41-1CF0-47D9-B80C-B4D77B027A61}.Debug-Intrinsics|Any CPU.Build.0 = Debug|Any CPU
+		{A7222F41-1CF0-47D9-B80C-B4D77B027A61}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{A7222F41-1CF0-47D9-B80C-B4D77B027A61}.Release|Any CPU.Build.0 = Release|Any CPU
+		{A7222F41-1CF0-47D9-B80C-B4D77B027A61}.Release-Intrinsics|Any CPU.ActiveCfg = Release|Any CPU
+		{A7222F41-1CF0-47D9-B80C-B4D77B027A61}.Release-Intrinsics|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -378,6 +388,7 @@ Global
 		{B4E55B2D-2A92-46E7-B72F-E76D6FD83440} = {7F13E156-3EBA-4021-84A5-CD56BA72F99E}
 		{3E4ABF07-7970-4BE6-B45B-A13D3C397545} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
 		{00E38F77-1E61-4CDF-8F97-1417D4E85053} = {09EADF06-BE25-4228-AB53-95AE3E15B530}
+		{A7222F41-1CF0-47D9-B80C-B4D77B027A61} = {09EADF06-BE25-4228-AB53-95AE3E15B530}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {41165AF1-35BB-4832-A189-73060F82B01D}

build/Dependencies.props

@@ -8,7 +8,7 @@
     <SystemReflectionEmitLightweightPackageVersion>4.3.0</SystemReflectionEmitLightweightPackageVersion>
     <PublishSymbolsPackageVersion>1.0.0-beta-62824-02</PublishSymbolsPackageVersion>
     <LightGBMPackageVersion>2.1.2.2</LightGBMPackageVersion>
-    <MlNetMklDepsPackageVersion>0.0.0.1</MlNetMklDepsPackageVersion>
+    <MlNetMklDepsPackageVersion>0.0.0.5</MlNetMklDepsPackageVersion>
     <SystemDrawingCommonPackageVersion>4.5.0</SystemDrawingCommonPackageVersion>
   </PropertyGroup>
 </Project>

pkg/Microsoft.ML.HalLearners/Microsoft.ML.HalLearners.nupkgproj

@@ -0,0 +1,13 @@
+<Project Sdk="Microsoft.NET.Sdk" DefaultTargets="Pack">
+
+  <PropertyGroup>
+    <TargetFramework>netstandard2.0</TargetFramework>
+    <PackageDescription>ML.NET additional learners making use of hardware acceleration. They depend on the MlNetMklDeps NuGet package.</PackageDescription>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="../Microsoft.ML/Microsoft.ML.nupkgproj" />
+    <PackageReference Include="MlNetMklDeps" Version="$(MlNetMklDepsPackageVersion)" />
+  </ItemGroup>
+
+</Project>

pkg/Microsoft.ML.HalLearners/Microsoft.ML.HalLearners.symbols.nupkgproj

@@ -0,0 +1,5 @@
+<Project DefaultTargets="Pack">
+
+  <Import Project="Microsoft.ML.HalLearners.nupkgproj" />
+
+</Project>

src/Microsoft.ML.Core/ComponentModel/ComponentCatalog.cs

@@ -343,7 +343,7 @@ private static bool ShouldSkipPath(string path)
             case "libvw.dll":
             case "matrixinterf.dll":
             case "Microsoft.ML.neuralnetworks.gpucuda.dll":
-            case "Microsoft.ML.mklimports.dll":
+            case "MklImports.dll":
             case "microsoft.research.controls.decisiontrees.dll":
             case "Microsoft.ML.neuralnetworks.sse.dll":
             case "neuraltreeevaluator.dll":

src/Microsoft.ML.HalLearners/Microsoft.ML.HalLearners.csproj

@@ -0,0 +1,16 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>netstandard2.0</TargetFramework>
+    <IncludeInPackage>Microsoft.ML.HalLearners</IncludeInPackage>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Microsoft.ML.Core\Microsoft.ML.Core.csproj" />
+    <ProjectReference Include="..\Microsoft.ML.Data\Microsoft.ML.Data.csproj" />
+    <ProjectReference Include="..\Microsoft.ML.StandardLearners\Microsoft.ML.StandardLearners.csproj" />
+    <ProjectReference Include="..\Microsoft.ML\Microsoft.ML.csproj" />
+    <PackageReference Include="MlNetMklDeps" Version="$(MlNetMklDepsPackageVersion)" />
+  </ItemGroup>
+
+</Project>

src/Microsoft.ML.StandardLearners/Standard/OlsLinearRegression.cs → src/Microsoft.ML.HalLearners/OlsLinearRegression.cs

@@ -7,14 +7,15 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
-using Microsoft.ML.Runtime.Internal.Internallearn;
 using Microsoft.ML.Runtime;
+using Microsoft.ML.Runtime.HalLearners;
+using Microsoft.ML.Runtime.Internal.Internallearn;
+using Microsoft.ML.Runtime.Internal.Utilities;
 using Microsoft.ML.Runtime.CommandLine;
 using Microsoft.ML.Runtime.Data;
 using Microsoft.ML.Runtime.EntryPoints;
 using Microsoft.ML.Runtime.Learners;
 using Microsoft.ML.Runtime.Model;
-using Microsoft.ML.Runtime.Internal.Utilities;
 using Microsoft.ML.Runtime.Training;
 using System.Runtime.InteropServices;
 
@@ -30,7 +31,7 @@
 
 [assembly: LoadableClass(typeof(void), typeof(OlsLinearRegressionTrainer), null, typeof(SignatureEntryPointModule), OlsLinearRegressionTrainer.LoadNameValue)]
 
-namespace Microsoft.ML.Runtime.Learners
+namespace Microsoft.ML.Runtime.HalLearners
 {
     /// <include file='doc.xml' path='doc/members/member[@name="OLS"]/*' />
     public sealed class OlsLinearRegressionTrainer : TrainerBase<OlsLinearRegressionPredictor>
@@ -220,7 +221,7 @@ private OlsLinearRegressionPredictor TrainCore(IChannel ch, FloatLabelCursor.Fac
             catch (DllNotFoundException)
             {
                 // REVIEW: Is there no better way?
-                throw ch.ExceptNotSupp("The MKL library (Microsoft.ML.MklImports.dll) or one of its dependencies is missing.");
+                throw ch.ExceptNotSupp("The MKL library (libMklImports) or one of its dependencies is missing.");
             }
             // Solve for beta in (LL')beta = X'y:
             Mkl.Pptrs(Mkl.Layout.RowMajor, Mkl.UpLo.Lo, m, 1, xtx, xty, 1);
@@ -329,7 +330,7 @@ private OlsLinearRegressionPredictor TrainCore(IChannel ch, FloatLabelCursor.Fac
 
         internal static class Mkl
         {
-            private const string DllName = "Microsoft.ML.MklImports.dll";
+            private const string DllName = "MklImports";
 
             public enum Layout
             {
@@ -466,7 +467,7 @@ public static void Pptri(Layout layout, UpLo uplo, int n, Double[] ap)
             Desc = "Train an OLS regression model.",
             UserName = UserNameValue,
             ShortName = ShortName,
-            XmlInclude = new[] { @"<include file='../Microsoft.ML.StandardLearners/Standard/doc.xml' path='doc/members/member[@name=""OLS""]/*' />" })]
+            XmlInclude = new[] { @"<include file='../Microsoft.ML.HalLearners/doc.xml' path='doc/members/member[@name=""OLS""]/*' />" })]
         public static CommonOutputs.RegressionOutput TrainRegression(IHostEnvironment env, Arguments input)
         {
             Contracts.CheckValue(env, nameof(env));

src/Microsoft.ML.HalLearners/doc.xml

@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="utf-8"?>
+<doc>
+  <members>
+
+    <member name="OLS">
+      <summary>
+        Train an OLS regression model.
+      </summary>
+      <remarks>
+        <a href='https://en.wikipedia.org/wiki/Ordinary_least_squares'>Ordinary least squares (OLS)</a> is a parameterized regression method. 
+        It assumes that the conditional mean of the dependent variable follows a linear function of the dependent variables.
+        The parameters of the regressor can be estimated by minimizing the squares of the difference between observed values and the predictions.
+      </remarks>
+       <example>
+        <code language="csharp">
+          new OrdinaryLeastSquaresRegressor
+          {
+            L2Weight = 0.1,
+            PerParameterSignificance = false,
+            NormalizeFeatures  = Microsoft.ML.Models.NormalizeOption.Yes
+          }
+        </code>
+      </example>
+    </member>
+
+  </members>
+</doc>

src/Microsoft.ML.StandardLearners/Microsoft.ML.StandardLearners.csproj

@@ -11,7 +11,6 @@
     <ProjectReference Include="..\Microsoft.ML.CpuMath\Microsoft.ML.CpuMath.csproj" />
     <ProjectReference Include="..\Microsoft.ML.Data\Microsoft.ML.Data.csproj" />
     <ProjectReference Include="..\Microsoft.ML\Microsoft.ML.csproj" />
-    <PackageReference Include="MlNetMklDeps" Version="$(MlNetMklDepsPackageVersion)" />
   </ItemGroup>
 
 </Project>

src/Microsoft.ML.StandardLearners/Standard/LinearPredictor.cs

@@ -553,7 +553,7 @@ public override void SaveAsIni(TextWriter writer, RoleMappedSchema schema, ICali
 
     public abstract class RegressionPredictor : LinearPredictor
     {
-        internal RegressionPredictor(IHostEnvironment env, string name, ref VBuffer<Float> weights, Float bias)
+        protected RegressionPredictor(IHostEnvironment env, string name, ref VBuffer<Float> weights, Float bias)
             : base(env, name, ref weights, bias)
         {
         }

src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LogisticRegression.cs

@@ -28,7 +28,6 @@
 
 namespace Microsoft.ML.Runtime.Learners
 {
-    using Mkl = Microsoft.ML.Runtime.Learners.OlsLinearRegressionTrainer.Mkl;
 
     /// <include file='doc.xml' path='doc/members/member[@name="LBFGS"]/*' />
     /// <include file='doc.xml' path='docs/members/example[@name="LogisticRegressionBinaryClassifier"]/*' />
@@ -282,64 +281,7 @@ protected override void ComputeTrainingStatistics(IChannel ch, FloatLabelCursor.
                 }
             }
 
-            // Apply Cholesky Decomposition to find the inverse of the Hessian.
-            Double[] invHessian = null;
-            try
-            {
-                // First, find the Cholesky decomposition LL' of the Hessian.
-                Mkl.Pptrf(Mkl.Layout.RowMajor, Mkl.UpLo.Lo, numParams, hessian);
-                // Note that hessian is already modified at this point. It is no longer the original Hessian,
-                // but instead represents the Cholesky decomposition L.
-                // Also note that the following routine is supposed to consume the Cholesky decomposition L instead
-                // of the original information matrix.
-                Mkl.Pptri(Mkl.Layout.RowMajor, Mkl.UpLo.Lo, numParams, hessian);
-                // At this point, hessian should contain the inverse of the original Hessian matrix.
-                // Swap hessian with invHessian to avoid confusion in the following context.
-                Utils.Swap(ref hessian, ref invHessian);
-                Contracts.Assert(hessian == null);
-            }
-            catch (DllNotFoundException)
-            {
-                throw ch.ExceptNotSupp("The MKL library (Microsoft.ML.MklImports.dll) or one of its dependencies is missing.");
-            }
-
-            Float[] stdErrorValues = new Float[numParams];
-            stdErrorValues[0] = (Float)Math.Sqrt(invHessian[0]);
-
-            for (int i = 1; i < numParams; i++)
-            {
-                // Initialize with inverse Hessian.
-                stdErrorValues[i] = (Single)invHessian[i * (i + 1) / 2 + i];
-            }
-
-            if (L2Weight > 0)
-            {
-                // Iterate through all entries of inverse Hessian to make adjustment to variance.
-                // A discussion on ridge regularized LR coefficient covariance matrix can be found here:
-                // http://www.ncbi.nlm.nih.gov/pmc/articles/PMC3228544/
-                // http://www.inf.unibz.it/dis/teaching/DWDM/project2010/LogisticRegression.pdf
-                int ioffset = 1;
-                for (int iRow = 1; iRow < numParams; iRow++)
-                {
-                    for (int iCol = 0; iCol <= iRow; iCol++)
-                    {
-                        var entry = (Single)invHessian[ioffset];
-                        var adjustment = -L2Weight * entry * entry;
-                        stdErrorValues[iRow] -= adjustment;
-                        if (0 < iCol && iCol < iRow)
-                            stdErrorValues[iCol] -= adjustment;
-                        ioffset++;
-                    }
-                }
-
-                Contracts.Assert(ioffset == invHessian.Length);
-            }
-
-            for (int i = 1; i < numParams; i++)
-                stdErrorValues[i] = (Float)Math.Sqrt(stdErrorValues[i]);
-
-            VBuffer<Float> stdErrors = new VBuffer<Float>(CurrentWeights.Length, numParams, stdErrorValues, weightIndices);
-            _stats = new LinearModelStatistics(Host, NumGoodRows, numParams, deviance, nullDeviance, ref stdErrors);
+            _stats = new LinearModelStatistics(Host, NumGoodRows, numParams, deviance, nullDeviance);
         }
 
         protected override void ProcessPriorDistribution(Float label, Float weight)
@@ -382,7 +324,7 @@ protected override ParameterMixingCalibratedPredictor CreatePredictor()
             CurrentWeights.GetItemOrDefault(0, ref bias);
             CurrentWeights.CopyTo(ref weights, 1, CurrentWeights.Length - 1);
             return new ParameterMixingCalibratedPredictor(Host,
-                new LinearBinaryPredictor(Host, ref weights, bias, _stats),
+                new LinearBinaryPredictor(Host, ref weights, bias),
                 new PlattCalibrator(Host, -1, 0));
         }
 

src/Microsoft.ML.StandardLearners/Standard/doc.xml

@@ -68,27 +68,6 @@
         </code>
       </example>
     </example>
-
-    <member name="OLS">
-      <summary>
-        Train an OLS regression model.
-      </summary>
-      <remarks>
-        <a href='https://en.wikipedia.org/wiki/Ordinary_least_squares'>Ordinary least squares (OLS)</a> is a parameterized regression method. 
-        It assumes that the conditional mean of the dependent variable follows a linear function of the dependent variables.
-        The parameters of the regressor can be estimated by minimizing the squares of the difference between observed values and the predictions.
-      </remarks>
-       <example>
-        <code language="csharp">
-          new OrdinaryLeastSquaresRegressor
-          {
-            L2Weight = 0.1,
-            PerParameterSignificance = false,
-            NormalizeFeatures  = Microsoft.ML.Models.NormalizeOption.Yes
-          }
-        </code>
-      </example>
-    </member>
 
   </members>
 </doc>

src/Microsoft.ML.Transforms/WhiteningTransform.cs

@@ -597,7 +597,7 @@ private static Float DotProduct(Float[] a, int aOffset, Float[] b, int[] indices
 
         private static class Mkl
         {
-            private const string DllName = "Microsoft.ML.MklImports.dll";
+            private const string DllName = "MklImports";
 
             public enum Layout
             {

src/Microsoft.ML/CSharpApi.cs

@@ -8911,7 +8911,7 @@ public OnlineGradientDescentRegressorPipelineStep(Output output)
     namespace Trainers
     {
 
-        /// <include file='../Microsoft.ML.StandardLearners/Standard/doc.xml' path='doc/members/member[@name="OLS"]/*' />
+        /// <include file='../Microsoft.ML.HalLearners/doc.xml' path='doc/members/member[@name="OLS"]/*' />
         public sealed partial class OrdinaryLeastSquaresRegressor : Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInputWithWeight, Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInputWithLabel, Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInput, Microsoft.ML.ILearningPipelineItem
         {
 

test/BaselineOutput/Common/EntryPoints/core_ep-list.tsv

@@ -59,7 +59,7 @@ Trainers.LogisticRegressionBinaryClassifier	Logistic Regression is a method in s
 Trainers.LogisticRegressionClassifier	Logistic Regression is a method in statistics used to predict the probability of occurrence of an event and can be used as a classification algorithm. The algorithm predicts the probability of occurrence of an event by fitting data to a logistical function.	Microsoft.ML.Runtime.Learners.LogisticRegression	TrainMultiClass	Microsoft.ML.Runtime.Learners.MulticlassLogisticRegression+Arguments	Microsoft.ML.Runtime.EntryPoints.CommonOutputs+MulticlassClassificationOutput
 Trainers.NaiveBayesClassifier	Train a MultiClassNaiveBayesTrainer.	Microsoft.ML.Runtime.Learners.MultiClassNaiveBayesTrainer	TrainMultiClassNaiveBayesTrainer	Microsoft.ML.Runtime.Learners.MultiClassNaiveBayesTrainer+Arguments	Microsoft.ML.Runtime.EntryPoints.CommonOutputs+MulticlassClassificationOutput
 Trainers.OnlineGradientDescentRegressor	Train a Online gradient descent perceptron.	Microsoft.ML.Runtime.Learners.OnlineGradientDescentTrainer	TrainRegression	Microsoft.ML.Runtime.Learners.OnlineGradientDescentTrainer+Arguments	Microsoft.ML.Runtime.EntryPoints.CommonOutputs+RegressionOutput
-Trainers.OrdinaryLeastSquaresRegressor	Train an OLS regression model.	Microsoft.ML.Runtime.Learners.OlsLinearRegressionTrainer	TrainRegression	Microsoft.ML.Runtime.Learners.OlsLinearRegressionTrainer+Arguments	Microsoft.ML.Runtime.EntryPoints.CommonOutputs+RegressionOutput
+Trainers.OrdinaryLeastSquaresRegressor	Train an OLS regression model.	Microsoft.ML.Runtime.HalLearners.OlsLinearRegressionTrainer	TrainRegression	Microsoft.ML.Runtime.HalLearners.OlsLinearRegressionTrainer+Arguments	Microsoft.ML.Runtime.EntryPoints.CommonOutputs+RegressionOutput
 Trainers.PcaAnomalyDetector	Train an PCA Anomaly model.	Microsoft.ML.Runtime.PCA.RandomizedPcaTrainer	TrainPcaAnomaly	Microsoft.ML.Runtime.PCA.RandomizedPcaTrainer+Arguments	Microsoft.ML.Runtime.EntryPoints.CommonOutputs+AnomalyDetectionOutput
 Trainers.PoissonRegressor	Train an Poisson regression model.	Microsoft.ML.Runtime.Learners.PoissonRegression	TrainRegression	Microsoft.ML.Runtime.Learners.PoissonRegression+Arguments	Microsoft.ML.Runtime.EntryPoints.CommonOutputs+RegressionOutput
 Trainers.StochasticDualCoordinateAscentBinaryClassifier	Train an SDCA binary model.	Microsoft.ML.Runtime.Learners.Sdca	TrainBinary	Microsoft.ML.Runtime.Learners.LinearClassificationTrainer+Arguments	Microsoft.ML.Runtime.EntryPoints.CommonOutputs+BinaryClassificationOutput

test/Microsoft.ML.Core.Tests/Microsoft.ML.Core.Tests.csproj

@@ -5,6 +5,7 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <ProjectReference Include="..\..\src\Microsoft.ML.HalLearners\Microsoft.ML.HalLearners.csproj" />
     <ProjectReference Include="..\..\src\Microsoft.ML.Api\Microsoft.ML.Api.csproj" />
     <ProjectReference Include="..\..\src\Microsoft.ML.Core\Microsoft.ML.Core.csproj" />
     <ProjectReference Include="..\..\src\Microsoft.ML.Data\Microsoft.ML.Data.csproj" />

test/Microsoft.ML.Predictor.Tests/Microsoft.ML.Predictor.Tests.csproj

@@ -9,6 +9,7 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <ProjectReference Include="..\..\src\Microsoft.ML.HalLearners\Microsoft.ML.HalLearners.csproj" />
     <ProjectReference Include="..\..\src\Microsoft.ML.Api\Microsoft.ML.Api.csproj" />
     <ProjectReference Include="..\..\src\Microsoft.ML.Core\Microsoft.ML.Core.csproj" />
     <ProjectReference Include="..\..\src\Microsoft.ML.Data\Microsoft.ML.Data.csproj" />

test/Microsoft.ML.Predictor.Tests/TestPredictors.cs

@@ -888,7 +888,7 @@ public void RegressorOlsTest()
         /// <summary>
         /// A test for ordinary least squares regression.
         /// </summary>
-        [Fact(Skip = "Need CoreTLC specific baseline update")]
+        [Fact]
         [TestCategory("Regressor")]
         public void RegressorOlsTestOne()
         {