diff --git a/Microsoft.ML.sln b/Microsoft.ML.sln index 28d1528d83..3d285d06d8 100644 --- a/Microsoft.ML.sln +++ b/Microsoft.ML.sln @@ -99,6 +99,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.CodeAnalyzer.T EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.ImageAnalytics", "src\Microsoft.ML.ImageAnalytics\Microsoft.ML.ImageAnalytics.csproj", "{00E38F77-1E61-4CDF-8F97-1417D4E85053}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.HalLearners", "src\Microsoft.ML.HalLearners\Microsoft.ML.HalLearners.csproj", "{A7222F41-1CF0-47D9-B80C-B4D77B027A61}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -339,6 +341,14 @@ Global {00E38F77-1E61-4CDF-8F97-1417D4E85053}.Release|Any CPU.Build.0 = Release|Any CPU {00E38F77-1E61-4CDF-8F97-1417D4E85053}.Release-Intrinsics|Any CPU.ActiveCfg = Release|Any CPU {00E38F77-1E61-4CDF-8F97-1417D4E85053}.Release-Intrinsics|Any CPU.Build.0 = Release|Any CPU + {A7222F41-1CF0-47D9-B80C-B4D77B027A61}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A7222F41-1CF0-47D9-B80C-B4D77B027A61}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A7222F41-1CF0-47D9-B80C-B4D77B027A61}.Debug-Intrinsics|Any CPU.ActiveCfg = Debug|Any CPU + {A7222F41-1CF0-47D9-B80C-B4D77B027A61}.Debug-Intrinsics|Any CPU.Build.0 = Debug|Any CPU + {A7222F41-1CF0-47D9-B80C-B4D77B027A61}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A7222F41-1CF0-47D9-B80C-B4D77B027A61}.Release|Any CPU.Build.0 = Release|Any CPU + {A7222F41-1CF0-47D9-B80C-B4D77B027A61}.Release-Intrinsics|Any CPU.ActiveCfg = Release|Any CPU + {A7222F41-1CF0-47D9-B80C-B4D77B027A61}.Release-Intrinsics|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -378,6 +388,7 @@ Global {B4E55B2D-2A92-46E7-B72F-E76D6FD83440} = {7F13E156-3EBA-4021-84A5-CD56BA72F99E} {3E4ABF07-7970-4BE6-B45B-A13D3C397545} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4} {00E38F77-1E61-4CDF-8F97-1417D4E85053} = {09EADF06-BE25-4228-AB53-95AE3E15B530} + {A7222F41-1CF0-47D9-B80C-B4D77B027A61} = {09EADF06-BE25-4228-AB53-95AE3E15B530} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {41165AF1-35BB-4832-A189-73060F82B01D} diff --git a/build/Dependencies.props b/build/Dependencies.props index 9667ddec04..79ae31c598 100644 --- a/build/Dependencies.props +++ b/build/Dependencies.props @@ -8,7 +8,7 @@ 4.3.0 1.0.0-beta-62824-02 2.1.2.2 - 0.0.0.1 + 0.0.0.5 4.5.0 diff --git a/pkg/Microsoft.ML.HalLearners/Microsoft.ML.HalLearners.nupkgproj b/pkg/Microsoft.ML.HalLearners/Microsoft.ML.HalLearners.nupkgproj new file mode 100644 index 0000000000..a531c8e403 --- /dev/null +++ b/pkg/Microsoft.ML.HalLearners/Microsoft.ML.HalLearners.nupkgproj @@ -0,0 +1,13 @@ + + + + netstandard2.0 + ML.NET additional learners making use of hardware acceleration. They depend on the MlNetMklDeps NuGet package. + + + + + + + + diff --git a/pkg/Microsoft.ML.HalLearners/Microsoft.ML.HalLearners.symbols.nupkgproj b/pkg/Microsoft.ML.HalLearners/Microsoft.ML.HalLearners.symbols.nupkgproj new file mode 100644 index 0000000000..248ae82414 --- /dev/null +++ b/pkg/Microsoft.ML.HalLearners/Microsoft.ML.HalLearners.symbols.nupkgproj @@ -0,0 +1,5 @@ + + + + + diff --git a/src/Microsoft.ML.Core/ComponentModel/ComponentCatalog.cs b/src/Microsoft.ML.Core/ComponentModel/ComponentCatalog.cs index 3b56e8bb36..a92c20a4e6 100644 --- a/src/Microsoft.ML.Core/ComponentModel/ComponentCatalog.cs +++ b/src/Microsoft.ML.Core/ComponentModel/ComponentCatalog.cs @@ -343,7 +343,7 @@ private static bool ShouldSkipPath(string path) case "libvw.dll": case "matrixinterf.dll": case "Microsoft.ML.neuralnetworks.gpucuda.dll": - case "Microsoft.ML.mklimports.dll": + case "MklImports.dll": case "microsoft.research.controls.decisiontrees.dll": case "Microsoft.ML.neuralnetworks.sse.dll": case "neuraltreeevaluator.dll": diff --git a/src/Microsoft.ML.HalLearners/Microsoft.ML.HalLearners.csproj b/src/Microsoft.ML.HalLearners/Microsoft.ML.HalLearners.csproj new file mode 100644 index 0000000000..a5f3c4b748 --- /dev/null +++ b/src/Microsoft.ML.HalLearners/Microsoft.ML.HalLearners.csproj @@ -0,0 +1,16 @@ + + + + netstandard2.0 + Microsoft.ML.HalLearners + + + + + + + + + + + diff --git a/src/Microsoft.ML.StandardLearners/Standard/OlsLinearRegression.cs b/src/Microsoft.ML.HalLearners/OlsLinearRegression.cs similarity index 99% rename from src/Microsoft.ML.StandardLearners/Standard/OlsLinearRegression.cs rename to src/Microsoft.ML.HalLearners/OlsLinearRegression.cs index 5fe8b62761..dbf2999657 100644 --- a/src/Microsoft.ML.StandardLearners/Standard/OlsLinearRegression.cs +++ b/src/Microsoft.ML.HalLearners/OlsLinearRegression.cs @@ -7,14 +7,15 @@ using System; using System.Collections.Generic; using System.IO; -using Microsoft.ML.Runtime.Internal.Internallearn; using Microsoft.ML.Runtime; +using Microsoft.ML.Runtime.HalLearners; +using Microsoft.ML.Runtime.Internal.Internallearn; +using Microsoft.ML.Runtime.Internal.Utilities; using Microsoft.ML.Runtime.CommandLine; using Microsoft.ML.Runtime.Data; using Microsoft.ML.Runtime.EntryPoints; using Microsoft.ML.Runtime.Learners; using Microsoft.ML.Runtime.Model; -using Microsoft.ML.Runtime.Internal.Utilities; using Microsoft.ML.Runtime.Training; using System.Runtime.InteropServices; @@ -30,7 +31,7 @@ [assembly: LoadableClass(typeof(void), typeof(OlsLinearRegressionTrainer), null, typeof(SignatureEntryPointModule), OlsLinearRegressionTrainer.LoadNameValue)] -namespace Microsoft.ML.Runtime.Learners +namespace Microsoft.ML.Runtime.HalLearners { /// public sealed class OlsLinearRegressionTrainer : TrainerBase @@ -220,7 +221,7 @@ private OlsLinearRegressionPredictor TrainCore(IChannel ch, FloatLabelCursor.Fac catch (DllNotFoundException) { // REVIEW: Is there no better way? - throw ch.ExceptNotSupp("The MKL library (Microsoft.ML.MklImports.dll) or one of its dependencies is missing."); + throw ch.ExceptNotSupp("The MKL library (libMklImports) or one of its dependencies is missing."); } // Solve for beta in (LL')beta = X'y: Mkl.Pptrs(Mkl.Layout.RowMajor, Mkl.UpLo.Lo, m, 1, xtx, xty, 1); @@ -329,7 +330,7 @@ private OlsLinearRegressionPredictor TrainCore(IChannel ch, FloatLabelCursor.Fac internal static class Mkl { - private const string DllName = "Microsoft.ML.MklImports.dll"; + private const string DllName = "MklImports"; public enum Layout { @@ -466,7 +467,7 @@ public static void Pptri(Layout layout, UpLo uplo, int n, Double[] ap) Desc = "Train an OLS regression model.", UserName = UserNameValue, ShortName = ShortName, - XmlInclude = new[] { @"" })] + XmlInclude = new[] { @"" })] public static CommonOutputs.RegressionOutput TrainRegression(IHostEnvironment env, Arguments input) { Contracts.CheckValue(env, nameof(env)); diff --git a/src/Microsoft.ML.HalLearners/doc.xml b/src/Microsoft.ML.HalLearners/doc.xml new file mode 100644 index 0000000000..d7ec04bb89 --- /dev/null +++ b/src/Microsoft.ML.HalLearners/doc.xml @@ -0,0 +1,27 @@ + + + + + + + Train an OLS regression model. + + + Ordinary least squares (OLS) is a parameterized regression method. + It assumes that the conditional mean of the dependent variable follows a linear function of the dependent variables. + The parameters of the regressor can be estimated by minimizing the squares of the difference between observed values and the predictions. + + + + new OrdinaryLeastSquaresRegressor + { + L2Weight = 0.1, + PerParameterSignificance = false, + NormalizeFeatures = Microsoft.ML.Models.NormalizeOption.Yes + } + + + + + + \ No newline at end of file diff --git a/src/Microsoft.ML.StandardLearners/Microsoft.ML.StandardLearners.csproj b/src/Microsoft.ML.StandardLearners/Microsoft.ML.StandardLearners.csproj index 72b0f912d5..6bada43299 100644 --- a/src/Microsoft.ML.StandardLearners/Microsoft.ML.StandardLearners.csproj +++ b/src/Microsoft.ML.StandardLearners/Microsoft.ML.StandardLearners.csproj @@ -11,7 +11,6 @@ - diff --git a/src/Microsoft.ML.StandardLearners/Standard/LinearPredictor.cs b/src/Microsoft.ML.StandardLearners/Standard/LinearPredictor.cs index 8f81324768..82069d413d 100644 --- a/src/Microsoft.ML.StandardLearners/Standard/LinearPredictor.cs +++ b/src/Microsoft.ML.StandardLearners/Standard/LinearPredictor.cs @@ -553,7 +553,7 @@ public override void SaveAsIni(TextWriter writer, RoleMappedSchema schema, ICali public abstract class RegressionPredictor : LinearPredictor { - internal RegressionPredictor(IHostEnvironment env, string name, ref VBuffer weights, Float bias) + protected RegressionPredictor(IHostEnvironment env, string name, ref VBuffer weights, Float bias) : base(env, name, ref weights, bias) { } diff --git a/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LogisticRegression.cs b/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LogisticRegression.cs index 09e2bbbcc4..47b08c586a 100644 --- a/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LogisticRegression.cs +++ b/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LogisticRegression.cs @@ -28,7 +28,6 @@ namespace Microsoft.ML.Runtime.Learners { - using Mkl = Microsoft.ML.Runtime.Learners.OlsLinearRegressionTrainer.Mkl; /// /// @@ -282,64 +281,7 @@ protected override void ComputeTrainingStatistics(IChannel ch, FloatLabelCursor. } } - // Apply Cholesky Decomposition to find the inverse of the Hessian. - Double[] invHessian = null; - try - { - // First, find the Cholesky decomposition LL' of the Hessian. - Mkl.Pptrf(Mkl.Layout.RowMajor, Mkl.UpLo.Lo, numParams, hessian); - // Note that hessian is already modified at this point. It is no longer the original Hessian, - // but instead represents the Cholesky decomposition L. - // Also note that the following routine is supposed to consume the Cholesky decomposition L instead - // of the original information matrix. - Mkl.Pptri(Mkl.Layout.RowMajor, Mkl.UpLo.Lo, numParams, hessian); - // At this point, hessian should contain the inverse of the original Hessian matrix. - // Swap hessian with invHessian to avoid confusion in the following context. - Utils.Swap(ref hessian, ref invHessian); - Contracts.Assert(hessian == null); - } - catch (DllNotFoundException) - { - throw ch.ExceptNotSupp("The MKL library (Microsoft.ML.MklImports.dll) or one of its dependencies is missing."); - } - - Float[] stdErrorValues = new Float[numParams]; - stdErrorValues[0] = (Float)Math.Sqrt(invHessian[0]); - - for (int i = 1; i < numParams; i++) - { - // Initialize with inverse Hessian. - stdErrorValues[i] = (Single)invHessian[i * (i + 1) / 2 + i]; - } - - if (L2Weight > 0) - { - // Iterate through all entries of inverse Hessian to make adjustment to variance. - // A discussion on ridge regularized LR coefficient covariance matrix can be found here: - // http://www.ncbi.nlm.nih.gov/pmc/articles/PMC3228544/ - // http://www.inf.unibz.it/dis/teaching/DWDM/project2010/LogisticRegression.pdf - int ioffset = 1; - for (int iRow = 1; iRow < numParams; iRow++) - { - for (int iCol = 0; iCol <= iRow; iCol++) - { - var entry = (Single)invHessian[ioffset]; - var adjustment = -L2Weight * entry * entry; - stdErrorValues[iRow] -= adjustment; - if (0 < iCol && iCol < iRow) - stdErrorValues[iCol] -= adjustment; - ioffset++; - } - } - - Contracts.Assert(ioffset == invHessian.Length); - } - - for (int i = 1; i < numParams; i++) - stdErrorValues[i] = (Float)Math.Sqrt(stdErrorValues[i]); - - VBuffer stdErrors = new VBuffer(CurrentWeights.Length, numParams, stdErrorValues, weightIndices); - _stats = new LinearModelStatistics(Host, NumGoodRows, numParams, deviance, nullDeviance, ref stdErrors); + _stats = new LinearModelStatistics(Host, NumGoodRows, numParams, deviance, nullDeviance); } protected override void ProcessPriorDistribution(Float label, Float weight) @@ -382,7 +324,7 @@ protected override ParameterMixingCalibratedPredictor CreatePredictor() CurrentWeights.GetItemOrDefault(0, ref bias); CurrentWeights.CopyTo(ref weights, 1, CurrentWeights.Length - 1); return new ParameterMixingCalibratedPredictor(Host, - new LinearBinaryPredictor(Host, ref weights, bias, _stats), + new LinearBinaryPredictor(Host, ref weights, bias), new PlattCalibrator(Host, -1, 0)); } diff --git a/src/Microsoft.ML.StandardLearners/Standard/doc.xml b/src/Microsoft.ML.StandardLearners/Standard/doc.xml index 8d06ef285f..eb87605232 100644 --- a/src/Microsoft.ML.StandardLearners/Standard/doc.xml +++ b/src/Microsoft.ML.StandardLearners/Standard/doc.xml @@ -68,27 +68,6 @@ - - - - Train an OLS regression model. - - - Ordinary least squares (OLS) is a parameterized regression method. - It assumes that the conditional mean of the dependent variable follows a linear function of the dependent variables. - The parameters of the regressor can be estimated by minimizing the squares of the difference between observed values and the predictions. - - - - new OrdinaryLeastSquaresRegressor - { - L2Weight = 0.1, - PerParameterSignificance = false, - NormalizeFeatures = Microsoft.ML.Models.NormalizeOption.Yes - } - - - \ No newline at end of file diff --git a/src/Microsoft.ML.Transforms/WhiteningTransform.cs b/src/Microsoft.ML.Transforms/WhiteningTransform.cs index 6854157f31..a46a764352 100644 --- a/src/Microsoft.ML.Transforms/WhiteningTransform.cs +++ b/src/Microsoft.ML.Transforms/WhiteningTransform.cs @@ -597,7 +597,7 @@ private static Float DotProduct(Float[] a, int aOffset, Float[] b, int[] indices private static class Mkl { - private const string DllName = "Microsoft.ML.MklImports.dll"; + private const string DllName = "MklImports"; public enum Layout { diff --git a/src/Microsoft.ML/CSharpApi.cs b/src/Microsoft.ML/CSharpApi.cs index b4a4c4bb94..bf753fe486 100644 --- a/src/Microsoft.ML/CSharpApi.cs +++ b/src/Microsoft.ML/CSharpApi.cs @@ -8911,7 +8911,7 @@ public OnlineGradientDescentRegressorPipelineStep(Output output) namespace Trainers { - /// + /// public sealed partial class OrdinaryLeastSquaresRegressor : Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInputWithWeight, Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInputWithLabel, Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInput, Microsoft.ML.ILearningPipelineItem { diff --git a/test/BaselineOutput/Common/EntryPoints/core_ep-list.tsv b/test/BaselineOutput/Common/EntryPoints/core_ep-list.tsv index ba4eceb25f..59d168cc3c 100644 --- a/test/BaselineOutput/Common/EntryPoints/core_ep-list.tsv +++ b/test/BaselineOutput/Common/EntryPoints/core_ep-list.tsv @@ -59,7 +59,7 @@ Trainers.LogisticRegressionBinaryClassifier Logistic Regression is a method in s Trainers.LogisticRegressionClassifier Logistic Regression is a method in statistics used to predict the probability of occurrence of an event and can be used as a classification algorithm. The algorithm predicts the probability of occurrence of an event by fitting data to a logistical function. Microsoft.ML.Runtime.Learners.LogisticRegression TrainMultiClass Microsoft.ML.Runtime.Learners.MulticlassLogisticRegression+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+MulticlassClassificationOutput Trainers.NaiveBayesClassifier Train a MultiClassNaiveBayesTrainer. Microsoft.ML.Runtime.Learners.MultiClassNaiveBayesTrainer TrainMultiClassNaiveBayesTrainer Microsoft.ML.Runtime.Learners.MultiClassNaiveBayesTrainer+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+MulticlassClassificationOutput Trainers.OnlineGradientDescentRegressor Train a Online gradient descent perceptron. Microsoft.ML.Runtime.Learners.OnlineGradientDescentTrainer TrainRegression Microsoft.ML.Runtime.Learners.OnlineGradientDescentTrainer+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+RegressionOutput -Trainers.OrdinaryLeastSquaresRegressor Train an OLS regression model. Microsoft.ML.Runtime.Learners.OlsLinearRegressionTrainer TrainRegression Microsoft.ML.Runtime.Learners.OlsLinearRegressionTrainer+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+RegressionOutput +Trainers.OrdinaryLeastSquaresRegressor Train an OLS regression model. Microsoft.ML.Runtime.HalLearners.OlsLinearRegressionTrainer TrainRegression Microsoft.ML.Runtime.HalLearners.OlsLinearRegressionTrainer+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+RegressionOutput Trainers.PcaAnomalyDetector Train an PCA Anomaly model. Microsoft.ML.Runtime.PCA.RandomizedPcaTrainer TrainPcaAnomaly Microsoft.ML.Runtime.PCA.RandomizedPcaTrainer+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+AnomalyDetectionOutput Trainers.PoissonRegressor Train an Poisson regression model. Microsoft.ML.Runtime.Learners.PoissonRegression TrainRegression Microsoft.ML.Runtime.Learners.PoissonRegression+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+RegressionOutput Trainers.StochasticDualCoordinateAscentBinaryClassifier Train an SDCA binary model. Microsoft.ML.Runtime.Learners.Sdca TrainBinary Microsoft.ML.Runtime.Learners.LinearClassificationTrainer+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+BinaryClassificationOutput diff --git a/test/Microsoft.ML.Core.Tests/Microsoft.ML.Core.Tests.csproj b/test/Microsoft.ML.Core.Tests/Microsoft.ML.Core.Tests.csproj index a9b1b991ae..6b99866749 100644 --- a/test/Microsoft.ML.Core.Tests/Microsoft.ML.Core.Tests.csproj +++ b/test/Microsoft.ML.Core.Tests/Microsoft.ML.Core.Tests.csproj @@ -5,6 +5,7 @@ + diff --git a/test/Microsoft.ML.Predictor.Tests/Microsoft.ML.Predictor.Tests.csproj b/test/Microsoft.ML.Predictor.Tests/Microsoft.ML.Predictor.Tests.csproj index 7f0a8d05f9..63234b0900 100644 --- a/test/Microsoft.ML.Predictor.Tests/Microsoft.ML.Predictor.Tests.csproj +++ b/test/Microsoft.ML.Predictor.Tests/Microsoft.ML.Predictor.Tests.csproj @@ -9,6 +9,7 @@ + diff --git a/test/Microsoft.ML.Predictor.Tests/TestPredictors.cs b/test/Microsoft.ML.Predictor.Tests/TestPredictors.cs index b8c8d9b497..60c9b21495 100644 --- a/test/Microsoft.ML.Predictor.Tests/TestPredictors.cs +++ b/test/Microsoft.ML.Predictor.Tests/TestPredictors.cs @@ -888,7 +888,7 @@ public void RegressorOlsTest() /// /// A test for ordinary least squares regression. /// - [Fact(Skip = "Need CoreTLC specific baseline update")] + [Fact] [TestCategory("Regressor")] public void RegressorOlsTestOne() {