From 9ea2076862712cb9856ecaade204d658cf2061f4 Mon Sep 17 00:00:00 2001 From: Abhishek Goswami Date: Wed, 27 Feb 2019 17:53:51 +0000 Subject: [PATCH 1/8] LogisticRegression --- .../ComputeLRTrainingStdThroughHal.cs | 4 +-- .../LogisticRegression/LbfgsPredictorBase.cs | 36 +++++++++---------- .../LogisticRegression/LogisticRegression.cs | 32 ++++++++--------- .../MulticlassLogisticRegression.cs | 2 +- .../PoissonRegression/PoissonRegression.cs | 2 +- .../StandardLearnersCatalog.cs | 6 ++-- src/Microsoft.ML.StaticPipe/LbfgsStatic.cs | 8 ++--- .../KMeansAndLogisticRegressionBench.cs | 2 +- .../UnitTests/TestEntryPoints.cs | 14 ++++---- .../Evaluation.cs | 4 +-- .../TestPredictors.cs | 8 ++--- .../Training.cs | 6 ++-- .../FeatureContributionTests.cs | 2 +- .../PermutationFeatureImportanceTests.cs | 6 ++-- .../TrainerEstimators/LbfgsTests.cs | 6 ++-- 15 files changed, 69 insertions(+), 69 deletions(-) diff --git a/src/Microsoft.ML.HalLearners/ComputeLRTrainingStdThroughHal.cs b/src/Microsoft.ML.HalLearners/ComputeLRTrainingStdThroughHal.cs index ce3bad8b99..5902503e70 100644 --- a/src/Microsoft.ML.HalLearners/ComputeLRTrainingStdThroughHal.cs +++ b/src/Microsoft.ML.HalLearners/ComputeLRTrainingStdThroughHal.cs @@ -11,7 +11,7 @@ namespace Microsoft.ML.Trainers { using Mkl = OlsLinearRegressionTrainer.Mkl; - public sealed class ComputeLRTrainingStdThroughHal : ComputeLRTrainingStd + public sealed class ComputeLRTrainingStdThroughHal : ComputeLogisticRegressionStandardDeviation { /// /// Computes the standart deviation matrix of each of the non-zero training weights, needed to calculate further the standart deviation, @@ -23,7 +23,7 @@ public sealed class ComputeLRTrainingStdThroughHal : ComputeLRTrainingStd /// /// The used for messaging. /// The L2Weight used for training. (Supply the same one that got used during training.) - public override VBuffer ComputeStd(double[] hessian, int[] weightIndices, int numSelectedParams, int currentWeightsCount, IChannel ch, float l2Weight) + public override VBuffer ComputeStandardDeviation(double[] hessian, int[] weightIndices, int numSelectedParams, int currentWeightsCount, IChannel ch, float l2Weight) { Contracts.AssertValue(ch); Contracts.AssertValue(hessian, nameof(hessian)); diff --git a/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LbfgsPredictorBase.cs b/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LbfgsPredictorBase.cs index 42b0b78c5b..948c5c33b0 100644 --- a/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LbfgsPredictorBase.cs +++ b/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LbfgsPredictorBase.cs @@ -36,7 +36,7 @@ public abstract class OptionsBase : LearnerInputBaseWithWeight ShortName = "ot", SortOrder = 50)] [TGUI(Label = "Optimization Tolerance", Description = "Threshold for optimizer convergence", SuggestedSweeps = "1e-4,1e-7")] [TlcModule.SweepableDiscreteParamAttribute(new object[] { 1e-4f, 1e-7f })] - public float OptTol = Defaults.OptTol; + public float OptmizationTolerance = Defaults.OptimizationTolerance; [Argument(ArgumentType.AtMostOnce, HelpText = "Memory size for L-BFGS. Low=faster, less accurate", ShortName = "m", SortOrder = 50)] @@ -47,7 +47,7 @@ public abstract class OptionsBase : LearnerInputBaseWithWeight [Argument(ArgumentType.AtMostOnce, HelpText = "Maximum iterations.", ShortName = "maxiter")] [TGUI(Label = "Max Number of Iterations")] [TlcModule.SweepableLongParamAttribute("MaxIterations", 1, int.MaxValue)] - public int MaxIterations = Defaults.MaxIterations; + public int MaximumIterations = Defaults.MaximumIterations; [Argument(ArgumentType.AtMostOnce, HelpText = "Run SGD to initialize LR weights, converging to this tolerance", ShortName = "sgd")] @@ -71,7 +71,7 @@ public abstract class OptionsBase : LearnerInputBaseWithWeight [Argument(ArgumentType.LastOccurenceWins, HelpText = "Init weights diameter", ShortName = "initwts", SortOrder = 140)] [TGUI(Label = "Initial Weights Scale", SuggestedSweeps = "0,0.1,0.5,1")] [TlcModule.SweepableFloatParamAttribute("InitWtsDiameter", 0.0f, 1.0f, numSteps: 5)] - public float InitWtsDiameter = 0; + public float InitialWeightsDiameter = 0; // Deprecated [Argument(ArgumentType.AtMostOnce, HelpText = "Whether or not to use threads. Default is true", @@ -82,7 +82,7 @@ public abstract class OptionsBase : LearnerInputBaseWithWeight /// Number of threads. Null means use the number of processors. /// [Argument(ArgumentType.AtMostOnce, HelpText = "Number of threads", ShortName = "nt")] - public int? NumThreads; + public int? NumberOfThreads; [Argument(ArgumentType.AtMostOnce, HelpText = "Force densification of the internal optimization vectors", ShortName = "do")] [TlcModule.SweepableDiscreteParamAttribute("DenseOptimizer", new object[] { false, true })] @@ -96,9 +96,9 @@ internal static class Defaults { public const float L2Weight = 1; public const float L1Weight = 1; - public const float OptTol = 1e-7f; + public const float OptimizationTolerance = 1e-7f; public const int MemorySize = 20; - public const int MaxIterations = int.MaxValue; + public const int MaximumIterations = int.MaxValue; public const bool EnforceNonNegativity = false; } } @@ -167,7 +167,7 @@ internal LbfgsTrainerBase(IHostEnvironment env, WeightColumn = weightColumn, L1Weight = l1Weight, L2Weight = l2Weight, - OptTol = optimizationTolerance, + OptmizationTolerance = optimizationTolerance, MemorySize = memorySize, EnforceNonNegativity = enforceNoNegativity }, @@ -191,31 +191,31 @@ internal LbfgsTrainerBase(IHostEnvironment env, options.FeatureColumn = FeatureColumn.Name; options.LabelColumn = LabelColumn.Name; options.WeightColumn = WeightColumn.Name; - Host.CheckUserArg(!LbfgsTrainerOptions.UseThreads || LbfgsTrainerOptions.NumThreads > 0 || LbfgsTrainerOptions.NumThreads == null, - nameof(LbfgsTrainerOptions.NumThreads), "numThreads must be positive (or empty for default)"); + Host.CheckUserArg(!LbfgsTrainerOptions.UseThreads || LbfgsTrainerOptions.NumberOfThreads > 0 || LbfgsTrainerOptions.NumberOfThreads == null, + nameof(LbfgsTrainerOptions.NumberOfThreads), "numThreads must be positive (or empty for default)"); Host.CheckUserArg(LbfgsTrainerOptions.L2Weight >= 0, nameof(LbfgsTrainerOptions.L2Weight), "Must be non-negative"); Host.CheckUserArg(LbfgsTrainerOptions.L1Weight >= 0, nameof(LbfgsTrainerOptions.L1Weight), "Must be non-negative"); - Host.CheckUserArg(LbfgsTrainerOptions.OptTol > 0, nameof(LbfgsTrainerOptions.OptTol), "Must be positive"); + Host.CheckUserArg(LbfgsTrainerOptions.OptmizationTolerance > 0, nameof(LbfgsTrainerOptions.OptmizationTolerance), "Must be positive"); Host.CheckUserArg(LbfgsTrainerOptions.MemorySize > 0, nameof(LbfgsTrainerOptions.MemorySize), "Must be positive"); - Host.CheckUserArg(LbfgsTrainerOptions.MaxIterations > 0, nameof(LbfgsTrainerOptions.MaxIterations), "Must be positive"); + Host.CheckUserArg(LbfgsTrainerOptions.MaximumIterations > 0, nameof(LbfgsTrainerOptions.MaximumIterations), "Must be positive"); Host.CheckUserArg(LbfgsTrainerOptions.SgdInitializationTolerance >= 0, nameof(LbfgsTrainerOptions.SgdInitializationTolerance), "Must be non-negative"); - Host.CheckUserArg(LbfgsTrainerOptions.NumThreads == null || LbfgsTrainerOptions.NumThreads.Value >= 0, nameof(LbfgsTrainerOptions.NumThreads), "Must be non-negative"); + Host.CheckUserArg(LbfgsTrainerOptions.NumberOfThreads == null || LbfgsTrainerOptions.NumberOfThreads.Value >= 0, nameof(LbfgsTrainerOptions.NumberOfThreads), "Must be non-negative"); Host.CheckParam(!(LbfgsTrainerOptions.L2Weight < 0), nameof(LbfgsTrainerOptions.L2Weight), "Must be non-negative, if provided."); Host.CheckParam(!(LbfgsTrainerOptions.L1Weight < 0), nameof(LbfgsTrainerOptions.L1Weight), "Must be non-negative, if provided"); - Host.CheckParam(!(LbfgsTrainerOptions.OptTol <= 0), nameof(LbfgsTrainerOptions.OptTol), "Must be positive, if provided."); + Host.CheckParam(!(LbfgsTrainerOptions.OptmizationTolerance <= 0), nameof(LbfgsTrainerOptions.OptmizationTolerance), "Must be positive, if provided."); Host.CheckParam(!(LbfgsTrainerOptions.MemorySize <= 0), nameof(LbfgsTrainerOptions.MemorySize), "Must be positive, if provided."); L2Weight = LbfgsTrainerOptions.L2Weight; L1Weight = LbfgsTrainerOptions.L1Weight; - OptTol = LbfgsTrainerOptions.OptTol; + OptTol = LbfgsTrainerOptions.OptmizationTolerance; MemorySize =LbfgsTrainerOptions.MemorySize; - MaxIterations = LbfgsTrainerOptions.MaxIterations; + MaxIterations = LbfgsTrainerOptions.MaximumIterations; SgdInitializationTolerance = LbfgsTrainerOptions.SgdInitializationTolerance; Quiet = LbfgsTrainerOptions.Quiet; - InitWtsDiameter = LbfgsTrainerOptions.InitWtsDiameter; + InitWtsDiameter = LbfgsTrainerOptions.InitialWeightsDiameter; UseThreads = LbfgsTrainerOptions.UseThreads; - NumThreads = LbfgsTrainerOptions.NumThreads; + NumThreads = LbfgsTrainerOptions.NumberOfThreads; DenseOptimizer = LbfgsTrainerOptions.DenseOptimizer; EnforceNonNegativity = LbfgsTrainerOptions.EnforceNonNegativity; @@ -247,7 +247,7 @@ private static TOptions ArgsInit(string featureColumn, SchemaShape.Column labelC WeightColumn = weightColumn, L1Weight = l1Weight, L2Weight = l2Weight, - OptTol = optimizationTolerance, + OptmizationTolerance = optimizationTolerance, MemorySize = memorySize, EnforceNonNegativity = enforceNoNegativity }; diff --git a/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LogisticRegression.cs b/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LogisticRegression.cs index 266135b551..7ca46da47b 100644 --- a/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LogisticRegression.cs +++ b/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LogisticRegression.cs @@ -45,19 +45,19 @@ public sealed class Options : OptionsBase /// If set to truetraining statistics will be generated at the end of training. /// If you have a large number of learned training parameters(more than 500), /// generating the training statistics might take a few seconds. - /// More than 1000 weights might take a few minutes. For those cases consider using the instance of + /// More than 1000 weights might take a few minutes. For those cases consider using the instance of /// present in the Microsoft.ML.HalLearners package. That computes the statistics using hardware acceleration. /// [Argument(ArgumentType.AtMostOnce, HelpText = "Show statistics of training examples.", ShortName = "stat", SortOrder = 50)] - public bool ShowTrainingStats = false; + public bool ShowTrainingStatistics = false; /// - /// The instance of that computes the std of the training statistics, at the end of training. + /// The instance of that computes the std of the training statistics, at the end of training. /// The calculations are not part of Microsoft.ML package, due to the size of MKL. - /// If you need these calculations, add the Microsoft.ML.HalLearners package, and initialize . - /// to the implementation in the Microsoft.ML.HalLearners package. + /// If you need these calculations, add the Microsoft.ML.HalLearners package, and initialize . + /// to the implementation in the Microsoft.ML.HalLearners package. /// - public ComputeLRTrainingStd StdComputer; + public ComputeLogisticRegressionStandardDeviation ComputeStandardDeviation; } private double _posWeight; @@ -81,7 +81,7 @@ internal LogisticRegression(IHostEnvironment env, string weights = null, float l1Weight = Options.Defaults.L1Weight, float l2Weight = Options.Defaults.L2Weight, - float optimizationTolerance = Options.Defaults.OptTol, + float optimizationTolerance = Options.Defaults.OptimizationTolerance, int memorySize = Options.Defaults.MemorySize, bool enforceNoNegativity = Options.Defaults.EnforceNonNegativity) : base(env, featureColumn, TrainerUtils.MakeBoolScalarLabel(labelColumn), weights, @@ -91,7 +91,7 @@ internal LogisticRegression(IHostEnvironment env, Host.CheckNonEmpty(labelColumn, nameof(labelColumn)); _posWeight = 0; - ShowTrainingStats = LbfgsTrainerOptions.ShowTrainingStats; + ShowTrainingStats = LbfgsTrainerOptions.ShowTrainingStatistics; } /// @@ -101,7 +101,7 @@ internal LogisticRegression(IHostEnvironment env, Options options) : base(env, options, TrainerUtils.MakeBoolScalarLabel(options.LabelColumn)) { _posWeight = 0; - ShowTrainingStats = LbfgsTrainerOptions.ShowTrainingStats; + ShowTrainingStats = LbfgsTrainerOptions.ShowTrainingStatistics; } private protected override PredictionKind PredictionKind => PredictionKind.BinaryClassification; @@ -354,11 +354,11 @@ private protected override void ComputeTrainingStatistics(IChannel ch, FloatLabe } } - if (LbfgsTrainerOptions.StdComputer == null) + if (LbfgsTrainerOptions.ComputeStandardDeviation == null) _stats = new LinearModelStatistics(Host, NumGoodRows, numParams, deviance, nullDeviance); else { - var std = LbfgsTrainerOptions.StdComputer.ComputeStd(hessian, weightIndices, numParams, CurrentWeights.Length, ch, L2Weight); + var std = LbfgsTrainerOptions.ComputeStandardDeviation.ComputeStandardDeviation(hessian, weightIndices, numParams, CurrentWeights.Length, ch, L2Weight); _stats = new LinearModelStatistics(Host, NumGoodRows, numParams, deviance, nullDeviance, std); } } @@ -429,20 +429,20 @@ internal static CommonOutputs.BinaryClassificationOutput TrainBinary(IHostEnviro /// /// Computes the standard deviation matrix of each of the non-zero training weights, needed to calculate further the standard deviation, /// p-value and z-Score. - /// If you need fast calculations, use the implementation in the Microsoft.ML.HALLearners package, + /// If you need fast calculations, use the implementation in the Microsoft.ML.HALLearners package, /// which makes use of hardware acceleration. /// Due to the existence of regularization, an approximation is used to compute the variances of the trained linear coefficients. /// - public abstract class ComputeLRTrainingStd + public abstract class ComputeLogisticRegressionStandardDeviation { /// /// Computes the standard deviation matrix of each of the non-zero training weights, needed to calculate further the standard deviation, /// p-value and z-Score. /// The calculations are not part of Microsoft.ML package, due to the size of MKL. - /// If you need these calculations, add the Microsoft.ML.HalLearners package, and initialize - /// to the implementation in the Microsoft.ML.HalLearners package. + /// If you need these calculations, add the Microsoft.ML.HalLearners package, and initialize + /// to the implementation in the Microsoft.ML.HalLearners package. /// Due to the existence of regularization, an approximation is used to compute the variances of the trained linear coefficients. /// - public abstract VBuffer ComputeStd(double[] hessian, int[] weightIndices, int parametersCount, int currentWeightsCount, IChannel ch, float l2Weight); + public abstract VBuffer ComputeStandardDeviation(double[] hessian, int[] weightIndices, int parametersCount, int currentWeightsCount, IChannel ch, float l2Weight); } } diff --git a/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/MulticlassLogisticRegression.cs b/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/MulticlassLogisticRegression.cs index 7a40960a68..3a3ae1a1b2 100644 --- a/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/MulticlassLogisticRegression.cs +++ b/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/MulticlassLogisticRegression.cs @@ -86,7 +86,7 @@ internal MulticlassLogisticRegression(IHostEnvironment env, string weights = null, float l1Weight = Options.Defaults.L1Weight, float l2Weight = Options.Defaults.L2Weight, - float optimizationTolerance = Options.Defaults.OptTol, + float optimizationTolerance = Options.Defaults.OptimizationTolerance, int memorySize = Options.Defaults.MemorySize, bool enforceNoNegativity = Options.Defaults.EnforceNonNegativity) : base(env, featureColumn, TrainerUtils.MakeU4ScalarColumn(labelColumn), weights, l1Weight, l2Weight, optimizationTolerance, memorySize, enforceNoNegativity) diff --git a/src/Microsoft.ML.StandardLearners/Standard/PoissonRegression/PoissonRegression.cs b/src/Microsoft.ML.StandardLearners/Standard/PoissonRegression/PoissonRegression.cs index 3315a0c01e..53a604194f 100644 --- a/src/Microsoft.ML.StandardLearners/Standard/PoissonRegression/PoissonRegression.cs +++ b/src/Microsoft.ML.StandardLearners/Standard/PoissonRegression/PoissonRegression.cs @@ -56,7 +56,7 @@ internal PoissonRegression(IHostEnvironment env, string weights = null, float l1Weight = Options.Defaults.L1Weight, float l2Weight = Options.Defaults.L2Weight, - float optimizationTolerance = Options.Defaults.OptTol, + float optimizationTolerance = Options.Defaults.OptimizationTolerance, int memorySize = Options.Defaults.MemorySize, bool enforceNoNegativity = Options.Defaults.EnforceNonNegativity) : base(env, featureColumn, TrainerUtils.MakeR4ScalarColumn(labelColumn), weights, diff --git a/src/Microsoft.ML.StandardLearners/StandardLearnersCatalog.cs b/src/Microsoft.ML.StandardLearners/StandardLearnersCatalog.cs index 906fe9529b..241528379f 100644 --- a/src/Microsoft.ML.StandardLearners/StandardLearnersCatalog.cs +++ b/src/Microsoft.ML.StandardLearners/StandardLearnersCatalog.cs @@ -430,7 +430,7 @@ public static LogisticRegression LogisticRegression(this BinaryClassificationCat string exampleWeightColumnName = null, float l1Weight = LROptions.Defaults.L1Weight, float l2Weight = LROptions.Defaults.L2Weight, - float optimizationTolerance = LROptions.Defaults.OptTol, + float optimizationTolerance = LROptions.Defaults.OptimizationTolerance, int memorySize = LROptions.Defaults.MemorySize, bool enforceNoNegativity = LROptions.Defaults.EnforceNonNegativity) { @@ -471,7 +471,7 @@ public static PoissonRegression PoissonRegression(this RegressionCatalog.Regress string exampleWeightColumnName = null, float l1Weight = LROptions.Defaults.L1Weight, float l2Weight = LROptions.Defaults.L2Weight, - float optimizationTolerance = LROptions.Defaults.OptTol, + float optimizationTolerance = LROptions.Defaults.OptimizationTolerance, int memorySize = LROptions.Defaults.MemorySize, bool enforceNoNegativity = LROptions.Defaults.EnforceNonNegativity) { @@ -512,7 +512,7 @@ public static MulticlassLogisticRegression LogisticRegression(this MulticlassCla string exampleWeightColumnName = null, float l1Weight = LROptions.Defaults.L1Weight, float l2Weight = LROptions.Defaults.L2Weight, - float optimizationTolerance = LROptions.Defaults.OptTol, + float optimizationTolerance = LROptions.Defaults.OptimizationTolerance, int memorySize = LROptions.Defaults.MemorySize, bool enforceNoNegativity = LROptions.Defaults.EnforceNonNegativity) { diff --git a/src/Microsoft.ML.StaticPipe/LbfgsStatic.cs b/src/Microsoft.ML.StaticPipe/LbfgsStatic.cs index 1c5eec3191..de6cd5b0ce 100644 --- a/src/Microsoft.ML.StaticPipe/LbfgsStatic.cs +++ b/src/Microsoft.ML.StaticPipe/LbfgsStatic.cs @@ -39,7 +39,7 @@ public static (Scalar score, Scalar probability, Scalar pred Scalar weights = null, float l1Weight = Options.Defaults.L1Weight, float l2Weight = Options.Defaults.L2Weight, - float optimizationTolerance = Options.Defaults.OptTol, + float optimizationTolerance = Options.Defaults.OptimizationTolerance, int memorySize = Options.Defaults.MemorySize, bool enoforceNoNegativity = Options.Defaults.EnforceNonNegativity, Action> onFit = null) @@ -135,7 +135,7 @@ public static Scalar PoissonRegression(this RegressionCatalog.RegressionT Scalar weights = null, float l1Weight = Options.Defaults.L1Weight, float l2Weight = Options.Defaults.L2Weight, - float optimizationTolerance = Options.Defaults.OptTol, + float optimizationTolerance = Options.Defaults.OptimizationTolerance, int memorySize = Options.Defaults.MemorySize, bool enoforceNoNegativity = Options.Defaults.EnforceNonNegativity, Action onFit = null) @@ -232,7 +232,7 @@ public static (Vector score, Key predictedLabel) Scalar weights = null, float l1Weight = Options.Defaults.L1Weight, float l2Weight = Options.Defaults.L2Weight, - float optimizationTolerance = Options.Defaults.OptTol, + float optimizationTolerance = Options.Defaults.OptimizationTolerance, int memorySize = Options.Defaults.MemorySize, bool enoforceNoNegativity = Options.Defaults.EnforceNonNegativity, Action onFit = null) @@ -305,7 +305,7 @@ internal static void ValidateParams(PipelineColumn label, Scalar weights = null, float l1Weight = Options.Defaults.L1Weight, float l2Weight = Options.Defaults.L2Weight, - float optimizationTolerance = Options.Defaults.OptTol, + float optimizationTolerance = Options.Defaults.OptimizationTolerance, int memorySize = Options.Defaults.MemorySize, bool enoforceNoNegativity = Options.Defaults.EnforceNonNegativity, Delegate onFit = null) diff --git a/test/Microsoft.ML.Benchmarks/KMeansAndLogisticRegressionBench.cs b/test/Microsoft.ML.Benchmarks/KMeansAndLogisticRegressionBench.cs index fd806a5d0b..f57247d65e 100644 --- a/test/Microsoft.ML.Benchmarks/KMeansAndLogisticRegressionBench.cs +++ b/test/Microsoft.ML.Benchmarks/KMeansAndLogisticRegressionBench.cs @@ -40,7 +40,7 @@ public CalibratedModelParametersBase (r.label, score: catalog.Trainers.PoissonRegression(r.label, r.features, null, - new PoissonRegression.Options { L2Weight = 2, EnforceNonNegativity = true, NumThreads = 1 }, + new PoissonRegression.Options { L2Weight = 2, EnforceNonNegativity = true, NumberOfThreads = 1 }, onFit: (p) => { pred = p; }))); var pipe = reader.Append(est); @@ -660,7 +660,7 @@ public void LogisticRegressionBinaryClassification() var est = reader.MakeNewEstimator() .Append(r => (r.label, preds: catalog.Trainers.LogisticRegressionBinaryClassifier(r.label, r.features, null, - new LogisticRegression.Options { L1Weight = 10, NumThreads = 1 }, onFit: (p) => { pred = p; }))); + new LogisticRegression.Options { L1Weight = 10, NumberOfThreads = 1 }, onFit: (p) => { pred = p; }))); var pipe = reader.Append(est); @@ -700,7 +700,7 @@ public void MulticlassLogisticRegression() r.label, r.features, null, - new MulticlassLogisticRegression.Options { NumThreads = 1 }, + new MulticlassLogisticRegression.Options { NumberOfThreads = 1 }, onFit: p => pred = p))); var pipe = reader.Append(est); diff --git a/test/Microsoft.ML.Tests/FeatureContributionTests.cs b/test/Microsoft.ML.Tests/FeatureContributionTests.cs index 4929ad8f52..80db67ee5e 100644 --- a/test/Microsoft.ML.Tests/FeatureContributionTests.cs +++ b/test/Microsoft.ML.Tests/FeatureContributionTests.cs @@ -88,7 +88,7 @@ public void TestOnlineGradientDescentRegression() public void TestPoissonRegression() { TestFeatureContribution(ML.Regression.Trainers.PoissonRegression( - new PoissonRegression.Options { NumThreads = 1 }), GetSparseDataset(numberOfInstances: 100), "PoissonRegression"); + new PoissonRegression.Options { NumberOfThreads = 1 }), GetSparseDataset(numberOfInstances: 100), "PoissonRegression"); } [Fact] diff --git a/test/Microsoft.ML.Tests/PermutationFeatureImportanceTests.cs b/test/Microsoft.ML.Tests/PermutationFeatureImportanceTests.cs index fee083d368..5ed9829090 100644 --- a/test/Microsoft.ML.Tests/PermutationFeatureImportanceTests.cs +++ b/test/Microsoft.ML.Tests/PermutationFeatureImportanceTests.cs @@ -153,7 +153,7 @@ public void TestPfiBinaryClassificationOnDenseFeatures() { var data = GetDenseDataset(TaskType.BinaryClassification); var model = ML.BinaryClassification.Trainers.LogisticRegression( - new LogisticRegression.Options { NumThreads = 1 }).Fit(data); + new LogisticRegression.Options { NumberOfThreads = 1 }).Fit(data); var pfi = ML.BinaryClassification.PermutationFeatureImportance(model, data); // Pfi Indices: @@ -191,7 +191,7 @@ public void TestPfiBinaryClassificationOnSparseFeatures() { var data = GetSparseDataset(TaskType.BinaryClassification); var model = ML.BinaryClassification.Trainers.LogisticRegression( - new LogisticRegression.Options { NumThreads = 1 }).Fit(data); + new LogisticRegression.Options { NumberOfThreads = 1 }).Fit(data); var pfi = ML.BinaryClassification.PermutationFeatureImportance(model, data); // Pfi Indices: @@ -270,7 +270,7 @@ public void TestPfiMulticlassClassificationOnSparseFeatures() { var data = GetSparseDataset(TaskType.MulticlassClassification); var model = ML.MulticlassClassification.Trainers.LogisticRegression( - new MulticlassLogisticRegression.Options { MaxIterations = 1000 }).Fit(data); + new MulticlassLogisticRegression.Options { MaximumIterations = 1000 }).Fit(data); var pfi = ML.MulticlassClassification.PermutationFeatureImportance(model, data); // Pfi Indices: diff --git a/test/Microsoft.ML.Tests/TrainerEstimators/LbfgsTests.cs b/test/Microsoft.ML.Tests/TrainerEstimators/LbfgsTests.cs index 2ba1e4dda7..b40a6fcadd 100644 --- a/test/Microsoft.ML.Tests/TrainerEstimators/LbfgsTests.cs +++ b/test/Microsoft.ML.Tests/TrainerEstimators/LbfgsTests.cs @@ -57,7 +57,7 @@ public void TestLogisticRegressionNoStats() { (IEstimator pipe, IDataView dataView) = GetBinaryClassificationPipeline(); - pipe = pipe.Append(ML.BinaryClassification.Trainers.LogisticRegression(new LogisticRegression.Options { ShowTrainingStats = true })); + pipe = pipe.Append(ML.BinaryClassification.Trainers.LogisticRegression(new LogisticRegression.Options { ShowTrainingStatistics = true })); var transformerChain = pipe.Fit(dataView) as TransformerChain>>; var linearModel = transformerChain.LastTransformer.Model.SubModel as LinearBinaryModelParameters; @@ -76,8 +76,8 @@ public void TestLogisticRegressionWithStats() pipe = pipe.Append(ML.BinaryClassification.Trainers.LogisticRegression( new LogisticRegression.Options { - ShowTrainingStats = true, - StdComputer = new ComputeLRTrainingStdThroughHal(), + ShowTrainingStatistics = true, + ComputeStandardDeviation = new ComputeLRTrainingStdThroughHal(), })); var transformer = pipe.Fit(dataView) as TransformerChain>>; From e49ca5937be4786dada63a73e47434c9c2498ba4 Mon Sep 17 00:00:00 2001 From: Abhishek Goswami Date: Wed, 27 Feb 2019 19:20:13 +0000 Subject: [PATCH 2/8] LogisticRegression - 2 --- .../LogisticRegression/LbfgsPredictorBase.cs | 78 ++++++++++++------- .../LogisticRegression/LogisticRegression.cs | 6 +- .../MulticlassLogisticRegression.cs | 6 +- .../PoissonRegression/PoissonRegression.cs | 6 +- .../StandardLearnersCatalog.cs | 18 ++--- src/Microsoft.ML.StaticPipe/LbfgsStatic.cs | 24 +++--- .../UnitTests/TestEntryPoints.cs | 12 +-- .../Training.cs | 4 +- .../PermutationFeatureImportanceTests.cs | 2 +- 9 files changed, 91 insertions(+), 65 deletions(-) diff --git a/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LbfgsPredictorBase.cs b/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LbfgsPredictorBase.cs index 948c5c33b0..38b16c0f31 100644 --- a/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LbfgsPredictorBase.cs +++ b/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LbfgsPredictorBase.cs @@ -22,33 +22,51 @@ public abstract class LbfgsTrainerBase : Trainer { public abstract class OptionsBase : LearnerInputBaseWithWeight { + /// + /// L2 regularization weight. + /// [Argument(ArgumentType.AtMostOnce, HelpText = "L2 regularization weight", ShortName = "l2", SortOrder = 50)] [TGUI(Label = "L2 Weight", Description = "Weight of L2 regularizer term", SuggestedSweeps = "0,0.1,1")] [TlcModule.SweepableFloatParamAttribute(0.0f, 1.0f, numSteps: 4)] - public float L2Weight = Defaults.L2Weight; + public float L2Regularization = Defaults.L2Regularization; + /// + /// L1 regularization weight. + /// [Argument(ArgumentType.AtMostOnce, HelpText = "L1 regularization weight", ShortName = "l1", SortOrder = 50)] [TGUI(Label = "L1 Weight", Description = "Weight of L1 regularizer term", SuggestedSweeps = "0,0.1,1")] [TlcModule.SweepableFloatParamAttribute(0.0f, 1.0f, numSteps: 4)] - public float L1Weight = Defaults.L1Weight; + public float L1Regularization = Defaults.L1Regularization; + /// + /// Tolerance parameter for optimization convergence. (Low = slower, more accurate). + /// [Argument(ArgumentType.AtMostOnce, HelpText = "Tolerance parameter for optimization convergence. Low = slower, more accurate", ShortName = "ot", SortOrder = 50)] [TGUI(Label = "Optimization Tolerance", Description = "Threshold for optimizer convergence", SuggestedSweeps = "1e-4,1e-7")] [TlcModule.SweepableDiscreteParamAttribute(new object[] { 1e-4f, 1e-7f })] public float OptmizationTolerance = Defaults.OptimizationTolerance; + /// + /// Number of previous iterations to remember for estimate of Hessian. + /// [Argument(ArgumentType.AtMostOnce, HelpText = "Memory size for L-BFGS. Low=faster, less accurate", ShortName = "m", SortOrder = 50)] [TGUI(Description = "Memory size for L-BFGS", SuggestedSweeps = "5,20,50")] [TlcModule.SweepableDiscreteParamAttribute("MemorySize", new object[] { 5, 20, 50 })] - public int MemorySize = Defaults.MemorySize; + public int NumberOfPreviousIterationsToRemember = Defaults.NumberOfPreviousIterationsToRemember; + /// + /// Number of iterations. + /// [Argument(ArgumentType.AtMostOnce, HelpText = "Maximum iterations.", ShortName = "maxiter")] [TGUI(Label = "Max Number of Iterations")] [TlcModule.SweepableLongParamAttribute("MaxIterations", 1, int.MaxValue)] - public int MaximumIterations = Defaults.MaximumIterations; + public int NumberOfIterations = Defaults.NumberOfIterations; + /// + /// Run SGD to initialize LR weights, converging to this tolerance. + /// [Argument(ArgumentType.AtMostOnce, HelpText = "Run SGD to initialize LR weights, converging to this tolerance", ShortName = "sgd")] public float SgdInitializationTolerance = 0; @@ -73,7 +91,9 @@ public abstract class OptionsBase : LearnerInputBaseWithWeight [TlcModule.SweepableFloatParamAttribute("InitWtsDiameter", 0.0f, 1.0f, numSteps: 5)] public float InitialWeightsDiameter = 0; - // Deprecated + /// + /// Whether or not to use threads. Default is true. + /// [Argument(ArgumentType.AtMostOnce, HelpText = "Whether or not to use threads. Default is true", ShortName = "t", Hide = true)] public bool UseThreads = true; @@ -84,21 +104,27 @@ public abstract class OptionsBase : LearnerInputBaseWithWeight [Argument(ArgumentType.AtMostOnce, HelpText = "Number of threads", ShortName = "nt")] public int? NumberOfThreads; + /// + /// Force densification of the internal optimization vectors. Default is false. + /// [Argument(ArgumentType.AtMostOnce, HelpText = "Force densification of the internal optimization vectors", ShortName = "do")] [TlcModule.SweepableDiscreteParamAttribute("DenseOptimizer", new object[] { false, true })] public bool DenseOptimizer = false; + /// + /// Enforce non-negative weights. Default is false. + /// [Argument(ArgumentType.AtMostOnce, HelpText = "Enforce non-negative weights", ShortName = "nn", SortOrder = 90)] public bool EnforceNonNegativity = Defaults.EnforceNonNegativity; [BestFriend] internal static class Defaults { - public const float L2Weight = 1; - public const float L1Weight = 1; + public const float L2Regularization = 1; + public const float L1Regularization = 1; public const float OptimizationTolerance = 1e-7f; - public const int MemorySize = 20; - public const int MaximumIterations = int.MaxValue; + public const int NumberOfPreviousIterationsToRemember = 20; + public const int NumberOfIterations = int.MaxValue; public const bool EnforceNonNegativity = false; } } @@ -165,10 +191,10 @@ internal LbfgsTrainerBase(IHostEnvironment env, FeatureColumn = featureColumn, LabelColumn = labelColumn.Name, WeightColumn = weightColumn, - L1Weight = l1Weight, - L2Weight = l2Weight, + L1Regularization = l1Weight, + L2Regularization = l2Weight, OptmizationTolerance = optimizationTolerance, - MemorySize = memorySize, + NumberOfPreviousIterationsToRemember = memorySize, EnforceNonNegativity = enforceNoNegativity }, labelColumn) @@ -193,24 +219,24 @@ internal LbfgsTrainerBase(IHostEnvironment env, options.WeightColumn = WeightColumn.Name; Host.CheckUserArg(!LbfgsTrainerOptions.UseThreads || LbfgsTrainerOptions.NumberOfThreads > 0 || LbfgsTrainerOptions.NumberOfThreads == null, nameof(LbfgsTrainerOptions.NumberOfThreads), "numThreads must be positive (or empty for default)"); - Host.CheckUserArg(LbfgsTrainerOptions.L2Weight >= 0, nameof(LbfgsTrainerOptions.L2Weight), "Must be non-negative"); - Host.CheckUserArg(LbfgsTrainerOptions.L1Weight >= 0, nameof(LbfgsTrainerOptions.L1Weight), "Must be non-negative"); + Host.CheckUserArg(LbfgsTrainerOptions.L2Regularization >= 0, nameof(LbfgsTrainerOptions.L2Regularization), "Must be non-negative"); + Host.CheckUserArg(LbfgsTrainerOptions.L1Regularization >= 0, nameof(LbfgsTrainerOptions.L1Regularization), "Must be non-negative"); Host.CheckUserArg(LbfgsTrainerOptions.OptmizationTolerance > 0, nameof(LbfgsTrainerOptions.OptmizationTolerance), "Must be positive"); - Host.CheckUserArg(LbfgsTrainerOptions.MemorySize > 0, nameof(LbfgsTrainerOptions.MemorySize), "Must be positive"); - Host.CheckUserArg(LbfgsTrainerOptions.MaximumIterations > 0, nameof(LbfgsTrainerOptions.MaximumIterations), "Must be positive"); + Host.CheckUserArg(LbfgsTrainerOptions.NumberOfPreviousIterationsToRemember > 0, nameof(LbfgsTrainerOptions.NumberOfPreviousIterationsToRemember), "Must be positive"); + Host.CheckUserArg(LbfgsTrainerOptions.NumberOfIterations > 0, nameof(LbfgsTrainerOptions.NumberOfIterations), "Must be positive"); Host.CheckUserArg(LbfgsTrainerOptions.SgdInitializationTolerance >= 0, nameof(LbfgsTrainerOptions.SgdInitializationTolerance), "Must be non-negative"); Host.CheckUserArg(LbfgsTrainerOptions.NumberOfThreads == null || LbfgsTrainerOptions.NumberOfThreads.Value >= 0, nameof(LbfgsTrainerOptions.NumberOfThreads), "Must be non-negative"); - Host.CheckParam(!(LbfgsTrainerOptions.L2Weight < 0), nameof(LbfgsTrainerOptions.L2Weight), "Must be non-negative, if provided."); - Host.CheckParam(!(LbfgsTrainerOptions.L1Weight < 0), nameof(LbfgsTrainerOptions.L1Weight), "Must be non-negative, if provided"); + Host.CheckParam(!(LbfgsTrainerOptions.L2Regularization < 0), nameof(LbfgsTrainerOptions.L2Regularization), "Must be non-negative, if provided."); + Host.CheckParam(!(LbfgsTrainerOptions.L1Regularization < 0), nameof(LbfgsTrainerOptions.L1Regularization), "Must be non-negative, if provided"); Host.CheckParam(!(LbfgsTrainerOptions.OptmizationTolerance <= 0), nameof(LbfgsTrainerOptions.OptmizationTolerance), "Must be positive, if provided."); - Host.CheckParam(!(LbfgsTrainerOptions.MemorySize <= 0), nameof(LbfgsTrainerOptions.MemorySize), "Must be positive, if provided."); + Host.CheckParam(!(LbfgsTrainerOptions.NumberOfPreviousIterationsToRemember <= 0), nameof(LbfgsTrainerOptions.NumberOfPreviousIterationsToRemember), "Must be positive, if provided."); - L2Weight = LbfgsTrainerOptions.L2Weight; - L1Weight = LbfgsTrainerOptions.L1Weight; + L2Weight = LbfgsTrainerOptions.L2Regularization; + L1Weight = LbfgsTrainerOptions.L1Regularization; OptTol = LbfgsTrainerOptions.OptmizationTolerance; - MemorySize =LbfgsTrainerOptions.MemorySize; - MaxIterations = LbfgsTrainerOptions.MaximumIterations; + MemorySize =LbfgsTrainerOptions.NumberOfPreviousIterationsToRemember; + MaxIterations = LbfgsTrainerOptions.NumberOfIterations; SgdInitializationTolerance = LbfgsTrainerOptions.SgdInitializationTolerance; Quiet = LbfgsTrainerOptions.Quiet; InitWtsDiameter = LbfgsTrainerOptions.InitialWeightsDiameter; @@ -245,10 +271,10 @@ private static TOptions ArgsInit(string featureColumn, SchemaShape.Column labelC FeatureColumn = featureColumn, LabelColumn = labelColumn.Name, WeightColumn = weightColumn, - L1Weight = l1Weight, - L2Weight = l2Weight, + L1Regularization = l1Weight, + L2Regularization = l2Weight, OptmizationTolerance = optimizationTolerance, - MemorySize = memorySize, + NumberOfPreviousIterationsToRemember = memorySize, EnforceNonNegativity = enforceNoNegativity }; diff --git a/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LogisticRegression.cs b/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LogisticRegression.cs index 7ca46da47b..7fd4135152 100644 --- a/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LogisticRegression.cs +++ b/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LogisticRegression.cs @@ -79,10 +79,10 @@ internal LogisticRegression(IHostEnvironment env, string labelColumn = DefaultColumnNames.Label, string featureColumn = DefaultColumnNames.Features, string weights = null, - float l1Weight = Options.Defaults.L1Weight, - float l2Weight = Options.Defaults.L2Weight, + float l1Weight = Options.Defaults.L1Regularization, + float l2Weight = Options.Defaults.L2Regularization, float optimizationTolerance = Options.Defaults.OptimizationTolerance, - int memorySize = Options.Defaults.MemorySize, + int memorySize = Options.Defaults.NumberOfPreviousIterationsToRemember, bool enforceNoNegativity = Options.Defaults.EnforceNonNegativity) : base(env, featureColumn, TrainerUtils.MakeBoolScalarLabel(labelColumn), weights, l1Weight, l2Weight, optimizationTolerance, memorySize, enforceNoNegativity) diff --git a/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/MulticlassLogisticRegression.cs b/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/MulticlassLogisticRegression.cs index 3a3ae1a1b2..8a6a43a0f0 100644 --- a/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/MulticlassLogisticRegression.cs +++ b/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/MulticlassLogisticRegression.cs @@ -84,10 +84,10 @@ internal MulticlassLogisticRegression(IHostEnvironment env, string labelColumn = DefaultColumnNames.Label, string featureColumn = DefaultColumnNames.Features, string weights = null, - float l1Weight = Options.Defaults.L1Weight, - float l2Weight = Options.Defaults.L2Weight, + float l1Weight = Options.Defaults.L1Regularization, + float l2Weight = Options.Defaults.L2Regularization, float optimizationTolerance = Options.Defaults.OptimizationTolerance, - int memorySize = Options.Defaults.MemorySize, + int memorySize = Options.Defaults.NumberOfPreviousIterationsToRemember, bool enforceNoNegativity = Options.Defaults.EnforceNonNegativity) : base(env, featureColumn, TrainerUtils.MakeU4ScalarColumn(labelColumn), weights, l1Weight, l2Weight, optimizationTolerance, memorySize, enforceNoNegativity) { diff --git a/src/Microsoft.ML.StandardLearners/Standard/PoissonRegression/PoissonRegression.cs b/src/Microsoft.ML.StandardLearners/Standard/PoissonRegression/PoissonRegression.cs index 53a604194f..ac3783cea6 100644 --- a/src/Microsoft.ML.StandardLearners/Standard/PoissonRegression/PoissonRegression.cs +++ b/src/Microsoft.ML.StandardLearners/Standard/PoissonRegression/PoissonRegression.cs @@ -54,10 +54,10 @@ internal PoissonRegression(IHostEnvironment env, string labelColumn = DefaultColumnNames.Label, string featureColumn = DefaultColumnNames.Features, string weights = null, - float l1Weight = Options.Defaults.L1Weight, - float l2Weight = Options.Defaults.L2Weight, + float l1Weight = Options.Defaults.L1Regularization, + float l2Weight = Options.Defaults.L2Regularization, float optimizationTolerance = Options.Defaults.OptimizationTolerance, - int memorySize = Options.Defaults.MemorySize, + int memorySize = Options.Defaults.NumberOfPreviousIterationsToRemember, bool enforceNoNegativity = Options.Defaults.EnforceNonNegativity) : base(env, featureColumn, TrainerUtils.MakeR4ScalarColumn(labelColumn), weights, l1Weight, l2Weight, optimizationTolerance, memorySize, enforceNoNegativity) diff --git a/src/Microsoft.ML.StandardLearners/StandardLearnersCatalog.cs b/src/Microsoft.ML.StandardLearners/StandardLearnersCatalog.cs index 241528379f..77b8d26367 100644 --- a/src/Microsoft.ML.StandardLearners/StandardLearnersCatalog.cs +++ b/src/Microsoft.ML.StandardLearners/StandardLearnersCatalog.cs @@ -428,10 +428,10 @@ public static LogisticRegression LogisticRegression(this BinaryClassificationCat string labelColumnName = DefaultColumnNames.Label, string featureColumnName = DefaultColumnNames.Features, string exampleWeightColumnName = null, - float l1Weight = LROptions.Defaults.L1Weight, - float l2Weight = LROptions.Defaults.L2Weight, + float l1Weight = LROptions.Defaults.L1Regularization, + float l2Weight = LROptions.Defaults.L2Regularization, float optimizationTolerance = LROptions.Defaults.OptimizationTolerance, - int memorySize = LROptions.Defaults.MemorySize, + int memorySize = LROptions.Defaults.NumberOfPreviousIterationsToRemember, bool enforceNoNegativity = LROptions.Defaults.EnforceNonNegativity) { Contracts.CheckValue(catalog, nameof(catalog)); @@ -469,10 +469,10 @@ public static PoissonRegression PoissonRegression(this RegressionCatalog.Regress string labelColumnName = DefaultColumnNames.Label, string featureColumnName = DefaultColumnNames.Features, string exampleWeightColumnName = null, - float l1Weight = LROptions.Defaults.L1Weight, - float l2Weight = LROptions.Defaults.L2Weight, + float l1Weight = LROptions.Defaults.L1Regularization, + float l2Weight = LROptions.Defaults.L2Regularization, float optimizationTolerance = LROptions.Defaults.OptimizationTolerance, - int memorySize = LROptions.Defaults.MemorySize, + int memorySize = LROptions.Defaults.NumberOfPreviousIterationsToRemember, bool enforceNoNegativity = LROptions.Defaults.EnforceNonNegativity) { Contracts.CheckValue(catalog, nameof(catalog)); @@ -510,10 +510,10 @@ public static MulticlassLogisticRegression LogisticRegression(this MulticlassCla string labelColumnName = DefaultColumnNames.Label, string featureColumnName = DefaultColumnNames.Features, string exampleWeightColumnName = null, - float l1Weight = LROptions.Defaults.L1Weight, - float l2Weight = LROptions.Defaults.L2Weight, + float l1Weight = LROptions.Defaults.L1Regularization, + float l2Weight = LROptions.Defaults.L2Regularization, float optimizationTolerance = LROptions.Defaults.OptimizationTolerance, - int memorySize = LROptions.Defaults.MemorySize, + int memorySize = LROptions.Defaults.NumberOfPreviousIterationsToRemember, bool enforceNoNegativity = LROptions.Defaults.EnforceNonNegativity) { Contracts.CheckValue(catalog, nameof(catalog)); diff --git a/src/Microsoft.ML.StaticPipe/LbfgsStatic.cs b/src/Microsoft.ML.StaticPipe/LbfgsStatic.cs index de6cd5b0ce..077b8ca9fd 100644 --- a/src/Microsoft.ML.StaticPipe/LbfgsStatic.cs +++ b/src/Microsoft.ML.StaticPipe/LbfgsStatic.cs @@ -37,10 +37,10 @@ public static (Scalar score, Scalar probability, Scalar pred Scalar label, Vector features, Scalar weights = null, - float l1Weight = Options.Defaults.L1Weight, - float l2Weight = Options.Defaults.L2Weight, + float l1Weight = Options.Defaults.L1Regularization, + float l2Weight = Options.Defaults.L2Regularization, float optimizationTolerance = Options.Defaults.OptimizationTolerance, - int memorySize = Options.Defaults.MemorySize, + int memorySize = Options.Defaults.NumberOfPreviousIterationsToRemember, bool enoforceNoNegativity = Options.Defaults.EnforceNonNegativity, Action> onFit = null) { @@ -133,10 +133,10 @@ public static Scalar PoissonRegression(this RegressionCatalog.RegressionT Scalar label, Vector features, Scalar weights = null, - float l1Weight = Options.Defaults.L1Weight, - float l2Weight = Options.Defaults.L2Weight, + float l1Weight = Options.Defaults.L1Regularization, + float l2Weight = Options.Defaults.L2Regularization, float optimizationTolerance = Options.Defaults.OptimizationTolerance, - int memorySize = Options.Defaults.MemorySize, + int memorySize = Options.Defaults.NumberOfPreviousIterationsToRemember, bool enoforceNoNegativity = Options.Defaults.EnforceNonNegativity, Action onFit = null) { @@ -230,10 +230,10 @@ public static (Vector score, Key predictedLabel) Key label, Vector features, Scalar weights = null, - float l1Weight = Options.Defaults.L1Weight, - float l2Weight = Options.Defaults.L2Weight, + float l1Weight = Options.Defaults.L1Regularization, + float l2Weight = Options.Defaults.L2Regularization, float optimizationTolerance = Options.Defaults.OptimizationTolerance, - int memorySize = Options.Defaults.MemorySize, + int memorySize = Options.Defaults.NumberOfPreviousIterationsToRemember, bool enoforceNoNegativity = Options.Defaults.EnforceNonNegativity, Action onFit = null) { @@ -303,10 +303,10 @@ internal static class LbfgsStaticUtils internal static void ValidateParams(PipelineColumn label, Vector features, Scalar weights = null, - float l1Weight = Options.Defaults.L1Weight, - float l2Weight = Options.Defaults.L2Weight, + float l1Weight = Options.Defaults.L1Regularization, + float l2Weight = Options.Defaults.L2Regularization, float optimizationTolerance = Options.Defaults.OptimizationTolerance, - int memorySize = Options.Defaults.MemorySize, + int memorySize = Options.Defaults.NumberOfPreviousIterationsToRemember, bool enoforceNoNegativity = Options.Defaults.EnforceNonNegativity, Delegate onFit = null) { diff --git a/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs b/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs index 26fa8be75d..e7ae78b3ea 100644 --- a/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs +++ b/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs @@ -431,8 +431,8 @@ public void EntryPointCreateEnsemble() var lrInput = new LogisticRegression.Options { TrainingData = data, - L1Weight = (Single)0.1 * i, - L2Weight = (Single)0.01 * (1 + i), + L1Regularization = (Single)0.1 * i, + L2Regularization = (Single)0.01 * (1 + i), NormalizeFeatures = NormalizeOption.No }; predictorModels[i] = LogisticRegression.TrainBinary(Env, lrInput).PredictorModel; @@ -735,8 +735,8 @@ public void EntryPointPipelineEnsemble() var lrInput = new LogisticRegression.Options { TrainingData = data, - L1Weight = (Single)0.1 * i, - L2Weight = (Single)0.01 * (1 + i), + L1Regularization = (Single)0.1 * i, + L2Regularization = (Single)0.01 * (1 + i), NormalizeFeatures = NormalizeOption.Yes }; predictorModels[i] = LogisticRegression.TrainBinary(Env, lrInput).PredictorModel; @@ -996,8 +996,8 @@ public void EntryPointPipelineEnsembleText() var lrInput = new LogisticRegression.Options { TrainingData = data, - L1Weight = (Single)0.1 * i, - L2Weight = (Single)0.01 * (1 + i), + L1Regularization = (Single)0.1 * i, + L2Regularization = (Single)0.01 * (1 + i), NormalizeFeatures = NormalizeOption.Yes }; predictorModels[i] = LogisticRegression.TrainBinary(Env, lrInput).PredictorModel; diff --git a/test/Microsoft.ML.StaticPipelineTesting/Training.cs b/test/Microsoft.ML.StaticPipelineTesting/Training.cs index ebc5622443..6eb5249f94 100644 --- a/test/Microsoft.ML.StaticPipelineTesting/Training.cs +++ b/test/Microsoft.ML.StaticPipelineTesting/Training.cs @@ -621,7 +621,7 @@ public void PoissonRegression() var est = reader.MakeNewEstimator() .Append(r => (r.label, score: catalog.Trainers.PoissonRegression(r.label, r.features, null, - new PoissonRegression.Options { L2Weight = 2, EnforceNonNegativity = true, NumberOfThreads = 1 }, + new PoissonRegression.Options { L2Regularization = 2, EnforceNonNegativity = true, NumberOfThreads = 1 }, onFit: (p) => { pred = p; }))); var pipe = reader.Append(est); @@ -660,7 +660,7 @@ public void LogisticRegressionBinaryClassification() var est = reader.MakeNewEstimator() .Append(r => (r.label, preds: catalog.Trainers.LogisticRegressionBinaryClassifier(r.label, r.features, null, - new LogisticRegression.Options { L1Weight = 10, NumberOfThreads = 1 }, onFit: (p) => { pred = p; }))); + new LogisticRegression.Options { L1Regularization = 10, NumberOfThreads = 1 }, onFit: (p) => { pred = p; }))); var pipe = reader.Append(est); diff --git a/test/Microsoft.ML.Tests/PermutationFeatureImportanceTests.cs b/test/Microsoft.ML.Tests/PermutationFeatureImportanceTests.cs index 5ed9829090..26af1402a3 100644 --- a/test/Microsoft.ML.Tests/PermutationFeatureImportanceTests.cs +++ b/test/Microsoft.ML.Tests/PermutationFeatureImportanceTests.cs @@ -270,7 +270,7 @@ public void TestPfiMulticlassClassificationOnSparseFeatures() { var data = GetSparseDataset(TaskType.MulticlassClassification); var model = ML.MulticlassClassification.Trainers.LogisticRegression( - new MulticlassLogisticRegression.Options { MaximumIterations = 1000 }).Fit(data); + new MulticlassLogisticRegression.Options { NumberOfIterations = 1000 }).Fit(data); var pfi = ML.MulticlassClassification.PermutationFeatureImportance(model, data); // Pfi Indices: From e805900e253f31675969e373f423fa215ef6abd4 Mon Sep 17 00:00:00 2001 From: Abhishek Goswami Date: Thu, 28 Feb 2019 17:53:12 +0000 Subject: [PATCH 3/8] added ShortName for entrypoint back compatibility; updated manifest files --- .../LogisticRegression/LbfgsPredictorBase.cs | 14 +-- .../LogisticRegression/LogisticRegression.cs | 2 +- .../Common/EntryPoints/core_manifest.json | 110 +++++++++++------- 3 files changed, 74 insertions(+), 52 deletions(-) diff --git a/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LbfgsPredictorBase.cs b/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LbfgsPredictorBase.cs index 38b16c0f31..68a442276f 100644 --- a/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LbfgsPredictorBase.cs +++ b/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LbfgsPredictorBase.cs @@ -25,7 +25,7 @@ public abstract class OptionsBase : LearnerInputBaseWithWeight /// /// L2 regularization weight. /// - [Argument(ArgumentType.AtMostOnce, HelpText = "L2 regularization weight", ShortName = "l2", SortOrder = 50)] + [Argument(ArgumentType.AtMostOnce, HelpText = "L2 regularization weight", ShortName = "l2, L2Weight", SortOrder = 50)] [TGUI(Label = "L2 Weight", Description = "Weight of L2 regularizer term", SuggestedSweeps = "0,0.1,1")] [TlcModule.SweepableFloatParamAttribute(0.0f, 1.0f, numSteps: 4)] public float L2Regularization = Defaults.L2Regularization; @@ -33,7 +33,7 @@ public abstract class OptionsBase : LearnerInputBaseWithWeight /// /// L1 regularization weight. /// - [Argument(ArgumentType.AtMostOnce, HelpText = "L1 regularization weight", ShortName = "l1", SortOrder = 50)] + [Argument(ArgumentType.AtMostOnce, HelpText = "L1 regularization weight", ShortName = "l1, L1Weight", SortOrder = 50)] [TGUI(Label = "L1 Weight", Description = "Weight of L1 regularizer term", SuggestedSweeps = "0,0.1,1")] [TlcModule.SweepableFloatParamAttribute(0.0f, 1.0f, numSteps: 4)] public float L1Regularization = Defaults.L1Regularization; @@ -42,7 +42,7 @@ public abstract class OptionsBase : LearnerInputBaseWithWeight /// Tolerance parameter for optimization convergence. (Low = slower, more accurate). /// [Argument(ArgumentType.AtMostOnce, HelpText = "Tolerance parameter for optimization convergence. Low = slower, more accurate", - ShortName = "ot", SortOrder = 50)] + ShortName = "ot, OptTol", SortOrder = 50)] [TGUI(Label = "Optimization Tolerance", Description = "Threshold for optimizer convergence", SuggestedSweeps = "1e-4,1e-7")] [TlcModule.SweepableDiscreteParamAttribute(new object[] { 1e-4f, 1e-7f })] public float OptmizationTolerance = Defaults.OptimizationTolerance; @@ -51,7 +51,7 @@ public abstract class OptionsBase : LearnerInputBaseWithWeight /// Number of previous iterations to remember for estimate of Hessian. /// [Argument(ArgumentType.AtMostOnce, HelpText = "Memory size for L-BFGS. Low=faster, less accurate", - ShortName = "m", SortOrder = 50)] + ShortName = "m, MemorySize", SortOrder = 50)] [TGUI(Description = "Memory size for L-BFGS", SuggestedSweeps = "5,20,50")] [TlcModule.SweepableDiscreteParamAttribute("MemorySize", new object[] { 5, 20, 50 })] public int NumberOfPreviousIterationsToRemember = Defaults.NumberOfPreviousIterationsToRemember; @@ -59,7 +59,7 @@ public abstract class OptionsBase : LearnerInputBaseWithWeight /// /// Number of iterations. /// - [Argument(ArgumentType.AtMostOnce, HelpText = "Maximum iterations.", ShortName = "maxiter")] + [Argument(ArgumentType.AtMostOnce, HelpText = "Maximum iterations.", ShortName = "maxiter, MaxIterations")] [TGUI(Label = "Max Number of Iterations")] [TlcModule.SweepableLongParamAttribute("MaxIterations", 1, int.MaxValue)] public int NumberOfIterations = Defaults.NumberOfIterations; @@ -86,7 +86,7 @@ public abstract class OptionsBase : LearnerInputBaseWithWeight /// /// Init Weights Diameter /// - [Argument(ArgumentType.LastOccurenceWins, HelpText = "Init weights diameter", ShortName = "initwts", SortOrder = 140)] + [Argument(ArgumentType.LastOccurenceWins, HelpText = "Init weights diameter", ShortName = "initwts, InitWtsDiameter", SortOrder = 140)] [TGUI(Label = "Initial Weights Scale", SuggestedSweeps = "0,0.1,0.5,1")] [TlcModule.SweepableFloatParamAttribute("InitWtsDiameter", 0.0f, 1.0f, numSteps: 5)] public float InitialWeightsDiameter = 0; @@ -101,7 +101,7 @@ public abstract class OptionsBase : LearnerInputBaseWithWeight /// /// Number of threads. Null means use the number of processors. /// - [Argument(ArgumentType.AtMostOnce, HelpText = "Number of threads", ShortName = "nt")] + [Argument(ArgumentType.AtMostOnce, HelpText = "Number of threads", ShortName = "nt, NumThreads")] public int? NumberOfThreads; /// diff --git a/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LogisticRegression.cs b/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LogisticRegression.cs index 7fd4135152..d0abbba694 100644 --- a/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LogisticRegression.cs +++ b/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LogisticRegression.cs @@ -48,7 +48,7 @@ public sealed class Options : OptionsBase /// More than 1000 weights might take a few minutes. For those cases consider using the instance of /// present in the Microsoft.ML.HalLearners package. That computes the statistics using hardware acceleration. /// - [Argument(ArgumentType.AtMostOnce, HelpText = "Show statistics of training examples.", ShortName = "stat", SortOrder = 50)] + [Argument(ArgumentType.AtMostOnce, HelpText = "Show statistics of training examples.", ShortName = "stat, ShowTrainingStats", SortOrder = 50)] public bool ShowTrainingStatistics = false; /// diff --git a/test/BaselineOutput/Common/EntryPoints/core_manifest.json b/test/BaselineOutput/Common/EntryPoints/core_manifest.json index 984e570840..c25ee2b28d 100644 --- a/test/BaselineOutput/Common/EntryPoints/core_manifest.json +++ b/test/BaselineOutput/Common/EntryPoints/core_manifest.json @@ -13495,11 +13495,12 @@ "Default": "Auto" }, { - "Name": "ShowTrainingStats", + "Name": "ShowTrainingStatistics", "Type": "Bool", "Desc": "Show statistics of training examples.", "Aliases": [ - "stat" + "stat", + "ShowTrainingStats" ], "Required": false, "SortOrder": 50.0, @@ -13507,11 +13508,12 @@ "Default": false }, { - "Name": "L2Weight", + "Name": "L2Regularization", "Type": "Float", "Desc": "L2 regularization weight", "Aliases": [ - "l2" + "l2", + "L2Weight" ], "Required": false, "SortOrder": 50.0, @@ -13525,11 +13527,12 @@ } }, { - "Name": "L1Weight", + "Name": "L1Regularization", "Type": "Float", "Desc": "L1 regularization weight", "Aliases": [ - "l1" + "l1", + "L1Weight" ], "Required": false, "SortOrder": 50.0, @@ -13543,11 +13546,12 @@ } }, { - "Name": "OptTol", + "Name": "OptmizationTolerance", "Type": "Float", "Desc": "Tolerance parameter for optimization convergence. Low = slower, more accurate", "Aliases": [ - "ot" + "ot", + "OptTol" ], "Required": false, "SortOrder": 50.0, @@ -13562,11 +13566,12 @@ } }, { - "Name": "MemorySize", + "Name": "NumberOfPreviousIterationsToRemember", "Type": "Int", "Desc": "Memory size for L-BFGS. Low=faster, less accurate", "Aliases": [ - "m" + "m", + "MemorySize" ], "Required": false, "SortOrder": 50.0, @@ -13594,11 +13599,12 @@ "Default": false }, { - "Name": "InitWtsDiameter", + "Name": "InitialWeightsDiameter", "Type": "Float", "Desc": "Init weights diameter", "Aliases": [ - "initwts" + "initwts", + "InitWtsDiameter" ], "Required": false, "SortOrder": 140.0, @@ -13612,11 +13618,12 @@ } }, { - "Name": "MaxIterations", + "Name": "NumberOfIterations", "Type": "Int", "Desc": "Maximum iterations.", "Aliases": [ - "maxiter" + "maxiter", + "MaxIterations" ], "Required": false, "SortOrder": 150.0, @@ -13665,11 +13672,12 @@ "Default": true }, { - "Name": "NumThreads", + "Name": "NumberOfThreads", "Type": "Int", "Desc": "Number of threads", "Aliases": [ - "nt" + "nt", + "NumThreads" ], "Required": false, "SortOrder": 150.0, @@ -13818,11 +13826,12 @@ "Default": false }, { - "Name": "L2Weight", + "Name": "L2Regularization", "Type": "Float", "Desc": "L2 regularization weight", "Aliases": [ - "l2" + "l2", + "L2Weight" ], "Required": false, "SortOrder": 50.0, @@ -13836,11 +13845,12 @@ } }, { - "Name": "L1Weight", + "Name": "L1Regularization", "Type": "Float", "Desc": "L1 regularization weight", "Aliases": [ - "l1" + "l1", + "L1Weight" ], "Required": false, "SortOrder": 50.0, @@ -13854,11 +13864,12 @@ } }, { - "Name": "OptTol", + "Name": "OptmizationTolerance", "Type": "Float", "Desc": "Tolerance parameter for optimization convergence. Low = slower, more accurate", "Aliases": [ - "ot" + "ot", + "OptTol" ], "Required": false, "SortOrder": 50.0, @@ -13873,11 +13884,12 @@ } }, { - "Name": "MemorySize", + "Name": "NumberOfPreviousIterationsToRemember", "Type": "Int", "Desc": "Memory size for L-BFGS. Low=faster, less accurate", "Aliases": [ - "m" + "m", + "MemorySize" ], "Required": false, "SortOrder": 50.0, @@ -13905,11 +13917,12 @@ "Default": false }, { - "Name": "InitWtsDiameter", + "Name": "InitialWeightsDiameter", "Type": "Float", "Desc": "Init weights diameter", "Aliases": [ - "initwts" + "initwts", + "InitWtsDiameter" ], "Required": false, "SortOrder": 140.0, @@ -13923,11 +13936,12 @@ } }, { - "Name": "MaxIterations", + "Name": "NumberOfIterations", "Type": "Int", "Desc": "Maximum iterations.", "Aliases": [ - "maxiter" + "maxiter", + "MaxIterations" ], "Required": false, "SortOrder": 150.0, @@ -13976,11 +13990,12 @@ "Default": true }, { - "Name": "NumThreads", + "Name": "NumberOfThreads", "Type": "Int", "Desc": "Number of threads", "Aliases": [ - "nt" + "nt", + "NumThreads" ], "Required": false, "SortOrder": 150.0, @@ -14835,11 +14850,12 @@ "Default": "Auto" }, { - "Name": "L2Weight", + "Name": "L2Regularization", "Type": "Float", "Desc": "L2 regularization weight", "Aliases": [ - "l2" + "l2", + "L2Weight" ], "Required": false, "SortOrder": 50.0, @@ -14853,11 +14869,12 @@ } }, { - "Name": "L1Weight", + "Name": "L1Regularization", "Type": "Float", "Desc": "L1 regularization weight", "Aliases": [ - "l1" + "l1", + "L1Weight" ], "Required": false, "SortOrder": 50.0, @@ -14871,11 +14888,12 @@ } }, { - "Name": "OptTol", + "Name": "OptmizationTolerance", "Type": "Float", "Desc": "Tolerance parameter for optimization convergence. Low = slower, more accurate", "Aliases": [ - "ot" + "ot", + "OptTol" ], "Required": false, "SortOrder": 50.0, @@ -14890,11 +14908,12 @@ } }, { - "Name": "MemorySize", + "Name": "NumberOfPreviousIterationsToRemember", "Type": "Int", "Desc": "Memory size for L-BFGS. Low=faster, less accurate", "Aliases": [ - "m" + "m", + "MemorySize" ], "Required": false, "SortOrder": 50.0, @@ -14922,11 +14941,12 @@ "Default": false }, { - "Name": "InitWtsDiameter", + "Name": "InitialWeightsDiameter", "Type": "Float", "Desc": "Init weights diameter", "Aliases": [ - "initwts" + "initwts", + "InitWtsDiameter" ], "Required": false, "SortOrder": 140.0, @@ -14940,11 +14960,12 @@ } }, { - "Name": "MaxIterations", + "Name": "NumberOfIterations", "Type": "Int", "Desc": "Maximum iterations.", "Aliases": [ - "maxiter" + "maxiter", + "MaxIterations" ], "Required": false, "SortOrder": 150.0, @@ -14993,11 +15014,12 @@ "Default": true }, { - "Name": "NumThreads", + "Name": "NumberOfThreads", "Type": "Int", "Desc": "Number of threads", "Aliases": [ - "nt" + "nt", + "NumThreads" ], "Required": false, "SortOrder": 150.0, From 2b3f88ff3b59fe0aff0e7d276c3ae936c452f9a1 Mon Sep 17 00:00:00 2001 From: Abhishek Goswami Date: Thu, 28 Feb 2019 19:24:22 +0000 Subject: [PATCH 4/8] also looked into MultiClass LR and PoissonRegression --- .../Standard/LinearModelParameters.cs | 3 +++ .../MulticlassLogisticRegression.cs | 14 ++++++++++---- .../Common/EntryPoints/core_manifest.json | 5 +++-- .../UnitTests/TestEntryPoints.cs | 2 +- 4 files changed, 17 insertions(+), 7 deletions(-) diff --git a/src/Microsoft.ML.StandardLearners/Standard/LinearModelParameters.cs b/src/Microsoft.ML.StandardLearners/Standard/LinearModelParameters.cs index 366bea0b11..6a4f72c6b9 100644 --- a/src/Microsoft.ML.StandardLearners/Standard/LinearModelParameters.cs +++ b/src/Microsoft.ML.StandardLearners/Standard/LinearModelParameters.cs @@ -662,6 +662,9 @@ IList> ICanGetSummaryInKeyValuePairs.GetSummaryInKe } } + /// + /// The model parameters class for Poisson Regression. + /// public sealed class PoissonRegressionModelParameters : RegressionModelParameters, IParameterMixer { internal const string LoaderSignature = "PoissonRegressionExec"; diff --git a/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/MulticlassLogisticRegression.cs b/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/MulticlassLogisticRegression.cs index 8a6a43a0f0..bc66737eed 100644 --- a/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/MulticlassLogisticRegression.cs +++ b/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/MulticlassLogisticRegression.cs @@ -45,8 +45,11 @@ public sealed class MulticlassLogisticRegression : LbfgsTrainerBase + /// If set to truetraining statistics will be generated at the end of training. + /// + [Argument(ArgumentType.AtMostOnce, HelpText = "Show statistics of training examples.", ShortName = "stat, ShowTrainingStats", SortOrder = 50)] + public bool ShowTrainingStatistics = false; } private int _numClasses; @@ -94,7 +97,7 @@ internal MulticlassLogisticRegression(IHostEnvironment env, Host.CheckNonEmpty(featureColumn, nameof(featureColumn)); Host.CheckNonEmpty(labelColumn, nameof(labelColumn)); - ShowTrainingStats = LbfgsTrainerOptions.ShowTrainingStats; + ShowTrainingStats = LbfgsTrainerOptions.ShowTrainingStatistics; } /// @@ -103,7 +106,7 @@ internal MulticlassLogisticRegression(IHostEnvironment env, internal MulticlassLogisticRegression(IHostEnvironment env, Options options) : base(env, options, TrainerUtils.MakeU4ScalarColumn(options.LabelColumn)) { - ShowTrainingStats = LbfgsTrainerOptions.ShowTrainingStats; + ShowTrainingStats = LbfgsTrainerOptions.ShowTrainingStatistics; } private protected override PredictionKind PredictionKind => PredictionKind.MultiClassClassification; @@ -333,6 +336,9 @@ public MulticlassPredictionTransformer TrainTransformer(trainData, initPredictor: modelParameters); } + /// + /// The model parameters class for Multiclass Logistic Regression. + /// public sealed class MulticlassLogisticRegressionModelParameters : ModelParametersBase>, IValueMapper, diff --git a/test/BaselineOutput/Common/EntryPoints/core_manifest.json b/test/BaselineOutput/Common/EntryPoints/core_manifest.json index c25ee2b28d..07598e4fd8 100644 --- a/test/BaselineOutput/Common/EntryPoints/core_manifest.json +++ b/test/BaselineOutput/Common/EntryPoints/core_manifest.json @@ -13814,11 +13814,12 @@ "Default": "Auto" }, { - "Name": "ShowTrainingStats", + "Name": "ShowTrainingStatistics", "Type": "Bool", "Desc": "Show statistics of training examples.", "Aliases": [ - "stat" + "stat", + "ShowTrainingStats" ], "Required": false, "SortOrder": 50.0, diff --git a/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs b/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs index e7ae78b3ea..0f09ba5eb6 100644 --- a/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs +++ b/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs @@ -3338,7 +3338,7 @@ public void EntryPointLinearPredictorSummary() TrainingData = dataView, NormalizeFeatures = NormalizeOption.Yes, NumberOfThreads = 1, - ShowTrainingStats = true + ShowTrainingStatistics = true }; var mcModel = LogisticRegression.TrainMultiClass(Env, mcLrInput).PredictorModel; From 04a4f7a367aed75978dad0d0b0cb1e1d8ea4301f Mon Sep 17 00:00:00 2001 From: Abhishek Goswami Date: Thu, 28 Feb 2019 23:05:56 +0000 Subject: [PATCH 5/8] review comments --- .../Standard/LogisticRegression/LbfgsPredictorBase.cs | 6 ++---- test/Microsoft.ML.Tests/OnnxConversionTest.cs | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LbfgsPredictorBase.cs b/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LbfgsPredictorBase.cs index 61f680fe66..e9692d6865 100644 --- a/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LbfgsPredictorBase.cs +++ b/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LbfgsPredictorBase.cs @@ -91,12 +91,10 @@ public abstract class OptionsBase : TrainerInputBaseWithWeight [TlcModule.SweepableFloatParamAttribute("InitWtsDiameter", 0.0f, 1.0f, numSteps: 5)] public float InitialWeightsDiameter = 0; - /// - /// Whether or not to use threads. Default is true. - /// + // Deprecated [Argument(ArgumentType.AtMostOnce, HelpText = "Whether or not to use threads. Default is true", ShortName = "t", Hide = true)] - public bool UseThreads = true; + internal bool UseThreads = true; /// /// Number of threads. Null means use the number of processors. diff --git a/test/Microsoft.ML.Tests/OnnxConversionTest.cs b/test/Microsoft.ML.Tests/OnnxConversionTest.cs index 492a71a501..4e83b3aad4 100644 --- a/test/Microsoft.ML.Tests/OnnxConversionTest.cs +++ b/test/Microsoft.ML.Tests/OnnxConversionTest.cs @@ -376,7 +376,7 @@ public void MulticlassLogisticRegressionOnnxConversionTest() var pipeline = mlContext.Transforms.Normalize("Features"). Append(mlContext.Transforms.Conversion.MapValueToKey("Label")). - Append(mlContext.MulticlassClassification.Trainers.LogisticRegression(new MulticlassLogisticRegression.Options() { UseThreads = false })); + Append(mlContext.MulticlassClassification.Trainers.LogisticRegression(new MulticlassLogisticRegression.Options() { NumberOfThreads = 1 })); var model = pipeline.Fit(data); var transformedData = model.Transform(data); From 7a7ac7852be6b29629731e18bce5ea69af5f7c3b Mon Sep 17 00:00:00 2001 From: Abhishek Goswami Date: Sun, 3 Mar 2019 17:21:53 +0000 Subject: [PATCH 6/8] review comments --- .../Prediction/Calibrator.cs | 2 +- .../Standard/LinearModelParameters.cs | 2 +- .../LogisticRegression/LbfgsPredictorBase.cs | 22 +++++++++---------- .../LogisticRegression/LogisticRegression.cs | 2 +- .../MulticlassLogisticRegression.cs | 2 +- .../PoissonRegression/PoissonRegression.cs | 2 +- .../StandardLearnersCatalog.cs | 6 ++--- src/Microsoft.ML.StaticPipe/LbfgsStatic.cs | 8 +++---- .../Common/EntryPoints/core_manifest.json | 21 ++++++++++-------- 9 files changed, 35 insertions(+), 32 deletions(-) diff --git a/src/Microsoft.ML.Data/Prediction/Calibrator.cs b/src/Microsoft.ML.Data/Prediction/Calibrator.cs index 46e8063df0..d72cb120c7 100644 --- a/src/Microsoft.ML.Data/Prediction/Calibrator.cs +++ b/src/Microsoft.ML.Data/Prediction/Calibrator.cs @@ -173,7 +173,7 @@ public abstract class CalibratedModelParametersBase : where TSubModel : class where TCalibrator : class, ICalibrator { - protected readonly IHost Host; + private protected readonly IHost Host; // Strongly-typed members. /// diff --git a/src/Microsoft.ML.StandardLearners/Standard/LinearModelParameters.cs b/src/Microsoft.ML.StandardLearners/Standard/LinearModelParameters.cs index 6afc687e48..34a7dc61d8 100644 --- a/src/Microsoft.ML.StandardLearners/Standard/LinearModelParameters.cs +++ b/src/Microsoft.ML.StandardLearners/Standard/LinearModelParameters.cs @@ -261,7 +261,7 @@ private protected virtual float Score(in VBuffer src) return Bias + VectorUtils.DotProduct(in _weightsDense, in src); } - protected virtual void GetFeatureContributions(in VBuffer features, ref VBuffer contributions, int top, int bottom, bool normalize) + private protected virtual void GetFeatureContributions(in VBuffer features, ref VBuffer contributions, int top, int bottom, bool normalize) { if (features.Length != Weight.Length) throw Contracts.Except("Input is of length {0} does not match expected length of weights {1}", features.Length, Weight.Length); diff --git a/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LbfgsPredictorBase.cs b/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LbfgsPredictorBase.cs index e9692d6865..b627eb0133 100644 --- a/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LbfgsPredictorBase.cs +++ b/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LbfgsPredictorBase.cs @@ -54,7 +54,7 @@ public abstract class OptionsBase : TrainerInputBaseWithWeight ShortName = "m, MemorySize", SortOrder = 50)] [TGUI(Description = "Memory size for L-BFGS", SuggestedSweeps = "5,20,50")] [TlcModule.SweepableDiscreteParamAttribute("MemorySize", new object[] { 5, 20, 50 })] - public int NumberOfPreviousIterationsToRemember = Defaults.NumberOfPreviousIterationsToRemember; + public int IterationsToRemember = Defaults.IterationsToRemember; /// /// Number of iterations. @@ -68,8 +68,8 @@ public abstract class OptionsBase : TrainerInputBaseWithWeight /// Run SGD to initialize LR weights, converging to this tolerance. /// [Argument(ArgumentType.AtMostOnce, HelpText = "Run SGD to initialize LR weights, converging to this tolerance", - ShortName = "sgd")] - public float SgdInitializationTolerance = 0; + ShortName = "sgd, SgdInitializationTolerance")] + public float StochasticGradientDescentInitilaizationTolerance = 0; /// /// Features must occur in at least this many instances to be included @@ -121,7 +121,7 @@ internal static class Defaults public const float L2Regularization = 1; public const float L1Regularization = 1; public const float OptimizationTolerance = 1e-7f; - public const int NumberOfPreviousIterationsToRemember = 20; + public const int IterationsToRemember = 20; public const int NumberOfIterations = int.MaxValue; public const bool EnforceNonNegativity = false; } @@ -192,7 +192,7 @@ internal LbfgsTrainerBase(IHostEnvironment env, L1Regularization = l1Weight, L2Regularization = l2Weight, OptmizationTolerance = optimizationTolerance, - NumberOfPreviousIterationsToRemember = memorySize, + IterationsToRemember = memorySize, EnforceNonNegativity = enforceNoNegativity }, labelColumn) @@ -220,22 +220,22 @@ internal LbfgsTrainerBase(IHostEnvironment env, Host.CheckUserArg(LbfgsTrainerOptions.L2Regularization >= 0, nameof(LbfgsTrainerOptions.L2Regularization), "Must be non-negative"); Host.CheckUserArg(LbfgsTrainerOptions.L1Regularization >= 0, nameof(LbfgsTrainerOptions.L1Regularization), "Must be non-negative"); Host.CheckUserArg(LbfgsTrainerOptions.OptmizationTolerance > 0, nameof(LbfgsTrainerOptions.OptmizationTolerance), "Must be positive"); - Host.CheckUserArg(LbfgsTrainerOptions.NumberOfPreviousIterationsToRemember > 0, nameof(LbfgsTrainerOptions.NumberOfPreviousIterationsToRemember), "Must be positive"); + Host.CheckUserArg(LbfgsTrainerOptions.IterationsToRemember > 0, nameof(LbfgsTrainerOptions.IterationsToRemember), "Must be positive"); Host.CheckUserArg(LbfgsTrainerOptions.NumberOfIterations > 0, nameof(LbfgsTrainerOptions.NumberOfIterations), "Must be positive"); - Host.CheckUserArg(LbfgsTrainerOptions.SgdInitializationTolerance >= 0, nameof(LbfgsTrainerOptions.SgdInitializationTolerance), "Must be non-negative"); + Host.CheckUserArg(LbfgsTrainerOptions.StochasticGradientDescentInitilaizationTolerance >= 0, nameof(LbfgsTrainerOptions.StochasticGradientDescentInitilaizationTolerance), "Must be non-negative"); Host.CheckUserArg(LbfgsTrainerOptions.NumberOfThreads == null || LbfgsTrainerOptions.NumberOfThreads.Value >= 0, nameof(LbfgsTrainerOptions.NumberOfThreads), "Must be non-negative"); Host.CheckParam(!(LbfgsTrainerOptions.L2Regularization < 0), nameof(LbfgsTrainerOptions.L2Regularization), "Must be non-negative, if provided."); Host.CheckParam(!(LbfgsTrainerOptions.L1Regularization < 0), nameof(LbfgsTrainerOptions.L1Regularization), "Must be non-negative, if provided"); Host.CheckParam(!(LbfgsTrainerOptions.OptmizationTolerance <= 0), nameof(LbfgsTrainerOptions.OptmizationTolerance), "Must be positive, if provided."); - Host.CheckParam(!(LbfgsTrainerOptions.NumberOfPreviousIterationsToRemember <= 0), nameof(LbfgsTrainerOptions.NumberOfPreviousIterationsToRemember), "Must be positive, if provided."); + Host.CheckParam(!(LbfgsTrainerOptions.IterationsToRemember <= 0), nameof(LbfgsTrainerOptions.IterationsToRemember), "Must be positive, if provided."); L2Weight = LbfgsTrainerOptions.L2Regularization; L1Weight = LbfgsTrainerOptions.L1Regularization; OptTol = LbfgsTrainerOptions.OptmizationTolerance; - MemorySize =LbfgsTrainerOptions.NumberOfPreviousIterationsToRemember; + MemorySize =LbfgsTrainerOptions.IterationsToRemember; MaxIterations = LbfgsTrainerOptions.NumberOfIterations; - SgdInitializationTolerance = LbfgsTrainerOptions.SgdInitializationTolerance; + SgdInitializationTolerance = LbfgsTrainerOptions.StochasticGradientDescentInitilaizationTolerance; Quiet = LbfgsTrainerOptions.Quiet; InitWtsDiameter = LbfgsTrainerOptions.InitialWeightsDiameter; UseThreads = LbfgsTrainerOptions.UseThreads; @@ -272,7 +272,7 @@ private static TOptions ArgsInit(string featureColumn, SchemaShape.Column labelC L1Regularization = l1Weight, L2Regularization = l2Weight, OptmizationTolerance = optimizationTolerance, - NumberOfPreviousIterationsToRemember = memorySize, + IterationsToRemember = memorySize, EnforceNonNegativity = enforceNoNegativity }; diff --git a/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LogisticRegression.cs b/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LogisticRegression.cs index 758c4acdc4..d0f33ff209 100644 --- a/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LogisticRegression.cs +++ b/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LogisticRegression.cs @@ -82,7 +82,7 @@ internal LogisticRegression(IHostEnvironment env, float l1Weight = Options.Defaults.L1Regularization, float l2Weight = Options.Defaults.L2Regularization, float optimizationTolerance = Options.Defaults.OptimizationTolerance, - int memorySize = Options.Defaults.NumberOfPreviousIterationsToRemember, + int memorySize = Options.Defaults.IterationsToRemember, bool enforceNoNegativity = Options.Defaults.EnforceNonNegativity) : base(env, featureColumn, TrainerUtils.MakeBoolScalarLabel(labelColumn), weights, l1Weight, l2Weight, optimizationTolerance, memorySize, enforceNoNegativity) diff --git a/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/MulticlassLogisticRegression.cs b/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/MulticlassLogisticRegression.cs index 25230b9b65..5e66c1fde9 100644 --- a/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/MulticlassLogisticRegression.cs +++ b/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/MulticlassLogisticRegression.cs @@ -90,7 +90,7 @@ internal MulticlassLogisticRegression(IHostEnvironment env, float l1Weight = Options.Defaults.L1Regularization, float l2Weight = Options.Defaults.L2Regularization, float optimizationTolerance = Options.Defaults.OptimizationTolerance, - int memorySize = Options.Defaults.NumberOfPreviousIterationsToRemember, + int memorySize = Options.Defaults.IterationsToRemember, bool enforceNoNegativity = Options.Defaults.EnforceNonNegativity) : base(env, featureColumn, TrainerUtils.MakeU4ScalarColumn(labelColumn), weights, l1Weight, l2Weight, optimizationTolerance, memorySize, enforceNoNegativity) { diff --git a/src/Microsoft.ML.StandardLearners/Standard/PoissonRegression/PoissonRegression.cs b/src/Microsoft.ML.StandardLearners/Standard/PoissonRegression/PoissonRegression.cs index c19acc736c..bb1e9eb792 100644 --- a/src/Microsoft.ML.StandardLearners/Standard/PoissonRegression/PoissonRegression.cs +++ b/src/Microsoft.ML.StandardLearners/Standard/PoissonRegression/PoissonRegression.cs @@ -57,7 +57,7 @@ internal PoissonRegression(IHostEnvironment env, float l1Weight = Options.Defaults.L1Regularization, float l2Weight = Options.Defaults.L2Regularization, float optimizationTolerance = Options.Defaults.OptimizationTolerance, - int memorySize = Options.Defaults.NumberOfPreviousIterationsToRemember, + int memorySize = Options.Defaults.IterationsToRemember, bool enforceNoNegativity = Options.Defaults.EnforceNonNegativity) : base(env, featureColumn, TrainerUtils.MakeR4ScalarColumn(labelColumn), weights, l1Weight, l2Weight, optimizationTolerance, memorySize, enforceNoNegativity) diff --git a/src/Microsoft.ML.StandardLearners/StandardLearnersCatalog.cs b/src/Microsoft.ML.StandardLearners/StandardLearnersCatalog.cs index 6c344ded34..589d93251c 100644 --- a/src/Microsoft.ML.StandardLearners/StandardLearnersCatalog.cs +++ b/src/Microsoft.ML.StandardLearners/StandardLearnersCatalog.cs @@ -444,7 +444,7 @@ public static LogisticRegression LogisticRegression(this BinaryClassificationCat float l1Weight = LROptions.Defaults.L1Regularization, float l2Weight = LROptions.Defaults.L2Regularization, float optimizationTolerance = LROptions.Defaults.OptimizationTolerance, - int memorySize = LROptions.Defaults.NumberOfPreviousIterationsToRemember, + int memorySize = LROptions.Defaults.IterationsToRemember, bool enforceNoNegativity = LROptions.Defaults.EnforceNonNegativity) { Contracts.CheckValue(catalog, nameof(catalog)); @@ -485,7 +485,7 @@ public static PoissonRegression PoissonRegression(this RegressionCatalog.Regress float l1Weight = LROptions.Defaults.L1Regularization, float l2Weight = LROptions.Defaults.L2Regularization, float optimizationTolerance = LROptions.Defaults.OptimizationTolerance, - int memorySize = LROptions.Defaults.NumberOfPreviousIterationsToRemember, + int memorySize = LROptions.Defaults.IterationsToRemember, bool enforceNoNegativity = LROptions.Defaults.EnforceNonNegativity) { Contracts.CheckValue(catalog, nameof(catalog)); @@ -526,7 +526,7 @@ public static MulticlassLogisticRegression LogisticRegression(this MulticlassCla float l1Weight = LROptions.Defaults.L1Regularization, float l2Weight = LROptions.Defaults.L2Regularization, float optimizationTolerance = LROptions.Defaults.OptimizationTolerance, - int memorySize = LROptions.Defaults.NumberOfPreviousIterationsToRemember, + int memorySize = LROptions.Defaults.IterationsToRemember, bool enforceNoNegativity = LROptions.Defaults.EnforceNonNegativity) { Contracts.CheckValue(catalog, nameof(catalog)); diff --git a/src/Microsoft.ML.StaticPipe/LbfgsStatic.cs b/src/Microsoft.ML.StaticPipe/LbfgsStatic.cs index 37d5860c4c..33149b7645 100644 --- a/src/Microsoft.ML.StaticPipe/LbfgsStatic.cs +++ b/src/Microsoft.ML.StaticPipe/LbfgsStatic.cs @@ -40,7 +40,7 @@ public static (Scalar score, Scalar probability, Scalar pred float l1Weight = Options.Defaults.L1Regularization, float l2Weight = Options.Defaults.L2Regularization, float optimizationTolerance = Options.Defaults.OptimizationTolerance, - int memorySize = Options.Defaults.NumberOfPreviousIterationsToRemember, + int memorySize = Options.Defaults.IterationsToRemember, bool enoforceNoNegativity = Options.Defaults.EnforceNonNegativity, Action> onFit = null) { @@ -136,7 +136,7 @@ public static Scalar PoissonRegression(this RegressionCatalog.RegressionT float l1Weight = Options.Defaults.L1Regularization, float l2Weight = Options.Defaults.L2Regularization, float optimizationTolerance = Options.Defaults.OptimizationTolerance, - int memorySize = Options.Defaults.NumberOfPreviousIterationsToRemember, + int memorySize = Options.Defaults.IterationsToRemember, bool enoforceNoNegativity = Options.Defaults.EnforceNonNegativity, Action onFit = null) { @@ -233,7 +233,7 @@ public static (Vector score, Key predictedLabel) float l1Weight = Options.Defaults.L1Regularization, float l2Weight = Options.Defaults.L2Regularization, float optimizationTolerance = Options.Defaults.OptimizationTolerance, - int memorySize = Options.Defaults.NumberOfPreviousIterationsToRemember, + int memorySize = Options.Defaults.IterationsToRemember, bool enoforceNoNegativity = Options.Defaults.EnforceNonNegativity, Action onFit = null) { @@ -306,7 +306,7 @@ internal static void ValidateParams(PipelineColumn label, float l1Weight = Options.Defaults.L1Regularization, float l2Weight = Options.Defaults.L2Regularization, float optimizationTolerance = Options.Defaults.OptimizationTolerance, - int memorySize = Options.Defaults.NumberOfPreviousIterationsToRemember, + int memorySize = Options.Defaults.IterationsToRemember, bool enoforceNoNegativity = Options.Defaults.EnforceNonNegativity, Delegate onFit = null) { diff --git a/test/BaselineOutput/Common/EntryPoints/core_manifest.json b/test/BaselineOutput/Common/EntryPoints/core_manifest.json index f6e666bbe3..0f99642c54 100644 --- a/test/BaselineOutput/Common/EntryPoints/core_manifest.json +++ b/test/BaselineOutput/Common/EntryPoints/core_manifest.json @@ -13566,7 +13566,7 @@ } }, { - "Name": "NumberOfPreviousIterationsToRemember", + "Name": "IterationsToRemember", "Type": "Int", "Desc": "Memory size for L-BFGS. Low=faster, less accurate", "Aliases": [ @@ -13636,11 +13636,12 @@ } }, { - "Name": "SgdInitializationTolerance", + "Name": "StochasticGradientDescentInitilaizationTolerance", "Type": "Float", "Desc": "Run SGD to initialize LR weights, converging to this tolerance", "Aliases": [ - "sgd" + "sgd", + "SgdInitializationTolerance" ], "Required": false, "SortOrder": 150.0, @@ -13885,7 +13886,7 @@ } }, { - "Name": "NumberOfPreviousIterationsToRemember", + "Name": "IterationsToRemember", "Type": "Int", "Desc": "Memory size for L-BFGS. Low=faster, less accurate", "Aliases": [ @@ -13955,11 +13956,12 @@ } }, { - "Name": "SgdInitializationTolerance", + "Name": "StochasticGradientDescentInitilaizationTolerance", "Type": "Float", "Desc": "Run SGD to initialize LR weights, converging to this tolerance", "Aliases": [ - "sgd" + "sgd", + "SgdInitializationTolerance" ], "Required": false, "SortOrder": 150.0, @@ -14909,7 +14911,7 @@ } }, { - "Name": "NumberOfPreviousIterationsToRemember", + "Name": "IterationsToRemember", "Type": "Int", "Desc": "Memory size for L-BFGS. Low=faster, less accurate", "Aliases": [ @@ -14979,11 +14981,12 @@ } }, { - "Name": "SgdInitializationTolerance", + "Name": "StochasticGradientDescentInitilaizationTolerance", "Type": "Float", "Desc": "Run SGD to initialize LR weights, converging to this tolerance", "Aliases": [ - "sgd" + "sgd", + "SgdInitializationTolerance" ], "Required": false, "SortOrder": 150.0, From c46837d8e1befd5864c1540a9482c811f09880f7 Mon Sep 17 00:00:00 2001 From: Abhishek Goswami Date: Mon, 4 Mar 2019 19:18:28 +0000 Subject: [PATCH 7/8] review comments --- .../Standard/LogisticRegression/LbfgsPredictorBase.cs | 2 +- .../Standard/LogisticRegression/LogisticRegression.cs | 2 +- .../Standard/LogisticRegression/MulticlassLogisticRegression.cs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LbfgsPredictorBase.cs b/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LbfgsPredictorBase.cs index b627eb0133..cdca6e8b66 100644 --- a/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LbfgsPredictorBase.cs +++ b/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LbfgsPredictorBase.cs @@ -216,7 +216,7 @@ internal LbfgsTrainerBase(IHostEnvironment env, options.LabelColumnName = LabelColumn.Name; options.ExampleWeightColumnName = WeightColumn.Name; Host.CheckUserArg(!LbfgsTrainerOptions.UseThreads || LbfgsTrainerOptions.NumberOfThreads > 0 || LbfgsTrainerOptions.NumberOfThreads == null, - nameof(LbfgsTrainerOptions.NumberOfThreads), "numThreads must be positive (or empty for default)"); + nameof(LbfgsTrainerOptions.NumberOfThreads), "Must be positive (or empty for default)"); Host.CheckUserArg(LbfgsTrainerOptions.L2Regularization >= 0, nameof(LbfgsTrainerOptions.L2Regularization), "Must be non-negative"); Host.CheckUserArg(LbfgsTrainerOptions.L1Regularization >= 0, nameof(LbfgsTrainerOptions.L1Regularization), "Must be non-negative"); Host.CheckUserArg(LbfgsTrainerOptions.OptmizationTolerance > 0, nameof(LbfgsTrainerOptions.OptmizationTolerance), "Must be positive"); diff --git a/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LogisticRegression.cs b/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LogisticRegression.cs index d0f33ff209..ea14a26f33 100644 --- a/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LogisticRegression.cs +++ b/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LogisticRegression.cs @@ -42,7 +42,7 @@ public sealed partial class LogisticRegression : LbfgsTrainerBase - /// If set to truetraining statistics will be generated at the end of training. + /// If set to true training statistics will be generated at the end of training. /// If you have a large number of learned training parameters(more than 500), /// generating the training statistics might take a few seconds. /// More than 1000 weights might take a few minutes. For those cases consider using the instance of diff --git a/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/MulticlassLogisticRegression.cs b/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/MulticlassLogisticRegression.cs index 5e66c1fde9..c6b69986fd 100644 --- a/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/MulticlassLogisticRegression.cs +++ b/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/MulticlassLogisticRegression.cs @@ -46,7 +46,7 @@ public sealed class MulticlassLogisticRegression : LbfgsTrainerBase - /// If set to truetraining statistics will be generated at the end of training. + /// If set to true training statistics will be generated at the end of training. /// [Argument(ArgumentType.AtMostOnce, HelpText = "Show statistics of training examples.", ShortName = "stat, ShowTrainingStats", SortOrder = 50)] public bool ShowTrainingStatistics = false; From 064d364c2e915ab7ed860509eacee88dc8504bea Mon Sep 17 00:00:00 2001 From: Abhishek Goswami Date: Mon, 4 Mar 2019 20:47:04 +0000 Subject: [PATCH 8/8] ensure MLcontext names are same as those used in Options --- .../StandardLearnersCatalog.cs | 54 +++++++------- src/Microsoft.ML.StaticPipe/LbfgsStatic.cs | 74 +++++++++---------- 2 files changed, 64 insertions(+), 64 deletions(-) diff --git a/src/Microsoft.ML.StandardLearners/StandardLearnersCatalog.cs b/src/Microsoft.ML.StandardLearners/StandardLearnersCatalog.cs index 931a771689..bff013f45a 100644 --- a/src/Microsoft.ML.StandardLearners/StandardLearnersCatalog.cs +++ b/src/Microsoft.ML.StandardLearners/StandardLearnersCatalog.cs @@ -449,10 +449,10 @@ public static OnlineGradientDescentTrainer OnlineGradientDescent(this Regression /// The name of the label column. /// The name of the feature column. /// The name of the example weight column (optional). - /// Enforce non-negative weights. - /// Weight of L1 regularization term. - /// Weight of L2 regularization term. - /// Memory size for . Low=faster, less accurate. + /// Enforce non-negative weights. + /// Weight of L1 regularization term. + /// Weight of L2 regularization term. + /// Memory size for . Low=faster, less accurate. /// Threshold for optimizer convergence. /// /// @@ -465,15 +465,15 @@ public static LogisticRegression LogisticRegression(this BinaryClassificationCat string labelColumnName = DefaultColumnNames.Label, string featureColumnName = DefaultColumnNames.Features, string exampleWeightColumnName = null, - float l1Weight = LROptions.Defaults.L1Regularization, - float l2Weight = LROptions.Defaults.L2Regularization, + float l1Regularization = LROptions.Defaults.L1Regularization, + float l2Regularization = LROptions.Defaults.L2Regularization, float optimizationTolerance = LROptions.Defaults.OptimizationTolerance, - int memorySize = LROptions.Defaults.IterationsToRemember, - bool enforceNoNegativity = LROptions.Defaults.EnforceNonNegativity) + int iterationsToRemember = LROptions.Defaults.IterationsToRemember, + bool enforceNonNegativity = LROptions.Defaults.EnforceNonNegativity) { Contracts.CheckValue(catalog, nameof(catalog)); var env = CatalogUtils.GetEnvironment(catalog); - return new LogisticRegression(env, labelColumnName, featureColumnName, exampleWeightColumnName, l1Weight, l2Weight, optimizationTolerance, memorySize, enforceNoNegativity); + return new LogisticRegression(env, labelColumnName, featureColumnName, exampleWeightColumnName, l1Regularization, l2Regularization, optimizationTolerance, iterationsToRemember, enforceNonNegativity); } /// @@ -497,24 +497,24 @@ public static LogisticRegression LogisticRegression(this BinaryClassificationCat /// The name of the label column. /// The name of the feature column. /// The name of the example weight column (optional). - /// Weight of L1 regularization term. - /// Weight of L2 regularization term. + /// Weight of L1 regularization term. + /// Weight of L2 regularization term. /// Threshold for optimizer convergence. - /// Memory size for . Low=faster, less accurate. - /// Enforce non-negative weights. + /// Memory size for . Low=faster, less accurate. + /// Enforce non-negative weights. public static PoissonRegression PoissonRegression(this RegressionCatalog.RegressionTrainers catalog, string labelColumnName = DefaultColumnNames.Label, string featureColumnName = DefaultColumnNames.Features, string exampleWeightColumnName = null, - float l1Weight = LROptions.Defaults.L1Regularization, - float l2Weight = LROptions.Defaults.L2Regularization, + float l1Regularization = LROptions.Defaults.L1Regularization, + float l2Regularization = LROptions.Defaults.L2Regularization, float optimizationTolerance = LROptions.Defaults.OptimizationTolerance, - int memorySize = LROptions.Defaults.IterationsToRemember, - bool enforceNoNegativity = LROptions.Defaults.EnforceNonNegativity) + int iterationsToRemember = LROptions.Defaults.IterationsToRemember, + bool enforceNonNegativity = LROptions.Defaults.EnforceNonNegativity) { Contracts.CheckValue(catalog, nameof(catalog)); var env = CatalogUtils.GetEnvironment(catalog); - return new PoissonRegression(env, labelColumnName, featureColumnName, exampleWeightColumnName, l1Weight, l2Weight, optimizationTolerance, memorySize, enforceNoNegativity); + return new PoissonRegression(env, labelColumnName, featureColumnName, exampleWeightColumnName, l1Regularization, l2Regularization, optimizationTolerance, iterationsToRemember, enforceNonNegativity); } /// @@ -538,24 +538,24 @@ public static PoissonRegression PoissonRegression(this RegressionCatalog.Regress /// The name of the label column. /// The name of the feature column. /// The name of the example weight column (optional). - /// Enforce non-negative weights. - /// Weight of L1 regularization term. - /// Weight of L2 regularization term. - /// Memory size for . Low=faster, less accurate. + /// Enforce non-negative weights. + /// Weight of L1 regularization term. + /// Weight of L2 regularization term. + /// Memory size for . Low=faster, less accurate. /// Threshold for optimizer convergence. public static MulticlassLogisticRegression LogisticRegression(this MulticlassClassificationCatalog.MulticlassClassificationTrainers catalog, string labelColumnName = DefaultColumnNames.Label, string featureColumnName = DefaultColumnNames.Features, string exampleWeightColumnName = null, - float l1Weight = LROptions.Defaults.L1Regularization, - float l2Weight = LROptions.Defaults.L2Regularization, + float l1Regularization = LROptions.Defaults.L1Regularization, + float l2Regularization = LROptions.Defaults.L2Regularization, float optimizationTolerance = LROptions.Defaults.OptimizationTolerance, - int memorySize = LROptions.Defaults.IterationsToRemember, - bool enforceNoNegativity = LROptions.Defaults.EnforceNonNegativity) + int iterationsToRemember = LROptions.Defaults.IterationsToRemember, + bool enforceNonNegativity = LROptions.Defaults.EnforceNonNegativity) { Contracts.CheckValue(catalog, nameof(catalog)); var env = CatalogUtils.GetEnvironment(catalog); - return new MulticlassLogisticRegression(env, labelColumnName, featureColumnName, exampleWeightColumnName, l1Weight, l2Weight, optimizationTolerance, memorySize, enforceNoNegativity); + return new MulticlassLogisticRegression(env, labelColumnName, featureColumnName, exampleWeightColumnName, l1Regularization, l2Regularization, optimizationTolerance, iterationsToRemember, enforceNonNegativity); } /// diff --git a/src/Microsoft.ML.StaticPipe/LbfgsStatic.cs b/src/Microsoft.ML.StaticPipe/LbfgsStatic.cs index 33149b7645..f2fe56401a 100644 --- a/src/Microsoft.ML.StaticPipe/LbfgsStatic.cs +++ b/src/Microsoft.ML.StaticPipe/LbfgsStatic.cs @@ -22,10 +22,10 @@ public static class LbfgsBinaryClassificationStaticExtensions /// The label, or dependent variable. /// The features, or independent variables. /// The optional example weights. - /// Enforce non-negative weights. - /// Weight of L1 regularization term. - /// Weight of L2 regularization term. - /// Memory size for . Low=faster, less accurate. + /// Enforce non-negative weights. + /// Weight of L1 regularization term. + /// Weight of L2 regularization term. + /// Memory size for . Low=faster, less accurate. /// Threshold for optimizer convergence. /// A delegate that is called every time the /// method is called on the @@ -37,20 +37,20 @@ public static (Scalar score, Scalar probability, Scalar pred Scalar label, Vector features, Scalar weights = null, - float l1Weight = Options.Defaults.L1Regularization, - float l2Weight = Options.Defaults.L2Regularization, + float l1Regularization = Options.Defaults.L1Regularization, + float l2Regularization = Options.Defaults.L2Regularization, float optimizationTolerance = Options.Defaults.OptimizationTolerance, - int memorySize = Options.Defaults.IterationsToRemember, - bool enoforceNoNegativity = Options.Defaults.EnforceNonNegativity, + int iterationsToRemember = Options.Defaults.IterationsToRemember, + bool enforceNonNegativity = Options.Defaults.EnforceNonNegativity, Action> onFit = null) { - LbfgsStaticUtils.ValidateParams(label, features, weights, l1Weight, l2Weight, optimizationTolerance, memorySize, enoforceNoNegativity, onFit); + LbfgsStaticUtils.ValidateParams(label, features, weights, l1Regularization, l2Regularization, optimizationTolerance, iterationsToRemember, enforceNonNegativity, onFit); var rec = new TrainerEstimatorReconciler.BinaryClassifier( (env, labelName, featuresName, weightsName) => { var trainer = new LogisticRegression(env, labelName, featuresName, weightsName, - l1Weight, l2Weight, optimizationTolerance, memorySize, enoforceNoNegativity); + l1Regularization, l2Regularization, optimizationTolerance, iterationsToRemember, enforceNonNegativity); if (onFit != null) return trainer.WithOnFitDelegate(trans => onFit(trans.Model)); @@ -118,10 +118,10 @@ public static class LbfgsRegressionExtensions /// The label, or dependent variable. /// The features, or independent variables. /// The optional example weights. - /// Enforce non-negative weights. - /// Weight of L1 regularization term. - /// Weight of L2 regularization term. - /// Memory size for . Low=faster, less accurate. + /// Enforce non-negative weights. + /// Weight of L1 regularization term. + /// Weight of L2 regularization term. + /// Memory size for . Low=faster, less accurate. /// Threshold for optimizer convergence. /// A delegate that is called every time the /// method is called on the @@ -133,20 +133,20 @@ public static Scalar PoissonRegression(this RegressionCatalog.RegressionT Scalar label, Vector features, Scalar weights = null, - float l1Weight = Options.Defaults.L1Regularization, - float l2Weight = Options.Defaults.L2Regularization, + float l1Regularization = Options.Defaults.L1Regularization, + float l2Regularization = Options.Defaults.L2Regularization, float optimizationTolerance = Options.Defaults.OptimizationTolerance, - int memorySize = Options.Defaults.IterationsToRemember, - bool enoforceNoNegativity = Options.Defaults.EnforceNonNegativity, + int iterationsToRemember = Options.Defaults.IterationsToRemember, + bool enforceNonNegativity = Options.Defaults.EnforceNonNegativity, Action onFit = null) { - LbfgsStaticUtils.ValidateParams(label, features, weights, l1Weight, l2Weight, optimizationTolerance, memorySize, enoforceNoNegativity, onFit); + LbfgsStaticUtils.ValidateParams(label, features, weights, l1Regularization, l2Regularization, optimizationTolerance, iterationsToRemember, enforceNonNegativity, onFit); var rec = new TrainerEstimatorReconciler.Regression( (env, labelName, featuresName, weightsName) => { var trainer = new PoissonRegression(env, labelName, featuresName, weightsName, - l1Weight, l2Weight, optimizationTolerance, memorySize, enoforceNoNegativity); + l1Regularization, l2Regularization, optimizationTolerance, iterationsToRemember, enforceNonNegativity); if (onFit != null) return trainer.WithOnFitDelegate(trans => onFit(trans.Model)); @@ -214,10 +214,10 @@ public static class LbfgsMulticlassExtensions /// The label, or dependent variable. /// The features, or independent variables. /// The optional example weights. - /// Enforce non-negative weights. - /// Weight of L1 regularization term. - /// Weight of L2 regularization term. - /// Memory size for . Low=faster, less accurate. + /// Enforce non-negative weights. + /// Weight of L1 regularization term. + /// Weight of L2 regularization term. + /// Memory size for . Low=faster, less accurate. /// Threshold for optimizer convergence. /// A delegate that is called every time the /// method is called on the @@ -230,20 +230,20 @@ public static (Vector score, Key predictedLabel) Key label, Vector features, Scalar weights = null, - float l1Weight = Options.Defaults.L1Regularization, - float l2Weight = Options.Defaults.L2Regularization, + float l1Regularization = Options.Defaults.L1Regularization, + float l2Regularization = Options.Defaults.L2Regularization, float optimizationTolerance = Options.Defaults.OptimizationTolerance, - int memorySize = Options.Defaults.IterationsToRemember, - bool enoforceNoNegativity = Options.Defaults.EnforceNonNegativity, + int iterationsToRemember = Options.Defaults.IterationsToRemember, + bool enforceNonNegativity = Options.Defaults.EnforceNonNegativity, Action onFit = null) { - LbfgsStaticUtils.ValidateParams(label, features, weights, l1Weight, l2Weight, optimizationTolerance, memorySize, enoforceNoNegativity, onFit); + LbfgsStaticUtils.ValidateParams(label, features, weights, l1Regularization, l2Regularization, optimizationTolerance, iterationsToRemember, enforceNonNegativity, onFit); var rec = new TrainerEstimatorReconciler.MulticlassClassifier( (env, labelName, featuresName, weightsName) => { var trainer = new MulticlassLogisticRegression(env, labelName, featuresName, weightsName, - l1Weight, l2Weight, optimizationTolerance, memorySize, enoforceNoNegativity); + l1Regularization, l2Regularization, optimizationTolerance, iterationsToRemember, enforceNonNegativity); if (onFit != null) return trainer.WithOnFitDelegate(trans => onFit(trans.Model)); @@ -303,19 +303,19 @@ internal static class LbfgsStaticUtils internal static void ValidateParams(PipelineColumn label, Vector features, Scalar weights = null, - float l1Weight = Options.Defaults.L1Regularization, - float l2Weight = Options.Defaults.L2Regularization, + float l1Regularization = Options.Defaults.L1Regularization, + float l2Regularization = Options.Defaults.L2Regularization, float optimizationTolerance = Options.Defaults.OptimizationTolerance, - int memorySize = Options.Defaults.IterationsToRemember, - bool enoforceNoNegativity = Options.Defaults.EnforceNonNegativity, + int iterationsToRemember = Options.Defaults.IterationsToRemember, + bool enforceNonNegativity = Options.Defaults.EnforceNonNegativity, Delegate onFit = null) { Contracts.CheckValue(label, nameof(label)); Contracts.CheckValue(features, nameof(features)); - Contracts.CheckParam(l2Weight >= 0, nameof(l2Weight), "Must be non-negative"); - Contracts.CheckParam(l1Weight >= 0, nameof(l1Weight), "Must be non-negative"); + Contracts.CheckParam(l2Regularization >= 0, nameof(l2Regularization), "Must be non-negative"); + Contracts.CheckParam(l1Regularization >= 0, nameof(l1Regularization), "Must be non-negative"); Contracts.CheckParam(optimizationTolerance > 0, nameof(optimizationTolerance), "Must be positive"); - Contracts.CheckParam(memorySize > 0, nameof(memorySize), "Must be positive"); + Contracts.CheckParam(iterationsToRemember > 0, nameof(iterationsToRemember), "Must be positive"); Contracts.CheckValueOrNull(onFit); } }