diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/SDCARegression.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/SDCARegression.cs
deleted file mode 100644
index bc67d1ee1b..0000000000
--- a/docs/samples/Microsoft.ML.Samples/Dynamic/SDCARegression.cs
+++ /dev/null
@@ -1,43 +0,0 @@
-using System;
-using System.Linq;
-using Microsoft.ML.Data;
-
-namespace Microsoft.ML.Samples.Dynamic
-{
- public static class SDCARegression
- {
- public static void Example()
- {
- // Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging,
- // as well as the source of randomness.
- var ml = new MLContext();
-
- // Get a small dataset as an IEnumerable and convert it to an IDataView.
- var data = SamplesUtils.DatasetUtils.GetInfertData();
- var trainData = ml.Data.LoadFromEnumerable(data);
-
- // Preview of the data.
- //
- // Age Case Education Induced Parity PooledStratum RowNum ...
- // 26 1 0-5yrs 1 6 3 1 ...
- // 42 1 0-5yrs 1 1 1 2 ...
- // 39 1 0-5yrs 2 6 4 3 ...
- // 34 1 0-5yrs 2 4 2 4 ...
- // 35 1 6-11yrs 1 3 32 5 ...
-
- // A pipeline for concatenating the Parity and Induced columns together in the Features column.
- // We will train a FastTreeRegression model with 1 tree on these two columns to predict Age.
- string outputColumnName = "Features";
- var pipeline = ml.Transforms.Concatenate(outputColumnName, new[] { "Parity", "Induced" })
- .Append(ml.Regression.Trainers.StochasticDualCoordinateAscent(labelColumnName: "Age", featureColumnName: outputColumnName, maxIterations:2));
-
- var model = pipeline.Fit(trainData);
-
- // Get the trained model parameters.
- var modelParams = model.LastTransformer.Model;
- // Inspect the bias and model weights.
- Console.WriteLine("The bias term is: " + modelParams.Bias);
- Console.WriteLine("The feature weights are: " + string.Join(", ", modelParams.Weights.ToArray()));
- }
- }
-}
diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/SDCALogisticRegression.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/StochasticDualCoordinateAscent.cs
similarity index 98%
rename from docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/SDCALogisticRegression.cs
rename to docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/StochasticDualCoordinateAscent.cs
index e5a01cd4f7..5e8e0fcc1f 100644
--- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/SDCALogisticRegression.cs
+++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/StochasticDualCoordinateAscent.cs
@@ -5,7 +5,7 @@
namespace Microsoft.ML.Samples.Dynamic.Trainers.BinaryClassification
{
- public static class SDCALogisticRegression
+ public static class StochasticDualCoordinateAscent
{
public static void Example()
{
diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/SDCASupportVectorMachine.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/StochasticDualCoordinateAscentNonCalibrated.cs
similarity index 97%
rename from docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/SDCASupportVectorMachine.cs
rename to docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/StochasticDualCoordinateAscentNonCalibrated.cs
index e0d331306b..9a7892fdb9 100644
--- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/SDCASupportVectorMachine.cs
+++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/StochasticDualCoordinateAscentNonCalibrated.cs
@@ -4,7 +4,7 @@
namespace Microsoft.ML.Samples.Dynamic.Trainers.BinaryClassification
{
- public static class SDCASupportVectorMachine
+ public static class StochasticDualCoordinateAscentNonCalibrated
{
public static void Example()
{
diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/StochasticDualCoordinateAscentWithOptions.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/StochasticDualCoordinateAscentWithOptions.cs
new file mode 100644
index 0000000000..db263ea96a
--- /dev/null
+++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/StochasticDualCoordinateAscentWithOptions.cs
@@ -0,0 +1,59 @@
+using Microsoft.ML;
+using Microsoft.ML.Trainers;
+
+namespace Microsoft.ML.Samples.Dynamic.Trainers.BinaryClassification
+{
+ public static class StochasticDualCoordinateAscentWithOptions
+ {
+ // In this examples we will use the adult income dataset. The goal is to predict
+ // if a person's income is above $50K or not, based on demographic information about that person.
+ // For more details about this dataset, please see https://archive.ics.uci.edu/ml/datasets/adult.
+ public static void Example()
+ {
+ // Create a new context for ML.NET operations. It can be used for exception tracking and logging,
+ // as a catalog of available operations and as the source of randomness.
+ // Setting the seed to a fixed number in this example to make outputs deterministic.
+ var mlContext = new MLContext(seed: 0);
+
+ // Download and featurize the dataset.
+ var data = SamplesUtils.DatasetUtils.LoadFeaturizedAdultDataset(mlContext);
+
+ // Leave out 10% of data for testing.
+ var trainTestData = mlContext.BinaryClassification.TrainTestSplit(data, testFraction: 0.1);
+
+ // Define the trainer options.
+ var options = new SdcaBinaryTrainer.Options()
+ {
+ // Make the convergence tolerance tighter.
+ ConvergenceTolerance = 0.05f,
+ // Increase the maximum number of passes over training data.
+ MaxIterations = 30,
+ // Give the instances of the positive class slightly more weight.
+ PositiveInstanceWeight = 1.2f,
+ };
+
+ // Create data training pipeline.
+ var pipeline = mlContext.BinaryClassification.Trainers.StochasticDualCoordinateAscent(options);
+
+ // Fit this pipeline to the training data.
+ var model = pipeline.Fit(trainTestData.TrainSet);
+
+ // Evaluate how the model is doing on the test data.
+ var dataWithPredictions = model.Transform(trainTestData.TestSet);
+ var metrics = mlContext.BinaryClassification.Evaluate(dataWithPredictions);
+ SamplesUtils.ConsoleUtils.PrintMetrics(metrics);
+
+ // Expected output:
+ // Accuracy: 0.85
+ // AUC: 0.90
+ // F1 Score: 0.66
+ // Negative Precision: 0.89
+ // Negative Recall: 0.92
+ // Positive Precision: 0.70
+ // Positive Recall: 0.63
+ // LogLoss: 0.47
+ // LogLossReduction: 39.77
+ // Entropy: 0.78
+ }
+ }
+}
\ No newline at end of file
diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/LightGbm.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/LightGbm.cs
index f1822b69be..399ddda16f 100644
--- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/LightGbm.cs
+++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/LightGbm.cs
@@ -5,7 +5,7 @@
namespace Microsoft.ML.Samples.Dynamic.Trainers.MulticlassClassification
{
- class LightGbm
+ public static class LightGbm
{
// This example requires installation of additional nuget package Microsoft.ML.LightGBM.
public static void Example()
@@ -14,10 +14,10 @@ public static void Example()
// as a catalog of available operations and as the source of randomness.
var mlContext = new MLContext();
- // Create in-memory examples as C# native class.
+ // Create a list of data examples.
var examples = DatasetUtils.GenerateRandomMulticlassClassificationExamples(1000);
- // Convert native C# class to IDataView, a consumble format to ML.NET functions.
+ // Convert the examples list to an IDataView object, which is consumable by ML.NET API.
var dataView = mlContext.Data.LoadFromEnumerable(examples);
//////////////////// Data Preview ////////////////////
diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/LightGbmWithOptions.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/LightGbmWithOptions.cs
index 4971fe6180..53a2db738d 100644
--- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/LightGbmWithOptions.cs
+++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/LightGbmWithOptions.cs
@@ -7,7 +7,7 @@
namespace Microsoft.ML.Samples.Dynamic.Trainers.MulticlassClassification
{
- class LightGbmWithOptions
+ public static class LightGbmWithOptions
{
// This example requires installation of additional nuget package Microsoft.ML.LightGBM.
public static void Example()
@@ -16,10 +16,10 @@ public static void Example()
// as a catalog of available operations and as the source of randomness.
var mlContext = new MLContext(seed: 0);
- // Create in-memory examples as C# native class.
+ // Create a list of data examples.
var examples = DatasetUtils.GenerateRandomMulticlassClassificationExamples(1000);
- // Convert native C# class to IDataView, a consumble format to ML.NET functions.
+ // Convert the examples list to an IDataView object, which is consumable by ML.NET API.
var dataView = mlContext.Data.LoadFromEnumerable(examples);
//////////////////// Data Preview ////////////////////
diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/StochasticDualCoordinateAscent.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/StochasticDualCoordinateAscent.cs
new file mode 100644
index 0000000000..d99d3368d7
--- /dev/null
+++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/StochasticDualCoordinateAscent.cs
@@ -0,0 +1,56 @@
+using Microsoft.ML.Data;
+using Microsoft.ML.SamplesUtils;
+
+namespace Microsoft.ML.Samples.Dynamic.Trainers.MulticlassClassification
+{
+ public static class StochasticDualCoordinateAscent
+ {
+ public static void Example()
+ {
+ // Create a new context for ML.NET operations. It can be used for exception tracking and logging,
+ // as a catalog of available operations and as the source of randomness.
+ // Setting the seed to a fixed number in this example to make outputs deterministic.
+ var mlContext = new MLContext(seed: 0);
+
+ // Create a list of data examples.
+ var examples = DatasetUtils.GenerateRandomMulticlassClassificationExamples(1000);
+
+ // Convert the examples list to an IDataView object, which is consumable by ML.NET API.
+ var dataView = mlContext.Data.LoadFromEnumerable(examples);
+
+ //////////////////// Data Preview ////////////////////
+ // Label Features
+ // AA 0.7262433,0.8173254,0.7680227,0.5581612,0.2060332,0.5588848,0.9060271,0.4421779,0.9775497,0.2737045
+ // BB 0.4919063,0.6673147,0.8326591,0.6695119,1.182151,0.230367,1.06237,1.195347,0.8771811,0.5145918
+ // CC 1.216908,1.248052,1.391902,0.4326252,1.099942,0.9262842,1.334019,1.08762,0.9468155,0.4811099
+ // DD 0.7871246,1.053327,0.8971719,1.588544,1.242697,1.362964,0.6303943,0.9810045,0.9431419,1.557455
+
+ // Create a pipeline.
+ var pipeline =
+ // Convert the string labels into key types.
+ mlContext.Transforms.Conversion.MapValueToKey("Label")
+ // Apply StochasticDualCoordinateAscent multiclass trainer.
+ .Append(mlContext.MulticlassClassification.Trainers.StochasticDualCoordinateAscent());
+
+ // Split the data into training and test sets. Only training set is used in fitting
+ // the created pipeline. Metrics are computed on the test.
+ var split = mlContext.MulticlassClassification.TrainTestSplit(dataView, testFraction: 0.1);
+
+ // Train the model.
+ var model = pipeline.Fit(split.TrainSet);
+
+ // Do prediction on the test set.
+ var dataWithPredictions = model.Transform(split.TestSet);
+
+ // Evaluate the trained model using the test set.
+ var metrics = mlContext.MulticlassClassification.Evaluate(dataWithPredictions);
+ SamplesUtils.ConsoleUtils.PrintMetrics(metrics);
+
+ // Expected output:
+ // Micro Accuracy: 0.82
+ // Macro Accuracy: 0.81
+ // Log Loss: 0.43
+ // Log Loss Reduction: 67.93
+ }
+ }
+}
diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/StochasticDualCoordinateAscentWithOptions.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/StochasticDualCoordinateAscentWithOptions.cs
new file mode 100644
index 0000000000..fe54dc2728
--- /dev/null
+++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/StochasticDualCoordinateAscentWithOptions.cs
@@ -0,0 +1,67 @@
+using Microsoft.ML.Data;
+using Microsoft.ML.SamplesUtils;
+using Microsoft.ML.Trainers;
+
+namespace Microsoft.ML.Samples.Dynamic.Trainers.MulticlassClassification
+{
+ public static class StochasticDualCoordinateAscentWithOptions
+ {
+ public static void Example()
+ {
+ // Create a new context for ML.NET operations. It can be used for exception tracking and logging,
+ // as a catalog of available operations and as the source of randomness.
+ // Setting the seed to a fixed number in this example to make outputs deterministic.
+ var mlContext = new MLContext(seed: 0);
+
+ // Create a list of data examples.
+ var examples = DatasetUtils.GenerateRandomMulticlassClassificationExamples(1000);
+
+ // Convert the examples list to an IDataView object, which is consumable by ML.NET API.
+ var dataView = mlContext.Data.LoadFromEnumerable(examples);
+
+ //////////////////// Data Preview ////////////////////
+ // Label Features
+ // AA 0.7262433,0.8173254,0.7680227,0.5581612,0.2060332,0.5588848,0.9060271,0.4421779,0.9775497,0.2737045
+ // BB 0.4919063,0.6673147,0.8326591,0.6695119,1.182151,0.230367,1.06237,1.195347,0.8771811,0.5145918
+ // CC 1.216908,1.248052,1.391902,0.4326252,1.099942,0.9262842,1.334019,1.08762,0.9468155,0.4811099
+ // DD 0.7871246,1.053327,0.8971719,1.588544,1.242697,1.362964,0.6303943,0.9810045,0.9431419,1.557455
+
+ var options = new SdcaMultiClassTrainer.Options
+ {
+ // Add custom loss
+ LossFunction = new HingeLoss.Options(),
+ // Make the convergence tolerance tighter.
+ ConvergenceTolerance = 0.05f,
+ // Increase the maximum number of passes over training data.
+ MaxIterations = 30,
+ };
+
+ // Create a pipeline.
+ var pipeline =
+ // Convert the string labels into key types.
+ mlContext.Transforms.Conversion.MapValueToKey("Label")
+ // Apply StochasticDualCoordinateAscent multiclass trainer.
+ .Append(mlContext.MulticlassClassification.Trainers.StochasticDualCoordinateAscent(options));
+
+ // Split the data into training and test sets. Only training set is used in fitting
+ // the created pipeline. Metrics are computed on the test.
+ var split = mlContext.MulticlassClassification.TrainTestSplit(dataView, testFraction: 0.1);
+
+ // Train the model.
+ var model = pipeline.Fit(split.TrainSet);
+
+ // Do prediction on the test set.
+ var dataWithPredictions = model.Transform(split.TestSet);
+
+ // Evaluate the trained model using the test set.
+ var metrics = mlContext.MulticlassClassification.Evaluate(dataWithPredictions);
+ SamplesUtils.ConsoleUtils.PrintMetrics(metrics);
+
+ // Expected output:
+ // Micro Accuracy: 0.82
+ // Macro Accuracy: 0.81
+ // Log Loss: 0.64
+ // Log Loss Reduction: 52.51
+ }
+ }
+}
diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/StochasticDualCoordinateAscent.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/StochasticDualCoordinateAscent.cs
new file mode 100644
index 0000000000..7ce8122f78
--- /dev/null
+++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/StochasticDualCoordinateAscent.cs
@@ -0,0 +1,43 @@
+using System;
+using System.Linq;
+using Microsoft.ML.Data;
+
+namespace Microsoft.ML.Samples.Dynamic.Trainers.Regression
+{
+ public static class StochasticDualCoordinateAscent
+ {
+ public static void Example()
+ {
+ // Create a new context for ML.NET operations. It can be used for exception tracking and logging,
+ // as a catalog of available operations and as the source of randomness.
+ // Setting the seed to a fixed number in this example to make outputs deterministic.
+ var mlContext = new MLContext(seed: 0);
+
+ // Create in-memory examples as C# native class and convert to IDataView
+ var data = SamplesUtils.DatasetUtils.GenerateFloatLabelFloatFeatureVectorSamples(1000);
+ var dataView = mlContext.Data.LoadFromEnumerable(data);
+
+ // Split the data into training and test sets. Only training set is used in fitting
+ // the created pipeline. Metrics are computed on the test.
+ var split = mlContext.MulticlassClassification.TrainTestSplit(dataView, testFraction: 0.1);
+
+ // Train the model.
+ var pipeline = mlContext.Regression.Trainers.StochasticDualCoordinateAscent();
+ var model = pipeline.Fit(split.TrainSet);
+
+ // Do prediction on the test set.
+ var dataWithPredictions = model.Transform(split.TestSet);
+
+ // Evaluate the trained model using the test set.
+ var metrics = mlContext.Regression.Evaluate(dataWithPredictions);
+ SamplesUtils.ConsoleUtils.PrintMetrics(metrics);
+
+ // Expected output:
+ // L1: 0.27
+ // L2: 0.11
+ // LossFunction: 0.11
+ // RMS: 0.33
+ // RSquared: 0.56
+ }
+ }
+}
diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/StochasticDualCoordinateAscentWithOptions.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/StochasticDualCoordinateAscentWithOptions.cs
new file mode 100644
index 0000000000..52c765d847
--- /dev/null
+++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/StochasticDualCoordinateAscentWithOptions.cs
@@ -0,0 +1,53 @@
+using Microsoft.ML.Data;
+using Microsoft.ML.Trainers;
+
+namespace Microsoft.ML.Samples.Dynamic.Trainers.Regression
+{
+ public static class StochasticDualCoordinateAscentWithOptions
+ {
+ public static void Example()
+ {
+ // Create a new context for ML.NET operations. It can be used for exception tracking and logging,
+ // as a catalog of available operations and as the source of randomness.
+ // Setting the seed to a fixed number in this example to make outputs deterministic.
+ var mlContext = new MLContext(seed: 0);
+
+ // Create in-memory examples as C# native class and convert to IDataView
+ var data = SamplesUtils.DatasetUtils.GenerateFloatLabelFloatFeatureVectorSamples(1000);
+ var dataView = mlContext.Data.LoadFromEnumerable(data);
+
+ // Split the data into training and test sets. Only training set is used in fitting
+ // the created pipeline. Metrics are computed on the test.
+ var split = mlContext.MulticlassClassification.TrainTestSplit(dataView, testFraction: 0.1);
+
+ // Create trainer options.
+ var options = new SdcaRegressionTrainer.Options
+ {
+ // Make the convergence tolerance tighter.
+ ConvergenceTolerance = 0.02f,
+ // Increase the maximum number of passes over training data.
+ MaxIterations = 30,
+ // Increase learning rate for bias
+ BiasLearningRate = 0.1f
+ };
+
+ // Train the model.
+ var pipeline = mlContext.Regression.Trainers.StochasticDualCoordinateAscent(options);
+ var model = pipeline.Fit(split.TrainSet);
+
+ // Do prediction on the test set.
+ var dataWithPredictions = model.Transform(split.TestSet);
+
+ // Evaluate the trained model using the test set.
+ var metrics = mlContext.Regression.Evaluate(dataWithPredictions);
+ SamplesUtils.ConsoleUtils.PrintMetrics(metrics);
+
+ // Expected output:
+ // L1: 0.26
+ // L2: 0.11
+ // LossFunction: 0.11
+ // RMS: 0.33
+ // RSquared: 0.56
+ }
+ }
+}
diff --git a/src/Microsoft.ML.SamplesUtils/ConsoleUtils.cs b/src/Microsoft.ML.SamplesUtils/ConsoleUtils.cs
index 2d868895b1..e3e34c3757 100644
--- a/src/Microsoft.ML.SamplesUtils/ConsoleUtils.cs
+++ b/src/Microsoft.ML.SamplesUtils/ConsoleUtils.cs
@@ -31,21 +31,32 @@ public static void PrintMetrics(BinaryClassificationMetrics metrics)
public static void PrintMetrics(CalibratedBinaryClassificationMetrics metrics)
{
PrintMetrics(metrics as BinaryClassificationMetrics);
- Console.WriteLine($"LogLoss: {metrics.LogLoss:F2}");
- Console.WriteLine($"LogLossReduction: {metrics.LogLossReduction:F2}");
+ Console.WriteLine($"Log Loss: {metrics.LogLoss:F2}");
+ Console.WriteLine($"Log Loss Reduction: {metrics.LogLossReduction:F2}");
Console.WriteLine($"Entropy: {metrics.Entropy:F2}");
}
+ ///
+ /// Pretty-print MultiClassClassifierMetrics objects.
+ ///
+ /// object.
+ public static void PrintMetrics(MultiClassClassifierMetrics metrics)
+ {
+ Console.WriteLine($"Micro Accuracy: {metrics.MicroAccuracy:F2}");
+ Console.WriteLine($"Macro Accuracy: {metrics.MacroAccuracy:F2}");
+ Console.WriteLine($"Log Loss: {metrics.LogLoss:F2}");
+ Console.WriteLine($"Log Loss Reduction: {metrics.LogLossReduction:F2}");
+ }
+
///
/// Pretty-print RegressionMetrics objects.
///
/// Regression metrics.
public static void PrintMetrics(RegressionMetrics metrics)
{
- Console.WriteLine($"L1: {metrics.MeanAbsoluteError:F2}");
- Console.WriteLine($"L2: {metrics.MeanSquaredError:F2}");
- Console.WriteLine($"LossFunction: {metrics.LossFunction:F2}");
- Console.WriteLine($"RMS: {metrics.RootMeanSquaredError:F2}");
+ Console.WriteLine($"Mean Absolute Error: {metrics.MeanAbsoluteError:F2}");
+ Console.WriteLine($"Mean Square dError: {metrics.MeanSquaredError:F2}");
+ Console.WriteLine($"Root Mean Squared Error: {metrics.RootMeanSquaredError:F2}");
Console.WriteLine($"RSquared: {metrics.RSquared:F2}");
}
diff --git a/src/Microsoft.ML.SamplesUtils/SamplesDatasetUtils.cs b/src/Microsoft.ML.SamplesUtils/SamplesDatasetUtils.cs
index 503db348ab..9f05a71ac8 100644
--- a/src/Microsoft.ML.SamplesUtils/SamplesDatasetUtils.cs
+++ b/src/Microsoft.ML.SamplesUtils/SamplesDatasetUtils.cs
@@ -677,7 +677,7 @@ public MulticlassClassificationExample()
}
///
- /// Helper function used to generate random s.
+ /// Helper function used to generate random objects.
///
/// Number of generated examples.
/// A list of random examples.
diff --git a/src/Microsoft.ML.StandardLearners/Standard/SdcaBinary.cs b/src/Microsoft.ML.StandardLearners/Standard/SdcaBinary.cs
index 9c1c628101..0a3305e409 100644
--- a/src/Microsoft.ML.StandardLearners/Standard/SdcaBinary.cs
+++ b/src/Microsoft.ML.StandardLearners/Standard/SdcaBinary.cs
@@ -157,40 +157,80 @@ public abstract class SdcaTrainerBase : Stochast
// 3. Don't "guess" the iteration to converge. It is very data-set dependent and hard to control. Always check for at least once to ensure convergence.
// 4. Use dual variable updates to infer whether a full iteration of convergence checking is necessary. Convergence checking iteration is time-consuming.
+ ///
+ /// Options for the SDCA-based trainers.
+ ///
public abstract class OptionsBase : TrainerInputBaseWithLabel
{
+ ///
+ /// The L2 regularization hyperparameter.
+ ///
[Argument(ArgumentType.AtMostOnce, HelpText = "L2 regularizer constant. By default the l2 constant is automatically inferred based on data set.", NullName = "", ShortName = "l2", SortOrder = 1)]
[TGUI(Label = "L2 Regularizer Constant", SuggestedSweeps = ",1e-7,1e-6,1e-5,1e-4,1e-3,1e-2")]
[TlcModule.SweepableDiscreteParam("L2Const", new object[] { "", 1e-7f, 1e-6f, 1e-5f, 1e-4f, 1e-3f, 1e-2f })]
public float? L2Const;
// REVIEW: make the default positive when we know how to consume a sparse model
+ ///
+ /// The L1 regularization hyperparameter.
+ ///
[Argument(ArgumentType.AtMostOnce, HelpText = "L1 soft threshold (L1/L2). Note that it is easier to control and sweep using the threshold parameter than the raw L1-regularizer constant. By default the l1 threshold is automatically inferred based on data set.", NullName = "", ShortName = "l1", SortOrder = 2)]
[TGUI(Label = "L1 Soft Threshold", SuggestedSweeps = ",0,0.25,0.5,0.75,1")]
[TlcModule.SweepableDiscreteParam("L1Threshold", new object[] { "", 0f, 0.25f, 0.5f, 0.75f, 1f })]
public float? L1Threshold;
+ ///
+ /// The degree of lock-free parallelism.
+ ///
+ ///
+ /// Defaults to automatic depending on data sparseness. Determinism is not guaranteed.
+ ///
[Argument(ArgumentType.AtMostOnce, HelpText = "Degree of lock-free parallelism. Defaults to automatic. Determinism not guaranteed.", NullName = "", ShortName = "nt,t,threads", SortOrder = 50)]
[TGUI(Label = "Number of threads", SuggestedSweeps = ",1,2,4")]
public int? NumThreads;
+ ///
+ /// The tolerance for the ratio between duality gap and primal loss for convergence checking.
+ ///
[Argument(ArgumentType.AtMostOnce, HelpText = "The tolerance for the ratio between duality gap and primal loss for convergence checking.", ShortName = "tol")]
[TGUI(SuggestedSweeps = "0.001, 0.01, 0.1, 0.2")]
[TlcModule.SweepableDiscreteParam("ConvergenceTolerance", new object[] { 0.001f, 0.01f, 0.1f, 0.2f })]
public float ConvergenceTolerance = 0.1f;
+ ///
+ /// The maximum number of passes to perform over the data.
+ ///
+ ///
+ /// Set to 1 to simulate online learning. Defaults to automatic.
+ ///
[Argument(ArgumentType.AtMostOnce, HelpText = "Maximum number of iterations; set to 1 to simulate online learning. Defaults to automatic.", NullName = "", ShortName = "iter")]
[TGUI(Label = "Max number of iterations", SuggestedSweeps = ",10,20,100")]
[TlcModule.SweepableDiscreteParam("MaxIterations", new object[] { "", 10, 20, 100 })]
public int? MaxIterations;
+ ///
+ /// Determines whether to shuffle data for each training iteration.
+ ///
+ ///
+ /// to shuffle data for each training iteration; otherwise, .
+ /// Default is .
+ ///
[Argument(ArgumentType.AtMostOnce, HelpText = "Shuffle data every epoch?", ShortName = "shuf")]
[TlcModule.SweepableDiscreteParam("Shuffle", null, isBool: true)]
public bool Shuffle = true;
+ ///
+ /// Determines the frequency of checking for convergence in terms of number of iterations.
+ ///
+ ///
+ /// Set to zero or negative value to disable checking. If , it defaults to ."
+ ///
[Argument(ArgumentType.AtMostOnce, HelpText = "Convergence check frequency (in terms of number of iterations). Set as negative or zero for not checking at all. If left blank, it defaults to check after every 'numThreads' iterations.", NullName = "", ShortName = "checkFreq")]
public int? CheckFrequency;
+ ///
+ /// The learning rate for adjusting bias from being regularized.
+ ///
[Argument(ArgumentType.AtMostOnce, HelpText = "The learning rate for adjusting bias from being regularized.", ShortName = "blr")]
[TGUI(SuggestedSweeps = "0, 0.01, 0.1, 1")]
[TlcModule.SweepableDiscreteParam("BiasLearningRate", new object[] { 0.0f, 0.01f, 0.1f, 1f })]
@@ -1419,8 +1459,17 @@ public abstract class SdcaBinaryTrainerBase :
public override TrainerInfo Info { get; }
+ ///
+ /// Options base class for binary SDCA trainers.
+ ///
public class BinaryOptionsBase : OptionsBase
{
+ ///
+ /// The weight to be applied to the positive class. This is useful for training with imbalanced data.
+ ///
+ ///
+ /// Default value is 1, which means no extra weight.
+ ///
[Argument(ArgumentType.AtMostOnce, HelpText = "Apply weight to the positive class, for imbalanced data", ShortName = "piw")]
public float PositiveInstanceWeight = 1;
@@ -1517,11 +1566,17 @@ private protected override BinaryPredictionTransformer MakeTra
=> new BinaryPredictionTransformer(Host, model, trainSchema, FeatureColumn.Name);
}
+ ///
+ /// The for training a binary logistic regression classification model using the stochastic dual coordinate ascent method.
+ /// The trained model is calibrated and can produce probability by feeding the output value of the
+ /// linear function to a .
+ ///
+ ///
public sealed class SdcaBinaryTrainer :
SdcaBinaryTrainerBase>
{
///
- /// Configuration to training logistic regression using SDCA.
+ /// Options for the .
///
public sealed class Options : BinaryOptionsBase
{
@@ -1577,13 +1632,23 @@ private protected override SchemaShape.Column[] ComputeSdcaBinaryClassifierSchem
}
}
+ ///
+ /// The for training a binary logistic regression classification model using the stochastic dual coordinate ascent method.
+ ///
+ ///
public sealed class SdcaNonCalibratedBinaryTrainer : SdcaBinaryTrainerBase
{
///
- /// General Configuration to training linear model using SDCA.
+ /// Options for the .
///
public sealed class Options : BinaryOptionsBase
{
+ ///
+ /// The custom loss.
+ ///
+ ///
+ /// If unspecified, will be used.
+ ///
[Argument(ArgumentType.Multiple, HelpText = "Loss Function", ShortName = "loss", SortOrder = 50)]
public ISupportSdcaClassificationLossFactory LossFunction = new LogLossFactory();
}
diff --git a/src/Microsoft.ML.StandardLearners/Standard/SdcaMultiClass.cs b/src/Microsoft.ML.StandardLearners/Standard/SdcaMultiClass.cs
index e1c9b195e5..cbb61a2c0b 100644
--- a/src/Microsoft.ML.StandardLearners/Standard/SdcaMultiClass.cs
+++ b/src/Microsoft.ML.StandardLearners/Standard/SdcaMultiClass.cs
@@ -24,8 +24,10 @@
namespace Microsoft.ML.Trainers
{
- // SDCA linear multiclass trainer.
- ///
+ ///
+ /// The for training a multiclass logistic regression classification model using the stochastic dual coordinate ascent method.
+ ///
+ ///
public class SdcaMultiClassTrainer : SdcaTrainerBase, MulticlassLogisticRegressionModelParameters>
{
internal const string LoadNameValue = "SDCAMC";
@@ -33,8 +35,17 @@ public class SdcaMultiClassTrainer : SdcaTrainerBase
+ /// Options for the .
+ ///
public sealed class Options : OptionsBase
{
+ ///
+ /// The custom loss.
+ ///
+ ///
+ /// If unspecified, will be used.
+ ///
[Argument(ArgumentType.Multiple, HelpText = "Loss Function", ShortName = "loss", SortOrder = 50)]
public ISupportSdcaClassificationLossFactory LossFunction = new LogLossFactory();
}
@@ -200,7 +211,7 @@ private protected override void TrainWithoutLock(IProgressChannelProvider progre
var output = labelOutput + labelPrimalUpdate * normSquared - WDot(in features, in weights[iClass], biasReg[iClass] + biasUnreg[iClass]);
var dualUpdate = _loss.DualUpdate(output, 1, dual, invariant, numThreads);
- // The successive over-relaxation apporach to adjust the sum of dual variables (biasReg) to zero.
+ // The successive over-relaxation approach to adjust the sum of dual variables (biasReg) to zero.
// Reference to details: http://stat.rutgers.edu/home/tzhang/papers/ml02_dual.pdf, pp. 16-17.
var adjustment = l1ThresholdZero ? lr * biasReg[iClass] : lr * l1IntermediateBias[iClass];
dualUpdate -= adjustment;
diff --git a/src/Microsoft.ML.StandardLearners/Standard/SdcaRegression.cs b/src/Microsoft.ML.StandardLearners/Standard/SdcaRegression.cs
index 3cb1a42dae..bc373fa3d0 100644
--- a/src/Microsoft.ML.StandardLearners/Standard/SdcaRegression.cs
+++ b/src/Microsoft.ML.StandardLearners/Standard/SdcaRegression.cs
@@ -21,7 +21,10 @@
namespace Microsoft.ML.Trainers
{
- ///
+ ///
+ /// The for training a regression model using the stochastic dual coordinate ascent method.
+ ///
+ ///
public sealed class SdcaRegressionTrainer : SdcaTrainerBase, LinearRegressionModelParameters>
{
internal const string LoadNameValue = "SDCAR";
@@ -29,11 +32,23 @@ public sealed class SdcaRegressionTrainer : SdcaTrainerBase
+ /// Options for the .
+ ///
public sealed class Options : OptionsBase
{
+ ///
+ /// A custom loss.
+ ///
+ ///
+ /// Defaults to
+ ///
[Argument(ArgumentType.Multiple, HelpText = "Loss Function", ShortName = "loss", SortOrder = 50)]
public ISupportSdcaRegressionLossFactory LossFunction = new SquaredLossFactory();
+ ///
+ /// Create the object.
+ ///
public Options()
{
// Using a higher default tolerance for better RMS.
diff --git a/src/Microsoft.ML.StandardLearners/Standard/doc.xml b/src/Microsoft.ML.StandardLearners/Standard/doc.xml
index 17677c9106..7f4ecf31f6 100644
--- a/src/Microsoft.ML.StandardLearners/Standard/doc.xml
+++ b/src/Microsoft.ML.StandardLearners/Standard/doc.xml
@@ -1,13 +1,13 @@
-
-
-
- Train an SDCA linear model.
-
+
+
- This classifier is a trainer based on the Stochastic Dual Coordinate Ascent(SDCA) method, a state-of-the-art optimization technique for convex objective functions.
+ This trainer is based on the Stochastic Dual Coordinate Ascent (SDCA) method, a state-of-the-art optimization technique for convex objective functions.
The algorithm can be scaled for use on large out-of-memory data sets due to a semi-asynchronized implementation that supports multi-threading.
Convergence is underwritten by periodically enforcing synchronization between primal and dual updates in a separate thread.
@@ -32,42 +32,6 @@
-
-
-
- new StochasticDualCoordinateAscentBinaryClassifier
- {
- MaxIterations = 100,
- NumThreads = 7,
- LossFunction = new SmoothedHingeLossSDCAClassificationLossFunction(),
- Caching = Microsoft.ML.Models.CachingOptions.Memory
- }
-
-
-
-
-
-
- new StochasticDualCoordinateAscentClassifier
- {
- MaxIterations = 100,
- NumThreads = 7,
- LossFunction = new SmoothedHingeLossSDCAClassificationLossFunction()
- }
-
-
-
-
-
-
- new StochasticDualCoordinateAscentRegressor
- {
- MaxIterations = 100,
- NumThreads = 5
- }
-
-
-
\ No newline at end of file
diff --git a/src/Microsoft.ML.StandardLearners/StandardLearnersCatalog.cs b/src/Microsoft.ML.StandardLearners/StandardLearnersCatalog.cs
index 5f3329792e..815006cd77 100644
--- a/src/Microsoft.ML.StandardLearners/StandardLearnersCatalog.cs
+++ b/src/Microsoft.ML.StandardLearners/StandardLearnersCatalog.cs
@@ -110,17 +110,22 @@ public static SgdNonCalibratedBinaryTrainer StochasticGradientDescentNonCalibrat
}
///
- /// Predict a target using a linear regression model trained with the SDCA trainer.
+ /// Predict a target using a linear regression model trained with .
///
/// The regression catalog trainer object.
/// The name of the label column.
/// The name of the feature column.
/// The name of the example weight column (optional).
- /// The L2 regularization hyperparameter.
- /// The L1 regularization hyperparameter. Higher values will tend to lead to more sparse model.
+ /// The L2 regularization hyperparameter.
+ /// The L1 regularization hyperparameter. Higher values will tend to lead to more sparse model.
/// The maximum number of passes to perform over the data.
- /// The custom loss, if unspecified will be .
-
+ /// The custom loss, if unspecified will be .
+ ///
+ ///
+ ///
+ ///
public static SdcaRegressionTrainer StochasticDualCoordinateAscent(this RegressionCatalog.RegressionTrainers catalog,
string labelColumnName = DefaultColumnNames.Label,
string featureColumnName = DefaultColumnNames.Features,
@@ -136,12 +141,18 @@ public static SdcaRegressionTrainer StochasticDualCoordinateAscent(this Regressi
}
///
- /// Predict a target using a linear regression model trained with the SDCA trainer.
+ /// Predict a target using a linear regression model trained with and advanced options.
///
/// The regression catalog trainer object.
- /// Advanced arguments to the algorithm.
+ /// Trainer options.
+ ///
+ ///
+ ///
+ ///
public static SdcaRegressionTrainer StochasticDualCoordinateAscent(this RegressionCatalog.RegressionTrainers catalog,
- SdcaRegressionTrainer.Options options)
+ SdcaRegressionTrainer.Options options)
{
Contracts.CheckValue(catalog, nameof(catalog));
Contracts.CheckValue(options, nameof(options));
@@ -151,21 +162,19 @@ public static SdcaRegressionTrainer StochasticDualCoordinateAscent(this Regressi
}
///
- /// Predict a target using a logistic regression model trained with the SDCA trainer.
- /// The trained model can produce probability by feeding the output value of the linear
- /// function to a .
+ /// Predict a target using a linear classification model trained with .
///
/// The binary classification catalog trainer object.
/// The name of the label column.
/// The name of the feature column.
/// The name of the example weight column (optional).
- /// The L2 regularization hyperparameter.
- /// The L1 regularization hyperparameter. Higher values will tend to lead to more sparse model.
+ /// The L2 regularization hyperparameter.
+ /// The L1 regularization hyperparameter. Higher values will tend to lead to more sparse model.
/// The maximum number of passes to perform over the data.
///
///
///
///
public static SdcaBinaryTrainer StochasticDualCoordinateAscent(
@@ -183,13 +192,16 @@ public static SdcaBinaryTrainer StochasticDualCoordinateAscent(
}
///
- /// Predict a target using a logistic regression model trained with the SDCA trainer.
- /// The trained model can produce probability via feeding output value of the linear
- /// function to a . Compared with ,
- /// this function allows more advanced settings by accepting .
+ /// Predict a target using a linear classification model trained with and advanced options.
///
/// The binary classification catalog trainer object.
- /// Advanced arguments to the algorithm.
+ /// Trainer options.
+ ///
+ ///
+ ///
+ ///
public static SdcaBinaryTrainer StochasticDualCoordinateAscent(
this BinaryClassificationCatalog.BinaryClassificationTrainers catalog,
SdcaBinaryTrainer.Options options)
@@ -202,20 +214,20 @@ public static SdcaBinaryTrainer StochasticDualCoordinateAscent(
}
///
- /// Predict a target using a linear binary classification model trained with the SDCA trainer.
+ /// Predict a target using a linear classification model trained with .
///
/// The binary classification catalog trainer object.
/// The name of the label column.
/// The name of the feature column.
/// The name of the example weight column (optional).
- /// The custom loss. Defaults to log-loss if not specified.
- /// The L2 regularization hyperparameter.
- /// The L1 regularization hyperparameter. Higher values will tend to lead to more sparse model.
+ /// The custom loss. Defaults to if not specified.
+ /// The L2 regularization hyperparameter.
+ /// The L1 regularization hyperparameter. Higher values will tend to lead to more sparse model.
/// The maximum number of passes to perform over the data.
///
///
///
///
public static SdcaNonCalibratedBinaryTrainer StochasticDualCoordinateAscentNonCalibrated(
@@ -234,10 +246,10 @@ public static SdcaNonCalibratedBinaryTrainer StochasticDualCoordinateAscentNonCa
}
///
- /// Predict a target using a linear binary classification model trained with the SDCA trainer.
+ /// Predict a target using a linear classification model trained with and advanced options.
///
/// The binary classification catalog trainer object.
- /// Advanced arguments to the algorithm.
+ /// Trainer options.
public static SdcaNonCalibratedBinaryTrainer StochasticDualCoordinateAscentNonCalibrated(
this BinaryClassificationCatalog.BinaryClassificationTrainers catalog,
SdcaNonCalibratedBinaryTrainer.Options options)
@@ -250,18 +262,24 @@ public static SdcaNonCalibratedBinaryTrainer StochasticDualCoordinateAscentNonCa
}
///
- /// Predict a target using a linear multiclass classification model trained with the SDCA trainer.
+ /// Predict a target using a linear multiclass classification model trained with .
///
/// The multiclass classification catalog trainer object.
/// The name of the label column.
/// The name of the feature column.
/// The name of the example weight column (optional).
- /// The optional custom loss.
- /// The L2 regularization hyperparameter.
- /// The L1 regularization hyperparameter. Higher values will tend to lead to more sparse model.
+ /// The custom loss. Defaults to if not specified.
+ /// The L2 regularization hyperparameter.
+ /// The L1 regularization hyperparameter. Higher values will tend to lead to more sparse model.
/// The maximum number of passes to perform over the data.
+ ///
+ ///
+ ///
+ ///
public static SdcaMultiClassTrainer StochasticDualCoordinateAscent(this MulticlassClassificationCatalog.MulticlassClassificationTrainers catalog,
- string labelColumnName = DefaultColumnNames.Label,
+ string labelColumnName = DefaultColumnNames.Label,
string featureColumnName = DefaultColumnNames.Features,
string exampleWeightColumnName = null,
ISupportSdcaClassificationLoss loss = null,
@@ -275,12 +293,18 @@ public static SdcaMultiClassTrainer StochasticDualCoordinateAscent(this Multicla
}
///
- /// Predict a target using a linear multiclass classification model trained with the SDCA trainer.
+ /// Predict a target using a linear multiclass classification model trained with and advanced options.
///
/// The multiclass classification catalog trainer object.
- /// Advanced arguments to the algorithm.
+ /// Trainer options.
+ ///
+ ///
+ ///
+ ///
public static SdcaMultiClassTrainer StochasticDualCoordinateAscent(this MulticlassClassificationCatalog.MulticlassClassificationTrainers catalog,
- SdcaMultiClassTrainer.Options options)
+ SdcaMultiClassTrainer.Options options)
{
Contracts.CheckValue(catalog, nameof(catalog));
Contracts.CheckValue(options, nameof(options));