Skip to content

Fixes onnx exports for binary classification trainers #4463

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Dec 2, 2019
Merged

Fixes onnx exports for binary classification trainers #4463

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
0d49e9c
Squashed commit of the following:
Lynx1820 Nov 14, 2019
edb0167
update name and remove some dependency code
Lynx1820 Nov 14, 2019
b8f694a
baseline file changes
Lynx1820 Nov 15, 2019
2450db0
Refactored function
Lynx1820 Dec 2, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 20 additions & 6 deletions src/Microsoft.ML.Data/Scorers/BinaryClassifierScorer.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,7 @@ private protected override void SaveAsOnnxCore(OnnxContext ctx)
{
Host.CheckValue(ctx, nameof(ctx));
Host.Assert(Bindable is IBindableCanSaveOnnx);
Host.Assert(Bindings.InfoCount >= 2);

if (!ctx.ContainsColumn(DefaultColumnNames.Features))
return;
Expand All @@ -197,15 +198,28 @@ private protected override void SaveAsOnnxCore(OnnxContext ctx)
for (int iinfo = 0; iinfo < Bindings.InfoCount; ++iinfo)
outColumnNames[iinfo] = Bindings.GetColumnName(Bindings.MapIinfoToCol(iinfo));

//Check if "Probability" column was generated by the base class, only then
//label can be predicted.
if (Bindings.InfoCount >= 3 && ctx.ContainsColumn(outColumnNames[2]))
/* If the probability column was generated, then the classification threshold is set to 0.5. Otherwise,
the predicted label is based on the sign of the score.
*/
string opType = "Binarizer";
OnnxNode node;
var binarizerOutput = ctx.AddIntermediateVariable(null, "BinarizerOutput", true);

if (Bindings.InfoCount >= 3)
{
string opType = "Binarizer";
var node = ctx.CreateNode(opType, new[] { ctx.GetVariableName(outColumnNames[2]) },
new[] { ctx.GetVariableName(outColumnNames[0]) }, ctx.GetNodeName(opType));
Host.Assert(ctx.ContainsColumn(outColumnNames[2]));
node = ctx.CreateNode(opType, ctx.GetVariableName(outColumnNames[2]), binarizerOutput, ctx.GetNodeName(opType));
node.AddAttribute("threshold", 0.5);
}
else
{
node = ctx.CreateNode(opType, ctx.GetVariableName(outColumnNames[1]), binarizerOutput, ctx.GetNodeName(opType));
node.AddAttribute("threshold", 0.0);
}
opType = "Cast";
node = ctx.CreateNode(opType, binarizerOutput, ctx.GetVariableName(outColumnNames[0]), ctx.GetNodeName(opType), "");
var t = InternalDataKindExtensions.ToInternalDataKind(DataKind.Boolean).ToType();
node.AddAttribute("to", t);
}

private protected override IDataTransform ApplyToDataCore(IHostEnvironment env, IDataView newSource)
Expand Down
4 changes: 3 additions & 1 deletion src/Microsoft.ML.FastTree/FastTree.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3049,6 +3049,7 @@ private enum AggregateFunction
private protected virtual bool SaveAsOnnx(OnnxContext ctx, string[] outputNames, string featureColumn)
{
Host.CheckValue(ctx, nameof(ctx));
Host.Check(Utils.Size(outputNames) >= 1);

//Nodes.
var nodesTreeids = new List<long>();
Expand Down Expand Up @@ -3111,7 +3112,8 @@ private protected virtual bool SaveAsOnnx(OnnxContext ctx, string[] outputNames,
}

string opType = "TreeEnsembleRegressor";
var node = ctx.CreateNode(opType, new[] { featureColumn }, outputNames, ctx.GetNodeName(opType));
string scoreVarName = (Utils.Size(outputNames) == 2) ? outputNames[1] : outputNames[0]; // Get Score from PredictedLabel and/or Score columns
var node = ctx.CreateNode(opType, new[] { featureColumn }, new[] { scoreVarName }, ctx.GetNodeName(opType));

node.AddAttribute("post_transform", PostTransform.None.GetDescription());
node.AddAttribute("n_targets", 1);
Expand Down
5 changes: 3 additions & 2 deletions src/Microsoft.ML.StandardTrainers/Standard/LinearModelParameters.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,9 +136,10 @@ internal LinearModelParameters(IHostEnvironment env, string name, in VBuffer<flo
private protected virtual bool SaveAsOnnx(OnnxContext ctx, string[] outputs, string featureColumn)
{
Host.CheckValue(ctx, nameof(ctx));
Host.Check(Utils.Size(outputs) == 1);
Host.Check(Utils.Size(outputs) >= 1);
string opType = "LinearRegressor";
var node = ctx.CreateNode(opType, new[] { featureColumn }, outputs, ctx.GetNodeName(opType));
string scoreVarName = (Utils.Size(outputs) == 2) ? outputs[1] : outputs[0]; // Get Score from PredictedLabel and/or Score columns
var node = ctx.CreateNode(opType, new[] { featureColumn }, new[] { scoreVarName }, ctx.GetNodeName(opType));
// Selection of logit or probit output transform. enum {'NONE', 'LOGIT', 'PROBIT}
node.AddAttribute("post_transform", "NONE");
node.AddAttribute("targets", 1);
Expand Down
19 changes: 18 additions & 1 deletion ...Output/Common/Onnx/BinaryClassification/BreastCancer/ExcludeVariablesInOnnxConversion.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -416,7 +416,7 @@
"Probability"
],
"output": [
"PredictedLabel"
"BinarizerOutput"
],
"name": "Binarizer",
"opType": "Binarizer",
Expand All @@ -429,6 +429,23 @@
],
"domain": "ai.onnx.ml"
},
{
"input": [
"BinarizerOutput"
],
"output": [
"PredictedLabel"
],
"name": "Cast1",
"opType": "Cast",
"attribute": [
{
"name": "to",
"i": "9",
"type": "INT"
}
]
},
{
"input": [
"PredictedLabel"
Expand Down
19 changes: 18 additions & 1 deletion test/BaselineOutput/Common/Onnx/BinaryClassification/BreastCancer/ModelWithLessIO.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -742,7 +742,7 @@
"Probability"
],
"output": [
"PredictedLabel"
"BinarizerOutput"
],
"name": "Binarizer",
"opType": "Binarizer",
Expand All @@ -755,6 +755,23 @@
],
"domain": "ai.onnx.ml"
},
{
"input": [
"BinarizerOutput"
],
"output": [
"PredictedLabel"
],
"name": "Cast1",
"opType": "Cast",
"attribute": [
{
"name": "to",
"i": "9",
"type": "INT"
}
]
},
{
"input": [
"PredictedLabel"
Expand Down
19 changes: 18 additions & 1 deletion test/BaselineOutput/Common/Onnx/BinaryClassification/BreastCancer/OneHotBagPipeline.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -369,7 +369,7 @@
"Probability"
],
"output": [
"PredictedLabel"
"BinarizerOutput"
],
"name": "Binarizer",
"opType": "Binarizer",
Expand All @@ -382,6 +382,23 @@
],
"domain": "ai.onnx.ml"
},
{
"input": [
"BinarizerOutput"
],
"output": [
"PredictedLabel"
],
"name": "Cast1",
"opType": "Cast",
"attribute": [
{
"name": "to",
"i": "9",
"type": "INT"
}
]
},
{
"input": [
"Label"
Expand Down
71 changes: 67 additions & 4 deletions test/Microsoft.ML.Tests/OnnxConversionTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,15 @@ private class BreastCancerMulticlassExample
public float[] Features;
}

private class BreastCancerBinaryClassification
{
[LoadColumn(0)]
public bool Label;

[LoadColumn(2, 9), VectorType(8)]
public float[] Features;
}

[LessThanNetCore30OrNotNetCoreFact("netcoreapp3.0 output differs from Baseline. Tracked by https://github.com/dotnet/machinelearning/issues/2087")]
public void KmeansOnnxConversionTest()
{
Expand Down Expand Up @@ -202,14 +211,15 @@ public void RegressionTrainersOnnxConversionTest()
List<IEstimator<ITransformer>> estimators = new List<IEstimator<ITransformer>>()
{
mlContext.Regression.Trainers.Sdca("Target","FeatureVector"),
mlContext.Regression.Trainers.Ols("Target","FeatureVector"),
mlContext.Regression.Trainers.Ols("Target","FeatureVector"),
mlContext.Regression.Trainers.OnlineGradientDescent("Target","FeatureVector"),
mlContext.Regression.Trainers.FastForest("Target", "FeatureVector"),
mlContext.Regression.Trainers.FastTree("Target", "FeatureVector"),
mlContext.Regression.Trainers.FastTreeTweedie("Target", "FeatureVector"),
mlContext.Regression.Trainers.LbfgsPoissonRegression("Target", "FeatureVector"),
};
if (Environment.Is64BitProcess) {
if (Environment.Is64BitProcess)
{
estimators.Add(mlContext.Regression.Trainers.LightGbm("Target", "FeatureVector"));
}
foreach (var estimator in estimators)
Expand All @@ -232,7 +242,7 @@ public void RegressionTrainersOnnxConversionTest()
CompareSelectedR4ScalarColumns(transformedData.Schema[2].Name, outputNames[2], transformedData, onnxResult, 3); // compare score results
}
// Compare the Onnx graph to a baseline if OnnxRuntime is not supported
else
else
{
var onnxFileName = $"{estimator.ToString()}.txt";
var subDir = Path.Combine("..", "..", "BaselineOutput", "Common", "Onnx", "Regression", "Adult");
Expand All @@ -244,6 +254,58 @@ public void RegressionTrainersOnnxConversionTest()
Done();
}

[Fact]
public void BinaryClassificationTrainersOnnxConversionTest()
{
var mlContext = new MLContext(seed: 1);
string dataPath = GetDataPath("breast-cancer.txt");
// Now read the file (remember though, readers are lazy, so the actual reading will happen when the data is accessed).
var dataView = mlContext.Data.LoadFromTextFile<BreastCancerBinaryClassification>(dataPath, separatorChar: '\t', hasHeader: true);
List<IEstimator<ITransformer>> estimators = new List<IEstimator<ITransformer>>()
{
mlContext.BinaryClassification.Trainers.AveragedPerceptron(),
mlContext.BinaryClassification.Trainers.FastForest(),
mlContext.BinaryClassification.Trainers.FastTree(),
mlContext.BinaryClassification.Trainers.LbfgsLogisticRegression(),
mlContext.BinaryClassification.Trainers.LinearSvm(),
mlContext.BinaryClassification.Trainers.SdcaLogisticRegression(),
mlContext.BinaryClassification.Trainers.SdcaNonCalibrated(),
mlContext.BinaryClassification.Trainers.SgdCalibrated(),
mlContext.BinaryClassification.Trainers.SgdNonCalibrated(),
mlContext.BinaryClassification.Trainers.SymbolicSgdLogisticRegression(),
};
if (Environment.Is64BitProcess)
{
estimators.Add(mlContext.BinaryClassification.Trainers.LightGbm());
}

var initialPipeline = mlContext.Transforms.ReplaceMissingValues("Features").
Append(mlContext.Transforms.NormalizeMinMax("Features"));
foreach (var estimator in estimators)
{
var pipeline = initialPipeline.Append(estimator);
var model = pipeline.Fit(dataView);
var transformedData = model.Transform(dataView);
var onnxModel = mlContext.Model.ConvertToOnnxProtobuf(model, dataView);
// Compare model scores produced by ML.NET and ONNX's runtime.
if (IsOnnxRuntimeSupported())
{
var onnxFileName = $"{estimator.ToString()}.onnx";
var onnxModelPath = GetOutputPath(onnxFileName);
SaveOnnxModel(onnxModel, onnxModelPath, null);
// Evaluate the saved ONNX model using the data used to train the ML.NET pipeline.
string[] inputNames = onnxModel.Graph.Input.Select(valueInfoProto => valueInfoProto.Name).ToArray();
string[] outputNames = onnxModel.Graph.Output.Select(valueInfoProto => valueInfoProto.Name).ToArray();
var onnxEstimator = mlContext.Transforms.ApplyOnnxModel(outputNames, inputNames, onnxModelPath);
var onnxTransformer = onnxEstimator.Fit(dataView);
var onnxResult = onnxTransformer.Transform(dataView);
CompareSelectedR4ScalarColumns(transformedData.Schema[5].Name, outputNames[3], transformedData, onnxResult, 3);
CompareSelectedScalarColumns<Boolean>(transformedData.Schema[4].Name, outputNames[2], transformedData, onnxResult);
}
}
Done();
}

private class DataPoint
{
[VectorType(3)]
Expand Down Expand Up @@ -1225,7 +1287,8 @@ private void CreateDummyExamplesToMakeComplierHappy()
var dummyExample = new BreastCancerFeatureVector() { Features = null };
var dummyExample1 = new BreastCancerCatFeatureExample() { Label = false, F1 = 0, F2 = "Amy" };
var dummyExample2 = new BreastCancerMulticlassExample() { Label = "Amy", Features = null };
var dummyExample3 = new SmallSentimentExample() { Tokens = null };
var dummyExample3 = new BreastCancerBinaryClassification() { Label = false, Features = null };
var dummyExample4 = new SmallSentimentExample() { Tokens = null };
}

private void CompareResults(string leftColumnName, string rightColumnName, IDataView left, IDataView right)
Expand Down