diff --git a/src/Microsoft.ML.Data/Prediction/Calibrator.cs b/src/Microsoft.ML.Data/Prediction/Calibrator.cs index f3a3b91792..effd0eac04 100644 --- a/src/Microsoft.ML.Data/Prediction/Calibrator.cs +++ b/src/Microsoft.ML.Data/Prediction/Calibrator.cs @@ -1581,10 +1581,10 @@ internal sealed class FixedPlattCalibratorTrainer : ICalibratorTrainer [TlcModule.Component(Name = "FixedPlattCalibrator", FriendlyName = "Fixed Platt Calibrator", Aliases = new[] { "FixedPlatt", "FixedSigmoid" })] public sealed class Arguments : ICalibratorTrainerFactory { - [Argument(ArgumentType.LastOccurrenceWins, HelpText = "The slope parameter of f(x) = 1 / (1 + exp(-slope * x + offset)", ShortName = "a")] - public Double Slope = 1; + [Argument(ArgumentType.LastOccurrenceWins, HelpText = "The slope parameter of f(x) = 1 / (1 + exp(slope * x + offset)", ShortName = "a")] + public Double Slope = -1; - [Argument(ArgumentType.LastOccurrenceWins, HelpText = "The offset parameter of f(x) = 1 / (1 + exp(-slope * x + offset)", ShortName = "b")] + [Argument(ArgumentType.LastOccurrenceWins, HelpText = "The offset parameter of f(x) = 1 / (1 + exp(slope * x + offset)", ShortName = "b")] public Double Offset = 0; public ICalibratorTrainer CreateComponent(IHostEnvironment env) @@ -1618,7 +1618,7 @@ internal FixedPlattCalibratorTrainer(IHostEnvironment env, Arguments args) /// /// The Platt calibrator calculates the probability following: - /// P(x) = 1 / (1 + exp(- * x + ) + /// P(x) = 1 / (1 + exp( * x + ) /// . public sealed class PlattCalibrator : ICalibrator, IParameterMixer, ICanSaveModel, ISingleCanSavePfa, ISingleCanSaveOnnx { @@ -2085,10 +2085,10 @@ public sealed class NoArgumentsInput : CalibrateInputBase public sealed class FixedPlattInput : CalibrateInputBase { - [Argument(ArgumentType.AtMostOnce, ShortName = "slope", HelpText = "The slope parameter of the calibration function 1 / (1 + exp(-slope * x + offset)", SortOrder = 1)] - public Double Slope = 1; + [Argument(ArgumentType.AtMostOnce, ShortName = "slope", HelpText = "The slope parameter of the calibration function 1 / (1 + exp(slope * x + offset)", SortOrder = 1)] + public Double Slope = -1; - [Argument(ArgumentType.AtMostOnce, ShortName = "offset", HelpText = "The offset parameter of the calibration function 1 / (1 + exp(-slope * x + offset)", SortOrder = 3)] + [Argument(ArgumentType.AtMostOnce, ShortName = "offset", HelpText = "The offset parameter of the calibration function 1 / (1 + exp(slope * x + offset)", SortOrder = 3)] public Double Offset = 0; } diff --git a/test/BaselineOutput/Common/EntryPoints/core_manifest.json b/test/BaselineOutput/Common/EntryPoints/core_manifest.json index 8c9c1092c0..d139710b10 100644 --- a/test/BaselineOutput/Common/EntryPoints/core_manifest.json +++ b/test/BaselineOutput/Common/EntryPoints/core_manifest.json @@ -1731,14 +1731,14 @@ { "Name": "Slope", "Type": "Float", - "Desc": "The slope parameter of the calibration function 1 / (1 + exp(-slope * x + offset)", + "Desc": "The slope parameter of the calibration function 1 / (1 + exp(slope * x + offset)", "Aliases": [ "slope" ], "Required": false, "SortOrder": 1.0, "IsNullable": false, - "Default": 1.0 + "Default": -1.0 }, { "Name": "Data", @@ -1762,7 +1762,7 @@ { "Name": "Offset", "Type": "Float", - "Desc": "The offset parameter of the calibration function 1 / (1 + exp(-slope * x + offset)", + "Desc": "The offset parameter of the calibration function 1 / (1 + exp(slope * x + offset)", "Aliases": [ "offset" ], @@ -25072,19 +25072,19 @@ { "Name": "Slope", "Type": "Float", - "Desc": "The slope parameter of f(x) = 1 / (1 + exp(-slope * x + offset)", + "Desc": "The slope parameter of f(x) = 1 / (1 + exp(slope * x + offset)", "Aliases": [ "a" ], "Required": false, "SortOrder": 150.0, "IsNullable": false, - "Default": 1.0 + "Default": -1.0 }, { "Name": "Offset", "Type": "Float", - "Desc": "The offset parameter of f(x) = 1 / (1 + exp(-slope * x + offset)", + "Desc": "The offset parameter of f(x) = 1 / (1 + exp(slope * x + offset)", "Aliases": [ "b" ], diff --git a/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs b/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs index f7dd2f4b58..d7eb9bec93 100644 --- a/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs +++ b/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs @@ -1536,6 +1536,7 @@ public void EntryPointCalibrate() var twiceCalibratedFfModel = Calibrate.Platt(Env, new Calibrate.NoArgumentsInput() { Data = splitOutput.TestData[0], UncalibratedPredictorModel = calibratedFfModel }).PredictorModel; var scoredFf = ScoreModel.Score(Env, new ScoreModel.Input() { Data = splitOutput.TestData[2], PredictorModel = twiceCalibratedFfModel }).ScoredData; + Done(); } [Fact] diff --git a/test/Microsoft.ML.Predictor.Tests/TestPredictors.cs b/test/Microsoft.ML.Predictor.Tests/TestPredictors.cs index 2feed686f1..d33b29b065 100644 --- a/test/Microsoft.ML.Predictor.Tests/TestPredictors.cs +++ b/test/Microsoft.ML.Predictor.Tests/TestPredictors.cs @@ -791,6 +791,7 @@ public void TestEnsembleCombiner() }; CombineAndTestEnsembles(dataView, "pe", "oc=average", PredictionKind.BinaryClassification, predictors); + Done(); } [X64Fact("x86 fails. Associated GitHubIssue: https://github.com/dotnet/machinelearning/issues/1216")] @@ -941,7 +942,7 @@ private void CombineAndTestEnsembles(IDataView idv, string name, string options, predGetters[i](ref preds[i]); } if (scores.All(s => !float.IsNaN(s))) - CompareNumbersWithTolerance(score, scores.Sum() / predCount); + CompareNumbersWithTolerance(score, scores.Sum() / predCount, digitsOfPrecision: 5); for (int i = 0; i < predCount; i++) Assert.Equal(vectorScore.Length, vectorScores[i].Length); for (int i = 0; i < vectorScore.Length; i++) diff --git a/test/Microsoft.ML.TestFramework/BaseTestBaseline.cs b/test/Microsoft.ML.TestFramework/BaseTestBaseline.cs index 2aea54ab75..d8b14e1432 100644 --- a/test/Microsoft.ML.TestFramework/BaseTestBaseline.cs +++ b/test/Microsoft.ML.TestFramework/BaseTestBaseline.cs @@ -4,8 +4,8 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.IO; -using System.Runtime.ExceptionServices; using System.Text; using System.Text.RegularExpressions; using System.Threading; @@ -81,7 +81,7 @@ protected BaseTestBaseline(ITestOutputHelper output) : base(output) protected IHostEnvironment Env => _env; protected MLContext ML; private bool _normal; - private readonly List _failures = new List(); + private int _failures = 0; protected override void Initialize() { @@ -126,6 +126,9 @@ protected override void Cleanup() _normal ? "completed normally" : "aborted", IsPassing ? "passed" : "failed"); + if (!_normal) + Assert.Equal(0, _failures); + Contracts.AssertValue(LogWriter); LogWriter.Dispose(); LogWriter = null; @@ -135,7 +138,7 @@ protected override void Cleanup() protected bool IsActive { get { return LogWriter != null; } } - protected bool IsPassing { get { return _failures.Count == 0; } } + protected bool IsPassing { get { return _failures == 0; } } // Called by a test to signal normal completion. If this is not called before the // TestScope is disposed, we assume the test was aborted. @@ -145,18 +148,7 @@ protected void Done() Contracts.Assert(!_normal, "Done() should only be called once!"); _normal = true; - switch (_failures.Count) - { - case 0: - break; - - case 1: - ExceptionDispatchInfo.Capture(_failures[0]).Throw(); - break; - - default: - throw new AggregateException(_failures.ToArray()); - } + Assert.Equal(0, _failures); } protected bool Check(bool f, string msg) @@ -176,16 +168,16 @@ protected bool Check(bool f, string msg, params object[] args) protected void Fail(string fmt, params object[] args) { Contracts.Assert(IsActive); - try - { - throw new InvalidOperationException(string.Format(fmt, args)); - } - catch (Exception ex) + _failures++; + Log($"*** Failure #{_failures}: " + fmt, args); + + var stackTrace = new StackTrace(true); + for (int i = 0; i < stackTrace.FrameCount; i++) { - _failures.Add(ex); + var frame = stackTrace.GetFrame(i); + Log($"\t\t{frame.GetMethod()} {frame.GetFileName()} {frame.GetFileLineNumber()}"); } - Log("*** Failure: " + fmt, args); } protected void Log(string msg) diff --git a/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs b/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs index 439eb9af0a..a6c0093242 100644 --- a/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs +++ b/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs @@ -485,10 +485,14 @@ public void OneClassMatrixFactorizationInMemoryDataZeroBaseIndex() var testPrediction = model.Transform(testDataView); var testResults = mlContext.Data.CreateEnumerable(testPrediction, false).ToList(); + + // TODO TEST_STABILITY: We are seeing lower precision on non-Windows platforms + int precision = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? 5 : 3; + // Positive example (i.e., examples can be found in dataMatrix) is close to 1. - CompareNumbersWithTolerance(0.982391, testResults[0].Score, digitsOfPrecision: 5); + CompareNumbersWithTolerance(0.982391, testResults[0].Score, digitsOfPrecision: precision); // Negative example (i.e., examples can not be found in dataMatrix) is close to 0.15 (specified by s.C = 0.15 in the trainer). - CompareNumbersWithTolerance(0.141411, testResults[1].Score, digitsOfPrecision: 5); + CompareNumbersWithTolerance(0.141411, testResults[1].Score, digitsOfPrecision: precision); } [MatrixFactorizationFact] diff --git a/test/Microsoft.ML.Tests/Transformers/TextFeaturizerTests.cs b/test/Microsoft.ML.Tests/Transformers/TextFeaturizerTests.cs index fd8dcabebe..9ab4a7fd9d 100644 --- a/test/Microsoft.ML.Tests/Transformers/TextFeaturizerTests.cs +++ b/test/Microsoft.ML.Tests/Transformers/TextFeaturizerTests.cs @@ -714,6 +714,7 @@ public void LdaWorkout() } [Fact] + [Trait("Category", "SkipInCI")] public void LdaWorkoutEstimatorCore() { var ml = new MLContext(1);