From f17077903e07bb88343d963ebfc63c461b8a5aec Mon Sep 17 00:00:00 2001 From: Anipik Date: Mon, 22 Oct 2018 11:27:33 -0700 Subject: [PATCH 1/4] -0 parsing added and digitsofPrecision added --- .../UnitTests/TestCSharpApi.cs | 4 +-- .../UnitTests/TestEntryPoints.cs | 10 +++---- .../TestPredictors.cs | 30 +++++++++---------- .../BaseTestBaseline.cs | 2 +- test/Microsoft.ML.Tests/OnnxTests.cs | 6 ++-- .../Transformers/PcaTests.cs | 2 +- 6 files changed, 27 insertions(+), 27 deletions(-) diff --git a/test/Microsoft.ML.Core.Tests/UnitTests/TestCSharpApi.cs b/test/Microsoft.ML.Core.Tests/UnitTests/TestCSharpApi.cs index 163db8e801..ec09f553d7 100644 --- a/test/Microsoft.ML.Core.Tests/UnitTests/TestCSharpApi.cs +++ b/test/Microsoft.ML.Core.Tests/UnitTests/TestCSharpApi.cs @@ -374,9 +374,9 @@ public void TestCrossValidationMacro() foldGetter(ref fold); Assert.True(ReadOnlyMemoryUtils.EqualsStr("Standard Deviation", fold)); if (w == 1) - Assert.Equal(1.584696, stdev, 6); + Assert.Equal(1.584696, stdev, 3); else - Assert.Equal(1.385165, stdev, 6); + Assert.Equal(1.385165, stdev, 2); isWeightedGetter(ref isWeighted); Assert.True(isWeighted == (w == 1)); } diff --git a/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs b/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs index 40c8dc1862..30a9ab6625 100644 --- a/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs +++ b/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs @@ -3630,10 +3630,10 @@ public void EntryPointLinearPredictorSummary() DataSaverUtils.SaveDataView(ch, saver, mcOutput.Stats, file); } - CheckEquality(@"../Common/EntryPoints", "lr-weights.txt"); - CheckEquality(@"../Common/EntryPoints", "lr-stats.txt"); - CheckEquality(@"../Common/EntryPoints", "mc-lr-weights.txt"); - CheckEquality(@"../Common/EntryPoints", "mc-lr-stats.txt"); + CheckEquality(@"../Common/EntryPoints", "lr-weights.txt", digitsOfPrecision: 6); + CheckEquality(@"../Common/EntryPoints", "lr-stats.txt", digitsOfPrecision: 6); + CheckEquality(@"../Common/EntryPoints", "mc-lr-weights.txt", digitsOfPrecision: 3); + CheckEquality(@"../Common/EntryPoints", "mc-lr-stats.txt", digitsOfPrecision: 5); Done(); } @@ -3675,7 +3675,7 @@ public void EntryPointPcaPredictorSummary() DataSaverUtils.SaveDataView(ch, saver, output.Summary, file); } - CheckEquality(@"../Common/EntryPoints", "pca-weights.txt"); + CheckEquality(@"../Common/EntryPoints", "pca-weights.txt", digitsOfPrecision: 4); Done(); } } diff --git a/test/Microsoft.ML.Predictor.Tests/TestPredictors.cs b/test/Microsoft.ML.Predictor.Tests/TestPredictors.cs index b2ea240e7f..27510e130d 100644 --- a/test/Microsoft.ML.Predictor.Tests/TestPredictors.cs +++ b/test/Microsoft.ML.Predictor.Tests/TestPredictors.cs @@ -108,7 +108,7 @@ public void BinaryClassifierPerceptronTest() { var binaryPredictors = new[] { TestLearners.perceptron }; var binaryClassificationDatasets = GetDatasetsForBinaryClassifierBaseTest(); - RunAllTests(binaryPredictors, binaryClassificationDatasets); + RunAllTests(binaryPredictors, binaryClassificationDatasets, digitsOfPrecision: 6); Done(); } @@ -161,7 +161,7 @@ public void EarlyStoppingTest() [TestCategory("Logistic Regression")] public void MulticlassLRTest() { - RunOneAllTests(TestLearners.multiclassLogisticRegression, TestDatasets.iris, digitsOfPrecision: 4); + RunOneAllTests(TestLearners.multiclassLogisticRegression, TestDatasets.iris, digitsOfPrecision: 2); Done(); } @@ -253,7 +253,7 @@ public void LinearClassifierTest() TestLearners.binarySgdHinge }; var binaryClassificationDatasets = GetDatasetsForBinaryClassifierBaseTest(); - RunAllTests(binaryPredictors, binaryClassificationDatasets); + RunAllTests(binaryPredictors, binaryClassificationDatasets, digitsOfPrecision: 5); Done(); } @@ -265,7 +265,7 @@ public void LinearClassifierTest() public void BinaryClassifierLogisticRegressionTest() { var binaryPredictors = new[] { TestLearners.logisticRegression }; - RunOneAllTests(TestLearners.logisticRegression, TestDatasets.breastCancer, summary: true); + RunOneAllTests(TestLearners.logisticRegression, TestDatasets.breastCancer, summary: true, digitsOfPrecision: 3); // RunOneAllTests(TestLearners.logisticRegression, TestDatasets.msm); Done(); } @@ -278,7 +278,7 @@ public void BinaryClassifierSymSgdTest() if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) return; - RunOneAllTests(TestLearners.symSGD, TestDatasets.breastCancer, summary: true); + RunOneAllTests(TestLearners.symSGD, TestDatasets.breastCancer, summary: true, digitsOfPrecision: 4); Done(); } @@ -288,7 +288,7 @@ public void BinaryClassifierTesterThresholdingTest() { var binaryPredictors = new[] { TestLearners.logisticRegression }; var binaryClassificationDatasets = new[] { TestDatasets.breastCancer }; - RunAllTests(binaryPredictors, binaryClassificationDatasets, new[] { "eval=BinaryClassifier{threshold=0.95 useRawScore=-}" }, "withThreshold"); + RunAllTests(binaryPredictors, binaryClassificationDatasets, new[] { "eval=BinaryClassifier{threshold=0.95 useRawScore=-}" }, "withThreshold", digitsOfPrecision: 3); Done(); } @@ -301,7 +301,7 @@ public void BinaryClassifierLogisticRegressionNormTest() { var binaryPredictors = new[] { TestLearners.logisticRegressionNorm }; var binaryClassificationDatasets = GetDatasetsForBinaryClassifierBaseTest(); - RunAllTests(binaryPredictors, binaryClassificationDatasets); + RunAllTests(binaryPredictors, binaryClassificationDatasets, digitsOfPrecision: 5); Done(); } @@ -327,7 +327,7 @@ public void BinaryClassifierLogisticRegressionBinNormTest() { var binaryPredictors = new[] { TestLearners.logisticRegressionBinNorm }; var binaryClassificationDatasets = GetDatasetsForBinaryClassifierBaseTest(); - RunAllTests(binaryPredictors, binaryClassificationDatasets); + RunAllTests(binaryPredictors, binaryClassificationDatasets, digitsOfPrecision: 6); Done(); } @@ -340,7 +340,7 @@ public void BinaryClassifierLogisticRegressionGaussianNormTest() { var binaryPredictors = new[] { TestLearners.logisticRegressionGaussianNorm }; var binaryClassificationDatasets = GetDatasetsForBinaryClassifierBaseTest(); - RunAllTests(binaryPredictors, binaryClassificationDatasets); + RunAllTests(binaryPredictors, binaryClassificationDatasets, digitsOfPrecision: 4); Done(); } @@ -934,7 +934,7 @@ public void RegressorOlsTest() [TestCategory("Regressor")] public void RegressorOlsTestOne() { - Run_TrainTest(TestLearners.Ols, TestDatasets.generatedRegressionDataset); + Run_TrainTest(TestLearners.Ols, TestDatasets.generatedRegressionDataset, digitsOfPrecision: 4); Done(); } @@ -1366,7 +1366,7 @@ public void PAVCalibratorPerceptronTest() public void RandomCalibratorPerceptronTest() { var datasets = GetDatasetsForCalibratorTest(); - RunAllTests(new[] { TestLearners.perceptronDefault }, datasets, new string[] { "numcali=200" }, "calibrateRandom"); + RunAllTests(new[] { TestLearners.perceptronDefault }, datasets, new string[] { "numcali=200" }, "calibrateRandom", digitsOfPrecision: 1); Done(); } @@ -1378,7 +1378,7 @@ public void RandomCalibratorPerceptronTest() public void NoCalibratorLinearSvmTest() { var datasets = GetDatasetsForCalibratorTest(); - RunAllTests(new[] { TestLearners.linearSVM }, datasets, new string[] { "cali={}" }, "nocalibration"); + RunAllTests(new[] { TestLearners.linearSVM }, datasets, new string[] { "cali={}" }, "nocalibration", digitsOfPrecision: 6); Done(); } @@ -1390,7 +1390,7 @@ public void NoCalibratorLinearSvmTest() public void PAVCalibratorLinearSvmTest() { var datasets = GetDatasetsForCalibratorTest(); - RunAllTests(new[] { TestLearners.linearSVM }, datasets, new string[] { "cali=PAV" }, "PAVcalibration"); + RunAllTests(new[] { TestLearners.linearSVM }, datasets, new string[] { "cali=PAV" }, "PAVcalibration", digitsOfPrecision: 5); Done(); } @@ -1520,8 +1520,8 @@ public void NnConfigTests() [TestCategory("Anomaly")] public void PcaAnomalyTest() { - Run_TrainTest(TestLearners.PCAAnomalyDefault, TestDatasets.mnistOneClass); - Run_TrainTest(TestLearners.PCAAnomalyNoNorm, TestDatasets.mnistOneClass); + Run_TrainTest(TestLearners.PCAAnomalyDefault, TestDatasets.mnistOneClass, digitsOfPrecision: 5); + Run_TrainTest(TestLearners.PCAAnomalyNoNorm, TestDatasets.mnistOneClass, digitsOfPrecision: 5); // REVIEW: This next test was misbehaving in a strange way that seems to have gone away // mysteriously (bad build?). diff --git a/test/Microsoft.ML.TestFramework/BaseTestBaseline.cs b/test/Microsoft.ML.TestFramework/BaseTestBaseline.cs index 4b14152011..83f6b8717e 100644 --- a/test/Microsoft.ML.TestFramework/BaseTestBaseline.cs +++ b/test/Microsoft.ML.TestFramework/BaseTestBaseline.cs @@ -506,7 +506,7 @@ protected bool CheckEqualityFromPathsCore(string relPath, string basePath, strin private void GetNumbersFromFile(ref string firstString, ref string secondString, int digitsOfPrecision) { - Regex _matchNumer = new Regex(@"\b[0-9]+\.?[0-9]*(E-[0-9]*)?\b", RegexOptions.IgnoreCase | RegexOptions.Compiled); + Regex _matchNumer = new Regex(@"-?[0-9]+\.?[0-9]*(E-[0-9]*)?\b", RegexOptions.IgnoreCase | RegexOptions.Compiled); MatchCollection firstCollection = _matchNumer.Matches(firstString); MatchCollection secondCollection = _matchNumer.Matches(secondString); diff --git a/test/Microsoft.ML.Tests/OnnxTests.cs b/test/Microsoft.ML.Tests/OnnxTests.cs index 09808501a6..b2f4b7f33c 100644 --- a/test/Microsoft.ML.Tests/OnnxTests.cs +++ b/test/Microsoft.ML.Tests/OnnxTests.cs @@ -329,7 +329,7 @@ public void KmeansTest() fileText = Regex.Replace(fileText, "\"producerVersion\": \"([^\"]+)\"", "\"producerVersion\": \"##VERSION##\""); File.WriteAllText(onnxAsJsonPath, fileText); - CheckEquality(subDir, "Kmeans.json"); + CheckEquality(subDir, "Kmeans.json", digitsOfPrecision: 2); Done(); } @@ -452,7 +452,7 @@ public void BinaryClassificationLRSaveModelToOnnxTest() fileText = Regex.Replace(fileText, "\"producerVersion\": \"([^\"]+)\"", "\"producerVersion\": \"##VERSION##\""); File.WriteAllText(onnxAsJsonPath, fileText); - CheckEquality(subDir, "BinaryClassificationLRSaveModelToOnnxTest.json"); + CheckEquality(subDir, "BinaryClassificationLRSaveModelToOnnxTest.json", digitsOfPrecision: 3); Done(); } @@ -514,7 +514,7 @@ public void MultiClassificationLRSaveModelToOnnxTest() fileText = Regex.Replace(fileText, "\"producerVersion\": \"([^\"]+)\"", "\"producerVersion\": \"##VERSION##\""); File.WriteAllText(onnxAsJsonPath, fileText); - CheckEquality(subDir, "MultiClassificationLRSaveModelToOnnxTest.json"); + CheckEquality(subDir, "MultiClassificationLRSaveModelToOnnxTest.json", digitsOfPrecision: 4); Done(); } diff --git a/test/Microsoft.ML.Tests/Transformers/PcaTests.cs b/test/Microsoft.ML.Tests/Transformers/PcaTests.cs index 5561b4df7d..4ca45dddc1 100644 --- a/test/Microsoft.ML.Tests/Transformers/PcaTests.cs +++ b/test/Microsoft.ML.Tests/Transformers/PcaTests.cs @@ -53,7 +53,7 @@ public void PcaWorkout() DataSaverUtils.SaveDataView(ch, saver, savedData, fs, keepHidden: true); } - CheckEquality("PCA", "pca.tsv"); + CheckEquality("PCA", "pca.tsv", digitsOfPrecision: 4); Done(); } } From 4c6969473d338400328c1914c1e7280519391969 Mon Sep 17 00:00:00 2001 From: Anipik Date: Tue, 23 Oct 2018 17:17:56 -0700 Subject: [PATCH 2/4] regex edited to include wordAssert and precisionDigit 2 removed --- test/Microsoft.ML.TestFramework/BaseTestBaseline.cs | 2 +- test/Microsoft.ML.Tests/OnnxTests.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/Microsoft.ML.TestFramework/BaseTestBaseline.cs b/test/Microsoft.ML.TestFramework/BaseTestBaseline.cs index 83f6b8717e..640540b902 100644 --- a/test/Microsoft.ML.TestFramework/BaseTestBaseline.cs +++ b/test/Microsoft.ML.TestFramework/BaseTestBaseline.cs @@ -506,7 +506,7 @@ protected bool CheckEqualityFromPathsCore(string relPath, string basePath, strin private void GetNumbersFromFile(ref string firstString, ref string secondString, int digitsOfPrecision) { - Regex _matchNumer = new Regex(@"-?[0-9]+\.?[0-9]*(E-[0-9]*)?\b", RegexOptions.IgnoreCase | RegexOptions.Compiled); + Regex _matchNumer = new Regex(@"-?\b[0-9]+\.?[0-9]*(E-[0-9]*)?\b", RegexOptions.IgnoreCase | RegexOptions.Compiled); MatchCollection firstCollection = _matchNumer.Matches(firstString); MatchCollection secondCollection = _matchNumer.Matches(secondString); diff --git a/test/Microsoft.ML.Tests/OnnxTests.cs b/test/Microsoft.ML.Tests/OnnxTests.cs index b2f4b7f33c..0d87140652 100644 --- a/test/Microsoft.ML.Tests/OnnxTests.cs +++ b/test/Microsoft.ML.Tests/OnnxTests.cs @@ -329,7 +329,7 @@ public void KmeansTest() fileText = Regex.Replace(fileText, "\"producerVersion\": \"([^\"]+)\"", "\"producerVersion\": \"##VERSION##\""); File.WriteAllText(onnxAsJsonPath, fileText); - CheckEquality(subDir, "Kmeans.json", digitsOfPrecision: 2); + CheckEquality(subDir, "Kmeans.json"); Done(); } From f55f4ed8041d4c732cb84113ecc0ad819b986275 Mon Sep 17 00:00:00 2001 From: Anipik Date: Thu, 25 Oct 2018 12:19:39 -0700 Subject: [PATCH 3/4] keeping the min for digitsof precision as 3 --- test/Microsoft.ML.Predictor.Tests/TestPredictors.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/Microsoft.ML.Predictor.Tests/TestPredictors.cs b/test/Microsoft.ML.Predictor.Tests/TestPredictors.cs index 27510e130d..b0158f60bb 100644 --- a/test/Microsoft.ML.Predictor.Tests/TestPredictors.cs +++ b/test/Microsoft.ML.Predictor.Tests/TestPredictors.cs @@ -161,7 +161,7 @@ public void EarlyStoppingTest() [TestCategory("Logistic Regression")] public void MulticlassLRTest() { - RunOneAllTests(TestLearners.multiclassLogisticRegression, TestDatasets.iris, digitsOfPrecision: 2); + RunOneAllTests(TestLearners.multiclassLogisticRegression, TestDatasets.iris, digitsOfPrecision: 4); Done(); } @@ -1366,7 +1366,7 @@ public void PAVCalibratorPerceptronTest() public void RandomCalibratorPerceptronTest() { var datasets = GetDatasetsForCalibratorTest(); - RunAllTests(new[] { TestLearners.perceptronDefault }, datasets, new string[] { "numcali=200" }, "calibrateRandom", digitsOfPrecision: 1); + RunAllTests(new[] { TestLearners.perceptronDefault }, datasets, new string[] { "numcali=200" }, "calibrateRandom"); Done(); } From c11bd976c9751aecb1d564af997e3cf986b855d6 Mon Sep 17 00:00:00 2001 From: Anipik Date: Thu, 25 Oct 2018 13:51:38 -0700 Subject: [PATCH 4/4] Adding comment about regex and removing extra digits --- test/Microsoft.ML.Core.Tests/UnitTests/TestCSharpApi.cs | 4 ++-- test/Microsoft.ML.TestFramework/BaseTestBaseline.cs | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/test/Microsoft.ML.Core.Tests/UnitTests/TestCSharpApi.cs b/test/Microsoft.ML.Core.Tests/UnitTests/TestCSharpApi.cs index ec09f553d7..7b2864fe36 100644 --- a/test/Microsoft.ML.Core.Tests/UnitTests/TestCSharpApi.cs +++ b/test/Microsoft.ML.Core.Tests/UnitTests/TestCSharpApi.cs @@ -374,9 +374,9 @@ public void TestCrossValidationMacro() foldGetter(ref fold); Assert.True(ReadOnlyMemoryUtils.EqualsStr("Standard Deviation", fold)); if (w == 1) - Assert.Equal(1.584696, stdev, 3); + Assert.Equal(1.585, stdev, 3); else - Assert.Equal(1.385165, stdev, 2); + Assert.Equal(1.39, stdev, 2); isWeightedGetter(ref isWeighted); Assert.True(isWeighted == (w == 1)); } diff --git a/test/Microsoft.ML.TestFramework/BaseTestBaseline.cs b/test/Microsoft.ML.TestFramework/BaseTestBaseline.cs index 640540b902..55bd13a328 100644 --- a/test/Microsoft.ML.TestFramework/BaseTestBaseline.cs +++ b/test/Microsoft.ML.TestFramework/BaseTestBaseline.cs @@ -506,6 +506,8 @@ protected bool CheckEqualityFromPathsCore(string relPath, string basePath, strin private void GetNumbersFromFile(ref string firstString, ref string secondString, int digitsOfPrecision) { + // The Regex matches both positive and negative decimal point numbers present in a string. + // The numbers could be a part of a word. They can also be in Exponential form eg. 3E-9 Regex _matchNumer = new Regex(@"-?\b[0-9]+\.?[0-9]*(E-[0-9]*)?\b", RegexOptions.IgnoreCase | RegexOptions.Compiled); MatchCollection firstCollection = _matchNumer.Matches(firstString); MatchCollection secondCollection = _matchNumer.Matches(secondString);