From af9455bae828c08c4f133ec114aaad1e7770b36e Mon Sep 17 00:00:00 2001 From: Keren Fuentes Date: Fri, 17 Jan 2020 17:05:11 -0800 Subject: [PATCH 1/4] fix for issue 4664 --- .../Evaluators/BinaryClassifierEvaluator.cs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.ML.Data/Evaluators/BinaryClassifierEvaluator.cs b/src/Microsoft.ML.Data/Evaluators/BinaryClassifierEvaluator.cs index 283e85a203..fa1f1c751d 100644 --- a/src/Microsoft.ML.Data/Evaluators/BinaryClassifierEvaluator.cs +++ b/src/Microsoft.ML.Data/Evaluators/BinaryClassifierEvaluator.cs @@ -480,7 +480,15 @@ public Double LogLossReduction } } - public Double F1 { get { return 2 * PrecisionPos * RecallPos / (PrecisionPos + RecallPos); } } + public Double F1 + { + get + { + if (PrecisionPos + RecallPos == 0) + return 0; + return 2 * PrecisionPos * RecallPos / (PrecisionPos + RecallPos); + } + } public Counters(bool useRaw, Single threshold) { From 12a77f4f76250ad7121233ab21e08b9c9181b73a Mon Sep 17 00:00:00 2001 From: Keren Fuentes Date: Fri, 17 Jan 2020 17:44:45 -0800 Subject: [PATCH 2/4] resolving comment --- .../Evaluators/BinaryClassifierEvaluator.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.ML.Data/Evaluators/BinaryClassifierEvaluator.cs b/src/Microsoft.ML.Data/Evaluators/BinaryClassifierEvaluator.cs index fa1f1c751d..bdcff62752 100644 --- a/src/Microsoft.ML.Data/Evaluators/BinaryClassifierEvaluator.cs +++ b/src/Microsoft.ML.Data/Evaluators/BinaryClassifierEvaluator.cs @@ -484,9 +484,10 @@ public Double F1 { get { - if (PrecisionPos + RecallPos == 0) + var precisionPlusRecall = PrecisionPos + RecallPos; + if (precisionPlusRecall == 0) return 0; - return 2 * PrecisionPos * RecallPos / (PrecisionPos + RecallPos); + return 2 * PrecisionPos * RecallPos / (precisionPlusRecall); } } From 29b594f2ad950b09762a0f6734018f61e7423cd6 Mon Sep 17 00:00:00 2001 From: Keren Fuentes Date: Fri, 17 Jan 2020 17:46:26 -0800 Subject: [PATCH 3/4] resolving comment --- src/Microsoft.ML.Data/Evaluators/BinaryClassifierEvaluator.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.ML.Data/Evaluators/BinaryClassifierEvaluator.cs b/src/Microsoft.ML.Data/Evaluators/BinaryClassifierEvaluator.cs index bdcff62752..deeecfc725 100644 --- a/src/Microsoft.ML.Data/Evaluators/BinaryClassifierEvaluator.cs +++ b/src/Microsoft.ML.Data/Evaluators/BinaryClassifierEvaluator.cs @@ -487,7 +487,7 @@ public Double F1 var precisionPlusRecall = PrecisionPos + RecallPos; if (precisionPlusRecall == 0) return 0; - return 2 * PrecisionPos * RecallPos / (precisionPlusRecall); + return 2 * PrecisionPos * RecallPos / precisionPlusRecall; } } From f449b99bc0268f07963271365f78e008cd89ae0a Mon Sep 17 00:00:00 2001 From: Keren Fuentes Date: Tue, 21 Jan 2020 10:46:35 -0800 Subject: [PATCH 4/4] change F1 in baselines --- .../FastTreeHighMinDocs-TrainTest-breast-cancer-out.txt | 2 +- .../FastTreeHighMinDocs-TrainTest-breast-cancer-rp.txt | 2 +- .../PriorPredictor/BinaryPrior-CV-breast-cancer-out.txt | 2 +- .../PriorPredictor/BinaryPrior-CV-breast-cancer-rp.txt | 2 +- .../PriorPredictor/BinaryPrior-TrainTest-breast-cancer-out.txt | 2 +- .../PriorPredictor/BinaryPrior-TrainTest-breast-cancer-rp.txt | 2 +- .../PriorPredictor/BinaryPrior-CV-breast-cancer-out.txt | 2 +- .../PriorPredictor/BinaryPrior-CV-breast-cancer-rp.txt | 2 +- .../PriorPredictor/BinaryPrior-TrainTest-breast-cancer-out.txt | 2 +- .../PriorPredictor/BinaryPrior-TrainTest-breast-cancer-rp.txt | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/test/BaselineOutput/Common/FastTreeBinaryClassification/FastTreeHighMinDocs-TrainTest-breast-cancer-out.txt b/test/BaselineOutput/Common/FastTreeBinaryClassification/FastTreeHighMinDocs-TrainTest-breast-cancer-out.txt index f16acb05b6..cc0142cc56 100644 --- a/test/BaselineOutput/Common/FastTreeBinaryClassification/FastTreeHighMinDocs-TrainTest-breast-cancer-out.txt +++ b/test/BaselineOutput/Common/FastTreeBinaryClassification/FastTreeHighMinDocs-TrainTest-breast-cancer-out.txt @@ -36,7 +36,7 @@ Negative precision: 0.655222 (0.0000) Negative recall: 1.000000 (0.0000) Log-loss: 1.000000 (0.0000) Log-loss reduction: -0.076058 (0.0000) -F1 Score: NaN (0.0000) +F1 Score: 0.000000 (0.0000) AUPRC: 0.415719 (0.0000) --------------------------------------- diff --git a/test/BaselineOutput/Common/FastTreeBinaryClassification/FastTreeHighMinDocs-TrainTest-breast-cancer-rp.txt b/test/BaselineOutput/Common/FastTreeBinaryClassification/FastTreeHighMinDocs-TrainTest-breast-cancer-rp.txt index 9d2862922b..d1d605b7e5 100644 --- a/test/BaselineOutput/Common/FastTreeBinaryClassification/FastTreeHighMinDocs-TrainTest-breast-cancer-rp.txt +++ b/test/BaselineOutput/Common/FastTreeBinaryClassification/FastTreeHighMinDocs-TrainTest-breast-cancer-rp.txt @@ -1,4 +1,4 @@ FastTreeBinaryClassification AUC Accuracy Positive precision Positive recall Negative precision Negative recall Log-loss Log-loss reduction F1 Score AUPRC /mil /iter Learner Name Train Dataset Test Dataset Results File Run Time Physical Memory Virtual Memory Command Line Settings -0.5 0.655222 0 0 0.655222 1 1 -0.076058 NaN 0.415719 10000 5 FastTreeBinaryClassification %Data% %Data% %Output% 99 0 0 maml.exe TrainTest test=%Data% tr=FastTreeBinaryClassification{mil=10000 iter=5} cache=- dout=%Output% loader=Text{sparse- col=Attr:TX:6 col=Label:0 col=Features:1-5,6,7-9} data=%Data% out=%Output% seed=1 /mil:10000;/iter:5 +0.5 0.655222 0 0 0.655222 1 1 -0.076058 0 0.415719 10000 5 FastTreeBinaryClassification %Data% %Data% %Output% 99 0 0 maml.exe TrainTest test=%Data% tr=FastTreeBinaryClassification{mil=10000 iter=5} cache=- dout=%Output% loader=Text{sparse- col=Attr:TX:6 col=Label:0 col=Features:1-5,6,7-9} data=%Data% out=%Output% seed=1 /mil:10000;/iter:5 diff --git a/test/BaselineOutput/SingleDebug/PriorPredictor/BinaryPrior-CV-breast-cancer-out.txt b/test/BaselineOutput/SingleDebug/PriorPredictor/BinaryPrior-CV-breast-cancer-out.txt index 8a5c288d72..5b8c118d2b 100644 --- a/test/BaselineOutput/SingleDebug/PriorPredictor/BinaryPrior-CV-breast-cancer-out.txt +++ b/test/BaselineOutput/SingleDebug/PriorPredictor/BinaryPrior-CV-breast-cancer-out.txt @@ -42,7 +42,7 @@ Negative precision: 0.656163 (0.0263) Negative recall: 1.000000 (0.0000) Log-loss: 0.935104 (0.0247) Log-loss reduction: -0.009590 (0.0001) -F1 Score: NaN (NaN) +F1 Score: 0.000000 (0.0000) AUPRC: 0.418968 (0.0212) --------------------------------------- diff --git a/test/BaselineOutput/SingleDebug/PriorPredictor/BinaryPrior-CV-breast-cancer-rp.txt b/test/BaselineOutput/SingleDebug/PriorPredictor/BinaryPrior-CV-breast-cancer-rp.txt index ca8471a9a8..b3fd773ef4 100644 --- a/test/BaselineOutput/SingleDebug/PriorPredictor/BinaryPrior-CV-breast-cancer-rp.txt +++ b/test/BaselineOutput/SingleDebug/PriorPredictor/BinaryPrior-CV-breast-cancer-rp.txt @@ -1,4 +1,4 @@ PriorPredictor AUC Accuracy Positive precision Positive recall Negative precision Negative recall Log-loss Log-loss reduction F1 Score AUPRC Learner Name Train Dataset Test Dataset Results File Run Time Physical Memory Virtual Memory Command Line Settings -0.5 0.656163 0 0 0.656163 1 0.935104 -0.00959 NaN 0.418968 PriorPredictor %Data% %Output% 99 0 0 maml.exe CV tr=PriorPredictor threads=- dout=%Output% loader=Text{col=Label:BL:0 col=Features:~} data=%Data% seed=1 +0.5 0.656163 0 0 0.656163 1 0.935104 -0.00959 0 0.418968 PriorPredictor %Data% %Output% 99 0 0 maml.exe CV tr=PriorPredictor threads=- dout=%Output% loader=Text{col=Label:BL:0 col=Features:~} data=%Data% seed=1 diff --git a/test/BaselineOutput/SingleDebug/PriorPredictor/BinaryPrior-TrainTest-breast-cancer-out.txt b/test/BaselineOutput/SingleDebug/PriorPredictor/BinaryPrior-TrainTest-breast-cancer-out.txt index 529400420f..d24983ea8e 100644 --- a/test/BaselineOutput/SingleDebug/PriorPredictor/BinaryPrior-TrainTest-breast-cancer-out.txt +++ b/test/BaselineOutput/SingleDebug/PriorPredictor/BinaryPrior-TrainTest-breast-cancer-out.txt @@ -26,7 +26,7 @@ Negative precision: 0.655222 (0.0000) Negative recall: 1.000000 (0.0000) Log-loss: 0.929318 (0.0000) Log-loss reduction: 0.000000 (0.0000) -F1 Score: NaN (0.0000) +F1 Score: 0.000000 (0.0000) AUPRC: 0.415719 (0.0000) --------------------------------------- diff --git a/test/BaselineOutput/SingleDebug/PriorPredictor/BinaryPrior-TrainTest-breast-cancer-rp.txt b/test/BaselineOutput/SingleDebug/PriorPredictor/BinaryPrior-TrainTest-breast-cancer-rp.txt index efb8a13e71..bfad05ac8e 100644 --- a/test/BaselineOutput/SingleDebug/PriorPredictor/BinaryPrior-TrainTest-breast-cancer-rp.txt +++ b/test/BaselineOutput/SingleDebug/PriorPredictor/BinaryPrior-TrainTest-breast-cancer-rp.txt @@ -1,4 +1,4 @@ PriorPredictor AUC Accuracy Positive precision Positive recall Negative precision Negative recall Log-loss Log-loss reduction F1 Score AUPRC Learner Name Train Dataset Test Dataset Results File Run Time Physical Memory Virtual Memory Command Line Settings -0.5 0.655222 0 0 0.655222 1 0.929318 0 NaN 0.415719 PriorPredictor %Data% %Data% %Output% 99 0 0 maml.exe TrainTest test=%Data% tr=PriorPredictor dout=%Output% loader=Text{col=Label:BL:0 col=Features:~} data=%Data% out=%Output% seed=1 +0.5 0.655222 0 0 0.655222 1 0.929318 0 0 0.415719 PriorPredictor %Data% %Data% %Output% 99 0 0 maml.exe TrainTest test=%Data% tr=PriorPredictor dout=%Output% loader=Text{col=Label:BL:0 col=Features:~} data=%Data% out=%Output% seed=1 diff --git a/test/BaselineOutput/SingleRelease/PriorPredictor/BinaryPrior-CV-breast-cancer-out.txt b/test/BaselineOutput/SingleRelease/PriorPredictor/BinaryPrior-CV-breast-cancer-out.txt index 8a5c288d72..5b8c118d2b 100644 --- a/test/BaselineOutput/SingleRelease/PriorPredictor/BinaryPrior-CV-breast-cancer-out.txt +++ b/test/BaselineOutput/SingleRelease/PriorPredictor/BinaryPrior-CV-breast-cancer-out.txt @@ -42,7 +42,7 @@ Negative precision: 0.656163 (0.0263) Negative recall: 1.000000 (0.0000) Log-loss: 0.935104 (0.0247) Log-loss reduction: -0.009590 (0.0001) -F1 Score: NaN (NaN) +F1 Score: 0.000000 (0.0000) AUPRC: 0.418968 (0.0212) --------------------------------------- diff --git a/test/BaselineOutput/SingleRelease/PriorPredictor/BinaryPrior-CV-breast-cancer-rp.txt b/test/BaselineOutput/SingleRelease/PriorPredictor/BinaryPrior-CV-breast-cancer-rp.txt index ca8471a9a8..b3fd773ef4 100644 --- a/test/BaselineOutput/SingleRelease/PriorPredictor/BinaryPrior-CV-breast-cancer-rp.txt +++ b/test/BaselineOutput/SingleRelease/PriorPredictor/BinaryPrior-CV-breast-cancer-rp.txt @@ -1,4 +1,4 @@ PriorPredictor AUC Accuracy Positive precision Positive recall Negative precision Negative recall Log-loss Log-loss reduction F1 Score AUPRC Learner Name Train Dataset Test Dataset Results File Run Time Physical Memory Virtual Memory Command Line Settings -0.5 0.656163 0 0 0.656163 1 0.935104 -0.00959 NaN 0.418968 PriorPredictor %Data% %Output% 99 0 0 maml.exe CV tr=PriorPredictor threads=- dout=%Output% loader=Text{col=Label:BL:0 col=Features:~} data=%Data% seed=1 +0.5 0.656163 0 0 0.656163 1 0.935104 -0.00959 0 0.418968 PriorPredictor %Data% %Output% 99 0 0 maml.exe CV tr=PriorPredictor threads=- dout=%Output% loader=Text{col=Label:BL:0 col=Features:~} data=%Data% seed=1 diff --git a/test/BaselineOutput/SingleRelease/PriorPredictor/BinaryPrior-TrainTest-breast-cancer-out.txt b/test/BaselineOutput/SingleRelease/PriorPredictor/BinaryPrior-TrainTest-breast-cancer-out.txt index 529400420f..d24983ea8e 100644 --- a/test/BaselineOutput/SingleRelease/PriorPredictor/BinaryPrior-TrainTest-breast-cancer-out.txt +++ b/test/BaselineOutput/SingleRelease/PriorPredictor/BinaryPrior-TrainTest-breast-cancer-out.txt @@ -26,7 +26,7 @@ Negative precision: 0.655222 (0.0000) Negative recall: 1.000000 (0.0000) Log-loss: 0.929318 (0.0000) Log-loss reduction: 0.000000 (0.0000) -F1 Score: NaN (0.0000) +F1 Score: 0.000000 (0.0000) AUPRC: 0.415719 (0.0000) --------------------------------------- diff --git a/test/BaselineOutput/SingleRelease/PriorPredictor/BinaryPrior-TrainTest-breast-cancer-rp.txt b/test/BaselineOutput/SingleRelease/PriorPredictor/BinaryPrior-TrainTest-breast-cancer-rp.txt index efb8a13e71..bfad05ac8e 100644 --- a/test/BaselineOutput/SingleRelease/PriorPredictor/BinaryPrior-TrainTest-breast-cancer-rp.txt +++ b/test/BaselineOutput/SingleRelease/PriorPredictor/BinaryPrior-TrainTest-breast-cancer-rp.txt @@ -1,4 +1,4 @@ PriorPredictor AUC Accuracy Positive precision Positive recall Negative precision Negative recall Log-loss Log-loss reduction F1 Score AUPRC Learner Name Train Dataset Test Dataset Results File Run Time Physical Memory Virtual Memory Command Line Settings -0.5 0.655222 0 0 0.655222 1 0.929318 0 NaN 0.415719 PriorPredictor %Data% %Data% %Output% 99 0 0 maml.exe TrainTest test=%Data% tr=PriorPredictor dout=%Output% loader=Text{col=Label:BL:0 col=Features:~} data=%Data% out=%Output% seed=1 +0.5 0.655222 0 0 0.655222 1 0.929318 0 0 0.415719 PriorPredictor %Data% %Data% %Output% 99 0 0 maml.exe TrainTest test=%Data% tr=PriorPredictor dout=%Output% loader=Text{col=Label:BL:0 col=Features:~} data=%Data% out=%Output% seed=1