diff --git a/Documentation/MSBuildIntegration.md b/Documentation/MSBuildIntegration.md index e036c0a51..cf4d5bb1e 100644 --- a/Documentation/MSBuildIntegration.md +++ b/Documentation/MSBuildIntegration.md @@ -98,7 +98,12 @@ The above command will automatically fail the build if the line, branch or metho dotnet test /p:CollectCoverage=true /p:Threshold=80 /p:ThresholdType=line ``` -You can specify multiple values for `ThresholdType` by separating them with commas. Valid values include `line`, `branch` and `method`. +You can specify multiple values for `ThresholdType` by separating them with commas. Valid values include `line`, `branch` and `method`. +You can do the same for `Threshold` as well. + +```bash +dotnet test /p:CollectCoverage=true /p:Threshold="80,100,70", /p:ThresholdType="line,branch,method" +``` By default, Coverlet will validate the threshold value against the coverage result of each module. The `/p:ThresholdStat` option allows you to change this behaviour and can have any of the following values: diff --git a/src/coverlet.console/Program.cs b/src/coverlet.console/Program.cs index 344ac0c5c..8ec689e3c 100644 --- a/src/coverlet.console/Program.cs +++ b/src/coverlet.console/Program.cs @@ -3,6 +3,7 @@ using System.ComponentModel; using System.Diagnostics; using System.IO; +using System.Linq; using System.Text; using ConsoleTables; @@ -129,13 +130,13 @@ static int Main(string[] args) process.WaitForExit(); var dOutput = output.HasValue() ? output.Value() : Directory.GetCurrentDirectory() + Path.DirectorySeparatorChar.ToString(); - var dThreshold = threshold.HasValue() ? double.Parse(threshold.Value()) : 0; var dThresholdTypes = thresholdTypes.HasValue() ? thresholdTypes.Values : new List(new string[] { "line", "branch", "method" }); var dThresholdStat = thresholdStat.HasValue() ? Enum.Parse(thresholdStat.Value(), true) : Enum.Parse("minimum", true); logger.LogInformation("\nCalculating coverage result..."); var result = coverage.GetCoverageResult(); + var directory = Path.GetDirectoryName(dOutput); if (directory == string.Empty) { @@ -173,27 +174,57 @@ static int Main(string[] args) } } - var thresholdTypeFlags = ThresholdTypeFlags.None; - + var thresholdTypeFlagQueue = new Queue(); + foreach (var thresholdType in dThresholdTypes) { if (thresholdType.Equals("line", StringComparison.OrdinalIgnoreCase)) { - thresholdTypeFlags |= ThresholdTypeFlags.Line; + thresholdTypeFlagQueue.Enqueue(ThresholdTypeFlags.Line); } else if (thresholdType.Equals("branch", StringComparison.OrdinalIgnoreCase)) { - thresholdTypeFlags |= ThresholdTypeFlags.Branch; + thresholdTypeFlagQueue.Enqueue(ThresholdTypeFlags.Branch); } else if (thresholdType.Equals("method", StringComparison.OrdinalIgnoreCase)) { - thresholdTypeFlags |= ThresholdTypeFlags.Method; + thresholdTypeFlagQueue.Enqueue(ThresholdTypeFlags.Method); + } + } + + Dictionary thresholdTypeFlagValues = new Dictionary(); + if (threshold.HasValue() && threshold.Value().Contains(',')) + { + var thresholdValues = threshold.Value().Split(',', StringSplitOptions.RemoveEmptyEntries).Select(t => t.Trim()); + if (thresholdValues.Count() != thresholdTypeFlagQueue.Count()) + { + throw new Exception($"Threshold type flag count ({thresholdTypeFlagQueue.Count()}) and values count ({thresholdValues.Count()}) doesn't match"); + } + + foreach (var thresholdValue in thresholdValues) + { + if (double.TryParse(thresholdValue, out var value)) + { + thresholdTypeFlagValues[thresholdTypeFlagQueue.Dequeue()] = value; + } + else + { + throw new Exception($"Invalid threshold value must be numeric"); + } + } + } + else + { + double thresholdValue = threshold.HasValue() ? double.Parse(threshold.Value()) : 0; + + while (thresholdTypeFlagQueue.Any()) + { + thresholdTypeFlagValues[thresholdTypeFlagQueue.Dequeue()] = thresholdValue; } } var coverageTable = new ConsoleTable("Module", "Line", "Branch", "Method"); var summary = new CoverageSummary(); - int numModules = result.Modules.Count; var linePercentCalculation = summary.CalculateLineCoverage(result.Modules); var branchPercentCalculation = summary.CalculateBranchCoverage(result.Modules); @@ -230,26 +261,26 @@ static int Main(string[] args) { exitCode += (int)CommandExitCodes.TestFailed; } - thresholdTypeFlags = result.GetThresholdTypesBelowThreshold(summary, dThreshold, thresholdTypeFlags, dThresholdStat); + + var thresholdTypeFlags = result.GetThresholdTypesBelowThreshold(summary, thresholdTypeFlagValues, dThresholdStat); if (thresholdTypeFlags != ThresholdTypeFlags.None) { exitCode += (int)CommandExitCodes.CoverageBelowThreshold; var exceptionMessageBuilder = new StringBuilder(); if ((thresholdTypeFlags & ThresholdTypeFlags.Line) != ThresholdTypeFlags.None) { - exceptionMessageBuilder.AppendLine($"The {dThresholdStat.ToString().ToLower()} line coverage is below the specified {dThreshold}"); + exceptionMessageBuilder.AppendLine($"The {dThresholdStat.ToString().ToLower()} line coverage is below the specified {thresholdTypeFlagValues[ThresholdTypeFlags.Line]}"); } if ((thresholdTypeFlags & ThresholdTypeFlags.Branch) != ThresholdTypeFlags.None) { - exceptionMessageBuilder.AppendLine($"The {dThresholdStat.ToString().ToLower()} branch coverage is below the specified {dThreshold}"); + exceptionMessageBuilder.AppendLine($"The {dThresholdStat.ToString().ToLower()} branch coverage is below the specified {thresholdTypeFlagValues[ThresholdTypeFlags.Branch]}"); } if ((thresholdTypeFlags & ThresholdTypeFlags.Method) != ThresholdTypeFlags.None) { - exceptionMessageBuilder.AppendLine($"The {dThresholdStat.ToString().ToLower()} method coverage is below the specified {dThreshold}"); + exceptionMessageBuilder.AppendLine($"The {dThresholdStat.ToString().ToLower()} method coverage is below the specified {thresholdTypeFlagValues[ThresholdTypeFlags.Method]}"); } - throw new Exception(exceptionMessageBuilder.ToString()); } diff --git a/src/coverlet.core/CoverageResult.cs b/src/coverlet.core/CoverageResult.cs index 470bb6c6a..e9a4b0391 100644 --- a/src/coverlet.core/CoverageResult.cs +++ b/src/coverlet.core/CoverageResult.cs @@ -111,7 +111,7 @@ internal void Merge(Modules modules) } } - public ThresholdTypeFlags GetThresholdTypesBelowThreshold(CoverageSummary summary, double threshold, ThresholdTypeFlags thresholdTypes, ThresholdStatistic thresholdStat) + public ThresholdTypeFlags GetThresholdTypesBelowThreshold(CoverageSummary summary, Dictionary thresholdTypeFlagValues, ThresholdStatistic thresholdStat) { var thresholdTypeFlags = ThresholdTypeFlags.None; switch (thresholdStat) @@ -123,23 +123,20 @@ public ThresholdTypeFlags GetThresholdTypesBelowThreshold(CoverageSummary summar double line = summary.CalculateLineCoverage(module.Value).Percent; double branch = summary.CalculateBranchCoverage(module.Value).Percent; double method = summary.CalculateMethodCoverage(module.Value).Percent; - - if ((thresholdTypes & ThresholdTypeFlags.Line) != ThresholdTypeFlags.None) + + if (thresholdTypeFlagValues.TryGetValue(ThresholdTypeFlags.Line, out var lineThresholdValue) && lineThresholdValue > line) { - if (line < threshold) - thresholdTypeFlags |= ThresholdTypeFlags.Line; + thresholdTypeFlags |= ThresholdTypeFlags.Line; } - if ((thresholdTypes & ThresholdTypeFlags.Branch) != ThresholdTypeFlags.None) + if (thresholdTypeFlagValues.TryGetValue(ThresholdTypeFlags.Branch, out var branchThresholdValue) && branchThresholdValue > branch) { - if (branch < threshold) - thresholdTypeFlags |= ThresholdTypeFlags.Branch; + thresholdTypeFlags |= ThresholdTypeFlags.Branch; } - if ((thresholdTypes & ThresholdTypeFlags.Method) != ThresholdTypeFlags.None) + if (thresholdTypeFlagValues.TryGetValue(ThresholdTypeFlags.Method, out var methodThresholdValue) && methodThresholdValue > method) { - if (method < threshold) - thresholdTypeFlags |= ThresholdTypeFlags.Method; + thresholdTypeFlags |= ThresholdTypeFlags.Method; } } } @@ -149,23 +146,20 @@ public ThresholdTypeFlags GetThresholdTypesBelowThreshold(CoverageSummary summar double line = summary.CalculateLineCoverage(Modules).AverageModulePercent; double branch = summary.CalculateBranchCoverage(Modules).AverageModulePercent; double method = summary.CalculateMethodCoverage(Modules).AverageModulePercent; - - if ((thresholdTypes & ThresholdTypeFlags.Line) != ThresholdTypeFlags.None) + + if (thresholdTypeFlagValues.TryGetValue(ThresholdTypeFlags.Line, out var lineThresholdValue) && lineThresholdValue > line) { - if (line < threshold) - thresholdTypeFlags |= ThresholdTypeFlags.Line; + thresholdTypeFlags |= ThresholdTypeFlags.Line; } - if ((thresholdTypes & ThresholdTypeFlags.Branch) != ThresholdTypeFlags.None) + if (thresholdTypeFlagValues.TryGetValue(ThresholdTypeFlags.Branch, out var branchThresholdValue) && branchThresholdValue > branch) { - if (branch < threshold) - thresholdTypeFlags |= ThresholdTypeFlags.Branch; + thresholdTypeFlags |= ThresholdTypeFlags.Branch; } - if ((thresholdTypes & ThresholdTypeFlags.Method) != ThresholdTypeFlags.None) + if (thresholdTypeFlagValues.TryGetValue(ThresholdTypeFlags.Method, out var methodThresholdValue) && methodThresholdValue > method) { - if (method < threshold) - thresholdTypeFlags |= ThresholdTypeFlags.Method; + thresholdTypeFlags |= ThresholdTypeFlags.Method; } } break; @@ -175,22 +169,19 @@ public ThresholdTypeFlags GetThresholdTypesBelowThreshold(CoverageSummary summar double branch = summary.CalculateBranchCoverage(Modules).Percent; double method = summary.CalculateMethodCoverage(Modules).Percent; - if ((thresholdTypes & ThresholdTypeFlags.Line) != ThresholdTypeFlags.None) + if (thresholdTypeFlagValues.TryGetValue(ThresholdTypeFlags.Line, out var lineThresholdValue) && lineThresholdValue > line) { - if (line < threshold) - thresholdTypeFlags |= ThresholdTypeFlags.Line; + thresholdTypeFlags |= ThresholdTypeFlags.Line; } - if ((thresholdTypes & ThresholdTypeFlags.Branch) != ThresholdTypeFlags.None) + if (thresholdTypeFlagValues.TryGetValue(ThresholdTypeFlags.Branch, out var branchThresholdValue) && branchThresholdValue > branch) { - if (branch < threshold) - thresholdTypeFlags |= ThresholdTypeFlags.Branch; + thresholdTypeFlags |= ThresholdTypeFlags.Branch; } - if ((thresholdTypes & ThresholdTypeFlags.Method) != ThresholdTypeFlags.None) + if (thresholdTypeFlagValues.TryGetValue(ThresholdTypeFlags.Method, out var methodThresholdValue) && methodThresholdValue > method) { - if (method < threshold) - thresholdTypeFlags |= ThresholdTypeFlags.Method; + thresholdTypeFlags |= ThresholdTypeFlags.Method; } } break; diff --git a/src/coverlet.msbuild.tasks/CoverageResultTask.cs b/src/coverlet.msbuild.tasks/CoverageResultTask.cs index fd3cdd9ca..2f50130e7 100644 --- a/src/coverlet.msbuild.tasks/CoverageResultTask.cs +++ b/src/coverlet.msbuild.tasks/CoverageResultTask.cs @@ -25,7 +25,7 @@ public class CoverageResultTask : BaseTask public string OutputFormat { get; set; } [Required] - public double Threshold { get; set; } + public string Threshold { get; set; } [Required] public string ThresholdType { get; set; } @@ -124,25 +124,56 @@ public override bool Execute() ReportItems = coverageReportPaths.ToArray(); - var thresholdTypeFlags = ThresholdTypeFlags.None; - var thresholdStat = ThresholdStatistic.Minimum; + var thresholdTypeFlagQueue = new Queue(); foreach (var thresholdType in ThresholdType.Split(',').Select(t => t.Trim())) { if (thresholdType.Equals("line", StringComparison.OrdinalIgnoreCase)) { - thresholdTypeFlags |= ThresholdTypeFlags.Line; + thresholdTypeFlagQueue.Enqueue(ThresholdTypeFlags.Line); } else if (thresholdType.Equals("branch", StringComparison.OrdinalIgnoreCase)) { - thresholdTypeFlags |= ThresholdTypeFlags.Branch; + thresholdTypeFlagQueue.Enqueue(ThresholdTypeFlags.Branch); } else if (thresholdType.Equals("method", StringComparison.OrdinalIgnoreCase)) { - thresholdTypeFlags |= ThresholdTypeFlags.Method; + thresholdTypeFlagQueue.Enqueue(ThresholdTypeFlags.Method); } } + + Dictionary thresholdTypeFlagValues = new Dictionary(); + if (Threshold.Contains(',')) + { + var thresholdValues = Threshold.Split(new char[] {','}, StringSplitOptions.RemoveEmptyEntries).Select(t => t.Trim()); + if(thresholdValues.Count() != thresholdTypeFlagQueue.Count()) + { + throw new Exception($"Threshold type flag count ({thresholdTypeFlagQueue.Count()}) and values count ({thresholdValues.Count()}) doesn't match"); + } + foreach (var threshold in thresholdValues) + { + if (double.TryParse(threshold, out var value)) + { + thresholdTypeFlagValues[thresholdTypeFlagQueue.Dequeue()] = value; + } + else + { + throw new Exception($"Invalid threshold value must be numeric"); + } + } + } + else + { + double thresholdValue = double.Parse(Threshold); + + while (thresholdTypeFlagQueue.Any()) + { + thresholdTypeFlagValues[thresholdTypeFlagQueue.Dequeue()] = thresholdValue; + } + } + + var thresholdStat = ThresholdStatistic.Minimum; if (ThresholdStat.Equals("average", StringComparison.OrdinalIgnoreCase)) { thresholdStat = ThresholdStatistic.Average; @@ -154,7 +185,6 @@ public override bool Execute() var coverageTable = new ConsoleTable("Module", "Line", "Branch", "Method"); var summary = new CoverageSummary(); - int numModules = result.Modules.Count; var linePercentCalculation = summary.CalculateLineCoverage(result.Modules); var branchPercentCalculation = summary.CalculateBranchCoverage(result.Modules); @@ -189,23 +219,26 @@ public override bool Execute() Console.WriteLine(coverageTable.ToStringAlternative()); - thresholdTypeFlags = result.GetThresholdTypesBelowThreshold(summary, Threshold, thresholdTypeFlags, thresholdStat); + var thresholdTypeFlags = result.GetThresholdTypesBelowThreshold(summary, thresholdTypeFlagValues, thresholdStat); if (thresholdTypeFlags != ThresholdTypeFlags.None) { var exceptionMessageBuilder = new StringBuilder(); if ((thresholdTypeFlags & ThresholdTypeFlags.Line) != ThresholdTypeFlags.None) { - exceptionMessageBuilder.AppendLine($"The {thresholdStat.ToString().ToLower()} line coverage is below the specified {Threshold}"); + exceptionMessageBuilder.AppendLine( + $"The {thresholdStat.ToString().ToLower()} line coverage is below the specified {thresholdTypeFlagValues[ThresholdTypeFlags.Line]}"); } if ((thresholdTypeFlags & ThresholdTypeFlags.Branch) != ThresholdTypeFlags.None) { - exceptionMessageBuilder.AppendLine($"The {thresholdStat.ToString().ToLower()} branch coverage is below the specified {Threshold}"); + exceptionMessageBuilder.AppendLine( + $"The {thresholdStat.ToString().ToLower()} branch coverage is below the specified {thresholdTypeFlagValues[ThresholdTypeFlags.Branch]}"); } if ((thresholdTypeFlags & ThresholdTypeFlags.Method) != ThresholdTypeFlags.None) { - exceptionMessageBuilder.AppendLine($"The {thresholdStat.ToString().ToLower()} method coverage is below the specified {Threshold}"); + exceptionMessageBuilder.AppendLine( + $"The {thresholdStat.ToString().ToLower()} method coverage is below the specified {thresholdTypeFlagValues[ThresholdTypeFlags.Method]}"); } throw new Exception(exceptionMessageBuilder.ToString()); diff --git a/test/coverlet.core.tests/CoverageResultTests.cs b/test/coverlet.core.tests/CoverageResultTests.cs new file mode 100644 index 000000000..c02d02ce4 --- /dev/null +++ b/test/coverlet.core.tests/CoverageResultTests.cs @@ -0,0 +1,152 @@ +using System.Collections.Generic; +using Coverlet.Core.Enums; +using Xunit; + +namespace Coverlet.Core.Tests +{ + public class CoverageResultTests + { + private Modules _modules; + + public CoverageResultTests() + { + Lines lines = new Lines(); + lines.Add(1, 1); + lines.Add(2, 1); + lines.Add(3, 1); + Branches branches = new Branches(); + branches.Add(new BranchInfo { Line = 1, Hits = 1, Offset = 1, Path = 0, Ordinal = 1 }); + branches.Add(new BranchInfo { Line = 1, Hits = 1, Offset = 1, Path = 1, Ordinal = 2 }); + branches.Add(new BranchInfo { Line = 2, Hits = 0, Offset = 1, Path = 0, Ordinal = 1 }); + + // System.Void Coverlet.Core.Tests.CoverageResultTests::CoverageResultTests - 3/3 100% line 2/3 66.7% branch coverage + Methods methods = new Methods(); + var methodString = "System.Void Coverlet.Core.Tests.CoverageResultTests::CoverageResultTests()"; + methods.Add(methodString, new Method()); + methods[methodString].Lines = lines; + methods[methodString].Branches = branches; + + // System.Void Coverlet.Core.Tests.CoverageResultTests::GetThresholdTypesBelowThreshold - 0/2 0% line + methodString = "System.Void Coverlet.Core.Tests.CoverageResultTests::GetThresholdTypesBelowThreshold()"; + methods.Add(methodString, new Method()); + methods[methodString].Lines = new Lines() + { + {1, 0}, + {2, 0}, + }; + + Classes classes = new Classes(); + classes.Add("Coverlet.Core.Tests.CoverageResultTests", methods); + // Methods - 1/2 (50%) + // Lines - 3/5 (60%) + // Branches - 2/3 (66.67%) + + Documents documents = new Documents(); + documents.Add("doc.cs", classes); + + _modules = new Modules(); + _modules.Add("module", documents); + } + + [Fact] + public void TestGetThresholdTypesBelowThresholdLine() + { + CoverageResult result = new CoverageResult(); + result.Modules = _modules; + + CoverageSummary summary = new CoverageSummary(); + Dictionary thresholdTypeFlagValues = new Dictionary() + { + { ThresholdTypeFlags.Line, 90 }, + { ThresholdTypeFlags.Method, 10 }, + { ThresholdTypeFlags.Branch, 10 }, + }; + + ThresholdStatistic thresholdStatic = ThresholdStatistic.Minimum; + + ThresholdTypeFlags resThresholdTypeFlags = result.GetThresholdTypesBelowThreshold(summary, thresholdTypeFlagValues, thresholdStatic); + Assert.Equal(ThresholdTypeFlags.Line, resThresholdTypeFlags); + } + + [Fact] + public void TestGetThresholdTypesBelowThresholdMethod() + { + CoverageResult result = new CoverageResult(); + result.Modules = _modules; + + CoverageSummary summary = new CoverageSummary(); + Dictionary thresholdTypeFlagValues = new Dictionary() + { + { ThresholdTypeFlags.Line, 50 }, + { ThresholdTypeFlags.Method, 75 }, + { ThresholdTypeFlags.Branch, 10 }, + }; + + ThresholdStatistic thresholdStatic = ThresholdStatistic.Minimum; + + ThresholdTypeFlags resThresholdTypeFlags = result.GetThresholdTypesBelowThreshold(summary, thresholdTypeFlagValues, thresholdStatic); + Assert.Equal(ThresholdTypeFlags.Method, resThresholdTypeFlags); + } + + [Fact] + public void TestGetThresholdTypesBelowThresholdBranch() + { + CoverageResult result = new CoverageResult(); + result.Modules = _modules; + + CoverageSummary summary = new CoverageSummary(); + Dictionary thresholdTypeFlagValues = new Dictionary() + { + { ThresholdTypeFlags.Line, 50 }, + { ThresholdTypeFlags.Method, 50 }, + { ThresholdTypeFlags.Branch, 90 }, + }; + + ThresholdStatistic thresholdStatic = ThresholdStatistic.Total; + + ThresholdTypeFlags resThresholdTypeFlags = result.GetThresholdTypesBelowThreshold(summary, thresholdTypeFlagValues, thresholdStatic); + Assert.Equal(ThresholdTypeFlags.Branch, resThresholdTypeFlags); + } + + [Fact] + public void TestGetThresholdTypesBelowThresholdAllGood() + { + CoverageResult result = new CoverageResult(); + result.Modules = _modules; + + CoverageSummary summary = new CoverageSummary(); + Dictionary thresholdTypeFlagValues = new Dictionary() + { + { ThresholdTypeFlags.Line, 50 }, + { ThresholdTypeFlags.Method, 50 }, + { ThresholdTypeFlags.Branch, 50 }, + }; + + ThresholdStatistic thresholdStatic = ThresholdStatistic.Average; + + ThresholdTypeFlags resThresholdTypeFlags = result.GetThresholdTypesBelowThreshold(summary, thresholdTypeFlagValues, thresholdStatic); + Assert.Equal(ThresholdTypeFlags.None, resThresholdTypeFlags); + } + + [Fact] + public void TestGetThresholdTypesBelowThresholdAllFail() + { + CoverageResult result = new CoverageResult(); + result.Modules = _modules; + + CoverageSummary summary = new CoverageSummary(); + Dictionary thresholdTypeFlagValues = new Dictionary() + { + { ThresholdTypeFlags.Line, 100 }, + { ThresholdTypeFlags.Method, 100 }, + { ThresholdTypeFlags.Branch, 100 }, + }; + + ThresholdTypeFlags thresholdTypeFlags = ThresholdTypeFlags.Line | ThresholdTypeFlags.Branch | ThresholdTypeFlags.Method; + ThresholdStatistic thresholdStatic = ThresholdStatistic.Minimum; + + ThresholdTypeFlags resThresholdTypeFlags = result.GetThresholdTypesBelowThreshold(summary, thresholdTypeFlagValues, thresholdStatic); + Assert.Equal(thresholdTypeFlags, resThresholdTypeFlags); + } + } +} \ No newline at end of file