diff --git a/Documentation/GlobalTool.md b/Documentation/GlobalTool.md index 84c1bbf25..1ca5f3852 100644 --- a/Documentation/GlobalTool.md +++ b/Documentation/GlobalTool.md @@ -9,12 +9,12 @@ coverlet --help The current options are (output of `coverlet --help`): ```bash -Cross platform .NET Core code coverage tool 1.0.0.0 +Cross platform .NET Core code coverage tool 3.0.0.0 Usage: coverlet [arguments] [options] Arguments: - Path to the test assembly. + Path to the test assembly or application directory. Options: -h|--help Show help information @@ -23,21 +23,21 @@ Options: -a|--targetargs Arguments to be passed to the test runner. -o|--output Output of the generated coverage report -v|--verbosity Sets the verbosity level of the command. Allowed values are quiet, minimal, normal, detailed. - -f|--format Format of the generated coverage report[multiple value]. + -f|--format Format of the generated coverage report. --threshold Exits with error if the coverage % is below value. - --threshold-type Coverage type to apply the threshold to[multiple value]. + --threshold-type Coverage type to apply the threshold to. --threshold-stat Coverage statistic used to enforce the threshold value. - --exclude Filter expressions to exclude specific modules and types[multiple value]. - --include Filter expressions to include specific modules and types[multiple value]. - --include-directory Include directories containing additional assemblies to be instrumented[multiple value]. - --exclude-by-file Glob patterns specifying source files to exclude[multiple value]. - --exclude-by-attribute Attributes to exclude from code coverage[multiple value]. + --exclude Filter expressions to exclude specific modules and types. + --include Filter expressions to include only specific modules and types. + --exclude-by-file Glob patterns specifying source files to exclude. + --include-directory Include directories containing additional assemblies to be instrumented. + --exclude-by-attribute Attributes to exclude from code coverage. --include-test-assembly Specifies whether to report code coverage of the test assembly. - --single-hit Specifies whether to limit code coverage hit reporting to a single hit for each location. + --single-hit Specifies whether to limit code coverage hit reporting to a single hit for each location + --skipautoprops Neither track nor record auto-implemented properties. --merge-with Path to existing coverage result to merge. --use-source-link Specifies whether to use SourceLink URIs in place of file system paths. - --skipautoprops Neither track nor record auto-implemented properties. - --does-not-return-attribute Attributes that mark methods that do not return[multiple value]. + --does-not-return-attribute Attributes that mark methods that do not return. ``` NB. For a [multiple value] options you have to specify values multiple times i.e. @@ -60,6 +60,20 @@ After the above command is run, a `coverage.json` file containing the results wi _Note: The `--no-build` flag is specified so that the `/path/to/test-assembly.dll` isn't rebuilt_ +## Code Coverage for integration tests and end-to-end tests. + +Sometimes, there are tests that doesn't use regular unit test frameworks like xunit. You may find yourself in a situation where your tests are driven by a custom executable/script, which when run, could do anything from making API calls to driving Selenium. + +As an example, suppose you have a folder `/integrationtest` which contains said executable (lets call it `runner.exe`) and everything it needs to successfully execute. You can use our tool to startup the executable and gather live coverage: + +```bash +coverlet "/integrationtest" --target "/application/runner.exe" +``` + +Coverlet will first instrument all .NET assemblies within the `integrationtests` folder, after which it will execute `runner.exe`. Finally, at shutdown of your `runner.exe`, it will generate the coverage report. You can use all parameters available to customize the report generation. Coverage results will be generated once `runner.exe` exits. You can use all parameters available to customize the report generation. + +_Note: Today, Coverlet relies on `AppDomain.CurrentDomain.ProcessExit` and `AppDomain.CurrentDomain.DomainUnload` to record hits to the filesystem, as a result, you need to ensure a graceful process shutdown. Forcefully, killing the process will result in an incomplete coverage report._ + ## Coverage Output Coverlet can generate coverage results in multiple formats, which is specified using the `--format` or `-f` options. For example, the following command emits coverage results in the `opencover` format instead of `json`: diff --git a/README.md b/README.md index 0c17bba7e..f07d36bbf 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ Coverlet can be used through three different *drivers* * VSTest engine integration * MSBuild task integration -* As a .NET Global tool +* As a .NET Global tool (supports standalone integration tests) Coverlet supports only SDK-style projects https://docs.microsoft.com/en-us/visualstudio/msbuild/how-to-use-project-sdk?view=vs-2019 diff --git a/src/coverlet.console/Program.cs b/src/coverlet.console/Program.cs index 1b4cbf219..344ac0c5c 100644 --- a/src/coverlet.console/Program.cs +++ b/src/coverlet.console/Program.cs @@ -44,7 +44,7 @@ static int Main(string[] args) app.VersionOption("-v|--version", GetAssemblyVersion()); int exitCode = (int)CommandExitCodes.Success; - CommandArgument module = app.Argument("", "Path to the test assembly."); + CommandArgument moduleOrAppDirectory = app.Argument("", "Path to the test assembly or application directory."); CommandOption target = app.Option("-t|--target", "Path to the test runner application.", CommandOptionType.SingleValue); CommandOption targs = app.Option("-a|--targetargs", "Arguments to be passed to the test runner.", CommandOptionType.SingleValue); CommandOption output = app.Option("-o|--output", "Output of the generated coverage report", CommandOptionType.SingleValue); @@ -67,8 +67,8 @@ static int Main(string[] args) app.OnExecute(() => { - if (string.IsNullOrEmpty(module.Value) || string.IsNullOrWhiteSpace(module.Value)) - throw new CommandParsingException(app, "No test assembly specified."); + if (string.IsNullOrEmpty(moduleOrAppDirectory.Value) || string.IsNullOrWhiteSpace(moduleOrAppDirectory.Value)) + throw new CommandParsingException(app, "No test assembly or application directory specified."); if (!target.HasValue()) throw new CommandParsingException(app, "Target must be specified."); @@ -94,7 +94,7 @@ static int Main(string[] args) DoesNotReturnAttributes = doesNotReturnAttributes.Values.ToArray() }; - Coverage coverage = new Coverage(module.Value, + Coverage coverage = new Coverage(moduleOrAppDirectory.Value, parameters, logger, serviceProvider.GetRequiredService(), diff --git a/src/coverlet.core/Coverage.cs b/src/coverlet.core/Coverage.cs index b71bb46d8..d64290b43 100644 --- a/src/coverlet.core/Coverage.cs +++ b/src/coverlet.core/Coverage.cs @@ -29,7 +29,7 @@ internal class CoverageParameters internal class Coverage { - private string _module; + private string _moduleOrAppDirectory; private string _identifier; private string[] _includeFilters; private string[] _includeDirectories; @@ -54,7 +54,7 @@ public string Identifier get { return _identifier; } } - public Coverage(string module, + public Coverage(string moduleOrDirectory, CoverageParameters parameters, ILogger logger, IInstrumentationHelper instrumentationHelper, @@ -62,7 +62,7 @@ public Coverage(string module, ISourceRootTranslator sourceRootTranslator, ICecilSymbolHelper cecilSymbolHelper) { - _module = module; + _moduleOrAppDirectory = moduleOrDirectory; _includeFilters = parameters.IncludeFilters; _includeDirectories = parameters.IncludeDirectories ?? Array.Empty(); _excludeFilters = parameters.ExcludeFilters; @@ -91,7 +91,7 @@ public Coverage(CoveragePrepareResult prepareResult, ISourceRootTranslator sourceRootTranslator) { _identifier = prepareResult.Identifier; - _module = prepareResult.Module; + _moduleOrAppDirectory = prepareResult.ModuleOrDirectory; _mergeWith = prepareResult.MergeWith; _useSourceLink = prepareResult.UseSourceLink; _results = new List(prepareResult.Results); @@ -103,7 +103,7 @@ public Coverage(CoveragePrepareResult prepareResult, public CoveragePrepareResult PrepareModules() { - string[] modules = _instrumentationHelper.GetCoverableModules(_module, _includeDirectories, _includeTestAssembly); + string[] modules = _instrumentationHelper.GetCoverableModules(_moduleOrAppDirectory, _includeDirectories, _includeTestAssembly); Array.ForEach(_excludeFilters ?? Array.Empty(), filter => _logger.LogVerbose($"Excluded module filter '{filter}'")); Array.ForEach(_includeFilters ?? Array.Empty(), filter => _logger.LogVerbose($"Included module filter '{filter}'")); @@ -161,7 +161,7 @@ public CoveragePrepareResult PrepareModules() return new CoveragePrepareResult() { Identifier = _identifier, - Module = _module, + ModuleOrDirectory = _moduleOrAppDirectory, MergeWith = _mergeWith, UseSourceLink = _useSourceLink, Results = _results.ToArray() diff --git a/src/coverlet.core/CoveragePrepareResult.cs b/src/coverlet.core/CoveragePrepareResult.cs index 9889c7b77..01389aa83 100644 --- a/src/coverlet.core/CoveragePrepareResult.cs +++ b/src/coverlet.core/CoveragePrepareResult.cs @@ -13,7 +13,7 @@ internal class CoveragePrepareResult [DataMember] public string Identifier { get; set; } [DataMember] - public string Module { get; set; } + public string ModuleOrDirectory { get; set; } [DataMember] public string MergeWith { get; set; } [DataMember] diff --git a/src/coverlet.core/Helpers/InstrumentationHelper.cs b/src/coverlet.core/Helpers/InstrumentationHelper.cs index 2441df40f..6c500ebb9 100644 --- a/src/coverlet.core/Helpers/InstrumentationHelper.cs +++ b/src/coverlet.core/Helpers/InstrumentationHelper.cs @@ -31,11 +31,14 @@ public InstrumentationHelper(IProcessExitHandler processExitHandler, IRetryHelpe _sourceRootTranslator = sourceRootTranslator; } - public string[] GetCoverableModules(string module, string[] directories, bool includeTestAssembly) + public string[] GetCoverableModules(string moduleOrAppDirectory, string[] directories, bool includeTestAssembly) { Debug.Assert(directories != null); + Debug.Assert(moduleOrAppDirectory != null); + + bool isAppDirectory = !File.Exists(moduleOrAppDirectory) && Directory.Exists(moduleOrAppDirectory); + string moduleDirectory = isAppDirectory ? moduleOrAppDirectory : Path.GetDirectoryName(moduleOrAppDirectory); - string moduleDirectory = Path.GetDirectoryName(module); if (moduleDirectory == string.Empty) { moduleDirectory = Directory.GetCurrentDirectory(); @@ -67,8 +70,8 @@ public string[] GetCoverableModules(string module, string[] directories, bool in // The module's name must be unique. var uniqueModules = new HashSet(); - if (!includeTestAssembly) - uniqueModules.Add(Path.GetFileName(module)); + if (!includeTestAssembly && !isAppDirectory) + uniqueModules.Add(Path.GetFileName(moduleOrAppDirectory)); return dirs.SelectMany(d => Directory.EnumerateFiles(d)) .Where(m => IsAssembly(m) && uniqueModules.Add(Path.GetFileName(m))) diff --git a/test/coverlet.core.tests/Helpers/InstrumentationHelperTests.cs b/test/coverlet.core.tests/Helpers/InstrumentationHelperTests.cs index 25d1553e5..a97e8d89a 100644 --- a/test/coverlet.core.tests/Helpers/InstrumentationHelperTests.cs +++ b/test/coverlet.core.tests/Helpers/InstrumentationHelperTests.cs @@ -208,9 +208,13 @@ public void TestIsTypeExcludedAndIncludedWithMatchingAndMismatchingFilter(string public void TestIncludeDirectories() { string module = typeof(InstrumentationHelperTests).Assembly.Location; - DirectoryInfo newDir = Directory.CreateDirectory("TestIncludeDirectories"); + newDir.Delete(true); + newDir.Create(); DirectoryInfo newDir2 = Directory.CreateDirectory("TestIncludeDirectories2"); + newDir2.Delete(true); + newDir2.Create(); + File.Copy(module, Path.Combine(newDir.FullName, Path.GetFileName(module))); module = Path.Combine(newDir.FullName, Path.GetFileName(module)); File.Copy("coverlet.msbuild.tasks.dll", Path.Combine(newDir.FullName, "coverlet.msbuild.tasks.dll")); @@ -220,11 +224,23 @@ public void TestIncludeDirectories() Assert.Single(currentDirModules); Assert.Equal("coverlet.msbuild.tasks.dll", Path.GetFileName(currentDirModules[0])); - var moreThanOneDirectory = _instrumentationHelper.GetCoverableModules(module, new string[] { newDir2.FullName }, false); + var moreThanOneDirectory = _instrumentationHelper + .GetCoverableModules(module, new string[] { newDir2.FullName }, false) + .OrderBy(f => f).ToArray(); + Assert.Equal(2, moreThanOneDirectory.Length); Assert.Equal("coverlet.msbuild.tasks.dll", Path.GetFileName(moreThanOneDirectory[0])); Assert.Equal("coverlet.core.dll", Path.GetFileName(moreThanOneDirectory[1])); + var moreThanOneDirectoryPlusTestAssembly = _instrumentationHelper + .GetCoverableModules(module, new string[] { newDir2.FullName }, true) + .OrderBy(f => f).ToArray(); + + Assert.Equal(3, moreThanOneDirectoryPlusTestAssembly.Length); + Assert.Equal("coverlet.core.tests.dll", Path.GetFileName(moreThanOneDirectoryPlusTestAssembly[0])); + Assert.Equal("coverlet.msbuild.tasks.dll", Path.GetFileName(moreThanOneDirectoryPlusTestAssembly[1])); + Assert.Equal("coverlet.core.dll", Path.GetFileName(moreThanOneDirectoryPlusTestAssembly[2])); + newDir.Delete(true); newDir2.Delete(true); } diff --git a/test/coverlet.core.tests/Instrumentation/InstrumenterResultTests.cs b/test/coverlet.core.tests/Instrumentation/InstrumenterResultTests.cs index 98001779b..7e2774a8c 100644 --- a/test/coverlet.core.tests/Instrumentation/InstrumenterResultTests.cs +++ b/test/coverlet.core.tests/Instrumentation/InstrumenterResultTests.cs @@ -29,7 +29,7 @@ public void CoveragePrepareResult_SerializationRoundTrip() CoveragePrepareResult cpr = new CoveragePrepareResult(); cpr.Identifier = "Identifier"; cpr.MergeWith = "MergeWith"; - cpr.Module = "Module"; + cpr.ModuleOrDirectory = "Module"; cpr.UseSourceLink = true; InstrumenterResult ir = new InstrumenterResult(); @@ -99,7 +99,7 @@ public void CoveragePrepareResult_SerializationRoundTrip() Assert.Equal(cpr.Identifier, roundTrip.Identifier); Assert.Equal(cpr.MergeWith, roundTrip.MergeWith); - Assert.Equal(cpr.Module, roundTrip.Module); + Assert.Equal(cpr.ModuleOrDirectory, roundTrip.ModuleOrDirectory); Assert.Equal(cpr.UseSourceLink, roundTrip.UseSourceLink); for (int i = 0; i < cpr.Results.Length; i++) diff --git a/test/coverlet.integration.template/Program.cs b/test/coverlet.integration.template/Program.cs new file mode 100644 index 000000000..ddaea102d --- /dev/null +++ b/test/coverlet.integration.template/Program.cs @@ -0,0 +1,16 @@ +using System; + +using Coverlet.Integration.Template; + +namespace HelloWorld +{ + class Program + { + static void Main(string[] args) + { + DeepThought dt = new DeepThought(); + dt.AnswerToTheUltimateQuestionOfLifeTheUniverseAndEverything(); + Console.WriteLine("Hello World!"); + } + } +} \ No newline at end of file diff --git a/test/coverlet.integration.template/coverlet.integration.template.csproj b/test/coverlet.integration.template/coverlet.integration.template.csproj index ddd56538f..6f834dc4c 100644 --- a/test/coverlet.integration.template/coverlet.integration.template.csproj +++ b/test/coverlet.integration.template/coverlet.integration.template.csproj @@ -5,6 +5,8 @@ false coverletsamplelib.integration.template false + Exe + false diff --git a/test/coverlet.integration.tests/DotnetTool.cs b/test/coverlet.integration.tests/DotnetTool.cs index 2338e75d7..51062008f 100644 --- a/test/coverlet.integration.tests/DotnetTool.cs +++ b/test/coverlet.integration.tests/DotnetTool.cs @@ -29,5 +29,20 @@ public void DotnetTool() Assert.Contains("Test Run Successful.", standardOutput); AssertCoverage(clonedTemplateProject, standardOutput: standardOutput); } + + [ConditionalFact] + [SkipOnOS(OS.Linux)] + [SkipOnOS(OS.MacOS)] + public void StandAlone() + { + using ClonedTemplateProject clonedTemplateProject = CloneTemplateProject(); + UpdateNugeConfigtWithLocalPackageFolder(clonedTemplateProject.ProjectRootPath!); + string coverletToolCommandPath = InstallTool(clonedTemplateProject.ProjectRootPath!); + DotnetCli($"build {clonedTemplateProject.ProjectRootPath}", out string standardOutput, out string standardError); + string publishedTestFile = clonedTemplateProject.GetFiles("*" + ClonedTemplateProject.AssemblyName + ".dll").Single(f => !f.Contains("obj")); + RunCommand(coverletToolCommandPath, $"\"{Path.GetDirectoryName(publishedTestFile)}\" --target \"dotnet\" --targetargs \"{publishedTestFile}\" --output \"{clonedTemplateProject.ProjectRootPath}\"\\", out standardOutput, out standardError); + Assert.Contains("Hello World!", standardOutput); + AssertCoverage(clonedTemplateProject, standardOutput: standardOutput); + } } }