Allow standalone coverlet usage for integration/end-to-end tests using .NET tool driver (#991)

Allow standalone coverlet usage for integration/end-to-end tests using .NET tool driver

Author: MarcoRossignoli

Documentation/GlobalTool.md

@@ -9,12 +9,12 @@ coverlet --help
 The current options are (output of `coverlet --help`):
 
 ```bash
-Cross platform .NET Core code coverage tool 1.0.0.0
+Cross platform .NET Core code coverage tool 3.0.0.0
 
 Usage: coverlet [arguments] [options]
 
 Arguments:
-  <ASSEMBLY>  Path to the test assembly.
+  <ASSEMBLY|DIRECTORY>         Path to the test assembly or application directory.
 
 Options:
   -h|--help                    Show help information
@@ -23,21 +23,21 @@ Options:
   -a|--targetargs              Arguments to be passed to the test runner.
   -o|--output                  Output of the generated coverage report
   -v|--verbosity               Sets the verbosity level of the command. Allowed values are quiet, minimal, normal, detailed.
-  -f|--format                  Format of the generated coverage report[multiple value].
+  -f|--format                  Format of the generated coverage report.
   --threshold                  Exits with error if the coverage % is below value.
-  --threshold-type             Coverage type to apply the threshold to[multiple value].
+  --threshold-type             Coverage type to apply the threshold to.
   --threshold-stat             Coverage statistic used to enforce the threshold value.
-  --exclude                    Filter expressions to exclude specific modules and types[multiple value].
-  --include                    Filter expressions to include specific modules and types[multiple value].
-  --include-directory          Include directories containing additional assemblies to be instrumented[multiple value].
-  --exclude-by-file            Glob patterns specifying source files to exclude[multiple value].
-  --exclude-by-attribute       Attributes to exclude from code coverage[multiple value].
+  --exclude                    Filter expressions to exclude specific modules and types.
+  --include                    Filter expressions to include only specific modules and types.
+  --exclude-by-file            Glob patterns specifying source files to exclude.
+  --include-directory          Include directories containing additional assemblies to be instrumented.
+  --exclude-by-attribute       Attributes to exclude from code coverage.
   --include-test-assembly      Specifies whether to report code coverage of the test assembly.
-  --single-hit                 Specifies whether to limit code coverage hit reporting to a single hit for each location.
+  --single-hit                 Specifies whether to limit code coverage hit reporting to a single hit for each location
+  --skipautoprops              Neither track nor record auto-implemented properties.
   --merge-with                 Path to existing coverage result to merge.
   --use-source-link            Specifies whether to use SourceLink URIs in place of file system paths.
-  --skipautoprops              Neither track nor record auto-implemented properties.
-  --does-not-return-attribute  Attributes that mark methods that do not return[multiple value].
+  --does-not-return-attribute  Attributes that mark methods that do not return.
 ```
 
 NB. For a [multiple value] options you have to specify values multiple times i.e.
@@ -60,6 +60,20 @@ After the above command is run, a `coverage.json` file containing the results wi
 
 _Note: The `--no-build` flag is specified so that the `/path/to/test-assembly.dll` isn't rebuilt_
 
+## Code Coverage for integration tests and end-to-end tests.
+
+Sometimes, there are tests that doesn't use regular unit test frameworks like xunit. You may find yourself in a situation where your tests are driven by a custom executable/script, which when run, could do anything from making API calls to driving Selenium.
+
+As an example, suppose you have a folder `/integrationtest` which contains said executable (lets call it `runner.exe`) and everything it needs to successfully execute. You can use our tool to startup the executable and gather live coverage:
+
+```bash
+coverlet "/integrationtest" --target "/application/runner.exe"
+```
+
+Coverlet will first instrument all .NET assemblies within the `integrationtests` folder, after which it will execute `runner.exe`. Finally, at shutdown of your `runner.exe`, it will generate the coverage report. You can use all parameters available to customize the report generation. Coverage results will be generated once `runner.exe` exits. You can use all parameters available to customize the report generation.
+
+_Note: Today, Coverlet relies on `AppDomain.CurrentDomain.ProcessExit` and `AppDomain.CurrentDomain.DomainUnload` to record hits to the filesystem, as a result, you need to ensure a graceful process shutdown. Forcefully, killing the process will result in an incomplete coverage report._
+
 ## Coverage Output
 
 Coverlet can generate coverage results in multiple formats, which is specified using the `--format` or `-f` options. For example, the following command emits coverage results in the `opencover` format instead of `json`:

README.md

@@ -29,7 +29,7 @@ Coverlet can be used through three different *drivers*
 
 * VSTest engine integration
 * MSBuild task integration
-* As a .NET Global tool  
+* As a .NET Global tool (supports standalone integration tests)
 
 Coverlet supports only SDK-style projects https://docs.microsoft.com/en-us/visualstudio/msbuild/how-to-use-project-sdk?view=vs-2019  
 

src/coverlet.console/Program.cs

@@ -44,7 +44,7 @@ static int Main(string[] args)
             app.VersionOption("-v|--version", GetAssemblyVersion());
             int exitCode = (int)CommandExitCodes.Success;
 
-            CommandArgument module = app.Argument("<ASSEMBLY>", "Path to the test assembly.");
+            CommandArgument moduleOrAppDirectory = app.Argument("<ASSEMBLY|DIRECTORY>", "Path to the test assembly or application directory.");
             CommandOption target = app.Option("-t|--target", "Path to the test runner application.", CommandOptionType.SingleValue);
             CommandOption targs = app.Option("-a|--targetargs", "Arguments to be passed to the test runner.", CommandOptionType.SingleValue);
             CommandOption output = app.Option("-o|--output", "Output of the generated coverage report", CommandOptionType.SingleValue);
@@ -67,8 +67,8 @@ static int Main(string[] args)
 
             app.OnExecute(() =>
             {
-                if (string.IsNullOrEmpty(module.Value) || string.IsNullOrWhiteSpace(module.Value))
-                    throw new CommandParsingException(app, "No test assembly specified.");
+                if (string.IsNullOrEmpty(moduleOrAppDirectory.Value) || string.IsNullOrWhiteSpace(moduleOrAppDirectory.Value))
+                    throw new CommandParsingException(app, "No test assembly or application directory specified.");
 
                 if (!target.HasValue())
                     throw new CommandParsingException(app, "Target must be specified.");
@@ -94,7 +94,7 @@ static int Main(string[] args)
                     DoesNotReturnAttributes = doesNotReturnAttributes.Values.ToArray()
                 };
 
-                Coverage coverage = new Coverage(module.Value,
+                Coverage coverage = new Coverage(moduleOrAppDirectory.Value,
                                                  parameters,
                                                  logger,
                                                  serviceProvider.GetRequiredService<IInstrumentationHelper>(),

src/coverlet.core/Coverage.cs

@@ -29,7 +29,7 @@ internal class CoverageParameters
 
     internal class Coverage
     {
-        private string _module;
+        private string _moduleOrAppDirectory;
         private string _identifier;
         private string[] _includeFilters;
         private string[] _includeDirectories;
@@ -54,15 +54,15 @@ public string Identifier
             get { return _identifier; }
         }
 
-        public Coverage(string module,
+        public Coverage(string moduleOrDirectory,
             CoverageParameters parameters,
             ILogger logger,
             IInstrumentationHelper instrumentationHelper,
             IFileSystem fileSystem,
             ISourceRootTranslator sourceRootTranslator,
             ICecilSymbolHelper cecilSymbolHelper)
         {
-            _module = module;
+            _moduleOrAppDirectory = moduleOrDirectory;
             _includeFilters = parameters.IncludeFilters;
             _includeDirectories = parameters.IncludeDirectories ?? Array.Empty<string>();
             _excludeFilters = parameters.ExcludeFilters;
@@ -91,7 +91,7 @@ public Coverage(CoveragePrepareResult prepareResult,
                         ISourceRootTranslator sourceRootTranslator)
         {
             _identifier = prepareResult.Identifier;
-            _module = prepareResult.Module;
+            _moduleOrAppDirectory = prepareResult.ModuleOrDirectory;
             _mergeWith = prepareResult.MergeWith;
             _useSourceLink = prepareResult.UseSourceLink;
             _results = new List<InstrumenterResult>(prepareResult.Results);
@@ -103,7 +103,7 @@ public Coverage(CoveragePrepareResult prepareResult,
 
         public CoveragePrepareResult PrepareModules()
         {
-            string[] modules = _instrumentationHelper.GetCoverableModules(_module, _includeDirectories, _includeTestAssembly);
+            string[] modules = _instrumentationHelper.GetCoverableModules(_moduleOrAppDirectory, _includeDirectories, _includeTestAssembly);
 
             Array.ForEach(_excludeFilters ?? Array.Empty<string>(), filter => _logger.LogVerbose($"Excluded module filter '{filter}'"));
             Array.ForEach(_includeFilters ?? Array.Empty<string>(), filter => _logger.LogVerbose($"Included module filter '{filter}'"));
@@ -161,7 +161,7 @@ public CoveragePrepareResult PrepareModules()
             return new CoveragePrepareResult()
             {
                 Identifier = _identifier,
-                Module = _module,
+                ModuleOrDirectory = _moduleOrAppDirectory,
                 MergeWith = _mergeWith,
                 UseSourceLink = _useSourceLink,
                 Results = _results.ToArray()

src/coverlet.core/CoveragePrepareResult.cs

@@ -13,7 +13,7 @@ internal class CoveragePrepareResult
         [DataMember]
         public string Identifier { get; set; }
         [DataMember]
-        public string Module { get; set; }
+        public string ModuleOrDirectory { get; set; }
         [DataMember]
         public string MergeWith { get; set; }
         [DataMember]

src/coverlet.core/Helpers/InstrumentationHelper.cs

@@ -31,11 +31,14 @@ public InstrumentationHelper(IProcessExitHandler processExitHandler, IRetryHelpe
             _sourceRootTranslator = sourceRootTranslator;
         }
 
-        public string[] GetCoverableModules(string module, string[] directories, bool includeTestAssembly)
+        public string[] GetCoverableModules(string moduleOrAppDirectory, string[] directories, bool includeTestAssembly)
         {
             Debug.Assert(directories != null);
+            Debug.Assert(moduleOrAppDirectory != null);
+
+            bool isAppDirectory = !File.Exists(moduleOrAppDirectory) && Directory.Exists(moduleOrAppDirectory);
+            string moduleDirectory = isAppDirectory ? moduleOrAppDirectory : Path.GetDirectoryName(moduleOrAppDirectory);
 
-            string moduleDirectory = Path.GetDirectoryName(module);
             if (moduleDirectory == string.Empty)
             {
                 moduleDirectory = Directory.GetCurrentDirectory();
@@ -67,8 +70,8 @@ public string[] GetCoverableModules(string module, string[] directories, bool in
             // The module's name must be unique.
             var uniqueModules = new HashSet<string>();
 
-            if (!includeTestAssembly)
-                uniqueModules.Add(Path.GetFileName(module));
+            if (!includeTestAssembly && !isAppDirectory)
+                uniqueModules.Add(Path.GetFileName(moduleOrAppDirectory));
 
             return dirs.SelectMany(d => Directory.EnumerateFiles(d))
                 .Where(m => IsAssembly(m) && uniqueModules.Add(Path.GetFileName(m)))

test/coverlet.core.tests/Helpers/InstrumentationHelperTests.cs

@@ -208,9 +208,13 @@ public void TestIsTypeExcludedAndIncludedWithMatchingAndMismatchingFilter(string
         public void TestIncludeDirectories()
         {
             string module = typeof(InstrumentationHelperTests).Assembly.Location;
-
             DirectoryInfo newDir = Directory.CreateDirectory("TestIncludeDirectories");
+            newDir.Delete(true);
+            newDir.Create();
             DirectoryInfo newDir2 = Directory.CreateDirectory("TestIncludeDirectories2");
+            newDir2.Delete(true);
+            newDir2.Create();
+
             File.Copy(module, Path.Combine(newDir.FullName, Path.GetFileName(module)));
             module = Path.Combine(newDir.FullName, Path.GetFileName(module));
             File.Copy("coverlet.msbuild.tasks.dll", Path.Combine(newDir.FullName, "coverlet.msbuild.tasks.dll"));
@@ -220,11 +224,23 @@ public void TestIncludeDirectories()
             Assert.Single(currentDirModules);
             Assert.Equal("coverlet.msbuild.tasks.dll", Path.GetFileName(currentDirModules[0]));
 
-            var moreThanOneDirectory = _instrumentationHelper.GetCoverableModules(module, new string[] { newDir2.FullName }, false);
+            var moreThanOneDirectory = _instrumentationHelper
+                                       .GetCoverableModules(module, new string[] { newDir2.FullName }, false)
+                                       .OrderBy(f => f).ToArray();
+
             Assert.Equal(2, moreThanOneDirectory.Length);
             Assert.Equal("coverlet.msbuild.tasks.dll", Path.GetFileName(moreThanOneDirectory[0]));
             Assert.Equal("coverlet.core.dll", Path.GetFileName(moreThanOneDirectory[1]));
 
+            var moreThanOneDirectoryPlusTestAssembly = _instrumentationHelper
+                                                       .GetCoverableModules(module, new string[] { newDir2.FullName }, true)
+                                                       .OrderBy(f => f).ToArray();
+
+            Assert.Equal(3, moreThanOneDirectoryPlusTestAssembly.Length);
+            Assert.Equal("coverlet.core.tests.dll", Path.GetFileName(moreThanOneDirectoryPlusTestAssembly[0]));
+            Assert.Equal("coverlet.msbuild.tasks.dll", Path.GetFileName(moreThanOneDirectoryPlusTestAssembly[1]));
+            Assert.Equal("coverlet.core.dll", Path.GetFileName(moreThanOneDirectoryPlusTestAssembly[2]));
+
             newDir.Delete(true);
             newDir2.Delete(true);
         }

test/coverlet.core.tests/Instrumentation/InstrumenterResultTests.cs

@@ -29,7 +29,7 @@ public void CoveragePrepareResult_SerializationRoundTrip()
             CoveragePrepareResult cpr = new CoveragePrepareResult();
             cpr.Identifier = "Identifier";
             cpr.MergeWith = "MergeWith";
-            cpr.Module = "Module";
+            cpr.ModuleOrDirectory = "Module";
             cpr.UseSourceLink = true;
 
             InstrumenterResult ir = new InstrumenterResult();
@@ -99,7 +99,7 @@ public void CoveragePrepareResult_SerializationRoundTrip()
 
             Assert.Equal(cpr.Identifier, roundTrip.Identifier);
             Assert.Equal(cpr.MergeWith, roundTrip.MergeWith);
-            Assert.Equal(cpr.Module, roundTrip.Module);
+            Assert.Equal(cpr.ModuleOrDirectory, roundTrip.ModuleOrDirectory);
             Assert.Equal(cpr.UseSourceLink, roundTrip.UseSourceLink);
 
             for (int i = 0; i < cpr.Results.Length; i++)

test/coverlet.integration.template/Program.cs

@@ -0,0 +1,16 @@
+using System;
+
+using Coverlet.Integration.Template;
+
+namespace HelloWorld
+{
+    class Program
+    {
+        static void Main(string[] args)
+        {
+            DeepThought dt = new DeepThought();
+            dt.AnswerToTheUltimateQuestionOfLifeTheUniverseAndEverything();
+            Console.WriteLine("Hello World!");
+        }
+    }
+}

test/coverlet.integration.template/coverlet.integration.template.csproj

@@ -5,6 +5,8 @@
     <IsPackable>false</IsPackable>
     <AssemblyName>coverletsamplelib.integration.template</AssemblyName>
     <IsTestProject>false</IsTestProject>
+    <OutputType>Exe</OutputType>
+    <GenerateProgramFile>false</GenerateProgramFile>
   </PropertyGroup>
 
   <ItemGroup>