Merge pull request #8 from tonerdo/master

Update fork
daveMueller · Oct 20, 2019 · a8198b6 · a8198b6
2 parents 28e038c + fd17f4b
commit a8198b6
Show file tree

Hide file tree

Showing 16 changed files with 255 additions and 15 deletions.
diff --git a/Documentation/Changelog.md b/Documentation/Changelog.md
@@ -0,0 +1,44 @@
+# Changelog
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+### Added
+-Add log to tracker [#553](https://github.com/tonerdo/coverlet/pull/553)  
+-Exclude by assembly level System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverage [#589](https://github.com/tonerdo/coverlet/pull/589)
+
+### Fixed
+
+-Fix and simplify async coverage [#549](https://github.com/tonerdo/coverlet/pull/549)  
+-Improve lambda scenario coverage [#583](https://github.com/tonerdo/coverlet/pull/583)
+
+### Improvements
+
+-Improve exception message for unsupported runtime [#569](https://github.com/tonerdo/coverlet/pull/569) by https://github.com/daveMueller
+
+## Release date 2019-09-23
+### Packages  
+coverlet.msbuild 2.7.0  
+coverlet.console 1.6.0  
+coverlet.collector 1.1.0
+
+### Added
+-Output multiple formats for vstest integration [#533](https://github.com/tonerdo/coverlet/pull/533) by https://github.com/daveMueller  
+-Different exit codes to indicate particular failures [#412](https://github.com/tonerdo/coverlet/pull/412) by https://github.com/sasivishnu
+
+
+### Changed
+
+-Skip instrumentation of module with embedded ppbd without local sources [#510](https://github.com/tonerdo/coverlet/pull/510), with this today xunit will be skipped in automatic way.
+
+### Fixed
+
+-Fix exclude by files [#524](https://github.com/tonerdo/coverlet/pull/524)  
+-Changed to calculate based on the average coverage of the module [#479](https://github.com/tonerdo/coverlet/pull/479) by https://github.com/dlplenin  
+-Fix property attribute detection [#477](https://github.com/tonerdo/coverlet/pull/477) by https://github.com/amweiss  
+-Fix instrumentation serialization bug [#458](https://github.com/tonerdo/coverlet/pull/458)  
+-Fix culture for cobertura xml report [#464](https://github.com/tonerdo/coverlet/pull/464)
+
diff --git a/Documentation/GlobalTool.md b/Documentation/GlobalTool.md
@@ -23,21 +23,25 @@ Options:
   -a|--targetargs          Arguments to be passed to the test runner.
   -o|--output              Output of the generated coverage report
   -v|--verbosity           Sets the verbosity level of the command. Allowed values are quiet, minimal, normal, detailed.
-  -f|--format              Format of the generated coverage report.
+  -f|--format              Format of the generated coverage report[multiple value].
   --threshold              Exits with error if the coverage % is below value.
-  --threshold-type         Coverage type to apply the threshold to.
+  --threshold-type         Coverage type to apply the threshold to[multiple value].
   --threshold-stat         Coverage statistic used to enforce the threshold value.
-  --exclude                Filter expressions to exclude specific modules and types.
-  --include                Filter expressions to include specific modules and types.
-  --include-directory      Include directories containing additional assemblies to be instrumented.
-  --exclude-by-file        Glob patterns specifying source files to exclude.
-  --exclude-by-attribute   Attributes to exclude from code coverage.
+  --exclude                Filter expressions to exclude specific modules and types[multiple value].
+  --include                Filter expressions to include specific modules and types[multiple value].
+  --include-directory      Include directories containing additional assemblies to be instrumented[multiple value].
+  --exclude-by-file        Glob patterns specifying source files to exclude[multiple value].
+  --exclude-by-attribute   Attributes to exclude from code coverage[multiple value].
   --include-test-assembly  Specifies whether to report code coverage of the test assembly.
   --single-hit             Specifies whether to limit code coverage hit reporting to a single hit for each location.
   --merge-with             Path to existing coverage result to merge.
   --use-source-link        Specifies whether to use SourceLink URIs in place of file system paths.
 ```
 
+NB. For a [multiple value] options you have to specify values multiple times i.e.
+```
+--exclude-by-attribute 'Obsolete' --exclude-by-attribute'GeneratedCode' --exclude-by-attribute 'CompilerGenerated'
+```
 For `--merge-with` [check the sample](Examples.md).
 
 ## Code Coverage
@@ -161,7 +165,7 @@ You can ignore a method or an entire class from code coverage by creating and ap
 You can also ignore additional attributes by using the `ExcludeByAttribute` property (short name or full name supported):
 
 ```bash
-coverlet <ASSEMBLY> --target <TARGET> --targetargs <TARGETARGS> --exclude-by-attribute "Obsolete,GeneratedCodeAttribute,CompilerGeneratedAttribute"
+coverlet <ASSEMBLY> --target <TARGET> --targetargs <TARGETARGS> --exclude-by-attribute 'Obsolete' --exclude-by-attribute'GeneratedCode' --exclude-by-attribute 'CompilerGenerated'
 ```
 
 ### Source Files

diff --git a/Documentation/KnowIssues.md b/Documentation/KnowIssues.md
@@ -0,0 +1,89 @@
+# Know Issues
+
+## 1) VSTest stops process execution early(`dotnet test`)  
+
+*Affected drivers*: msbuild(`dotnet test`) , dotnet tool(if you're using ` --targetargs "test ... --no-build"`)  
+
+ *Symptoms:* 
+ * warning or error like
+
+ `Unable to read beyond end of stream`  
+
+ `warning : [coverlet] Hits file:'C:\Users\REDACTED\AppData\Local\Temp\testApp_ac32258b-fd4a-4bb4-824c-a79061e97c31' not found for module: 'testApp'`
+
+ *  zero coverage result (often only on CI but not on local)  
+```
+Calculating coverage result...
+C:\Users\REDACTED\.nuget\packages\coverlet.msbuild\2.6.0\build\netstandard2.0\coverlet.msbuild.targets(21,5): warning : [coverlet] Hits file:'C:\Users\REDACTED\AppData\Local\Temp\testApp_ac32258b-fd4a-4bb4-824c-a79061e97c31' not found for module: 'testApp' [C:\Users\REDACTED\Documents\repo\testapp\testapp.Tests\testapp.Tests.csproj]
+C:\Users\REDACTED\.nuget\packages\coverlet.msbuild\2.6.0\build\netstandard2.0\coverlet.msbuild.targets(21,5): warning : [coverlet] Hits file:'C:\Users\REDACTED\AppData\Local\Temp\testApp.Tests_ac32258b-fd4a-4bb4-824c-a79061e97c31' not found for module: 'testApp.Tests' [C:\Users\REDACTED\Documents\repo\testapp\testapp.Tests\testapp.Tests.csproj]
+  Generating report 'C:\Users\REDACTED\Documents\repo\testapp\lcov.info'
+
++---------------+------+--------+--------+
+| Module        | Line | Branch | Method |
++---------------+------+--------+--------+
+| testApp       | 0%   | 0%     | 0%     |
++---------------+------+--------+--------+
+| testApp.Tests | 0%   | 100%   | 0%     |
++---------------+------+--------+--------+
+
++---------+------+--------+--------+
+|         | Line | Branch | Method |
++---------+------+--------+--------+
+| Total   | 0%   | 0%     | 0%     |
++---------+------+--------+--------+
+| Average | 0%   | 0%     | 0%     |
++---------+------+--------+--------+
+```
+
+The issue is related to vstest platform https://github.com/microsoft/vstest/issues/1900#issuecomment-457488472  
+```
+However if testhost doesn't shut down within 100ms(as the execution is completed, we expect it to shutdown fast). vstest.console forcefully kills the process.
+```
+
+Coverlet collect and write hits data on process exist, if for some reason process is too slow to close will be killed and we cannot collect coverage result.
+This happen also if there are other "piece of code" during testing that slow down process exit.
+We found problem for instance with test that uses RabbitMQ.
+
+*Solution:* 
+The only way to solve this issue is to use collectors integration https://github.com/tonerdo/coverlet#vstest-integration.  
+With collector we're injected in test host throught a in-proc collector that talk with vstest platform so we can signal when we end our work.  
+Check requirements https://github.com/tonerdo/coverlet#requirements you need to run *.NET Core SDK v2.2.401 or newer*.
+
+## 2) Upgrade `coverlet.collector` to version > 1.0.0
+
+*Affected drivers*: vstest integration `dotnet test --collect:"XPlat Code Coverage"`  
+
+ *Symptoms:* The same of know issue 1.  
+
+There is a bug inside vstest platform https://github.com/microsoft/vstest/issues/2205.  
+If you upgrade collector package with version greather than 1.0.0 in-proc collector won't be loaded so you could incur in know issue number 1 and get zero coverage result
+
+*Solution:* you need to pass custom *runsetting* file like this
+```xml
+<?xml version="1.0" encoding="utf-8" ?>
+<RunSettings>
+  <DataCollectionRunSettings>
+    <DataCollectors>
+      <DataCollector friendlyName="XPlat code coverage">
+        <Configuration>
+          <Format>cobertura</Format>
+        </Configuration>
+      </DataCollector>
+    </DataCollectors>
+  </DataCollectionRunSettings>
+  <InProcDataCollectionRunSettings>
+    <InProcDataCollectors>
+      <InProcDataCollector assemblyQualifiedName="Coverlet.Collector.DataCollection.CoverletInProcDataCollector, coverlet.collector, Version=1.1.0.0, Culture=neutral, PublicKeyToken=null"
+                     friendlyName="XPlat Code Coverage"
+                     enabled="True"
+                     codebase="coverlet.collector.dll" />
+    </InProcDataCollectors>
+  </InProcDataCollectionRunSettings>
+</RunSettings>
+```
+And pass it to command line
+```
+dotnet test --settings runsetting
+```
+
+
diff --git a/Documentation/MSBuildIntegration.md b/Documentation/MSBuildIntegration.md
@@ -109,7 +109,7 @@ dotnet test /p:CollectCoverage=true /p:Threshold=80 /p:ThresholdType=line /p:Thr
 
 ### Attributes
 
-You can ignore a method or an entire class from code coverage by creating and applying the `ExcludeFromCodeCoverage` attribute present in the `System.Diagnostics.CodeAnalysis` namespace.
+You can ignore a method an entire class or assembly from code coverage by creating and applying the `ExcludeFromCodeCoverage` attribute present in the `System.Diagnostics.CodeAnalysis` namespace.
 
 You can also ignore additional attributes by using the `ExcludeByAttribute` property (short name or full name supported):
 

diff --git a/Documentation/VSTestIntegration.md b/Documentation/VSTestIntegration.md
@@ -84,3 +84,7 @@ The datacollectors will be bundled as a separate NuGet package, the reference to
     <PackageReference Include="coverlet.collector" Version="1.0.0" />
 </ItemGroup>
 ```
+
+## Know issue
+
+Thre is a know issue, check it here https://github.com/tonerdo/coverlet/blob/master/Documentation/KnowIssues.md#2-upgrade-coverletcollector-to-version--100
diff --git a/README.md b/README.md
@@ -4,6 +4,15 @@
 
 Coverlet is a cross platform code coverage framework for .NET, with support for line, branch and method coverage. It works with .NET Framework on Windows and .NET Core on all supported platforms.
 
+# Main contents
+* [Installation](#Installation)
+* [QuickStart](#Quick-Start)
+* [How It Works](#How-It-Works)
+* [Know Issues](#Know-Issues)
+* [Consume nightly build](#Consume-nightly-build)
+* [Feature samples](Documentation/Examples.md)
+* [Cake Add-In](#Cake.-Add-In)
+* [Changelog](Documentation/Changelog.md)
 ## Installation
 
 **VSTest Integration**:
@@ -102,6 +111,10 @@ _Note: The assembly you'd like to get coverage for must be different from the as
 
 ## Are you in trouble with some feature? Check on [examples](Documentation/Examples.md)!
 
+## Know Issues
+
+Unfortunately we have some know issues, check it [here](Documentation/KnowIssues.md) 
+
 ## Cake Add-In
 
 If you're using [Cake Build](https://cakebuild.net) for your build script you can use the [Cake.Coverlet](https://github.com/Romanx/Cake.Coverlet) add-in to provide you extensions to dotnet test for passing Coverlet arguments in a strongly typed manner.

diff --git a/coverlet.sln b/coverlet.sln
@@ -41,6 +41,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution
 		nuget.config = nuget.config
 	EndProjectSection
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "coverlet.tests.projectsample.excludedbyattribute", "test\coverlet.tests.projectsample.excludedbyattribute\coverlet.tests.projectsample.excludedbyattribute.csproj", "{D6B14F2F-9E7D-4D2C-BAC8-48834F853ED6}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -87,6 +89,10 @@ Global
 		{085A3AFB-C086-4E98-86F1-1B481446EC5E}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{085A3AFB-C086-4E98-86F1-1B481446EC5E}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{085A3AFB-C086-4E98-86F1-1B481446EC5E}.Release|Any CPU.Build.0 = Release|Any CPU
+		{D6B14F2F-9E7D-4D2C-BAC8-48834F853ED6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{D6B14F2F-9E7D-4D2C-BAC8-48834F853ED6}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{D6B14F2F-9E7D-4D2C-BAC8-48834F853ED6}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{D6B14F2F-9E7D-4D2C-BAC8-48834F853ED6}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -102,6 +108,7 @@ Global
 		{5ED4FA81-8F8C-4211-BA88-7573BD63262E} = {2FEBDE1B-83E3-445B-B9F8-5644B0E0E134}
 		{3E0F9E47-A1D7-4DF5-841D-A633486E2475} = {2FEBDE1B-83E3-445B-B9F8-5644B0E0E134}
 		{085A3AFB-C086-4E98-86F1-1B481446EC5E} = {2FEBDE1B-83E3-445B-B9F8-5644B0E0E134}
+		{D6B14F2F-9E7D-4D2C-BAC8-48834F853ED6} = {2FEBDE1B-83E3-445B-B9F8-5644B0E0E134}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {9CA57C02-97B0-4C38-A027-EA61E8741F10}

diff --git a/src/coverlet.core/Coverage.cs b/src/coverlet.core/Coverage.cs
@@ -106,9 +106,12 @@ public CoveragePrepareResult PrepareModules()
                     // Guard code path and restore if instrumentation fails.
                     try
                     {
-                        var result = instrumenter.Instrument();
-                        _results.Add(result);
-                        _logger.LogVerbose($"Instrumented module: '{module}'");
+                        InstrumenterResult result = instrumenter.Instrument();
+                        if (!instrumenter.SkipModule)
+                        {
+                            _results.Add(result);
+                            _logger.LogVerbose($"Instrumented module: '{module}'");
+                        }
                     }
                     catch (Exception ex)
                     {

diff --git a/src/coverlet.core/Helpers/FileSystem.cs b/src/coverlet.core/Helpers/FileSystem.cs
@@ -42,7 +42,8 @@ public virtual Stream NewFileStream(string path, FileMode mode)
             return new FileStream(path, mode);
         }
 
-        public Stream NewFileStream(string path, FileMode mode, FileAccess access)
+        // We need to partial mock this method on tests
+        public virtual Stream NewFileStream(string path, FileMode mode, FileAccess access)
         {
             return new FileStream(path, mode, access);
         }

diff --git a/src/coverlet.core/Instrumentation/Instrumenter.cs b/src/coverlet.core/Instrumentation/Instrumenter.cs
@@ -40,6 +40,8 @@ internal class Instrumenter
         private List<string> _excludedSourceFiles;
         private List<string> _branchesInCompiledGeneratedClass;
 
+        public bool SkipModule { get; set; } = false;
+
         public Instrumenter(
             string module,
             string identifier,
@@ -151,6 +153,16 @@ private void InstrumentModule()
 
                 using (var module = ModuleDefinition.ReadModule(stream, parameters))
                 {
+                    foreach (CustomAttribute customAttribute in module.Assembly.CustomAttributes)
+                    {
+                        if (customAttribute.AttributeType.FullName == "System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverageAttribute")
+                        {
+                            _logger.LogVerbose($"Excluded module: '{module}' for assembly level attribute 'System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverageAttribute'");
+                            SkipModule = true;
+                            return;
+                        }
+                    }
+
                     var containsAppContext = module.GetType(nameof(System), nameof(AppContext)) != null;
                     var types = module.GetTypes();
                     AddCustomModuleTrackerToModule(module);

diff --git a/test/coverlet.core.tests/CoverageTests.cs b/test/coverlet.core.tests/CoverageTests.cs
@@ -1,5 +1,6 @@
 using System;
 using System.IO;
+using System.Linq;
 using System.Threading.Tasks;
 using Coverlet.Core.Abstracts;
 using Coverlet.Core.Helpers;
@@ -40,6 +41,25 @@ public void TestCoverage()
             directory.Delete(true);
         }
 
+        [Fact]
+        public void TestCoverageSkipModule__AssemblyMarkedAsExcludeFromCodeCoverage()
+        {
+            Mock<FileSystem> partialMockFileSystem = new Mock<FileSystem>();
+            partialMockFileSystem.CallBase = true;
+            partialMockFileSystem.Setup(fs => fs.NewFileStream(It.IsAny<string>(), It.IsAny<FileMode>(), It.IsAny<FileAccess>())).Returns((string path, FileMode mode, FileAccess access) =>
+            {
+                return new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read);
+            });
+            var loggerMock = new Mock<ILogger>();
+
+            string excludedbyattributeDll = Directory.GetFiles(Path.Combine(Directory.GetCurrentDirectory(), "TestAssets"), "coverlet.tests.projectsample.excludedbyattribute.dll").First();
+            // test skip module includint test assembly feature
+            var coverage = new Coverage(excludedbyattributeDll, new string[] { "[coverlet.tests.projectsample.excludedbyattribute*]*" }, Array.Empty<string>(), Array.Empty<string>(), Array.Empty<string>(), Array.Empty<string>(), true, false, string.Empty, false, loggerMock.Object, _instrumentationHelper, partialMockFileSystem.Object);
+            CoveragePrepareResult result = coverage.PrepareModules();
+            Assert.Empty(result.Results);
+            loggerMock.Verify(l => l.LogVerbose(It.IsAny<string>()));
+        }
+
         [Fact]
         public void TestCoverageWithTestAssembly()
         {

diff --git a/test/coverlet.core.tests/Instrumentation/InstrumenterTests.cs b/test/coverlet.core.tests/Instrumentation/InstrumenterTests.cs
@@ -416,5 +416,22 @@ public void TestInstrument_MissingModule()
             loggerMock.Verify(l => l.LogWarning(It.IsAny<string>()));
         }
 
+        [Fact]
+        public void TestInstrument_AssemblyMarkedAsExcludeFromCodeCoverage()
+        {
+            Mock<FileSystem> partialMockFileSystem = new Mock<FileSystem>();
+            partialMockFileSystem.CallBase = true;
+            partialMockFileSystem.Setup(fs => fs.NewFileStream(It.IsAny<string>(), It.IsAny<FileMode>(), It.IsAny<FileAccess>())).Returns((string path, FileMode mode, FileAccess access) =>
+            {
+                return new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read);
+            });
+            var loggerMock = new Mock<ILogger>();
+
+            string excludedbyattributeDll = Directory.GetFiles(Path.Combine(Directory.GetCurrentDirectory(), "TestAssets"), "coverlet.tests.projectsample.excludedbyattribute.dll").First();
+            Instrumenter instrumenter = new Instrumenter(excludedbyattributeDll, "_xunit_excludedbyattribute", Array.Empty<string>(), Array.Empty<string>(), Array.Empty<string>(), Array.Empty<string>(), false, loggerMock.Object, _instrumentationHelper, partialMockFileSystem.Object);
+            InstrumenterResult result = instrumenter.Instrument();
+            Assert.Empty(result.Documents);
+            loggerMock.Verify(l => l.LogVerbose(It.IsAny<string>()));
+        }
     }
 }
diff --git a/test/coverlet.core.tests/coverlet.core.tests.csproj b/test/coverlet.core.tests/coverlet.core.tests.csproj
@@ -2,7 +2,7 @@
   <Import Project="$(RepoRoot)src\coverlet.msbuild.tasks\coverlet.msbuild.props" />
 
   <PropertyGroup>
-    <TargetFramework>netcoreapp2.0</TargetFramework>
+    <TargetFramework>netcoreapp2.2</TargetFramework>
     <IsPackable>false</IsPackable>
     <LangVersion>preview</LangVersion>
     <NoWarn>$(NoWarn);CS8002</NoWarn>
@@ -39,6 +39,8 @@
   <Target Name="PostBuild" AfterTargets="PostBuildEvent">
     <Exec Condition="$([MSBuild]::IsOSPlatform('Windows'))" Command="copy $(ProjectDir)..\coverlet.tests.remoteexecutor\$(OutDir) $(TargetDir) /Y" />
     <Exec Condition="$([MSBuild]::IsOSPlatform('OSX')) OR $([MSBuild]::IsOSPlatform('Linux'))" Command="cp -r $(ProjectDir)..\coverlet.tests.remoteexecutor\$(OutDir). $(TargetDir)" />
+    <Exec Condition="$([MSBuild]::IsOSPlatform('Windows'))" Command="copy $(ProjectDir)..\coverlet.tests.projectsample.excludedbyattribute\$(OutDir) $(TargetDir)TestAssets /Y" />
+    <Exec Condition="$([MSBuild]::IsOSPlatform('OSX')) OR $([MSBuild]::IsOSPlatform('Linux'))" Command="cp -r $(ProjectDir)..\coverlet.tests.projectsample.excludedbyattribute\$(OutDir). $(TargetDir)TestAssets" />
   </Target>
 
   <Import Project="$(RepoRoot)src\coverlet.msbuild.tasks\coverlet.msbuild.targets" />