Merge pull request #297 from csoltenborn/develop

Release 0.17.0

GoogleTestAdapter/Common/Common.csproj

@@ -62,6 +62,7 @@
     <Compile Include="ProcessWaiter.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
     <Compile Include="Strings.cs" />
+    <Compile Include="MissingTestsReportMode.cs" />
     <Compile Include="SummaryMode.cs" />
     <Compile Include="Win32Utils.cs" />
   </ItemGroup>

GoogleTestAdapter/Common/MissingTestsReportMode.cs

@@ -0,0 +1,31 @@
+using System.Collections.Generic;
+using System.ComponentModel;
+
+namespace GoogleTestAdapter.Common
+{
+    [TypeConverter(typeof(MissingTestsReportModeConverter))]
+    public enum MissingTestsReportMode
+    {
+        DoNotReport,
+        ReportAsNotFound,
+        ReportAsSkipped,
+        ReportAsFailed
+    }
+
+    public class MissingTestsReportModeConverter : EnumConverterBase<MissingTestsReportMode>
+    {
+        public const string DoNotReport = "Do not report";
+        public const string ReportAsNotFound = "Report as not found";
+        public const string ReportAsSkipped = "Report as skipped";
+        public const string ReportAsFailed = "Report as failed";
+
+        public MissingTestsReportModeConverter() : base(new Dictionary<MissingTestsReportMode, string>
+        {
+            { MissingTestsReportMode.DoNotReport, DoNotReport},
+            { MissingTestsReportMode.ReportAsNotFound, ReportAsNotFound},
+            { MissingTestsReportMode.ReportAsSkipped, ReportAsSkipped},
+            { MissingTestsReportMode.ReportAsFailed, ReportAsFailed},
+        }) {}
+    }
+
+}

GoogleTestAdapter/Core.Tests/GoogleTestDiscovererTests.cs

@@ -169,7 +169,7 @@ public void GetTestsFromExecutable_SampleTestsDebugWithExitCodeTest_FindsTestsWi
             var exitCodeTestCase = testCases.Single(tc => tc.FullyQualifiedName == finalName);
             exitCodeTestCase.DisplayName.Should().Be(finalName);
             exitCodeTestCase.Source.Should().Be(TestResources.Tests_DebugX86);
-            exitCodeTestCase.CodeFilePath.Should().Contain(@"sampletests\tests\main.cpp");
+            exitCodeTestCase.CodeFilePath.Should().ContainEquivalentOf(@"sampletests\tests\main.cpp");
             exitCodeTestCase.LineNumber.Should().Be(8);
 
             MockLogger.Verify(l => l.DebugInfo(It.Is<string>(msg => msg.Contains("Exit code") && msg.Contains("ignored"))), Times.Once);
@@ -445,12 +445,12 @@ private IList<TestCase> FindTests(string location, int expectedNrOfTestCases = T
 
             TestCase testCase = testCases.Single(tc => tc.FullyQualifiedName == "TheFixture.AddFails");
             testCase.DisplayName.Should().Be("TheFixture.AddFails");
-            testCase.CodeFilePath.Should().EndWith(@"sampletests\tests\fixturetests.cpp");
+            testCase.CodeFilePath.Should().EndWithEquivalent(@"sampletests\tests\fixturetests.cpp");
             testCase.LineNumber.Should().Be(11);
 
             testCase = testCases.Single(tc => tc.FullyQualifiedName == "Arr/TypeParameterizedTests/1.CanDefeatMath");
             testCase.DisplayName.Should().Be("Arr/TypeParameterizedTests/1.CanDefeatMath<MyStrangeArray>");
-            testCase.CodeFilePath.Should().EndWith(@"sampletests\tests\typeparameterizedtests.cpp");
+            testCase.CodeFilePath.Should().EndWithEquivalent(@"sampletests\tests\typeparameterizedtests.cpp");
             testCase.LineNumber.Should().Be(56);
 
             return testCases;
@@ -465,11 +465,11 @@ private void FindExternallyLinkedTests(string location)
 
             string expectedCodeFilePath = Path.GetFullPath($@"{TestResources.SampleTestsSolutionDir}dlldependentproject\dlltests.cpp").ToLower();
             testCases[0].DisplayName.Should().Be("Passing.InvokeFunction");
-            testCases[0].CodeFilePath.Should().Be(expectedCodeFilePath);
+            testCases[0].CodeFilePath.Should().BeEquivalentTo(expectedCodeFilePath);
             testCases[0].LineNumber.Should().Be(5);
 
             testCases[1].DisplayName.Should().Be("Failing.InvokeFunction");
-            testCases[1].CodeFilePath.Should().Be(expectedCodeFilePath);
+            testCases[1].CodeFilePath.Should().BeEquivalentTo(expectedCodeFilePath);
             testCases[1].LineNumber.Should().Be(10);
         }
 

GoogleTestAdapter/Core/Runners/SequentialTestRunner.cs

@@ -161,7 +161,7 @@ private IEnumerable<TestResult> TryRunTests(string executable, string workingDir
                 arguments.TestCases
                     .Except(streamingParser.TestResults.Select(tr => tr.TestCase))
                     .Where(tc => !tc.IsExitCodeTestCase);
-            var testResults = new TestResultCollector(_logger, _threadName)
+            var testResults = new TestResultCollector(_logger, _threadName, _settings)
                 .CollectTestResults(remainingTestCases, executable, resultXmlFile, consoleOutput, streamingParser.CrashedTestCase);
             testResults = testResults.OrderBy(tr => tr.TestCase.FullyQualifiedName).ToList();
 

GoogleTestAdapter/Core/Runners/TestResultCollector.cs

@@ -3,19 +3,29 @@
 using System.Linq;
 using GoogleTestAdapter.Common;
 using GoogleTestAdapter.Model;
+using GoogleTestAdapter.Settings;
 using GoogleTestAdapter.TestResults;
 
 namespace GoogleTestAdapter.Runners
 {
     public class TestResultCollector
     {
+        private static readonly IReadOnlyList<string> TestsNotRunCauses = new List<string>
+        {
+            "A test run is repeated, but tests have changed in the meantime",
+            "A test dependency has been removed or changed without Visual Studio noticing"
+        };
+
         private readonly ILogger _logger;
+        private readonly SettingsWrapper _settings;
+
         private readonly string _threadName;
 
-        public TestResultCollector(ILogger logger, string threadName)
+        public TestResultCollector(ILogger logger, string threadName, SettingsWrapper settings)
         {
             _logger = logger;
             _threadName = threadName;
+            _settings = settings;
         }
 
         public List<TestResult> CollectTestResults(IEnumerable<TestCase> testCasesRun, string testExecutable, string resultXmlFile, List<string> consoleOutput, TestCase crashedTestCase)
@@ -42,7 +52,7 @@ public List<TestResult> CollectTestResults(IEnumerable<TestCase> testCasesRun, s
                 if (crashedTestCase != null)
                     CreateMissingResults(remainingTestCases, crashedTestCase, testResults);
                 else
-                    ReportSuspiciousTestCases(remainingTestCases);
+                    ReportSuspiciousTestCases(remainingTestCases, testResults);
             }
 
             return testResults;
@@ -84,25 +94,55 @@ private void CreateMissingResults(TestCase[] testCases, TestCase crashedTestCase
             var errorStackTrace = ErrorMessageParser.CreateStackTraceEntry("crash suspect",
                 crashedTestCase.CodeFilePath, crashedTestCase.LineNumber.ToString());
 
-            foreach (TestCase testCase in testCases)
-            {
-                testResults.Add(new TestResult(testCase)
-                {
-                    ComputerName = Environment.MachineName,
-                    Outcome = TestOutcome.Skipped,
-                    ErrorMessage = errorMessage,
-                    ErrorStackTrace = errorStackTrace
-                });
-            }
+            testResults.AddRange(testCases.Select(testCase => 
+                CreateTestResult(testCase, TestOutcome.Skipped, errorMessage, errorStackTrace)));
             if (testCases.Length > 0)
                 _logger.DebugInfo($"{_threadName}Created {testCases.Length} test results for tests which were neither found in result XML file nor in console output");
         }
 
-        private void ReportSuspiciousTestCases(TestCase[] testCases)
+        private void ReportSuspiciousTestCases(TestCase[] testCases, List<TestResult> testResults)
         {
+            string causesAsString = $" - possible causes:{Environment.NewLine}{string.Join(Environment.NewLine, TestsNotRunCauses.Select(s => $"- {s}"))}";
             string testCasesAsString = string.Join(Environment.NewLine, testCases.Select(tc => tc.DisplayName));
-            _logger.DebugWarning(
-                $"{_threadName}{testCases.Length} test cases seem to not have been run - are you repeating a test run, but tests have changed in the meantime? Test cases:{Environment.NewLine}{testCasesAsString}");
+
+            _logger.DebugWarning($"{_threadName}{testCases.Length} test cases seem to not have been run{causesAsString}");
+            _logger.VerboseInfo($"{_threadName}Test cases:{Environment.NewLine}{testCasesAsString}");
+
+            TestOutcome? testOutcome = GetTestOutcomeOfMissingTests();
+            if (testOutcome.HasValue)
+            {
+                string errorMessage = $"Test case has not been run{causesAsString}";
+                testResults.AddRange(testCases.Select(tc => CreateTestResult(tc, testOutcome.Value, errorMessage, null)));
+            }
+        }
+
+        private TestOutcome? GetTestOutcomeOfMissingTests()
+        {
+            switch (_settings.MissingTestsReportMode)
+            {
+                case MissingTestsReportMode.DoNotReport:
+                    return null;
+                case MissingTestsReportMode.ReportAsFailed:
+                    return TestOutcome.Failed;
+                case MissingTestsReportMode.ReportAsSkipped:
+                    return TestOutcome.Skipped;
+                case MissingTestsReportMode.ReportAsNotFound:
+                    return TestOutcome.NotFound;
+                default:
+                    throw new InvalidOperationException($"Unknown {nameof(MissingTestsReportMode)}: {_settings.MissingTestsReportMode}");
+            }
+        }
+
+        private TestResult CreateTestResult(TestCase testCase, TestOutcome outcome, string errorMessage, string errorStackTrace)
+        {
+            return new TestResult(testCase)
+            {
+                ComputerName = Environment.MachineName,
+                Outcome = outcome,
+                ErrorMessage = errorMessage,
+                ErrorStackTrace = errorStackTrace
+
+            };
         }
 
     }

GoogleTestAdapter/Core/Settings/IGoogleTestAdapterSettings.cs

@@ -53,6 +53,7 @@ public interface IGoogleTestAdapterSettings
         bool? KillProcessesOnCancel { get; set; }
         bool? SkipOriginCheck { get; set; }
         string ExitCodeTestCase { get; set; }
+        MissingTestsReportMode? MissingTestsReportMode { get; set; }
 
         bool? UseNewTestExecutionFramework { get; set; }
         DebuggerKind? DebuggerKind { get; set; }
@@ -100,6 +101,7 @@ public static void GetUnsetValuesFrom(this IGoogleTestAdapterSettings self, IGoo
             self.KillProcessesOnCancel = self.KillProcessesOnCancel ?? other.KillProcessesOnCancel;
             self.SkipOriginCheck = self.SkipOriginCheck ?? other.SkipOriginCheck;
             self.ExitCodeTestCase = self.ExitCodeTestCase ?? other.ExitCodeTestCase;
+            self.MissingTestsReportMode = self.MissingTestsReportMode ?? other.MissingTestsReportMode;
 
             self.UseNewTestExecutionFramework = self.UseNewTestExecutionFramework ?? other.UseNewTestExecutionFramework;
             self.DebuggerKind = self.DebuggerKind ?? other.DebuggerKind;

GoogleTestAdapter/Core/Settings/RunSettings.cs

@@ -115,6 +115,9 @@ public RunSettings(string projectRegex)
         public string ExitCodeTestCase { get; set; }
         public bool ShouldSerializeExitCodeTestCase() { return ExitCodeTestCase != null; }
 
+        public MissingTestsReportMode? MissingTestsReportMode { get; set; }
+        public bool ShouldSerializeMissingTestsReportMode() { return MissingTestsReportMode != null; }
+
 
         public virtual bool? UseNewTestExecutionFramework { get; set; }
         public bool ShouldSerializeUseNewTestExecutionFramework() { return UseNewTestExecutionFramework != null; }

GoogleTestAdapter/Core/Settings/SettingsWrapper.cs

@@ -452,6 +452,15 @@ public virtual int MaxNrOfThreads
             }
         }
 
+        public const string OptionMissingTestsReportMode = "Behavior for missing test results";
+        public const string OptionMissingTestsReportModeDescription =
+            "If a test can not be run (e.g. because a dependency has been removed since discovery without VS noticing), this option allows to configure how that test will be reported to the VS test framework." + 
+            "\nDefault: " + MissingTestsReportModeConverter.ReportAsNotFound;
+        public const MissingTestsReportMode OptionMissingTestsReportModeDefaultValue = MissingTestsReportMode.ReportAsNotFound;
+
+        public virtual MissingTestsReportMode MissingTestsReportMode =>
+            _currentSettings.MissingTestsReportMode ?? OptionMissingTestsReportModeDefaultValue;
+
         #endregion
 
         #region TestDiscoveryOptionsPage

GoogleTestAdapter/DiaResolver.Tests/DiaResolverTests.cs

@@ -34,7 +34,7 @@ public void GetFunctions_X86_EverythingMatches_ResultSizeIsCorrect()
                 TestResources.LoadTests_ReleaseX86, 
                 "*", 
                 TestMetadata.VersionUnderTest == VsVersion.VS2017 ? 628 : 728, 
-                90);
+                88);
         }
 
         [TestMethod]

GoogleTestAdapter/Packaging.GTA/VsPackage.nuspec

@@ -2,7 +2,7 @@
 <package>
   <metadata minClientVersion="2.8">
     <id>GoogleTestAdapter</id>
-    <version>0.16.1</version>
+    <version>0.17.0</version>
     <title>Google Test Adapter</title>
     <authors>Christian Soltenborn</authors>
     <owners>Christian Soltenborn</owners>

GoogleTestAdapter/Resources/AllTestSettings.gta.runsettings

@@ -32,6 +32,7 @@
 				<UseNewTestExecutionFramework>true</UseNewTestExecutionFramework>
 				<KillProcessesOnCancel>false</KillProcessesOnCancel>
 				<ExitCodeTestCase/>
+				<MissingTestsReportMode>ReportAsFailed</MissingTestsReportMode>
 			</Settings>
 		</SolutionSettings>
 		<ProjectSettings/>

GoogleTestAdapter/TestAdapter.Tests/TestExecutorParallelTests.cs

@@ -18,9 +18,9 @@ public class TestExecutorParallelTests : TestExecutorTestsBase
         public TestExecutorParallelTests() : base(true, Environment.ProcessorCount) { }
 
 
-        protected override void CheckMockInvocations(int nrOfPassedTests, int nrOfFailedTests, int nrOfUnexecutedTests, int nrOfSkippedTests)
+        protected override void CheckMockInvocations(int nrOfPassedTests, int nrOfFailedTests, int nrOfUnexecutedTests, int nrOfSkippedTests, int nrOfNotFoundTests)
         {
-            base.CheckMockInvocations(nrOfPassedTests, nrOfFailedTests, nrOfUnexecutedTests, nrOfSkippedTests);
+            base.CheckMockInvocations(nrOfPassedTests, nrOfFailedTests, nrOfUnexecutedTests, nrOfSkippedTests, nrOfNotFoundTests);
 
             if (nrOfPassedTests > 0)
             {
@@ -38,6 +38,14 @@ protected override void CheckMockInvocations(int nrOfPassedTests, int nrOfFailed
                     Times.AtLeast(nrOfFailedTests));
             }
 
+            if (nrOfNotFoundTests > 0)
+            {
+                MockFrameworkHandle.Verify(h => h.RecordResult(It.Is<TestResult>(tr => tr.Outcome == TestOutcome.NotFound)),
+                    Times.AtLeast(nrOfNotFoundTests));
+                MockFrameworkHandle.Verify(h => h.RecordEnd(It.IsAny<TestCase>(), It.Is<TestOutcome>(to => to == TestOutcome.NotFound)),
+                    Times.AtLeast(nrOfNotFoundTests));
+            }
+
             MockFrameworkHandle.Verify(h => h.RecordResult(It.Is<TestResult>(tr => tr.Outcome == TestOutcome.Skipped)),
                 Times.AtMost(nrOfSkippedTests));
             MockFrameworkHandle.Verify(h => h.RecordEnd(It.IsAny<TestCase>(), It.Is<TestOutcome>(to => to == TestOutcome.Skipped)),

GoogleTestAdapter/TestAdapter.Tests/TestExecutorSequentialTests.cs

@@ -23,9 +23,9 @@ public class TestExecutorSequentialTests : TestExecutorTestsBase
 
         public TestExecutorSequentialTests() : base(false, 1) { }
 
-        protected override void CheckMockInvocations(int nrOfPassedTests, int nrOfFailedTests, int nrOfUnexecutedTests, int nrOfSkippedTests)
+        protected override void CheckMockInvocations(int nrOfPassedTests, int nrOfFailedTests, int nrOfUnexecutedTests, int nrOfSkippedTests, int nrOfNotFoundTests)
         {
-            base.CheckMockInvocations(nrOfPassedTests, nrOfFailedTests, nrOfUnexecutedTests, nrOfSkippedTests);
+            base.CheckMockInvocations(nrOfPassedTests, nrOfFailedTests, nrOfUnexecutedTests, nrOfSkippedTests, nrOfNotFoundTests);
 
             MockFrameworkHandle.Verify(h => h.RecordResult(It.Is<TestResult>(tr => tr.Outcome == TestOutcome.Passed)),
                 Times.Exactly(nrOfPassedTests));
@@ -41,6 +41,11 @@ protected override void CheckMockInvocations(int nrOfPassedTests, int nrOfFailed
                 Times.Exactly(nrOfSkippedTests));
             MockFrameworkHandle.Verify(h => h.RecordEnd(It.IsAny<TestCase>(), It.Is<TestOutcome>(to => to == TestOutcome.Skipped)),
                 Times.Exactly(nrOfSkippedTests));
+
+            MockFrameworkHandle.Verify(h => h.RecordResult(It.Is<TestResult>(tr => tr.Outcome == TestOutcome.NotFound)),
+                Times.Exactly(nrOfNotFoundTests));
+            MockFrameworkHandle.Verify(h => h.RecordEnd(It.IsAny<TestCase>(), It.Is<TestOutcome>(to => to == TestOutcome.NotFound)),
+                Times.Exactly(nrOfNotFoundTests));
         }