Fix skipped Test isn't shown as skipped/not executed in Trx Report (#…

…3773)
microsoft · Sep 9, 2024 · f31d77d · f31d77d
1 parent 6e86d38
commit f31d77d
Show file tree

Hide file tree

Showing 5 changed files with 49 additions and 26 deletions.
diff --git a/src/Platform/Microsoft.Testing.Extensions.TrxReport/TrxDataConsumer.cs b/src/Platform/Microsoft.Testing.Extensions.TrxReport/TrxDataConsumer.cs
@@ -48,6 +48,7 @@ internal sealed class TrxReportGenerator :
     private DateTimeOffset? _testStartTime;
     private int _failedTestsCount;
     private int _passedTestsCount;
+    private int _notExecutedTestsCount;
     private bool _adapterSupportTrxCapability;
 
     public TrxReportGenerator(
@@ -128,6 +129,7 @@ public Task ConsumeAsync(IDataProducer dataProducer, IData value, CancellationTo
                     else if (nodeState is SkippedTestNodeStateProperty)
                     {
                         _tests.Add(nodeChangedMessage);
+                        _notExecutedTestsCount++;
                     }
 
                     break;
@@ -225,7 +227,7 @@ public async Task OnTestSessionFinishingAsync(SessionUid sessionUid, Cancellatio
             ApplicationStateGuard.Ensure(_testStartTime is not null);
 
             TrxReportEngine trxReportGeneratorEngine = new(_testApplicationModuleInfo, _environment, _commandLineOptionsService, _configuration,
-            _clock, _tests.ToArray(), _failedTestsCount, _passedTestsCount, _artifactsByExtension, _artifactsByTestNode,
+            _clock, _tests.ToArray(), _failedTestsCount, _passedTestsCount, _notExecutedTestsCount, _artifactsByExtension, _artifactsByTestNode,
             _adapterSupportTrxCapability, _testFramework, _testStartTime.Value, cancellationToken);
             string reportFileName = await trxReportGeneratorEngine.GenerateReportAsync();
 

diff --git a/src/Platform/Microsoft.Testing.Extensions.TrxReport/TrxProcessLifetimeHandler.cs b/src/Platform/Microsoft.Testing.Extensions.TrxReport/TrxProcessLifetimeHandler.cs
@@ -158,7 +158,7 @@ public async Task OnTestHostProcessExitedAsync(ITestHostProcessInformation testH
         if (!testHostProcessInformation.HasExitedGracefully)
         {
             TrxReportEngine trxReportGeneratorEngine = new(_testApplicationModuleInfo, _environment, _commandLineOptions, _configuration,
-                _clock, [], 0, 0,
+                _clock, [], 0, 0, 0,
                 artifacts,
                 new Dictionary<TestNodeUid, List<SessionFileArtifact>>(),
                 adapterSupportTrxCapability: null,
@@ -188,7 +188,7 @@ await _messageBus.PublishAsync(
         if (_fileArtifacts.Count > 0)
         {
             TrxReportEngine trxReportGeneratorEngine = new(_testApplicationModuleInfo, _environment, _commandLineOptions, _configuration,
-               _clock, [], 0, 0,
+               _clock, [], 0, 0, 0,
                artifacts,
                new Dictionary<TestNodeUid, List<SessionFileArtifact>>(),
                false,

diff --git a/src/Platform/Microsoft.Testing.Extensions.TrxReport/TrxReportEngine.cs b/src/Platform/Microsoft.Testing.Extensions.TrxReport/TrxReportEngine.cs
@@ -94,6 +94,7 @@ internal sealed partial class TrxReportEngine
     private readonly TestNodeUpdateMessage[] _testNodeUpdatedMessages;
     private readonly int _failedTestsCount;
     private readonly int _passedTestsCount;
+    private readonly int _notExecutedTestsCount;
     private readonly Dictionary<IExtension, List<SessionFileArtifact>> _artifactsByExtension;
     private readonly Dictionary<TestNodeUid, List<SessionFileArtifact>> _artifactsByTestNode;
     private readonly bool? _adapterSupportTrxCapability;
@@ -104,27 +105,28 @@ internal sealed partial class TrxReportEngine
     private readonly IFileSystem _fileSystem;
     private readonly bool _isCopyingFileAllowed;
 
-    public TrxReportEngine(ITestApplicationModuleInfo testApplicationModuleInfo, IEnvironment environment, ICommandLineOptions commandLineOptionsService, IConfiguration configuration, IClock clock, TestNodeUpdateMessage[] testNodeUpdatedMessages, int failedTestsCount, int passedTestsCount, Dictionary<IExtension, List<SessionFileArtifact>> artifactsByExtension, Dictionary<TestNodeUid, List<SessionFileArtifact>> artifactsByTestNode, bool? adapterSupportTrxCapability, ITestFramework testFrameworkAdapter, DateTimeOffset testStartTime, CancellationToken cancellationToken)
-        : this(
-            new SystemFileSystem(),
-            testApplicationModuleInfo,
-            environment,
-            commandLineOptionsService,
-            configuration,
-            clock,
-            testNodeUpdatedMessages,
-            failedTestsCount,
-            passedTestsCount,
-            artifactsByExtension,
-            artifactsByTestNode,
-            adapterSupportTrxCapability,
-            testFrameworkAdapter,
-            testStartTime,
-            cancellationToken)
+    public TrxReportEngine(ITestApplicationModuleInfo testApplicationModuleInfo, IEnvironment environment, ICommandLineOptions commandLineOptionsService, IConfiguration configuration, IClock clock, TestNodeUpdateMessage[] testNodeUpdatedMessages, int failedTestsCount, int passedTestsCount, int notExecutedTestsCount, Dictionary<IExtension, List<SessionFileArtifact>> artifactsByExtension, Dictionary<TestNodeUid, List<SessionFileArtifact>> artifactsByTestNode, bool? adapterSupportTrxCapability, ITestFramework testFrameworkAdapter, DateTimeOffset testStartTime, CancellationToken cancellationToken)
+    : this(
+        new SystemFileSystem(),
+        testApplicationModuleInfo,
+        environment,
+        commandLineOptionsService,
+        configuration,
+        clock,
+        testNodeUpdatedMessages,
+        failedTestsCount,
+        passedTestsCount,
+        notExecutedTestsCount,
+        artifactsByExtension,
+        artifactsByTestNode,
+        adapterSupportTrxCapability,
+        testFrameworkAdapter,
+        testStartTime,
+        cancellationToken)
     {
     }
 
-    internal TrxReportEngine(IFileSystem fileSystem, ITestApplicationModuleInfo testApplicationModuleInfo, IEnvironment environment, ICommandLineOptions commandLineOptionsService, IConfiguration configuration, IClock clock, TestNodeUpdateMessage[] testNodeUpdatedMessages, int failedTestsCount, int passedTestsCount, Dictionary<IExtension, List<SessionFileArtifact>> artifactsByExtension, Dictionary<TestNodeUid, List<SessionFileArtifact>> artifactsByTestNode, bool? adapterSupportTrxCapability, ITestFramework testFrameworkAdapter, DateTimeOffset testStartTime, CancellationToken cancellationToken, bool isCopyingFileAllowed = true)
+    public TrxReportEngine(IFileSystem fileSystem, ITestApplicationModuleInfo testApplicationModuleInfo, IEnvironment environment, ICommandLineOptions commandLineOptionsService, IConfiguration configuration, IClock clock, TestNodeUpdateMessage[] testNodeUpdatedMessages, int failedTestsCount, int passedTestsCount, int notExecutedTestsCount, Dictionary<IExtension, List<SessionFileArtifact>> artifactsByExtension, Dictionary<TestNodeUid, List<SessionFileArtifact>> artifactsByTestNode, bool? adapterSupportTrxCapability, ITestFramework testFrameworkAdapter, DateTimeOffset testStartTime, CancellationToken cancellationToken, bool isCopyingFileAllowed = true)
     {
         _testApplicationModuleInfo = testApplicationModuleInfo;
         _environment = environment;
@@ -134,6 +136,7 @@ internal TrxReportEngine(IFileSystem fileSystem, ITestApplicationModuleInfo test
         _testNodeUpdatedMessages = testNodeUpdatedMessages;
         _failedTestsCount = failedTestsCount;
         _passedTestsCount = passedTestsCount;
+        _notExecutedTestsCount = notExecutedTestsCount;
         _artifactsByExtension = artifactsByExtension;
         _artifactsByTestNode = artifactsByTestNode;
         _adapterSupportTrxCapability = adapterSupportTrxCapability;
@@ -302,7 +305,7 @@ private async Task AddResultSummaryAsync(XElement testRun, string resultSummaryO
             new XAttribute("inconclusive", 0),
             new XAttribute("passedButRunAborted", 0),
             new XAttribute("notRunnable", 0),
-            new XAttribute("notExecuted", 0),
+            new XAttribute("notExecuted", _notExecutedTestsCount),
             new XAttribute("disconnected", 0),
             new XAttribute("warning", 0),
             new XAttribute("completed", 0),

diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/TrxTests.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/TrxTests.cs
@@ -81,7 +81,7 @@ public async Task Trx_WhenSkipTest_ItAppearsAsExpectedInsideTheTrx(string tfm)
         Assert.That(trxContent.Contains(@"<UnitTest name=""TestMethod1"), trxContent);
         Assert.That(trxContent.Contains(@"<TestEntry "), trxContent);
         Assert.That(trxContent.Contains("""<ResultSummary outcome="Completed">"""), trxContent);
-        Assert.That(trxContent.Contains("""<Counters total="2" executed="0" passed="0" failed="0" error="0" timeout="0" aborted="0" inconclusive="0" passedButRunAborted="0" notRunnable="0" notExecuted="0" disconnected="0" warning="0" completed="0" inProgress="0" pending="0" />"""), trxContent);
+        Assert.That(trxContent.Contains("""<Counters total="2" executed="0" passed="0" failed="0" error="0" timeout="0" aborted="0" inconclusive="0" passedButRunAborted="0" notRunnable="0" notExecuted="2" disconnected="0" warning="0" completed="0" inProgress="0" pending="0" />"""), trxContent);
     }
 
     [ArgumentsProvider(nameof(TargetFrameworks.Net), typeof(TargetFrameworks))]

diff --git a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/TrxTests.cs b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/TrxTests.cs
@@ -51,6 +51,24 @@ public async Task TrxReportEngine_GenerateReportAsyncWithNullAdapterSupportTrxCa
         Assert.IsFalse(trxContent.Contains(@"className="));
     }
 
+    public async Task TrxReportEngine_GenerateReportAsyncWithNotExecutedTests_TrxExecutedTestsCountHasIt()
+    {
+        // Arrange
+        using MemoryFileStream memoryStream = new();
+        PropertyBag propertyBag = new(new PassedTestNodeStateProperty());
+        TrxReportEngine trxReportEngine = GenerateTrxReportEngine(1, 0, propertyBag, memoryStream, notExecutedTestsCount: 1);
+
+        // Act
+        string fileName = await trxReportEngine.GenerateReportAsync(keepReportFileStreamOpen: true);
+
+        // Assert
+        AssertExpectedTrxFileName(fileName);
+        XDocument xml = GetTrxContent(memoryStream);
+        AssertTrxOutcome(xml, "Completed");
+        string trxContent = xml.ToString();
+        Assert.IsTrue(trxContent.Contains(@"notExecuted=""1"""));
+    }
+
     public async Task TrxReportEngine_GenerateReportAsync_WithArgumentTrxReportFileName_FileIsCorrectlyGenerated()
     {
         // Arrange
@@ -359,7 +377,7 @@ public async Task TrxReportEngine_GenerateReportAsync_FileAlreadyExists_WillRetr
         _ = _environmentMock.SetupGet(_ => _.MachineName).Returns("MachineName");
         _ = _testApplicationModuleInfoMock.Setup(_ => _.GetCurrentTestApplicationFullPath()).Returns("TestAppPath");
         TrxReportEngine trxReportEngine = new(_fileSystem.Object, _testApplicationModuleInfoMock.Object, _environmentMock.Object, _commandLineOptionsMock.Object,
-            _configurationMock.Object, _clockMock.Object, [], 0, 0,
+            _configurationMock.Object, _clockMock.Object, [], 0, 0, 0,
             _artifactsByExtension, _artifactsByTestNode, true, _testFrameworkMock.Object, DateTime.UtcNow, CancellationToken.None,
             isCopyingFileAllowed: false);
 
@@ -394,7 +412,7 @@ private static void AssertExpectedTrxFileName(string fileName)
            => Assert.IsTrue(fileName.Equals("_MachineName_0001-01-01_00_00_00.000.trx", StringComparison.Ordinal));
 
     private TrxReportEngine GenerateTrxReportEngine(int passedTestsCount, int failedTestsCount, PropertyBag propertyBag, MemoryFileStream memoryStream,
-           bool? adapterSupportTrxCapability = null)
+           bool? adapterSupportTrxCapability = null, int notExecutedTestsCount = 0)
     {
         var testNode = new TestNodeUpdateMessage(
             new SessionUid("1"),
@@ -414,7 +432,7 @@ private TrxReportEngine GenerateTrxReportEngine(int passedTestsCount, int failed
         _ = _testApplicationModuleInfoMock.Setup(_ => _.GetCurrentTestApplicationFullPath()).Returns("TestAppPath");
 
         return new TrxReportEngine(_fileSystem.Object, _testApplicationModuleInfoMock.Object, _environmentMock.Object, _commandLineOptionsMock.Object,
-                   _configurationMock.Object, _clockMock.Object, testNodeUpdatedMessages, failedTestsCount, passedTestsCount,
+                   _configurationMock.Object, _clockMock.Object, testNodeUpdatedMessages, failedTestsCount, passedTestsCount, notExecutedTestsCount,
                    _artifactsByExtension, _artifactsByTestNode, adapterSupportTrxCapability, _testFrameworkMock.Object, testStartTime, cancellationToken,
                    isCopyingFileAllowed: false);
     }