diff --git a/src/Microsoft.TestPlatform.Client/Discovery/DiscoveryRequest.cs b/src/Microsoft.TestPlatform.Client/Discovery/DiscoveryRequest.cs index 343dd8bee7..459bc31656 100644 --- a/src/Microsoft.TestPlatform.Client/Discovery/DiscoveryRequest.cs +++ b/src/Microsoft.TestPlatform.Client/Discovery/DiscoveryRequest.cs @@ -6,6 +6,8 @@ using System.Linq; using System.Threading; +using Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework; +using Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework.Utilities; using Microsoft.VisualStudio.TestPlatform.Common.Telemetry; using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities; using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Interfaces; @@ -248,6 +250,22 @@ public void HandleDiscoveryComplete(DiscoveryCompleteEventArgs discoveryComplete OnDiscoveredTests.SafeInvoke(this, discoveredTestsEvent, "DiscoveryRequest.DiscoveryComplete"); } + // Add extensions discovered by vstest.console. + // + // TODO(copoiena): Writing telemetry twice is less than ideal. + // We first write telemetry data in the _requestData variable in the ParallelRunEventsHandler + // and then we write again here. We should refactor this code and write only once. + discoveryCompleteEventArgs.DiscoveredExtensions = TestExtensions.CreateMergedDictionary( + discoveryCompleteEventArgs.DiscoveredExtensions, + TestPluginCache.Instance.TestExtensions.GetCachedExtensions()); + + if (RequestData.IsTelemetryOptedIn) + { + TestExtensions.AddExtensionTelemetry( + discoveryCompleteEventArgs.Metrics, + discoveryCompleteEventArgs.DiscoveredExtensions); + } + LoggerManager.HandleDiscoveryComplete(discoveryCompleteEventArgs); OnDiscoveryComplete.SafeInvoke(this, discoveryCompleteEventArgs, "DiscoveryRequest.DiscoveryComplete"); } @@ -406,6 +424,25 @@ private string UpdateRawMessageWithTelemetryInfo(DiscoveryCompletePayload discov // Collecting Total Time Taken discoveryCompletePayload.Metrics[TelemetryDataConstants.TimeTakenInSecForDiscovery] = discoveryFinalTimeTakenForDesignMode.TotalSeconds; + + // Add extensions discovered by vstest.console. + // + // TODO(copoiena): + // Doing extension merging here is incorrect because we can end up not merging the + // cached extensions for the current process (i.e. vstest.console) and hence have + // an incomplete list of discovered extensions. This can happen because this method + // is called only if telemetry is opted in (see: HandleRawMessage). We should handle + // this merge a level above in order to be consistent, but that means we'd have to + // deserialize all raw messages no matter if telemetry is opted in or not and that + // would probably mean a performance hit. + discoveryCompletePayload.DiscoveredExtensions = TestExtensions.CreateMergedDictionary( + discoveryCompletePayload.DiscoveredExtensions, + TestPluginCache.Instance.TestExtensions.GetCachedExtensions()); + + // Write extensions to telemetry data. + TestExtensions.AddExtensionTelemetry( + discoveryCompletePayload.Metrics, + discoveryCompletePayload.DiscoveredExtensions); } if (message is VersionedMessage message1) diff --git a/src/Microsoft.TestPlatform.Client/Execution/TestRunRequest.cs b/src/Microsoft.TestPlatform.Client/Execution/TestRunRequest.cs index c74ea3560e..0efe23df94 100644 --- a/src/Microsoft.TestPlatform.Client/Execution/TestRunRequest.cs +++ b/src/Microsoft.TestPlatform.Client/Execution/TestRunRequest.cs @@ -8,6 +8,8 @@ using System.Linq; using System.Threading; +using Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework; +using Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework.Utilities; using Microsoft.VisualStudio.TestPlatform.Common.Telemetry; using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities; using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Interfaces; @@ -400,6 +402,22 @@ public void HandleTestRunComplete(TestRunCompleteEventArgs runCompleteArgs!!, Te runCompleteArgs.InvokedDataCollectors, _runRequestTimeTracker.Elapsed); + // Add extensions discovered by vstest.console. + // + // TODO(copoiena): Writing telemetry twice is less than ideal. + // We first write telemetry data in the _requestData variable in the ParallelRunEventsHandler + // and then we write again here. We should refactor this code and write only once. + runCompleteArgs.DiscoveredExtensions = TestExtensions.CreateMergedDictionary( + runCompleteArgs.DiscoveredExtensions, + TestPluginCache.Instance.TestExtensions.GetCachedExtensions()); + + if (_requestData.IsTelemetryOptedIn) + { + TestExtensions.AddExtensionTelemetry( + runCompleteArgs.Metrics, + runCompleteArgs.DiscoveredExtensions); + } + // Ignore the time sent (runCompleteArgs.ElapsedTimeInRunningTests) // by either engines - as both calculate at different points // If we use them, it would be an incorrect comparison between TAEF and Rocksteady @@ -420,7 +438,6 @@ public void HandleTestRunComplete(TestRunCompleteEventArgs runCompleteArgs!!, Te // Notify the waiting handle that run is complete _runCompletionEvent.Set(); - var executionTotalTimeTaken = DateTime.UtcNow - _executionStartTime; // Fill in the time taken to complete the run @@ -583,6 +600,25 @@ private string UpdateRawMessageWithTelemetryInfo(TestRunCompletePayload testRunC // Fill in the time taken to complete the run var executionTotalTimeTakenForDesignMode = DateTime.UtcNow - _executionStartTime; testRunCompletePayload.TestRunCompleteArgs.Metrics[TelemetryDataConstants.TimeTakenInSecForRun] = executionTotalTimeTakenForDesignMode.TotalSeconds; + + // Add extensions discovered by vstest.console. + // + // TODO(copoiena): + // Doing extension merging here is incorrect because we can end up not merging the + // cached extensions for the current process (i.e. vstest.console) and hence have + // an incomplete list of discovered extensions. This can happen because this method + // is called only if telemetry is opted in (see: HandleRawMessage). We should handle + // this merge a level above in order to be consistent, but that means we'd have to + // deserialize all raw messages no matter if telemetry is opted in or not and that + // would probably mean a performance hit. + testRunCompletePayload.TestRunCompleteArgs.DiscoveredExtensions = TestExtensions.CreateMergedDictionary( + testRunCompletePayload.TestRunCompleteArgs.DiscoveredExtensions, + TestPluginCache.Instance.TestExtensions.GetCachedExtensions()); + + // Write extensions to telemetry data. + TestExtensions.AddExtensionTelemetry( + testRunCompletePayload.TestRunCompleteArgs.Metrics, + testRunCompletePayload.TestRunCompleteArgs.DiscoveredExtensions); } if (message is VersionedMessage message1) diff --git a/src/Microsoft.TestPlatform.Common/ExtensionFramework/TestPluginCache.cs b/src/Microsoft.TestPlatform.Common/ExtensionFramework/TestPluginCache.cs index d0b0393a1e..ce95ce8c0c 100644 --- a/src/Microsoft.TestPlatform.Common/ExtensionFramework/TestPluginCache.cs +++ b/src/Microsoft.TestPlatform.Common/ExtensionFramework/TestPluginCache.cs @@ -117,7 +117,8 @@ public List GetExtensionPaths(string endsWithPattern, bool skipDefaultEx /// /// The . of test plugin info. /// - public Dictionary DiscoverTestExtensions(string endsWithPattern) + public Dictionary DiscoverTestExtensions( + string endsWithPattern) where TPluginInfo : TestPluginInformation { EqtTrace.Verbose("TestPluginCache.DiscoverTestExtensions: finding test extensions in assemblies ends with: {0} TPluginInfo: {1} TExtension: {2}", endsWithPattern, typeof(TPluginInfo), typeof(TExtension)); @@ -309,35 +310,37 @@ internal IEnumerable DefaultExtensionPaths /// /// The . /// - internal Dictionary GetTestExtensions(string extensionAssembly, bool skipCache = false) where TPluginInfo : TestPluginInformation + internal Dictionary GetTestExtensions( + string extensionAssembly, + bool skipCache = false) + where TPluginInfo : TestPluginInformation { if (skipCache) { return GetTestExtensions(new List() { extensionAssembly }); } - else - { - // Check if extensions from this assembly have already been discovered. - var extensions = TestExtensions?.GetExtensionsDiscoveredFromAssembly( - TestExtensions.GetTestExtensionCache(), - extensionAssembly); - if (extensions != null && extensions.Count > 0) - { - return extensions; - } + // Check if extensions from this assembly have already been discovered. + var extensions = TestExtensions?.GetExtensionsDiscoveredFromAssembly( + TestExtensions.GetTestExtensionCache(), + extensionAssembly); - var pluginInfos = GetTestExtensions(new List() { extensionAssembly }); + if (extensions?.Count > 0) + { + return extensions; + } - // Add extensions discovered to the cache. - if (TestExtensions == null) - { - TestExtensions = new TestExtensions(); - } + var pluginInfos = GetTestExtensions(new List() { extensionAssembly }); - TestExtensions.AddExtension(pluginInfos); - return pluginInfos; + // Add extensions discovered to the cache. + if (TestExtensions == null) + { + TestExtensions = new TestExtensions(); } + + TestExtensions.AddExtension(pluginInfos); + + return pluginInfos; } /// diff --git a/src/Microsoft.TestPlatform.Common/ExtensionFramework/Utilities/TestExtensions.cs b/src/Microsoft.TestPlatform.Common/ExtensionFramework/Utilities/TestExtensions.cs index a1a0afd360..224a00815d 100644 --- a/src/Microsoft.TestPlatform.Common/ExtensionFramework/Utilities/TestExtensions.cs +++ b/src/Microsoft.TestPlatform.Common/ExtensionFramework/Utilities/TestExtensions.cs @@ -5,11 +5,11 @@ using System.Collections.Generic; using System.Linq; using System.Reflection; +using System.Text; using Microsoft.VisualStudio.TestPlatform.Common.DataCollector; - +using Microsoft.VisualStudio.TestPlatform.Common.Telemetry; using Microsoft.VisualStudio.TestPlatform.ObjectModel; - using Microsoft.VisualStudio.TestPlatform.PlatformAbstractions; #nullable disable @@ -91,6 +91,80 @@ public class TestExtensions /// internal bool AreDataCollectorsCached { get; set; } + /// + /// Merge two extension dictionaries. + /// + /// + /// First extension dictionary. + /// Second extension dictionary. + /// + /// + /// A dictionary representing the merger between the two input dictionaries. + /// + internal static Dictionary> CreateMergedDictionary( + Dictionary> first, + Dictionary> second) + { + var isFirstNullOrEmpty = first == null || first.Count == 0; + var isSecondNullOrEmpty = second == null || second.Count == 0; + + // Sanity checks. + if (isFirstNullOrEmpty && isSecondNullOrEmpty) + { + return new Dictionary>(); + } + if (isFirstNullOrEmpty) + { + return new Dictionary>(second); + } + if (isSecondNullOrEmpty) + { + return new Dictionary>(first); + } + + // Copy all the keys in the first dictionary into the resulting dictionary. + var result = new Dictionary>(first); + + foreach (var kvp in second) + { + // If the "source" set is empty there's no reason to continue merging for this key. + if (kvp.Value == null || kvp.Value.Count == 0) + { + continue; + } + + // If there's no key-value pair entry in the "destination" dictionary for the current + // key in the "source" dictionary, we copy the "source" set wholesale. + if (!result.ContainsKey(kvp.Key)) + { + result.Add(kvp.Key, kvp.Value); + continue; + } + + // Getting here means there's already an entry for the "source" key in the "destination" + // dictionary which means we need to copy individual set elements from the "source" set + // to the "destination" set. + result[kvp.Key] = MergeSets(result[kvp.Key], kvp.Value); + } + + return result; + } + + /// + /// Add extension-related telemetry. + /// + /// + /// A collection representing the telemetry data. + /// The input extension collection. + internal static void AddExtensionTelemetry( + IDictionary metrics, + Dictionary> extensions) + { + metrics.Add( + TelemetryDataConstants.DiscoveredExtensions, + SerializeExtensionDictionary(extensions)); + } + /// /// Adds the extensions specified to the current set of extensions. /// @@ -307,6 +381,27 @@ internal void SetTestExtensionsCacheStatusToTrue() where TPluginInf } } + /// + /// Gets the cached extensions for the current process. + /// + /// + /// A dictionary representing the cached extensions for the current process. + internal Dictionary> GetCachedExtensions() + { + var extensions = new Dictionary>(); + + // Write all "known" cached extension. + AddCachedExtensionToDictionary(extensions, "TestDiscoverers", TestDiscoverers?.Values); + AddCachedExtensionToDictionary(extensions, "TestExecutors", TestExecutors?.Values); + AddCachedExtensionToDictionary(extensions, "TestExecutors2", TestExecutors2?.Values); + AddCachedExtensionToDictionary(extensions, "TestSettingsProviders", TestSettingsProviders?.Values); + AddCachedExtensionToDictionary(extensions, "TestLoggers", TestLoggers?.Values); + AddCachedExtensionToDictionary(extensions, "TestHosts", TestHosts?.Values); + AddCachedExtensionToDictionary(extensions, "DataCollectors", DataCollectors?.Values); + + return extensions; + } + /// /// The invalidate cache of plugin infos. /// @@ -390,4 +485,45 @@ private void SetTestExtensionCache(Dictionary } } + private void AddCachedExtensionToDictionary( + Dictionary> extensionDict, + string extensionType, + IEnumerable extensions) + where T : TestPluginInformation + { + if (extensions == null) + { + return; + } + + extensionDict.Add(extensionType, new HashSet(extensions.Select(e => e.IdentifierData))); + } + + private static string SerializeExtensionDictionary(IDictionary> extensions) + { + StringBuilder sb = new(); + + foreach (var kvp in extensions) + { + if (kvp.Value?.Count > 0) + { + sb.AppendFormat("{0}=[{1}];", kvp.Key, string.Join(",", kvp.Value)); + } + } + + return sb.ToString(); + } + + private static HashSet MergeSets(HashSet firstSet, HashSet secondSet) + { + var mergedSet = new HashSet(firstSet); + + // No need to worry about duplicates as the set implementation handles this already. + foreach (var key in secondSet) + { + mergedSet.Add(key); + } + + return mergedSet; + } } diff --git a/src/Microsoft.TestPlatform.Common/Telemetry/TelemetryDataConstants.cs b/src/Microsoft.TestPlatform.Common/Telemetry/TelemetryDataConstants.cs index 4db3fce97d..ba3c2f8fd5 100644 --- a/src/Microsoft.TestPlatform.Common/Telemetry/TelemetryDataConstants.cs +++ b/src/Microsoft.TestPlatform.Common/Telemetry/TelemetryDataConstants.cs @@ -10,6 +10,9 @@ namespace Microsoft.VisualStudio.TestPlatform.Common.Telemetry; /// internal static class TelemetryDataConstants { + // ******************** General *********************** + public static readonly string DiscoveredExtensions = "VS.TestPlatform.DiscoveredExtensions"; + // ******************** Execution *********************** public static readonly string ParallelEnabledDuringExecution = "VS.TestRun.ParallelEnabled"; diff --git a/src/Microsoft.TestPlatform.CommunicationUtilities/Messages/DiscoveryCompletePayload.cs b/src/Microsoft.TestPlatform.CommunicationUtilities/Messages/DiscoveryCompletePayload.cs index dd76b70ea8..d08b59dc2d 100644 --- a/src/Microsoft.TestPlatform.CommunicationUtilities/Messages/DiscoveryCompletePayload.cs +++ b/src/Microsoft.TestPlatform.CommunicationUtilities/Messages/DiscoveryCompletePayload.cs @@ -48,4 +48,9 @@ public class DiscoveryCompletePayload /// Gets or sets list of sources which were not discovered at all. /// public IList NotDiscoveredSources { get; set; } = new List(); + + /// + /// Gets or sets the collection of discovered extensions. + /// + public Dictionary> DiscoveredExtensions { get; set; } = new(); } diff --git a/src/Microsoft.TestPlatform.CommunicationUtilities/PublicAPI/PublicAPI.Unshipped.txt b/src/Microsoft.TestPlatform.CommunicationUtilities/PublicAPI/PublicAPI.Unshipped.txt index 47a461296e..52a4e630e0 100644 --- a/src/Microsoft.TestPlatform.CommunicationUtilities/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/Microsoft.TestPlatform.CommunicationUtilities/PublicAPI/PublicAPI.Unshipped.txt @@ -6,3 +6,5 @@ Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel.Discovery Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel.DiscoveryCompletePayload.PartiallyDiscoveredSources.get -> System.Collections.Generic.IList Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel.DiscoveryCompletePayload.PartiallyDiscoveredSources.set -> void Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.TestRequestSender.SendDiscoveryAbort() -> void +Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel.DiscoveryCompletePayload.DiscoveredExtensions.get -> System.Collections.Generic.Dictionary> +Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel.DiscoveryCompletePayload.DiscoveredExtensions.set -> void diff --git a/src/Microsoft.TestPlatform.CommunicationUtilities/TestRequestSender.cs b/src/Microsoft.TestPlatform.CommunicationUtilities/TestRequestSender.cs index 5e4a0b4d71..5320a70e6f 100644 --- a/src/Microsoft.TestPlatform.CommunicationUtilities/TestRequestSender.cs +++ b/src/Microsoft.TestPlatform.CommunicationUtilities/TestRequestSender.cs @@ -579,7 +579,9 @@ private void OnDiscoveryMessageReceived(ITestDiscoveryEventsHandler2 discoveryEv discoveryCompletePayload.IsAborted, discoveryCompletePayload.FullyDiscoveredSources, discoveryCompletePayload.PartiallyDiscoveredSources, - discoveryCompletePayload.NotDiscoveredSources); + discoveryCompletePayload.NotDiscoveredSources, + discoveryCompletePayload.DiscoveredExtensions); + discoveryCompleteEventArgs.Metrics = discoveryCompletePayload.Metrics; discoveryEventsHandler.HandleDiscoveryComplete( discoveryCompleteEventArgs, diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelDiscoveryDataAggregator.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelDiscoveryDataAggregator.cs index c6f557e2c2..654688a4ed 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelDiscoveryDataAggregator.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelDiscoveryDataAggregator.cs @@ -6,6 +6,7 @@ using System.Collections.Generic; using System.Linq; +using Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework.Utilities; using Microsoft.VisualStudio.TestPlatform.Common.Telemetry; using Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Discovery; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine; @@ -27,6 +28,7 @@ public ParallelDiscoveryDataAggregator() IsAborted = false; TotalTests = 0; _metricsAggregator = new ConcurrentDictionary(); + DiscoveredExtensions = new(); } /// @@ -39,6 +41,10 @@ public ParallelDiscoveryDataAggregator() /// public long TotalTests { get; private set; } + /// + /// A collection of aggregated discovered extensions. + /// + public Dictionary> DiscoveredExtensions { get; private set; } /// /// Dictionary which stores source with corresponding discoveryStatus @@ -84,7 +90,10 @@ public IDictionary GetAggregatedDiscoveryDataMetrics() /// Aggregate discovery data /// Must be thread-safe as this is expected to be called by parallel managers /// - public void Aggregate(long totalTests, bool isAborted) + public void Aggregate( + long totalTests, + bool isAborted, + Dictionary> discoveredExtensions) { lock (_dataUpdateSyncObject) { @@ -100,6 +109,9 @@ public void Aggregate(long totalTests, bool isAborted) } TotalTests += totalTests; + + // Aggregate the discovered extensions. + DiscoveredExtensions = TestExtensions.CreateMergedDictionary(DiscoveredExtensions, discoveredExtensions); } } diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelDiscoveryEventsHandler.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelDiscoveryEventsHandler.cs index 463f42ee8a..acb32d96c6 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelDiscoveryEventsHandler.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelDiscoveryEventsHandler.cs @@ -4,6 +4,7 @@ using System.Collections.Generic; using System.Linq; +using Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework; using Microsoft.VisualStudio.TestPlatform.Common.Telemetry; using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities; using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Interfaces; @@ -69,7 +70,7 @@ public void HandleDiscoveryComplete(DiscoveryCompleteEventArgs discoveryComplete var isAborted = discoveryCompleteEventArgs.IsAborted; // Aggregate for final discovery complete - _discoveryDataAggregator.Aggregate(totalTests, isAborted); + _discoveryDataAggregator.Aggregate(totalTests, isAborted, discoveryCompleteEventArgs.DiscoveredExtensions); // Aggregate Discovery Data Metrics _discoveryDataAggregator.AggregateDiscoveryDataMetrics(discoveryCompleteEventArgs.Metrics); @@ -106,7 +107,7 @@ public void HandleDiscoveryComplete(DiscoveryCompleteEventArgs discoveryComplete // we need to set isAborted = true and totalTests = -1 if (_parallelProxyDiscoveryManager.IsAbortRequested) { - _discoveryDataAggregator.Aggregate(-1, true); + _discoveryDataAggregator.Aggregate(-1, true, null); } // In case of sequential discovery - RawMessage would have contained a 'DiscoveryCompletePayload' object @@ -118,7 +119,8 @@ public void HandleDiscoveryComplete(DiscoveryCompleteEventArgs discoveryComplete LastDiscoveredTests = null, FullyDiscoveredSources = fullyDiscovered, PartiallyDiscoveredSources = partiallyDiscovered, - NotDiscoveredSources = notDiscovered + NotDiscoveredSources = notDiscovered, + DiscoveredExtensions = _discoveryDataAggregator.DiscoveredExtensions, }; // Collecting Final Discovery State @@ -136,7 +138,8 @@ public void HandleDiscoveryComplete(DiscoveryCompleteEventArgs discoveryComplete _discoveryDataAggregator.IsAborted, fullyDiscovered, partiallyDiscovered, - notDiscovered); + notDiscovered, + _discoveryDataAggregator.DiscoveredExtensions); finalDiscoveryCompleteEventArgs.Metrics = aggregatedDiscoveryDataMetrics; diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelRunDataAggregator.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelRunDataAggregator.cs index 5a42d7303b..0869b061fc 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelRunDataAggregator.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelRunDataAggregator.cs @@ -7,6 +7,7 @@ using System.Collections.ObjectModel; using System.Linq; +using Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework.Utilities; using Microsoft.VisualStudio.TestPlatform.Common.Telemetry; using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; @@ -37,6 +38,7 @@ public ParallelRunDataAggregator(string runSettingsXml!!) RunCompleteArgsAttachments = new List(); InvokedDataCollectors = new Collection(); Exceptions = new List(); + DiscoveredExtensions = new Dictionary>(); _executorUris = new List(); _testRunStatsList = new List(); @@ -58,6 +60,11 @@ public ParallelRunDataAggregator(string runSettingsXml!!) public HashSet ExecutorUris => new(_executorUris); + /// + /// A collection of aggregated discovered extensions. + /// + public Dictionary> DiscoveredExtensions { get; private set; } + public bool IsAborted { get; private set; } public bool IsCanceled { get; private set; } @@ -136,7 +143,8 @@ public void Aggregate( bool isCanceled, ICollection runContextAttachments, Collection runCompleteArgsAttachments, - Collection invokedDataCollectors) + Collection invokedDataCollectors, + Dictionary> discoveredExtensions) { lock (_dataUpdateSyncObject) { @@ -167,6 +175,9 @@ public void Aggregate( } } } + + // Aggregate the discovered extensions. + DiscoveredExtensions = TestExtensions.CreateMergedDictionary(DiscoveredExtensions, discoveredExtensions); } } @@ -199,5 +210,4 @@ public void AggregateRunDataMetrics(IDictionary metrics) } } } - } diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelRunEventsHandler.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelRunEventsHandler.cs index 3af43fd63d..41c71266ed 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelRunEventsHandler.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelRunEventsHandler.cs @@ -80,12 +80,21 @@ public virtual void HandleTestRunComplete( _runDataAggregator.ElapsedTime); // Collect Final RunState - _requestData.MetricsCollection.Add(TelemetryDataConstants.RunState, _runDataAggregator.IsAborted ? "Aborted" : _runDataAggregator.IsCanceled ? "Canceled" : "Completed"); + _requestData.MetricsCollection.Add( + TelemetryDataConstants.RunState, + _runDataAggregator.IsAborted + ? "Aborted" + : _runDataAggregator.IsCanceled ? "Canceled" : "Completed"); + + // Add the map containing discovered extensions to the event args. This map contains + // only extensions discovered by the testhost processes. Current process extensions + // (i.e. vstest.console) are added later on. + completedArgs.DiscoveredExtensions = _runDataAggregator.DiscoveredExtensions; // Collect Aggregated Metrics Data var aggregatedRunDataMetrics = _runDataAggregator.GetAggregatedRunDataMetrics(); - completedArgs.Metrics = aggregatedRunDataMetrics; + HandleParallelTestRunComplete(completedArgs); } } @@ -115,7 +124,8 @@ protected bool HandleSingleTestRunComplete(TestRunCompleteEventArgs testRunCompl testRunCompleteArgs.IsCanceled, runContextAttachments, testRunCompleteArgs.AttachmentSets, - testRunCompleteArgs.InvokedDataCollectors); + testRunCompleteArgs.InvokedDataCollectors, + testRunCompleteArgs.DiscoveredExtensions); // Aggregate Run Data Metrics _runDataAggregator.AggregateRunDataMetrics(testRunCompleteArgs.Metrics); diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Discovery/DiscoveryManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Discovery/DiscoveryManager.cs index 7cd5cb30b1..7610bef448 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Discovery/DiscoveryManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Discovery/DiscoveryManager.cs @@ -165,6 +165,7 @@ public void DiscoverTests(DiscoveryCriteria discoveryCriteria, ITestDiscoveryEve GetSourcesWithStatus(DiscoveryStatus.PartiallyDiscovered, _sourcesWithDiscoveryStatus), GetSourcesWithStatus(DiscoveryStatus.NotDiscovered, _sourcesWithDiscoveryStatus)); + discoveryCompleteEventsArgs.DiscoveredExtensions = TestPluginCache.Instance.TestExtensions?.GetCachedExtensions(); discoveryCompleteEventsArgs.Metrics = _requestData.MetricsCollection.Metrics; eventHandler.HandleDiscoveryComplete(discoveryCompleteEventsArgs, lastChunk); diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/EventHandlers/TestRequestHandler.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/EventHandlers/TestRequestHandler.cs index 1a9f998dc3..f28429113d 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/EventHandlers/TestRequestHandler.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/EventHandlers/TestRequestHandler.cs @@ -242,6 +242,7 @@ public void DiscoveryComplete(DiscoveryCompleteEventArgs discoveryCompleteEventA FullyDiscoveredSources = discoveryCompleteEventArgs.FullyDiscoveredSources, PartiallyDiscoveredSources = discoveryCompleteEventArgs.PartiallyDiscoveredSources, NotDiscoveredSources = discoveryCompleteEventArgs.NotDiscoveredSources, + DiscoveredExtensions = discoveryCompleteEventArgs.DiscoveredExtensions, }, _protocolVersion); SendData(data); diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Execution/BaseRunTests.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Execution/BaseRunTests.cs index cf82ebf397..2abbe4c6cd 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Execution/BaseRunTests.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Execution/BaseRunTests.cs @@ -614,6 +614,8 @@ private void RaiseTestRunComplete( // Today we don't offer an extension to run collectors for test adapters. new Collection(), elapsedTime); + + testRunCompleteEventArgs.DiscoveredExtensions = TestPluginCache.Instance.TestExtensions?.GetCachedExtensions(); testRunCompleteEventArgs.Metrics = _requestData.MetricsCollection.Metrics; TestRunEventsHandler.HandleTestRunComplete( diff --git a/src/Microsoft.TestPlatform.ObjectModel/Client/Events/DiscoveryCompleteEventArgs.cs b/src/Microsoft.TestPlatform.ObjectModel/Client/Events/DiscoveryCompleteEventArgs.cs index 95547d7a22..1b63046197 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/Client/Events/DiscoveryCompleteEventArgs.cs +++ b/src/Microsoft.TestPlatform.ObjectModel/Client/Events/DiscoveryCompleteEventArgs.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.Diagnostics; +using System.Runtime.Serialization; #nullable disable @@ -12,6 +13,7 @@ namespace Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; /// /// Event arguments used on completion of discovery /// +[DataContract] public class DiscoveryCompleteEventArgs : EventArgs { /// @@ -22,10 +24,37 @@ public class DiscoveryCompleteEventArgs : EventArgs /// List of fully discovered sources /// List of partially discovered sources /// List of not discovered sources - public DiscoveryCompleteEventArgs(long totalTests, bool isAborted, + public DiscoveryCompleteEventArgs( + long totalTests, + bool isAborted, IList fullyDiscoveredSources, IList partiallyDiscoveredSources, IList notDiscoveredSources) + : this( + totalTests, + isAborted, + fullyDiscoveredSources, + partiallyDiscoveredSources, + notDiscoveredSources, + new Dictionary>()) + { } + + /// + /// Constructor for creating event args object + /// + /// Total tests which got discovered + /// Specifies if discovery has been aborted. + /// List of fully discovered sources + /// List of partially discovered sources + /// List of not discovered sources + /// Map containing discovered extensions. + public DiscoveryCompleteEventArgs( + long totalTests, + bool isAborted, + IList fullyDiscoveredSources, + IList partiallyDiscoveredSources, + IList notDiscoveredSources, + Dictionary> discoveredExtensions) { // This event is always raised from the client side, while the total count of tests is maintained // only at the testhost end. In case of a discovery abort (various reasons including crash), it is @@ -38,6 +67,8 @@ public DiscoveryCompleteEventArgs(long totalTests, bool isAborted, FullyDiscoveredSources = fullyDiscoveredSources ?? new List(); PartiallyDiscoveredSources = partiallyDiscoveredSources ?? new List(); NotDiscoveredSources = notDiscoveredSources ?? new List(); + + DiscoveredExtensions = discoveredExtensions; } /// @@ -53,30 +84,42 @@ public DiscoveryCompleteEventArgs(long totalTests, bool isAborted) /// /// Indicates the total tests which got discovered in this request. /// + [DataMember] public long TotalCount { get; private set; } /// /// Specifies if discovery has been aborted. If true TotalCount is also set to -1. /// + [DataMember] public bool IsAborted { get; private set; } /// /// Metrics /// + [DataMember] public IDictionary Metrics { get; set; } /// /// Gets the list of sources which were fully discovered. /// + [DataMember] public IList FullyDiscoveredSources { get; set; } /// /// Gets the list of sources which were partially discovered (started discover tests, but then discovery aborted). /// + [DataMember] public IList PartiallyDiscoveredSources { get; set; } /// /// Gets the list of sources which were not discovered at all. /// + [DataMember] public IList NotDiscoveredSources { get; set; } + + /// + /// Gets or sets the collection of discovered extensions. + /// + [DataMember] + public Dictionary> DiscoveredExtensions { get; set; } } diff --git a/src/Microsoft.TestPlatform.ObjectModel/Client/Events/TestRunCompleteEventArgs.cs b/src/Microsoft.TestPlatform.ObjectModel/Client/Events/TestRunCompleteEventArgs.cs index b2420a36d5..f287f25e66 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/Client/Events/TestRunCompleteEventArgs.cs +++ b/src/Microsoft.TestPlatform.ObjectModel/Client/Events/TestRunCompleteEventArgs.cs @@ -33,8 +33,21 @@ private TestRunCompleteEventArgs() /// Specifies the error encountered during the execution of the test run. /// Attachment sets associated with the run. /// Time elapsed in just running tests - public TestRunCompleteEventArgs(ITestRunStatistics stats, bool isCanceled, bool isAborted, Exception error, Collection attachmentSets, TimeSpan elapsedTime) - : this(stats, isCanceled, isAborted, error, attachmentSets, null, elapsedTime) + public TestRunCompleteEventArgs( + ITestRunStatistics stats, + bool isCanceled, + bool isAborted, + Exception error, + Collection attachmentSets, + TimeSpan elapsedTime) + : this( + stats, + isCanceled, + isAborted, + error, + attachmentSets, + null, + elapsedTime) { } /// @@ -47,7 +60,14 @@ public TestRunCompleteEventArgs(ITestRunStatistics stats, bool isCanceled, bool /// Attachment sets associated with the run. /// Invoked data collectors /// Time elapsed in just running tests - public TestRunCompleteEventArgs(ITestRunStatistics stats, bool isCanceled, bool isAborted, Exception error, Collection attachmentSets, Collection invokedDataCollectors, TimeSpan elapsedTime) + public TestRunCompleteEventArgs( + ITestRunStatistics stats, + bool isCanceled, + bool isAborted, + Exception error, + Collection attachmentSets, + Collection invokedDataCollectors, + TimeSpan elapsedTime) { TestRunStatistics = stats; IsCanceled = isCanceled; @@ -56,6 +76,8 @@ public TestRunCompleteEventArgs(ITestRunStatistics stats, bool isCanceled, bool AttachmentSets = attachmentSets ?? new Collection(); // Ensuring attachmentSets are not null, so that new attachmentSets can be combined whenever required. InvokedDataCollectors = invokedDataCollectors ?? new Collection(); // Ensuring that invoked data collectors are not null. ElapsedTimeInRunningTests = elapsedTime; + + DiscoveredExtensions = new Dictionary>(); } /// @@ -106,4 +128,10 @@ public TestRunCompleteEventArgs(ITestRunStatistics stats, bool isCanceled, bool /// [DataMember] public IDictionary Metrics { get; set; } + + /// + /// Gets or sets the collection of discovered extensions. + /// + [DataMember] + public Dictionary> DiscoveredExtensions { get; set; } } diff --git a/src/Microsoft.TestPlatform.ObjectModel/PublicAPI/PublicAPI.Unshipped.txt b/src/Microsoft.TestPlatform.ObjectModel/PublicAPI/PublicAPI.Unshipped.txt index da26b150f1..6c64dac8b5 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/Microsoft.TestPlatform.ObjectModel/PublicAPI/PublicAPI.Unshipped.txt @@ -39,3 +39,9 @@ Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.StopTestSessionCompleteEv Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.StopTestSessionCompleteEventArgs.StopTestSessionCompleteEventArgs(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestSessionInfo testSessionInfo) -> void Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.StopTestSessionCompleteEventArgs.TestSessionInfo.get -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestSessionInfo Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.StopTestSessionCompleteEventArgs.TestSessionInfo.set -> void +Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.DiscoveryCompleteEventArgs.DiscoveredExtensions.get -> System.Collections.Generic.Dictionary> +Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.DiscoveryCompleteEventArgs.DiscoveredExtensions.set -> void +Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.DiscoveryCompleteEventArgs.DiscoveryCompleteEventArgs(long totalTests, bool isAborted, System.Collections.Generic.IList fullyDiscoveredSources, System.Collections.Generic.IList partiallyDiscoveredSources, System.Collections.Generic.IList notDiscoveredSources, System.Collections.Generic.Dictionary> discoveredExtensions) -> void +Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestRunCompleteEventArgs.DiscoveredExtensions.get -> System.Collections.Generic.Dictionary> +Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestRunCompleteEventArgs.DiscoveredExtensions.set -> void + diff --git a/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/VsTestConsoleRequestSender.cs b/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/VsTestConsoleRequestSender.cs index d4d2e13172..27074d6cfb 100644 --- a/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/VsTestConsoleRequestSender.cs +++ b/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/VsTestConsoleRequestSender.cs @@ -977,7 +977,8 @@ private void SendMessageAndListenAndReportTestCases( discoveryCompletePayload.IsAborted, discoveryCompletePayload.FullyDiscoveredSources, discoveryCompletePayload.PartiallyDiscoveredSources, - discoveryCompletePayload.NotDiscoveredSources); + discoveryCompletePayload.NotDiscoveredSources, + discoveryCompletePayload.DiscoveredExtensions); // Adding metrics from vstest.console. discoveryCompleteEventArgs.Metrics = discoveryCompletePayload.Metrics; @@ -1067,7 +1068,8 @@ private async Task SendMessageAndListenAndReportTestCasesAsync( discoveryCompletePayload.IsAborted, discoveryCompletePayload.FullyDiscoveredSources, discoveryCompletePayload.PartiallyDiscoveredSources, - discoveryCompletePayload.NotDiscoveredSources); + discoveryCompletePayload.NotDiscoveredSources, + discoveryCompletePayload.DiscoveredExtensions); // Adding Metrics from VsTestConsole discoveryCompleteEventArgs.Metrics = discoveryCompletePayload.Metrics; diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelDiscoveryDataAggregatorTests.cs b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelDiscoveryDataAggregatorTests.cs index d107dcc5b5..a273bbea68 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelDiscoveryDataAggregatorTests.cs +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelDiscoveryDataAggregatorTests.cs @@ -19,13 +19,13 @@ public void AggregateShouldAggregateAbortedCorrectly() { var aggregator = new ParallelDiscoveryDataAggregator(); - aggregator.Aggregate(totalTests: 5, isAborted: false); + aggregator.Aggregate(totalTests: 5, isAborted: false, discoveredExtensions: null); Assert.IsFalse(aggregator.IsAborted, "Aborted must be false"); - aggregator.Aggregate(totalTests: 5, isAborted: true); + aggregator.Aggregate(totalTests: 5, isAborted: true, discoveredExtensions: null); Assert.IsTrue(aggregator.IsAborted, "Aborted must be true"); - aggregator.Aggregate(totalTests: 5, isAborted: false); + aggregator.Aggregate(totalTests: 5, isAborted: false, discoveredExtensions: null); Assert.IsTrue(aggregator.IsAborted, "Aborted must be true"); Assert.AreEqual(-1, aggregator.TotalTests, "Aggregator shouldn't count tests if one host aborts"); @@ -35,13 +35,13 @@ public void AggregateShouldAggregateAbortedCorrectly() public void AggregateShouldAggregateTotalTestsCorrectly() { var aggregator = new ParallelDiscoveryDataAggregator(); - aggregator.Aggregate(totalTests: 2, isAborted: false); + aggregator.Aggregate(totalTests: 2, isAborted: false, discoveredExtensions: null); Assert.AreEqual(2, aggregator.TotalTests, "Aggregated totalTests count does not match"); - aggregator.Aggregate(totalTests: 5, isAborted: false); + aggregator.Aggregate(totalTests: 5, isAborted: false, discoveredExtensions: null); Assert.AreEqual(7, aggregator.TotalTests, "Aggregated totalTests count does not match"); - aggregator.Aggregate(totalTests: 3, isAborted: false); + aggregator.Aggregate(totalTests: 3, isAborted: false, discoveredExtensions: null); Assert.AreEqual(10, aggregator.TotalTests, "Aggregated totalTests count does not match"); } diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelRunDataAggregatorTests.cs b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelRunDataAggregatorTests.cs index 16a35c398b..38e28b51b6 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelRunDataAggregatorTests.cs +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelRunDataAggregatorTests.cs @@ -48,7 +48,7 @@ public void AggregateShouldAggregateRunCompleteAttachmentsCorrectly() new AttachmentSet(new Uri("x://hello1"), "hello1") }; - aggregator.Aggregate(null, null, null, TimeSpan.Zero, false, false, null, attachmentSet1, null); + aggregator.Aggregate(null, null, null, TimeSpan.Zero, false, false, null, attachmentSet1, null, null); Assert.AreEqual(1, aggregator.RunCompleteArgsAttachments.Count, "RunCompleteArgsAttachments List must have data."); @@ -58,7 +58,7 @@ public void AggregateShouldAggregateRunCompleteAttachmentsCorrectly() new AttachmentSet(new Uri("x://hello2"), "hello2") }; - aggregator.Aggregate(null, null, null, TimeSpan.Zero, false, false, null, attachmentSet2, null); + aggregator.Aggregate(null, null, null, TimeSpan.Zero, false, false, null, attachmentSet2, null, null); Assert.AreEqual(2, aggregator.RunCompleteArgsAttachments.Count, "RunCompleteArgsAttachments List must have aggregated data."); } @@ -73,7 +73,7 @@ public void AggregateShouldAggregateRunContextAttachmentsCorrectly() new AttachmentSet(new Uri("x://hello1"), "hello1") }; - aggregator.Aggregate(null, null, null, TimeSpan.Zero, false, false, attachmentSet1, null, null); + aggregator.Aggregate(null, null, null, TimeSpan.Zero, false, false, attachmentSet1, null, null, null); Assert.AreEqual(1, aggregator.RunContextAttachments.Count, "RunContextAttachments List must have data."); @@ -82,7 +82,7 @@ public void AggregateShouldAggregateRunContextAttachmentsCorrectly() new AttachmentSet(new Uri("x://hello2"), "hello2") }; - aggregator.Aggregate(null, null, null, TimeSpan.Zero, false, false, attachmentSet2, null, null); + aggregator.Aggregate(null, null, null, TimeSpan.Zero, false, false, attachmentSet2, null, null, null); Assert.AreEqual(2, aggregator.RunContextAttachments.Count, "RunContextAttachments List must have aggregated data."); } @@ -96,18 +96,18 @@ public void AggregateShouldAggregateInvokedCollectorsCorrectly() { new InvokedDataCollector(new Uri("datacollector://sample"),"sample", typeof(string).AssemblyQualifiedName,typeof(string).Assembly.Location,false) }; - aggregator.Aggregate(null, null, null, TimeSpan.Zero, false, false, null, null, invokedDataCollectors); + aggregator.Aggregate(null, null, null, TimeSpan.Zero, false, false, null, null, invokedDataCollectors, null); Assert.AreEqual(1, aggregator.InvokedDataCollectors.Count, "InvokedDataCollectors List must have data."); var invokedDataCollectors2 = new Collection() { new InvokedDataCollector(new Uri("datacollector://sample2"),"sample2", typeof(int).AssemblyQualifiedName,typeof(int).Assembly.Location,false) }; - aggregator.Aggregate(null, null, null, TimeSpan.Zero, false, false, null, null, invokedDataCollectors2); + aggregator.Aggregate(null, null, null, TimeSpan.Zero, false, false, null, null, invokedDataCollectors2, null); Assert.AreEqual(2, aggregator.InvokedDataCollectors.Count, "InvokedDataCollectors List must have aggregated data."); - aggregator.Aggregate(null, null, null, TimeSpan.Zero, false, false, null, null, invokedDataCollectors); - aggregator.Aggregate(null, null, null, TimeSpan.Zero, false, false, null, null, invokedDataCollectors2); + aggregator.Aggregate(null, null, null, TimeSpan.Zero, false, false, null, null, invokedDataCollectors, null); + aggregator.Aggregate(null, null, null, TimeSpan.Zero, false, false, null, null, invokedDataCollectors2, null); Assert.AreEqual(2, aggregator.InvokedDataCollectors.Count, "InvokedDataCollectors List must have aggregated data."); Assert.AreEqual(invokedDataCollectors[0].AssemblyQualifiedName, aggregator.InvokedDataCollectors[0].AssemblyQualifiedName); @@ -124,28 +124,28 @@ public void AggregateShouldAggregateAbortedAndCanceledCorrectly() var aggregator = new ParallelRunDataAggregator(Constants.EmptyRunSettings); aggregator.Aggregate(null, null, null, TimeSpan.Zero, isAborted: false, isCanceled: false, runContextAttachments: null, - runCompleteArgsAttachments: null, invokedDataCollectors: null); + runCompleteArgsAttachments: null, invokedDataCollectors: null, discoveredExtensions: null); Assert.IsFalse(aggregator.IsAborted, "Aborted must be false"); Assert.IsFalse(aggregator.IsCanceled, "Canceled must be false"); aggregator.Aggregate(null, null, null, TimeSpan.Zero, isAborted: true, isCanceled: false, runContextAttachments: null, - runCompleteArgsAttachments: null, invokedDataCollectors: null); + runCompleteArgsAttachments: null, invokedDataCollectors: null, discoveredExtensions: null); Assert.IsTrue(aggregator.IsAborted, "Aborted must be true"); Assert.IsFalse(aggregator.IsCanceled, "Canceled must still be false"); aggregator.Aggregate(null, null, null, TimeSpan.Zero, isAborted: false, isCanceled: true, runContextAttachments: null, - runCompleteArgsAttachments: null, invokedDataCollectors: null); + runCompleteArgsAttachments: null, invokedDataCollectors: null, discoveredExtensions: null); Assert.IsTrue(aggregator.IsAborted, "Aborted must continue be true"); Assert.IsTrue(aggregator.IsCanceled, "Canceled must be true"); aggregator.Aggregate(null, null, null, TimeSpan.Zero, isAborted: false, isCanceled: false, runContextAttachments: null, - runCompleteArgsAttachments: null, invokedDataCollectors: null); + runCompleteArgsAttachments: null, invokedDataCollectors: null, discoveredExtensions: null); Assert.IsTrue(aggregator.IsAborted, "Aborted must continue be true"); @@ -159,23 +159,23 @@ public void AggregateShouldAggregateTimeSpanCorrectly() var aggregator = new ParallelRunDataAggregator(Constants.EmptyRunSettings); aggregator.Aggregate(null, null, null, TimeSpan.Zero, isAborted: false, isCanceled: false, runContextAttachments: null, - runCompleteArgsAttachments: null, invokedDataCollectors: null); + runCompleteArgsAttachments: null, invokedDataCollectors: null, discoveredExtensions: null); Assert.AreEqual(TimeSpan.Zero, aggregator.ElapsedTime, "Timespan must be zero"); aggregator.Aggregate(null, null, null, TimeSpan.FromMilliseconds(100), isAborted: false, isCanceled: false, runContextAttachments: null, - runCompleteArgsAttachments: null, invokedDataCollectors: null); + runCompleteArgsAttachments: null, invokedDataCollectors: null, discoveredExtensions: null); Assert.AreEqual(TimeSpan.FromMilliseconds(100), aggregator.ElapsedTime, "Timespan must be 100ms"); aggregator.Aggregate(null, null, null, TimeSpan.FromMilliseconds(200), isAborted: false, isCanceled: false, runContextAttachments: null, - runCompleteArgsAttachments: null, invokedDataCollectors: null); + runCompleteArgsAttachments: null, invokedDataCollectors: null, discoveredExtensions: null); Assert.AreEqual(TimeSpan.FromMilliseconds(200), aggregator.ElapsedTime, "Timespan should be Max of all 200ms"); aggregator.Aggregate(null, null, null, TimeSpan.FromMilliseconds(150), isAborted: false, isCanceled: false, runContextAttachments: null, - runCompleteArgsAttachments: null, invokedDataCollectors: null); + runCompleteArgsAttachments: null, invokedDataCollectors: null, discoveredExtensions: null); Assert.AreEqual(TimeSpan.FromMilliseconds(200), aggregator.ElapsedTime, "Timespan should be Max of all i.e. 200ms"); } @@ -186,13 +186,13 @@ public void AggregateShouldAggregateExceptionsCorrectly() var aggregator = new ParallelRunDataAggregator(Constants.EmptyRunSettings); aggregator.Aggregate(null, null, exception: null, elapsedTime: TimeSpan.Zero, isAborted: false, isCanceled: false, runContextAttachments: null, - runCompleteArgsAttachments: null, invokedDataCollectors: null); + runCompleteArgsAttachments: null, invokedDataCollectors: null, discoveredExtensions: null); Assert.IsNull(aggregator.GetAggregatedException(), "Aggregated exception must be null"); var exception1 = new NotImplementedException(); aggregator.Aggregate(null, null, exception: exception1, elapsedTime: TimeSpan.Zero, isAborted: false, isCanceled: false, runContextAttachments: null, - runCompleteArgsAttachments: null, invokedDataCollectors: null); + runCompleteArgsAttachments: null, invokedDataCollectors: null, discoveredExtensions: null); var aggregatedException = aggregator.GetAggregatedException() as AggregateException; Assert.IsNotNull(aggregatedException, "Aggregated exception must NOT be null"); @@ -202,7 +202,7 @@ public void AggregateShouldAggregateExceptionsCorrectly() var exception2 = new NotSupportedException(); aggregator.Aggregate(null, null, exception: exception2, elapsedTime: TimeSpan.Zero, isAborted: false, isCanceled: false, runContextAttachments: null, - runCompleteArgsAttachments: null, invokedDataCollectors: null); + runCompleteArgsAttachments: null, invokedDataCollectors: null, discoveredExtensions: null); aggregatedException = aggregator.GetAggregatedException() as AggregateException; Assert.IsNotNull(aggregatedException, "Aggregated exception must NOT be null"); @@ -216,18 +216,18 @@ public void AggregateShouldAggregateExecutorUrisCorrectly() { var aggregator = new ParallelRunDataAggregator(Constants.EmptyRunSettings); - aggregator.Aggregate(null, null, null, TimeSpan.Zero, false, false, null, null, null); + aggregator.Aggregate(null, null, null, TimeSpan.Zero, false, false, null, null, null, null); Assert.AreEqual(0, aggregator.ExecutorUris.Count, "ExecutorUris List must not have data."); var uri1 = "x://hello1"; - aggregator.Aggregate(null, new List() { uri1 }, null, TimeSpan.Zero, false, false, null, null, null); + aggregator.Aggregate(null, new List() { uri1 }, null, TimeSpan.Zero, false, false, null, null, null, null); Assert.AreEqual(1, aggregator.ExecutorUris.Count, "ExecutorUris List must have data."); Assert.IsTrue(aggregator.ExecutorUris.Contains(uri1), "ExecutorUris List must have correct data."); var uri2 = "x://hello2"; - aggregator.Aggregate(null, new List() { uri2 }, null, TimeSpan.Zero, false, false, null, null, null); + aggregator.Aggregate(null, new List() { uri2 }, null, TimeSpan.Zero, false, false, null, null, null, null); Assert.AreEqual(2, aggregator.ExecutorUris.Count, "ExecutorUris List must have aggregated data."); Assert.IsTrue(aggregator.ExecutorUris.Contains(uri2), "ExecutorUris List must have correct data."); @@ -238,7 +238,7 @@ public void AggregateShouldAggregateRunStatsCorrectly() { var aggregator = new ParallelRunDataAggregator(Constants.EmptyRunSettings); - aggregator.Aggregate(null, null, null, TimeSpan.Zero, false, false, null, null, null); + aggregator.Aggregate(null, null, null, TimeSpan.Zero, false, false, null, null, null, null); var runStats = aggregator.GetAggregatedRunStats(); Assert.AreEqual(0, runStats.ExecutedTests, "RunStats must not have data."); @@ -252,7 +252,7 @@ public void AggregateShouldAggregateRunStatsCorrectly() { TestOutcome.None, 2 } }; - aggregator.Aggregate(new TestRunStatistics(12, stats1), null, null, TimeSpan.Zero, false, false, null, null, null); + aggregator.Aggregate(new TestRunStatistics(12, stats1), null, null, TimeSpan.Zero, false, false, null, null, null, null); runStats = aggregator.GetAggregatedRunStats(); Assert.AreEqual(12, runStats.ExecutedTests, "RunStats must have aggregated data."); @@ -272,7 +272,7 @@ public void AggregateShouldAggregateRunStatsCorrectly() { TestOutcome.None, 3 } }; - aggregator.Aggregate(new TestRunStatistics(11, stats2), null, null, TimeSpan.Zero, false, false, null, null, null); + aggregator.Aggregate(new TestRunStatistics(11, stats2), null, null, TimeSpan.Zero, false, false, null, null, null, null); runStats = aggregator.GetAggregatedRunStats(); Assert.AreEqual(23, runStats.ExecutedTests, "RunStats must have aggregated data.");