diff --git a/tracer/src/Datadog.Trace/Ci/Agent/Payloads/EventPlatformPayload.cs b/tracer/src/Datadog.Trace/Ci/Agent/Payloads/EventPlatformPayload.cs index b7b0305c052f..e251e9ba4982 100644 --- a/tracer/src/Datadog.Trace/Ci/Agent/Payloads/EventPlatformPayload.cs +++ b/tracer/src/Datadog.Trace/Ci/Agent/Payloads/EventPlatformPayload.cs @@ -4,6 +4,7 @@ // using System; +using Datadog.Trace.Agent; using Datadog.Trace.Ci.Configuration; using Datadog.Trace.Telemetry.Metrics; using Datadog.Trace.Util; @@ -142,7 +143,18 @@ private void EnsureUrl() else { // Use Agent EVP Proxy - builder = new UriBuilder(_settings.TracerSettings.ExporterInternal.AgentUriInternal); + switch (_settings.TracerSettings.ExporterInternal.TracesTransport) + { + case TracesTransportType.WindowsNamedPipe: + case TracesTransportType.UnixDomainSocket: + builder = new UriBuilder("http://localhost"); + break; + case TracesTransportType.Default: + default: + builder = new UriBuilder(_settings.TracerSettings.ExporterInternal.AgentUriInternal); + break; + } + if (CIVisibility.EventPlatformProxySupport == EventPlatformProxySupport.V4) { builder.Path = $"/evp_proxy/v4/{EventPlatformPath}"; diff --git a/tracer/src/Datadog.Trace/Ci/CIVisibility.cs b/tracer/src/Datadog.Trace/Ci/CIVisibility.cs index 76d71e1d3e05..e62532a02263 100644 --- a/tracer/src/Datadog.Trace/Ci/CIVisibility.cs +++ b/tracer/src/Datadog.Trace/Ci/CIVisibility.cs @@ -12,10 +12,12 @@ using System.Threading.Tasks; using Datadog.Trace.Agent; using Datadog.Trace.Agent.DiscoveryService; +using Datadog.Trace.Agent.StreamFactories; using Datadog.Trace.Agent.Transports; using Datadog.Trace.Ci.CiEnvironment; using Datadog.Trace.Ci.Configuration; using Datadog.Trace.Configuration; +using Datadog.Trace.HttpOverStreams; using Datadog.Trace.Logging; using Datadog.Trace.Pdb; using Datadog.Trace.PlatformHelpers; @@ -425,48 +427,57 @@ internal static IApiRequestFactory GetRequestFactory(ImmutableTracerSettings set internal static IApiRequestFactory GetRequestFactory(ImmutableTracerSettings tracerSettings, TimeSpan timeout) { IApiRequestFactory? factory = null; - + var exporterSettings = tracerSettings.ExporterInternal; + if (exporterSettings.TracesTransport != TracesTransportType.Default) + { + factory = AgentTransportStrategy.Get( + exporterSettings, + productName: "CI Visibility", + tcpTimeout: null, + AgentHttpHeaderNames.DefaultHeaders, + () => new TraceAgentHttpHeaderHelper(), + uri => uri); + } + else + { #if NETCOREAPP - Log.Information("Using {FactoryType} for trace transport.", nameof(HttpClientRequestFactory)); - factory = new HttpClientRequestFactory( - tracerSettings.ExporterInternal.AgentUriInternal, - AgentHttpHeaderNames.DefaultHeaders, - handler: new System.Net.Http.HttpClientHandler - { - AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate, - }, - timeout: timeout); + Log.Information("Using {FactoryType} for trace transport.", nameof(HttpClientRequestFactory)); + factory = new HttpClientRequestFactory( + exporterSettings.AgentUriInternal, + AgentHttpHeaderNames.DefaultHeaders, + handler: new System.Net.Http.HttpClientHandler { AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate, }, + timeout: timeout); #else - Log.Information("Using {FactoryType} for trace transport.", nameof(ApiWebRequestFactory)); - factory = new ApiWebRequestFactory(tracerSettings.ExporterInternal.AgentUriInternal, AgentHttpHeaderNames.DefaultHeaders, timeout: timeout); + Log.Information("Using {FactoryType} for trace transport.", nameof(ApiWebRequestFactory)); + factory = new ApiWebRequestFactory(tracerSettings.ExporterInternal.AgentUriInternal, AgentHttpHeaderNames.DefaultHeaders, timeout: timeout); #endif + var settings = Settings; + if (!string.IsNullOrWhiteSpace(settings.ProxyHttps)) + { + var proxyHttpsUriBuilder = new UriBuilder(settings.ProxyHttps); - var settings = Settings; - if (!string.IsNullOrWhiteSpace(settings.ProxyHttps)) - { - var proxyHttpsUriBuilder = new UriBuilder(settings.ProxyHttps); + var userName = proxyHttpsUriBuilder.UserName; + var password = proxyHttpsUriBuilder.Password; - var userName = proxyHttpsUriBuilder.UserName; - var password = proxyHttpsUriBuilder.Password; + proxyHttpsUriBuilder.UserName = string.Empty; + proxyHttpsUriBuilder.Password = string.Empty; - proxyHttpsUriBuilder.UserName = string.Empty; - proxyHttpsUriBuilder.Password = string.Empty; + if (proxyHttpsUriBuilder.Scheme == "https") + { + // HTTPS proxy is not supported by .NET BCL + Log.Error("HTTPS proxy is not supported. ({ProxyHttpsUriBuilder})", proxyHttpsUriBuilder); + return factory; + } - if (proxyHttpsUriBuilder.Scheme == "https") - { - // HTTPS proxy is not supported by .NET BCL - Log.Error("HTTPS proxy is not supported. ({ProxyHttpsUriBuilder})", proxyHttpsUriBuilder); - return factory; - } + NetworkCredential? credential = null; + if (!string.IsNullOrWhiteSpace(userName)) + { + credential = new NetworkCredential(userName, password); + } - NetworkCredential? credential = null; - if (!string.IsNullOrWhiteSpace(userName)) - { - credential = new NetworkCredential(userName, password); + Log.Information("Setting proxy to: {ProxyHttps}", proxyHttpsUriBuilder.Uri.ToString()); + factory.SetProxy(new WebProxy(proxyHttpsUriBuilder.Uri, true, settings.ProxyNoProxy, credential), credential); } - - Log.Information("Setting proxy to: {ProxyHttps}", proxyHttpsUriBuilder.Uri.ToString()); - factory.SetProxy(new WebProxy(proxyHttpsUriBuilder.Uri, true, settings.ProxyNoProxy, credential), credential); } return factory; @@ -608,7 +619,8 @@ private static bool InternalEnabled() Environment.CommandLine.IndexOf("dotnet.dll\" test", StringComparison.OrdinalIgnoreCase) == -1 && Environment.CommandLine.IndexOf("dotnet.dll' test", StringComparison.OrdinalIgnoreCase) == -1 && Environment.CommandLine.IndexOf(" test ", StringComparison.OrdinalIgnoreCase) == -1 && - Environment.CommandLine.IndexOf("datacollector", StringComparison.OrdinalIgnoreCase) == -1) + Environment.CommandLine.IndexOf("datacollector", StringComparison.OrdinalIgnoreCase) == -1 && + Environment.CommandLine.IndexOf("vstest.console.dll", StringComparison.OrdinalIgnoreCase) == -1) { Log.Information("CI Visibility disabled because the process name is 'dotnet' but the commandline doesn't contain 'testhost.dll': {Cmdline}", Environment.CommandLine); return false; diff --git a/tracer/src/Datadog.Trace/Ci/IntelligentTestRunnerClient.cs b/tracer/src/Datadog.Trace/Ci/IntelligentTestRunnerClient.cs index 639775feccce..9452e9c2bcd4 100644 --- a/tracer/src/Datadog.Trace/Ci/IntelligentTestRunnerClient.cs +++ b/tracer/src/Datadog.Trace/Ci/IntelligentTestRunnerClient.cs @@ -143,23 +143,22 @@ public IntelligentTestRunnerClient(string? workingDirectory, CIVisibilitySetting else { // Use Agent EVP Proxy - var agentUrl = _settings.TracerSettings.ExporterInternal.AgentUriInternal; _eventPlatformProxySupport = CIVisibility.EventPlatformProxySupport; switch (_eventPlatformProxySupport) { case EventPlatformProxySupport.V2: - _settingsUrl = UriHelpers.Combine(agentUrl, $"evp_proxy/v2/{settingsUrlPath}"); - _searchCommitsUrl = UriHelpers.Combine(agentUrl, $"evp_proxy/v2/{searchCommitsUrlPath}"); - _packFileUrl = UriHelpers.Combine(agentUrl, $"evp_proxy/v2/{packFileUrlPath}"); - _skippableTestsUrl = UriHelpers.Combine(agentUrl, $"evp_proxy/v2/{skippableTestsUrlPath}"); - _earlyFlakeDetectionTestsUrl = UriHelpers.Combine(agentUrl, $"evp_proxy/v2/{efdTestsUrlPath}"); + _settingsUrl = _apiRequestFactory.GetEndpoint($"evp_proxy/v2/{settingsUrlPath}"); + _searchCommitsUrl = _apiRequestFactory.GetEndpoint($"evp_proxy/v2/{searchCommitsUrlPath}"); + _packFileUrl = _apiRequestFactory.GetEndpoint($"evp_proxy/v2/{packFileUrlPath}"); + _skippableTestsUrl = _apiRequestFactory.GetEndpoint($"evp_proxy/v2/{skippableTestsUrlPath}"); + _earlyFlakeDetectionTestsUrl = _apiRequestFactory.GetEndpoint($"evp_proxy/v2/{efdTestsUrlPath}"); break; case EventPlatformProxySupport.V4: - _settingsUrl = UriHelpers.Combine(agentUrl, $"evp_proxy/v4/{settingsUrlPath}"); - _searchCommitsUrl = UriHelpers.Combine(agentUrl, $"evp_proxy/v4/{searchCommitsUrlPath}"); - _packFileUrl = UriHelpers.Combine(agentUrl, $"evp_proxy/v4/{packFileUrlPath}"); - _skippableTestsUrl = UriHelpers.Combine(agentUrl, $"evp_proxy/v4/{skippableTestsUrlPath}"); - _earlyFlakeDetectionTestsUrl = UriHelpers.Combine(agentUrl, $"evp_proxy/v4/{efdTestsUrlPath}"); + _settingsUrl = _apiRequestFactory.GetEndpoint($"evp_proxy/v4/{settingsUrlPath}"); + _searchCommitsUrl = _apiRequestFactory.GetEndpoint($"evp_proxy/v4/{searchCommitsUrlPath}"); + _packFileUrl = _apiRequestFactory.GetEndpoint($"evp_proxy/v4/{packFileUrlPath}"); + _skippableTestsUrl = _apiRequestFactory.GetEndpoint($"evp_proxy/v4/{skippableTestsUrlPath}"); + _earlyFlakeDetectionTestsUrl = _apiRequestFactory.GetEndpoint($"evp_proxy/v4/{efdTestsUrlPath}"); break; default: throw new NotSupportedException("Event platform proxy not supported by the agent."); diff --git a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Testing/DotnetTest/DotnetCommon.cs b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Testing/DotnetTest/DotnetCommon.cs index d21a660f1cfe..fd5cc529d420 100644 --- a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Testing/DotnetTest/DotnetCommon.cs +++ b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Testing/DotnetTest/DotnetCommon.cs @@ -15,6 +15,7 @@ using Datadog.Trace.Configuration; using Datadog.Trace.ExtensionMethods; using Datadog.Trace.Logging; +using Datadog.Trace.Propagators; using Datadog.Trace.Util; using Datadog.Trace.Vendors.Serilog.Events; @@ -51,6 +52,16 @@ internal static bool IsDataCollectorDomain return null; } + // Let's detect if we already have a session for this test process + if (SpanContextPropagator.Instance.Extract( + EnvironmentHelpers.GetEnvironmentVariables(), + new DictionaryGetterAndSetter(DictionaryGetterAndSetter.EnvironmentVariableKeyProcessor)) is not null) + { + // Session found in the environment variables + // let's bail-out + return null; + } + var ciVisibilitySettings = CIVisibility.Settings; var agentless = ciVisibilitySettings.Agentless; var isEvpProxy = CIVisibility.EventPlatformProxySupport != EventPlatformProxySupport.None; diff --git a/tracer/src/Datadog.Tracer.Native/cor_profiler.cpp b/tracer/src/Datadog.Tracer.Native/cor_profiler.cpp index 107fbe01d283..74ce33ac0582 100644 --- a/tracer/src/Datadog.Tracer.Native/cor_profiler.cpp +++ b/tracer/src/Datadog.Tracer.Native/cor_profiler.cpp @@ -90,7 +90,8 @@ HRESULT STDMETHODCALLTYPE CorProfiler::Initialize(IUnknown* cor_profiler_info_un // these are executed with exec, so we could check for that, but the // below check is more conservative, so leaving at that process_command_line.find(WStr("testhost")) == WSTRING::npos && - process_command_line.find(WStr("datacollector")) == WSTRING::npos) + process_command_line.find(WStr("datacollector")) == WSTRING::npos && + process_command_line.find(WStr("vstest.console.dll")) == WSTRING::npos) { Logger::Info("The Tracer Profiler has been disabled because the process is running in CI Visibility " "mode, the name is 'dotnet' but the commandline doesn't contain 'testhost'"); diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/PipesXUnitEvpTests.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/PipesXUnitEvpTests.cs new file mode 100644 index 000000000000..344a94afe670 --- /dev/null +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/PipesXUnitEvpTests.cs @@ -0,0 +1,75 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// +#if NETCOREAPP3_1_OR_GREATER +using System; +using System.Threading.Tasks; +using Datadog.Trace.Configuration; +using Datadog.Trace.TestHelpers; +using Xunit; +using Xunit.Abstractions; + +namespace Datadog.Trace.ClrProfiler.IntegrationTests.CI; + +[Collection(nameof(TransportTestsCollection))] +public class PipesXUnitEvpTests(ITestOutputHelper output) : XUnitEvpTests(output) +{ + [SkippableTheory] + [MemberData(nameof(GetData))] + [Trait("RunOnWindows", "True")] + [Trait("Category", "EndToEnd")] + [Trait("Category", "TestIntegrations")] + public override async Task SubmitTraces(string packageVersion, string evpVersionToRemove, bool expectedGzip) + { + SkipOn.Platform(SkipOn.PlatformValue.MacOs); + SkipOn.Platform(SkipOn.PlatformValue.Linux); + EnvironmentHelper.EnableWindowsNamedPipes(); + + // The server implementation of named pipes is flaky so have 5 attempts + var attemptsRemaining = 5; + while (true) + { + try + { + attemptsRemaining--; + await base.SubmitTraces(packageVersion, evpVersionToRemove, expectedGzip); + return; + } + catch (Exception ex) when (attemptsRemaining > 0 && ex is not SkipException) + { + await ReportRetry(Output, attemptsRemaining, ex); + } + } + } + + [SkippableTheory] + [MemberData(nameof(GetDataForEarlyFlakeDetection))] + [Trait("RunOnWindows", "True")] + [Trait("Category", "EndToEnd")] + [Trait("Category", "TestIntegrations")] + [Trait("Category", "EarlyFlakeDetection")] + public override async Task EarlyFlakeDetection(string packageVersion, string evpVersionToRemove, bool expectedGzip, string settingsJson, string testsJson, int expectedSpans, string friendlyName) + { + SkipOn.Platform(SkipOn.PlatformValue.MacOs); + SkipOn.Platform(SkipOn.PlatformValue.Linux); + EnvironmentHelper.EnableWindowsNamedPipes(); + + // The server implementation of named pipes is flaky so have 5 attempts + var attemptsRemaining = 5; + while (true) + { + try + { + attemptsRemaining--; + await base.EarlyFlakeDetection(packageVersion, evpVersionToRemove, expectedGzip, settingsJson, testsJson, expectedSpans, friendlyName); + return; + } + catch (Exception ex) when (attemptsRemaining > 0 && ex is not SkipException) + { + await ReportRetry(Output, attemptsRemaining, ex); + } + } + } +} +#endif diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/PipesXUnitTests.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/PipesXUnitTests.cs new file mode 100644 index 000000000000..80476281756e --- /dev/null +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/PipesXUnitTests.cs @@ -0,0 +1,48 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#if NETCOREAPP3_1_OR_GREATER +using System; +using System.Threading.Tasks; +using Datadog.Trace.Configuration; +using Datadog.Trace.TestHelpers; +using Xunit; +using Xunit.Abstractions; + +namespace Datadog.Trace.ClrProfiler.IntegrationTests.CI; + +[Collection(nameof(TransportTestsCollection))] +public class PipesXUnitTests(ITestOutputHelper output) : XUnitTests(output) +{ + [SkippableTheory] + [MemberData(nameof(PackageVersions.XUnit), MemberType = typeof(PackageVersions))] + [Trait("RunOnWindows", "True")] + [Trait("Category", "EndToEnd")] + [Trait("Category", "TestIntegrations")] + public override async Task SubmitTraces(string packageVersion) + { + SkipOn.Platform(SkipOn.PlatformValue.MacOs); + SkipOn.Platform(SkipOn.PlatformValue.Linux); + EnvironmentHelper.EnableWindowsNamedPipes(); + + // The server implementation of named pipes is flaky so have 5 attempts + var attemptsRemaining = 5; + while (true) + { + try + { + attemptsRemaining--; + await base.SubmitTraces(packageVersion); + return; + } + catch (Exception ex) when (attemptsRemaining > 0 && ex is not SkipException) + { + await ReportRetry(Output, attemptsRemaining, ex); + } + } + } +} + +#endif diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/TcpXUnitEvpTests.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/TcpXUnitEvpTests.cs new file mode 100644 index 000000000000..d552987bbefc --- /dev/null +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/TcpXUnitEvpTests.cs @@ -0,0 +1,36 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// +#if NETCOREAPP3_1_OR_GREATER +using System.Threading.Tasks; +using Xunit; +using Xunit.Abstractions; + +namespace Datadog.Trace.ClrProfiler.IntegrationTests.CI; + +[Collection(nameof(TransportTestsCollection))] +public class TcpXUnitEvpTests(ITestOutputHelper output) : XUnitEvpTests(output) +{ + [SkippableTheory] + [MemberData(nameof(GetData))] + [Trait("Category", "EndToEnd")] + [Trait("Category", "TestIntegrations")] + public override Task SubmitTraces(string packageVersion, string evpVersionToRemove, bool expectedGzip) + { + EnvironmentHelper.EnableDefaultTransport(); + return base.SubmitTraces(packageVersion, evpVersionToRemove, expectedGzip); + } + + [SkippableTheory] + [MemberData(nameof(GetDataForEarlyFlakeDetection))] + [Trait("Category", "EndToEnd")] + [Trait("Category", "TestIntegrations")] + [Trait("Category", "EarlyFlakeDetection")] + public override Task EarlyFlakeDetection(string packageVersion, string evpVersionToRemove, bool expectedGzip, string settingsJson, string testsJson, int expectedSpans, string friendlyName) + { + EnvironmentHelper.EnableDefaultTransport(); + return base.EarlyFlakeDetection(packageVersion, evpVersionToRemove, expectedGzip, settingsJson, testsJson, expectedSpans, friendlyName); + } +} +#endif diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/TcpXUnitTests.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/TcpXUnitTests.cs new file mode 100644 index 000000000000..33b0676363f6 --- /dev/null +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/TcpXUnitTests.cs @@ -0,0 +1,27 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#if NETCOREAPP3_1_OR_GREATER +using System.Threading.Tasks; +using Xunit; +using Xunit.Abstractions; + +namespace Datadog.Trace.ClrProfiler.IntegrationTests.CI; + +[Collection(nameof(TransportTestsCollection))] +public class TcpXUnitTests(ITestOutputHelper output) : XUnitTests(output) +{ + [SkippableTheory] + [MemberData(nameof(PackageVersions.XUnit), MemberType = typeof(PackageVersions))] + [Trait("Category", "EndToEnd")] + [Trait("Category", "TestIntegrations")] + public override Task SubmitTraces(string packageVersion) + { + EnvironmentHelper.EnableDefaultTransport(); + return base.SubmitTraces(packageVersion); + } +} + +#endif diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/TestingFrameworkEvpTest.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/TestingFrameworkEvpTest.cs index 95552b76c52e..ac7f75fba56a 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/TestingFrameworkEvpTest.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/TestingFrameworkEvpTest.cs @@ -19,32 +19,47 @@ namespace Datadog.Trace.ClrProfiler.IntegrationTests.CI; public abstract class TestingFrameworkEvpTest : TestHelper { + private readonly GacFixture _gacFixture; + protected TestingFrameworkEvpTest(string sampleAppName, string samplePathOverrides, ITestOutputHelper output) : base(sampleAppName, samplePathOverrides, output) { SetCIEnvironmentValues(); + _gacFixture = new GacFixture(); + _gacFixture.AddAssembliesToGac(); } protected TestingFrameworkEvpTest(string sampleAppName, string samplePathOverrides, ITestOutputHelper output, bool prependSamplesToAppName) : base(sampleAppName, samplePathOverrides, output, prependSamplesToAppName) { SetCIEnvironmentValues(); + _gacFixture = new GacFixture(); + _gacFixture.AddAssembliesToGac(); } protected TestingFrameworkEvpTest(string sampleAppName, ITestOutputHelper output) : base(sampleAppName, output) { SetCIEnvironmentValues(); + _gacFixture = new GacFixture(); + _gacFixture.AddAssembliesToGac(); } protected TestingFrameworkEvpTest(EnvironmentHelper environmentHelper, ITestOutputHelper output) : base(environmentHelper, output) { SetCIEnvironmentValues(); + _gacFixture = new GacFixture(); + _gacFixture.AddAssembliesToGac(); } protected object? CIValues { get; private set; } + public override void Dispose() + { + _gacFixture.RemoveAssembliesFromGac(); + } + protected virtual void WriteSpans(List? tests) { if (tests is null || tests.Count == 0) diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/TestingFrameworkTest.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/TestingFrameworkTest.cs index 9d20619a71b4..45c04c06b1d3 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/TestingFrameworkTest.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/TestingFrameworkTest.cs @@ -19,32 +19,47 @@ namespace Datadog.Trace.ClrProfiler.IntegrationTests.CI; public abstract class TestingFrameworkTest : TestHelper { + private readonly GacFixture _gacFixture; + protected TestingFrameworkTest(string sampleAppName, string samplePathOverrides, ITestOutputHelper output) : base(sampleAppName, samplePathOverrides, output) { SetCIEnvironmentValues(); + _gacFixture = new GacFixture(); + _gacFixture.AddAssembliesToGac(); } protected TestingFrameworkTest(string sampleAppName, string samplePathOverrides, ITestOutputHelper output, bool prependSamplesToAppName) : base(sampleAppName, samplePathOverrides, output, prependSamplesToAppName) { SetCIEnvironmentValues(); + _gacFixture = new GacFixture(); + _gacFixture.AddAssembliesToGac(); } protected TestingFrameworkTest(string sampleAppName, ITestOutputHelper output) : base(sampleAppName, output) { SetCIEnvironmentValues(); + _gacFixture = new GacFixture(); + _gacFixture.AddAssembliesToGac(); } protected TestingFrameworkTest(EnvironmentHelper environmentHelper, ITestOutputHelper output) : base(environmentHelper, output) { SetCIEnvironmentValues(); + _gacFixture = new GacFixture(); + _gacFixture.AddAssembliesToGac(); } protected object? CIValues { get; private set; } + public override void Dispose() + { + _gacFixture.RemoveAssembliesFromGac(); + } + protected virtual void WriteSpans(List? spans) { if (spans is null || spans.Count == 0) diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/TransportTestsCollection.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/TransportTestsCollection.cs new file mode 100644 index 000000000000..d32c5fe8b711 --- /dev/null +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/TransportTestsCollection.cs @@ -0,0 +1,15 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// +#if NETCOREAPP3_1_OR_GREATER +using Xunit; + +namespace Datadog.Trace.ClrProfiler.IntegrationTests.CI; + +[CollectionDefinition(nameof(TransportTestsCollection), DisableParallelization = true)] +public class TransportTestsCollection +{ +} + +#endif diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/UdsXUnitEvpTests.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/UdsXUnitEvpTests.cs new file mode 100644 index 000000000000..d27cc9a9086e --- /dev/null +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/UdsXUnitEvpTests.cs @@ -0,0 +1,36 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// +#if NETCOREAPP3_1_OR_GREATER +using System.Threading.Tasks; +using Xunit; +using Xunit.Abstractions; + +namespace Datadog.Trace.ClrProfiler.IntegrationTests.CI; + +[Collection(nameof(TransportTestsCollection))] +public class UdsXUnitEvpTests(ITestOutputHelper output) : XUnitEvpTests(output) +{ + [SkippableTheory] + [MemberData(nameof(GetData))] + [Trait("Category", "EndToEnd")] + [Trait("Category", "TestIntegrations")] + public override Task SubmitTraces(string packageVersion, string evpVersionToRemove, bool expectedGzip) + { + EnvironmentHelper.EnableUnixDomainSockets(); + return base.SubmitTraces(packageVersion, evpVersionToRemove, expectedGzip); + } + + [SkippableTheory] + [MemberData(nameof(GetDataForEarlyFlakeDetection))] + [Trait("Category", "EndToEnd")] + [Trait("Category", "TestIntegrations")] + [Trait("Category", "EarlyFlakeDetection")] + public override Task EarlyFlakeDetection(string packageVersion, string evpVersionToRemove, bool expectedGzip, string settingsJson, string testsJson, int expectedSpans, string friendlyName) + { + EnvironmentHelper.EnableUnixDomainSockets(); + return base.EarlyFlakeDetection(packageVersion, evpVersionToRemove, expectedGzip, settingsJson, testsJson, expectedSpans, friendlyName); + } +} +#endif diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/UdsXUnitTests.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/UdsXUnitTests.cs new file mode 100644 index 000000000000..d09cd91cb5e6 --- /dev/null +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/UdsXUnitTests.cs @@ -0,0 +1,26 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// +#if NETCOREAPP3_1_OR_GREATER +using System.Threading.Tasks; +using Xunit; +using Xunit.Abstractions; + +namespace Datadog.Trace.ClrProfiler.IntegrationTests.CI; + +[Collection(nameof(TransportTestsCollection))] +public class UdsXUnitTests(ITestOutputHelper output) : XUnitTests(output) +{ + [SkippableTheory] + [MemberData(nameof(PackageVersions.XUnit), MemberType = typeof(PackageVersions))] + [Trait("Category", "EndToEnd")] + [Trait("Category", "TestIntegrations")] + public override Task SubmitTraces(string packageVersion) + { + EnvironmentHelper.EnableUnixDomainSockets(); + return base.SubmitTraces(packageVersion); + } +} + +#endif diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/XUnitEvpTests.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/XUnitEvpTests.cs index d717a979b767..0bde4aaa3ae4 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/XUnitEvpTests.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/XUnitEvpTests.cs @@ -28,7 +28,7 @@ namespace Datadog.Trace.ClrProfiler.IntegrationTests.CI; [UsesVerify] -public class XUnitEvpTests : TestingFrameworkEvpTest +public abstract class XUnitEvpTests : TestingFrameworkEvpTest { private const string TestBundleName = "Samples.XUnitTests"; private const string TestSuiteName = "Samples.XUnitTests.TestSuite"; @@ -74,13 +74,10 @@ public static IEnumerable GetDataForEarlyFlakeDetection() } } - [SkippableTheory] - [MemberData(nameof(GetData))] - [Trait("Category", "EndToEnd")] - [Trait("Category", "TestIntegrations")] - public async Task SubmitTraces(string packageVersion, string evpVersionToRemove, bool expectedGzip) + public virtual async Task SubmitTraces(string packageVersion, string evpVersionToRemove, bool expectedGzip) { var tests = new List(); + var testsCopy = new List(); var testSuites = new List(); var testModules = new List(); @@ -110,265 +107,248 @@ public async Task SubmitTraces(string packageVersion, string evpVersionToRemove, }); string[] messages = null; - try - { - SetEnvironmentVariable(ConfigurationKeys.CIVisibility.Enabled, "1"); - using var logsIntake = new MockLogsIntakeForCiVisibility(); - EnableDirectLogSubmission(logsIntake.Port, nameof(IntegrationId.XUnit), nameof(XUnitTests)); - SetEnvironmentVariable(ConfigurationKeys.CIVisibility.Logs, "1"); + SetEnvironmentVariable(ConfigurationKeys.CIVisibility.Enabled, "1"); + + using var logsIntake = new MockLogsIntakeForCiVisibility(); + EnableDirectLogSubmission(logsIntake.Port, nameof(IntegrationId.XUnit), nameof(XUnitTests)); + SetEnvironmentVariable(ConfigurationKeys.CIVisibility.Logs, "1"); - using (var agent = EnvironmentHelper.GetMockAgent()) + using var agent = EnvironmentHelper.GetMockAgent(); + agent.Configuration.Endpoints = agent.Configuration.Endpoints.Where(e => !e.Contains(evpVersionToRemove)).ToArray(); + + const string correlationId = "2e8a36bda770b683345957cc6c15baf9"; + agent.EventPlatformProxyPayloadReceived += (sender, e) => + { + if (e.Value.PathAndQuery.EndsWith("api/v2/libraries/tests/services/setting")) { - agent.Configuration.Endpoints = agent.Configuration.Endpoints.Where(e => !e.Contains(evpVersionToRemove)).ToArray(); + e.Value.Response = new MockTracerResponse("""{"data":{"id":"b5a855bffe6c0b2ae5d150fb6ad674363464c816","type":"ci_app_tracers_test_service_settings","attributes":{"code_coverage":false,"efd_enabled":false,"flaky_test_retries_enabled":false,"itr_enabled":true,"require_git":false,"tests_skipping":true}}} """, 200); + return; + } - const string correlationId = "2e8a36bda770b683345957cc6c15baf9"; - agent.EventPlatformProxyPayloadReceived += (sender, e) => - { - if (e.Value.PathAndQuery.EndsWith("api/v2/libraries/tests/services/setting")) - { - e.Value.Response = new MockTracerResponse("""{"data":{"id":"b5a855bffe6c0b2ae5d150fb6ad674363464c816","type":"ci_app_tracers_test_service_settings","attributes":{"code_coverage":false,"efd_enabled":false,"flaky_test_retries_enabled":false,"itr_enabled":true,"require_git":false,"tests_skipping":true}}} """, 200); - return; - } + if (e.Value.PathAndQuery.EndsWith("api/v2/ci/tests/skippable")) + { + e.Value.Response = new MockTracerResponse($"{{\"data\":[],\"meta\":{{\"correlation_id\":\"{correlationId}\"}}}}", 200); + return; + } - if (e.Value.PathAndQuery.EndsWith("api/v2/ci/tests/skippable")) - { - e.Value.Response = new MockTracerResponse($"{{\"data\":[],\"meta\":{{\"correlation_id\":\"{correlationId}\"}}}}", 200); - return; - } + if (e.Value.PathAndQuery.EndsWith("api/v2/citestcycle")) + { + e.Value.Headers["Content-Encoding"].Should().Be(expectedGzip ? "gzip" : null); - if (e.Value.PathAndQuery.EndsWith("api/v2/citestcycle")) + var payload = JsonConvert.DeserializeObject(e.Value.BodyInJson); + if (payload.Events?.Length > 0) + { + foreach (var @event in payload.Events) { - e.Value.Headers["Content-Encoding"].Should().Be(expectedGzip ? "gzip" : null); - - var payload = JsonConvert.DeserializeObject(e.Value.BodyInJson); - if (payload.Events?.Length > 0) + if (@event.Content.ToString() is { } eventContent) { - foreach (var @event in payload.Events) + if (@event.Type == SpanTypes.Test) { - if (@event.Content.ToString() is { } eventContent) - { - if (@event.Type == SpanTypes.Test) - { - tests.Add(JsonConvert.DeserializeObject(eventContent)); - } - else if (@event.Type == SpanTypes.TestSuite) - { - testSuites.Add(JsonConvert.DeserializeObject(eventContent)); - } - else if (@event.Type == SpanTypes.TestModule) - { - testModules.Add(JsonConvert.DeserializeObject(eventContent)); - } - } + tests.Add(JsonConvert.DeserializeObject(eventContent)); + testsCopy.Add(JsonConvert.DeserializeObject(eventContent)); + } + else if (@event.Type == SpanTypes.TestSuite) + { + testSuites.Add(JsonConvert.DeserializeObject(eventContent)); + } + else if (@event.Type == SpanTypes.TestModule) + { + testModules.Add(JsonConvert.DeserializeObject(eventContent)); } } } - }; - - using var processResult = await RunDotnetTestSampleAndWaitForExit( - agent, - arguments: "--collect:\"XPlat Code Coverage\"", - packageVersion: packageVersion); + } + } + }; - var settings = VerifyHelper.GetCIVisibilitySpanVerifierSettings(); - settings.UseTextForParameters("packageVersion=all"); - settings.DisableRequireUniquePrefix(); - await Verifier.Verify(tests.OrderBy(s => s.Resource).ThenBy(s => s.Meta.GetValueOrDefault(TestTags.Parameters)), settings); + using var processResult = await RunDotnetTestSampleAndWaitForExit( + agent, + arguments: "--collect:\"XPlat Code Coverage\"", + packageVersion: packageVersion); - // Check the tests, suites and modules count - Assert.Equal(ExpectedTestCount, tests.Count); - Assert.Equal(2, testSuites.Count); - Assert.Single(testModules); + // Check the tests, suites and modules count + Assert.Equal(ExpectedTestCount, tests.Count); + Assert.Equal(2, testSuites.Count); + Assert.Single(testModules); - var testSuite = testSuites.First(suite => suite.Resource == TestSuiteName); - var unskippableTestSuite = testSuites.First(suite => suite.Resource == UnSkippableSuiteName); - var testModule = testModules[0]; + var testSuite = testSuites.First(suite => suite.Resource == TestSuiteName); + var unskippableTestSuite = testSuites.First(suite => suite.Resource == UnSkippableSuiteName); + var testModule = testModules[0]; - // Check Suite - Assert.True(tests.All(t => t.TestSuiteId == testSuite.TestSuiteId || t.TestSuiteId == unskippableTestSuite.TestSuiteId)); - testSuite.TestModuleId.Should().Be(testModule.TestModuleId); - unskippableTestSuite.TestModuleId.Should().Be(testModule.TestModuleId); + // Check Suite + Assert.True(tests.All(t => t.TestSuiteId == testSuite.TestSuiteId || t.TestSuiteId == unskippableTestSuite.TestSuiteId)); + testSuite.TestModuleId.Should().Be(testModule.TestModuleId); + unskippableTestSuite.TestModuleId.Should().Be(testModule.TestModuleId); - // ITR tags inside the test suite - testSuite.Metrics.Should().Contain(IntelligentTestRunnerTags.SkippingCount, 1); + // ITR tags inside the test suite + testSuite.Metrics.Should().Contain(IntelligentTestRunnerTags.SkippingCount, 1); - // Check Module - Assert.True(tests.All(t => t.TestModuleId == testSuite.TestModuleId)); + // Check Module + Assert.True(tests.All(t => t.TestModuleId == testSuite.TestModuleId)); - // ITR tags inside the test module - testModule.Metrics.Should().Contain(IntelligentTestRunnerTags.SkippingCount, 1); - testModule.Meta.Should().Contain(IntelligentTestRunnerTags.SkippingType, IntelligentTestRunnerTags.SkippingTypeTest); - testModule.Meta.Should().Contain(IntelligentTestRunnerTags.TestsSkipped, "true"); + // ITR tags inside the test module + testModule.Metrics.Should().Contain(IntelligentTestRunnerTags.SkippingCount, 1); + testModule.Meta.Should().Contain(IntelligentTestRunnerTags.SkippingType, IntelligentTestRunnerTags.SkippingTypeTest); + testModule.Meta.Should().Contain(IntelligentTestRunnerTags.TestsSkipped, "true"); - // Check Session - tests.Should().OnlyContain(t => t.TestSessionId == testSuite.TestSessionId); - testSuite.TestSessionId.Should().Be(testModule.TestSessionId); - unskippableTestSuite.TestSessionId.Should().Be(testModule.TestSessionId); - testModule.TestSessionId.Should().Be(sessionId); + // Check Session + tests.Should().OnlyContain(t => t.TestSessionId == testSuite.TestSessionId); + testSuite.TestSessionId.Should().Be(testModule.TestSessionId); + unskippableTestSuite.TestSessionId.Should().Be(testModule.TestSessionId); + testModule.TestSessionId.Should().Be(sessionId); - // *************************************************************************** + // *************************************************************************** + try + { + foreach (var targetTest in tests) + { + // Remove decision maker tag (not used by the backend for civisibility) + targetTest.Meta.Remove(Tags.Propagated.DecisionMaker); - foreach (var targetTest in tests) - { - // Remove decision maker tag (not used by the backend for civisibility) - targetTest.Meta.Remove(Tags.Propagated.DecisionMaker); + // Remove EFD tags + targetTest.Meta.Remove(EarlyFlakeDetectionTags.TestIsNew); + targetTest.Meta.Remove(EarlyFlakeDetectionTags.TestIsRetry); - // Remove EFD tags - targetTest.Meta.Remove(EarlyFlakeDetectionTags.TestIsNew); - targetTest.Meta.Remove(EarlyFlakeDetectionTags.TestIsRetry); + // check the name + Assert.Equal("xunit.test", targetTest.Name); - // check the name - Assert.Equal("xunit.test", targetTest.Name); + // check correlationId + Assert.Equal(correlationId, targetTest.CorrelationId); - // check correlationId - Assert.Equal(correlationId, targetTest.CorrelationId); + // check the CIEnvironmentValues decoration. + CheckCIEnvironmentValuesDecoration(targetTest, gitRepositoryUrl, gitBranch, gitCommitSha); - // check the CIEnvironmentValues decoration. - CheckCIEnvironmentValuesDecoration(targetTest, gitRepositoryUrl, gitBranch, gitCommitSha); + // check the runtime values + CheckRuntimeValues(targetTest); - // check the runtime values - CheckRuntimeValues(targetTest); + // check the bundle name + AssertTargetSpanEqual(targetTest, TestTags.Bundle, TestBundleName); + AssertTargetSpanEqual(targetTest, TestTags.Module, TestBundleName); - // check the bundle name - AssertTargetSpanEqual(targetTest, TestTags.Bundle, TestBundleName); - AssertTargetSpanEqual(targetTest, TestTags.Module, TestBundleName); + // check the suite name + AssertTargetSpanAnyOf(targetTest, TestTags.Suite, TestSuiteName, UnSkippableSuiteName); - // check the suite name - AssertTargetSpanAnyOf(targetTest, TestTags.Suite, TestSuiteName, UnSkippableSuiteName); + // check the test type + AssertTargetSpanEqual(targetTest, TestTags.Type, TestTags.TypeTest); - // check the test type - AssertTargetSpanEqual(targetTest, TestTags.Type, TestTags.TypeTest); + // check the test framework + AssertTargetSpanContains(targetTest, TestTags.Framework, "xUnit"); + Assert.True(targetTest.Meta.Remove(TestTags.FrameworkVersion)); - // check the test framework - AssertTargetSpanContains(targetTest, TestTags.Framework, "xUnit"); - Assert.True(targetTest.Meta.Remove(TestTags.FrameworkVersion)); + // check the version + AssertTargetSpanEqual(targetTest, "version", "1.0.0"); - // check the version - AssertTargetSpanEqual(targetTest, "version", "1.0.0"); + // checks the origin tag + CheckOriginTag(targetTest); - // checks the origin tag - CheckOriginTag(targetTest); + // checks the source tags + AssertTargetSpanExists(targetTest, TestTags.SourceFile); - // checks the source tags - AssertTargetSpanExists(targetTest, TestTags.SourceFile); + // checks code owners + AssertTargetSpanExists(targetTest, TestTags.CodeOwners); - // checks code owners - AssertTargetSpanExists(targetTest, TestTags.CodeOwners); + // Check the Environment + AssertTargetSpanEqual(targetTest, Tags.Env, "integration_tests"); - // Check the Environment - AssertTargetSpanEqual(targetTest, Tags.Env, "integration_tests"); + // Language + AssertTargetSpanEqual(targetTest, Tags.Language, TracerConstants.Language); - // Language - AssertTargetSpanEqual(targetTest, Tags.Language, TracerConstants.Language); + // CI Library Language + AssertTargetSpanEqual(targetTest, CommonTags.LibraryVersion, TracerConstants.AssemblyVersion); - // CI Library Language - AssertTargetSpanEqual(targetTest, CommonTags.LibraryVersion, TracerConstants.AssemblyVersion); + // Check Session data + AssertTargetSpanEqual(targetTest, TestTags.Command, sessionCommand); + AssertTargetSpanEqual(targetTest, TestTags.CommandWorkingDirectory, sessionWorkingDirectory); - // Check Session data - AssertTargetSpanEqual(targetTest, TestTags.Command, sessionCommand); - AssertTargetSpanEqual(targetTest, TestTags.CommandWorkingDirectory, sessionWorkingDirectory); + // Unskippable data + if (targetTest.Meta[TestTags.Name] != "UnskippableTest") + { + AssertTargetSpanEqual(targetTest, IntelligentTestRunnerTags.UnskippableTag, "false"); + AssertTargetSpanEqual(targetTest, IntelligentTestRunnerTags.ForcedRunTag, "false"); + } - // Unskippable data - if (targetTest.Meta[TestTags.Name] != "UnskippableTest") - { - AssertTargetSpanEqual(targetTest, IntelligentTestRunnerTags.UnskippableTag, "false"); + // check specific test span + switch (targetTest.Meta[TestTags.Name]) + { + case "SimplePassTest": + CheckSimpleTestSpan(targetTest); + break; + + case "SimpleSkipFromAttributeTest": + CheckSimpleSkipFromAttributeTest(targetTest); + AssertTargetSpanEqual(targetTest, IntelligentTestRunnerTags.SkippedBy, "false"); + break; + + case "SkipByITRSimulation": + AssertTargetSpanEqual(targetTest, TestTags.Status, TestTags.StatusSkip); + AssertTargetSpanEqual(targetTest, TestTags.SkipReason, IntelligentTestRunnerTags.SkippedByReason); + AssertTargetSpanEqual(targetTest, IntelligentTestRunnerTags.SkippedBy, "true"); + break; + + case "SimpleErrorTest": + CheckSimpleErrorTest(targetTest); + break; + + case "TraitPassTest": + CheckSimpleTestSpan(targetTest); + CheckTraitsValues(targetTest); + break; + + case "TraitSkipFromAttributeTest": + CheckSimpleSkipFromAttributeTest(targetTest); + CheckTraitsValues(targetTest); + AssertTargetSpanEqual(targetTest, IntelligentTestRunnerTags.SkippedBy, "false"); + break; + + case "TraitErrorTest": + CheckSimpleErrorTest(targetTest); + CheckTraitsValues(targetTest); + break; + + case "SimpleParameterizedTest": + CheckSimpleTestSpan(targetTest); + AssertTargetSpanAnyOf( + targetTest, + TestTags.Parameters, + "{\"metadata\":{\"test_name\":\"Samples.XUnitTests.TestSuite.SimpleParameterizedTest(xValue: 1, yValue: 1, expectedResult: 2)\"},\"arguments\":{\"xValue\":\"1\",\"yValue\":\"1\",\"expectedResult\":\"2\"}}", + "{\"metadata\":{\"test_name\":\"Samples.XUnitTests.TestSuite.SimpleParameterizedTest(xValue: 2, yValue: 2, expectedResult: 4)\"},\"arguments\":{\"xValue\":\"2\",\"yValue\":\"2\",\"expectedResult\":\"4\"}}", + "{\"metadata\":{\"test_name\":\"Samples.XUnitTests.TestSuite.SimpleParameterizedTest(xValue: 3, yValue: 3, expectedResult: 6)\"},\"arguments\":{\"xValue\":\"3\",\"yValue\":\"3\",\"expectedResult\":\"6\"}}", + "{\"metadata\":{\"test_name\":\"SimpleParameterizedTest(xValue: 1, yValue: 1, expectedResult: 2)\"},\"arguments\":{\"xValue\":\"1\",\"yValue\":\"1\",\"expectedResult\":\"2\"}}", + "{\"metadata\":{\"test_name\":\"SimpleParameterizedTest(xValue: 2, yValue: 2, expectedResult: 4)\"},\"arguments\":{\"xValue\":\"2\",\"yValue\":\"2\",\"expectedResult\":\"4\"}}", + "{\"metadata\":{\"test_name\":\"SimpleParameterizedTest(xValue: 3, yValue: 3, expectedResult: 6)\"},\"arguments\":{\"xValue\":\"3\",\"yValue\":\"3\",\"expectedResult\":\"6\"}}"); + break; + + case "SimpleSkipParameterizedTest": + CheckSimpleSkipFromAttributeTest(targetTest); + AssertTargetSpanEqual(targetTest, IntelligentTestRunnerTags.SkippedBy, "false"); + break; + + case "SimpleErrorParameterizedTest": + CheckSimpleErrorTest(targetTest); + AssertTargetSpanAnyOf( + targetTest, + TestTags.Parameters, + "{\"metadata\":{\"test_name\":\"Samples.XUnitTests.TestSuite.SimpleErrorParameterizedTest(xValue: 1, yValue: 0, expectedResult: 2)\"},\"arguments\":{\"xValue\":\"1\",\"yValue\":\"0\",\"expectedResult\":\"2\"}}", + "{\"metadata\":{\"test_name\":\"Samples.XUnitTests.TestSuite.SimpleErrorParameterizedTest(xValue: 2, yValue: 0, expectedResult: 4)\"},\"arguments\":{\"xValue\":\"2\",\"yValue\":\"0\",\"expectedResult\":\"4\"}}", + "{\"metadata\":{\"test_name\":\"Samples.XUnitTests.TestSuite.SimpleErrorParameterizedTest(xValue: 3, yValue: 0, expectedResult: 6)\"},\"arguments\":{\"xValue\":\"3\",\"yValue\":\"0\",\"expectedResult\":\"6\"}}", + "{\"metadata\":{\"test_name\":\"SimpleErrorParameterizedTest(xValue: 1, yValue: 0, expectedResult: 2)\"},\"arguments\":{\"xValue\":\"1\",\"yValue\":\"0\",\"expectedResult\":\"2\"}}", + "{\"metadata\":{\"test_name\":\"SimpleErrorParameterizedTest(xValue: 2, yValue: 0, expectedResult: 4)\"},\"arguments\":{\"xValue\":\"2\",\"yValue\":\"0\",\"expectedResult\":\"4\"}}", + "{\"metadata\":{\"test_name\":\"SimpleErrorParameterizedTest(xValue: 3, yValue: 0, expectedResult: 6)\"},\"arguments\":{\"xValue\":\"3\",\"yValue\":\"0\",\"expectedResult\":\"6\"}}"); + break; + + case "UnskippableTest": + AssertTargetSpanEqual(targetTest, IntelligentTestRunnerTags.UnskippableTag, "true"); AssertTargetSpanEqual(targetTest, IntelligentTestRunnerTags.ForcedRunTag, "false"); - } - - // check specific test span - switch (targetTest.Meta[TestTags.Name]) - { - case "SimplePassTest": - CheckSimpleTestSpan(targetTest); - break; - - case "SimpleSkipFromAttributeTest": - CheckSimpleSkipFromAttributeTest(targetTest); - AssertTargetSpanEqual(targetTest, IntelligentTestRunnerTags.SkippedBy, "false"); - break; - - case "SkipByITRSimulation": - AssertTargetSpanEqual(targetTest, TestTags.Status, TestTags.StatusSkip); - AssertTargetSpanEqual(targetTest, TestTags.SkipReason, IntelligentTestRunnerTags.SkippedByReason); - AssertTargetSpanEqual(targetTest, IntelligentTestRunnerTags.SkippedBy, "true"); - break; - - case "SimpleErrorTest": - CheckSimpleErrorTest(targetTest); - break; - - case "TraitPassTest": - CheckSimpleTestSpan(targetTest); - CheckTraitsValues(targetTest); - break; - - case "TraitSkipFromAttributeTest": - CheckSimpleSkipFromAttributeTest(targetTest); - CheckTraitsValues(targetTest); - AssertTargetSpanEqual(targetTest, IntelligentTestRunnerTags.SkippedBy, "false"); - break; - - case "TraitErrorTest": - CheckSimpleErrorTest(targetTest); - CheckTraitsValues(targetTest); - break; - - case "SimpleParameterizedTest": - CheckSimpleTestSpan(targetTest); - AssertTargetSpanAnyOf( - targetTest, - TestTags.Parameters, - "{\"metadata\":{\"test_name\":\"Samples.XUnitTests.TestSuite.SimpleParameterizedTest(xValue: 1, yValue: 1, expectedResult: 2)\"},\"arguments\":{\"xValue\":\"1\",\"yValue\":\"1\",\"expectedResult\":\"2\"}}", - "{\"metadata\":{\"test_name\":\"Samples.XUnitTests.TestSuite.SimpleParameterizedTest(xValue: 2, yValue: 2, expectedResult: 4)\"},\"arguments\":{\"xValue\":\"2\",\"yValue\":\"2\",\"expectedResult\":\"4\"}}", - "{\"metadata\":{\"test_name\":\"Samples.XUnitTests.TestSuite.SimpleParameterizedTest(xValue: 3, yValue: 3, expectedResult: 6)\"},\"arguments\":{\"xValue\":\"3\",\"yValue\":\"3\",\"expectedResult\":\"6\"}}", - "{\"metadata\":{\"test_name\":\"SimpleParameterizedTest(xValue: 1, yValue: 1, expectedResult: 2)\"},\"arguments\":{\"xValue\":\"1\",\"yValue\":\"1\",\"expectedResult\":\"2\"}}", - "{\"metadata\":{\"test_name\":\"SimpleParameterizedTest(xValue: 2, yValue: 2, expectedResult: 4)\"},\"arguments\":{\"xValue\":\"2\",\"yValue\":\"2\",\"expectedResult\":\"4\"}}", - "{\"metadata\":{\"test_name\":\"SimpleParameterizedTest(xValue: 3, yValue: 3, expectedResult: 6)\"},\"arguments\":{\"xValue\":\"3\",\"yValue\":\"3\",\"expectedResult\":\"6\"}}"); - break; - - case "SimpleSkipParameterizedTest": - CheckSimpleSkipFromAttributeTest(targetTest); - AssertTargetSpanEqual(targetTest, IntelligentTestRunnerTags.SkippedBy, "false"); - break; - - case "SimpleErrorParameterizedTest": - CheckSimpleErrorTest(targetTest); - AssertTargetSpanAnyOf( - targetTest, - TestTags.Parameters, - "{\"metadata\":{\"test_name\":\"Samples.XUnitTests.TestSuite.SimpleErrorParameterizedTest(xValue: 1, yValue: 0, expectedResult: 2)\"},\"arguments\":{\"xValue\":\"1\",\"yValue\":\"0\",\"expectedResult\":\"2\"}}", - "{\"metadata\":{\"test_name\":\"Samples.XUnitTests.TestSuite.SimpleErrorParameterizedTest(xValue: 2, yValue: 0, expectedResult: 4)\"},\"arguments\":{\"xValue\":\"2\",\"yValue\":\"0\",\"expectedResult\":\"4\"}}", - "{\"metadata\":{\"test_name\":\"Samples.XUnitTests.TestSuite.SimpleErrorParameterizedTest(xValue: 3, yValue: 0, expectedResult: 6)\"},\"arguments\":{\"xValue\":\"3\",\"yValue\":\"0\",\"expectedResult\":\"6\"}}", - "{\"metadata\":{\"test_name\":\"SimpleErrorParameterizedTest(xValue: 1, yValue: 0, expectedResult: 2)\"},\"arguments\":{\"xValue\":\"1\",\"yValue\":\"0\",\"expectedResult\":\"2\"}}", - "{\"metadata\":{\"test_name\":\"SimpleErrorParameterizedTest(xValue: 2, yValue: 0, expectedResult: 4)\"},\"arguments\":{\"xValue\":\"2\",\"yValue\":\"0\",\"expectedResult\":\"4\"}}", - "{\"metadata\":{\"test_name\":\"SimpleErrorParameterizedTest(xValue: 3, yValue: 0, expectedResult: 6)\"},\"arguments\":{\"xValue\":\"3\",\"yValue\":\"0\",\"expectedResult\":\"6\"}}"); - break; - - case "UnskippableTest": - AssertTargetSpanEqual(targetTest, IntelligentTestRunnerTags.UnskippableTag, "true"); - AssertTargetSpanEqual(targetTest, IntelligentTestRunnerTags.ForcedRunTag, "false"); - CheckSimpleTestSpan(targetTest); - break; - } - - // check remaining tag (only the name) - Assert.Single(targetTest.Meta); - - // check if we received code coverage information at session level - codeCoverageReceived.Value.Should().BeTrue(); + CheckSimpleTestSpan(targetTest); + break; } - // *************************************************************************** - // Check logs - messages = logsIntake.Logs.Select(i => i.Message).Where(m => m.StartsWith("Test:")).ToArray(); + // check remaining tag (only the name) + Assert.Single(targetTest.Meta); - Assert.Contains(messages, m => m.StartsWith("Test:SimplePassTest")); - Assert.Contains(messages, m => m.StartsWith("Test:SimpleErrorTest")); - Assert.Contains(messages, m => m.StartsWith("Test:TraitPassTest")); - Assert.Contains(messages, m => m.StartsWith("Test:TraitErrorTest")); - Assert.Contains(messages, m => m.StartsWith("Test:SimpleParameterizedTest")); - Assert.Contains(messages, m => m.StartsWith("Test:SimpleErrorParameterizedTest")); + // check if we received code coverage information at session level + codeCoverageReceived.Value.Should().BeTrue(); } } catch @@ -376,14 +356,27 @@ public async Task SubmitTraces(string packageVersion, string evpVersionToRemove, WriteSpans(tests); throw; } + + // Snapshots + var settings = VerifyHelper.GetCIVisibilitySpanVerifierSettings(); + settings.UseTextForParameters("packageVersion=all"); + settings.DisableRequireUniquePrefix(); + settings.UseTypeName(nameof(XUnitEvpTests)); + await Verifier.Verify(testsCopy.OrderBy(s => s.Resource).ThenBy(s => s.Meta.GetValueOrDefault(TestTags.Parameters)), settings); + + // *************************************************************************** + // Check logs + messages = logsIntake.Logs.Select(i => i.Message).Where(m => m.StartsWith("Test:")).ToArray(); + + Assert.Contains(messages, m => m.StartsWith("Test:SimplePassTest")); + Assert.Contains(messages, m => m.StartsWith("Test:SimpleErrorTest")); + Assert.Contains(messages, m => m.StartsWith("Test:TraitPassTest")); + Assert.Contains(messages, m => m.StartsWith("Test:TraitErrorTest")); + Assert.Contains(messages, m => m.StartsWith("Test:SimpleParameterizedTest")); + Assert.Contains(messages, m => m.StartsWith("Test:SimpleErrorParameterizedTest")); } - [SkippableTheory] - [MemberData(nameof(GetDataForEarlyFlakeDetection))] - [Trait("Category", "EndToEnd")] - [Trait("Category", "TestIntegrations")] - [Trait("Category", "EarlyFlakeDetection")] - public async Task EarlyFlakeDetection(string packageVersion, string evpVersionToRemove, bool expectedGzip, string settingsJson, string testsJson, int expectedSpans, string friendlyName) + public virtual async Task EarlyFlakeDetection(string packageVersion, string evpVersionToRemove, bool expectedGzip, string settingsJson, string testsJson, int expectedSpans, string friendlyName) { var tests = new List(); var testSuites = new List(); @@ -468,9 +461,16 @@ public async Task EarlyFlakeDetection(string packageVersion, string evpVersionTo using var processResult = await RunDotnetTestSampleAndWaitForExit(agent, packageVersion: packageVersion); + // Check the tests, suites and modules count + Assert.Equal(expectedSpans, tests.Count); + Assert.Equal(2, testSuites.Count); + Assert.Single(testModules); + + // Snapshot testing var settings = VerifyHelper.GetCIVisibilitySpanVerifierSettings(); settings.UseTextForParameters(friendlyName); settings.DisableRequireUniquePrefix(); + settings.UseTypeName(nameof(XUnitEvpTests)); await Verifier.Verify( tests .OrderBy(s => s.Resource) @@ -479,11 +479,6 @@ await Verifier.Verify( .ThenBy(s => s.Meta.GetValueOrDefault(EarlyFlakeDetectionTags.TestIsRetry)) .ThenBy(s => s.Meta.GetValueOrDefault(EarlyFlakeDetectionTags.AbortReason)), settings); - - // Check the tests, suites and modules count - Assert.Equal(expectedSpans, tests.Count); - Assert.Equal(2, testSuites.Count); - Assert.Single(testModules); } catch { diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/XUnitTests.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/XUnitTests.cs index 23d5983c3649..841b3e77dee7 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/XUnitTests.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/XUnitTests.cs @@ -3,13 +3,9 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. // -using System; using System.Collections.Generic; -using System.IO; using System.Linq; -using System.Threading; using System.Threading.Tasks; -using Datadog.Trace.Ci; using Datadog.Trace.Ci.Tags; using Datadog.Trace.Configuration; using Datadog.Trace.ExtensionMethods; @@ -22,7 +18,7 @@ namespace Datadog.Trace.ClrProfiler.IntegrationTests.CI { [UsesVerify] - public class XUnitTests : TestingFrameworkTest + public abstract class XUnitTests : TestingFrameworkTest { private const string TestBundleName = "Samples.XUnitTests"; private const string TestSuiteName = "Samples.XUnitTests.TestSuite"; @@ -36,204 +32,182 @@ public XUnitTests(ITestOutputHelper output) SetServiceVersion("1.0.0"); } - [SkippableTheory] - [MemberData(nameof(PackageVersions.XUnit), MemberType = typeof(PackageVersions))] - [Trait("Category", "EndToEnd")] - [Trait("Category", "TestIntegrations")] - public async Task SubmitTraces(string packageVersion) + public virtual async Task SubmitTraces(string packageVersion) { - List spans = null; string[] messages = null; + SetEnvironmentVariable(ConfigurationKeys.CIVisibility.Enabled, "1"); + + using var logsIntake = new MockLogsIntakeForCiVisibility(); + EnableDirectLogSubmission(logsIntake.Port, nameof(IntegrationId.XUnit), nameof(XUnitTests)); + SetEnvironmentVariable(ConfigurationKeys.CIVisibility.Logs, "1"); + + using var agent = EnvironmentHelper.GetMockAgent(); + + // We remove the evp_proxy endpoint to force the APM protocol compatibility + agent.Configuration.Endpoints = agent.Configuration.Endpoints.Where(e => !e.Contains("evp_proxy/v2") && !e.Contains("evp_proxy/v4")).ToArray(); + using var processResult = await RunDotnetTestSampleAndWaitForExit(agent, packageVersion: packageVersion); + var spans = agent.WaitForSpans(ExpectedSpanCount) + .Where(s => !(s.Tags.TryGetValue(Tags.InstrumentationName, out var sValue) && sValue == "HttpMessageHandler")) + .ToList(); + var spansCopy = JsonConvert.DeserializeObject>(JsonConvert.SerializeObject(spans)); + + // Check the span count + Assert.Equal(ExpectedSpanCount, spans.Count); + + // *************************************************************************** + try { - SetEnvironmentVariable(ConfigurationKeys.CIVisibility.Enabled, "1"); + foreach (var targetSpan in spans) + { + // Remove decision maker tag (not used by the backend for civisibility) + targetSpan.Tags.Remove(Tags.Propagated.DecisionMaker); + targetSpan.Tags.Remove(Tags.Propagated.TraceIdUpper); - using var logsIntake = new MockLogsIntakeForCiVisibility(); - EnableDirectLogSubmission(logsIntake.Port, nameof(IntegrationId.XUnit), nameof(XUnitTests)); - SetEnvironmentVariable(ConfigurationKeys.CIVisibility.Logs, "1"); + // Remove git metadata added by the apm agent writer. + targetSpan.Tags.Remove(Tags.GitCommitSha); + targetSpan.Tags.Remove(Tags.GitRepositoryUrl); - using (var agent = EnvironmentHelper.GetMockAgent()) - { - // We remove the evp_proxy endpoint to force the APM protocol compatibility - agent.Configuration.Endpoints = agent.Configuration.Endpoints.Where(e => !e.Contains("evp_proxy/v2") && !e.Contains("evp_proxy/v4")).ToArray(); - using (ProcessResult processResult = await RunDotnetTestSampleAndWaitForExit(agent, packageVersion: packageVersion)) - { - spans = agent.WaitForSpans(ExpectedSpanCount) - .Where(s => !(s.Tags.TryGetValue(Tags.InstrumentationName, out var sValue) && sValue == "HttpMessageHandler")) - .ToList(); + // Remove EFD tags + targetSpan.Tags.Remove(EarlyFlakeDetectionTags.TestIsNew); + targetSpan.Tags.Remove(EarlyFlakeDetectionTags.TestIsRetry); + + // check the name + Assert.Equal("xunit.test", targetSpan.Name); + + // check the CIEnvironmentValues decoration. + CheckCIEnvironmentValuesDecoration(targetSpan); + + // check the runtime values + CheckRuntimeValues(targetSpan); + + // check the bundle name + AssertTargetSpanEqual(targetSpan, TestTags.Bundle, TestBundleName); + AssertTargetSpanEqual(targetSpan, TestTags.Module, TestBundleName); - var settings = VerifyHelper.GetCIVisibilitySpanVerifierSettings("all"); - settings.DisableRequireUniquePrefix(); - await Verifier.Verify(spans.OrderBy(s => s.Resource).ThenBy(s => s.Tags.GetValueOrDefault(TestTags.Parameters)), settings); + // check the suite name + AssertTargetSpanAnyOf(targetSpan, TestTags.Suite, TestSuiteName, UnSkippableSuiteName); - // Check the span count - Assert.Equal(ExpectedSpanCount, spans.Count); + // check the test type + AssertTargetSpanEqual(targetSpan, TestTags.Type, TestTags.TypeTest); - // *************************************************************************** + // check the test framework + AssertTargetSpanContains(targetSpan, TestTags.Framework, "xUnit"); + Assert.True(targetSpan.Tags.Remove(TestTags.FrameworkVersion)); - foreach (var targetSpan in spans) - { - // Remove decision maker tag (not used by the backend for civisibility) - targetSpan.Tags.Remove(Tags.Propagated.DecisionMaker); - targetSpan.Tags.Remove(Tags.Propagated.TraceIdUpper); + // check the version + AssertTargetSpanEqual(targetSpan, "version", "1.0.0"); - // Remove git metadata added by the apm agent writer. - targetSpan.Tags.Remove(Tags.GitCommitSha); - targetSpan.Tags.Remove(Tags.GitRepositoryUrl); + // checks the origin tag + CheckOriginTag(targetSpan); - // Remove EFD tags - targetSpan.Tags.Remove(EarlyFlakeDetectionTags.TestIsNew); - targetSpan.Tags.Remove(EarlyFlakeDetectionTags.TestIsRetry); + // checks the runtime id tag + AssertTargetSpanExists(targetSpan, Tags.RuntimeId); - // check the name - Assert.Equal("xunit.test", targetSpan.Name); - - // check the CIEnvironmentValues decoration. - CheckCIEnvironmentValuesDecoration(targetSpan); - - // check the runtime values - CheckRuntimeValues(targetSpan); - - // check the bundle name - AssertTargetSpanEqual(targetSpan, TestTags.Bundle, TestBundleName); - AssertTargetSpanEqual(targetSpan, TestTags.Module, TestBundleName); - - // check the suite name - AssertTargetSpanAnyOf(targetSpan, TestTags.Suite, TestSuiteName, UnSkippableSuiteName); - - // check the test type - AssertTargetSpanEqual(targetSpan, TestTags.Type, TestTags.TypeTest); - - // check the test framework - AssertTargetSpanContains(targetSpan, TestTags.Framework, "xUnit"); - Assert.True(targetSpan.Tags.Remove(TestTags.FrameworkVersion)); - - // check the version - AssertTargetSpanEqual(targetSpan, "version", "1.0.0"); - - // checks the origin tag - CheckOriginTag(targetSpan); - - // checks the runtime id tag - AssertTargetSpanExists(targetSpan, Tags.RuntimeId); - - // checks the source tags - AssertTargetSpanExists(targetSpan, TestTags.SourceFile); - - // checks code owners - AssertTargetSpanExists(targetSpan, TestTags.CodeOwners); - - // Check the Environment - AssertTargetSpanEqual(targetSpan, Tags.Env, "integration_tests"); - - // Language - AssertTargetSpanEqual(targetSpan, Tags.Language, TracerConstants.Language); - - // CI Library Language - AssertTargetSpanEqual(targetSpan, CommonTags.LibraryVersion, TracerConstants.AssemblyVersion); - - // Session data - AssertTargetSpanExists(targetSpan, TestTags.Command); - AssertTargetSpanExists(targetSpan, TestTags.CommandWorkingDirectory); - - // Unskippable data - if (targetSpan.Tags[TestTags.Name] != "UnskippableTest") - { - AssertTargetSpanEqual(targetSpan, IntelligentTestRunnerTags.UnskippableTag, "false"); - AssertTargetSpanEqual(targetSpan, IntelligentTestRunnerTags.ForcedRunTag, "false"); - } - - // check specific test span - switch (targetSpan.Tags[TestTags.Name]) - { - case "SimplePassTest": - CheckSimpleTestSpan(targetSpan); - break; - - case "SimpleSkipFromAttributeTest": - CheckSimpleSkipFromAttributeTest(targetSpan); - AssertTargetSpanEqual(targetSpan, IntelligentTestRunnerTags.SkippedBy, "false"); - break; - - case "SkipByITRSimulation": - AssertTargetSpanEqual(targetSpan, TestTags.Status, TestTags.StatusSkip); - AssertTargetSpanEqual(targetSpan, TestTags.SkipReason, IntelligentTestRunnerTags.SkippedByReason); - AssertTargetSpanEqual(targetSpan, IntelligentTestRunnerTags.SkippedBy, "true"); - break; - - case "SimpleErrorTest": - CheckSimpleErrorTest(targetSpan); - break; - - case "TraitPassTest": - CheckSimpleTestSpan(targetSpan); - CheckTraitsValues(targetSpan); - break; - - case "TraitSkipFromAttributeTest": - CheckSimpleSkipFromAttributeTest(targetSpan); - CheckTraitsValues(targetSpan); - AssertTargetSpanEqual(targetSpan, IntelligentTestRunnerTags.SkippedBy, "false"); - break; - - case "TraitErrorTest": - CheckSimpleErrorTest(targetSpan); - CheckTraitsValues(targetSpan); - break; - - case "SimpleParameterizedTest": - CheckSimpleTestSpan(targetSpan); - AssertTargetSpanAnyOf( - targetSpan, - TestTags.Parameters, - "{\"metadata\":{\"test_name\":\"Samples.XUnitTests.TestSuite.SimpleParameterizedTest(xValue: 1, yValue: 1, expectedResult: 2)\"},\"arguments\":{\"xValue\":\"1\",\"yValue\":\"1\",\"expectedResult\":\"2\"}}", - "{\"metadata\":{\"test_name\":\"Samples.XUnitTests.TestSuite.SimpleParameterizedTest(xValue: 2, yValue: 2, expectedResult: 4)\"},\"arguments\":{\"xValue\":\"2\",\"yValue\":\"2\",\"expectedResult\":\"4\"}}", - "{\"metadata\":{\"test_name\":\"Samples.XUnitTests.TestSuite.SimpleParameterizedTest(xValue: 3, yValue: 3, expectedResult: 6)\"},\"arguments\":{\"xValue\":\"3\",\"yValue\":\"3\",\"expectedResult\":\"6\"}}", - "{\"metadata\":{\"test_name\":\"SimpleParameterizedTest(xValue: 1, yValue: 1, expectedResult: 2)\"},\"arguments\":{\"xValue\":\"1\",\"yValue\":\"1\",\"expectedResult\":\"2\"}}", - "{\"metadata\":{\"test_name\":\"SimpleParameterizedTest(xValue: 2, yValue: 2, expectedResult: 4)\"},\"arguments\":{\"xValue\":\"2\",\"yValue\":\"2\",\"expectedResult\":\"4\"}}", - "{\"metadata\":{\"test_name\":\"SimpleParameterizedTest(xValue: 3, yValue: 3, expectedResult: 6)\"},\"arguments\":{\"xValue\":\"3\",\"yValue\":\"3\",\"expectedResult\":\"6\"}}"); - break; - - case "SimpleSkipParameterizedTest": - CheckSimpleSkipFromAttributeTest(targetSpan); - AssertTargetSpanEqual(targetSpan, IntelligentTestRunnerTags.SkippedBy, "false"); - break; - - case "SimpleErrorParameterizedTest": - CheckSimpleErrorTest(targetSpan); - AssertTargetSpanAnyOf( - targetSpan, - TestTags.Parameters, - "{\"metadata\":{\"test_name\":\"Samples.XUnitTests.TestSuite.SimpleErrorParameterizedTest(xValue: 1, yValue: 0, expectedResult: 2)\"},\"arguments\":{\"xValue\":\"1\",\"yValue\":\"0\",\"expectedResult\":\"2\"}}", - "{\"metadata\":{\"test_name\":\"Samples.XUnitTests.TestSuite.SimpleErrorParameterizedTest(xValue: 2, yValue: 0, expectedResult: 4)\"},\"arguments\":{\"xValue\":\"2\",\"yValue\":\"0\",\"expectedResult\":\"4\"}}", - "{\"metadata\":{\"test_name\":\"Samples.XUnitTests.TestSuite.SimpleErrorParameterizedTest(xValue: 3, yValue: 0, expectedResult: 6)\"},\"arguments\":{\"xValue\":\"3\",\"yValue\":\"0\",\"expectedResult\":\"6\"}}", - "{\"metadata\":{\"test_name\":\"SimpleErrorParameterizedTest(xValue: 1, yValue: 0, expectedResult: 2)\"},\"arguments\":{\"xValue\":\"1\",\"yValue\":\"0\",\"expectedResult\":\"2\"}}", - "{\"metadata\":{\"test_name\":\"SimpleErrorParameterizedTest(xValue: 2, yValue: 0, expectedResult: 4)\"},\"arguments\":{\"xValue\":\"2\",\"yValue\":\"0\",\"expectedResult\":\"4\"}}", - "{\"metadata\":{\"test_name\":\"SimpleErrorParameterizedTest(xValue: 3, yValue: 0, expectedResult: 6)\"},\"arguments\":{\"xValue\":\"3\",\"yValue\":\"0\",\"expectedResult\":\"6\"}}"); - break; - - case "UnskippableTest": - AssertTargetSpanEqual(targetSpan, IntelligentTestRunnerTags.UnskippableTag, "true"); - AssertTargetSpanEqual(targetSpan, IntelligentTestRunnerTags.ForcedRunTag, "false"); - CheckSimpleTestSpan(targetSpan); - break; - } - - // check remaining tag (only the name) - Assert.Single(targetSpan.Tags); - } - - // *************************************************************************** - // Check logs - messages = logsIntake.Logs.Select(i => i.Message).Where(m => m.StartsWith("Test:")).ToArray(); - - Assert.Contains(messages, m => m.StartsWith("Test:SimplePassTest")); - Assert.Contains(messages, m => m.StartsWith("Test:SimpleErrorTest")); - Assert.Contains(messages, m => m.StartsWith("Test:TraitPassTest")); - Assert.Contains(messages, m => m.StartsWith("Test:TraitErrorTest")); - Assert.Contains(messages, m => m.StartsWith("Test:SimpleParameterizedTest")); - Assert.Contains(messages, m => m.StartsWith("Test:SimpleErrorParameterizedTest")); + // checks the source tags + AssertTargetSpanExists(targetSpan, TestTags.SourceFile); + + // checks code owners + AssertTargetSpanExists(targetSpan, TestTags.CodeOwners); + + // Check the Environment + AssertTargetSpanEqual(targetSpan, Tags.Env, "integration_tests"); + + // Language + AssertTargetSpanEqual(targetSpan, Tags.Language, TracerConstants.Language); + + // CI Library Language + AssertTargetSpanEqual(targetSpan, CommonTags.LibraryVersion, TracerConstants.AssemblyVersion); + + // Session data + AssertTargetSpanExists(targetSpan, TestTags.Command); + AssertTargetSpanExists(targetSpan, TestTags.CommandWorkingDirectory); + + // Unskippable data + if (targetSpan.Tags[TestTags.Name] != "UnskippableTest") + { + AssertTargetSpanEqual(targetSpan, IntelligentTestRunnerTags.UnskippableTag, "false"); + AssertTargetSpanEqual(targetSpan, IntelligentTestRunnerTags.ForcedRunTag, "false"); + } + + // check specific test span + switch (targetSpan.Tags[TestTags.Name]) + { + case "SimplePassTest": + CheckSimpleTestSpan(targetSpan); + break; + + case "SimpleSkipFromAttributeTest": + CheckSimpleSkipFromAttributeTest(targetSpan); + AssertTargetSpanEqual(targetSpan, IntelligentTestRunnerTags.SkippedBy, "false"); + break; + + case "SkipByITRSimulation": + AssertTargetSpanEqual(targetSpan, TestTags.Status, TestTags.StatusSkip); + AssertTargetSpanEqual(targetSpan, TestTags.SkipReason, IntelligentTestRunnerTags.SkippedByReason); + AssertTargetSpanEqual(targetSpan, IntelligentTestRunnerTags.SkippedBy, "true"); + break; + + case "SimpleErrorTest": + CheckSimpleErrorTest(targetSpan); + break; + + case "TraitPassTest": + CheckSimpleTestSpan(targetSpan); + CheckTraitsValues(targetSpan); + break; + + case "TraitSkipFromAttributeTest": + CheckSimpleSkipFromAttributeTest(targetSpan); + CheckTraitsValues(targetSpan); + AssertTargetSpanEqual(targetSpan, IntelligentTestRunnerTags.SkippedBy, "false"); + break; + + case "TraitErrorTest": + CheckSimpleErrorTest(targetSpan); + CheckTraitsValues(targetSpan); + break; + + case "SimpleParameterizedTest": + CheckSimpleTestSpan(targetSpan); + AssertTargetSpanAnyOf( + targetSpan, + TestTags.Parameters, + "{\"metadata\":{\"test_name\":\"Samples.XUnitTests.TestSuite.SimpleParameterizedTest(xValue: 1, yValue: 1, expectedResult: 2)\"},\"arguments\":{\"xValue\":\"1\",\"yValue\":\"1\",\"expectedResult\":\"2\"}}", + "{\"metadata\":{\"test_name\":\"Samples.XUnitTests.TestSuite.SimpleParameterizedTest(xValue: 2, yValue: 2, expectedResult: 4)\"},\"arguments\":{\"xValue\":\"2\",\"yValue\":\"2\",\"expectedResult\":\"4\"}}", + "{\"metadata\":{\"test_name\":\"Samples.XUnitTests.TestSuite.SimpleParameterizedTest(xValue: 3, yValue: 3, expectedResult: 6)\"},\"arguments\":{\"xValue\":\"3\",\"yValue\":\"3\",\"expectedResult\":\"6\"}}", + "{\"metadata\":{\"test_name\":\"SimpleParameterizedTest(xValue: 1, yValue: 1, expectedResult: 2)\"},\"arguments\":{\"xValue\":\"1\",\"yValue\":\"1\",\"expectedResult\":\"2\"}}", + "{\"metadata\":{\"test_name\":\"SimpleParameterizedTest(xValue: 2, yValue: 2, expectedResult: 4)\"},\"arguments\":{\"xValue\":\"2\",\"yValue\":\"2\",\"expectedResult\":\"4\"}}", + "{\"metadata\":{\"test_name\":\"SimpleParameterizedTest(xValue: 3, yValue: 3, expectedResult: 6)\"},\"arguments\":{\"xValue\":\"3\",\"yValue\":\"3\",\"expectedResult\":\"6\"}}"); + break; + + case "SimpleSkipParameterizedTest": + CheckSimpleSkipFromAttributeTest(targetSpan); + AssertTargetSpanEqual(targetSpan, IntelligentTestRunnerTags.SkippedBy, "false"); + break; + + case "SimpleErrorParameterizedTest": + CheckSimpleErrorTest(targetSpan); + AssertTargetSpanAnyOf( + targetSpan, + TestTags.Parameters, + "{\"metadata\":{\"test_name\":\"Samples.XUnitTests.TestSuite.SimpleErrorParameterizedTest(xValue: 1, yValue: 0, expectedResult: 2)\"},\"arguments\":{\"xValue\":\"1\",\"yValue\":\"0\",\"expectedResult\":\"2\"}}", + "{\"metadata\":{\"test_name\":\"Samples.XUnitTests.TestSuite.SimpleErrorParameterizedTest(xValue: 2, yValue: 0, expectedResult: 4)\"},\"arguments\":{\"xValue\":\"2\",\"yValue\":\"0\",\"expectedResult\":\"4\"}}", + "{\"metadata\":{\"test_name\":\"Samples.XUnitTests.TestSuite.SimpleErrorParameterizedTest(xValue: 3, yValue: 0, expectedResult: 6)\"},\"arguments\":{\"xValue\":\"3\",\"yValue\":\"0\",\"expectedResult\":\"6\"}}", + "{\"metadata\":{\"test_name\":\"SimpleErrorParameterizedTest(xValue: 1, yValue: 0, expectedResult: 2)\"},\"arguments\":{\"xValue\":\"1\",\"yValue\":\"0\",\"expectedResult\":\"2\"}}", + "{\"metadata\":{\"test_name\":\"SimpleErrorParameterizedTest(xValue: 2, yValue: 0, expectedResult: 4)\"},\"arguments\":{\"xValue\":\"2\",\"yValue\":\"0\",\"expectedResult\":\"4\"}}", + "{\"metadata\":{\"test_name\":\"SimpleErrorParameterizedTest(xValue: 3, yValue: 0, expectedResult: 6)\"},\"arguments\":{\"xValue\":\"3\",\"yValue\":\"0\",\"expectedResult\":\"6\"}}"); + break; + + case "UnskippableTest": + AssertTargetSpanEqual(targetSpan, IntelligentTestRunnerTags.UnskippableTag, "true"); + AssertTargetSpanEqual(targetSpan, IntelligentTestRunnerTags.ForcedRunTag, "false"); + CheckSimpleTestSpan(targetSpan); + break; } + + // check remaining tag (only the name) + Assert.Single(targetSpan.Tags); } } catch @@ -241,6 +215,23 @@ public async Task SubmitTraces(string packageVersion) WriteSpans(spans); throw; } + + // Snapshot testing + var settings = VerifyHelper.GetCIVisibilitySpanVerifierSettings("all"); + settings.DisableRequireUniquePrefix(); + settings.UseTypeName(nameof(XUnitTests)); + await Verifier.Verify(spansCopy.OrderBy(s => s.Resource).ThenBy(s => s.Tags.GetValueOrDefault(TestTags.Parameters)), settings); + + // *************************************************************************** + // Check logs + messages = logsIntake.Logs.Select(i => i.Message).Where(m => m.StartsWith("Test:")).ToArray(); + + Assert.Contains(messages, m => m.StartsWith("Test:SimplePassTest")); + Assert.Contains(messages, m => m.StartsWith("Test:SimpleErrorTest")); + Assert.Contains(messages, m => m.StartsWith("Test:TraitPassTest")); + Assert.Contains(messages, m => m.StartsWith("Test:TraitErrorTest")); + Assert.Contains(messages, m => m.StartsWith("Test:SimpleParameterizedTest")); + Assert.Contains(messages, m => m.StartsWith("Test:SimpleErrorParameterizedTest")); } private class MockLogsIntakeForCiVisibility : MockLogsIntake diff --git a/tracer/test/Datadog.Trace.TestHelpers.AutoInstrumentation/TestHelper.cs b/tracer/test/Datadog.Trace.TestHelpers.AutoInstrumentation/TestHelper.cs index c48bfa852323..3e42c3f2a7da 100644 --- a/tracer/test/Datadog.Trace.TestHelpers.AutoInstrumentation/TestHelper.cs +++ b/tracer/test/Datadog.Trace.TestHelpers.AutoInstrumentation/TestHelper.cs @@ -94,7 +94,7 @@ public async Task StartDotnetTestSample(MockTracerAgent agent, string a Output.WriteLine($"Starting Application: {sampleAppPath} {arguments ?? string.Empty}"); string testCli = forceVsTestParam ? EnvironmentHelper.GetDotnetExe() : EnvironmentHelper.GetDotNetTest(); string exec = testCli; - string appPath = testCli.StartsWith("dotnet") || forceVsTestParam ? $"vstest {sampleAppPath}" : sampleAppPath; + string appPath = testCli.StartsWith("dotnet") || testCli.Contains("dotnet.exe") || forceVsTestParam ? $"vstest {sampleAppPath}" : sampleAppPath; Output.WriteLine("Executable: " + exec); Output.WriteLine($"ApplicationPath: {appPath} {arguments ?? string.Empty}"); var process = await ProfilerHelper.StartProcessWithProfiler( @@ -139,6 +139,10 @@ public async Task RunDotnetTestSampleAndWaitForExit(MockTracerAge { Output.WriteLine($"StandardOutput:{Environment.NewLine}{standardOutput}"); } + else + { + Output.WriteLine($"StandardOutput: (empty)"); + } var standardError = helper.ErrorOutput; @@ -146,6 +150,10 @@ public async Task RunDotnetTestSampleAndWaitForExit(MockTracerAge { Output.WriteLine($"StandardError:{Environment.NewLine}{standardError}"); } + else + { + Output.WriteLine($"StandardError: (empty)"); + } return new ProcessResult(process, standardOutput, standardError, exitCode); } diff --git a/tracer/test/test-applications/integrations/Samples.XUnitTests/Samples.XUnitTests.csproj b/tracer/test/test-applications/integrations/Samples.XUnitTests/Samples.XUnitTests.csproj index e132c51a8d48..1bf53e938448 100644 --- a/tracer/test/test-applications/integrations/Samples.XUnitTests/Samples.XUnitTests.csproj +++ b/tracer/test/test-applications/integrations/Samples.XUnitTests/Samples.XUnitTests.csproj @@ -3,7 +3,7 @@ false Library - 2.7.0 + 2.4.2 16.7.1 16.2.0 15.9.0 @@ -15,6 +15,10 @@ false true false + + true + true + PackageReference @@ -36,4 +40,16 @@ + + + + + + + + + + + + diff --git a/tracer/test/test-applications/integrations/Samples.XUnitTests/TestSuite.cs b/tracer/test/test-applications/integrations/Samples.XUnitTests/TestSuite.cs index 415104553e0d..63da2fdf018b 100644 --- a/tracer/test/test-applications/integrations/Samples.XUnitTests/TestSuite.cs +++ b/tracer/test/test-applications/integrations/Samples.XUnitTests/TestSuite.cs @@ -13,13 +13,6 @@ public class TestSuite public TestSuite(ITestOutputHelper output) { - output.WriteLine($"Pid: {Process.GetCurrentProcess().Id}"); - output.WriteLine("Environment Variables:"); - foreach (DictionaryEntry entry in Environment.GetEnvironmentVariables()) - { - output.WriteLine($" {entry.Key} = {entry.Value}"); - } - output.WriteLine(string.Empty); _output = output; }