Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add attribute for skipping tests in CI without using [Fact(Skip = "")] #5756

Merged
merged 2 commits into from
Jul 2, 2024
Merged

Add attribute for skipping tests in CI without using [Fact(Skip = "")] #5756

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0ad27c5
Replace explicit skips with soft skips for non-flake
andrewlock Jul 1, 2024
804306a
Change default filters to all include (SkipInCI!=True)
andrewlock Jul 1, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions tracer/build/_build/Build.Steps.Debugger.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,9 +125,9 @@ string GetTestFilter()
{
var filter = (IsWin, IsArm64) switch
{
(true, _) => "(RunOnWindows=True)",
(_, true) => "(Category!=ArmUnsupported)",
_ => "(Category!=LinuxUnsupported)",
(true, _) => "(RunOnWindows=True)&(SkipInCI!=True)",
(_, true) => "(Category!=ArmUnsupported)&(SkipInCI!=True)",
_ => "(Category!=LinuxUnsupported)&(SkipInCI!=True)",
};

return Filter is null ? filter : $"{Filter}&{filter}";
Expand Down
16 changes: 8 additions & 8 deletions tracer/build/_build/Build.Steps.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1009,7 +1009,7 @@ async Task DownloadWafVersion(string libddwafVersion = null, string uncompressFo
.ToList();

testProjects.ForEach(EnsureResultsDirectory);
var filter = string.IsNullOrWhiteSpace(Filter) && IsArm64 ? "(Category!=ArmUnsupported)&(Category!=AzureFunctions)" : Filter;
var filter = string.IsNullOrWhiteSpace(Filter) && IsArm64 ? "(Category!=ArmUnsupported)&(Category!=AzureFunctions)&(SkipInCI!=True)" : Filter;
var exceptions = new List<Exception>();
try
{
Expand Down Expand Up @@ -1424,7 +1424,7 @@ _ when exclude.Contains(project.Path) => false,
//.WithMemoryDumpAfter(timeoutInMinutes: 30)
.EnableNoRestore()
.EnableNoBuild()
.SetFilter(string.IsNullOrWhiteSpace(Filter) ? "RunOnWindows=True&LoadFromGAC!=True&IIS!=True&Category!=AzureFunctions" : Filter)
.SetFilter(string.IsNullOrWhiteSpace(Filter) ? "(RunOnWindows=True)&(LoadFromGAC!=True)&(IIS!=True)&(Category!=AzureFunctions)&(SkipInCI!=True)" : Filter)
.SetTestTargetPlatform(TargetPlatform)
.SetIsDebugRun(isDebugRun)
.SetProcessEnvironmentVariable("MonitoringHomeDirectory", MonitoringHomeDirectory)
Expand Down Expand Up @@ -1494,7 +1494,7 @@ _ when exclude.Contains(project.Path) => false,
//.WithMemoryDumpAfter(timeoutInMinutes: 30)
.EnableNoRestore()
.EnableNoBuild()
.SetFilter(string.IsNullOrWhiteSpace(Filter) ? "RunOnWindows=True&Category=AzureFunctions" : Filter)
.SetFilter(string.IsNullOrWhiteSpace(Filter) ? "(RunOnWindows=True)&(Category=AzureFunctions)&(SkipInCI!=True)" : Filter)
.SetTestTargetPlatform(TargetPlatform)
.SetIsDebugRun(isDebugRun)
.SetProcessEnvironmentVariable("MonitoringHomeDirectory", MonitoringHomeDirectory)
Expand Down Expand Up @@ -1535,7 +1535,7 @@ _ when exclude.Contains(project.Path) => false,
.EnableCrashDumps()
.EnableNoRestore()
.EnableNoBuild()
.SetFilter(string.IsNullOrWhiteSpace(Filter) ? "Category=Smoke&LoadFromGAC!=True&Category!=AzureFunctions" : Filter)
.SetFilter(string.IsNullOrWhiteSpace(Filter) ? "(Category=Smoke)&(LoadFromGAC!=True)&(Category!=AzureFunctions)&(SkipInCI!=True)" : Filter)
.SetTestTargetPlatform(TargetPlatform)
.SetIsDebugRun(isDebugRun)
.SetProcessEnvironmentVariable("MonitoringHomeDirectory", MonitoringHomeDirectory)
Expand Down Expand Up @@ -1587,7 +1587,7 @@ async Task RunWindowsIisIntegrationTests(Project project)
.SetFramework(Framework)
.EnableNoRestore()
.EnableNoBuild()
.SetFilter(string.IsNullOrWhiteSpace(Filter) ? "(RunOnWindows=True)&LoadFromGAC=True&Category!=AzureFunctions" : Filter)
.SetFilter(string.IsNullOrWhiteSpace(Filter) ? "(RunOnWindows=True)&(LoadFromGAC=True)&(Category!=AzureFunctions)&(SkipInCI!=True)" : Filter)
.SetTestTargetPlatform(TargetPlatform)
.SetIsDebugRun(isDebugRun)
.SetProcessEnvironmentVariable("MonitoringHomeDirectory", MonitoringHomeDirectory)
Expand Down Expand Up @@ -1626,7 +1626,7 @@ async Task RunWindowsIisIntegrationTests(Project project)
.SetFramework(Framework)
.EnableNoRestore()
.EnableNoBuild()
.SetFilter(string.IsNullOrWhiteSpace(Filter) ? "(RunOnWindows=True)&MSI=True&Category!=AzureFunctions" : Filter)
.SetFilter(string.IsNullOrWhiteSpace(Filter) ? "(RunOnWindows=True)&(MSI=True)&(Category!=AzureFunctions)&(SkipInCI!=True)" : Filter)
.SetTestTargetPlatform(TargetPlatform)
.SetIsDebugRun(isDebugRun)
.SetProcessEnvironmentVariable("MonitoringHomeDirectory", MonitoringHomeDirectory)
Expand Down Expand Up @@ -1927,7 +1927,7 @@ var name when multiPackageProjects.Contains(name) => false,
var filter = string.IsNullOrWhiteSpace(Filter) switch
{
false => $"({Filter}){dockerFilter}{armFilter}",
true => $"(Category!=LinuxUnsupported)&(Category!=Lambda)&(Category!=AzureFunctions){dockerFilter}{armFilter}",
true => $"(Category!=LinuxUnsupported)&(Category!=Lambda)&(Category!=AzureFunctions)&(SkipInCI!=True){dockerFilter}{armFilter}",
};

try
Expand Down Expand Up @@ -2008,7 +2008,7 @@ var name when multiPackageProjects.Contains(name) => false,
var filter = string.IsNullOrWhiteSpace(Filter) switch
{
false => Filter,
true => $"(Category!=LinuxUnsupported)&(Category!=Lambda)&(Category!=AzureFunctions){dockerFilter}{armFilter}",
true => $"(Category!=LinuxUnsupported)&(Category!=Lambda)&(Category!=AzureFunctions)&(SkipInCI!=True){dockerFilter}{armFilter}",
};

var targetPlatform = IsArm64 ? (MSBuildTargetPlatform)"arm64" : TargetPlatform;
Expand Down
3 changes: 2 additions & 1 deletion tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AspNet/AspNetWebFormsTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,10 +63,11 @@ await AssertAspNetSpanOnly(
"1.0.0");
}

[Fact(Skip = "This test requires Elasticsearch to be running on the host, which is not currently enabled in CI.")]
[Fact]
[Trait("Category", "EndToEnd")]
[Trait("RunOnWindows", "True")]
[Trait("LoadFromGAC", "True")]
[Trait("SkipInCI", "True")] // This test requires Elasticsearch to be running on the host, which is not currently enabled in CI.
public async Task NestedAsyncElasticCallSubmitsTrace()
{
var testStart = DateTime.UtcNow;
Expand Down
3 changes: 2 additions & 1 deletion tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/Azure/AzureServiceBusTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,9 +41,10 @@ public static IEnumerable<object[]> GetEnabledConfig()
_ => throw new ArgumentException($"span.Tags[\"span.kind\"] is not a supported value for the AWS SQS integration: {span.Tags["span.kind"]}", nameof(span)),
};

[SkippableTheory(Skip = "We are unable to test all the features of Azure Service Bus with an emulator. For now, run only locally with a connection string to a live Azure Service Bus namespace")]
[SkippableTheory]
[MemberData(nameof(GetEnabledConfig))]
[Trait("Category", "EndToEnd")]
[Trait("SkipInCI", "True")] // We are unable to test all the features of Azure Service Bus with an emulator. For now, run only locally with a connection string to a live Azure Service Bus namespace
public async Task SubmitsTraces(string packageVersion, string metadataSchemaVersion)
{
SetEnvironmentVariable("DD_TRACE_SPAN_ATTRIBUTE_SCHEMA", metadataSchemaVersion);
Expand Down
6 changes: 4 additions & 2 deletions ...dog.Trace.ClrProfiler.IntegrationTests/Azure/DataStreamsMonitoringAzureServiceBusTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,10 @@ public static IEnumerable<object[]> GetPackageVersions()
=> from packageVersionArray in new string[] { string.Empty }
select new string[] { packageVersionArray };

[SkippableTheory(Skip = "This has only been tested on a live Azure Service Bus namespace using a connection string. Unskip this if you'd like to run locally or if you've correctly configured piotr-rojek/devopsifyme-sbemulator in CI")]
[SkippableTheory]
[MemberData(nameof(GetPackageVersions))]
[Trait("Category", "EndToEnd")]
[Trait("SkipInCI", "True")] // "This has only been tested on a live Azure Service Bus namespace using a connection string. Unskip this if you'd like to run locally or if you've correctly configured piotr-rojek/devopsifyme-sbemulator in CI"
public async Task HandleProduceAndConsume(string packageVersion)
{
SetEnvironmentVariable(ConfigurationKeys.DataStreamsMonitoring.Enabled, "1");
Expand All @@ -59,9 +60,10 @@ await Verifier.Verify(PayloadsToPoints(agent.DataStreams), settings)
}
}

[SkippableTheory(Skip = "This has only been tested on a live Azure Service Bus namespace using a connection string. Unskip this if you'd like to run locally or if you've correctly configured piotr-rojek/devopsifyme-sbemulator in CI")]
[SkippableTheory]
[MemberData(nameof(GetPackageVersions))]
[Trait("Category", "EndToEnd")]
[Trait("SkipInCI", "True")] // This has only been tested on a live Azure Service Bus namespace using a connection string. Unskip this if you'd like to run locally or if you've correctly configured piotr-rojek/devopsifyme-sbemulator in CI
public async Task ValidateSpanTags(string packageVersion)
{
SetEnvironmentVariable(ConfigurationKeys.DataStreamsMonitoring.Enabled, "1");
Expand Down
3 changes: 2 additions & 1 deletion tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CosmosTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,12 +34,13 @@ public static IEnumerable<object[]> GetEnabledConfig()

public override Result ValidateIntegrationSpan(MockSpan span, string metadataSchemaVersion) => span.IsCosmosDb(metadataSchemaVersion);

[SkippableTheory(Skip = "Cosmos emulator is too flaky at the moment")]
[SkippableTheory]
[MemberData(nameof(GetEnabledConfig))]
[Trait("Category", "EndToEnd")]
[Trait("RunOnWindows", "True")]
[Trait("Category", "LinuxUnsupported")]
[Trait("Category", "ArmUnsupported")]
[Trait("SkipInCI", "True")] // Cosmos emulator is too flaky in CI at the moment
public async Task SubmitTraces(string packageVersion, string metadataSchemaVersion)
{
var expectedSpanCount = 14;
Expand Down
3 changes: 2 additions & 1 deletion ...r.IntegrationTests/SmokeTests/StackExchangeRedisAssemblyConflictLegacyProjectSmokeTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,9 @@ public StackExchangeRedisAssemblyConflictLegacyProjectSmokeTest(ITestOutputHelpe
{
}

[Fact(Skip = ".NET Framework test, but cannot run on Windows because it requires Redis")]
[Fact]
[Trait("Category", "Smoke")]
[Trait("SkipInCI", "True")] // .NET Framework test, but cannot run on Windows in CI because it requires Redis
public async Task NoExceptions()
{
await CheckForSmoke(shouldDeserializeTraces: false);
Expand Down
3 changes: 2 additions & 1 deletion tracer/test/Datadog.Trace.Security.IntegrationTests/AspNetCore2RateLimiter.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ public override void Dispose()
this.fixture.SetOutput(null);
}

[SkippableTheory(Skip = "Don't run in CI as test is slow, can be run manually by removing this attribute")]
[SkippableTheory]
[InlineData(true, 90, 100)]
[InlineData(false, 90, 100)]
[InlineData(true, 110, 100)]
Expand All @@ -43,6 +43,7 @@ public override void Dispose()
[InlineData(false, 30, 20)]
[Trait("RunOnWindows", "True")]
[Trait("Category", "ArmUnsupported")]
[Trait("SkipInCI", "True")] // Don't run in CI as test is slow, can be run manually by removing this attribute
public async Task TestRateLimiterSecurity(bool enableSecurity, int totalRequests, int traceRateLimit, string url = DefaultAttackUrl)
{
EnvironmentHelper.CustomEnvironmentVariables.Add("DD_APPSEC_TRACE_RATE_LIMIT", traceRateLimit.ToString());
Expand Down
3 changes: 2 additions & 1 deletion tracer/test/Datadog.Trace.Security.IntegrationTests/AspNetCore5RateLimiter.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ public override void Dispose()
this.fixture.SetOutput(null);
}

[SkippableTheory(Skip = "Don't run in CI as test is slow, can be run manually by removing this attribute")]
[SkippableTheory]
[InlineData(true, 90, 100)]
[InlineData(false, 90, 100)]
[InlineData(true, 110, 100)]
Expand All @@ -38,6 +38,7 @@ public override void Dispose()
[InlineData(false, 30, 20)]
[Trait("RunOnWindows", "True")]
[Trait("Category", "ArmUnsupported")]
[Trait("SkipInCI", "True")] // Don't run in CI as test is slow, can be run manually by removing this attribute
public async Task TestRateLimiterSecurity(bool enableSecurity, int totalRequests, int? traceRateLimit, string url = DefaultAttackUrl)
{
EnvironmentHelper.CustomEnvironmentVariables.Add("DD_APPSEC_TRACE_RATE_LIMIT", traceRateLimit.ToString());
Expand Down
3 changes: 2 additions & 1 deletion tracer/test/Datadog.Trace.Security.IntegrationTests/AspNetMvc5RateLimiter.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,10 +112,11 @@ public AspNetMvc5RateLimiter(IisFixture iisFixture, ITestOutputHelper output, bo
}
}

[SkippableTheory]
[Trait("Category", "EndToEnd")]
[Trait("RunOnWindows", "True")]
[Trait("LoadFromGAC", "True")]
[SkippableTheory(Skip = "Don't run in CI as test is slow, can be run manually by removing this attribute")]
[Trait("SkipInCI", "True")] // Don't run in CI as test is slow, can be run manually by removing this attribute
[InlineData(110, DefaultAttackUrl)]
[InlineData(30, DefaultAttackUrl)]
public async Task TestRateLimiterSecurity(int totalRequests, string url = DefaultAttackUrl)
Expand Down
Loading