From c43ef73ccc593ff3a65b45c3333731c7594a71fb Mon Sep 17 00:00:00 2001 From: Kevin Gosse Date: Thu, 21 Mar 2024 11:06:56 +0100 Subject: [PATCH 1/4] Force a crash when too many threads --- .../RuntimeMetrics/RuntimeMetricsWriterTests.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tracer/test/Datadog.Trace.Tests/RuntimeMetrics/RuntimeMetricsWriterTests.cs b/tracer/test/Datadog.Trace.Tests/RuntimeMetrics/RuntimeMetricsWriterTests.cs index 5a4e05cc8faa..d92119e2b27e 100644 --- a/tracer/test/Datadog.Trace.Tests/RuntimeMetrics/RuntimeMetricsWriterTests.cs +++ b/tracer/test/Datadog.Trace.Tests/RuntimeMetrics/RuntimeMetricsWriterTests.cs @@ -157,6 +157,12 @@ public async Task ShouldCaptureProcessMetrics() actualNumberOfThreads.Should().NotBeNull(); + if (actualNumberOfThreads > expectedNumberOfThreads + 100) + { + // Too many thread, try to force a memory dump + Environment.FailFast("(╯°□°)╯︵ ┻━┻"); + } + // To future generations: if 100 is not enough, feel free to bump it up. We're really just checking that the value is "realistic". actualNumberOfThreads.Should().NotBeNull().And.BeGreaterThan(0).And.BeInRange(expectedNumberOfThreads - 100, expectedNumberOfThreads + 100); From 4edd3cd6db4ff0ef14ff5fc743e7a15cbd6198e7 Mon Sep 17 00:00:00 2001 From: Kevin Gosse Date: Thu, 21 Mar 2024 11:49:15 +0100 Subject: [PATCH 2/4] Try another way --- .../RuntimeMetrics/RuntimeMetricsWriterTests.cs | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/tracer/test/Datadog.Trace.Tests/RuntimeMetrics/RuntimeMetricsWriterTests.cs b/tracer/test/Datadog.Trace.Tests/RuntimeMetrics/RuntimeMetricsWriterTests.cs index d92119e2b27e..94f21085f834 100644 --- a/tracer/test/Datadog.Trace.Tests/RuntimeMetrics/RuntimeMetricsWriterTests.cs +++ b/tracer/test/Datadog.Trace.Tests/RuntimeMetrics/RuntimeMetricsWriterTests.cs @@ -14,6 +14,7 @@ using FluentAssertions; using Moq; using Xunit; +using Xunit.Abstractions; namespace Datadog.Trace.Tests.RuntimeMetrics { @@ -21,6 +22,13 @@ namespace Datadog.Trace.Tests.RuntimeMetrics [Collection(nameof(RuntimeMetricsWriterTests))] public class RuntimeMetricsWriterTests { + private readonly ITestOutputHelper _output; + + public RuntimeMetricsWriterTests(ITestOutputHelper output) + { + _output = output; + } + [Fact] public void PushEvents() { @@ -160,7 +168,13 @@ public async Task ShouldCaptureProcessMetrics() if (actualNumberOfThreads > expectedNumberOfThreads + 100) { // Too many thread, try to force a memory dump - Environment.FailFast("(╯°□°)╯︵ ┻━┻"); + if (!MemoryDumpHelper.IsAvailable) + { + await MemoryDumpHelper.InitializeAsync(new Progress(s => _output.WriteLine(s))); + } + + var dumpResult = MemoryDumpHelper.CaptureMemoryDump(Process.GetCurrentProcess()); + Environment.FailFast(dumpResult.ToString()); } // To future generations: if 100 is not enough, feel free to bump it up. We're really just checking that the value is "realistic". From b58a103c483b2f4a3aad6e48b603ebc8b43e5f6d Mon Sep 17 00:00:00 2001 From: Kevin Gosse Date: Thu, 21 Mar 2024 14:34:37 +0100 Subject: [PATCH 3/4] Increate number of threads --- .../RuntimeMetrics/RuntimeMetricsWriterTests.cs | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/tracer/test/Datadog.Trace.Tests/RuntimeMetrics/RuntimeMetricsWriterTests.cs b/tracer/test/Datadog.Trace.Tests/RuntimeMetrics/RuntimeMetricsWriterTests.cs index 94f21085f834..db28aae2f68e 100644 --- a/tracer/test/Datadog.Trace.Tests/RuntimeMetrics/RuntimeMetricsWriterTests.cs +++ b/tracer/test/Datadog.Trace.Tests/RuntimeMetrics/RuntimeMetricsWriterTests.cs @@ -165,20 +165,8 @@ public async Task ShouldCaptureProcessMetrics() actualNumberOfThreads.Should().NotBeNull(); - if (actualNumberOfThreads > expectedNumberOfThreads + 100) - { - // Too many thread, try to force a memory dump - if (!MemoryDumpHelper.IsAvailable) - { - await MemoryDumpHelper.InitializeAsync(new Progress(s => _output.WriteLine(s))); - } - - var dumpResult = MemoryDumpHelper.CaptureMemoryDump(Process.GetCurrentProcess()); - Environment.FailFast(dumpResult.ToString()); - } - - // To future generations: if 100 is not enough, feel free to bump it up. We're really just checking that the value is "realistic". - actualNumberOfThreads.Should().NotBeNull().And.BeGreaterThan(0).And.BeInRange(expectedNumberOfThreads - 100, expectedNumberOfThreads + 100); + // A margin of 500 threads seem like a lot, but we have tests that spawn a large number of threads to try to find race conditions + actualNumberOfThreads.Should().NotBeNull().And.BeGreaterThan(0).And.BeInRange(expectedNumberOfThreads - 500, expectedNumberOfThreads + 500); // CPU time and memory usage can vary wildly, so don't try too hard to validate userCpuTime.Should().NotBeNull().And.BeGreaterThan(0); From 3e4f41ea1a71287dcf300146e352fc50c946c193 Mon Sep 17 00:00:00 2001 From: Kevin Gosse Date: Thu, 21 Mar 2024 14:35:15 +0100 Subject: [PATCH 4/4] Cleanup --- .../RuntimeMetrics/RuntimeMetricsWriterTests.cs | 8 -------- 1 file changed, 8 deletions(-) diff --git a/tracer/test/Datadog.Trace.Tests/RuntimeMetrics/RuntimeMetricsWriterTests.cs b/tracer/test/Datadog.Trace.Tests/RuntimeMetrics/RuntimeMetricsWriterTests.cs index db28aae2f68e..cacfa1cf1981 100644 --- a/tracer/test/Datadog.Trace.Tests/RuntimeMetrics/RuntimeMetricsWriterTests.cs +++ b/tracer/test/Datadog.Trace.Tests/RuntimeMetrics/RuntimeMetricsWriterTests.cs @@ -14,7 +14,6 @@ using FluentAssertions; using Moq; using Xunit; -using Xunit.Abstractions; namespace Datadog.Trace.Tests.RuntimeMetrics { @@ -22,13 +21,6 @@ namespace Datadog.Trace.Tests.RuntimeMetrics [Collection(nameof(RuntimeMetricsWriterTests))] public class RuntimeMetricsWriterTests { - private readonly ITestOutputHelper _output; - - public RuntimeMetricsWriterTests(ITestOutputHelper output) - { - _output = output; - } - [Fact] public void PushEvents() {