From 924dc9481d32c2661fcdfedba31f6322ff73fba9 Mon Sep 17 00:00:00 2001 From: Kevin Gosse Date: Fri, 1 Dec 2023 11:04:10 +0100 Subject: [PATCH] Fix memory dumps on artifact tests (#4906) --- .azure-pipelines/ultimate-pipeline.yml | 16 ++++++++++++++++ tracer/build/_build/Build.Steps.cs | 14 ++++++++++++-- .../CustomTestFramework.cs | 14 ++++++++++++++ .../MemoryDumpHelper.cs | 18 +++++++++--------- .../Checks/ConsoleTestHelper.cs | 7 ++++--- .../Checks/ProcessBasicChecksTests.cs | 2 +- .../CustomTestFramework.cs | 2 +- 7 files changed, 57 insertions(+), 16 deletions(-) diff --git a/.azure-pipelines/ultimate-pipeline.yml b/.azure-pipelines/ultimate-pipeline.yml index 46cc653d41fa..671199b7d9f6 100644 --- a/.azure-pipelines/ultimate-pipeline.yml +++ b/.azure-pipelines/ultimate-pipeline.yml @@ -3396,6 +3396,12 @@ stages: DD_LOGGER_DD_API_KEY: $(ddApiKey) enable_crash_dumps: true + - publish: tracer/build_data + displayName: Uploading tool_artifacts_tests_windows logs + artifact: _$(System.StageName)_$(Agent.JobName)_logs_$(System.JobAttempt) + condition: succeededOrFailed() + continueOnError: true + - task: PublishTestResults@2 displayName: publish test results inputs: @@ -3500,6 +3506,16 @@ stages: baseImage: $(baseImage) command: "BuildAndRunDdDotnetArtifactTests --framework net7.0" + - script: | + sudo chmod -R 644 tracer/build_data/dumps/* || true + displayName: Make dumps uploadable to AzDo + condition: succeededOrFailed() + + - publish: tracer/build_data + artifact: _$(System.StageName)_$(Agent.JobName)_logs_$(System.JobAttempt) + condition: succeededOrFailed() + continueOnError: true + - task: PublishTestResults@2 displayName: publish test results inputs: diff --git a/tracer/build/_build/Build.Steps.cs b/tracer/build/_build/Build.Steps.cs index 4d0634e293b5..8190f392a949 100644 --- a/tracer/build/_build/Build.Steps.cs +++ b/tracer/build/_build/Build.Steps.cs @@ -2706,17 +2706,27 @@ private void CopyDumpsToBuildData() private void CopyDumpsTo(AbsolutePath root) { + var dumpFolder = root / "dumps"; + if (Directory.Exists(TempDirectory)) { foreach (var dump in GlobFiles(TempDirectory, "coredump*", "*.dmp")) { - MoveFileToDirectory(dump, root / "dumps", FileExistsPolicy.Overwrite); + Logger.Information("Moving file '{Dump}' to '{Root}'", dump, dumpFolder); + + MoveFileToDirectory(dump, dumpFolder, FileExistsPolicy.Overwrite); } } foreach (var file in Directory.EnumerateFiles(TracerDirectory, "*.dmp", SearchOption.AllDirectories)) { - CopyFileToDirectory(file, root, FileExistsPolicy.OverwriteIfNewer); + if (Path.GetDirectoryName(file) == dumpFolder) + { + // The dump is already in the right location + continue; + } + + CopyFileToDirectory(file, dumpFolder, FileExistsPolicy.OverwriteIfNewer); } } diff --git a/tracer/test/Datadog.Trace.TestHelpers/CustomTestFramework.cs b/tracer/test/Datadog.Trace.TestHelpers/CustomTestFramework.cs index 2c8d0ff7fc0f..4086e1270270 100644 --- a/tracer/test/Datadog.Trace.TestHelpers/CustomTestFramework.cs +++ b/tracer/test/Datadog.Trace.TestHelpers/CustomTestFramework.cs @@ -24,6 +24,20 @@ public CustomTestFramework(IMessageSink messageSink) : base(messageSink) { FluentAssertions.Formatting.Formatter.AddFormatter(new DiffPaneModelFormatter()); + + if (bool.Parse(Environment.GetEnvironmentVariable("enable_crash_dumps") ?? "false")) + { + var progress = new Progress(message => messageSink.OnMessage(new DiagnosticMessage(message))); + + try + { + MemoryDumpHelper.InitializeAsync(progress).GetAwaiter().GetResult(); + } + catch (Exception ex) + { + messageSink.OnMessage(new DiagnosticMessage($"MemoryDumpHelper initialization failed: {ex}")); + } + } } public CustomTestFramework(IMessageSink messageSink, Type typeTestedAssembly) diff --git a/tracer/test/Datadog.Trace.TestHelpers/MemoryDumpHelper.cs b/tracer/test/Datadog.Trace.TestHelpers/MemoryDumpHelper.cs index 22d37f80044c..9339fba7a9fc 100644 --- a/tracer/test/Datadog.Trace.TestHelpers/MemoryDumpHelper.cs +++ b/tracer/test/Datadog.Trace.TestHelpers/MemoryDumpHelper.cs @@ -87,7 +87,7 @@ public static void MonitorCrashes(int pid) }); } - public static bool CaptureMemoryDump(Process process) + public static bool CaptureMemoryDump(Process process, IProgress output = null) { if (!IsAvailable) { @@ -96,8 +96,8 @@ public static bool CaptureMemoryDump(Process process) try { - var args = EnvironmentTools.IsWindows() ? $"-ma {process.Id} -accepteula" : process.Id.ToString(); - return CaptureMemoryDump(args); + var args = EnvironmentTools.IsWindows() ? $"-ma -accepteula {process.Id} {Path.GetTempPath()}" : process.Id.ToString(); + return CaptureMemoryDump(args, output ?? _output); } catch (Exception ex) { @@ -106,9 +106,9 @@ public static bool CaptureMemoryDump(Process process) } } - private static bool CaptureMemoryDump(string args) + private static bool CaptureMemoryDump(string args, IProgress output) { - _output?.Report($"Capturing memory dump using '{_path} {args}'"); + output?.Report($"Capturing memory dump using '{_path} {args}'"); using var dumpToolProcess = Process.Start(new ProcessStartInfo(_path, args) { @@ -121,16 +121,16 @@ private static bool CaptureMemoryDump(string args) using var helper = new ProcessHelper(dumpToolProcess); dumpToolProcess.WaitForExit(30_000); helper.Drain(); - _output?.Report($"[dump][stdout] {helper.StandardOutput}"); - _output?.Report($"[dump][stderr] {helper.ErrorOutput}"); + output?.Report($"[dump][stdout] {helper.StandardOutput}"); + output?.Report($"[dump][stderr] {helper.ErrorOutput}"); if (dumpToolProcess.ExitCode == 0) { - _output?.Report($"Memory dump successfully captured using '{_path} {args}'."); + output?.Report($"Memory dump successfully captured using '{_path} {args}'."); } else { - _output?.Report($"Failed to capture memory dump using '{_path} {args}'. Exit code was {dumpToolProcess.ExitCode}."); + output?.Report($"Failed to capture memory dump using '{_path} {args}'. Exit code was {dumpToolProcess.ExitCode}."); } return true; diff --git a/tracer/test/Datadog.Trace.Tools.dd_dotnet.ArtifactTests/Checks/ConsoleTestHelper.cs b/tracer/test/Datadog.Trace.Tools.dd_dotnet.ArtifactTests/Checks/ConsoleTestHelper.cs index 169f9aa18ae5..1cd7ce72b250 100644 --- a/tracer/test/Datadog.Trace.Tools.dd_dotnet.ArtifactTests/Checks/ConsoleTestHelper.cs +++ b/tracer/test/Datadog.Trace.Tools.dd_dotnet.ArtifactTests/Checks/ConsoleTestHelper.cs @@ -87,15 +87,14 @@ protected async Task StartConsole(EnvironmentHelper environmentHe return helper; } - helper.Dispose(); - if (completed == helper.Task) { + helper.Dispose(); throw new Exception("The target process unexpectedly exited"); } // Try to capture a memory dump before giving up - if (MemoryDumpHelper.CaptureMemoryDump(helper.Process)) + if (MemoryDumpHelper.CaptureMemoryDump(helper.Process, new Progress(Output.WriteLine))) { Output.WriteLine("Successfully captured a memory dump"); } @@ -104,6 +103,8 @@ protected async Task StartConsole(EnvironmentHelper environmentHe Output.WriteLine("Failed to capture a memory dump"); } + helper.Dispose(); + throw new TimeoutException("Timeout when waiting for the target process to start"); } diff --git a/tracer/test/Datadog.Trace.Tools.dd_dotnet.ArtifactTests/Checks/ProcessBasicChecksTests.cs b/tracer/test/Datadog.Trace.Tools.dd_dotnet.ArtifactTests/Checks/ProcessBasicChecksTests.cs index 087b7cc54cc4..1245359d1b77 100644 --- a/tracer/test/Datadog.Trace.Tools.dd_dotnet.ArtifactTests/Checks/ProcessBasicChecksTests.cs +++ b/tracer/test/Datadog.Trace.Tools.dd_dotnet.ArtifactTests/Checks/ProcessBasicChecksTests.cs @@ -47,7 +47,7 @@ public ProcessBasicChecksTests(ITestOutputHelper output) public async Task DetectRuntime() { SkipOn.Platform(SkipOn.PlatformValue.MacOs); - using var helper = await StartConsole(enableProfiler: false); + using var helper = await StartConsole(enableProfiler: false, ("crash", "1")); var (standardOutput, errorOutput, _) = await RunTool($"check process {helper.Process.Id}"); diff --git a/tracer/test/Datadog.Trace.Tools.dd_dotnet.ArtifactTests/CustomTestFramework.cs b/tracer/test/Datadog.Trace.Tools.dd_dotnet.ArtifactTests/CustomTestFramework.cs index 00379e71f0d3..c0d7f9dee4a0 100644 --- a/tracer/test/Datadog.Trace.Tools.dd_dotnet.ArtifactTests/CustomTestFramework.cs +++ b/tracer/test/Datadog.Trace.Tools.dd_dotnet.ArtifactTests/CustomTestFramework.cs @@ -6,7 +6,7 @@ using Xunit; using Xunit.Abstractions; -[assembly: TestFramework("Datadog.Trace.Tools.Runner.ArtifactTests.CustomTestFramework", "Datadog.Trace.Tools.Runner.ArtifactTests")] +[assembly: TestFramework("Datadog.Trace.Tools.dd_dotnet.ArtifactTests.CustomTestFramework", "Datadog.Trace.Tools.dd_dotnet.ArtifactTests")] namespace Datadog.Trace.Tools.dd_dotnet.ArtifactTests;