Fix memory dumps on artifact tests (#4906)

DataDog · Dec 1, 2023 · 924dc94 · 924dc94
1 parent b38c6b7
commit 924dc94
Show file tree

Hide file tree

Showing 7 changed files with 57 additions and 16 deletions.
diff --git a/.azure-pipelines/ultimate-pipeline.yml b/.azure-pipelines/ultimate-pipeline.yml
@@ -3396,6 +3396,12 @@ stages:
         DD_LOGGER_DD_API_KEY: $(ddApiKey)
         enable_crash_dumps: true
 
+    - publish: tracer/build_data
+      displayName: Uploading tool_artifacts_tests_windows logs
+      artifact: _$(System.StageName)_$(Agent.JobName)_logs_$(System.JobAttempt)
+      condition: succeededOrFailed()
+      continueOnError: true
+
     - task: PublishTestResults@2
       displayName: publish test results
       inputs:
@@ -3500,6 +3506,16 @@ stages:
         baseImage: $(baseImage)
         command: "BuildAndRunDdDotnetArtifactTests --framework net7.0"
 
+    - script: |
+        sudo chmod -R 644 tracer/build_data/dumps/* || true
+      displayName: Make dumps uploadable to AzDo
+      condition: succeededOrFailed()
+
+    - publish: tracer/build_data
+      artifact: _$(System.StageName)_$(Agent.JobName)_logs_$(System.JobAttempt)
+      condition: succeededOrFailed()
+      continueOnError: true
+
     - task: PublishTestResults@2
       displayName: publish test results
       inputs:

diff --git a/tracer/build/_build/Build.Steps.cs b/tracer/build/_build/Build.Steps.cs
@@ -2706,17 +2706,27 @@ private void CopyDumpsToBuildData()
 
     private void CopyDumpsTo(AbsolutePath root)
     {
+        var dumpFolder = root / "dumps";
+
         if (Directory.Exists(TempDirectory))
         {
             foreach (var dump in GlobFiles(TempDirectory, "coredump*", "*.dmp"))
             {
-                MoveFileToDirectory(dump, root / "dumps", FileExistsPolicy.Overwrite);
+                Logger.Information("Moving file '{Dump}' to '{Root}'", dump, dumpFolder);
+
+                MoveFileToDirectory(dump, dumpFolder, FileExistsPolicy.Overwrite);
             }
         }
 
         foreach (var file in Directory.EnumerateFiles(TracerDirectory, "*.dmp", SearchOption.AllDirectories))
         {
-            CopyFileToDirectory(file, root, FileExistsPolicy.OverwriteIfNewer);
+            if (Path.GetDirectoryName(file) == dumpFolder)
+            {
+                // The dump is already in the right location
+                continue;
+            }
+
+            CopyFileToDirectory(file, dumpFolder, FileExistsPolicy.OverwriteIfNewer);
         }
     }
 

diff --git a/tracer/test/Datadog.Trace.TestHelpers/CustomTestFramework.cs b/tracer/test/Datadog.Trace.TestHelpers/CustomTestFramework.cs
@@ -24,6 +24,20 @@ public CustomTestFramework(IMessageSink messageSink)
             : base(messageSink)
         {
             FluentAssertions.Formatting.Formatter.AddFormatter(new DiffPaneModelFormatter());
+
+            if (bool.Parse(Environment.GetEnvironmentVariable("enable_crash_dumps") ?? "false"))
+            {
+                var progress = new Progress<string>(message => messageSink.OnMessage(new DiagnosticMessage(message)));
+
+                try
+                {
+                    MemoryDumpHelper.InitializeAsync(progress).GetAwaiter().GetResult();
+                }
+                catch (Exception ex)
+                {
+                    messageSink.OnMessage(new DiagnosticMessage($"MemoryDumpHelper initialization failed: {ex}"));
+                }
+            }
         }
 
         public CustomTestFramework(IMessageSink messageSink, Type typeTestedAssembly)

diff --git a/tracer/test/Datadog.Trace.TestHelpers/MemoryDumpHelper.cs b/tracer/test/Datadog.Trace.TestHelpers/MemoryDumpHelper.cs
@@ -87,7 +87,7 @@ public static void MonitorCrashes(int pid)
             });
         }
 
-        public static bool CaptureMemoryDump(Process process)
+        public static bool CaptureMemoryDump(Process process, IProgress<string> output = null)
         {
             if (!IsAvailable)
             {
@@ -96,8 +96,8 @@ public static bool CaptureMemoryDump(Process process)
 
             try
             {
-                var args = EnvironmentTools.IsWindows() ? $"-ma {process.Id} -accepteula" : process.Id.ToString();
-                return CaptureMemoryDump(args);
+                var args = EnvironmentTools.IsWindows() ? $"-ma -accepteula {process.Id} {Path.GetTempPath()}" : process.Id.ToString();
+                return CaptureMemoryDump(args, output ?? _output);
             }
             catch (Exception ex)
             {
@@ -106,9 +106,9 @@ public static bool CaptureMemoryDump(Process process)
             }
         }
 
-        private static bool CaptureMemoryDump(string args)
+        private static bool CaptureMemoryDump(string args, IProgress<string> output)
         {
-            _output?.Report($"Capturing memory dump using '{_path} {args}'");
+            output?.Report($"Capturing memory dump using '{_path} {args}'");
 
             using var dumpToolProcess = Process.Start(new ProcessStartInfo(_path, args)
             {
@@ -121,16 +121,16 @@ private static bool CaptureMemoryDump(string args)
             using var helper = new ProcessHelper(dumpToolProcess);
             dumpToolProcess.WaitForExit(30_000);
             helper.Drain();
-            _output?.Report($"[dump][stdout] {helper.StandardOutput}");
-            _output?.Report($"[dump][stderr] {helper.ErrorOutput}");
+            output?.Report($"[dump][stdout] {helper.StandardOutput}");
+            output?.Report($"[dump][stderr] {helper.ErrorOutput}");
 
             if (dumpToolProcess.ExitCode == 0)
             {
-                _output?.Report($"Memory dump successfully captured using '{_path} {args}'.");
+                output?.Report($"Memory dump successfully captured using '{_path} {args}'.");
             }
             else
             {
-                _output?.Report($"Failed to capture memory dump using '{_path} {args}'. Exit code was {dumpToolProcess.ExitCode}.");
+                output?.Report($"Failed to capture memory dump using '{_path} {args}'. Exit code was {dumpToolProcess.ExitCode}.");
             }
 
             return true;

diff --git a/tracer/test/Datadog.Trace.Tools.dd_dotnet.ArtifactTests/Checks/ConsoleTestHelper.cs b/tracer/test/Datadog.Trace.Tools.dd_dotnet.ArtifactTests/Checks/ConsoleTestHelper.cs
@@ -87,15 +87,14 @@ protected async Task<ProcessHelper> StartConsole(EnvironmentHelper environmentHe
             return helper;
         }
 
-        helper.Dispose();
-
         if (completed == helper.Task)
         {
+            helper.Dispose();
             throw new Exception("The target process unexpectedly exited");
         }
 
         // Try to capture a memory dump before giving up
-        if (MemoryDumpHelper.CaptureMemoryDump(helper.Process))
+        if (MemoryDumpHelper.CaptureMemoryDump(helper.Process, new Progress<string>(Output.WriteLine)))
         {
             Output.WriteLine("Successfully captured a memory dump");
         }
@@ -104,6 +103,8 @@ protected async Task<ProcessHelper> StartConsole(EnvironmentHelper environmentHe
             Output.WriteLine("Failed to capture a memory dump");
         }
 
+        helper.Dispose();
+
         throw new TimeoutException("Timeout when waiting for the target process to start");
     }
 

diff --git a/tracer/test/Datadog.Trace.Tools.dd_dotnet.ArtifactTests/Checks/ProcessBasicChecksTests.cs b/tracer/test/Datadog.Trace.Tools.dd_dotnet.ArtifactTests/Checks/ProcessBasicChecksTests.cs
@@ -47,7 +47,7 @@ public ProcessBasicChecksTests(ITestOutputHelper output)
         public async Task DetectRuntime()
         {
             SkipOn.Platform(SkipOn.PlatformValue.MacOs);
-            using var helper = await StartConsole(enableProfiler: false);
+            using var helper = await StartConsole(enableProfiler: false, ("crash", "1"));
 
             var (standardOutput, errorOutput, _) = await RunTool($"check process {helper.Process.Id}");
 

diff --git a/tracer/test/Datadog.Trace.Tools.dd_dotnet.ArtifactTests/CustomTestFramework.cs b/tracer/test/Datadog.Trace.Tools.dd_dotnet.ArtifactTests/CustomTestFramework.cs
@@ -6,7 +6,7 @@
 using Xunit;
 using Xunit.Abstractions;
 
-[assembly: TestFramework("Datadog.Trace.Tools.Runner.ArtifactTests.CustomTestFramework", "Datadog.Trace.Tools.Runner.ArtifactTests")]
+[assembly: TestFramework("Datadog.Trace.Tools.dd_dotnet.ArtifactTests.CustomTestFramework", "Datadog.Trace.Tools.dd_dotnet.ArtifactTests")]
 
 namespace Datadog.Trace.Tools.dd_dotnet.ArtifactTests;