[Tool] update continuous profiler diagnostics (#6014)

## Summary of changes

Fix the diagnostics associated with the expected state of the continuous
profiler

## Reason for change

In #5240, the continuous profiler gained two new "modes" of operation:
delayed enablement based on heuristics, and "monitoring" mode. The tool
should be updated to understand those possibilities + handle it in the
tests

## Implementation details

- Reset the `DD_INJECTION_ENABLED` variable before running the tests -
the presence/absence of the variable changes the expectations, so we
need to account for it
- Explicitly test the various combinations of `DD_INJECTION_ENABLED` and
`DD_PROFILING_ENABLED`.

## Test coverage

Added additional integration and artifact tests

## Other details

This should address most of the issues in the scheduled SSI run. I
suspect the windows integration tests _may_ need additional work, as
currently the profiler library isn't copied to those tests. Will fix in
this PR if that ends up being the case

Supersedes 
- #5991

---------

Co-authored-by: chrisnas <chrisnas@users.noreply.github.com>

tracer/src/Datadog.Trace.Tools.dd_dotnet/Checks/ProcessBasicCheck.cs

@@ -258,7 +258,7 @@ public static bool Run(ProcessInfo process, IRegistryService? registryService =
                     {
                         if (profilingEnabled == "auto")
                         {
-                            Utils.WriteInfo(ContinuousProfilerSsiDeployed);
+                            Utils.WriteInfo(ContinuousProfilerEnabledWithHeuristics);
                             isContinuousProfilerEnabled = true;
                         }
                         else
@@ -270,10 +270,16 @@ public static bool Run(ProcessInfo process, IRegistryService? registryService =
                 }
                 else
                 {
-                    isContinuousProfilerEnabled = process.EnvironmentVariables.TryGetValue("DD_INJECTION_ENABLED", out var _);
-                    if (isContinuousProfilerEnabled)
+                    if (process.EnvironmentVariables.TryGetValue("DD_INJECTION_ENABLED", out var injectionEnabled))
                     {
-                        Utils.WriteInfo(ContinuousProfilerSsiDeployed);
+                        if (injectionEnabled.Contains("profiler", StringComparison.OrdinalIgnoreCase))
+                        {
+                            Utils.WriteInfo(ContinuousProfilerSsiEnabledWithHeuristics);
+                        }
+                        else
+                        {
+                            Utils.WriteInfo(ContinuousProfilerSsiMonitoring);
+                        }
                     }
                     else
                     {

tracer/src/Datadog.Trace.Tools.dd_dotnet/Checks/Resources.cs

@@ -46,9 +46,11 @@ internal static class Resources
         public const string DdAgentChecks = "---- DATADOG AGENT CHECKS -----";
 
         public const string ContinuousProfilerEnabled = "DD_PROFILING_ENABLED is set.";
-        public const string ContinuousProfilerSsiDeployed = "DD_INJECTION_ENABLED is set. Profiler is deployed through SSI.";
+        public const string ContinuousProfilerEnabledWithHeuristics = "DD_PROFILING_ENABLED is set to 'auto'. The continuous profiler is enabled and may begin profiling based on heuristics.";
+        public const string ContinuousProfilerSsiEnabledWithHeuristics = "DD_INJECTION_ENABLED contains 'profiler'. The continuous profiler is enabled through SSI and may begin profiling based on heuristics.";
+        public const string ContinuousProfilerSsiMonitoring = "DD_INJECTION_ENABLED is set but does not contain 'profiler'. The continuous profiler is monitoring but will not generate profiles.";
         public const string ContinuousProfilerDisabled = "The continuous profiler is explicitly disabled through DD_PROFILING_ENABLED.";
-        public const string ContinuousProfilerNotSet = "DD_PROFILING_ENABLED is not set, the continuous profiler is disabled.";
+        public const string ContinuousProfilerNotSet = "DD_INJECTION_ENABLED and DD_PROFILING_ENABLED are not set, the continuous profiler is disabled.";
         public const string ContinuousProfilerNotLoaded = "The continuous profiler library is not loaded into the process.";
         public const string ContinuousProfilerWithoutLoader = "The continuous profiler needs the Datadog.Trace.ClrProfiler.Native module and the loader.conf file to work. Try reinstalling the tracer in version 2.14+.";
 

tracer/test/Datadog.Trace.Tools.dd_dotnet.ArtifactTests/Checks/ProcessBasicChecksTests.cs

@@ -4,7 +4,9 @@
 // </copyright>
 
 using System;
+using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 using System.Runtime.InteropServices;
 using System.Threading.Tasks;
 using Datadog.Trace.TestHelpers;
@@ -18,6 +20,8 @@
 
 namespace Datadog.Trace.Tools.dd_dotnet.ArtifactTests.Checks
 {
+    // Some of these tests use SSI variables, so we have to explicitly reset them
+    [EnvironmentRestorer("DD_INJECTION_ENABLED")]
     public class ProcessBasicChecksTests : ConsoleTestHelper
     {
 #if NETFRAMEWORK
@@ -194,17 +198,48 @@ public async Task Working()
             exitCode.Should().Be(0);
         }
 
-        [SkippableFact]
-        public async Task WorkingWithContinuousProfiler()
+        [SkippableTheory]
+        [InlineData("auto", null)]
+        [InlineData("1", null)]
+        [InlineData("0", null)]
+        [InlineData(null, null)]
+        [InlineData("auto", false)]
+        [InlineData("1", false)]
+        [InlineData("0", false)]
+        [InlineData(null, false)]
+        [InlineData("auto", true)]
+        [InlineData("1", true)]
+        [InlineData("0", true)]
+        [InlineData(null, true)]
+        public async Task WorkingWithContinuousProfiler(string enabled, bool? ssiInjectionEnabled)
         {
             SkipOn.Platform(SkipOn.PlatformValue.MacOs);
 
+            var ssiInjection = ssiInjectionEnabled is null
+                                   ? null
+                                   : ssiInjectionEnabled == true ? "tracer,profiler" : "tracer";
+
+            var expected = (enabled, ssiInjectionEnabled) switch
+            {
+                ("0", _) => ContinuousProfilerDisabled,
+                ("1", _) => ContinuousProfilerEnabled,
+                ("auto", _) => ContinuousProfilerEnabledWithHeuristics,
+                (null, null) => ContinuousProfilerNotSet,
+                (null, true) => ContinuousProfilerSsiEnabledWithHeuristics,
+                (null, false) => ContinuousProfilerSsiMonitoring,
+                _ => throw new InvalidOperationException("Unexpected test combination"),
+            };
+
             var apiWrapperPath = Utils.GetApiWrapperPath();
+            (string, string)[] envVars = (enabled, ssiInjection) switch
+            {
+                (null, null) => [("LD_PRELOAD", apiWrapperPath)],
+                ({ } prof, null) => [("LD_PRELOAD", apiWrapperPath), ("DD_PROFILING_ENABLED", prof)],
+                (null, { } ssi) => [("LD_PRELOAD", apiWrapperPath), ("DD_INJECTION_ENABLED", ssi)],
+                ({ } prof, { } ssi) => [("LD_PRELOAD", apiWrapperPath), ("DD_PROFILING_ENABLED", prof), ("DD_INJECTION_ENABLED", ssi)],
+            };
 
-            using var helper = await StartConsole(
-                                   enableProfiler: true,
-                                   ("DD_PROFILING_ENABLED", "1"),
-                                   ("LD_PRELOAD", apiWrapperPath));
+            using var helper = await StartConsole(enableProfiler: true, envVars);
 
             var (standardOutput, errorOutput, exitCode) = await RunTool($"check process {helper.Process.Id}");
 
@@ -215,19 +250,34 @@ public async Task WorkingWithContinuousProfiler()
 
             standardOutput.Should().ContainAll(
                 TracerVersion(TracerConstants.AssemblyVersion),
-                ContinuousProfilerEnabled,
+                expected,
                 CorrectlySetupEnvironment(CorProfilerKey, Profilerid),
                 CorrectlySetupEnvironment(CorEnableKey, "1"));
 
             standardOutput.Replace(" ", string.Empty).Should().Contain(CorrectlySetupEnvironment(CorProfilerPathKey, ProfilerPath).Replace(" ", string.Empty));
 
-            standardOutput.Should().NotContainAny(
-                NativeTracerNotLoaded,
-                TracerNotLoaded,
+            // All of the possible messages about Continuous profiler state
+            var profilerMessages = new[]
+            {
                 ContinuousProfilerNotSet,
-                ContinuousProfilerNotLoaded,
-                "LD_PRELOAD",
-                TracerHomeNotFoundFormat("DD_DOTNET_TRACER_HOME"));
+                ContinuousProfilerEnabled,
+                ContinuousProfilerEnabledWithHeuristics,
+                ContinuousProfilerDisabled,
+                ContinuousProfilerSsiEnabledWithHeuristics,
+                ContinuousProfilerSsiMonitoring
+            };
+
+            standardOutput.Should()
+                          .NotContainAny(
+                               profilerMessages
+                                  .Concat(
+                                   [
+                                       NativeTracerNotLoaded,
+                                       TracerNotLoaded,
+                                       "LD_PRELOAD",
+                                       TracerHomeNotFoundFormat("DD_DOTNET_TRACER_HOME")
+                                   ])
+                                  .Except([expected]));
 
             errorOutput.Should().BeEmpty();
             exitCode.Should().Be(0);

tracer/test/Datadog.Trace.Tools.dd_dotnet.IntegrationTests/Checks/ProcessBasicChecksTests.cs

@@ -23,6 +23,8 @@ namespace Datadog.Trace.Tools.dd_dotnet.IntegrationTests.Checks;
 
 #pragma warning disable CS0436 // Type conflicts with imported type
 
+// Some of these tests use SSI variables, so we have to explicitly reset them
+[EnvironmentRestorer("DD_INJECTION_ENABLED")]
 [Collection(nameof(ConsoleTestsCollection))]
 public class ProcessBasicChecksTests : ConsoleTestHelper
 {
@@ -331,16 +333,46 @@ public async Task LdPreloadNotFound()
     }
 
     [SkippableTheory]
-    [InlineData(true)]
-    [InlineData(false)]
-    [InlineData(null)]
-    public async Task DetectContinousProfilerState(bool? enabled)
+    [InlineData("auto", null)]
+    [InlineData("1", null)]
+    [InlineData("0", null)]
+    [InlineData(null, null)]
+    [InlineData("auto", false)]
+    [InlineData("1", false)]
+    [InlineData("0", false)]
+    [InlineData(null, false)]
+    [InlineData("auto", true)]
+    [InlineData("1", true)]
+    [InlineData("0", true)]
+    [InlineData(null, true)]
+    public async Task DetectContinuousProfilerState(string enabled, bool? ssiInjectionEnabled)
     {
         SkipOn.Platform(SkipOn.PlatformValue.MacOs);
-        var environmentVariables = enabled == null ? Array.Empty<(string, string)>()
-            : new[] { ("DD_PROFILING_ENABLED", enabled == true ? "1" : "0") };
+        var ssiInjection = ssiInjectionEnabled is null
+                               ? null
+                               : ssiInjectionEnabled == true ? "tracer,profiler" : "tracer";
+
+        var expected = (enabled, ssiInjectionEnabled) switch
+        {
+            ("0", _) => ContinuousProfilerDisabled,
+            ("1", _) => ContinuousProfilerEnabled,
+            ("auto", _) => ContinuousProfilerEnabledWithHeuristics,
+            (null, null) => ContinuousProfilerNotSet,
+            (null, true) => ContinuousProfilerSsiEnabledWithHeuristics,
+            (null, false) => ContinuousProfilerSsiMonitoring,
+            _ => throw new InvalidOperationException("Unexpected test combination"),
+        };
+
+        (string, string)[] envVars = (enabled, ssiInjection) switch
+        {
+            (null, null) => [],
+            ({ } prof, null) => [("DD_PROFILING_ENABLED", prof)],
+            (null, { } ssi) => [("DD_INJECTION_ENABLED", ssi)],
+            ({ } prof, { } ssi) => [("DD_PROFILING_ENABLED", prof), ("DD_INJECTION_ENABLED", ssi)],
+        };
+
+        using var helper = await StartConsole(enableProfiler: true, envVars);
 
-        using var helper = await StartConsole(enableProfiler: true, environmentVariables);
         var processInfo = ProcessInfo.GetProcessInfo(helper.Process.Id);
 
         processInfo.Should().NotBeNull();
@@ -352,18 +384,7 @@ public async Task DetectContinousProfilerState(bool? enabled)
         using var scope = new AssertionScope();
         scope.AddReportable("Output", console.Output);
 
-        if (enabled == null)
-        {
-            console.Output.Should().Contain(ContinuousProfilerNotSet);
-        }
-        else if (enabled == true)
-        {
-            console.Output.Should().Contain(ContinuousProfilerEnabled);
-        }
-        else
-        {
-            console.Output.Should().Contain(ContinuousProfilerDisabled);
-        }
+        console.Output.Should().Contain(expected);
     }
 
     [SkippableFact]