Config refactor - Add telemetry to otel config (#5717 -> v2)

tracer/src/Datadog.Trace.Tools.Runner/Utils.cs

@@ -17,6 +17,7 @@
 using System.Xml;
 using Datadog.Trace.Agent.DiscoveryService;
 using Datadog.Trace.Configuration;
+using Datadog.Trace.Configuration.ConfigurationSources.Telemetry;
 using Datadog.Trace.Configuration.Telemetry;
 using Datadog.Trace.Logging;
 using Datadog.Trace.Util;
@@ -405,7 +406,7 @@ public static string GetEnvironmentVariable(string key, string defaultValue = nu
             configurationSource.AddInternal(new NameValueConfigurationSource(env, ConfigurationOrigins.EnvVars));
             configurationSource.AddInternal(GlobalConfigurationSource.Instance);
 
-            var tracerSettings = new TracerSettings(configurationSource, new ConfigurationTelemetry());
+            var tracerSettings = new TracerSettings(configurationSource, new ConfigurationTelemetry(), new OverrideErrorLog());
             var settings = new ImmutableTracerSettings(tracerSettings, unusedParamNotToUsePublicApi: true);
 
             Log.Debug("Creating DiscoveryService for: {AgentUriInternal}", settings.ExporterInternal.AgentUriInternal);

tracer/src/Datadog.Trace/Ci/Configuration/CIVisibilitySettings.cs

@@ -8,6 +8,7 @@
 using System.Collections.Specialized;
 using System.Threading;
 using Datadog.Trace.Configuration;
+using Datadog.Trace.Configuration.ConfigurationSources.Telemetry;
 using Datadog.Trace.Configuration.Telemetry;
 using Datadog.Trace.Telemetry;
 using Datadog.Trace.Util;
@@ -231,7 +232,7 @@ private TracerSettings InitializeTracerSettings()
                                       },
                                       ConfigurationOrigins.Calculated));
 
-            var tracerSettings = new TracerSettings(source, new ConfigurationTelemetry());
+            var tracerSettings = new TracerSettings(source, new ConfigurationTelemetry(), new OverrideErrorLog());
 
             if (Logs)
             {

tracer/src/Datadog.Trace/Configuration/ConfigurationSources/Telemetry/ConfigurationBuilder.cs

@@ -59,32 +59,6 @@ public ConfigurationBuilder(IConfigurationSource source, IConfigurationTelemetry
 
     public HasKeys WithKeys(string key, string fallbackKey1, string fallbackKey2, string fallbackKey3) => new(_source, _telemetry, key, fallbackKey1, fallbackKey2, fallbackKey3);
 
-    private static bool TryHandleOverrides<T>(
-        ConfigurationResult<T> datadogConfigResult,
-        ConfigurationResult<T> otelConfigResult,
-        [NotNullWhen(true)] out T? value)
-    {
-        if (datadogConfigResult.IsPresent && otelConfigResult.IsPresent)
-        {
-            // TODO Log to user and report "otel.env.hiding" telemetry metric
-        }
-        else if (otelConfigResult is { IsPresent: true } config)
-        {
-            if (config is { Result: { } openTelemetryValue, IsValid: true })
-            {
-                {
-                    value = openTelemetryValue;
-                    return true;
-                }
-            }
-
-            // TODO Log to user and report "otel.env.invalid" telemetry metric
-        }
-
-        value = default;
-        return false;
-    }
-
     private static bool TryHandleResult<T>(
         IConfigurationTelemetry telemetry,
         string key,
@@ -526,19 +500,19 @@ public T WithDefault(Func<DefaultResult<T>> getDefaultValue)
             return default; // should never be invoked because we have a value for getDefaultValue
         }
 
-        public T? OverrideWith(in StructConfigurationResultWithKey<T> otelConfig)
-            => CalculateOverrides(in otelConfig, getDefaultValue: null);
+        public T? OverrideWith(in StructConfigurationResultWithKey<T> otelConfig, IConfigurationOverrideHandler overrideHandler)
+            => CalculateOverrides(in otelConfig, overrideHandler, getDefaultValue: null);
 
-        public T OverrideWith(in StructConfigurationResultWithKey<T> otelConfig, T defaultValue)
-            => CalculateOverrides(in otelConfig, getDefaultValue: () => defaultValue).Value;
+        public T OverrideWith(in StructConfigurationResultWithKey<T> otelConfig, IConfigurationOverrideHandler overrideHandler, T defaultValue)
+            => CalculateOverrides(in otelConfig, overrideHandler, getDefaultValue: () => defaultValue).Value;
 
-        public T OverrideWith(in StructConfigurationResultWithKey<T> otelConfig, Func<DefaultResult<T>> getDefaultValue)
-            => CalculateOverrides(in otelConfig, getDefaultValue).Value;
+        public T OverrideWith(in StructConfigurationResultWithKey<T> otelConfig, IConfigurationOverrideHandler overrideHandler, Func<DefaultResult<T>> getDefaultValue)
+            => CalculateOverrides(in otelConfig, overrideHandler, getDefaultValue).Value;
 
         [return: NotNullIfNotNull(nameof(getDefaultValue))]
-        private T? CalculateOverrides(in StructConfigurationResultWithKey<T> otelConfig, Func<DefaultResult<T>>? getDefaultValue)
+        private T? CalculateOverrides(in StructConfigurationResultWithKey<T> otelConfig, IConfigurationOverrideHandler overrideHandler, Func<DefaultResult<T>>? getDefaultValue)
         {
-            if (TryHandleOverrides(ConfigurationResult, otelConfig.ConfigurationResult, out var overridden))
+            if (overrideHandler.TryHandleOverrides(Key, ConfigurationResult, otelConfig.Key, otelConfig.ConfigurationResult, out var overridden))
             {
                 return overridden;
             }
@@ -574,19 +548,19 @@ public T WithDefault(Func<DefaultResult<T>> getDefaultValue)
             return default!; // should never be invoked because we have a value for getDefaultValue
         }
 
-        public T? OverrideWith(in ClassConfigurationResultWithKey<T> otelConfig)
-            => CalculateOverrides(in otelConfig, getDefaultValue: null);
+        public T? OverrideWith(in ClassConfigurationResultWithKey<T> otelConfig, IConfigurationOverrideHandler overrideHandler)
+            => CalculateOverrides(in otelConfig, overrideHandler, getDefaultValue: null);
 
-        public T OverrideWith(in ClassConfigurationResultWithKey<T> otelConfig, T defaultValue)
-            => CalculateOverrides(in otelConfig, getDefaultValue: () => defaultValue);
+        public T OverrideWith(in ClassConfigurationResultWithKey<T> otelConfig, IConfigurationOverrideHandler overrideHandler, T defaultValue)
+            => CalculateOverrides(in otelConfig, overrideHandler, getDefaultValue: () => defaultValue);
 
-        public T OverrideWith(in ClassConfigurationResultWithKey<T> otelConfig, Func<DefaultResult<T>> getDefaultValue)
-            => CalculateOverrides(in otelConfig, getDefaultValue);
+        public T OverrideWith(in ClassConfigurationResultWithKey<T> otelConfig, IConfigurationOverrideHandler overrideHandler, Func<DefaultResult<T>> getDefaultValue)
+            => CalculateOverrides(in otelConfig, overrideHandler, getDefaultValue);
 
         [return: NotNullIfNotNull(nameof(getDefaultValue))]
-        private T? CalculateOverrides(in ClassConfigurationResultWithKey<T> otelConfig, Func<DefaultResult<T>>? getDefaultValue)
+        private T? CalculateOverrides(in ClassConfigurationResultWithKey<T> otelConfig, IConfigurationOverrideHandler overrideHandler, Func<DefaultResult<T>>? getDefaultValue)
         {
-            if (TryHandleOverrides(ConfigurationResult, otelConfig.ConfigurationResult, out var overridden))
+            if (overrideHandler.TryHandleOverrides(Key, ConfigurationResult, otelConfig.Key, otelConfig.ConfigurationResult, out var overridden))
             {
                 return overridden;
             }

tracer/src/Datadog.Trace/Configuration/ConfigurationSources/Telemetry/IConfigurationOverrideHandler.cs

@@ -0,0 +1,20 @@
+// <copyright file="IConfigurationOverrideHandler.cs" company="Datadog">
+// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
+// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
+// </copyright>
+
+#nullable enable
+
+using System.Diagnostics.CodeAnalysis;
+
+namespace Datadog.Trace.Configuration.ConfigurationSources.Telemetry;
+
+internal interface IConfigurationOverrideHandler
+{
+    bool TryHandleOverrides<T>(
+        string datadogKey,
+        ConfigurationResult<T> datadogConfigResult,
+        string otelKey,
+        ConfigurationResult<T> otelConfigResult,
+        [NotNullWhen(true)] out T? value);
+}

tracer/src/Datadog.Trace/Configuration/ConfigurationSources/Telemetry/OpenTelemetryHelpers.cs

@@ -0,0 +1,72 @@
+// <copyright file="OpenTelemetryHelpers.cs" company="Datadog">
+// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
+// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
+// </copyright>
+
+#nullable enable
+
+using System;
+using Datadog.Trace.Telemetry.Metrics;
+
+namespace Datadog.Trace.Configuration.ConfigurationSources.Telemetry
+{
+    internal static class OpenTelemetryHelpers
+    {
+        public static void GetConfigurationMetricTags(
+                string openTelemetryKey,
+                out MetricTags.OpenTelemetryConfiguration openTelemetryConfig,
+                out MetricTags.DatadogConfiguration datadogConfig)
+        {
+            if (string.Equals(openTelemetryKey, "OTEL_SERVICE_NAME", StringComparison.OrdinalIgnoreCase))
+            {
+                openTelemetryConfig = MetricTags.OpenTelemetryConfiguration.ServiceName;
+                datadogConfig = MetricTags.DatadogConfiguration.Service;
+            }
+            else if (string.Equals(openTelemetryKey, "OTEL_LOG_LEVEL", StringComparison.OrdinalIgnoreCase))
+            {
+                openTelemetryConfig = MetricTags.OpenTelemetryConfiguration.LogLevel;
+                datadogConfig = MetricTags.DatadogConfiguration.DebugEnabled;
+            }
+            else if (string.Equals(openTelemetryKey, "OTEL_PROPAGATORS", StringComparison.OrdinalIgnoreCase))
+            {
+                openTelemetryConfig = MetricTags.OpenTelemetryConfiguration.Propagators;
+                datadogConfig = MetricTags.DatadogConfiguration.PropagationStyle;
+            }
+            else if (string.Equals(openTelemetryKey, "OTEL_TRACES_SAMPLER", StringComparison.OrdinalIgnoreCase))
+            {
+                openTelemetryConfig = MetricTags.OpenTelemetryConfiguration.TracesSampler;
+                datadogConfig = MetricTags.DatadogConfiguration.SampleRate;
+            }
+            else if (string.Equals(openTelemetryKey, "OTEL_TRACES_SAMPLER_ARG", StringComparison.OrdinalIgnoreCase))
+            {
+                openTelemetryConfig = MetricTags.OpenTelemetryConfiguration.TracesSamplerArg;
+                datadogConfig = MetricTags.DatadogConfiguration.SampleRate;
+            }
+            else if (string.Equals(openTelemetryKey, "OTEL_TRACES_EXPORTER", StringComparison.OrdinalIgnoreCase))
+            {
+                openTelemetryConfig = MetricTags.OpenTelemetryConfiguration.TracesExporter;
+                datadogConfig = MetricTags.DatadogConfiguration.TraceEnabled;
+            }
+            else if (string.Equals(openTelemetryKey, "OTEL_METRICS_EXPORTER", StringComparison.OrdinalIgnoreCase))
+            {
+                openTelemetryConfig = MetricTags.OpenTelemetryConfiguration.MetricsExporter;
+                datadogConfig = MetricTags.DatadogConfiguration.RuntimeMetricsEnabled;
+            }
+            else if (string.Equals(openTelemetryKey, "OTEL_RESOURCE_ATTRIBUTES", StringComparison.OrdinalIgnoreCase))
+            {
+                openTelemetryConfig = MetricTags.OpenTelemetryConfiguration.ResourceAttributes;
+                datadogConfig = MetricTags.DatadogConfiguration.Tags;
+            }
+            else if (string.Equals(openTelemetryKey, "OTEL_SDK_DISABLED", StringComparison.OrdinalIgnoreCase))
+            {
+                openTelemetryConfig = MetricTags.OpenTelemetryConfiguration.SdkDisabled;
+                datadogConfig = MetricTags.DatadogConfiguration.OpenTelemetryEnabled;
+            }
+            else
+            {
+                openTelemetryConfig = MetricTags.OpenTelemetryConfiguration.Unknown;
+                datadogConfig = MetricTags.DatadogConfiguration.Unknown;
+            }
+        }
+    }
+}

tracer/src/Datadog.Trace/Configuration/ConfigurationSources/Telemetry/OverrideErrorLog.cs

@@ -0,0 +1,137 @@
+// <copyright file="OverrideErrorLog.cs" company="Datadog">
+// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
+// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
+// </copyright>
+
+#nullable enable
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using System.Linq;
+using System.Threading;
+using Datadog.Trace.Logging;
+using Datadog.Trace.Telemetry;
+
+namespace Datadog.Trace.Configuration.ConfigurationSources.Telemetry;
+
+/// <summary>
+/// Records cases where configuration was overridden by startup telemetry,
+/// so they can be written once the tracer has been fully initialized
+/// </summary>
+internal sealed class OverrideErrorLog : IConfigurationOverrideHandler
+{
+    private readonly object _lock = new();
+    private List<Action<IDatadogLogger, IMetricsTelemetryCollector>>? _actions;
+
+    public static OverrideErrorLog Instance { get; } = new();
+
+    /// <summary>
+    /// Enqueue an action to be executed
+    /// </summary>
+    public void EnqueueAction(Action<IDatadogLogger, IMetricsTelemetryCollector> action)
+    {
+        lock (_lock)
+        {
+            _actions ??= new();
+            _actions.Add(action);
+        }
+    }
+
+    public void ProcessAndClearActions(IDatadogLogger log, IMetricsTelemetryCollector metrics)
+    {
+        List<Action<IDatadogLogger, IMetricsTelemetryCollector>>? actions;
+
+        lock (_lock)
+        {
+            actions = _actions;
+            _actions = null;
+        }
+
+        if (actions is not null)
+        {
+            foreach (var logAction in actions)
+            {
+                logAction(log, metrics);
+            }
+        }
+    }
+
+    public OverrideErrorLog Clone()
+    {
+        var clone = new OverrideErrorLog();
+        lock (_lock)
+        {
+            clone._actions = _actions?.ToList();
+        }
+
+        return clone;
+    }
+
+    public void LogDuplicateConfiguration(string datadogKey, string otelKey)
+    {
+        EnqueueAction(
+            (log, metrics) =>
+            {
+                log.Warning(
+                    "Both Datadog configuration {DatadogConfiguration} and OpenTelemetry configuration {OpenTelemetryConfiguration} are set. The Datadog configuration will be used.",
+                    datadogKey,
+                    otelKey);
+                OpenTelemetryHelpers.GetConfigurationMetricTags(otelKey, out var openTelemetryConfig, out var datadogConfig);
+                metrics.RecordCountOpenTelemetryConfigHiddenByDatadogConfig(datadogConfig, openTelemetryConfig);
+            });
+    }
+
+    public void LogInvalidConfiguration(string otelKey)
+    {
+        EnqueueAction(
+            (log, metrics) =>
+            {
+                log.Warning("OpenTelemetry configuration {OpenTelemetryConfiguration} is invalid.", otelKey);
+                OpenTelemetryHelpers.GetConfigurationMetricTags(otelKey, out var openTelemetryConfig, out var datadogConfig);
+                metrics.RecordCountOpenTelemetryConfigInvalid(datadogConfig, openTelemetryConfig);
+            });
+    }
+
+    public void LogUnsupportedConfiguration(string otelKey, string otelValue, string replacementValue)
+    {
+        EnqueueAction(
+            (log, _) =>
+            {
+                log.Warning(
+                    "OpenTelemetry configuration {OpenTelemetryConfiguration}={OpenTelemetryValue} is not supported. {ModifiedValue} will be used instead.",
+                    otelKey,
+                    otelValue,
+                    replacementValue);
+            });
+    }
+
+    bool IConfigurationOverrideHandler.TryHandleOverrides<T>(
+        string datadogKey,
+        ConfigurationResult<T> datadogConfigResult,
+        string otelKey,
+        ConfigurationResult<T> otelConfigResult,
+        [NotNullWhen(true)] out T? value)
+        where T : default
+    {
+        if (datadogConfigResult.IsPresent && otelConfigResult.IsPresent)
+        {
+            LogDuplicateConfiguration(datadogKey, otelKey);
+        }
+        else if (otelConfigResult is { IsPresent: true } config)
+        {
+            if (config is { Result: { } openTelemetryValue, IsValid: true })
+            {
+                {
+                    value = openTelemetryValue;
+                    return true;
+                }
+            }
+
+            LogInvalidConfiguration(otelKey);
+        }
+
+        value = default;
+        return false;
+    }
+}

tracer/src/Datadog.Trace/Configuration/GlobalSettings.cs

@@ -27,7 +27,11 @@ public class GlobalSettings
         /// </summary>
         /// <param name="source">The <see cref="IConfigurationSource"/> to use when retrieving configuration values.</param>
         /// <param name="telemetry">Records the origin of telemetry values</param>
-        internal GlobalSettings(IConfigurationSource source, IConfigurationTelemetry telemetry)
+        /// <param name="overrideHandler">Records any errors </param>
+        internal GlobalSettings(
+            IConfigurationSource source,
+            IConfigurationTelemetry telemetry,
+            IConfigurationOverrideHandler overrideHandler)
         {
             var builder = new ConfigurationBuilder(source, telemetry);
 
@@ -41,7 +45,7 @@ internal GlobalSettings(IConfigurationSource source, IConfigurationTelemetry tel
             DebugEnabledInternal = builder
                                   .WithKeys(ConfigurationKeys.DebugEnabled)
                                   .AsBoolResult()
-                                  .OverrideWith(in otelConfig, false);
+                                  .OverrideWith(in otelConfig, overrideHandler, false);
 
             DiagnosticSourceEnabled = builder
                                      .WithKeys(ConfigurationKeys.DiagnosticSourceEnabled)
@@ -132,6 +136,6 @@ public static GlobalSettings FromDefaultSources()
         }
 
         private static GlobalSettings CreateFromDefaultSources()
-            => new(GlobalConfigurationSource.Instance, TelemetryFactory.Config);
+            => new(GlobalConfigurationSource.Instance, TelemetryFactory.Config, OverrideErrorLog.Instance);
     }
 }