[sdk-metrics] Support setting exemplar filter from spec envvar key (#…

…5412)

src/OpenTelemetry/CHANGELOG.md

@@ -70,6 +70,13 @@
   Specification](https://github.com/open-telemetry/opentelemetry-specification/pull/3820).
   ([#5404](https://github.com/open-telemetry/opentelemetry-dotnet/pull/5404))
 
+* **Experimental (pre-release builds only):** The `ExemplarFilter` used by SDK
+  `MeterProvider`s can now be controlled via the `OTEL_METRICS_EXEMPLAR_FILTER`
+  environment variable. The supported values are: `always_off`, `always_on`, and
+  `trace_based`. For details see: [OpenTelemetry Environment Variable
+  Specification](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/configuration/sdk-environment-variables.md#exemplar).
+  ([#5412](https://github.com/open-telemetry/opentelemetry-dotnet/pull/5412))
+
 ## 1.7.0
 
 Released 2023-Dec-08

src/OpenTelemetry/Metrics/MeterProviderSdk.cs

@@ -13,16 +13,19 @@ namespace OpenTelemetry.Metrics;
 
 internal sealed class MeterProviderSdk : MeterProvider
 {
+    internal const string EmitOverFlowAttributeConfigKey = "OTEL_DOTNET_EXPERIMENTAL_METRICS_EMIT_OVERFLOW_ATTRIBUTE";
+    internal const string ReclaimUnusedMetricPointsConfigKey = "OTEL_DOTNET_EXPERIMENTAL_METRICS_RECLAIM_UNUSED_METRIC_POINTS";
+    internal const string ExemplarFilterConfigKey = "OTEL_METRICS_EXEMPLAR_FILTER";
+
     internal readonly IServiceProvider ServiceProvider;
     internal readonly IDisposable? OwnedServiceProvider;
     internal int ShutdownCount;
     internal bool Disposed;
-    internal bool ShouldReclaimUnusedMetricPoints;
+    internal bool EmitOverflowAttribute;
+    internal bool ReclaimUnusedMetricPoints;
+    internal ExemplarFilterType? ExemplarFilter;
     internal Action? OnCollectObservableInstruments;
 
-    private const string EmitOverFlowAttributeConfigKey = "OTEL_DOTNET_EXPERIMENTAL_METRICS_EMIT_OVERFLOW_ATTRIBUTE";
-    private const string ReclaimUnusedMetricPointsConfigKey = "OTEL_DOTNET_EXPERIMENTAL_METRICS_RECLAIM_UNUSED_METRIC_POINTS";
-
     private readonly List<object> instrumentations = new();
     private readonly List<Func<Instrument, MetricStreamConfiguration?>> viewConfigs;
     private readonly object collectLock = new();
@@ -40,10 +43,6 @@ internal MeterProviderSdk(
         var state = serviceProvider!.GetRequiredService<MeterProviderBuilderSdk>();
         state.RegisterProvider(this);
 
-        var config = serviceProvider!.GetRequiredService<IConfiguration>();
-        _ = config.TryGetBoolValue(EmitOverFlowAttributeConfigKey, out bool isEmitOverflowAttributeKeySet);
-        _ = config.TryGetBoolValue(ReclaimUnusedMetricPointsConfigKey, out this.ShouldReclaimUnusedMetricPoints);
-
         this.ServiceProvider = serviceProvider!;
 
         if (ownsServiceProvider)
@@ -54,14 +53,16 @@ internal MeterProviderSdk(
 
         OpenTelemetrySdkEventSource.Log.MeterProviderSdkEvent("Building MeterProvider.");
 
-        OpenTelemetrySdkEventSource.Log.MeterProviderSdkEvent($"Metric overflow attribute key set to: {isEmitOverflowAttributeKeySet}");
-
         var configureProviderBuilders = serviceProvider!.GetServices<IConfigureMeterProviderBuilder>();
         foreach (var configureProviderBuilder in configureProviderBuilders)
         {
             configureProviderBuilder.ConfigureBuilder(serviceProvider!, state);
         }
 
+        this.ExemplarFilter = state.ExemplarFilter;
+
+        this.ApplySpecificationConfigurationKeys(serviceProvider!.GetRequiredService<IConfiguration>());
+
         StringBuilder exportersAdded = new StringBuilder();
         StringBuilder instrumentationFactoriesAdded = new StringBuilder();
 
@@ -80,8 +81,9 @@ internal MeterProviderSdk(
             reader.ApplyParentProviderSettings(
                 state.MetricLimit,
                 state.CardinalityLimit,
-                state.ExemplarFilter,
-                isEmitOverflowAttributeKeySet);
+                this.EmitOverflowAttribute,
+                this.ReclaimUnusedMetricPoints,
+                this.ExemplarFilter);
 
             if (this.reader == null)
             {
@@ -475,4 +477,58 @@ protected override void Dispose(bool disposing)
 
         base.Dispose(disposing);
     }
+
+    private void ApplySpecificationConfigurationKeys(IConfiguration configuration)
+    {
+        if (configuration.TryGetBoolValue(EmitOverFlowAttributeConfigKey, out this.EmitOverflowAttribute))
+        {
+            OpenTelemetrySdkEventSource.Log.MeterProviderSdkEvent("Overflow attribute feature enabled via configuration.");
+        }
+
+        if (configuration.TryGetBoolValue(ReclaimUnusedMetricPointsConfigKey, out this.ReclaimUnusedMetricPoints))
+        {
+            OpenTelemetrySdkEventSource.Log.MeterProviderSdkEvent("Reclaim unused metric point feature enabled via configuration.");
+        }
+
+#if EXPOSE_EXPERIMENTAL_FEATURES
+        if (configuration.TryGetStringValue(ExemplarFilterConfigKey, out var configValue))
+        {
+            if (this.ExemplarFilter.HasValue)
+            {
+                OpenTelemetrySdkEventSource.Log.MeterProviderSdkEvent(
+                    $"Exemplar filter configuration value '{configValue}' has been ignored because a value '{this.ExemplarFilter}' was set programmatically.");
+                return;
+            }
+
+            ExemplarFilterType? exemplarFilter;
+            if (string.Equals("always_off", configValue, StringComparison.OrdinalIgnoreCase))
+            {
+                exemplarFilter = ExemplarFilterType.AlwaysOff;
+            }
+            else if (string.Equals("always_on", configValue, StringComparison.OrdinalIgnoreCase))
+            {
+                exemplarFilter = ExemplarFilterType.AlwaysOn;
+            }
+            else if (string.Equals("trace_based", configValue, StringComparison.OrdinalIgnoreCase))
+            {
+                exemplarFilter = ExemplarFilterType.TraceBased;
+            }
+            else
+            {
+                OpenTelemetrySdkEventSource.Log.MeterProviderSdkEvent($"Exemplar filter configuration was found but the value '{configValue}' is invalid and will be ignored.");
+                return;
+            }
+
+            this.ExemplarFilter = exemplarFilter;
+
+            OpenTelemetrySdkEventSource.Log.MeterProviderSdkEvent($"Exemplar filter set to '{exemplarFilter}' from configuration.");
+        }
+#else
+        if (configuration.TryGetStringValue(ExemplarFilterConfigKey, out var configValue))
+        {
+            OpenTelemetrySdkEventSource.Log.MeterProviderSdkEvent(
+                $"Exemplar filter configuration value '{configValue}' has been ignored because exemplars are an experimental feature not available in stable builds.");
+        }
+#endif
+    }
 }

src/OpenTelemetry/Metrics/MetricReaderExt.cs

@@ -23,7 +23,7 @@ public abstract partial class MetricReader
     private Metric[]? metricsCurrentBatch;
     private int metricIndex = -1;
     private bool emitOverflowAttribute;
-
+    private bool reclaimUnusedMetricPoints;
     private ExemplarFilterType? exemplarFilter;
 
     internal static void DeactivateMetric(Metric metric)
@@ -72,8 +72,7 @@ internal virtual List<Metric> AddMetricWithNoViews(Instrument instrument)
                 Metric? metric = null;
                 try
                 {
-                    bool shouldReclaimUnusedMetricPoints = this.parentProvider is MeterProviderSdk meterProviderSdk && meterProviderSdk.ShouldReclaimUnusedMetricPoints;
-                    metric = new Metric(metricStreamIdentity, this.GetAggregationTemporality(metricStreamIdentity.InstrumentType), this.cardinalityLimit, this.emitOverflowAttribute, shouldReclaimUnusedMetricPoints, this.exemplarFilter);
+                    metric = new Metric(metricStreamIdentity, this.GetAggregationTemporality(metricStreamIdentity.InstrumentType), this.cardinalityLimit, this.emitOverflowAttribute, this.reclaimUnusedMetricPoints, this.exemplarFilter);
                 }
                 catch (NotSupportedException nse)
                 {
@@ -145,14 +144,12 @@ internal virtual List<Metric> AddMetricWithViews(Instrument instrument, List<Met
                 }
                 else
                 {
-                    bool shouldReclaimUnusedMetricPoints = this.parentProvider is MeterProviderSdk meterProviderSdk && meterProviderSdk.ShouldReclaimUnusedMetricPoints;
-
                     Metric metric = new(
                         metricStreamIdentity,
                         this.GetAggregationTemporality(metricStreamIdentity.InstrumentType),
                         metricStreamConfig?.CardinalityLimit ?? this.cardinalityLimit,
                         this.emitOverflowAttribute,
-                        shouldReclaimUnusedMetricPoints,
+                        this.reclaimUnusedMetricPoints,
                         this.exemplarFilter,
                         metricStreamConfig?.ExemplarReservoirFactory);
 
@@ -171,16 +168,18 @@ internal virtual List<Metric> AddMetricWithViews(Instrument instrument, List<Met
     internal void ApplyParentProviderSettings(
         int metricLimit,
         int cardinalityLimit,
-        ExemplarFilterType? exemplarFilter,
-        bool isEmitOverflowAttributeKeySet)
+        bool emitOverflowAttribute,
+        bool reclaimUnusedMetricPoints,
+        ExemplarFilterType? exemplarFilter)
     {
         this.metricLimit = metricLimit;
         this.metrics = new Metric[metricLimit];
         this.metricsCurrentBatch = new Metric[metricLimit];
         this.cardinalityLimit = cardinalityLimit;
+        this.reclaimUnusedMetricPoints = reclaimUnusedMetricPoints;
         this.exemplarFilter = exemplarFilter;
 
-        if (isEmitOverflowAttributeKeySet)
+        if (emitOverflowAttribute)
         {
             // We need at least two metric points. One is reserved for zero tags and the other one for overflow attribute
             if (cardinalityLimit > 1)

test/OpenTelemetry.Extensions.Hosting.Tests/OpenTelemetry.Extensions.Hosting.Tests.csproj

@@ -14,6 +14,7 @@
 
   <ItemGroup>
     <Compile Include="$(RepoRoot)\test\OpenTelemetry.Tests\Shared\EventSourceTestHelper.cs" Link="Includes\EventSourceTestHelper.cs" />
+    <Compile Include="$(RepoRoot)\test\OpenTelemetry.Tests\Shared\SkipUnlessTrueTheoryAttribute.cs" Link="Includes\SkipUnlessTrueTheoryAttribute.cs" />
     <Compile Include="$(RepoRoot)\test\OpenTelemetry.Tests\Shared\TestEventListener.cs" Link="Includes\TestEventListener.cs" />
     <Compile Include="$(RepoRoot)\test\OpenTelemetry.Tests\Shared\Utils.cs" Link="Includes\Utils.cs" />
   </ItemGroup>

test/OpenTelemetry.Tests/Metrics/MetricExemplarTests.cs

@@ -5,6 +5,8 @@
 
 using System.Diagnostics;
 using System.Diagnostics.Metrics;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
 using OpenTelemetry.Tests;
 using Xunit;
 
@@ -13,6 +15,45 @@ namespace OpenTelemetry.Metrics.Tests;
 public class MetricExemplarTests : MetricTestsBase
 {
     private const int MaxTimeToAllowForFlush = 10000;
+    private static readonly Func<bool> IsExemplarApiExposed = () => typeof(ExemplarFilterType).IsVisible;
+
+    [SkipUnlessTrueTheory(typeof(MetricExemplarTests), nameof(IsExemplarApiExposed), "ExemplarFilter config tests skipped for stable builds")]
+    [InlineData(null, null, null)]
+    [InlineData(null, "always_off", (int)ExemplarFilterType.AlwaysOff)]
+    [InlineData(null, "ALWays_ON", (int)ExemplarFilterType.AlwaysOn)]
+    [InlineData(null, "trace_based", (int)ExemplarFilterType.TraceBased)]
+    [InlineData(null, "invalid", null)]
+    [InlineData((int)ExemplarFilterType.AlwaysOn, "trace_based", (int)ExemplarFilterType.AlwaysOn)]
+    public void TestExemplarFilterSetFromConfiguration(
+        int? programmaticValue,
+        string? configValue,
+        int? expectedValue)
+    {
+        var configBuilder = new ConfigurationBuilder();
+        if (!string.IsNullOrEmpty(configValue))
+        {
+            configBuilder.AddInMemoryCollection(new Dictionary<string, string?>
+            {
+                [MeterProviderSdk.ExemplarFilterConfigKey] = configValue,
+            });
+        }
+
+        using var container = this.BuildMeterProvider(out var meterProvider, b =>
+        {
+            b.ConfigureServices(
+                s => s.AddSingleton<IConfiguration>(configBuilder.Build()));
+
+            if (programmaticValue.HasValue)
+            {
+                b.SetExemplarFilter(((ExemplarFilterType?)programmaticValue).Value);
+            }
+        });
+
+        var meterProviderSdk = meterProvider as MeterProviderSdk;
+
+        Assert.NotNull(meterProviderSdk);
+        Assert.Equal((ExemplarFilterType?)expectedValue, meterProviderSdk.ExemplarFilter);
+    }
 
     [Theory]
     [InlineData(MetricReaderTemporalityPreference.Cumulative)]

test/OpenTelemetry.Tests/Metrics/MetricPointReclaimTestsBase.cs

@@ -57,7 +57,7 @@ public void TestReclaimAttributeConfigWithEnvVar(string value, bool isReclaimAtt
             .Build();
 
         var meterProviderSdk = meterProvider as MeterProviderSdk;
-        Assert.Equal(isReclaimAttributeKeySet, meterProviderSdk.ShouldReclaimUnusedMetricPoints);
+        Assert.Equal(isReclaimAttributeKeySet, meterProviderSdk.ReclaimUnusedMetricPoints);
     }
 
     [Theory]
@@ -87,7 +87,7 @@ public void TestReclaimAttributeConfigWithOtherConfigProvider(string value, bool
             .Build();
 
         var meterProviderSdk = meterProvider as MeterProviderSdk;
-        Assert.Equal(isReclaimAttributeKeySet, meterProviderSdk.ShouldReclaimUnusedMetricPoints);
+        Assert.Equal(isReclaimAttributeKeySet, meterProviderSdk.ReclaimUnusedMetricPoints);
     }
 
     [Theory]

test/OpenTelemetry.Tests/Shared/SkipUnlessTrueTheoryAttribute.cs

@@ -0,0 +1,33 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+using System.Reflection;
+using OpenTelemetry.Internal;
+using Xunit;
+
+namespace OpenTelemetry.Tests;
+
+internal sealed class SkipUnlessTrueTheoryAttribute : TheoryAttribute
+{
+    public SkipUnlessTrueTheoryAttribute(Type typeContainingTest, string testFieldName, string skipMessage)
+    {
+        Guard.ThrowIfNull(typeContainingTest);
+        Guard.ThrowIfNullOrEmpty(testFieldName);
+        Guard.ThrowIfNullOrEmpty(skipMessage);
+
+        var field = typeContainingTest.GetField(testFieldName, BindingFlags.Static | BindingFlags.Public | BindingFlags.NonPublic)
+            ?? throw new InvalidOperationException($"Static field '{testFieldName}' could not be found on '{typeContainingTest}' type.");
+
+        if (field.FieldType != typeof(Func<bool>))
+        {
+            throw new InvalidOperationException($"Field '{testFieldName}' on '{typeContainingTest}' type should be defined as '{typeof(Func<bool>)}'.");
+        }
+
+        var testFunc = (Func<bool>)field.GetValue(null);
+
+        if (!testFunc())
+        {
+            this.Skip = skipMessage;
+        }
+    }
+}