Automatic Tags and Span Aggregates for Metrics (#3191)

CHANGELOG.md

@@ -5,6 +5,7 @@
 ### Features
 
 - Added Crons support via `SentrySdk.CaptureCheckIn` and an integration with Hangfire ([#3128](https://github.com/getsentry/sentry-dotnet/pull/3128))
+- Common tags set automatically for metrics and metrics summaries are attached to Spans ([#3191](https://github.com/getsentry/sentry-dotnet/pull/3191))
 
 ### Fixes
 

src/Sentry/IMetricHub.cs

@@ -18,4 +18,7 @@ internal interface IMetricHub
     /// Starts a child span for the current transaction or, if there is no active transaction, starts a new transaction.
     /// </summary>
     ISpan StartSpan(string operation, string description);
+
+    /// <inheritdoc cref="IHub.GetSpan"/>
+    ISpan? GetSpan();
 }

src/Sentry/Internal/Extensions/DictionaryExtensions.cs

@@ -0,0 +1,13 @@
+namespace Sentry.Internal.Extensions;
+
+internal static class DictionaryExtensions
+{
+    public static void AddIfNotNullOrEmpty<TKey>(this IDictionary<TKey, string> dictionary, TKey key, string? value)
+        where TKey : notnull
+    {
+        if (!string.IsNullOrEmpty(value))
+        {
+            dictionary.Add(key, value);
+        }
+    }
+}

src/Sentry/MetricAggregator.cs

@@ -58,61 +58,6 @@ internal MetricAggregator(SentryOptions options, IMetricHub metricHub,
         }
     }
 
-    internal static string GetMetricBucketKey(MetricType type, string metricKey, MeasurementUnit unit,
-        IDictionary<string, string>? tags)
-    {
-        var typePrefix = type.ToStatsdType();
-        var serializedTags = GetTagsKey(tags);
-
-        return $"{typePrefix}_{metricKey}_{unit}_{serializedTags}";
-    }
-
-    internal static string GetTagsKey(IDictionary<string, string>? tags)
-    {
-        if (tags == null || tags.Count == 0)
-        {
-            return string.Empty;
-        }
-
-        const char pairDelimiter = ',';  // Delimiter between key-value pairs
-        const char keyValueDelimiter = '=';  // Delimiter between key and value
-        const char escapeChar = '\\';
-
-        var builder = new StringBuilder();
-
-        foreach (var tag in tags)
-        {
-            // Escape delimiters in key and value
-            var key = EscapeString(tag.Key, pairDelimiter, keyValueDelimiter, escapeChar);
-            var value = EscapeString(tag.Value, pairDelimiter, keyValueDelimiter, escapeChar);
-
-            if (builder.Length > 0)
-            {
-                builder.Append(pairDelimiter);
-            }
-
-            builder.Append(key).Append(keyValueDelimiter).Append(value);
-        }
-
-        return builder.ToString();
-
-        static string EscapeString(string input, params char[] charsToEscape)
-        {
-            var escapedString = new StringBuilder(input.Length);
-
-            foreach (var ch in input)
-            {
-                if (charsToEscape.Contains(ch))
-                {
-                    escapedString.Append(escapeChar);  // Prefix with escape character
-                }
-                escapedString.Append(ch);
-            }
-
-            return escapedString.ToString();
-        }
-    }
-
     /// <inheritdoc cref="IMetricAggregator.Increment"/>
     public void Increment(string key,
         double value = 1.0,
@@ -186,19 +131,28 @@ private void Emit(
         timestamp ??= DateTimeOffset.UtcNow;
         unit ??= MeasurementUnit.None;
 
+        var updatedTags = tags != null ? new Dictionary<string, string>(tags) : new Dictionary<string, string>();
+        updatedTags.AddIfNotNullOrEmpty("release", _options.Release);
+        updatedTags.AddIfNotNullOrEmpty("environment", _options.Environment);
+        var span = _metricHub.GetSpan();
+        if (span?.GetTransaction() is { } transaction)
+        {
+            updatedTags.AddIfNotNullOrEmpty("transaction", transaction.TransactionName);
+        }
+
         Func<string, Metric> addValuesFactory = type switch
         {
-            MetricType.Counter => _ => new CounterMetric(key, value, unit.Value, tags, timestamp),
-            MetricType.Gauge => _ => new GaugeMetric(key, value, unit.Value, tags, timestamp),
-            MetricType.Distribution => _ => new DistributionMetric(key, value, unit.Value, tags, timestamp),
-            MetricType.Set => _ => new SetMetric(key, (int)value, unit.Value, tags, timestamp),
+            MetricType.Counter => _ => new CounterMetric(key, value, unit.Value, updatedTags, timestamp),
+            MetricType.Gauge => _ => new GaugeMetric(key, value, unit.Value, updatedTags, timestamp),
+            MetricType.Distribution => _ => new DistributionMetric(key, value, unit.Value, updatedTags, timestamp),
+            MetricType.Set => _ => new SetMetric(key, (int)value, unit.Value, updatedTags, timestamp),
             _ => throw new ArgumentOutOfRangeException(nameof(type), type, "Unknown MetricType")
         };
 
         var timeBucket = GetOrAddTimeBucket(timestamp.Value.GetTimeBucketKey());
 
         timeBucket.AddOrUpdate(
-            GetMetricBucketKey(type, key, unit.Value, tags),
+            MetricHelper.GetMetricBucketKey(type, key, unit.Value, updatedTags),
             addValuesFactory,
             (_, metric) =>
             {
@@ -223,6 +177,16 @@ private void Emit(
         {
             RecordCodeLocation(type, key, unit.Value, stackLevel + 1, timestamp.Value);
         }
+
+        switch (span)
+        {
+            case TransactionTracer transactionTracer:
+                transactionTracer.MetricsSummary.Add(type, key, value, unit, tags);
+                break;
+            case SpanTracer spanTracer:
+                spanTracer.MetricsSummary.Add(type, key, value, unit, tags);
+                break;
+        }
     }
 
     private ConcurrentDictionary<string, Metric> GetOrAddTimeBucket(long bucketKey)

src/Sentry/MetricHelper.cs

@@ -1,4 +1,5 @@
 using Sentry.Internal;
+using Sentry.Protocol.Metrics;
 
 namespace Sentry;
 
@@ -64,4 +65,59 @@ internal static DateTimeOffset GetCutoff() => DateTimeOffset.UtcNow
     private static readonly Regex InvalidMetricUnitCharacters = new(InvalidMetricUnitCharactersPattern, RegexOptions.Compiled);
     internal static string SanitizeMetricUnit(string input) => InvalidMetricUnitCharacters.Replace(input, "_");
 #endif
+
+    public static string GetMetricBucketKey(MetricType type, string metricKey, MeasurementUnit unit,
+        IDictionary<string, string>? tags)
+    {
+        var typePrefix = type.ToStatsdType();
+        var serializedTags = GetTagsKey(tags);
+
+        return $"{typePrefix}_{metricKey}_{unit}_{serializedTags}";
+    }
+
+    internal static string GetTagsKey(IDictionary<string, string>? tags)
+    {
+        if (tags == null || tags.Count == 0)
+        {
+            return string.Empty;
+        }
+
+        const char pairDelimiter = ',';  // Delimiter between key-value pairs
+        const char keyValueDelimiter = '=';  // Delimiter between key and value
+        const char escapeChar = '\\';
+
+        var builder = new StringBuilder();
+
+        foreach (var tag in tags)
+        {
+            // Escape delimiters in key and value
+            var key = EscapeString(tag.Key, pairDelimiter, keyValueDelimiter, escapeChar);
+            var value = EscapeString(tag.Value, pairDelimiter, keyValueDelimiter, escapeChar);
+
+            if (builder.Length > 0)
+            {
+                builder.Append(pairDelimiter);
+            }
+
+            builder.Append(key).Append(keyValueDelimiter).Append(value);
+        }
+
+        return builder.ToString();
+
+        static string EscapeString(string input, params char[] charsToEscape)
+        {
+            var escapedString = new StringBuilder(input.Length);
+
+            foreach (var ch in input)
+            {
+                if (charsToEscape.Contains(ch))
+                {
+                    escapedString.Append(escapeChar);  // Prefix with escape character
+                }
+                escapedString.Append(ch);
+            }
+
+            return escapedString.ToString();
+        }
+    }
 }

src/Sentry/MetricsSummaryAggregator.cs

@@ -0,0 +1,37 @@
+using Sentry.Protocol.Metrics;
+
+namespace Sentry;
+
+internal class MetricsSummaryAggregator
+{
+    private Lazy<ConcurrentDictionary<string, SpanMetric>> LazyMeasurements { get; } = new();
+    internal ConcurrentDictionary<string, SpanMetric> Measurements => LazyMeasurements.Value;
+
+    public void Add(
+        MetricType ty,
+        string key,
+        double value = 1.0,
+        MeasurementUnit? unit = null,
+        IDictionary<string, string>? tags = null
+        )
+    {
+        unit ??= MeasurementUnit.None;
+
+        var bucketKey = MetricHelper.GetMetricBucketKey(ty, key, unit.Value, tags);
+
+        Measurements.AddOrUpdate(
+            bucketKey,
+            _ => new SpanMetric(ty, key, value, unit.Value, tags),
+            (_, metric) =>
+            {
+                // This prevents multiple threads from trying to mutate the metric at the same time. The only other
+                // operation performed against metrics is adding one to the bucket (guaranteed to be atomic due to
+                // the use of a ConcurrentDictionary for the timeBucket).
+                lock (metric)
+                {
+                    metric.Add(value);
+                }
+                return metric;
+            });
+    }
+}

src/Sentry/Protocol/Metrics/MetricsSummary.cs

@@ -0,0 +1,47 @@
+using Sentry.Extensibility;
+
+namespace Sentry.Protocol.Metrics;
+
+internal class MetricsSummary : ISentryJsonSerializable
+{
+    private readonly IDictionary<string, List<SpanMetric>> _measurements;
+
+    public MetricsSummary(MetricsSummaryAggregator aggregator)
+    {
+        // For the Metrics Summary we group all the metrics by an export key.
+        // See https://github.com/getsentry/rfcs/blob/main/text/0123-metrics-correlation.md#basics
+        var measurements = new Dictionary<string, List<SpanMetric>>();
+        foreach (var (_, value) in aggregator.Measurements)
+        {
+            var exportKey = value.ExportKey;
+#if NET6_0_OR_GREATER
+            measurements.TryAdd(exportKey, new List<SpanMetric>());
+#else
+            if (!measurements.ContainsKey(exportKey))
+            {
+                measurements.Add(exportKey, new List<SpanMetric>());
+            }
+#endif
+            measurements[exportKey].Add(value);
+        }
+        _measurements = measurements.ToImmutableSortedDictionary();
+    }
+
+    public void WriteTo(Utf8JsonWriter writer, IDiagnosticLogger? logger)
+    {
+        writer.WriteStartObject();
+
+        foreach (var (exportKey, value) in _measurements)
+        {
+            writer.WritePropertyName(exportKey);
+            writer.WriteStartArray();
+            foreach (var metric in value.OrderBy(x => MetricHelper.GetMetricBucketKey(x.MetricType, x.Key, x.Unit, x.Tags)))
+            {
+                metric.WriteTo(writer, logger);
+            }
+            writer.WriteEndArray();
+        }
+
+        writer.WriteEndObject();
+    }
+}

src/Sentry/Protocol/Metrics/SpanMetric.cs

@@ -0,0 +1,54 @@
+using Sentry.Extensibility;
+using Sentry.Internal.Extensions;
+
+namespace Sentry.Protocol.Metrics;
+
+internal record SpanMetric
+{
+    public SpanMetric(MetricType MetricType,
+        string key,
+        double value,
+        MeasurementUnit unit,
+        IDictionary<string, string>? tags = null)
+    {
+        this.MetricType = MetricType;
+        Key = key;
+        Unit = unit;
+        Tags = tags;
+        Min = value;
+        Max = value;
+        Sum = value;
+    }
+
+    public MetricType MetricType { get; init; }
+    public string Key { get; init; }
+    public MeasurementUnit Unit { get; init; }
+    public IDictionary<string, string>? Tags { get; init; }
+
+    public double Min { get; private set; }
+    public double Max { get; private set; }
+    public double Sum { get; private set; }
+    public double Count { get; private set; } = 1;
+
+    public string ExportKey => $"{MetricType.ToStatsdType()}:{Key}@{Unit}";
+
+    public void Add(double value)
+    {
+        Min = Math.Min(Min, value);
+        Max = Math.Max(Max, value);
+        Sum += value;
+        Count++;
+    }
+
+    /// <inheritdoc cref="ISentryJsonSerializable.WriteTo"/>
+    public void WriteTo(Utf8JsonWriter writer, IDiagnosticLogger? logger)
+    {
+        writer.WriteStartObject();
+        writer.WriteNumber("min", Min);
+        writer.WriteNumber("max", Max);
+        writer.WriteNumber("count", Count);
+        writer.WriteNumber("sum", Sum);
+        writer.WriteStringDictionaryIfNotEmpty("tags", (IEnumerable<KeyValuePair<string, string?>>?)Tags);
+        writer.WriteEndObject();
+    }
+}

src/Sentry/SentrySpan.cs

@@ -2,6 +2,7 @@
 using Sentry.Internal;
 using Sentry.Internal.Extensions;
 using Sentry.Protocol;
+using Sentry.Protocol.Metrics;
 
 namespace Sentry;
 
@@ -66,6 +67,7 @@ public void UnsetTag(string key) =>
 
     // Aka 'data'
     private Dictionary<string, object?>? _extra;
+    private readonly MetricsSummary? _metricsSummary;
 
     /// <inheritdoc />
     public IReadOnlyDictionary<string, object?> Extra => _extra ??= new Dictionary<string, object?>();
@@ -104,6 +106,10 @@ public SentrySpan(ISpan tracer)
         {
             _measurements = spanTracer.InternalMeasurements?.ToDict();
             _tags = spanTracer.InternalTags?.ToDict();
+            if (spanTracer.HasMetrics)
+            {
+                _metricsSummary = new MetricsSummary(spanTracer.MetricsSummary);
+            }
         }
         else
         {
@@ -134,6 +140,7 @@ public void WriteTo(Utf8JsonWriter writer, IDiagnosticLogger? logger)
         writer.WriteStringDictionaryIfNotEmpty("tags", _tags!);
         writer.WriteDictionaryIfNotEmpty("data", _extra!, logger);
         writer.WriteDictionaryIfNotEmpty("measurements", _measurements, logger);
+        writer.WriteSerializableIfNotNull("_metrics_summary", _metricsSummary, logger);
 
         writer.WriteEndObject();
     }