Hashing exceptions with normalization + improved diagnostic capabilities

tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/CachedDoneExceptions.cs

@@ -20,17 +20,17 @@ internal static class CachedDoneExceptions
     {
         private static readonly CachedItems _cachedDoneExceptions = new CachedItems();
 
-        internal static void Add(string item)
+        internal static void Add(int item)
         {
             _cachedDoneExceptions.Add(item);
         }
 
-        internal static bool Remove(string item)
+        internal static bool Remove(int item)
         {
             return _cachedDoneExceptions.Remove(item);
         }
 
-        internal static bool Contains(string item)
+        internal static bool Contains(int item)
         {
             return _cachedDoneExceptions.Contains(item);
         }

tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/CachedItems.cs

@@ -18,48 +18,46 @@ namespace Datadog.Trace.Debugger.ExceptionAutoInstrumentation
 {
     internal class CachedItems
     {
-        private readonly HashSet<int> cache = new();
-        private readonly ReaderWriterLockSlim cacheLocker = new();
+        private readonly HashSet<int> _cache = new();
+        private readonly ReaderWriterLockSlim _cacheLocker = new();
 
-        internal void Add(string item)
+        internal void Add(int item)
         {
-            cacheLocker.EnterWriteLock();
+            _cacheLocker.EnterWriteLock();
             try
             {
-                cache.Add(Hash(item));
+                _cache.Add(item);
             }
             finally
             {
-                cacheLocker.ExitWriteLock();
+                _cacheLocker.ExitWriteLock();
             }
         }
 
-        internal bool Remove(string item)
+        internal bool Remove(int item)
         {
-            cacheLocker.EnterWriteLock();
+            _cacheLocker.EnterWriteLock();
             try
             {
-                return cache.Remove(Hash(item));
+                return _cache.Remove(item);
             }
             finally
             {
-                cacheLocker.ExitWriteLock();
+                _cacheLocker.ExitWriteLock();
             }
         }
 
-        internal bool Contains(string item)
+        internal bool Contains(int item)
         {
-            cacheLocker.EnterReadLock();
+            _cacheLocker.EnterReadLock();
             try
             {
-                return cache.Contains(Hash(item));
+                return _cache.Contains(item);
             }
             finally
             {
-                cacheLocker.ExitReadLock();
+                _cacheLocker.ExitReadLock();
             }
         }
-
-        private int Hash(string item) => Fnv1aHash.GetFNVHashCode(StringEncoding.UTF8.GetBytes(item));
     }
 }

tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionNormalizer.cs

@@ -0,0 +1,99 @@
+// <copyright file="ExceptionNormalizer.cs" company="Datadog">
+// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
+// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
+// </copyright>
+
+using System;
+using System.Numerics;
+using System.Runtime.CompilerServices;
+using System.Text;
+using Datadog.Trace.VendoredMicrosoftCode.System;
+using Fnv1aHash = Datadog.Trace.VendoredMicrosoftCode.System.Reflection.Internal.Hash;
+using MemoryExtensions = Datadog.Trace.Debugger.Helpers.MemoryExtensions;
+
+#nullable enable
+namespace Datadog.Trace.Debugger.ExceptionAutoInstrumentation
+{
+    internal class ExceptionNormalizer
+    {
+        /// <summary>
+        /// Given the string representation of an exception alongside it's FQN of the outer and (potential) inner exception,
+        /// this function cleanse the stack trace from error messages, customized information attached to the exception and PDB line info if present.
+        /// It returns a hash representing the resulting cleansed exception and inner exceptions.
+        /// Used to aggregate same/similar exceptions that only differ by non-relevant bits.
+        /// </summary>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        internal int NormalizeAndHashException(string exceptionString, string outerExceptionType, string? innerExceptionType)
+        {
+            if (string.IsNullOrEmpty(exceptionString))
+            {
+                throw new ArgumentException(@"Exception string cannot be null or empty", nameof(exceptionString));
+            }
+
+            var fnvHashCode = HashLine(VendoredMicrosoftCode.System.MemoryExtensions.AsSpan(outerExceptionType), Fnv1aHash.FnvOffsetBias);
+
+            if (innerExceptionType != null)
+            {
+                fnvHashCode = HashLine(VendoredMicrosoftCode.System.MemoryExtensions.AsSpan(innerExceptionType), fnvHashCode);
+            }
+
+            var exceptionSpan = VendoredMicrosoftCode.System.MemoryExtensions.AsSpan(exceptionString);
+            var inSpan = VendoredMicrosoftCode.System.MemoryExtensions.AsSpan(" in ");
+            var atSpan = VendoredMicrosoftCode.System.MemoryExtensions.AsSpan("at ");
+            var lambdaSpan = VendoredMicrosoftCode.System.MemoryExtensions.AsSpan("lambda_");
+            var microsoftSpan = VendoredMicrosoftCode.System.MemoryExtensions.AsSpan("at Microsoft.");
+            var systemSpan = VendoredMicrosoftCode.System.MemoryExtensions.AsSpan("at System.");
+            var datadogSpan = VendoredMicrosoftCode.System.MemoryExtensions.AsSpan("at Datadog.");
+
+            while (!exceptionSpan.IsEmpty)
+            {
+                var lineEndIndex = exceptionSpan.IndexOfAny('\r', '\n');
+                VendoredMicrosoftCode.System.ReadOnlySpan<char> line;
+
+                if (lineEndIndex >= 0)
+                {
+                    line = exceptionSpan.Slice(0, lineEndIndex);
+                    exceptionSpan = exceptionSpan.Slice(lineEndIndex + 1);
+                    if (!exceptionSpan.IsEmpty && exceptionSpan[0] == '\n')
+                    {
+                        exceptionSpan = exceptionSpan.Slice(1);
+                    }
+                }
+                else
+                {
+                    line = exceptionSpan;
+                    exceptionSpan = default;
+                }
+
+                // Is frame line (starts with `in `).
+                if (VendoredMicrosoftCode.System.MemoryExtensions.StartsWith(line.TrimStart(), atSpan, StringComparison.Ordinal))
+                {
+                    var index = VendoredMicrosoftCode.System.MemoryExtensions.IndexOf(line, inSpan, StringComparison.Ordinal);
+                    line = index > 0 ? line.Slice(0, index) : line;
+
+                    if (VendoredMicrosoftCode.System.MemoryExtensions.Contains(line, lambdaSpan, StringComparison.Ordinal) ||
+                        VendoredMicrosoftCode.System.MemoryExtensions.Contains(line, microsoftSpan, StringComparison.Ordinal) ||
+                        VendoredMicrosoftCode.System.MemoryExtensions.Contains(line, datadogSpan, StringComparison.Ordinal) ||
+                        VendoredMicrosoftCode.System.MemoryExtensions.Contains(line, systemSpan, StringComparison.Ordinal))
+                    {
+                        continue;
+                    }
+
+                    fnvHashCode = HashLine(line, fnvHashCode);
+                }
+            }
+
+            return fnvHashCode;
+        }
+
+        protected virtual int HashLine(VendoredMicrosoftCode.System.ReadOnlySpan<char> line, int fnvHashCode)
+        {
+            for (var i = 0; i < line.Length; i++)
+            {
+                fnvHashCode = Fnv1aHash.Combine((uint)line[i], fnvHashCode);
+            }
+
+            return fnvHashCode;
+        }
+    }
+}

tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionReplayDiagnosticTagNames.cs

@@ -18,16 +18,17 @@ namespace Datadog.Trace.Debugger.ExceptionAutoInstrumentation
     internal static class ExceptionReplayDiagnosticTagNames
     {
         public const string Eligible = nameof(Eligible);
-        public const string EmptyShadowStack = nameof(EmptyShadowStack);
+        public const string NotEligible = nameof(NotEligible);
         public const string ExceptionTrackManagerNotInitialized = nameof(ExceptionTrackManagerNotInitialized);
         public const string NotRootSpan = nameof(NotRootSpan);
         public const string ExceptionObjectIsNull = nameof(ExceptionObjectIsNull);
         public const string NonSupportedExceptionType = nameof(NonSupportedExceptionType);
         public const string CachedDoneExceptionCase = nameof(CachedDoneExceptionCase);
+        public const string CachedInvalidatedExceptionCase = nameof(CachedInvalidatedExceptionCase);
         public const string InvalidatedExceptionCase = nameof(InvalidatedExceptionCase);
         public const string CircuitBreakerIsOpen = nameof(CircuitBreakerIsOpen);
         public const string NonCachedDoneExceptionCase = nameof(NonCachedDoneExceptionCase);
-        public const string NotSupportedExceptionType = nameof(NotSupportedExceptionType);
+        public const string NoCustomerFrames = nameof(NoCustomerFrames);
         public const string NoFramesToInstrument = nameof(NoFramesToInstrument);
         public const string EmptyCallStackTreeWhileCollecting = nameof(EmptyCallStackTreeWhileCollecting);
         public const string InvalidatedCase = nameof(InvalidatedCase);