From 57fc175a5dd3769fbef6ab989e2c18c9f200aae3 Mon Sep 17 00:00:00 2001 From: Matan Green Date: Mon, 29 Jan 2024 14:22:05 +0200 Subject: [PATCH] Exception Debugging Core --- .../build/Datadog.Trace.Trimming.xml | 2 + .../AggregateExceptionRelatedFrames.cs | 37 ++ .../DebuggerSnapshot.cs | 29 ++ .../ErrorOriginType.cs | 44 ++ .../ExceptionCollectionState.cs | 22 + .../ExceptionDebuggingProcessor.cs | 259 +++++++++++ .../ExceptionProbe.cs | 30 ++ .../ExceptionProbeProcessor.cs | 241 ++++++++++ .../ExceptionRelatedFrames.cs | 73 +++ .../ExceptionSnapshotCreator.cs | 28 ++ .../ExceptionStackNodeRecord.cs | 32 ++ .../ExceptionStackTreeRecord.cs | 34 ++ .../ExceptionTrackManager.cs | 424 ++++++++++++++++++ .../ExceptionAutoInstrumentation/Fnv1aHash.cs | 30 ++ .../FrameFilter.cs | 130 ++++++ .../MD5HashProvider.cs | 19 + .../MethodUniqueIdentifier.cs | 281 ++++++++++++ .../ParticipatingFrame.cs | 66 +++ .../ShadowStackContainer.cs | 42 ++ .../ShadowStackTree.cs | 158 +++++++ .../SnapshotCapturingStrategy.cs | 13 + .../TrackedExceptionCase.cs | 106 +++++ .../TrackedStackFrameNode.cs | 324 +++++++++++++ .../TrackingFrameData.cs | 26 ++ .../Debugger/Expressions/CaptureInfo.cs | 4 + .../Debugger/Expressions/IProbeProcessor.cs | 25 ++ .../Expressions/ProbeExpressionsProcessor.cs | 17 +- .../Debugger/Expressions/ProbeProcessor.cs | 46 +- .../Debugger/Helpers/ExceptionExtensions.cs | 28 ++ .../Debugger/Helpers/LazyExtensions.cs | 18 + .../Debugger/Helpers/MethodExtensions.cs | 183 ++++++++ .../Debugger/Helpers/StackTraceExtensions.cs | 62 +++ .../AsyncLineDebuggerInvoker.cs | 28 +- .../Instrumentation/AsyncLineDebuggerState.cs | 19 +- .../AsyncMethodDebuggerInvoker.SingleProbe.cs | 65 ++- .../AsyncMethodDebuggerState.cs | 5 +- .../Instrumentation/Collections/ProbeData.cs | 6 +- .../Instrumentation/LineDebuggerInvoker.cs | 33 +- .../Instrumentation/LineDebuggerState.cs | 19 +- .../MethodDebuggerInvoker.SingleProbe.cs | 61 ++- .../Instrumentation/MethodDebuggerState.cs | 20 +- .../Debugger/RateLimiting/AdaptiveSampler.cs | 10 +- .../Debugger/RateLimiting/IAdaptiveSampler.cs | 18 + .../RateLimiting/NopAdaptiveSampler.cs | 32 ++ .../Debugger/RateLimiting/ProbeRateLimiter.cs | 9 +- .../Snapshots/DebuggerSnapshotCreator.cs | 27 +- .../Snapshots/IDebuggerSnapshotCreator.cs | 13 + tracer/src/Datadog.Trace/Span.cs | 1 + 48 files changed, 3056 insertions(+), 143 deletions(-) create mode 100644 tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/AggregateExceptionRelatedFrames.cs create mode 100644 tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/DebuggerSnapshot.cs create mode 100644 tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ErrorOriginType.cs create mode 100644 tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionCollectionState.cs create mode 100644 tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionDebuggingProcessor.cs create mode 100644 tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionProbe.cs create mode 100644 tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionProbeProcessor.cs create mode 100644 tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionRelatedFrames.cs create mode 100644 tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionSnapshotCreator.cs create mode 100644 tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionStackNodeRecord.cs create mode 100644 tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionStackTreeRecord.cs create mode 100644 tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionTrackManager.cs create mode 100644 tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/Fnv1aHash.cs create mode 100644 tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/FrameFilter.cs create mode 100644 tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/MD5HashProvider.cs create mode 100644 tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/MethodUniqueIdentifier.cs create mode 100644 tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ParticipatingFrame.cs create mode 100644 tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ShadowStackContainer.cs create mode 100644 tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ShadowStackTree.cs create mode 100644 tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/SnapshotCapturingStrategy.cs create mode 100644 tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/TrackedExceptionCase.cs create mode 100644 tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/TrackedStackFrameNode.cs create mode 100644 tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/TrackingFrameData.cs create mode 100644 tracer/src/Datadog.Trace/Debugger/Expressions/IProbeProcessor.cs create mode 100644 tracer/src/Datadog.Trace/Debugger/Helpers/ExceptionExtensions.cs create mode 100644 tracer/src/Datadog.Trace/Debugger/Helpers/LazyExtensions.cs create mode 100644 tracer/src/Datadog.Trace/Debugger/Helpers/MethodExtensions.cs create mode 100644 tracer/src/Datadog.Trace/Debugger/Helpers/StackTraceExtensions.cs create mode 100644 tracer/src/Datadog.Trace/Debugger/RateLimiting/IAdaptiveSampler.cs create mode 100644 tracer/src/Datadog.Trace/Debugger/RateLimiting/NopAdaptiveSampler.cs create mode 100644 tracer/src/Datadog.Trace/Debugger/Snapshots/IDebuggerSnapshotCreator.cs diff --git a/tracer/src/Datadog.Trace.Trimming/build/Datadog.Trace.Trimming.xml b/tracer/src/Datadog.Trace.Trimming/build/Datadog.Trace.Trimming.xml index 4356d91921e3..327fa0fdc99f 100644 --- a/tracer/src/Datadog.Trace.Trimming/build/Datadog.Trace.Trimming.xml +++ b/tracer/src/Datadog.Trace.Trimming/build/Datadog.Trace.Trimming.xml @@ -597,6 +597,7 @@ + @@ -824,6 +825,7 @@ + diff --git a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/AggregateExceptionRelatedFrames.cs b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/AggregateExceptionRelatedFrames.cs new file mode 100644 index 000000000000..2a6d3a5af32c --- /dev/null +++ b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/AggregateExceptionRelatedFrames.cs @@ -0,0 +1,37 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Datadog.Trace.Debugger.ExceptionAutoInstrumentation +{ + internal class AggregateExceptionRelatedFrames : ExceptionRelatedFrames + { + public AggregateExceptionRelatedFrames(AggregateException ex, ParticipatingFrame[] frames, ExceptionRelatedFrames[] innerFrames) + : base(ex, frames) + { + InnerFrames = innerFrames; + } + + public ExceptionRelatedFrames[] InnerFrames { get; } + + public override IEnumerable GetAllFlattenedFrames() + { + foreach (var frame in InnerFrames.SelectMany(innerFrame => innerFrame?.GetAllFlattenedFrames())) + { + yield return frame; + } + + foreach (var frame in Frames) + { + yield return frame; + } + } + } +} diff --git a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/DebuggerSnapshot.cs b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/DebuggerSnapshot.cs new file mode 100644 index 000000000000..279368832d28 --- /dev/null +++ b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/DebuggerSnapshot.cs @@ -0,0 +1,29 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +using System; + +namespace Datadog.Trace.Debugger.ExceptionAutoInstrumentation; + +internal class DebuggerSnapshot +{ + public DebuggerSnapshot(string probeId, string snapshot, Exception exceptionThrown, Guid snapshotId) + { + ProbeId = probeId; + Snapshot = snapshot; + ExceptionThrown = exceptionThrown; + SnapshotId = snapshotId; + } + + public DebuggerSnapshot Child { get; set; } + + public string ProbeId { get; } + + public string Snapshot { get; } + + public Exception ExceptionThrown { get; } + + public Guid SnapshotId { get; } +} diff --git a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ErrorOriginType.cs b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ErrorOriginType.cs new file mode 100644 index 000000000000..aeb5a87d88ad --- /dev/null +++ b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ErrorOriginType.cs @@ -0,0 +1,44 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Datadog.Trace.Debugger.ExceptionAutoInstrumentation +{ + internal enum ErrorOriginType : byte + { + /// + /// A first-chance exception is any exception that is initially thrown. + /// + FirstChanceException, + + /// + /// A second chance exception is an unhandled exception that has propagated + /// to the top of the stack and is about to crash the process. + /// + SecondChanceException, + + /// + /// An exception that was sent to a logging tool - e.g. via a call to Logger.Error(...) + /// + LoggedError, + + /// + /// In ASP.NET / ASP.NET Core / WebAPI etc, an HTTP request failure exception is + /// an exception in that will cause the request to return a failure response code (usually HTTP 500). + /// + HttpRequestFailure, + + /// + /// The shadow stack knows when a catch statement ends. When it does, it calls an exception event. + /// For first-chance exception it means this instance will have the full StackTrace. + /// + ExceptionCaught + } +} diff --git a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionCollectionState.cs b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionCollectionState.cs new file mode 100644 index 000000000000..41f1cffe9c8e --- /dev/null +++ b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionCollectionState.cs @@ -0,0 +1,22 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Datadog.Trace.Debugger.ExceptionAutoInstrumentation +{ + internal enum ExceptionCollectionState : byte + { + Done, + Initializing, + Collecting, + Finalizing, + None + } +} diff --git a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionDebuggingProcessor.cs b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionDebuggingProcessor.cs new file mode 100644 index 000000000000..10f8ef2f0ddb --- /dev/null +++ b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionDebuggingProcessor.cs @@ -0,0 +1,259 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +using System; +using Datadog.Trace.Debugger.Configurations.Models; +using Datadog.Trace.Debugger.Expressions; +using Datadog.Trace.Debugger.Instrumentation.Collections; +using Datadog.Trace.Debugger.Snapshots; +using Datadog.Trace.Logging; +using Datadog.Trace.VendoredMicrosoftCode.System.Collections.Immutable; + +namespace Datadog.Trace.Debugger.ExceptionAutoInstrumentation +{ + internal class ExceptionDebuggingProcessor : IProbeProcessor + { + private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor(typeof(ExceptionDebuggingProcessor)); + private readonly object _lock = new(); + private ExceptionProbeProcessor[] _processors; + + internal ExceptionDebuggingProcessor(string probeId, MethodUniqueIdentifier method) + { + _processors = Array.Empty(); + ProbeId = probeId; + Method = method; + } + + public string ProbeId { get; } + + public MethodUniqueIdentifier Method { get; } + + public bool ShouldProcess(in ProbeData probeData) + { + var shadowStack = ShadowStackContainer.EnsureShadowStackEnabled(); + + if (shadowStack.CurrentStackFrameNode?.IsInvalidPath == true) + { + return false; + } + + return true; + } + + public bool Process(ref CaptureInfo info, IDebuggerSnapshotCreator inSnapshotCreator, in ProbeData probeData) + { + var snapshotCreator = (ExceptionSnapshotCreator)inSnapshotCreator; + ShadowStackTree shadowStack; + + try + { + switch (info.MethodState) + { + case MethodState.BeginLine: + case MethodState.BeginLineAsync: + break; + case MethodState.EntryStart: + case MethodState.EntryAsync: + shadowStack = ShadowStackContainer.EnsureShadowStackEnabled(); + snapshotCreator.EnterHash = shadowStack.CurrentStackFrameNode?.EnterSequenceHash ?? 0; + + var shouldProcess = false; + foreach (var processor in snapshotCreator.Processors) + { + if (processor.ShouldProcess(snapshotCreator.EnterHash)) + { + // TODO consider if the processors that successfully entered, should be used solely on Leave without any noise. + // TODO this requirement arises due to Invalidation that could happen to EnterHash if a new method pops in the call path. + shouldProcess = true; + } + } + + var enteredNode = shadowStack.Enter(info.Method, isInvalidPath: !shouldProcess); + snapshotCreator.TrackedStackFrameNode = enteredNode; + return true; + case MethodState.ExitStart: + case MethodState.ExitStartAsync: + if (snapshotCreator.TrackedStackFrameNode.TrackingFrameData.IsFrameUnwound) + { + Log.Warning("ExceptionDebuggingProcessor: Frame is already unwound. Probe Id: {ProbeId}", ProbeId); + return false; + } + + shadowStack = ShadowStackContainer.EnsureShadowStackEnabled(); + + if (snapshotCreator.TrackedStackFrameNode.IsInvalidPath) + { + shadowStack.Leave(snapshotCreator.TrackedStackFrameNode); + return false; + } + + if (info.MemberKind != ScopeMemberKind.Exception || info.Value == null) + { + shadowStack.Leave(snapshotCreator.TrackedStackFrameNode); + return false; + } + + var exception = info.Value as Exception; + snapshotCreator.TrackedStackFrameNode.LeavingException = exception; + snapshotCreator.LeaveHash = shadowStack.CurrentStackFrameNode?.LeaveSequenceHash ?? 0; + + var leavingExceptionType = info.Value.GetType(); + + foreach (var processor in snapshotCreator.Processors) + { + if (processor.Leave(leavingExceptionType, snapshotCreator)) + { + shadowStack.LeaveWithException(snapshotCreator.TrackedStackFrameNode, exception); + + // Full Snapshot / Lightweight + + var sequenceHash = snapshotCreator.TrackedStackFrameNode.SequenceHash; + if (!shadowStack.ContainsUniqueId(sequenceHash)) + { + shadowStack.AddUniqueId(sequenceHash); + snapshotCreator.TrackedStackFrameNode.CapturingStrategy = SnapshotCapturingStrategy.FullSnapshot; + } + else + { + snapshotCreator.TrackedStackFrameNode.CapturingStrategy = SnapshotCapturingStrategy.LightweightSnapshot; + } + + snapshotCreator.TrackedStackFrameNode.AddScopeMember(info.Name, info.Type, info.Value, info.MemberKind); + + return true; + } + } + + shadowStack.Leave(snapshotCreator.TrackedStackFrameNode); + return false; + case MethodState.EntryEnd: + case MethodState.EndLine: + case MethodState.EndLineAsync: + break; + case MethodState.ExitEnd: + case MethodState.ExitEndAsync: + if (snapshotCreator.TrackedStackFrameNode.CapturingStrategy != SnapshotCapturingStrategy.None) + { + snapshotCreator.TrackedStackFrameNode.AddScopeMember(info.Name, info.Type, info.Value, info.MemberKind); + + snapshotCreator.TrackedStackFrameNode.MethodMetadataIndex = info.MethodMetadataIndex; + snapshotCreator.TrackedStackFrameNode.ProbeId = ProbeId; + snapshotCreator.TrackedStackFrameNode.HasArgumentsOrLocals = info.HasLocalOrArgument; + + if (info.IsAsyncCapture()) + { + snapshotCreator.TrackedStackFrameNode.IsAsyncMethod = true; + snapshotCreator.TrackedStackFrameNode.MoveNextInvocationTarget = info.AsyncCaptureInfo.MoveNextInvocationTarget; + snapshotCreator.TrackedStackFrameNode.KickoffInvocationTarget = info.AsyncCaptureInfo.KickoffInvocationTarget; + } + + if (snapshotCreator.TrackedStackFrameNode.CapturingStrategy == SnapshotCapturingStrategy.FullSnapshot) + { + var snapshot = snapshotCreator.TrackedStackFrameNode.Snapshot; + Log.Information("Snapshot: {Snapshot}", snapshot); + } + } + + return true; + case MethodState.LogLocal: + case MethodState.LogArg: + if (snapshotCreator.TrackedStackFrameNode.CapturingStrategy != SnapshotCapturingStrategy.None) + { + snapshotCreator.TrackedStackFrameNode.AddScopeMember(info.Name, info.Type, info.Value, info.MemberKind); + } + + break; + } + } + catch (Exception e) + { + Log.Error(e, "ExceptionDebuggingProcessor: Failed to process probe. Probe Id: {ProbeId}", ProbeId); + return false; + } + + return true; + } + + public void LogException(Exception ex, IDebuggerSnapshotCreator inSnapshotCreator) + { + var snapshotCreator = (ExceptionSnapshotCreator)inSnapshotCreator; + + if (snapshotCreator.TrackedStackFrameNode == null) + { + return; + } + + var shadowStack = ShadowStackContainer.EnsureShadowStackEnabled(); + shadowStack.Leave(snapshotCreator.TrackedStackFrameNode); + } + + public IProbeProcessor UpdateProbeProcessor(ProbeDefinition probe) + { + return this; + } + + public IDebuggerSnapshotCreator CreateSnapshotCreator() + { + // ReSharper disable once InconsistentlySynchronizedField + return new ExceptionSnapshotCreator(_processors, ProbeId); + } + + public void AddProbeProcessor(ExceptionProbeProcessor processor) + { + lock (_lock) + { + var newProcessors = new ExceptionProbeProcessor[_processors.Length + 1]; + Array.Copy(_processors, newProcessors, _processors.Length); + newProcessors[newProcessors.Length - 1] = processor; + _processors = newProcessors; + } + } + + public int RemoveProbeProcessor(ExceptionProbeProcessor processor) + { + lock (_lock) + { + var index = Array.IndexOf(_processors, processor); + if (index < 0) + { + return -1; + } + + if (_processors.Length == 1) + { + _processors = Array.Empty(); + return 0; + } + + var newProcessors = new ExceptionProbeProcessor[_processors.Length - 1]; + + if (index > 0) + { + Array.Copy(_processors, 0, newProcessors, 0, index); + } + + if (index < _processors.Length - 1) + { + Array.Copy(_processors, index + 1, newProcessors, index, _processors.Length - index - 1); + } + + _processors = newProcessors; + + return _processors.Length; + } + } + + public void InvalidateEnterLeave() + { + lock (_lock) + { + foreach (var processor in _processors) + { + processor.InvalidateEnterLeave(); + } + } + } + } +} diff --git a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionProbe.cs b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionProbe.cs new file mode 100644 index 000000000000..43c99576937a --- /dev/null +++ b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionProbe.cs @@ -0,0 +1,30 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Datadog.Trace.Debugger.Configurations.Models; + +namespace Datadog.Trace.Debugger.ExceptionAutoInstrumentation +{ + internal class ExceptionProbe + { + internal ExceptionProbe(HashSet exceptionTypes, ExceptionDebuggingProbe[] parentProbes, ExceptionDebuggingProbe[] childProbes) + { + ExceptionTypes = exceptionTypes; + ParentProbes = parentProbes; + ChildProbes = childProbes; + } + + internal HashSet ExceptionTypes { get; } + + internal ExceptionDebuggingProbe[] ChildProbes { get; } + + internal ExceptionDebuggingProbe[] ParentProbes { get; } + } +} diff --git a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionProbeProcessor.cs b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionProbeProcessor.cs new file mode 100644 index 000000000000..59e305f61b2c --- /dev/null +++ b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionProbeProcessor.cs @@ -0,0 +1,241 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Datadog.Trace.Debugger.Configurations.Models; +using Datadog.Trace.Debugger.Expressions; +using Datadog.Trace.Debugger.Helpers; +using Datadog.Trace.Debugger.Instrumentation.Collections; +using Datadog.Trace.Debugger.PInvoke; +using Datadog.Trace.Debugger.Sink.Models; +using Datadog.Trace.Debugger.Snapshots; +using Datadog.Trace.Logging; + +namespace Datadog.Trace.Debugger.ExceptionAutoInstrumentation +{ + internal class ExceptionProbeProcessor + { + private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor(typeof(ExceptionProbeProcessor)); + private readonly HashSet _exceptionTypes; + private readonly Type _singleExceptionType; + private readonly ExceptionDebuggingProbe[] _childProbes; + private readonly ExceptionDebuggingProbe[] _parentProbes; + private readonly object _locker = new(); + private uint? _enterSequenceHash; + private uint? _leaveSequenceHash; + + internal ExceptionProbeProcessor(ExceptionDebuggingProbe probe, HashSet exceptionTypes, ExceptionDebuggingProbe[] parentProbes, ExceptionDebuggingProbe[] childProbes) + { + ExceptionDebuggingProcessor = probe.ExceptionDebuggingProcessor; + _exceptionTypes = exceptionTypes; + _singleExceptionType = _exceptionTypes.Count == 1 ? _exceptionTypes.Single() : null; + _childProbes = childProbes; + _parentProbes = parentProbes; + } + + internal ExceptionDebuggingProcessor ExceptionDebuggingProcessor { get; } + + internal bool ShouldProcess(uint enterSequenceHash) + { + if (!EnsureEnterHashComputed()) + { + return false; + } + + return enterSequenceHash == _enterSequenceHash; + } + + private bool EnsureEnterHashComputed() + { + if (_enterSequenceHash == null) + { + // Build _callSequenceHashing based on _parentProbes and their ProbeStatus. If we failed to instrument a method, it shall not be part of the hash. + // If there are no parents (i.e they are all failed in instrumentation, or we're the first in the chain) then the shadowStack.CallSequenceHash should also be + // an empty string. Thus keeping the check as is. + // TODO keeping it as an empty string is not good for these cases, because then we will enter into this if statement A LOT. + // Fact: The ShadowStack hashing will be expanded in the same manner the parents are built. + // TODO We can not base the callSequenceHash building on ProbeIds since it should serve a broader uses. There might be different exceptions with a similar + // TODO call path. We have to accomodate for that by buildings it on the methods. Possibly MethodToken. Watch out for Async methods. + // TODO We have to have all the child probes (as we do for parent probes) since the next probe in the chain could be broken (i.e instrumentation failure) + // TODO As such we will have to pick up the next in the chain or call it a day and consider ourselves as the first one if we propagate with an interesting + // TODO exception. + // TODO We could possibly use the same algorithm we do for callSequence of the entries, to the leaving with exceptions. Such as, we will build a similar hash + // TODO and account for failed to instrument probe statuses. It will be more accurate than merely comparing the next probe in the chain. + // TODO What do you do with recursive methods? they are normalized in ExceptionTrackManager... + // TODO Consolidate all ExceptionProbeProcessor into one per method. We want the decision to Enter/Leave a method + // TODO to be related to the method not a probe. We don't want to Enter multiple times just because we have multiple + // TODO Probes defined on that method. + // TODO Use the full qualified name of the method when requesting instrumentation. + // TODO We want to have two hashes: CallSequenceHash, LeaveSequenceHash. The first hash will be extended upon entering methods and reduce upon leaves. + // TODO the second hash will be extended upon method leave and reduce upon method enter. + // TODO When a method leaves with an exception AND has a child, the relevant child should be picked up, if any. Meaning, if you leave with exception + // TODO FooException, then your child should have the same exception instance (either itself or as inner) of the exception you leave with. + // TODO If your child does not have an exception that is correspondent with the exception you leave with, the child should be abandoned. + // TODO including all the grandchildren along the path. When leaving with a parent, the parent should have your leaving information with them. + // TODO In essence, a parent should leave with the same exception instance (as self or inner) as it's child. + // TODO This point is also relevant for ExceptionTrackManager, where the exception it reports should be correspondent with the exception of it's + // TODO active shadow stack. If the exception in the shadow stack is not the reporting exception (either self or inner), then the shadow stack + // TODO should be abandoned, and the case should stay upon of the exception the Shadow Stack had and the exception that visits the ExceptionTrackManager. + // TODO We have to be mindful when we mutate the ShadowStack. It's shared within the execution context and different ProbeProcessors of the same method + // TODO might put their lags on one another. In other words: entrances to the ProbeProcessor does not mean an entrance of a method. It might be + // TODO that there are other ProbeProcessors acting on the same method, they are all in essence should act as a single unit. + // TODO When a method leaves, only it's correct child should be kept, if any. + // TODO How can we be sure we try to capture full snapshot only once in the execution context? + // TODO Get rid of ExecutionId? + // TODO Compute the hash correctly. + // TODO Call Leave correctly on all exit paths, including LogException (when an exception is thrown from the instrumentation). + // TODO Check `return allActiveExceptions.Intersect(allChildExceptions).Any()` algorithm. + // TODO Build uniqueness based on frames that are part of existing exception cases: + // TODO When I want to instrument bunch of methods, I need to update the frames to all the cases they play part of. + // TODO The idea is to differentiate exceptions. + + lock (_locker) + { + if (_enterSequenceHash == null) + { + var instrumentedProbes = _parentProbes.Where(p => p.IsInstrumented).ToArray(); + + var probeIds = instrumentedProbes + .Where(p => p.ProbeStatus == Status.RECEIVED) + .Select(p => p.ProbeId) + .ToArray(); + + var statuses = DebuggerNativeMethods.GetProbesStatuses(probeIds); // Expensive, maybe save in a cache & create circuit-breaker on top + if (!statuses.All(status => status.Status is Status.INSTALLED or Status.ERROR)) + { + // Come back here later on, not all probes has been instrumented yet... + return false; + } + + foreach (var status in statuses) + { + var parent = instrumentedProbes.First(p => p.ProbeId == status.ProbeId); + parent.ProbeStatus = status.Status; + } + + var installedParentProbes = instrumentedProbes + .Where(parent => parent.ProbeStatus == Status.INSTALLED) + .ToArray(); + + if (!installedParentProbes.Any()) + { + // We have to mark the callSequence of current ExceptionProbeProcessor as evaluated so it won't be evaluated any more. + // There should not be any parent visible in the ShadowStack as we are left with 0 installations for the probes. meaning, there + // are either no parent or all of them has failed in instrumentation. Either way, the shadow stack should be empty upon entrance of the current method. + _enterSequenceHash = 0; + } + else + { + uint hash = 0; + + foreach (var parent in installedParentProbes) + { + hash = Fnv1aHash.ComputeHash(hash, parent.Method.MethodToken); + } + + _enterSequenceHash = hash; + } + } + } + } + + return _enterSequenceHash != null; + } + + private bool EnsureLeaveHashComputed() + { + if (_leaveSequenceHash == null) + { + lock (_locker) + { + if (_leaveSequenceHash == null) + { + var instrumentedProbes = _childProbes.Where(p => p.IsInstrumented).ToArray(); + + var probeIds = instrumentedProbes + .Where(p => p.ProbeStatus == Status.RECEIVED) + .Select(p => p.ProbeId) + .ToArray(); + + var statuses = DebuggerNativeMethods.GetProbesStatuses(probeIds); // Expensive, maybe save in a cache & create circuit-breaker on top + if (!statuses.All(status => status.Status is Status.INSTALLED or Status.ERROR)) + { + // Come back here later on, not all probes has been instrumented yet... + return false; + } + + foreach (var status in statuses) + { + var child = instrumentedProbes.First(p => p.ProbeId == status.ProbeId); + child.ProbeStatus = status.Status; + } + + var installedProbes = instrumentedProbes + .Where(parent => parent.ProbeStatus == Status.INSTALLED) + .ToArray(); + + if (!installedProbes.Any()) + { + // We have to mark the callSequence of current ExceptionProbeProcessor as evaluated so it won't be evaluated any more. + // There should not be any parent visible in the ShadowStack as we are left with 0 installations for the probes. meaning, there + // are either no parent or all of them has failed in instrumentation. Either way, the shadow stack should be empty upon entrance of the current method. + _leaveSequenceHash = 0; + } + else + { + uint hash = 0; + + foreach (var child in installedProbes.Reverse()) + { + hash = Fnv1aHash.ComputeHash(hash, child.Method.MethodToken); + } + + _leaveSequenceHash = hash; + } + } + } + } + + return _leaveSequenceHash != null; + } + + internal void InvalidateEnterLeave() + { + lock (_locker) + { + _enterSequenceHash = null; + _leaveSequenceHash = null; + } + } + + internal void Enter(ref CaptureInfo info, ExceptionSnapshotCreator snapshotCreator, in ProbeData probeData) + { + } + + internal bool Leave(Type exceptionType, ExceptionSnapshotCreator snapshotCreator) + { + if (!EnsureEnterHashComputed() || _enterSequenceHash != snapshotCreator.EnterHash) + { + return false; + } + + if (_singleExceptionType == exceptionType || _exceptionTypes.Contains(exceptionType)) + { + if (!EnsureLeaveHashComputed() || _leaveSequenceHash != snapshotCreator.LeaveHash) + { + return false; + } + + return true; + } + + return false; + } + } +} diff --git a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionRelatedFrames.cs b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionRelatedFrames.cs new file mode 100644 index 000000000000..a39f84b8bfb6 --- /dev/null +++ b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionRelatedFrames.cs @@ -0,0 +1,73 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Datadog.Trace.Debugger.ExceptionAutoInstrumentation +{ + internal class ExceptionRelatedFrames + { + public static readonly ExceptionRelatedFrames Empty = new(ex: null, frames: null); + + public ExceptionRelatedFrames(Exception ex, ParticipatingFrame[] frames) + { + Exception = ex; + Frames = frames; + } + + public ExceptionRelatedFrames(Exception ex, ParticipatingFrame[] frames, ExceptionRelatedFrames innerFrame) + : this(ex, frames) + { + InnerFrame = innerFrame; + } + + public ParticipatingFrame[] Frames { get; } + + public Exception Exception { get; } + + public ExceptionRelatedFrames InnerFrame { get; } + + public virtual IEnumerable GetAllFlattenedFrames() + { + if (InnerFrame != null) + { + // Determine if we should skip the first method of the InnerException. + // When an exception is placed as InnerException and rethrown, it leads to duplications of the rethrowing frame. + // In other words, the last method of the outer exception is presented as the first method of the inner exception. + // for example: outer trail: A -> B -> C, inner trail: C -> D -> E, to avoid being misled and think the full + // exception trail is: A -> B -> C -> C -> D -> E (while we met C only once), we try to match the last method of the outer (this.Frames) + // with the first method of the inner exception (this.InnerFrame). In case they match, we skip 1 frame so we'll have, + // considering the previously mentioned example, the following trail: A -> B -> C -> D -> E, instead of A -> B -> C -> C -> D -> E. + // For reference, see the following test: ExceptionCaughtAndRethrownAsInnerTest. + + var firstFrame = Frames?.FirstOrDefault(); + var lastFrameOfInner = InnerFrame.Frames?.FirstOrDefault(); + + var skipDuplicatedMethod = 0; + if (lastFrameOfInner?.Method == firstFrame?.Method) + { + skipDuplicatedMethod = 1; + } + + foreach (var frame in InnerFrame.GetAllFlattenedFrames().Skip(skipDuplicatedMethod)) + { + yield return frame; + } + } + + if (Frames != null) + { + foreach (var frame in Frames) + { + yield return frame; + } + } + } + } +} diff --git a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionSnapshotCreator.cs b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionSnapshotCreator.cs new file mode 100644 index 000000000000..ae5f242216e8 --- /dev/null +++ b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionSnapshotCreator.cs @@ -0,0 +1,28 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +using Datadog.Trace.Debugger.Snapshots; + +namespace Datadog.Trace.Debugger.ExceptionAutoInstrumentation +{ + internal class ExceptionSnapshotCreator : IDebuggerSnapshotCreator + { + public ExceptionSnapshotCreator(ExceptionProbeProcessor[] processors, string probeId) + { + Processors = processors; + ProbeId = probeId; + } + + public string ProbeId { get; } + + public ExceptionProbeProcessor[] Processors { get; } + + public uint EnterHash { get; set; } + + public uint LeaveHash { get; set; } + + public TrackedStackFrameNode TrackedStackFrameNode { get; set; } + } +} diff --git a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionStackNodeRecord.cs b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionStackNodeRecord.cs new file mode 100644 index 000000000000..62d4d669e35b --- /dev/null +++ b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionStackNodeRecord.cs @@ -0,0 +1,32 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +using System; +using System.Reflection; +using Datadog.Trace.Debugger.Instrumentation.Collections; + +namespace Datadog.Trace.Debugger.ExceptionAutoInstrumentation; + +internal class ExceptionStackNodeRecord +{ + public ExceptionStackNodeRecord(int level, TrackedStackFrameNode node) + { + Level = level; + ProbeId = node.ProbeId; + MethodInfo = MethodMetadataCollection.Instance.Get(node.MethodMetadataIndex); + Snapshot = node.Snapshot; + SnapshotId = node.SnapshotId; + } + + public int Level { get; } + + public string ProbeId { get; } + + public MethodMetadataInfo MethodInfo { get; } + + public string Snapshot { get; } + + public string SnapshotId { get; } +} diff --git a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionStackTreeRecord.cs b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionStackTreeRecord.cs new file mode 100644 index 000000000000..78df66823df6 --- /dev/null +++ b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionStackTreeRecord.cs @@ -0,0 +1,34 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Datadog.Trace.Debugger.ExceptionAutoInstrumentation +{ + internal class ExceptionStackTreeRecord + { + private readonly List _methods; + + public ExceptionStackTreeRecord() + { + _methods = new List(); + } + + public IList Frames => _methods.AsReadOnly(); + + public void Add(int level, TrackedStackFrameNode node) + { + _methods.Add(new ExceptionStackNodeRecord(level, node)); + } + + public void Add(ExceptionStackNodeRecord recordedMethodData) + { + _methods.Add(recordedMethodData); + } + } +} diff --git a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionTrackManager.cs b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionTrackManager.cs new file mode 100644 index 000000000000..8def45b639e8 --- /dev/null +++ b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionTrackManager.cs @@ -0,0 +1,424 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Threading; +using System.Xml.Linq; +using Datadog.Trace.Debugger.Configurations.Models; +using Datadog.Trace.Debugger.Expressions; +using Datadog.Trace.Debugger.Helpers; +using Datadog.Trace.Debugger.PInvoke; +using Datadog.Trace.Debugger.RateLimiting; +using Datadog.Trace.Debugger.Sink.Models; +using Datadog.Trace.Debugger.Symbols; +using Datadog.Trace.Logging; +using Datadog.Trace.Telemetry.Metrics; +using Datadog.Trace.Util; +using Datadog.Trace.VendoredMicrosoftCode.System.Buffers; +using Datadog.Trace.Vendors.Serilog.Events; + +namespace Datadog.Trace.Debugger.ExceptionAutoInstrumentation +{ + internal class ExceptionTrackManager + { + private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor(); + private static readonly ConcurrentDictionary TrackedExceptionCases = new(); + private static readonly ConcurrentDictionary MethodToProbe = new(); + + public static void Report(Span span, Exception exception) + { + // For V1 of Exception Debugging, we only care about exceptions propagating up the stack + // and marked as error by the service entry span (aka root span). + if (!span.IsRootSpan || exception == null || !IsSupportedExceptionType(exception)) + { + Log.Information(exception, "Skipping the processing of the exception. Span = {Span}", span); + return; + } + + if (EnvironmentHelpers.GetEnvironmentVariable("DD_EXCEPTION_DEBUGGING_ENABLED") == null) + { + return; + } + + try + { + ReportInternal(exception, ErrorOriginType.HttpRequestFailure, span); + } + catch (Exception ex) + { + Log.Error(ex, "An exception was thrown while processing an exception for tracking."); + } + } + + private static void ReportInternal(Exception exception, ErrorOriginType errorOrigin, Span rootSpan) + { + // TODO if IsShadowStackTrackingEnabled, the operation shall not be async. + // Otherwise, handle all async + + var allParticipatingFrames = GetAllExceptionRelatedStackFrames(exception); + var allParticipatingFramesFlattened = allParticipatingFrames.GetAllFlattenedFrames().Reverse().ToArray(); + + if (allParticipatingFramesFlattened.Length == 0) + { + return; + } + + if (!ShouldReportException(exception, allParticipatingFramesFlattened)) + { + Log.Information(exception, "Skipping the processing of an exception by Exception Debugging."); + return; + } + + var exceptionTypes = new HashSet(); + var currentFrame = allParticipatingFrames; + + while (currentFrame != null) + { + exceptionTypes.Add(currentFrame.Exception.GetType()); + currentFrame = currentFrame.InnerFrame; + } + + var exceptionId = new ExceptionIdentifier(exceptionTypes, allParticipatingFramesFlattened, errorOrigin); + + var trackedExceptionCase = TrackedExceptionCases.GetOrAdd(exceptionId, _ => new TrackedExceptionCase(exceptionId, TimeSpan.FromSeconds(1))); + + if (trackedExceptionCase.IsDone) + { + } + else if (trackedExceptionCase.IsCollecting) + { + Log.Information("Exception case re-occurred, data can be collected. Exception details: {FullName} {Message}.", exception.GetType().FullName, exception.Message); + + if (!ShadowStackContainer.IsShadowStackTrackingEnabled) + { + Log.Warning("The shadow stack is not enabled, while processing IsCollecting state of an exception. Exception details: {FullName} {Message}.", exception.GetType().FullName, exception.Message); + return; + } + + var resultCallStackTree = ShadowStackContainer.ShadowStack.CreateResultReport(exceptionPath: exception); + if (resultCallStackTree == null || !resultCallStackTree.Frames.Any()) + { + Log.Error("ExceptionTrackManager: Received an empty tree from the shadow stack for exception: {Exception}.", exception.ToString()); + System.Diagnostics.Debugger.Break(); + } + else + { + // TODO Ensure all snapshots present. I.e we are not in partial capturing scenario. + + // Attach tags to the root span + var debugErrorPrefix = "_dd.debug.error"; + rootSpan.Tags.SetTag("error.debug_info_captured", "true"); + rootSpan.Tags.SetTag($"{debugErrorPrefix}.exception_id", trackedExceptionCase.ErrorHash); + + var @case = trackedExceptionCase.ExceptionCase; + var frames = resultCallStackTree.Frames; + var allFrames = @case.ExceptionId.StackTrace; + var capturedFrameIndex = 0; + + for (var frameIndex = 0; frameIndex < allFrames.Length; frameIndex++) + { + if (capturedFrameIndex >= frames.Count) + { + break; + } + + var frame = frames[capturedFrameIndex]; + + if (!allFrames[frameIndex].Method.Equals(frame.MethodInfo.Method)) + { + continue; + } + + capturedFrameIndex++; + + var realIndex = allFrames.Length - frameIndex - 1 + 1; + + var prefix = $"{debugErrorPrefix}.{realIndex}."; + rootSpan.Tags.SetTag(prefix + "frame_data.function", frame.MethodInfo.Method.Name); + rootSpan.Tags.SetTag(prefix + "frame_data.class_name", frame.MethodInfo.Method.DeclaringType.Name); + rootSpan.Tags.SetTag(prefix + "snapshot_id", frame.SnapshotId); + + LiveDebugger.Instance.AddSnapshot(frame.ProbeId, frame.Snapshot); + } + + if (trackedExceptionCase.BeginTeardown()) + { + foreach (var probe in trackedExceptionCase.ExceptionCase.Probes) + { + probe.RemoveExceptionCase(trackedExceptionCase.ExceptionCase); + } + + var revertProbeIds = new HashSet(); + + foreach (var processor in trackedExceptionCase.ExceptionCase.Processors.Keys) + { + if (processor.ExceptionDebuggingProcessor.RemoveProbeProcessor(processor) == 0) + { + MethodToProbe.TryRemove(processor.ExceptionDebuggingProcessor.Method, out _); + revertProbeIds.Add(processor.ExceptionDebuggingProcessor.ProbeId); + } + } + + if (revertProbeIds.Count > 0) + { + Log.Information("ExceptionTrackManager: Reverting {RevertCount} Probes.", revertProbeIds.Count.ToString()); + + var removeProbesRequests = revertProbeIds.Select(p => new NativeRemoveProbeRequest(p)).ToArray(); + DebuggerNativeMethods.InstrumentProbes( + Array.Empty(), + Array.Empty(), + Array.Empty(), + removeProbesRequests); + } + + trackedExceptionCase.EndTeardown(); + } + } + } + else + { + // else - If there is a concurrent initialization or tearing down, ignore this case + if (!trackedExceptionCase.Initialized()) + { + return; + } + + Log.Information("New exception case occurred, initiating data collection for exception: {Name}, Message: {Message}, StackTrace: {StackTrace}", exception.GetType().Name, exception.Message, exception.StackTrace); + + trackedExceptionCase.ExceptionCase = InstrumentFrames(trackedExceptionCase.ExceptionIdentifier); + + trackedExceptionCase.BeginCollect(); + } + } + + private static bool ShouldReportException(Exception ex, ParticipatingFrame[] framesToRejit) + { + try + { + // Both OutOfMemoryException and ThreadAbortException may be thrown anywhere from non-deterministic reasons, which means it won't + // necessarily re-occur with the same callstack. We should not attempt to rejit methods nor track these exceptions. + + // ThreadAbortException is particularly problematic because is it often thrown in legacy ASP.NET apps whenever + // there is an HTTP Redirect. See also https://stackoverflow.com/questions/2777105/why-response-redirect-causes-system-threading-threadabortexception + if (ex is OutOfMemoryException || ex is ThreadAbortException) + { + return false; + } + + return AtLeastOneFrameBelongToUserCode() && ThereIsNoFrameThatBelongsToDatadogClrProfilerAgentCode(); + } + catch + { + // When in doubt, report it. + return true; + } + + bool AtLeastOneFrameBelongToUserCode() => framesToRejit.All(f => !FrameFilter.IsUserCode(f)) == false; + bool ThereIsNoFrameThatBelongsToDatadogClrProfilerAgentCode() => framesToRejit.Any(f => FrameFilter.IsDatadogAssembly(f.Method.Module.Assembly.GetName().Name)) == false; + } + + private static bool IsSupportedExceptionType(Exception ex) => + IsSupportedExceptionType(ex?.GetType()); + + public static bool IsSupportedExceptionType(Type ex) => + ex != typeof(BadImageFormatException) && + ex != typeof(InvalidProgramException) && + ex != typeof(TypeInitializationException) && + ex != typeof(TypeLoadException) && + ex != typeof(OutOfMemoryException); + + private static List GetMethodsToRejit(ParticipatingFrame[] allFrames) + { + var methodsToRejit = new List(); + + foreach (var frame in allFrames) + { + try + { + // HasMethod? + + if (frame.State == ParticipatingFrameState.Blacklist) + { + continue; + } + + var frameMethod = frame.Method; + if (frameMethod.IsAbstract) + { + continue; + } + + methodsToRejit.Add(frame.MethodIdentifier); + } + catch (Exception ex) + { + Log.Error(ex, "Failed to instrument frame the frame: {FrameToRejit}", frame); + } + } + + return methodsToRejit; + } + + private static ExceptionCase InstrumentFrames(ExceptionIdentifier exceptionIdentifier) + { + var participatingUserMethods = GetMethodsToRejit(exceptionIdentifier.StackTrace); + + var uniqueMethods = participatingUserMethods + .Distinct(EqualityComparer.Default) + .ToArray(); + + var neverSeenBeforeMethods = uniqueMethods + .Where(frame => !MethodToProbe.ContainsKey(frame)) + .ToArray(); + + foreach (var frame in neverSeenBeforeMethods) + { + MethodToProbe.TryAdd(frame, new ExceptionDebuggingProbe(frame)); + } + + var probes = participatingUserMethods.Select((m, frameIndex) => MethodToProbe[m]).ToArray(); + + var lastNElements = 5; + var thresholdIndex = participatingUserMethods.Count - lastNElements; + var targetMethods = new HashSet(); + + for (var index = 0; index < probes.Length; index++) + { + if (ShouldInstrumentFrameAtIndex(index)) + { + targetMethods.Add(probes[index].Method); + } + } + + var newCase = new ExceptionCase(exceptionIdentifier, probes); + + foreach (var method in uniqueMethods) + { + var probe = MethodToProbe[method]; + probe.AddExceptionCase(newCase, targetMethods.Contains(method)); + } + + // TODO decide if a sampler is needed, ExceptionProbeProcessor does not use any sampler for now. + // TODO InnerExceptions poses struggle in ExceptionProbeProcessor leaving logic. + // TODO Capture arguments on exit upon first leave, collect lightweight snapshot for subsequent re-entrances. + // TODO AsyncLocal cleansing when done dealing with exception from the Exception Debugging instrumentation (ShadowStack cleansing) + // TODO In ExceptionProbeProcessor.ShouldProcess, maybe negotiate with the ShadowStack to determine if the top of the stack + // TODO is relevant for the specific exception case it manages. Maybe instead of ShouldProcess we can do that + // TODO in the Process method, in the branch where the exception type is checked to see if the previous method is relevant. + // TODO there's a gotcha in doing it - it might be the next method has not been instrumented (failed to instrument) + // TODO so it won't be there because it should. We will have to accommodate for that by checking the probe status and cache it. + // TODO When leaving with an exception, we can negotiate with the ShadowStack to determine if the previous frame + // TODO Is holding the same exception instance (either as inner / itself) to better decide if we should keep on collecting + // TODO or not. + // TODO Multiple AppDomains issue. The ProbeProcessor might not be there. Also relevant for DI probes. To assess how big + // TODO the issue is, we should determine how many people are using .NET Framework .VS. .NET Core. + // TODO For Exception Debugging we can possibly choose to ditch this altogether since if the same exception will + // TODO happen multiple times in different AppDomains, then they will all capture the exception. The only problem is + // TODO over-instrumenting which is not ideal. + // TODO In AsyncMethodProbe Invoker, is it always MultiProbe even when there is only one? + // TODO What do you do with empty shadow stack? meaning, all the participating methods has failed in the instrumentation process OR they are all 3rd party code? + // TODO There might be two different exceptions, that yield the same snapshots. Consider A -> B -> C with exception "InvalidOperationException" + // TODO and K -> B -> D with exception "InvalidOperationException". If we fail to instrument: A, B, K, D then there will be the same causality chain for both exceptions. + // TODO That's why ExceptionTrackManager is the only place where snapshots are uploaded, based on the exception in hand, to be able to stop tracking an exception + // TODO and keep on tracking the other. + // TODO For Lightweight/Full snapshot capturing: + // TODO Consider keeping a cache in ShadowStackTree's AsyncLocal (in ShadowStackContainer), where the cached key + // TODO will be the hash of parents & children (Enter/Leave) and the MethodToken of the method. This way, + // TODO the method that is leaving with an interesting exception can ask this AsyncLocal (top-thread-tree) cache + // TODO if it's hash (EnterHash+LeaveHash+MethodToken) is in there. If it is, collect lightweight snapshot. + // TODO if it's not, collect full snapshot. + // TODO In this technique we will have to verify AsyncLocal safety in terms of memory leaking and the cleansing timing. + // TODO we don't want this cache to be alive for a longer time than is needed or being reused by another execution + // TODO context in a later time. This cache will have to be thread-safe since many threads may access it at the same + // TODO time. Consider using Readers/Writer lock pattern or another one that is prioritizing readings than writings. + // TODO Or any other lock-free pattern that may be suitable in this case. + // TODO Better handle multiple exceptions related to concurrency - AggregateException. It's InnerException & + // TODO InnerExceptions properties. + + return newCase; + + bool ShouldInstrumentFrameAtIndex(int i) + { + return i == 0 || i >= thresholdIndex || participatingUserMethods.Count <= lastNElements + 1; + } + } + + public static ExceptionRelatedFrames GetAllExceptionRelatedStackFrames(Exception exception) + { + if (exception == null) + { + return ExceptionRelatedFrames.Empty; + } + + return CreateExceptionPath(exception, true); + + ExceptionRelatedFrames CreateExceptionPath(Exception e, bool isTopFrame) + { + var frames = GetParticipatingFrames(new StackTrace(e, false), isTopFrame, ParticipatingFrameState.Default); + + ExceptionRelatedFrames innerFrame = null; + + // the first inner exception in the inner exceptions list of the aggregate exception is similar to the inner exception` + innerFrame = e.InnerException != null ? CreateExceptionPath(e.InnerException, false) : null; + + return new ExceptionRelatedFrames(e, frames.ToArray(), innerFrame); + } + } + + /// + /// Getting a stack trace and creating list of ParticipatingFrame according to requested parameters and filters + /// + /// Stack trace to get frames from + /// If it's a top frame, we should skip on the above method (e.g. ASP Net methods) + /// Default state of all method that are not + /// All the frames of the exception. + public static IEnumerable GetParticipatingFrames(StackTrace stackTrace, bool isTopFrame, ParticipatingFrameState defaultState) + { + var frames = isTopFrame + ? stackTrace.GetFrames()?. + Reverse(). + SkipWhile(frame => FrameFilter.ShouldSkipNamespaceIfOnTopOfStack(frame.GetMethod())). + Reverse(). + GetAsyncFriendlyFrameMethods() + : stackTrace.GetFrames()?.GetAsyncFriendlyFrameMethods(); + + if (frames == null) + { + yield break; + } + + foreach (var frame in frames) + { + var method = frame?.GetMethod(); + + if (method == null) + { + continue; + } + + var assembly = method.Module.Assembly; + var assemblyName = assembly.GetName().Name; + + if (FrameFilter.IsDatadogAssembly(assemblyName)) + { + continue; + } + + if (FrameFilter.ShouldSkip(method)) + { + yield return new ParticipatingFrame(frame, ParticipatingFrameState.Blacklist); + } + else + { + yield return new ParticipatingFrame(frame, defaultState); + } + } + } + } +} diff --git a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/Fnv1aHash.cs b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/Fnv1aHash.cs new file mode 100644 index 000000000000..4ffefb0be52b --- /dev/null +++ b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/Fnv1aHash.cs @@ -0,0 +1,30 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +using System; +using System.Text; + +namespace Datadog.Trace.Debugger.ExceptionAutoInstrumentation +{ + internal static class Fnv1aHash + { + private const uint FnvPrime = 16777619; + private const uint OffsetBasis = 2166136261; + + public static uint ComputeHash(uint previousHash, int input) + { + var hash = previousHash == 0 ? OffsetBasis : previousHash; + + // Processing the bytes of the integer `input` + foreach (var data in BitConverter.GetBytes(input)) + { + hash ^= data; + hash *= FnvPrime; + } + + return hash; + } + } +} diff --git a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/FrameFilter.cs b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/FrameFilter.cs new file mode 100644 index 000000000000..bf4d83d9a244 --- /dev/null +++ b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/FrameFilter.cs @@ -0,0 +1,130 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Reflection.Emit; +using System.Threading; +using System.Threading.Tasks; +using Datadog.Trace.Debugger.Helpers; +using Datadog.Trace.Debugger.Symbols; +using Datadog.Trace.Logging; +using Datadog.Trace.VendoredMicrosoftCode.System.Collections.Immutable; + +namespace Datadog.Trace.Debugger.ExceptionAutoInstrumentation +{ + internal static class FrameFilter + { + private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor(typeof(FrameFilter)); + private static readonly HashSet ThirdPartyModuleNames = new() + { + "Serilog.AspNetCore" + }; + + internal static bool IsDatadogAssembly(string assemblyName) + { + return assemblyName?.StartsWith("datadog.", StringComparison.OrdinalIgnoreCase) == true; + } + + internal static bool IsUserCode(in ParticipatingFrame participatingFrameToRejit) + { + return IsUserCode(participatingFrameToRejit.Method); + } + + private static bool IsUserCode(MethodBase method) + { + if (method == null) + { + return false; + } + + var declaringType = method.DeclaringType; + + if (declaringType == null) + { + return false; + } + + var namespaceName = declaringType.Namespace; + + if (namespaceName == null) + { + return false; + } + + return ShouldSkip(method) == false; + } + + internal static bool ShouldSkip(MethodBase method) + { + if (method == null) + { + return true; + } + + if (method is DynamicMethod || method.Module.Assembly.IsDynamic) + { + return true; + } + + if (method.GetType().Name.Equals("RTDynamicMethod", StringComparison.OrdinalIgnoreCase)) + { + return true; + } + + var moduleName = GetModuleNameWithoutExtension(method.Module.Name); + + return string.IsNullOrEmpty(moduleName) || + ThirdPartyModuleNames.Contains(moduleName) || + AssemblyFilter.ShouldSkipAssembly(method.Module.Assembly); + } + + private static string GetModuleNameWithoutExtension(string moduleName) + { + if (string.IsNullOrEmpty(moduleName)) + { + return moduleName; + } + + try + { + var lastPeriod = moduleName.LastIndexOf('.'); + if (lastPeriod == -1) + { + return moduleName; + } + + if (lastPeriod == moduleName.Length - 1) + { + return moduleName.Substring(0, moduleName.Length - 1); + } + + var ext = moduleName.Remove(0, lastPeriod + 1).ToLower(); + + if (ext == "dll" || + ext == "exe" || + ext == "so") + { + return moduleName.Substring(0, lastPeriod); + } + + return moduleName; + } + catch (Exception e) + { + Log.Error(e, "Failed tlo get the name of {ModuleName} without extension", moduleName); + + throw; + } + } + + internal static bool ShouldSkipNamespaceIfOnTopOfStack(MethodBase method) + { + return ShouldSkip(method); + } + } +} diff --git a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/MD5HashProvider.cs b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/MD5HashProvider.cs new file mode 100644 index 000000000000..48c3989c1a8d --- /dev/null +++ b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/MD5HashProvider.cs @@ -0,0 +1,19 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +using System.Collections.Generic; +using System.Linq; +using Datadog.Trace.Debugger.Helpers; + +namespace Datadog.Trace.Debugger.ExceptionAutoInstrumentation +{ + internal static class MD5HashProvider + { + internal static string GetHash(ExceptionIdentifier exceptionId) + { + return ((byte)exceptionId.ErrorOrigin + string.Concat(exceptionId.ExceptionTypes.Select(ex => ex.FullName)) + string.Concat(exceptionId.StackTrace.Select(method => method.Method.GetFullyQualifiedName()))).ToUUID(); + } + } +} diff --git a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/MethodUniqueIdentifier.cs b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/MethodUniqueIdentifier.cs new file mode 100644 index 000000000000..23f8872e3629 --- /dev/null +++ b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/MethodUniqueIdentifier.cs @@ -0,0 +1,281 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Reflection; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Datadog.Trace.Debugger.Expressions; +using Datadog.Trace.Debugger.PInvoke; +using Datadog.Trace.Debugger.RateLimiting; +using Datadog.Trace.Debugger.Sink.Models; +using Datadog.Trace.Vendors.Serilog; + +namespace Datadog.Trace.Debugger.ExceptionAutoInstrumentation +{ + internal readonly record struct MethodUniqueIdentifier(Guid Mvid, int MethodToken, MethodBase Method) + { + public override int GetHashCode() + { + return HashCode.Combine(Mvid, MethodToken); + } + } + + internal readonly struct ExceptionCase : IEquatable + { + private readonly int _hashCode; + + public ExceptionCase(ExceptionIdentifier exceptionId, ExceptionDebuggingProbe[] probes) + { + ExceptionId = exceptionId; + Probes = probes; + _hashCode = ComputeHashCode(); + } + + public ExceptionIdentifier ExceptionId { get; } + + public ExceptionDebuggingProbe[] Probes { get; } + + public ConcurrentDictionary Processors { get; } = new(); + + private int ComputeHashCode() + { + var hashCode = new HashCode(); + + foreach (var probe in Probes) + { + hashCode.Add(probe); + } + + return hashCode.ToHashCode(); + } + + public bool Equals(ExceptionCase other) + { + return Probes.SequenceEqual(other.Probes); + } + + public override bool Equals(object obj) + { + if (obj.GetType() != this.GetType()) + { + return false; + } + + return Equals((ExceptionCase)obj); + } + + public override int GetHashCode() + { + return _hashCode; + } + } + + internal readonly struct ExceptionIdentifier : IEquatable + { + private readonly int _hashCode; + + public ExceptionIdentifier(HashSet exceptionTypes, ParticipatingFrame[] stackTrace, ErrorOriginType errorOrigin) + { + ExceptionTypes = exceptionTypes; + StackTrace = stackTrace; + ErrorOrigin = errorOrigin; + _hashCode = ComputeHashCode(); + } + + public HashSet ExceptionTypes { get; } + + public ParticipatingFrame[] StackTrace { get; } + + public ErrorOriginType ErrorOrigin { get; } + + private int ComputeHashCode() + { + var hashCode = new HashCode(); + hashCode.Add(ErrorOrigin); + + foreach (var exceptionType in ExceptionTypes) + { + hashCode.Add(exceptionType); + } + + foreach (var frame in StackTrace) + { + hashCode.Add(frame); + } + + return hashCode.ToHashCode(); + } + + public bool Equals(ExceptionIdentifier other) + { + return ErrorOrigin == other.ErrorOrigin && + ExceptionTypes.SequenceEqual(other.ExceptionTypes) && + StackTrace.SequenceEqual(other.StackTrace); + } + + public override bool Equals(object obj) + { + if (obj.GetType() != this.GetType()) + { + return false; + } + + return Equals((ExceptionIdentifier)obj); + } + + public override int GetHashCode() + { + return _hashCode; + } + } + + internal class ExceptionDebuggingProbe + { + private readonly int _hashCode; + private readonly object _locker = new(); + private readonly List _exceptionCases = new(); + private int _isInstrumented = 0; + + public ExceptionDebuggingProbe(MethodUniqueIdentifier method) + { + Method = method; + _hashCode = ComputeHashCode(); + } + + internal string ProbeId { get; private set; } + + internal MethodUniqueIdentifier Method { get; } + + internal ExceptionDebuggingProcessor ExceptionDebuggingProcessor { get; private set; } + + internal Status ProbeStatus { get; set; } + + internal bool IsInstrumented + { + get + { + return _isInstrumented == 1 && ExceptionDebuggingProcessor != null && !string.IsNullOrEmpty(ProbeId); + } + } + + private bool ShouldInstrument() + { + if (Interlocked.CompareExchange(ref _isInstrumented, 1, 0) == 0) + { + ProbeId = Guid.NewGuid().ToString(); + ExceptionDebuggingProcessor = new ExceptionDebuggingProcessor(ProbeId, Method); + + return true; + } + + return false; + } + + private void ProcessCase(ExceptionCase @case) + { + if (!IsInstrumented) + { + return; + } + + var probes = @case.Probes; + + for (var index = 0; index < probes.Length; index++) + { + var probe = probes[index]; + + if (probe.Method.Equals(Method)) + { + var parentProbes = probes.Take(index).ToArray(); + var childProbes = probes.Skip(index + 1).ToArray(); + + var processor = new ExceptionProbeProcessor(probe, @case.ExceptionId.ExceptionTypes, parentProbes: parentProbes, childProbes: childProbes); + @case.Processors.TryAdd(processor, 0); + ExceptionDebuggingProcessor.AddProbeProcessor(processor); + } + } + + foreach (var probe in probes.Where(p => p.IsInstrumented)) + { + probe.ExceptionDebuggingProcessor.InvalidateEnterLeave(); + } + } + + internal void AddExceptionCase(ExceptionCase @case, bool isPartOfCase) + { + lock (_locker) + { + if (isPartOfCase && ShouldInstrument()) + { + foreach (var existingCase in _exceptionCases) + { + ProcessCase(existingCase); + } + + // We don't care about sampling Exception Probes. To save memory, NopAdaptiveSampler is used. + ProbeRateLimiter.Instance.TryAddSampler(ProbeId, NopAdaptiveSampler.Instance); + if (!ProbeExpressionsProcessor.Instance.TryAddProbeProcessor(ProbeId, ExceptionDebuggingProcessor)) + { + Log.Error("Could not add ExceptionDebuggingProcessor. Method: {TypeName}.{MethodName}", Method.Method.DeclaringType.Name, Method.Method.Name); + System.Diagnostics.Debugger.Break(); + } + + // TODO use full name + var rejitRequest = new NativeMethodProbeDefinition(ProbeId, Method.Method.DeclaringType.Name, Method.Method.Name, targetParameterTypesFullName: null); + + DebuggerNativeMethods.InstrumentProbes( + new[] { rejitRequest }, + Array.Empty(), + Array.Empty(), + Array.Empty()); + } + + _exceptionCases.Add(@case); + ProcessCase(@case); + } + } + + internal void RemoveExceptionCase(ExceptionCase @case) + { + lock (_locker) + { + _exceptionCases.Remove(@case); + } + } + + private int ComputeHashCode() + { + var hashCode = new HashCode(); + hashCode.Add(Method); + return hashCode.ToHashCode(); + } + + public bool Equals(ExceptionDebuggingProbe other) + { + return Method.Equals(other.Method); + } + + public override bool Equals(object obj) + { + if (obj.GetType() != this.GetType()) + { + return false; + } + + return Equals((ExceptionDebuggingProbe)obj); + } + + public override int GetHashCode() + { + return _hashCode; + } + } +} diff --git a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ParticipatingFrame.cs b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ParticipatingFrame.cs new file mode 100644 index 000000000000..58af16323d44 --- /dev/null +++ b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ParticipatingFrame.cs @@ -0,0 +1,66 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +using System; +using System.Diagnostics; +using System.Reflection; + +namespace Datadog.Trace.Debugger.ExceptionAutoInstrumentation +{ + internal enum ParticipatingFrameState + { + Default, + Blacklist + } + + internal readonly struct ParticipatingFrame + { + private const int UndefinedIlOffset = -1; + + private ParticipatingFrame(MethodBase method, ParticipatingFrameState state, int ilOffset = UndefinedIlOffset, bool isInBlackList = false) + { + Method = method; + MethodIdentifier = new MethodUniqueIdentifier(method.Module.ModuleVersionId, method.MetadataToken, method); + IsInBlackList = isInBlackList; + ILOffset = ilOffset; + State = state; + } + + public ParticipatingFrame(StackFrame stackFrame, ParticipatingFrameState state, bool isInBlackList = false) + : this(stackFrame?.GetMethod(), state, stackFrame?.GetILOffset() ?? UndefinedIlOffset, isInBlackList) + { + } + + public ParticipatingFrameState State { get; } + + public MethodBase Method { get; } + + public MethodUniqueIdentifier MethodIdentifier { get; } + + public bool IsInBlackList { get; } + + public int ILOffset { get; } + + public override int GetHashCode() + { + return MethodIdentifier.GetHashCode(); + } + + public bool Equals(ParticipatingFrame other) + { + return MethodIdentifier.Equals(other.MethodIdentifier); + } + + public override bool Equals(object obj) + { + if (obj.GetType() != this.GetType()) + { + return false; + } + + return Equals((ParticipatingFrame)obj); + } + } +} diff --git a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ShadowStackContainer.cs b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ShadowStackContainer.cs new file mode 100644 index 000000000000..9aacd43b3a76 --- /dev/null +++ b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ShadowStackContainer.cs @@ -0,0 +1,42 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +using System; +using System.Threading; +using System.Threading.Tasks; +using Datadog.Trace.Logging; + +namespace Datadog.Trace.Debugger.ExceptionAutoInstrumentation +{ + internal class ShadowStackContainer + { + private static readonly IDatadogLogger Logger = DatadogLogging.GetLoggerFor(); + private static readonly AsyncLocal ShadowStackTree = new(); + [ThreadStatic] + private static ShadowStackTree _lastShadowStackTreeOnThisThread; + + public static ShadowStackTree ShadowStack + { + get => ShadowStackTree.Value ?? _lastShadowStackTreeOnThisThread; + set => ShadowStackTree.Value = value; + } + + public static bool IsShadowStackTrackingEnabled => ShadowStack != null; + + public static ShadowStackTree EnsureShadowStackEnabled() + { + ShadowStackTree.Value ??= new ShadowStackTree(); + _lastShadowStackTreeOnThisThread = ShadowStackTree.Value; + return _lastShadowStackTreeOnThisThread; + } + + public static void DisableShadowStackTracking() + { + Logger.Information("DisableShadowStackTracking called on threadID {ManagedThreadId} and taskId {CurrentId}.", Thread.CurrentThread.ManagedThreadId, Task.CurrentId); + ShadowStackTree.Value = null; + _lastShadowStackTreeOnThisThread = null; + } + } +} diff --git a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ShadowStackTree.cs b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ShadowStackTree.cs new file mode 100644 index 000000000000..53bf2a740281 --- /dev/null +++ b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ShadowStackTree.cs @@ -0,0 +1,158 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using System.Xml; +using Datadog.Trace.Debugger.Instrumentation.Collections; +using Datadog.Trace.Logging; +using Datadog.Trace.Vendors.Serilog.Events; + +namespace Datadog.Trace.Debugger.ExceptionAutoInstrumentation +{ + internal class ShadowStackTree + { + private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor(typeof(ShadowStackTree)); + + private readonly AsyncLocal _trackedStackFrameActiveNode = new(); + private readonly HashSet _uniqueSequencesLeaves = new(); + private readonly ReaderWriterLockSlim _lock = new(); + private TrackedStackFrameNode _trackedStackFrameRootNode; + + public TrackedStackFrameNode CurrentStackFrameNode => _trackedStackFrameActiveNode.Value; + + public TrackedStackFrameNode Enter(MethodBase method, bool isInvalidPath = false) + { + var trackingFrameData = new TrackingFrameData(method); + _trackedStackFrameActiveNode.Value = new TrackedStackFrameNode(_trackedStackFrameActiveNode.Value, ref trackingFrameData, isInvalidPath); + return _trackedStackFrameActiveNode.Value; + } + + public void LeaveWithException(TrackedStackFrameNode trackedStackFrameNode, Exception ex) + { + var currentActiveNode = _trackedStackFrameActiveNode.Value; + + if (trackedStackFrameNode != currentActiveNode) + { + System.Diagnostics.Debugger.Break(); + } + + var parent = currentActiveNode.RecordFunctionExit(ex); + + if (parent == null) + { + _trackedStackFrameRootNode = currentActiveNode; + } + + _trackedStackFrameActiveNode.Value = parent; + } + + public void Leave(TrackedStackFrameNode trackedStackFrameNode) + { + var currentActiveNode = _trackedStackFrameActiveNode.Value; + + if (trackedStackFrameNode != currentActiveNode) + { + System.Diagnostics.Debugger.Break(); + } + + _trackedStackFrameActiveNode.Value = currentActiveNode.RecordFunctionExit(); + } + + public ExceptionStackTreeRecord CreateResultReport(Exception exceptionPath, int stackSize = int.MaxValue) + { + var tree = new ExceptionStackTreeRecord(); + + if (_trackedStackFrameRootNode == null && _trackedStackFrameActiveNode.Value == null) + { + Log.Warning($"{nameof(ShadowStackTree)}: returning an empty tree."); + return tree; + } + + var rootNode = _trackedStackFrameRootNode ?? _trackedStackFrameActiveNode.Value; + + if (!rootNode.HasChildException(exceptionPath)) + { + Log.Warning("The root node has a different exception. Root Node Exception: {RootNodeException}, Reported Exception: {ReportedException}", rootNode.LeavingException?.ToString(), exceptionPath?.ToString()); + return tree; + } + + var knownNodes = new Stack>(); + knownNodes.Push(new Tuple(0, rootNode)); + + while (knownNodes.Count > 0 && tree.Frames.Count < stackSize) + { + var node = knownNodes.Pop(); + var level = node.Item1; + var trackedStackFrame = node.Item2; + + if (trackedStackFrame.IsDisposed) + { + throw new InvalidOperationException("Attempting to generate result report from a TrackedStackFrameNode which has already been disposed."); + } + + if (trackedStackFrame.CapturingStrategy != SnapshotCapturingStrategy.None) + { + tree.Add(level, trackedStackFrame); + } + + if (trackedStackFrame.ChildNodes == null || !trackedStackFrame.ChildNodes.Any()) + { + break; + } + + // TODO multiple children in AggregateException? + knownNodes.Push(new Tuple(level + 1, trackedStackFrame.ChildNodes.First())); + } + + return tree; + } + + public bool AddUniqueId(uint id) + { + _lock.EnterWriteLock(); + try + { + return _uniqueSequencesLeaves.Add(id); + } + finally + { + _lock.ExitWriteLock(); + } + } + + public bool ContainsUniqueId(uint id) + { + _lock.EnterReadLock(); + try + { + return _uniqueSequencesLeaves.Contains(id); + } + finally + { + _lock.ExitReadLock(); + } + } + + public bool RemoveUniqueId(uint id) + { + _lock.EnterWriteLock(); + try + { + return _uniqueSequencesLeaves.Remove(id); + } + finally + { + _lock.ExitWriteLock(); + } + } + } +} diff --git a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/SnapshotCapturingStrategy.cs b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/SnapshotCapturingStrategy.cs new file mode 100644 index 000000000000..429d69bb1a93 --- /dev/null +++ b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/SnapshotCapturingStrategy.cs @@ -0,0 +1,13 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +namespace Datadog.Trace.Debugger.ExceptionAutoInstrumentation; + +internal enum SnapshotCapturingStrategy +{ + None, + FullSnapshot, + LightweightSnapshot +} diff --git a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/TrackedExceptionCase.cs b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/TrackedExceptionCase.cs new file mode 100644 index 000000000000..cbb611792dbb --- /dev/null +++ b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/TrackedExceptionCase.cs @@ -0,0 +1,106 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +using System; +using System.Threading; +using Datadog.Trace.Debugger.RateLimiting; + +namespace Datadog.Trace.Debugger.ExceptionAutoInstrumentation +{ + /// + /// None: Initialization has not been started (yet) + /// Initialization and teardown process takes time and can happen concurrently + /// Initialized: rejit for all exception related methods have been requested + /// IsCollecting: all exception related methods have been rejited and data is being collected + /// IsCreatingSnapshot: there is a full snapshot (not partial) that is yet to be handled by the snapshot factory ("in flight"). + /// Done: Exception information was collected and reported + /// + internal class TrackedExceptionCase + { + private volatile int _initializationOrTearDownInProgress; + + public TrackedExceptionCase(ExceptionIdentifier exceptionId, TimeSpan windowDuration) + { + ExceptionIdentifier = exceptionId; + ErrorHash = MD5HashProvider.GetHash(exceptionId); + StartCollectingTime = DateTime.MaxValue; + Sampler = new AdaptiveSampler(windowDuration, 1, 180, 16, null); + } + + public bool IsCollecting => TrackingExceptionCollectionState == ExceptionCollectionState.Collecting; + + public ExceptionIdentifier ExceptionIdentifier { get; } + + public string ErrorHash { get; } + + public ExceptionCollectionState TrackingExceptionCollectionState { get; private set; } = ExceptionCollectionState.None; + + public bool IsDone => TrackingExceptionCollectionState == ExceptionCollectionState.Finalizing || TrackingExceptionCollectionState == ExceptionCollectionState.Done; + + public DateTime StartCollectingTime { get; private set; } + + public ExceptionCase ExceptionCase { get; set; } + + public AdaptiveSampler Sampler { get; } + + private bool BeginInProgressState(ExceptionCollectionState exceptionCollectionState) + { + if (Interlocked.CompareExchange(ref _initializationOrTearDownInProgress, 1, 0) == 0) + { + TrackingExceptionCollectionState = exceptionCollectionState; + return true; + } + + return false; + } + + private void EndInProgressState(ExceptionCollectionState exceptionCollectionState) + { + TrackingExceptionCollectionState = exceptionCollectionState; + _initializationOrTearDownInProgress = 0; + } + + public bool Initialized() + { + return BeginInProgressState(ExceptionCollectionState.Initializing); + } + + public void BeginCollect() + { + EndInProgressState(ExceptionCollectionState.Collecting); + StartCollectingTime = DateTime.UtcNow; + } + + public bool BeginTeardown() + { + return BeginInProgressState(ExceptionCollectionState.Finalizing); + } + + public void EndTeardown() + { + EndInProgressState(ExceptionCollectionState.Done); + } + + public override int GetHashCode() + { + return ExceptionIdentifier.GetHashCode(); + } + + public override bool Equals(object obj) + { + if (obj is TrackedExceptionCase trackedExceptionCase) + { + if (ReferenceEquals(this, trackedExceptionCase)) + { + return true; + } + + return trackedExceptionCase.ExceptionIdentifier.Equals(ExceptionIdentifier); + } + + return false; + } + } +} diff --git a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/TrackedStackFrameNode.cs b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/TrackedStackFrameNode.cs new file mode 100644 index 000000000000..95a7da2bf2ca --- /dev/null +++ b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/TrackedStackFrameNode.cs @@ -0,0 +1,324 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +using System; +using System.Collections.Generic; +using System.Linq; +using Datadog.Trace.Debugger.Expressions; +using Datadog.Trace.Debugger.Instrumentation.Collections; +using Datadog.Trace.Debugger.Snapshots; + +namespace Datadog.Trace.Debugger.ExceptionAutoInstrumentation +{ + internal class TrackedStackFrameNode + { + private TrackedStackFrameNode _parent; + private List _activeChildNodes; + private bool _disposed; + private uint? _enterSequenceHash; + private uint? _leaveSequenceHash; + private bool _childNodesAlreadyCleansed; + private string _snapshot; + private string _snapshotId; + + public TrackedStackFrameNode(TrackedStackFrameNode parent, ref TrackingFrameData trackingFrameData, bool isInvalidPath = false) + { + _parent = parent; + TrackingFrameData = trackingFrameData; + IsInvalidPath = isInvalidPath; + } + + public uint EnterSequenceHash + { + get + { + _enterSequenceHash ??= ComputeEnterSequenceHash(); + return _enterSequenceHash.Value; + } + } + + public uint LeaveSequenceHash + { + get + { + _leaveSequenceHash ??= ComputeLeaveSequenceHash(); + return _leaveSequenceHash.Value; + } + } + + public uint SequenceHash + { + get + { + return EnterSequenceHash ^ LeaveSequenceHash; + } + } + + public TrackedStackFrameNode Parent => _parent; + + public Exception LeavingException { get; set; } + + public string Snapshot + { + get + { + _snapshot ??= CapturingStrategy == SnapshotCapturingStrategy.None ? string.Empty : CreateSnapshot(); + return _snapshot; + } + } + + public string SnapshotId + { + get + { + if (_snapshotId == null) + { + _ = Snapshot; + } + + return _snapshotId; + } + } + + public bool IsDisposed => _disposed; + + public bool IsInvalidPath { get; } + + public IEnumerable ChildNodes => _activeChildNodes?.ToList() ?? Enumerable.Empty(); + + public TrackingFrameData TrackingFrameData { get; private set; } + + internal string ProbeId { get; set; } + + internal MethodScopeMembers Members { get; set; } + + internal int MethodMetadataIndex { get; set; } + + internal bool IsAsyncMethod { get; set; } + + internal object MoveNextInvocationTarget { get; set; } + + internal object KickoffInvocationTarget { get; set; } + + internal bool? HasArgumentsOrLocals { get; set; } + + internal SnapshotCapturingStrategy CapturingStrategy { get; set; } + + private string CreateSnapshot() + { + using var snapshotCreator = new DebuggerSnapshotCreator(isFullSnapshot: true, location: ProbeLocation.Method, hasCondition: false, Array.Empty(), Members); + + _snapshotId = snapshotCreator.SnapshotId; + + ref var methodMetadataInfo = ref MethodMetadataCollection.Instance.Get(MethodMetadataIndex); + + CaptureInfo info; + + if (IsAsyncMethod) + { + var asyncCaptureInfo = new AsyncCaptureInfo(MoveNextInvocationTarget, KickoffInvocationTarget, methodMetadataInfo.KickoffInvocationTargetType, methodMetadataInfo.KickoffMethod, methodMetadataInfo.AsyncMethodHoistedArguments, methodMetadataInfo.AsyncMethodHoistedLocals); + info = new CaptureInfo(MethodMetadataIndex, value: asyncCaptureInfo.KickoffInvocationTarget, type: asyncCaptureInfo.KickoffInvocationTargetType, methodState: MethodState.ExitEndAsync, memberKind: ScopeMemberKind.This, asyncCaptureInfo: asyncCaptureInfo, hasLocalOrArgument: HasArgumentsOrLocals); + + ProbeProcessor.AddAsyncMethodArguments(snapshotCreator, ref info); + ProbeProcessor.AddAsyncMethodLocals(snapshotCreator, ref info); + } + else + { + info = new CaptureInfo(MethodMetadataIndex, value: Members.InvocationTarget, type: methodMetadataInfo.DeclaringType, invocationTargetType: methodMetadataInfo.DeclaringType, memberKind: ScopeMemberKind.This, methodState: MethodState.ExitEnd, hasLocalOrArgument: HasArgumentsOrLocals, method: methodMetadataInfo.Method); + } + + snapshotCreator.CaptureBehaviour = CaptureBehaviour.Evaluate; + snapshotCreator.ProcessDelayedSnapshot(ref info, hasCondition: true); + snapshotCreator.CaptureExitMethodEndMarker(ref info); + return snapshotCreator.FinalizeMethodSnapshot(ProbeId, 1, ref info); + } + + internal void AddScopeMember(string name, Type type, T value, ScopeMemberKind memberKind) + { + if (Members == null) + { + Members = new MethodScopeMembers(0, 0); + } + + type = (type.IsGenericTypeDefinition ? value?.GetType() : type) ?? type; + switch (memberKind) + { + case ScopeMemberKind.This: + Members.InvocationTarget = new ScopeMember(name, type, value, ScopeMemberKind.This); + return; + case ScopeMemberKind.Exception: + Members.Exception = value as Exception; + return; + case ScopeMemberKind.Return: + Members.Return = new ScopeMember("return", type, value, ScopeMemberKind.Return); + return; + case ScopeMemberKind.None: + return; + } + + Members.AddMember(new ScopeMember(name, type, value, memberKind)); + } + + private IEnumerable FlattenException(Exception exception) + { + var exceptionList = new Stack(); + + exceptionList.Push(exception); + + while (exceptionList.Count > 0) + { + var e = exceptionList.Pop(); + + yield return e; + + if (e == null) + { + continue; + } + + if (e.InnerException != null && !(e is AggregateException)) + { + // Aggregate exceptions contains the InnerException in its InnerExceptions list + exceptionList.Push(e.InnerException); + } + + if (e is AggregateException aggregateException) + { + foreach (var aggExp in aggregateException.InnerExceptions) + { + exceptionList.Push(aggExp); + } + } + } + } + + private uint ComputeEnterSequenceHash() + { + return Fnv1aHash.ComputeHash(_parent?.EnterSequenceHash ?? 0, TrackingFrameData.Method.MetadataToken); + } + + /// + /// TODO take not only first child. + /// + private uint ComputeLeaveSequenceHash() + { + lock (this) + { + ClearNonRelevantChildNodes(); + + if (_activeChildNodes?.Any() == true) + { + var firstChild = _activeChildNodes.First(); + return Fnv1aHash.ComputeHash(firstChild.LeaveSequenceHash, firstChild.TrackingFrameData.Method.MetadataToken); + } + + return 0; + } + } + + public TrackedStackFrameNode RecordFunctionExit(Exception ex) + { + ClearNonRelevantChildNodes(); + + if (_parent == null) + { + return null; + } + + lock (_parent) + { + _parent._activeChildNodes ??= new List(); + _parent._activeChildNodes.Add(this); + } + + return _parent; + } + + public TrackedStackFrameNode RecordFunctionExit() + { + var parent = _parent; + Dispose(); + return parent; + } + + public void Dispose() + { + if (_disposed) + { + return; + } + + _parent = null; + if (_activeChildNodes != null) + { + foreach (var node in _activeChildNodes) + { + node.Dispose(); + } + } + + _activeChildNodes = null; + TrackingFrameData = default; + TrackingFrameData.MarkAsUnwound(); + _disposed = true; + } + + private void ClearNonRelevantChildNodes() + { + // ReSharper disable once InconsistentlySynchronizedField + if (_activeChildNodes == null || _childNodesAlreadyCleansed) + { + return; + } + + lock (this) + { + if (_childNodesAlreadyCleansed) + { + return; + } + + if (!_activeChildNodes.Any()) + { + _activeChildNodes = null; + return; + } + + for (var i = _activeChildNodes.Count - 1; i >= 0; i--) + { + var frame = _activeChildNodes[i]; + if (frame.LeavingException == null || !HasChildException(frame.LeavingException)) + { + _activeChildNodes.RemoveAt(i); + frame.Dispose(); + } + } + + if (LeavingException != null) + { + _childNodesAlreadyCleansed = true; + } + } + } + + public bool HasChildException(Exception exception) + { + if (LeavingException == exception || LeavingException == exception?.InnerException) + { + return true; + } + + var allActiveExceptions = FlattenException(exception); + var allChildExceptions = FlattenException(LeavingException); + + return allActiveExceptions.Intersect(allChildExceptions).Any(); + } + + public override string ToString() + { + return $"{nameof(TrackedStackFrameNode)}(Child Count = {_activeChildNodes?.Count}, {nameof(TrackingFrameData)} = {TrackingFrameData})"; + } + } +} diff --git a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/TrackingFrameData.cs b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/TrackingFrameData.cs new file mode 100644 index 000000000000..215ec0631370 --- /dev/null +++ b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/TrackingFrameData.cs @@ -0,0 +1,26 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +using System; +using System.Reflection; + +namespace Datadog.Trace.Debugger.ExceptionAutoInstrumentation; + +internal struct TrackingFrameData +{ + public TrackingFrameData(MethodBase method) + { + Method = method; + } + + public bool IsFrameUnwound { get; private set; } + + public MethodBase Method { get; } + + public void MarkAsUnwound() + { + IsFrameUnwound = true; + } +} diff --git a/tracer/src/Datadog.Trace/Debugger/Expressions/CaptureInfo.cs b/tracer/src/Datadog.Trace/Debugger/Expressions/CaptureInfo.cs index 9f8b12290632..ac64696fb340 100644 --- a/tracer/src/Datadog.Trace/Debugger/Expressions/CaptureInfo.cs +++ b/tracer/src/Datadog.Trace/Debugger/Expressions/CaptureInfo.cs @@ -12,6 +12,7 @@ namespace Datadog.Trace.Debugger.Expressions; internal readonly ref struct CaptureInfo { internal CaptureInfo( + int methodMetadataIndex, MethodState methodState, TCapture value = default, MethodBase method = null, @@ -25,6 +26,7 @@ internal CaptureInfo( LineCaptureInfo lineCaptureInfo = default, AsyncCaptureInfo asyncCaptureInfo = default) { + MethodMetadataIndex = methodMetadataIndex; Value = value; MemberKind = memberKind; Type = type ?? value?.GetType() ?? typeof(TCapture); @@ -39,6 +41,8 @@ internal CaptureInfo( ArgumentsCount = argumentsCount; } + internal int MethodMetadataIndex { get; } + internal TCapture Value { get; } internal Type Type { get; } diff --git a/tracer/src/Datadog.Trace/Debugger/Expressions/IProbeProcessor.cs b/tracer/src/Datadog.Trace/Debugger/Expressions/IProbeProcessor.cs new file mode 100644 index 000000000000..0e70a90926e0 --- /dev/null +++ b/tracer/src/Datadog.Trace/Debugger/Expressions/IProbeProcessor.cs @@ -0,0 +1,25 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +using System; +using Datadog.Trace.Debugger.Configurations.Models; +using Datadog.Trace.Debugger.Instrumentation.Collections; +using Datadog.Trace.Debugger.Snapshots; + +namespace Datadog.Trace.Debugger.Expressions +{ + internal interface IProbeProcessor + { + bool ShouldProcess(in ProbeData probeData); + + bool Process(ref CaptureInfo info, IDebuggerSnapshotCreator snapshotCreator, in ProbeData probeData); + + void LogException(Exception ex, IDebuggerSnapshotCreator snapshotCreator); + + IProbeProcessor UpdateProbeProcessor(ProbeDefinition probe); + + IDebuggerSnapshotCreator CreateSnapshotCreator(); + } +} diff --git a/tracer/src/Datadog.Trace/Debugger/Expressions/ProbeExpressionsProcessor.cs b/tracer/src/Datadog.Trace/Debugger/Expressions/ProbeExpressionsProcessor.cs index e8a30f38136f..a4fc54c8bdfc 100644 --- a/tracer/src/Datadog.Trace/Debugger/Expressions/ProbeExpressionsProcessor.cs +++ b/tracer/src/Datadog.Trace/Debugger/Expressions/ProbeExpressionsProcessor.cs @@ -22,7 +22,7 @@ internal class ProbeExpressionsProcessor private static ProbeExpressionsProcessor _instance; - private readonly ConcurrentDictionary _processors = new(); + private readonly ConcurrentDictionary _processors = new(); internal static ProbeExpressionsProcessor Instance { @@ -50,12 +50,25 @@ internal void AddProbeProcessor(ProbeDefinition probe) } } + internal bool TryAddProbeProcessor(string probeId, IProbeProcessor probeProcessor) + { + try + { + return _processors.TryAdd(probeId, probeProcessor); + } + catch (Exception e) + { + Log.Error(e, "Failed to create probe processor for probe: {Id}", probeId); + return false; + } + } + internal void Remove(string probeId) { _processors.TryRemove(probeId, out _); } - internal ProbeProcessor Get(string probeId) + internal IProbeProcessor Get(string probeId) { _processors.TryGetValue(probeId, out var probeProcessor); return probeProcessor; diff --git a/tracer/src/Datadog.Trace/Debugger/Expressions/ProbeProcessor.cs b/tracer/src/Datadog.Trace/Debugger/Expressions/ProbeProcessor.cs index 352531167133..74ff810a532a 100644 --- a/tracer/src/Datadog.Trace/Debugger/Expressions/ProbeProcessor.cs +++ b/tracer/src/Datadog.Trace/Debugger/Expressions/ProbeProcessor.cs @@ -9,6 +9,8 @@ using System.Linq; using System.Threading; using Datadog.Trace.Debugger.Configurations.Models; +using Datadog.Trace.Debugger.Helpers; +using Datadog.Trace.Debugger.Instrumentation.Collections; using Datadog.Trace.Debugger.Models; using Datadog.Trace.Debugger.RateLimiting; using Datadog.Trace.Debugger.Snapshots; @@ -16,7 +18,7 @@ namespace Datadog.Trace.Debugger.Expressions { - internal class ProbeProcessor + internal class ProbeProcessor : IProbeProcessor { private const string DynamicPrefix = "_dd.di."; private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor(typeof(ProbeProcessor)); @@ -88,12 +90,21 @@ private void InitializeProbeProcessor(ProbeDefinition probe) return segment == null ? null : new DebuggerExpression(segment.Dsl, segment.Json?.ToString(), segment.Str); } - internal ProbeProcessor UpdateProbeProcessor(ProbeDefinition probe) + public void LogException(Exception ex, IDebuggerSnapshotCreator snapshotCreator) + { + } + + public IProbeProcessor UpdateProbeProcessor(ProbeDefinition probe) { InitializeProbeProcessor(probe); return this; } + public IDebuggerSnapshotCreator CreateSnapshotCreator() + { + return new DebuggerSnapshotCreator(ProbeInfo.IsFullSnapshot, ProbeInfo.ProbeLocation, ProbeInfo.HasCondition, ProbeInfo.Tags); + } + private void SetExpressions(ProbeDefinition probe) { // ReSharper disable once PossibleInvalidOperationException @@ -115,11 +126,20 @@ private ProbeExpressionEvaluator GetOrCreateEvaluator() return _evaluator; } - internal bool Process(ref CaptureInfo info, DebuggerSnapshotCreator snapshotCreator) + public bool ShouldProcess(in ProbeData probeData) { + return HasCondition() || probeData.Sampler.Sample(); + } + + public bool Process(ref CaptureInfo info, IDebuggerSnapshotCreator inSnapshotCreator, in ProbeData probeData) + { + var snapshotCreator = (DebuggerSnapshotCreator)inSnapshotCreator; + ExpressionEvaluationResult evaluationResult = default; try { + snapshotCreator.StopSampling(); + switch (info.MethodState) { case MethodState.BeginLine: @@ -131,7 +151,7 @@ internal bool Process(ref CaptureInfo info, DebuggerSnapshot { AddAsyncMethodArguments(snapshotCreator, ref info); snapshotCreator.AddScopeMember(info.Name, info.Type, info.Value, info.MemberKind); - evaluationResult = Evaluate(snapshotCreator, out var shouldStopCapture); + evaluationResult = Evaluate(snapshotCreator, out var shouldStopCapture, probeData.Sampler); if (shouldStopCapture) { snapshotCreator.Stop(); @@ -197,7 +217,7 @@ internal bool Process(ref CaptureInfo info, DebuggerSnapshot } snapshotCreator.AddScopeMember(info.Name, info.Type, info.Value, info.MemberKind); - evaluationResult = Evaluate(snapshotCreator, out var shouldStopCapture); + evaluationResult = Evaluate(snapshotCreator, out var shouldStopCapture, probeData.Sampler); if (shouldStopCapture) { snapshotCreator.Stop(); @@ -237,16 +257,20 @@ internal bool Process(ref CaptureInfo info, DebuggerSnapshot $"{info.MethodState} is not valid value here"); } - return ProcessCapture(ref info, ref snapshotCreator, ref evaluationResult); + return ProcessCapture(ref info, snapshotCreator, ref evaluationResult); } catch (Exception e) { Log.Error(e, "Failed to process probe. Probe Id: {ProbeId}", ProbeInfo.ProbeId); return false; } + finally + { + snapshotCreator.StartSampling(); + } } - private ExpressionEvaluationResult Evaluate(DebuggerSnapshotCreator snapshotCreator, out bool shouldStopCapture) + private ExpressionEvaluationResult Evaluate(DebuggerSnapshotCreator snapshotCreator, out bool shouldStopCapture, IAdaptiveSampler sampler) { ExpressionEvaluationResult evaluationResult = default; shouldStopCapture = false; @@ -301,7 +325,7 @@ private ExpressionEvaluationResult Evaluate(DebuggerSnapshotCreator snapshotCrea if (evaluationResult.Condition != null && // meaning not metric, span probe or span decoration (evaluationResult.Condition is false || - !ProbeRateLimiter.Instance.Sample(ProbeInfo.ProbeId))) + !sampler.Sample())) { // if the expression evaluated to false, or there is a rate limit, stop capture shouldStopCapture = true; @@ -365,7 +389,7 @@ private void CheckSpanDecoration(DebuggerSnapshotCreator snapshotCreator, ref bo } } - private void AddAsyncMethodArguments(DebuggerSnapshotCreator snapshotCreator, ref CaptureInfo captureInfo) + internal static void AddAsyncMethodArguments(DebuggerSnapshotCreator snapshotCreator, ref CaptureInfo captureInfo) { var asyncCaptureInfo = captureInfo.AsyncCaptureInfo; for (int i = 0; i < asyncCaptureInfo.HoistedArguments.Length; i++) @@ -381,7 +405,7 @@ private void AddAsyncMethodArguments(DebuggerSnapshotCreator snapshotCreator, } } - private void AddAsyncMethodLocals(DebuggerSnapshotCreator snapshotCreator, ref CaptureInfo captureInfo) + internal static void AddAsyncMethodLocals(DebuggerSnapshotCreator snapshotCreator, ref CaptureInfo captureInfo) { var asyncCaptureInfo = captureInfo.AsyncCaptureInfo; for (int i = 0; i < asyncCaptureInfo.HoistedLocals.Length; i++) @@ -397,7 +421,7 @@ private void AddAsyncMethodLocals(DebuggerSnapshotCreator snapshotCreator, re } } - private bool ProcessCapture(ref CaptureInfo info, ref DebuggerSnapshotCreator snapshotCreator, ref ExpressionEvaluationResult evaluationResult) + private bool ProcessCapture(ref CaptureInfo info, DebuggerSnapshotCreator snapshotCreator, ref ExpressionEvaluationResult evaluationResult) { switch (ProbeInfo.ProbeLocation) { diff --git a/tracer/src/Datadog.Trace/Debugger/Helpers/ExceptionExtensions.cs b/tracer/src/Datadog.Trace/Debugger/Helpers/ExceptionExtensions.cs new file mode 100644 index 000000000000..1553d784e2ad --- /dev/null +++ b/tracer/src/Datadog.Trace/Debugger/Helpers/ExceptionExtensions.cs @@ -0,0 +1,28 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Datadog.Trace.Debugger.Helpers +{ + internal static class ExceptionExtensions + { + public static bool IsSelfOrInnerExceptionEquals(this Exception checkSelfAndInner, Exception toCheckAgainst, out Exception matchedException) + { + matchedException = checkSelfAndInner; + + if (checkSelfAndInner == null) + { + return false; + } + + return object.ReferenceEquals(checkSelfAndInner, toCheckAgainst) || (checkSelfAndInner is AggregateException == false && IsSelfOrInnerExceptionEquals(checkSelfAndInner.InnerException, toCheckAgainst, out matchedException)); + } + } +} diff --git a/tracer/src/Datadog.Trace/Debugger/Helpers/LazyExtensions.cs b/tracer/src/Datadog.Trace/Debugger/Helpers/LazyExtensions.cs new file mode 100644 index 000000000000..cac8facdf6f7 --- /dev/null +++ b/tracer/src/Datadog.Trace/Debugger/Helpers/LazyExtensions.cs @@ -0,0 +1,18 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +using System; + +namespace Datadog.Trace.Debugger.Helpers +{ + internal static class LazyExtensions + { + public static Lazy Cast(this Lazy lazy) + where TConcrete : class, TInterface + { + return new Lazy(() => (TConcrete)lazy.Value); + } + } +} diff --git a/tracer/src/Datadog.Trace/Debugger/Helpers/MethodExtensions.cs b/tracer/src/Datadog.Trace/Debugger/Helpers/MethodExtensions.cs new file mode 100644 index 000000000000..92d4a77a03a5 --- /dev/null +++ b/tracer/src/Datadog.Trace/Debugger/Helpers/MethodExtensions.cs @@ -0,0 +1,183 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +using System; +using System.Collections; +using System.Collections.Generic; +using System.Diagnostics; +using System.Reflection; +using System.Reflection.Emit; +using System.Runtime.CompilerServices; +using System.Security.Cryptography; +using System.Text; + +namespace Datadog.Trace.Debugger.Helpers +{ + internal static class MethodExtensions + { + /// + /// Gets fully qualified name of a method with parameters and generics. For example SkyApm.Sample.ConsoleApp.Program.Main(String[] args). + /// Code was copied from System.Diagnostics.StackTrace.ToString() - .NET Standard implementation, not .NET Framework + /// + internal static string GetFullyQualifiedName(this MethodBase mb) + { + try + { + var sb = new StringBuilder(255); + + sb.Append(mb.Name + "_"); + + var declaringType = mb.DeclaringType; + var methodName = mb.Name; + var methodChanged = false; + if (declaringType != null && declaringType.IsDefined(typeof(CompilerGeneratedAttribute), inherit: false)) + { + var isAsync = typeof(IAsyncStateMachine).IsAssignableFrom(declaringType); + if (isAsync || typeof(IEnumerator).IsAssignableFrom(declaringType)) + { + methodChanged = TryResolveStateMachineMethod(ref mb, out declaringType); + } + } + + // if there is a type (non global method) print it + // ResolveStateMachineMethod may have set declaringType to null + if (declaringType != null) + { + // Append t.FullName, replacing '+' with '.' + var fullName = declaringType.FullName; + if (fullName == null) + { + return null; + } + + for (var i = 0; i < fullName.Length; i++) + { + var ch = fullName[i]; + sb.Append(ch == '+' ? '.' : ch); + } + + sb.Append('.'); + } + + sb.Append(mb.Name); + + // deal with the generic portion of the method + if (mb is MethodInfo mi && mi.IsGenericMethod) + { + var typars = mi.GetGenericArguments(); + sb.Append('['); + var k = 0; + var fFirstTyParam = true; + while (k < typars.Length) + { + if (fFirstTyParam == false) + { + sb.Append(','); + } + else + { + fFirstTyParam = false; + } + + sb.Append(typars[k].Name); + k++; + } + + sb.Append(']'); + } + + ParameterInfo[] pi = null; + try + { + pi = mb.GetParameters(); + } + catch + { + // The parameter info cannot be loaded, so we don't + // append the parameter list. + } + + if (pi != null) + { + // arguments printing + sb.Append('('); + var fFirstParam = true; + for (var j = 0; j < pi.Length; j++) + { + if (fFirstParam == false) + { + sb.Append(", "); + } + else + { + fFirstParam = false; + } + + var typeName = pi[j].ParameterType?.Name ?? ""; + sb.Append(typeName); + sb.Append(' '); + sb.Append(pi[j].Name); + } + + sb.Append(')'); + } + + if (methodChanged) + { + // Append original method name e.g. +MoveNext() + sb.Append('+'); + sb.Append(methodName); + sb.Append('(').Append(')'); + } + + return sb.ToString(); + } + catch + { + return null; + } + } + + private static bool TryResolveStateMachineMethod(ref MethodBase method, out Type declaringType) + { + declaringType = method.DeclaringType; + + var parentType = declaringType.DeclaringType; + if (parentType == null) + { + return false; + } + + var methods = parentType.GetMethods(BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Instance | BindingFlags.DeclaredOnly); + + foreach (MethodInfo candidateMethod in methods) + { + var attributes = candidateMethod.GetCustomAttributes(inherit: false); + + bool foundAttribute = false, foundIteratorAttribute = false; + foreach (var attr in attributes) + { + if (attr.StateMachineType == declaringType) + { + foundAttribute = true; + foundIteratorAttribute |= attr is IteratorStateMachineAttribute || attr.GetType().Name == "AsyncIteratorStateMachineAttribute"; + } + } + + if (foundAttribute) + { + // If this is an iterator (sync or async), mark the iterator as changed, so it gets the + annotation + // of the original method. Non-iterator async state machines resolve directly to their builder methods + // so aren't marked as changed. + method = candidateMethod; + declaringType = candidateMethod.DeclaringType; + return foundIteratorAttribute; + } + } + + return false; + } + } +} diff --git a/tracer/src/Datadog.Trace/Debugger/Helpers/StackTraceExtensions.cs b/tracer/src/Datadog.Trace/Debugger/Helpers/StackTraceExtensions.cs new file mode 100644 index 000000000000..f0dd5daaeb5b --- /dev/null +++ b/tracer/src/Datadog.Trace/Debugger/Helpers/StackTraceExtensions.cs @@ -0,0 +1,62 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +using System.Collections.Generic; +using System.Diagnostics; +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.ExceptionServices; + +namespace Datadog.Trace.Debugger.Helpers +{ + /// + /// Provides extension methods for . + /// + internal static class StackTraceExtensions + { + /// + /// Produces an async-friendly readable representation of the stack trace. + /// + /// + /// The async-friendly formatting is archived by: + /// * Skipping all awaiter frames (all methods in types implementing ). + /// * Inferring the original method name from the async state machine class () + /// and removing the "MoveNext" - currently only for C#. + /// * Adding the "async" prefix after "at" on each line for async invocations. + /// * Appending "(?)" to the method signature to indicate that parameter information is missing. + /// * Removing the "End of stack trace from previous location..." text. + /// + /// The stack frames. + /// An async-friendly readable representation of the stack trace. + public static IEnumerable GetAsyncFriendlyFrameMethods(this IEnumerable stackFrames) + { + if (stackFrames == null) + { + yield break; + } + + foreach (var frame in stackFrames) + { + var method = frame.GetMethod(); + + if (method == null) + { + continue; + } + + var declaringType = method.DeclaringType?.GetTypeInfo(); + // skip awaiters + if (declaringType != null && + (typeof(INotifyCompletion).GetTypeInfo().IsAssignableFrom(declaringType) || + method.DeclaringType == typeof(ExceptionDispatchInfo))) + { + continue; + } + + yield return frame; + } + } + } +} diff --git a/tracer/src/Datadog.Trace/Debugger/Instrumentation/AsyncLineDebuggerInvoker.cs b/tracer/src/Datadog.Trace/Debugger/Instrumentation/AsyncLineDebuggerInvoker.cs index 0645f20bc983..7c189d169e10 100644 --- a/tracer/src/Datadog.Trace/Debugger/Instrumentation/AsyncLineDebuggerInvoker.cs +++ b/tracer/src/Datadog.Trace/Debugger/Instrumentation/AsyncLineDebuggerInvoker.cs @@ -47,23 +47,21 @@ public static void LogLocal(ref TLocal local, int index, ref AsyncLineDe return; } - state.SnapshotCreator.StopSampling(); var localVariableNames = state.MethodMetadataInfo.LocalVariableNames; if (!MethodDebuggerInvoker.TryGetLocalName(index, localVariableNames, out var localName)) { - state.SnapshotCreator.StartSampling(); return; } - var captureInfo = new CaptureInfo(value: local, methodState: MethodState.LogLocal, name: localName, memberKind: ScopeMemberKind.Local); + var captureInfo = new CaptureInfo(state.MethodMetadataIndex, value: local, methodState: MethodState.LogLocal, name: localName, memberKind: ScopeMemberKind.Local); + var probeData = state.ProbeData; - if (!state.ProbeData.Processor.Process(ref captureInfo, state.SnapshotCreator)) + if (!state.ProbeData.Processor.Process(ref captureInfo, state.SnapshotCreator, in probeData)) { state.IsActive = false; } state.HasLocalsOrReturnValue = true; - state.SnapshotCreator.StartSampling(); } catch (Exception e) { @@ -157,24 +155,22 @@ public static AsyncLineDebuggerState BeginLine(string probeId, int prob return CreateInvalidatedAsyncLineDebuggerState(); } - var kickoffParentObject = AsyncHelper.GetAsyncKickoffThisObject(instance); - var state = new AsyncLineDebuggerState(probeId, scope: default, methodMetadataIndex, ref probeData, lineNumber, probeFilePath, instance, kickoffParentObject); - - if (!state.SnapshotCreator.ProbeHasCondition && - !state.ProbeData.Sampler.Sample()) + if (!probeData.Processor.ShouldProcess(in probeData)) { + Log.Warning("[Async]BeginLine: Skipping the instrumentation. type = {Type}, instance type name = {Name}, probeMetadataIndex = {ProbeMetadataIndex}, probeId = {ProbeId}", new object[] { typeof(TTarget), instance?.GetType().Name, probeMetadataIndex, probeId }); return CreateInvalidatedAsyncLineDebuggerState(); } + var kickoffParentObject = AsyncHelper.GetAsyncKickoffThisObject(instance); + var state = new AsyncLineDebuggerState(probeId, scope: default, methodMetadataIndex, ref probeData, lineNumber, probeFilePath, instance, kickoffParentObject); var asyncInfo = new AsyncCaptureInfo(state.MoveNextInvocationTarget, state.KickoffInvocationTarget, state.MethodMetadataInfo.KickoffInvocationTargetType, hoistedLocals: state.MethodMetadataInfo.AsyncMethodHoistedLocals, hoistedArgs: state.MethodMetadataInfo.AsyncMethodHoistedArguments); - var captureInfo = new CaptureInfo(value: null, type: state.MethodMetadataInfo.DeclaringType, methodState: MethodState.BeginLineAsync, localsCount: state.MethodMetadataInfo.LocalVariableNames.Length, argumentsCount: state.MethodMetadataInfo.ParameterNames.Length, lineCaptureInfo: new LineCaptureInfo(lineNumber, probeFilePath), asyncCaptureInfo: asyncInfo); + var captureInfo = new CaptureInfo(state.MethodMetadataIndex, value: null, type: state.MethodMetadataInfo.DeclaringType, methodState: MethodState.BeginLineAsync, localsCount: state.MethodMetadataInfo.LocalVariableNames.Length, argumentsCount: state.MethodMetadataInfo.ParameterNames.Length, lineCaptureInfo: new LineCaptureInfo(lineNumber, probeFilePath), asyncCaptureInfo: asyncInfo); - if (!state.ProbeData.Processor.Process(ref captureInfo, state.SnapshotCreator)) + if (!state.ProbeData.Processor.Process(ref captureInfo, state.SnapshotCreator, in probeData)) { state.IsActive = false; } - state.SnapshotCreator.StartSampling(); return state; } catch (Exception e) @@ -199,15 +195,15 @@ public static void EndLine(ref AsyncLineDebuggerState state) return; } - state.SnapshotCreator.StopSampling(); var hasArgumentsOrLocals = state.HasLocalsOrReturnValue || state.MethodMetadataInfo.AsyncMethodHoistedArguments.Length > 0 || state.KickoffInvocationTarget != null; state.HasLocalsOrReturnValue = false; var asyncCaptureInfo = new AsyncCaptureInfo(state.MoveNextInvocationTarget, state.KickoffInvocationTarget, state.MethodMetadataInfo.KickoffInvocationTargetType, kickoffMethod: state.MethodMetadataInfo.KickoffMethod, hoistedArgs: state.MethodMetadataInfo.AsyncMethodHoistedArguments, hoistedLocals: state.MethodMetadataInfo.AsyncMethodHoistedLocals); - var captureInfo = new CaptureInfo(value: state.KickoffInvocationTarget, type: state.MethodMetadataInfo.KickoffInvocationTargetType, name: "this", memberKind: ScopeMemberKind.This, methodState: MethodState.EndLineAsync, hasLocalOrArgument: hasArgumentsOrLocals, asyncCaptureInfo: asyncCaptureInfo, lineCaptureInfo: new LineCaptureInfo(state.LineNumber, state.ProbeFilePath)); - state.ProbeData.Processor.Process(ref captureInfo, state.SnapshotCreator); + var captureInfo = new CaptureInfo(state.MethodMetadataIndex, value: state.KickoffInvocationTarget, type: state.MethodMetadataInfo.KickoffInvocationTargetType, name: "this", memberKind: ScopeMemberKind.This, methodState: MethodState.EndLineAsync, hasLocalOrArgument: hasArgumentsOrLocals, asyncCaptureInfo: asyncCaptureInfo, lineCaptureInfo: new LineCaptureInfo(state.LineNumber, state.ProbeFilePath)); + var probeData = state.ProbeData; + state.ProbeData.Processor.Process(ref captureInfo, state.SnapshotCreator, in probeData); } catch (Exception e) { diff --git a/tracer/src/Datadog.Trace/Debugger/Instrumentation/AsyncLineDebuggerState.cs b/tracer/src/Datadog.Trace/Debugger/Instrumentation/AsyncLineDebuggerState.cs index 03683fc41909..f34dcc5d96ef 100644 --- a/tracer/src/Datadog.Trace/Debugger/Instrumentation/AsyncLineDebuggerState.cs +++ b/tracer/src/Datadog.Trace/Debugger/Instrumentation/AsyncLineDebuggerState.cs @@ -25,6 +25,7 @@ public ref struct AsyncLineDebuggerState private readonly int _lineNumber; /// + /// Backing field of . /// Used to perform a fast lookup to grab the proper . /// This index is hard-coded into the method's instrumented bytecode. /// @@ -35,7 +36,6 @@ public ref struct AsyncLineDebuggerState // Determines whether we should still be capturing values, or halt for any reason (e.g an exception was caused by our instrumentation, rate limiter threshold reached). internal bool IsActive = true; - internal bool HasLocalsOrReturnValue; /// @@ -57,12 +57,25 @@ internal AsyncLineDebuggerState(string probeId, Scope scope, int methodMetadataI _lineNumber = lineNumber; _probeFilePath = probeFilePath; HasLocalsOrReturnValue = false; + var processor = probeData.Processor; + SnapshotCreator = processor.CreateSnapshotCreator(); ProbeData = probeData; - SnapshotCreator = DebuggerSnapshotCreator.BuildSnapshotCreator(probeData.Processor); _moveNextInvocationTarget = invocationTarget; _kickoffInvocationTarget = kickoffInvocationTarget; } + /// + /// Gets an index that is used as a fast lookup to grab the proper . + /// This index is hard-coded into the method's instrumented bytecode. + /// + internal int MethodMetadataIndex + { + get + { + return _methodMetadataIndex; + } + } + internal ref MethodMetadataInfo MethodMetadataInfo => ref MethodMetadataCollection.Instance.Get(_methodMetadataIndex); internal ProbeData ProbeData { get; } @@ -70,7 +83,7 @@ internal AsyncLineDebuggerState(string probeId, Scope scope, int methodMetadataI /// /// Gets the LiveDebugger SnapshotCreator /// - internal DebuggerSnapshotCreator SnapshotCreator { get; } + internal IDebuggerSnapshotCreator SnapshotCreator { get; } /// /// Gets the LiveDebugger BeginMethod scope diff --git a/tracer/src/Datadog.Trace/Debugger/Instrumentation/AsyncMethodDebuggerInvoker.SingleProbe.cs b/tracer/src/Datadog.Trace/Debugger/Instrumentation/AsyncMethodDebuggerInvoker.SingleProbe.cs index 70aaa880f40f..3a877810b237 100644 --- a/tracer/src/Datadog.Trace/Debugger/Instrumentation/AsyncMethodDebuggerInvoker.SingleProbe.cs +++ b/tracer/src/Datadog.Trace/Debugger/Instrumentation/AsyncMethodDebuggerInvoker.SingleProbe.cs @@ -165,6 +165,13 @@ public static void BeginMethod(TTarget instance, int methodMetadataInde return; } + if (!probeData.Processor.ShouldProcess(in probeData)) + { + Log.Warning("BeginMethod: Skipping the instrumentation. type = {Type}, instance type name = {Name}, probeMetadataIndex = {ProbeMetadataIndex}, probeId = {ProbeId}", new object[] { typeof(TTarget), instance?.GetType().Name, probeMetadataIndex, probeId }); + state = AsyncMethodDebuggerState.CreateInvalidatedDebuggerState(); + return; + } + var asyncState = new AsyncMethodDebuggerState(probeId, ref probeData) { KickoffInvocationTarget = kickoffInfo.KickoffParentObject, @@ -173,31 +180,22 @@ public static void BeginMethod(TTarget instance, int methodMetadataInde MoveNextInvocationTarget = instance }; - if (!asyncState.SnapshotCreator.ProbeHasCondition && - !asyncState.ProbeData.Sampler.Sample()) - { - state = AsyncMethodDebuggerState.CreateInvalidatedDebuggerState(); - return; - } - var hasArgumentsOrLocals = asyncState.HasLocalsOrReturnValue || asyncState.HasArguments || asyncState.KickoffInvocationTarget != null; var asyncCaptureInfo = new AsyncCaptureInfo(asyncState.MoveNextInvocationTarget, asyncState.KickoffInvocationTarget, asyncState.MethodMetadataInfo.KickoffInvocationTargetType, hoistedLocals: asyncState.MethodMetadataInfo.AsyncMethodHoistedLocals, hoistedArgs: asyncState.MethodMetadataInfo.AsyncMethodHoistedArguments); - var capture = new CaptureInfo(value: asyncState.KickoffInvocationTarget, type: asyncState.MethodMetadataInfo.KickoffInvocationTargetType, methodState: MethodState.EntryAsync, hasLocalOrArgument: hasArgumentsOrLocals, asyncCaptureInfo: asyncCaptureInfo, memberKind: ScopeMemberKind.This, localsCount: asyncState.MethodMetadataInfo.LocalVariableNames.Length, argumentsCount: asyncState.MethodMetadataInfo.ParameterNames.Length); + var capture = new CaptureInfo(asyncState.MethodMetadataIndex, method: asyncState.MethodMetadataInfo.Method, value: asyncState.KickoffInvocationTarget, type: asyncState.MethodMetadataInfo.KickoffInvocationTargetType, methodState: MethodState.EntryAsync, hasLocalOrArgument: hasArgumentsOrLocals, asyncCaptureInfo: asyncCaptureInfo, memberKind: ScopeMemberKind.This, localsCount: asyncState.MethodMetadataInfo.LocalVariableNames.Length, argumentsCount: asyncState.MethodMetadataInfo.ParameterNames.Length); asyncState.HasLocalsOrReturnValue = false; asyncState.HasArguments = false; state = asyncState; // Denotes that subsequent re-entries of the `MoveNext` will be ignored by `BeginMethod`. - if (!asyncState.ProbeData.Processor.Process(ref capture, asyncState.SnapshotCreator)) + if (!asyncState.ProbeData.Processor.Process(ref capture, asyncState.SnapshotCreator, in probeData)) { asyncState.IsActive = false; } - - asyncState.SnapshotCreator.StartSampling(); } /// @@ -215,23 +213,22 @@ public static void LogLocal(ref TLocal local, int index, ref AsyncMethod return; } - asyncState.SnapshotCreator.StopSampling(); var localVariableNames = asyncState.MethodMetadataInfo.LocalVariableNames; if (!MethodDebuggerInvoker.TryGetLocalName(index, localVariableNames, out var localName)) { - asyncState.SnapshotCreator.StartSampling(); return; } - var captureInfo = new CaptureInfo(value: local, methodState: MethodState.LogLocal, name: localName, memberKind: ScopeMemberKind.Local); - if (!asyncState.ProbeData.Processor.Process(ref captureInfo, asyncState.SnapshotCreator)) + var captureInfo = new CaptureInfo(asyncState.MethodMetadataIndex, value: local, methodState: MethodState.LogLocal, name: localName, memberKind: ScopeMemberKind.Local); + var probeData = asyncState.ProbeData; + + if (!asyncState.ProbeData.Processor.Process(ref captureInfo, asyncState.SnapshotCreator, in probeData)) { asyncState.IsActive = false; } asyncState.HasLocalsOrReturnValue = true; asyncState.HasArguments = false; - asyncState.SnapshotCreator.StartSampling(); } /// @@ -252,19 +249,17 @@ public static DebuggerReturn EndMethod_StartMarker(TTarget instance, Ex return DebuggerReturn.GetDefault(); } - asyncState.SnapshotCreator.StopSampling(); - asyncState.MoveNextInvocationTarget = instance; var asyncCaptureInfo = new AsyncCaptureInfo(asyncState.MoveNextInvocationTarget, asyncState.KickoffInvocationTarget, asyncState.MethodMetadataInfo.KickoffInvocationTargetType, hoistedLocals: asyncState.MethodMetadataInfo.AsyncMethodHoistedLocals, hoistedArgs: asyncState.MethodMetadataInfo.AsyncMethodHoistedArguments); - var capture = new CaptureInfo(value: exception, methodState: MethodState.ExitStartAsync, asyncCaptureInfo: asyncCaptureInfo, memberKind: ScopeMemberKind.Exception); + var capture = new CaptureInfo(asyncState.MethodMetadataIndex, value: exception, methodState: MethodState.ExitStartAsync, asyncCaptureInfo: asyncCaptureInfo, memberKind: ScopeMemberKind.Exception); + var probeData = asyncState.ProbeData; - if (!asyncState.ProbeData.Processor.Process(ref capture, asyncState.SnapshotCreator)) + if (!asyncState.ProbeData.Processor.Process(ref capture, asyncState.SnapshotCreator, in probeData)) { asyncState.IsActive = false; } - asyncState.SnapshotCreator.StartSampling(); return DebuggerReturn.GetDefault(); } @@ -289,23 +284,23 @@ public static DebuggerReturn EndMethod_StartMarker(TT return new DebuggerReturn(returnValue); } - asyncState.SnapshotCreator.StopSampling(); - asyncState.MoveNextInvocationTarget = instance; var asyncCaptureInfo = new AsyncCaptureInfo(asyncState.MoveNextInvocationTarget, asyncState.KickoffInvocationTarget, asyncState.MethodMetadataInfo.KickoffInvocationTargetType, hoistedLocals: asyncState.MethodMetadataInfo.AsyncMethodHoistedLocals, hoistedArgs: asyncState.MethodMetadataInfo.AsyncMethodHoistedArguments); + var probeData = asyncState.ProbeData; + if (exception != null) { - var captureInfo = new CaptureInfo(value: exception, methodState: MethodState.ExitStartAsync, memberKind: ScopeMemberKind.Exception, asyncCaptureInfo: asyncCaptureInfo); - if (!asyncState.ProbeData.Processor.Process(ref captureInfo, asyncState.SnapshotCreator)) + var captureInfo = new CaptureInfo(asyncState.MethodMetadataIndex, value: exception, methodState: MethodState.ExitStartAsync, memberKind: ScopeMemberKind.Exception, asyncCaptureInfo: asyncCaptureInfo); + if (!asyncState.ProbeData.Processor.Process(ref captureInfo, asyncState.SnapshotCreator, in probeData)) { asyncState.IsActive = false; } } else if (returnValue != null) { - var captureInfo = new CaptureInfo(value: returnValue, name: "@return", methodState: MethodState.ExitStartAsync, memberKind: ScopeMemberKind.Return, asyncCaptureInfo: asyncCaptureInfo); - if (!asyncState.ProbeData.Processor.Process(ref captureInfo, asyncState.SnapshotCreator)) + var captureInfo = new CaptureInfo(asyncState.MethodMetadataIndex, value: returnValue, name: "@return", methodState: MethodState.ExitStartAsync, memberKind: ScopeMemberKind.Return, asyncCaptureInfo: asyncCaptureInfo); + if (!asyncState.ProbeData.Processor.Process(ref captureInfo, asyncState.SnapshotCreator, in probeData)) { asyncState.IsActive = false; } @@ -313,7 +308,6 @@ public static DebuggerReturn EndMethod_StartMarker(TT asyncState.HasLocalsOrReturnValue = true; } - asyncState.SnapshotCreator.StartSampling(); return new DebuggerReturn(returnValue); } @@ -329,14 +323,15 @@ public static void EndMethod_EndMarker(ref AsyncMethodDebuggerState asyncState) return; } - asyncState.SnapshotCreator.StopSampling(); var hasArgumentsOrLocals = asyncState.HasLocalsOrReturnValue || asyncState.HasArguments || !asyncState.MethodMetadataInfo.Method.IsStatic; var asyncCaptureInfo = new AsyncCaptureInfo(asyncState.MoveNextInvocationTarget, asyncState.KickoffInvocationTarget, asyncState.MethodMetadataInfo.KickoffInvocationTargetType, asyncState.MethodMetadataInfo.KickoffMethod, asyncState.MethodMetadataInfo.AsyncMethodHoistedArguments, asyncState.MethodMetadataInfo.AsyncMethodHoistedLocals); - var captureInfo = new CaptureInfo(value: asyncCaptureInfo.KickoffInvocationTarget, type: asyncCaptureInfo.KickoffInvocationTargetType, methodState: MethodState.ExitEndAsync, memberKind: ScopeMemberKind.This, asyncCaptureInfo: asyncCaptureInfo, hasLocalOrArgument: hasArgumentsOrLocals); - if (!asyncState.ProbeData.Processor.Process(ref captureInfo, asyncState.SnapshotCreator)) + var captureInfo = new CaptureInfo(asyncState.MethodMetadataIndex, value: asyncCaptureInfo.KickoffInvocationTarget, type: asyncCaptureInfo.KickoffInvocationTargetType, methodState: MethodState.ExitEndAsync, memberKind: ScopeMemberKind.This, asyncCaptureInfo: asyncCaptureInfo, hasLocalOrArgument: hasArgumentsOrLocals); + var probeData = asyncState.ProbeData; + + if (!asyncState.ProbeData.Processor.Process(ref captureInfo, asyncState.SnapshotCreator, in probeData)) { asyncState.IsActive = false; } @@ -359,11 +354,15 @@ public static void LogException(Exception exception, ref AsyncMethodDebuggerStat } Log.Warning(exception, "Error caused by our instrumentation"); - asyncState = AsyncMethodDebuggerState.CreateInvalidatedDebuggerState(); + asyncState.ProbeData.Processor.LogException(exception, asyncState.SnapshotCreator); } catch { - // ignored + // Ignored + } + finally + { + asyncState = AsyncMethodDebuggerState.CreateInvalidatedDebuggerState(); } } } diff --git a/tracer/src/Datadog.Trace/Debugger/Instrumentation/AsyncMethodDebuggerState.cs b/tracer/src/Datadog.Trace/Debugger/Instrumentation/AsyncMethodDebuggerState.cs index cabd66def8e5..8685265d3f5e 100644 --- a/tracer/src/Datadog.Trace/Debugger/Instrumentation/AsyncMethodDebuggerState.cs +++ b/tracer/src/Datadog.Trace/Debugger/Instrumentation/AsyncMethodDebuggerState.cs @@ -32,7 +32,8 @@ internal AsyncMethodDebuggerState(string probeId, ref ProbeData probeData) ProbeId = probeId; HasLocalsOrReturnValue = false; HasArguments = false; - SnapshotCreator = DebuggerSnapshotCreator.BuildSnapshotCreator(probeData.Processor); + var processor = probeData.Processor; + SnapshotCreator = processor.CreateSnapshotCreator(); ProbeData = probeData; } @@ -69,7 +70,7 @@ private AsyncMethodDebuggerState() /// /// Gets the LiveDebugger SnapshotCreator /// - internal DebuggerSnapshotCreator SnapshotCreator { get; } + internal IDebuggerSnapshotCreator SnapshotCreator { get; } /// /// Gets or sets the LiveDebugger BeginMethod scope diff --git a/tracer/src/Datadog.Trace/Debugger/Instrumentation/Collections/ProbeData.cs b/tracer/src/Datadog.Trace/Debugger/Instrumentation/Collections/ProbeData.cs index 6ef28d95c512..f9221f4ba18d 100644 --- a/tracer/src/Datadog.Trace/Debugger/Instrumentation/Collections/ProbeData.cs +++ b/tracer/src/Datadog.Trace/Debugger/Instrumentation/Collections/ProbeData.cs @@ -14,15 +14,15 @@ namespace Datadog.Trace.Debugger.Instrumentation.Collections /// /// Holds data needed during Debugger instrumentation execution. /// - internal readonly record struct ProbeData(string ProbeId, AdaptiveSampler Sampler, ProbeProcessor Processor) + internal readonly record struct ProbeData(string ProbeId, IAdaptiveSampler Sampler, IProbeProcessor Processor) { internal static ProbeData Empty = new(string.Empty, null, null); public string ProbeId { get; } = ProbeId; - public AdaptiveSampler Sampler { get; } = Sampler; + public IAdaptiveSampler Sampler { get; } = Sampler; - public ProbeProcessor Processor { get; } = Processor; + public IProbeProcessor Processor { get; } = Processor; public bool IsEmpty() => this == Empty; } diff --git a/tracer/src/Datadog.Trace/Debugger/Instrumentation/LineDebuggerInvoker.cs b/tracer/src/Datadog.Trace/Debugger/Instrumentation/LineDebuggerInvoker.cs index 933c44037937..25d3b3cec3a3 100644 --- a/tracer/src/Datadog.Trace/Debugger/Instrumentation/LineDebuggerInvoker.cs +++ b/tracer/src/Datadog.Trace/Debugger/Instrumentation/LineDebuggerInvoker.cs @@ -46,17 +46,16 @@ public static void LogArg(ref TArg arg, int index, ref LineDebuggerState s return; } - state.SnapshotCreator.StopSampling(); var paramName = state.MethodMetadataInfo.ParameterNames[index]; - var captureInfo = new CaptureInfo(value: arg, methodState: MethodState.LogArg, name: paramName, memberKind: ScopeMemberKind.Argument); + var captureInfo = new CaptureInfo(state.MethodMetadataIndex, value: arg, methodState: MethodState.LogArg, name: paramName, memberKind: ScopeMemberKind.Argument); + var probeData = state.ProbeData; - if (!state.ProbeData.Processor.Process(ref captureInfo, state.SnapshotCreator)) + if (!state.ProbeData.Processor.Process(ref captureInfo, state.SnapshotCreator, in probeData)) { state.IsActive = false; } state.HasLocalsOrReturnValue = false; - state.SnapshotCreator.StartSampling(); } catch (Exception e) { @@ -81,23 +80,21 @@ public static void LogLocal(ref TLocal local, int index, ref LineDebugge return; } - state.SnapshotCreator.StopSampling(); var localVariableNames = state.MethodMetadataInfo.LocalVariableNames; if (!MethodDebuggerInvoker.TryGetLocalName(index, localVariableNames, out var localName)) { - state.SnapshotCreator.StartSampling(); return; } - var captureInfo = new CaptureInfo(value: local, methodState: MethodState.LogLocal, name: localName, memberKind: ScopeMemberKind.Local); + var captureInfo = new CaptureInfo(state.MethodMetadataIndex, value: local, methodState: MethodState.LogLocal, name: localName, memberKind: ScopeMemberKind.Local); + var probeData = state.ProbeData; - if (!state.ProbeData.Processor.Process(ref captureInfo, state.SnapshotCreator)) + if (!state.ProbeData.Processor.Process(ref captureInfo, state.SnapshotCreator, in probeData)) { state.IsActive = false; } state.HasLocalsOrReturnValue = true; - state.SnapshotCreator.StartSampling(); } catch (Exception e) { @@ -169,22 +166,20 @@ public static LineDebuggerState BeginLine(string probeId, int probeMeta return CreateInvalidatedLineDebuggerState(); } - var state = new LineDebuggerState(probeId, scope: default, methodMetadataIndex, ref probeData, lineNumber, probeFilePath, instance); - - if (!state.SnapshotCreator.ProbeHasCondition && - !state.ProbeData.Sampler.Sample()) + if (!probeData.Processor.ShouldProcess(in probeData)) { + Log.Warning("BeginLine: Skipping the instrumentation. type = {Type}, instance type name = {Name}, probeMetadataIndex = {ProbeMetadataIndex}, probeId = {ProbeId}", new object[] { typeof(TTarget), instance?.GetType().Name, probeMetadataIndex, probeId }); return CreateInvalidatedLineDebuggerState(); } - var captureInfo = new CaptureInfo(invocationTargetType: state.MethodMetadataInfo.DeclaringType, methodState: MethodState.BeginLine, localsCount: state.MethodMetadataInfo.LocalVariableNames.Length, argumentsCount: state.MethodMetadataInfo.ParameterNames.Length, lineCaptureInfo: new LineCaptureInfo(lineNumber, probeFilePath)); + var state = new LineDebuggerState(probeId, scope: default, methodMetadataIndex, ref probeData, lineNumber, probeFilePath, instance); + var captureInfo = new CaptureInfo(state.MethodMetadataIndex, invocationTargetType: state.MethodMetadataInfo.DeclaringType, methodState: MethodState.BeginLine, localsCount: state.MethodMetadataInfo.LocalVariableNames.Length, argumentsCount: state.MethodMetadataInfo.ParameterNames.Length, lineCaptureInfo: new LineCaptureInfo(lineNumber, probeFilePath)); - if (!state.ProbeData.Processor.Process(ref captureInfo, state.SnapshotCreator)) + if (!state.ProbeData.Processor.Process(ref captureInfo, state.SnapshotCreator, in probeData)) { state.IsActive = false; } - state.SnapshotCreator.StartSampling(); return state; } catch (Exception e) @@ -209,13 +204,13 @@ public static void EndLine(ref LineDebuggerState state) return; } - state.SnapshotCreator.StopSampling(); var hasArgumentsOrLocals = state.HasLocalsOrReturnValue || state.MethodMetadataInfo.ParameterNames.Length > 0 || !state.MethodMetadataInfo.Method.IsStatic; - var captureInfo = new CaptureInfo(value: state.InvocationTarget, type: state.MethodMetadataInfo.DeclaringType, invocationTargetType: state.MethodMetadataInfo.DeclaringType, memberKind: ScopeMemberKind.This, methodState: MethodState.EndLine, hasLocalOrArgument: hasArgumentsOrLocals, method: state.MethodMetadataInfo.Method, lineCaptureInfo: new LineCaptureInfo(state.LineNumber, state.ProbeFilePath)); - state.ProbeData.Processor.Process(ref captureInfo, state.SnapshotCreator); + var captureInfo = new CaptureInfo(state.MethodMetadataIndex, value: state.InvocationTarget, type: state.MethodMetadataInfo.DeclaringType, invocationTargetType: state.MethodMetadataInfo.DeclaringType, memberKind: ScopeMemberKind.This, methodState: MethodState.EndLine, hasLocalOrArgument: hasArgumentsOrLocals, method: state.MethodMetadataInfo.Method, lineCaptureInfo: new LineCaptureInfo(state.LineNumber, state.ProbeFilePath)); + var probeData = state.ProbeData; + state.ProbeData.Processor.Process(ref captureInfo, state.SnapshotCreator, in probeData); state.HasLocalsOrReturnValue = false; } diff --git a/tracer/src/Datadog.Trace/Debugger/Instrumentation/LineDebuggerState.cs b/tracer/src/Datadog.Trace/Debugger/Instrumentation/LineDebuggerState.cs index 072e2e594245..ef18066975fa 100644 --- a/tracer/src/Datadog.Trace/Debugger/Instrumentation/LineDebuggerState.cs +++ b/tracer/src/Datadog.Trace/Debugger/Instrumentation/LineDebuggerState.cs @@ -24,6 +24,7 @@ public ref struct LineDebuggerState private readonly int _lineNumber; /// + /// Backing field of . /// Used to perform a fast lookup to grab the proper . /// This index is hard-coded into the method's instrumented bytecode. /// @@ -31,7 +32,6 @@ public ref struct LineDebuggerState // Determines whether we should still be capturing values, or halt for any reason (e.g an exception was caused by our instrumentation, rate limiter threshold reached). internal bool IsActive = true; - internal bool HasLocalsOrReturnValue; /// @@ -52,11 +52,24 @@ internal LineDebuggerState(string probeId, Scope scope, int methodMetadataIndex, _lineNumber = lineNumber; _probeFilePath = probeFilePath; HasLocalsOrReturnValue = false; - SnapshotCreator = DebuggerSnapshotCreator.BuildSnapshotCreator(probeData.Processor); + var processor = probeData.Processor; + SnapshotCreator = processor.CreateSnapshotCreator(); ProbeData = probeData; InvocationTarget = invocationTarget; } + /// + /// Gets an index that is used as a fast lookup to grab the proper . + /// This index is hard-coded into the method's instrumented bytecode. + /// + internal int MethodMetadataIndex + { + get + { + return _methodMetadataIndex; + } + } + internal ref MethodMetadataInfo MethodMetadataInfo => ref MethodMetadataCollection.Instance.Get(_methodMetadataIndex); internal ProbeData ProbeData { get; } @@ -64,7 +77,7 @@ internal LineDebuggerState(string probeId, Scope scope, int methodMetadataIndex, /// /// Gets the LiveDebugger SnapshotCreator /// - internal DebuggerSnapshotCreator SnapshotCreator { get; } + internal IDebuggerSnapshotCreator SnapshotCreator { get; } /// /// Gets the LiveDebugger BeginMethod scope diff --git a/tracer/src/Datadog.Trace/Debugger/Instrumentation/MethodDebuggerInvoker.SingleProbe.cs b/tracer/src/Datadog.Trace/Debugger/Instrumentation/MethodDebuggerInvoker.SingleProbe.cs index 669199459fc8..195b8145fc22 100644 --- a/tracer/src/Datadog.Trace/Debugger/Instrumentation/MethodDebuggerInvoker.SingleProbe.cs +++ b/tracer/src/Datadog.Trace/Debugger/Instrumentation/MethodDebuggerInvoker.SingleProbe.cs @@ -91,22 +91,21 @@ public static MethodDebuggerState BeginMethod_StartMarker(TTarget insta return CreateInvalidatedDebuggerState(); } - var state = new MethodDebuggerState(probeId, scope: default, methodMetadataIndex, ref probeData, instance); - - if (!state.SnapshotCreator.ProbeHasCondition && - !state.ProbeData.Sampler.Sample()) + if (!probeData.Processor.ShouldProcess(in probeData)) { + Log.Warning("BeginMethod_StartMarker: Skipping the instrumentation. type = {Type}, instance type name = {Name}, probeMetadataIndex = {ProbeMetadataIndex}, probeId = {ProbeId}", new object[] { typeof(TTarget), instance?.GetType().Name, probeMetadataIndex, probeId }); return CreateInvalidatedDebuggerState(); } - var captureInfo = new CaptureInfo(value: null, type: state.MethodMetadataInfo.DeclaringType, invocationTargetType: state.MethodMetadataInfo.DeclaringType, methodState: MethodState.EntryStart, localsCount: state.MethodMetadataInfo.LocalVariableNames.Length, argumentsCount: state.MethodMetadataInfo.ParameterNames.Length); + var state = new MethodDebuggerState(probeId, scope: default, methodMetadataIndex, ref probeData, instance); - if (!state.ProbeData.Processor.Process(ref captureInfo, state.SnapshotCreator)) + var captureInfo = new CaptureInfo(state.MethodMetadataIndex, value: null, method: state.MethodMetadataInfo.Method, type: state.MethodMetadataInfo.DeclaringType, invocationTargetType: state.MethodMetadataInfo.DeclaringType, methodState: MethodState.EntryStart, localsCount: state.MethodMetadataInfo.LocalVariableNames.Length, argumentsCount: state.MethodMetadataInfo.ParameterNames.Length); + + if (!state.ProbeData.Processor.Process(ref captureInfo, state.SnapshotCreator, in probeData)) { state.IsActive = false; } - state.SnapshotCreator.StartSampling(); return state; } @@ -122,20 +121,19 @@ public static void BeginMethod_EndMarker(ref MethodDebuggerState state) return; } - state.SnapshotCreator.StopSampling(); var hasArgumentsOrLocals = state.HasLocalsOrReturnValue || state.MethodMetadataInfo.ParameterNames.Length > 0 || !state.MethodMetadataInfo.Method.IsStatic; - var captureInfo = new CaptureInfo(value: state.InvocationTarget, type: state.MethodMetadataInfo.DeclaringType, invocationTargetType: state.MethodMetadataInfo.DeclaringType, methodState: MethodState.EntryEnd, hasLocalOrArgument: hasArgumentsOrLocals, memberKind: ScopeMemberKind.This); + var captureInfo = new CaptureInfo(state.MethodMetadataIndex, value: state.InvocationTarget, type: state.MethodMetadataInfo.DeclaringType, invocationTargetType: state.MethodMetadataInfo.DeclaringType, methodState: MethodState.EntryEnd, hasLocalOrArgument: hasArgumentsOrLocals, memberKind: ScopeMemberKind.This); + var probeData = state.ProbeData; - if (!state.ProbeData.Processor.Process(ref captureInfo, state.SnapshotCreator)) + if (!state.ProbeData.Processor.Process(ref captureInfo, state.SnapshotCreator, in probeData)) { state.IsActive = false; } state.HasLocalsOrReturnValue = false; - state.SnapshotCreator.StartSampling(); } /// @@ -153,17 +151,16 @@ public static void LogArg(ref TArg arg, int index, ref MethodDebuggerState return; } - state.SnapshotCreator.StopSampling(); var paramName = state.MethodMetadataInfo.ParameterNames[index]; - var captureInfo = new CaptureInfo(value: arg, methodState: MethodState.LogArg, name: paramName, memberKind: ScopeMemberKind.Argument); + var captureInfo = new CaptureInfo(state.MethodMetadataIndex, value: arg, methodState: MethodState.LogArg, name: paramName, memberKind: ScopeMemberKind.Argument); + var probeData = state.ProbeData; - if (!state.ProbeData.Processor.Process(ref captureInfo, state.SnapshotCreator)) + if (!state.ProbeData.Processor.Process(ref captureInfo, state.SnapshotCreator, in probeData)) { state.IsActive = false; } state.HasLocalsOrReturnValue = false; - state.SnapshotCreator.StartSampling(); } /// @@ -181,23 +178,21 @@ public static void LogLocal(ref TLocal local, int index, ref MethodDebug return; } - state.SnapshotCreator.StopSampling(); var localVariableNames = state.MethodMetadataInfo.LocalVariableNames; if (!TryGetLocalName(index, localVariableNames, out var localName)) { - state.SnapshotCreator.StartSampling(); return; } - var captureInfo = new CaptureInfo(value: local, methodState: MethodState.LogLocal, name: localName, memberKind: ScopeMemberKind.Local); + var captureInfo = new CaptureInfo(state.MethodMetadataIndex, value: local, methodState: MethodState.LogLocal, name: localName, memberKind: ScopeMemberKind.Local); + var probeData = state.ProbeData; - if (!state.ProbeData.Processor.Process(ref captureInfo, state.SnapshotCreator)) + if (!state.ProbeData.Processor.Process(ref captureInfo, state.SnapshotCreator, in probeData)) { state.IsActive = false; } state.HasLocalsOrReturnValue = true; - state.SnapshotCreator.StartSampling(); } /// @@ -216,18 +211,17 @@ public static DebuggerReturn EndMethod_StartMarker(TTarget instance, Ex return DebuggerReturn.GetDefault(); } - state.SnapshotCreator.StopSampling(); state.MethodPhase = EvaluateAt.Exit; state.InvocationTarget = instance; - var captureInfo = new CaptureInfo(value: exception, invocationTargetType: state.MethodMetadataInfo.DeclaringType, methodState: MethodState.ExitStart, memberKind: ScopeMemberKind.Exception, localsCount: state.MethodMetadataInfo.LocalVariableNames.Length, argumentsCount: state.MethodMetadataInfo.ParameterNames.Length); + var captureInfo = new CaptureInfo(state.MethodMetadataIndex, value: exception, invocationTargetType: state.MethodMetadataInfo.DeclaringType, methodState: MethodState.ExitStart, memberKind: ScopeMemberKind.Exception, localsCount: state.MethodMetadataInfo.LocalVariableNames.Length, argumentsCount: state.MethodMetadataInfo.ParameterNames.Length); + var probeData = state.ProbeData; - if (!state.ProbeData.Processor.Process(ref captureInfo, state.SnapshotCreator)) + if (!state.ProbeData.Processor.Process(ref captureInfo, state.SnapshotCreator, in probeData)) { state.IsActive = false; } - state.SnapshotCreator.StartSampling(); return DebuggerReturn.GetDefault(); } @@ -249,22 +243,22 @@ public static DebuggerReturn EndMethod_StartMarker(TT return new DebuggerReturn(returnValue); } - state.SnapshotCreator.StopSampling(); state.MethodPhase = EvaluateAt.Exit; state.InvocationTarget = instance; + var probeData = state.ProbeData; if (exception != null) { - var captureInfo = new CaptureInfo(value: exception, invocationTargetType: state.MethodMetadataInfo.DeclaringType, methodState: MethodState.ExitStart, memberKind: ScopeMemberKind.Exception, localsCount: state.MethodMetadataInfo.LocalVariableNames.Length, argumentsCount: state.MethodMetadataInfo.ParameterNames.Length); - if (!state.ProbeData.Processor.Process(ref captureInfo, state.SnapshotCreator)) + var captureInfo = new CaptureInfo(state.MethodMetadataIndex, value: exception, invocationTargetType: state.MethodMetadataInfo.DeclaringType, methodState: MethodState.ExitStart, memberKind: ScopeMemberKind.Exception, localsCount: state.MethodMetadataInfo.LocalVariableNames.Length, argumentsCount: state.MethodMetadataInfo.ParameterNames.Length); + if (!state.ProbeData.Processor.Process(ref captureInfo, state.SnapshotCreator, in probeData)) { state.IsActive = false; } } else { - var captureInfo = new CaptureInfo(value: returnValue, name: "@return", invocationTargetType: state.MethodMetadataInfo.DeclaringType, methodState: MethodState.ExitStart, memberKind: ScopeMemberKind.Return, localsCount: state.MethodMetadataInfo.LocalVariableNames.Length, argumentsCount: state.MethodMetadataInfo.ParameterNames.Length); - if (!state.ProbeData.Processor.Process(ref captureInfo, state.SnapshotCreator)) + var captureInfo = new CaptureInfo(state.MethodMetadataIndex, value: returnValue, name: "@return", invocationTargetType: state.MethodMetadataInfo.DeclaringType, methodState: MethodState.ExitStart, memberKind: ScopeMemberKind.Return, localsCount: state.MethodMetadataInfo.LocalVariableNames.Length, argumentsCount: state.MethodMetadataInfo.ParameterNames.Length); + if (!state.ProbeData.Processor.Process(ref captureInfo, state.SnapshotCreator, in probeData)) { state.IsActive = false; } @@ -272,7 +266,6 @@ public static DebuggerReturn EndMethod_StartMarker(TT state.HasLocalsOrReturnValue = true; } - state.SnapshotCreator.StartSampling(); return new DebuggerReturn(returnValue); } @@ -288,13 +281,14 @@ public static void EndMethod_EndMarker(ref MethodDebuggerState state) return; } - state.SnapshotCreator.StopSampling(); var hasArgumentsOrLocals = state.HasLocalsOrReturnValue || state.MethodMetadataInfo.ParameterNames.Length > 0 || !state.MethodMetadataInfo.Method.IsStatic; - var captureInfo = new CaptureInfo(value: state.InvocationTarget, type: state.MethodMetadataInfo.DeclaringType, invocationTargetType: state.MethodMetadataInfo.DeclaringType, memberKind: ScopeMemberKind.This, methodState: MethodState.ExitEnd, hasLocalOrArgument: hasArgumentsOrLocals, method: state.MethodMetadataInfo.Method); - state.ProbeData.Processor.Process(ref captureInfo, state.SnapshotCreator); + var captureInfo = new CaptureInfo(state.MethodMetadataIndex, value: state.InvocationTarget, type: state.MethodMetadataInfo.DeclaringType, invocationTargetType: state.MethodMetadataInfo.DeclaringType, memberKind: ScopeMemberKind.This, methodState: MethodState.ExitEnd, hasLocalOrArgument: hasArgumentsOrLocals, method: state.MethodMetadataInfo.Method); + var probeData = state.ProbeData; + + state.ProbeData.Processor.Process(ref captureInfo, state.SnapshotCreator, in probeData); state.HasLocalsOrReturnValue = false; } @@ -347,6 +341,7 @@ public static void LogException(Exception exception, ref MethodDebuggerState sta Log.Warning(exception, "Error caused by our instrumentation"); state.IsActive = false; + state.ProbeData.Processor.LogException(exception, state.SnapshotCreator); } catch { diff --git a/tracer/src/Datadog.Trace/Debugger/Instrumentation/MethodDebuggerState.cs b/tracer/src/Datadog.Trace/Debugger/Instrumentation/MethodDebuggerState.cs index 08ea9546fb19..483bad451f3e 100644 --- a/tracer/src/Datadog.Trace/Debugger/Instrumentation/MethodDebuggerState.cs +++ b/tracer/src/Datadog.Trace/Debugger/Instrumentation/MethodDebuggerState.cs @@ -7,6 +7,7 @@ using System.ComponentModel; using System.Runtime.CompilerServices; using Datadog.Trace.Debugger.Configurations.Models; +using Datadog.Trace.Debugger.Expressions; using Datadog.Trace.Debugger.Instrumentation.Collections; using Datadog.Trace.Debugger.Snapshots; @@ -28,6 +29,7 @@ public struct MethodDebuggerState private readonly Scope _scope; /// + /// Backing field of . /// Used to perform a fast lookup to grab the proper . /// This index is hard-coded into the method's instrumented bytecode. /// @@ -35,7 +37,6 @@ public struct MethodDebuggerState // Determines whether we should still be capturing values, or halt for any reason (e.g an exception was caused by our instrumentation, rate limiter threshold reached). internal bool IsActive = true; - internal bool HasLocalsOrReturnValue; internal object InvocationTarget; @@ -54,11 +55,24 @@ internal MethodDebuggerState(string probeId, Scope scope, int methodMetadataInde _methodMetadataIndex = methodMetadataIndex; HasLocalsOrReturnValue = false; InvocationTarget = invocationTarget; - SnapshotCreator = DebuggerSnapshotCreator.BuildSnapshotCreator(probeData.Processor); + var processor = probeData.Processor; + SnapshotCreator = processor.CreateSnapshotCreator(); ProbeData = probeData; MethodPhase = EvaluateAt.Entry; } + /// + /// Gets an index that is used as a fast lookup to grab the proper . + /// This index is hard-coded into the method's instrumented bytecode. + /// + internal int MethodMetadataIndex + { + get + { + return _methodMetadataIndex; + } + } + internal EvaluateAt MethodPhase { get; set; } internal ref MethodMetadataInfo MethodMetadataInfo => ref MethodMetadataCollection.Instance.Get(_methodMetadataIndex); @@ -68,7 +82,7 @@ internal MethodDebuggerState(string probeId, Scope scope, int methodMetadataInde /// /// Gets the LiveDebugger SnapshotCreator /// - internal DebuggerSnapshotCreator SnapshotCreator { get; } + internal IDebuggerSnapshotCreator SnapshotCreator { get; } /// /// Gets the LiveDebugger BeginMethod scope diff --git a/tracer/src/Datadog.Trace/Debugger/RateLimiting/AdaptiveSampler.cs b/tracer/src/Datadog.Trace/Debugger/RateLimiting/AdaptiveSampler.cs index cf8dc9a530e3..adabb2f6d8bf 100644 --- a/tracer/src/Datadog.Trace/Debugger/RateLimiting/AdaptiveSampler.cs +++ b/tracer/src/Datadog.Trace/Debugger/RateLimiting/AdaptiveSampler.cs @@ -32,7 +32,7 @@ namespace Datadog.Trace.Debugger.RateLimiting /// to compensate for too rapid changes in the incoming events rate and maintain the target average /// number of samples per window. /// - internal class AdaptiveSampler + internal class AdaptiveSampler : IAdaptiveSampler { private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor(); @@ -104,7 +104,7 @@ internal AdaptiveSampler( } } - internal bool Sample() + public bool Sample() { _countsRef.AddTest(); @@ -116,20 +116,20 @@ internal bool Sample() return false; } - internal bool Keep() + public bool Keep() { _countsRef.AddTest(); _countsRef.AddSample(); return true; } - internal bool Drop() + public bool Drop() { _countsRef.AddTest(); return false; } - internal double NextDouble() + public double NextDouble() { return ThreadSafeRandom.Shared.NextDouble(); } diff --git a/tracer/src/Datadog.Trace/Debugger/RateLimiting/IAdaptiveSampler.cs b/tracer/src/Datadog.Trace/Debugger/RateLimiting/IAdaptiveSampler.cs new file mode 100644 index 000000000000..43f636de8131 --- /dev/null +++ b/tracer/src/Datadog.Trace/Debugger/RateLimiting/IAdaptiveSampler.cs @@ -0,0 +1,18 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +namespace Datadog.Trace.Debugger.RateLimiting +{ + internal interface IAdaptiveSampler + { + bool Sample(); + + bool Keep(); + + bool Drop(); + + double NextDouble(); + } +} diff --git a/tracer/src/Datadog.Trace/Debugger/RateLimiting/NopAdaptiveSampler.cs b/tracer/src/Datadog.Trace/Debugger/RateLimiting/NopAdaptiveSampler.cs new file mode 100644 index 000000000000..331b76b1cbef --- /dev/null +++ b/tracer/src/Datadog.Trace/Debugger/RateLimiting/NopAdaptiveSampler.cs @@ -0,0 +1,32 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +namespace Datadog.Trace.Debugger.RateLimiting +{ + internal class NopAdaptiveSampler : IAdaptiveSampler + { + internal static readonly NopAdaptiveSampler Instance = new(); + + public bool Sample() + { + return true; + } + + public bool Keep() + { + return true; + } + + public bool Drop() + { + return true; + } + + public double NextDouble() + { + return 1.0; + } + } +} diff --git a/tracer/src/Datadog.Trace/Debugger/RateLimiting/ProbeRateLimiter.cs b/tracer/src/Datadog.Trace/Debugger/RateLimiting/ProbeRateLimiter.cs index 5e29c5d88894..090e1428f393 100644 --- a/tracer/src/Datadog.Trace/Debugger/RateLimiting/ProbeRateLimiter.cs +++ b/tracer/src/Datadog.Trace/Debugger/RateLimiting/ProbeRateLimiter.cs @@ -25,7 +25,7 @@ internal class ProbeRateLimiter private readonly AdaptiveSampler _globalSampler = CreateSampler(DefaultGlobalSamplesPerSecond); - private readonly ConcurrentDictionary _samplers = new(); + private readonly ConcurrentDictionary _samplers = new(); internal static ProbeRateLimiter Instance { @@ -41,11 +41,16 @@ internal static ProbeRateLimiter Instance private static AdaptiveSampler CreateSampler(int samplesPerSecond = DefaultSamplesPerSecond) => new(TimeSpan.FromSeconds(1), samplesPerSecond, 180, 16, null); - public AdaptiveSampler GerOrAddSampler(string probeId) + public IAdaptiveSampler GerOrAddSampler(string probeId) { return _samplers.GetOrAdd(probeId, _ => CreateSampler(1)); } + public bool TryAddSampler(string probeId, IAdaptiveSampler sampler) + { + return _samplers.TryAdd(probeId, sampler); + } + public bool Sample(string probeId) { // Rate limiter is engaged at ~1 probe per second (1 probes per 1s time window) diff --git a/tracer/src/Datadog.Trace/Debugger/Snapshots/DebuggerSnapshotCreator.cs b/tracer/src/Datadog.Trace/Debugger/Snapshots/DebuggerSnapshotCreator.cs index 46b2483e0763..f7d4ecb0d11c 100644 --- a/tracer/src/Datadog.Trace/Debugger/Snapshots/DebuggerSnapshotCreator.cs +++ b/tracer/src/Datadog.Trace/Debugger/Snapshots/DebuggerSnapshotCreator.cs @@ -22,7 +22,7 @@ namespace Datadog.Trace.Debugger.Snapshots { - internal class DebuggerSnapshotCreator : IDisposable + internal class DebuggerSnapshotCreator : IDebuggerSnapshotCreator, IDisposable { private const string LoggerVersion = "2"; private const string DDSource = "dd_debugger"; @@ -37,6 +37,7 @@ internal class DebuggerSnapshotCreator : IDisposable private CaptureBehaviour _captureBehaviour; private string _message; private List _errors; + private string _snapshotId; public DebuggerSnapshotCreator(bool isFullSnapshot, ProbeLocation location, bool hasCondition, string[] tags) { @@ -54,6 +55,21 @@ public DebuggerSnapshotCreator(bool isFullSnapshot, ProbeLocation location, bool Initialize(); } + public DebuggerSnapshotCreator(bool isFullSnapshot, ProbeLocation location, bool hasCondition, string[] tags, MethodScopeMembers methodScopeMembers) + : this(isFullSnapshot, location, hasCondition, tags) + { + MethodScopeMembers = methodScopeMembers; + } + + internal string SnapshotId + { + get + { + _snapshotId ??= Guid.NewGuid().ToString(); + return _snapshotId; + } + } + internal MethodScopeMembers MethodScopeMembers { get; private set; } internal bool ProbeHasCondition { get; } @@ -84,11 +100,6 @@ internal void StopSampling() _accumulatedDuration += StopwatchHelpers.GetElapsed(Stopwatch.GetTimestamp() - _lastSampledTime); } - public static DebuggerSnapshotCreator BuildSnapshotCreator(ProbeProcessor processor) - { - return new DebuggerSnapshotCreator(processor.ProbeInfo.IsFullSnapshot, processor.ProbeInfo.ProbeLocation, processor.ProbeInfo.HasCondition, processor.ProbeInfo.Tags); - } - internal CaptureBehaviour DefineSnapshotBehavior(ref CaptureInfo info, EvaluateAt evaluateAt, bool hasCondition) { if (CaptureBehaviour == CaptureBehaviour.Stop) @@ -308,7 +319,7 @@ internal DebuggerSnapshotCreator EndDebugger() internal DebuggerSnapshotCreator EndSnapshot() { _jsonWriter.WritePropertyName("id"); - _jsonWriter.WriteValue(Guid.NewGuid()); + _jsonWriter.WriteValue(SnapshotId); _jsonWriter.WritePropertyName("timestamp"); _jsonWriter.WriteValue(DateTimeOffset.Now.ToUnixTimeMilliseconds()); @@ -882,7 +893,7 @@ public void Dispose() try { Stop(); - MethodScopeMembers.Dispose(); + MethodScopeMembers?.Dispose(); MethodScopeMembers = null; _jsonWriter?.Close(); } diff --git a/tracer/src/Datadog.Trace/Debugger/Snapshots/IDebuggerSnapshotCreator.cs b/tracer/src/Datadog.Trace/Debugger/Snapshots/IDebuggerSnapshotCreator.cs new file mode 100644 index 000000000000..3191f5c11ae6 --- /dev/null +++ b/tracer/src/Datadog.Trace/Debugger/Snapshots/IDebuggerSnapshotCreator.cs @@ -0,0 +1,13 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +using Datadog.Trace.Debugger.Expressions; + +namespace Datadog.Trace.Debugger.Snapshots +{ + internal interface IDebuggerSnapshotCreator + { + } +} diff --git a/tracer/src/Datadog.Trace/Span.cs b/tracer/src/Datadog.Trace/Span.cs index a693f7bb7b78..f287a12b92e2 100644 --- a/tracer/src/Datadog.Trace/Span.cs +++ b/tracer/src/Datadog.Trace/Span.cs @@ -394,6 +394,7 @@ internal void SetException(Exception exception) { Error = true; SetExceptionTags(exception); + Debugger.ExceptionAutoInstrumentation.ExceptionTrackManager.Report(this, exception); } ///