Add rate limit for trace rate log

DataDog · Feb 27, 2024 · d381d47 · d381d47
1 parent 7df424a
commit d381d47
Show file tree

Hide file tree

Showing 2 changed files with 22 additions and 1 deletion.
diff --git a/tracer/src/Datadog.Trace/Sampling/RateLimiter.cs b/tracer/src/Datadog.Trace/Sampling/RateLimiter.cs
@@ -80,6 +80,10 @@ public bool Allowed(Span span)
 
         public abstract void OnDisallowed(Span span, int count, int intervalMs, int maxTracesPerInterval);
 
+        public virtual void OnRefresh(int intervalMs, int checksInLastInterval, int allowedInLastInterval)
+        {
+        }
+
         public abstract void OnFinally(Span span);
 
         public float GetEffectiveRate()
@@ -134,6 +138,7 @@ private void WaitForRefresh()
                     _windowAllowed = 0;
                     _windowChecks = 0;
                     _windowBegin = now;
+                    OnRefresh(_intervalMilliseconds, _previousWindowChecks, _previousWindowAllowed);
                 }
 
                 while (_intervalQueue.TryPeek(out var time) && now.Subtract(time) > _interval)

diff --git a/tracer/src/Datadog.Trace/Sampling/TracerRateLimiter.cs b/tracer/src/Datadog.Trace/Sampling/TracerRateLimiter.cs
@@ -3,13 +3,15 @@
 // This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
 // </copyright>
 
+using System.Threading;
 using Datadog.Trace.Logging;
 
 namespace Datadog.Trace.Sampling
 {
     internal class TracerRateLimiter : RateLimiter
     {
         private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor<TracerRateLimiter>();
+        private bool _warningWritten;
 
         public TracerRateLimiter(int? maxTracesPerInterval)
             : base(maxTracesPerInterval)
@@ -18,7 +20,21 @@ public TracerRateLimiter(int? maxTracesPerInterval)
 
         public override void OnDisallowed(Span span, int count, int intervalMs, int maxTracesPerInterval)
         {
-            Log.Warning<string, int, int>("Dropping trace id {TraceId} with count of {Count} for last {Interval}ms.", span.Context.RawTraceId, count, intervalMs);
+            if (!Volatile.Read(ref _warningWritten))
+            {
+                Log.Warning<string, int, int>("Dropping trace id {TraceId} with count of {Count} for last {Interval}ms.", span.Context.RawTraceId, count, intervalMs);
+                _warningWritten = true;
+            }
+        }
+
+        public override void OnRefresh(int intervalMs, int checksInLastInterval, int allowedInLastInterval)
+        {
+            if (Volatile.Read(ref _warningWritten))
+            {
+                Log.Warning<int, int, int>("Trace rate limit interval reset: Allowed {Allowed} traces out of {Checked} in last {Interval}ms.", allowedInLastInterval, checksInLastInterval, intervalMs);
+            }
+
+            _warningWritten = false;
         }
 
         public override void OnFinally(Span span)