Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add rate limit for the log written by TraceRateLimiter #5229

Merged
merged 2 commits into from
Mar 26, 2024
Merged

Add rate limit for the log written by TraceRateLimiter #5229

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9d7d00f
Add rate limit for trace rate log
andrewlock Feb 22, 2024
e9292ef
Update tracer/src/Datadog.Trace/Sampling/TracerRateLimiter.cs
andrewlock Mar 11, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions tracer/src/Datadog.Trace/Sampling/RateLimiter.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,10 @@ public bool Allowed(Span span)

public abstract void OnDisallowed(Span span, int count, int intervalMs, int maxTracesPerInterval);

public virtual void OnRefresh(int intervalMs, int checksInLastInterval, int allowedInLastInterval)
{
}

public abstract void OnFinally(Span span);

public float GetEffectiveRate()
Expand Down Expand Up @@ -134,6 +138,7 @@ private void WaitForRefresh()
_windowAllowed = 0;
_windowChecks = 0;
_windowBegin = now;
OnRefresh(_intervalMilliseconds, _previousWindowChecks, _previousWindowAllowed);
}

while (_intervalQueue.TryPeek(out var time) && now.Subtract(time) > _interval)
Expand Down
18 changes: 17 additions & 1 deletion tracer/src/Datadog.Trace/Sampling/TracerRateLimiter.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,15 @@
// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
// </copyright>

using System.Threading;
using Datadog.Trace.Logging;

namespace Datadog.Trace.Sampling
{
internal class TracerRateLimiter : RateLimiter
{
private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor<TracerRateLimiter>();
private bool _warningWritten;

public TracerRateLimiter(int? maxTracesPerInterval)
: base(maxTracesPerInterval)
Expand All @@ -18,7 +20,21 @@ public TracerRateLimiter(int? maxTracesPerInterval)

public override void OnDisallowed(Span span, int count, int intervalMs, int maxTracesPerInterval)
{
Log.Warning<string, int, int>("Dropping trace id {TraceId} with count of {Count} for last {Interval}ms.", span.Context.RawTraceId, count, intervalMs);
if (!Volatile.Read(ref _warningWritten))
{
Log.Warning<string, int, int>("Rate limiter dropped a trace ({TraceId}) after reaching {Count} traces in the last {Interval}ms.", span.Context.RawTraceId, count, intervalMs);
_warningWritten = true;
}
}

public override void OnRefresh(int intervalMs, int checksInLastInterval, int allowedInLastInterval)
{
if (Volatile.Read(ref _warningWritten))
{
Log.Warning<int, int, int>("Trace rate limit interval reset: Allowed {Allowed} traces out of {Checked} in last {Interval}ms.", allowedInLastInterval, checksInLastInterval, intervalMs);
}

_warningWritten = false;
}

public override void OnFinally(Span span)
Expand Down
Loading