tracing: cap registry size at 5k

This caps the size of the span registry at 5k, evicting "random" (map
order) existing entries when at the limit.
The purpose of this is to serve as a guardrail against leaked spans,
which would otherwise lead to unbounded memory growth.

Touches cockroachdb#59188.

Release justification: low risk, high benefit changes to existing
functionality
Release note: None

pkg/util/tracing/tracer.go

@@ -49,6 +49,9 @@ const (
 	maxStructuredEventsPerSpan = 50
 	// maxChildrenPerSpan limits the number of (direct) child spans in a Span.
 	maxChildrenPerSpan = 1000
+	// maxSpanRegistrySize limits the number of local root spans tracked in
+	// a Tracer's registry.
+	maxSpanRegistrySize = 5000
 )
 
 // These constants are used to form keys to represent tracing context
@@ -413,6 +416,20 @@ func (t *Tracer) startSpanGeneric(
 			// Local root span - put it into the registry of active local root
 			// spans. `Span.Finish` takes care of deleting it again.
 			t.activeSpans.Lock()
+
+			// Ensure that the registry does not grow unboundedly in case there
+			// is a leak. When the registry reaches max size, each new span added
+			// kicks out some old span. We rely on map iteration order here to
+			// make this cheap.
+			if toDelete := len(t.activeSpans.m) - maxSpanRegistrySize + 1; toDelete > 0 {
+				for k := range t.activeSpans.m {
+					delete(t.activeSpans.m, k)
+					toDelete--
+					if toDelete <= 0 {
+						break
+					}
+				}
+			}
 			t.activeSpans.m[spanID] = s
 			t.activeSpans.Unlock()
 		}

pkg/util/tracing/tracer_test.go

@@ -429,6 +429,18 @@ func TestShadowTracerNilTracer(t *testing.T) {
 	})
 }
 
+func TestTracer_RegistryMaxSize(t *testing.T) {
+	tr := NewTracer()
+	for i := 0; i < maxSpanRegistrySize+10; i++ {
+		_ = tr.StartSpan("foo", WithForceRealSpan()) // intentionally not closed
+		exp := i + 1
+		if exp > maxSpanRegistrySize {
+			exp = maxSpanRegistrySize
+		}
+		require.Len(t, tr.activeSpans.m, exp)
+	}
+}
+
 // TestActiveSpanVisitorErrors confirms that the visitor of the Tracer's
 // activeSpans registry gracefully exits upon receiving a sentinel error from
 // `iterutil.StopIteration()`.