[Tracer] Set _dd.base_service tag whenever a span's service name is…

… different than the default value (`DD_SERVICE`) (#5502)

## Summary of changes

This fixes the setting of the `ServiceName` so that instead of just
adding a `_dd.base_service` tag when `Span.ServiceName` is overridden
that it'll be computed during MessagePack serialization and checked to
see if it is different than the default value (aka `DD_SERVICE`).

If it is different from the default then we add the default name as
`_dd.base_service`

## Reason for change

We want to record the base service name after the service name has been
updated.

## Implementation details

- Remove the setting of this in `Span.ServiceName`
- Add check in `SpanMessagePackFormatter` for that.

## Test coverage

- Updated snapshots

## Other details
<!-- Fixes #{issue} -->

<!-- ⚠️ Note: where possible, please obtain 2 approvals prior to
merging. Unless CODEOWNERS specifies otherwise, for external teams it is
typically best to have one review from a team member, and one review
from apm-dotnet. Trivial changes do not require 2 reviews. -->

---------

Co-authored-by: Andrew Lock <andrew.lock@datadoghq.com>

.gitignore

@@ -381,4 +381,6 @@ cmake-build-debug/
 .cache/jb
 
 #datadog coverage
-datadog-coverage-*
+datadog-coverage-*
+
+.localstack/*

tracer/build/smoke_test_snapshots/smoke_test_snapshots.json

@@ -17,7 +17,8 @@
         "language": "dotnet",
         "out.host": "localhost",
         "runtime-id": "11c61d09-16bb-477f-87ab-4f81d656c5ca",
-        "span.kind": "client"
+        "span.kind": "client",
+        "_dd.base_service": "AspNetCoreSmokeTest"
       },
       "metrics": {
         "_dd.top_level": 1.0,

tracer/build/smoke_test_snapshots/smoke_test_snapshots_2_1.json

@@ -17,7 +17,8 @@
         "language": "dotnet",
         "out.host": "localhost",
         "runtime-id": "b34b2bed-9444-4597-87f6-b8202b03b1b8",
-        "span.kind": "client"
+        "span.kind": "client",
+        "_dd.base_service": "AspNetCoreSmokeTest"
       },
       "metrics": {
         "_dd.top_level": 1.0,

tracer/src/Datadog.Trace/Agent/MessagePack/MessagePackStringCache.cs

@@ -29,6 +29,9 @@ internal static class MessagePackStringCache
     [ThreadStatic]
     private static CachedBytes _origin;
 
+    [ThreadStatic]
+    private static CachedBytes _service;
+
     private static CachedBytes _gitCommitSha;
     private static CachedBytes _gitRepositoryUrl;
     private static CachedBytes _aasSiteNameBytes;
@@ -48,6 +51,7 @@ public static void Clear()
         _env = default;
         _version = default;
         _origin = default;
+        _service = default;
         _gitCommitSha = default;
         _gitRepositoryUrl = default;
         _aasSiteNameBytes = default;
@@ -88,6 +92,11 @@ public static void Clear()
         return GetBytes(origin, ref _origin);
     }
 
+    public static byte[]? GetServiceBytes(string? service)
+    {
+        return GetBytes(service, ref _service);
+    }
+
     public static byte[]? GetAzureAppServiceKeyBytes(string key, string? value)
     {
         switch (key)

tracer/src/Datadog.Trace/Agent/MessagePack/SpanMessagePackFormatter.cs

@@ -65,6 +65,7 @@ internal class SpanMessagePackFormatter : IMessagePackFormatter<TraceChunkModel>
 
         private readonly byte[] _originNameBytes = StringEncoding.UTF8.GetBytes(Trace.Tags.Origin);
         private readonly byte[] _lastParentIdBytes = StringEncoding.UTF8.GetBytes(Trace.Tags.LastParentId);
+        private readonly byte[] _baseServiceNameBytes = StringEncoding.UTF8.GetBytes(Trace.Tags.BaseService);
 
         // numeric tags
         private readonly byte[] _metricsBytes = StringEncoding.UTF8.GetBytes("metrics");
@@ -412,7 +413,8 @@ private int WriteTags(ref byte[] bytes, int offset, in SpanModel model, ITagProc
             offset += MessagePackBinary.WriteStringBytes(ref bytes, offset, _languageValueBytes);
 
             // add "version" tags to all spans whose service name is the default service name
-            if (string.Equals(span.Context.ServiceNameInternal, model.TraceChunk.DefaultServiceName, StringComparison.OrdinalIgnoreCase))
+            var serviceNameEqualsDefault = string.Equals(span.Context.ServiceNameInternal, model.TraceChunk.DefaultServiceName, StringComparison.OrdinalIgnoreCase);
+            if (serviceNameEqualsDefault)
             {
                 var versionRawBytes = MessagePackStringCache.GetVersionBytes(model.TraceChunk.ServiceVersion);
 
@@ -424,7 +426,20 @@ private int WriteTags(ref byte[] bytes, int offset, in SpanModel model, ITagProc
                 }
             }
 
-            // SCI tags will be sent only once per trace chunk
+            // add _dd.base_service tag to spans where the service name has been overrideen
+            if (!serviceNameEqualsDefault && !string.IsNullOrEmpty(model.TraceChunk.DefaultServiceName))
+            {
+                var serviceNameRawBytes = MessagePackStringCache.GetServiceBytes(model.TraceChunk.DefaultServiceName);
+
+                if (serviceNameRawBytes is not null)
+                {
+                    count++;
+                    offset += MessagePackBinary.WriteStringBytes(ref bytes, offset, _baseServiceNameBytes);
+                    offset += MessagePackBinary.WriteRaw(ref bytes, offset, serviceNameRawBytes);
+                }
+            }
+
+            // SCI tags will be sent only once per trace
             if (model.IsFirstSpanInChunk)
             {
                 var gitCommitShaRawBytes = MessagePackStringCache.GetGitCommitShaBytes(model.TraceChunk.GitCommitSha);

tracer/src/Datadog.Trace/Span.cs

@@ -31,7 +31,6 @@ internal partial class Span : ISpan
         private static readonly bool IsLogLevelDebugEnabled = Log.IsEnabled(LogEventLevel.Debug);
 
         private int _isFinished;
-        private bool _baseServiceTagSet;
 
         internal Span(SpanContext context, DateTimeOffset? start)
             : this(context, start, null)
@@ -80,13 +79,6 @@ internal string ServiceName
             get => Context.ServiceNameInternal;
             set
             {
-                // Ignore case because service name and _dd.base_service are normalized in the agent and backend
-                if (!_baseServiceTagSet && !string.Equals(value, Context.ServiceNameInternal, StringComparison.OrdinalIgnoreCase))
-                {
-                    Tags.SetTag(Trace.Tags.BaseService, Context.ServiceNameInternal);
-                    _baseServiceTagSet = true;
-                }
-
                 Context.ServiceNameInternal = value;
             }
         }

tracer/test/Datadog.Trace.TestHelpers/SpanMetadataV0Rules.cs

@@ -150,6 +150,7 @@ public static Result IsAwsDynamoDbV0(this MockSpan span) => Result.FromSpan(span
                 .IsPresent("http.status_code")
                 .IsPresent("http.url")
                 .Matches("component", "aws-sdk")
+                .IsOptional("_dd.base_service")
                 .Matches("span.kind", "client"));
 
         public static Result IsAwsKinesisOutboundV0(this MockSpan span) => Result.FromSpan(span)
@@ -170,6 +171,7 @@ public static Result IsAwsKinesisOutboundV0(this MockSpan span) => Result.FromSp
                 .IsPresent("http.status_code")
                 .IsPresent("http.url")
                 .Matches("component", "aws-sdk")
+                .IsOptional("_dd.base_service")
                 .Matches("span.kind", "producer"));
 
         public static Result IsAwsSqsRequestV0(this MockSpan span) => Result.FromSpan(span)
@@ -237,6 +239,7 @@ public static Result IsAzureServiceBusInboundV0(this MockSpan span, ISet<string>
                 .IsOptional("otel.status_description")
                 .IfPresentMatches("component", "servicebus")
                 .IfPresentMatches("kind", "consumer")
+                .IsOptional("_dd.base_service")
                 .Matches("span.kind", "consumer"));
 
         public static Result IsAzureServiceBusOutboundV0(this MockSpan span, ISet<string> excludeTags = null) => Result.FromSpan(span, excludeTags)
@@ -258,6 +261,7 @@ public static Result IsAzureServiceBusOutboundV0(this MockSpan span, ISet<string
                 .IsOptional("otel.status_description")
                 .IfPresentMatches("component", "servicebus")
                 .IfPresentMatches("kind", "producer")
+                .IsOptional("_dd.base_service")
                 .Matches("span.kind", "producer"));
 
         public static Result IsAzureServiceBusRequestV0(this MockSpan span, ISet<string> excludeTags = null) => Result.FromSpan(span, excludeTags)
@@ -281,6 +285,7 @@ public static Result IsAzureServiceBusRequestV0(this MockSpan span, ISet<string>
                 .IsOptional("otel.status_description")
                 .IfPresentMatches("component", "servicebus")
                 .IfPresentMatches("kind", "client")
+                .IsOptional("_dd.base_service")
                 .Matches("span.kind", "client"));
 
         public static Result IsCosmosDbV0(this MockSpan span) => Result.FromSpan(span)
@@ -379,6 +384,7 @@ public static Result IsHotChocolateV0(this MockSpan span) => Result.FromSpan(spa
                 .IsOptional("graphql.operation.type")
                 .IsPresent("graphql.source")
                 .Matches("component", "HotChocolate")
+                .IsOptional("_dd.base_service")
                 .Matches("span.kind", "server"));
 
         public static Result IsHttpMessageHandlerV0(this MockSpan span) => Result.FromSpan(span)
@@ -470,6 +476,7 @@ public static Result IsMySqlV0(this MockSpan span) => Result.FromSpan(span)
                 .Matches("db.type", "mysql")
                 .Matches("component", "MySql")
                 .Matches("span.kind", "client")
+                .IsOptional("_dd.base_service")
                 .IsOptional("_dd.dbm_trace_injected"));
 
         public static Result IsNpgsqlV0(this MockSpan span) => Result.FromSpan(span)
@@ -547,6 +554,7 @@ public static Result IsRemotingClientV0(this MockSpan span) => Result.FromSpan(s
                .IsPresent("rpc.method")
                .Matches("rpc.system", "dotnet_remoting")
                .Matches("component", "Remoting")
+               .IsOptional("_dd.base_service")
                .Matches("span.kind", "client"));
 
         public static Result IsRemotingServerV0(this MockSpan span) => Result.FromSpan(span)
@@ -556,6 +564,7 @@ public static Result IsRemotingServerV0(this MockSpan span) => Result.FromSpan(s
                .IsPresent("rpc.method")
                .Matches("rpc.system", "dotnet_remoting")
                .Matches("component", "Remoting")
+               .IsOptional("_dd.base_service")
                .Matches("span.kind", "server"));
 
         public static Result IsServiceRemotingClientV0(this MockSpan span) => Result.FromSpan(span)

tracer/test/Datadog.Trace.TestHelpers/SpanMetadataV1Rules.cs

@@ -158,6 +158,7 @@ public static Result IsAwsDynamoDbV1(this MockSpan span) => Result.FromSpan(span
                 .IsOptional("peer.service.remapped_from")
                 .MatchesOneOf("_dd.peer.service.source", "tablename", "peer.service")
                 .Matches("component", "aws-sdk")
+                .IsOptional("_dd.base_service")
                 .Matches("span.kind", "client"));
 
         public static Result IsAwsKinesisOutboundV1(this MockSpan span) => Result.FromSpan(span)
@@ -181,6 +182,7 @@ public static Result IsAwsKinesisOutboundV1(this MockSpan span) => Result.FromSp
                 .IsOptional("peer.service.remapped_from")
                 .MatchesOneOf("_dd.peer.service.source", "streamname", "peer.service")
                 .Matches("component", "aws-sdk")
+                .IsOptional("_dd.base_service")
                 .Matches("span.kind", "producer"));
 
         public static Result IsAwsSqsInboundV1(this MockSpan span) => Result.FromSpan(span)
@@ -252,6 +254,7 @@ public static Result IsAwsSqsRequestV1(this MockSpan span) => Result.FromSpan(sp
                 .IsOptional("peer.service.remapped_from")
                 .MatchesOneOf("_dd.peer.service.source", "queuename", "peer.service")
                 .Matches("component", "aws-sdk")
+                .IsOptional("_dd.base_service")
                 .Matches("span.kind", "client"));
 
         public static Result IsAwsSnsInboundV1(this MockSpan span) => Result.FromSpan(span)
@@ -298,6 +301,7 @@ public static Result IsAwsSnsOutboundV1(this MockSpan span) => Result.FromSpan(s
                 .IsOptional("peer.service.remapped_from")
                 .MatchesOneOf("_dd.peer.service.source", "topicname", "peer.service")
                 .Matches("component", "aws-sdk")
+                .IsOptional("_dd.base_service")
                 .Matches("span.kind", "producer"));
 
         public static Result IsAwsSnsRequestV1(this MockSpan span) => Result.FromSpan(span)
@@ -322,6 +326,7 @@ public static Result IsAwsSnsRequestV1(this MockSpan span) => Result.FromSpan(sp
                 .IsOptional("peer.service.remapped_from")
                 .MatchesOneOf("_dd.peer.service.source", "topicname", "peer.service")
                 .Matches("component", "aws-sdk")
+                .IsOptional("_dd.base_service")
                 .Matches("span.kind", "client"));
 
         public static Result IsAzureServiceBusInboundV1(this MockSpan span, ISet<string> excludeTags = null) => Result.FromSpan(span, excludeTags)
@@ -345,6 +350,7 @@ public static Result IsAzureServiceBusInboundV1(this MockSpan span, ISet<string>
                 .IsOptional("otel.status_description")
                 .IfPresentMatches("component", "servicebus")
                 .IfPresentMatches("kind", "consumer")
+                .IsOptional("_dd.base_service")
                 .Matches("span.kind", "consumer"));
 
         public static Result IsAzureServiceBusOutboundV1(this MockSpan span, ISet<string> excludeTags = null) => Result.FromSpan(span, excludeTags)
@@ -369,6 +375,7 @@ public static Result IsAzureServiceBusOutboundV1(this MockSpan span, ISet<string
                 .MatchesOneOf("_dd.peer.service.source", "messaging.destination.name", "message_bus.destination", "peer.service")
                 .IfPresentMatches("component", "servicebus")
                 .IfPresentMatches("kind", "producer")
+                .IsOptional("_dd.base_service")
                 .Matches("span.kind", "producer"));
 
         public static Result IsAzureServiceBusRequestV1(this MockSpan span, ISet<string> excludeTags = null) => Result.FromSpan(span, excludeTags)
@@ -395,6 +402,7 @@ public static Result IsAzureServiceBusRequestV1(this MockSpan span, ISet<string>
                 .MatchesOneOf("_dd.peer.service.source", "messaging.destination.name", "message_bus.destination", "peer.service")
                 .IfPresentMatches("component", "servicebus")
                 .IfPresentMatches("kind", "client")
+                .IsOptional("_dd.base_service")
                 .Matches("span.kind", "client"));
 
         public static Result IsCosmosDbV1(this MockSpan span) => Result.FromSpan(span)
@@ -612,6 +620,7 @@ public static Result IsMsmqOutboundV1(this MockSpan span) => Result.FromSpan(spa
                 .IsOptional("peer.service.remapped_from")
                 .MatchesOneOf("_dd.peer.service.source", "out.host", "peer.service")
                 .Matches("component", "msmq")
+                .IsOptional("_dd.base_service")
                 .Matches("span.kind", "producer"));
 
         public static Result IsMsmqClientV1(this MockSpan span) => Result.FromSpan(span)
@@ -773,6 +782,7 @@ public static Result IsRemotingClientV1(this MockSpan span) => Result.FromSpan(s
                      .Matches("rpc.system", "dotnet_remoting")
                      .Matches("component", "Remoting")
                      .IfPresentMatchesOneOf("_dd.peer.service.source", "peer.service", "rpc.service")
+                     .IsOptional("_dd.base_service")
                      .MatchesOneOf("span.kind", "client", "server"));
 
         public static Result IsRemotingServerV1(this MockSpan span) => Result.FromSpan(span)
@@ -782,6 +792,7 @@ public static Result IsRemotingServerV1(this MockSpan span) => Result.FromSpan(s
                .IsPresent("rpc.method")
                .Matches("rpc.system", "dotnet_remoting")
                .Matches("component", "Remoting")
+               .IsOptional("_dd.base_service")
                .MatchesOneOf("span.kind", "client", "server"));
 
         public static Result IsServiceRemotingClientV1(this MockSpan span) => Result.FromSpan(span)

tracer/test/Datadog.Trace.Tests/Agent/AgentWriterTests.cs

@@ -142,11 +142,11 @@ public async Task SpanSampling_ShouldSend_MultipleMatchedSpans_WhenStatsDropsOne
 
             var traceContext = new TraceContext(tracer);
             traceContext.SetSamplingPriority(SamplingPriorityValues.UserReject, SamplingMechanism.Manual);
-            var rootSpanContext = new SpanContext(null, traceContext, "service");
+            var rootSpanContext = new SpanContext(null, traceContext, "testhost");
             var rootSpan = new Span(rootSpanContext, DateTimeOffset.UtcNow) { OperationName = "operation" };
-            var droppedChildSpan = new Span(new SpanContext(rootSpanContext, traceContext, "service"), DateTimeOffset.UtcNow) { OperationName = "drop_me" };
-            var droppedChildSpan2 = new Span(new SpanContext(rootSpanContext, traceContext, "service"), DateTimeOffset.UtcNow) { OperationName = "drop_me_also" };
-            var keptChildSpan = new Span(new SpanContext(rootSpanContext, traceContext, "service"), DateTimeOffset.UtcNow) { OperationName = "operation" };
+            var droppedChildSpan = new Span(new SpanContext(rootSpanContext, traceContext, "testhost"), DateTimeOffset.UtcNow) { OperationName = "drop_me" };
+            var droppedChildSpan2 = new Span(new SpanContext(rootSpanContext, traceContext, "testhost"), DateTimeOffset.UtcNow) { OperationName = "drop_me_also" };
+            var keptChildSpan = new Span(new SpanContext(rootSpanContext, traceContext, "testhost"), DateTimeOffset.UtcNow) { OperationName = "operation" };
             traceContext.AddSpan(rootSpan); // IS single span sampled
             traceContext.AddSpan(droppedChildSpan); // is NOT single span sampled
             traceContext.AddSpan(droppedChildSpan2); // is NOT single span sampled

tracer/test/Datadog.Trace.Tests/Agent/MessagePack/SpanMessagePackFormatterTests.cs

@@ -31,13 +31,13 @@ public class SpanMessagePackFormatterTests
     public void SerializeSpans()
     {
         var formatter = SpanFormatterResolver.Instance.GetFormatter<TraceChunkModel>();
-
+        var traceContext = new TraceContext(Mock.Of<IDatadogTracer>());
         var parentContext = new SpanContext(new TraceId(0, 1), 2, (int)SamplingPriority.UserKeep, "ServiceName1", "origin1");
 
         var spans = new[]
         {
             new Span(parentContext, DateTimeOffset.UtcNow),
-            new Span(new SpanContext(parentContext, new TraceContext(Mock.Of<IDatadogTracer>()), "ServiceName1"), DateTimeOffset.UtcNow),
+            new Span(new SpanContext(parentContext, traceContext, "ServiceName1"), DateTimeOffset.UtcNow),
             new Span(new SpanContext(new TraceId(0, 5), 6, (int)SamplingPriority.UserKeep, "ServiceName3", "origin3"), DateTimeOffset.UtcNow),
         };
 

tracer/test/Datadog.Trace.Tests/SpanTests.cs

@@ -357,63 +357,5 @@ public void SetTag_Double()
             span.GetTag(stringKey).Should().Be(stringValue);
             span.GetTag(numericKey).Should().BeNull();
         }
-
-        [Fact]
-        public void ServiceOverride_WhenSet_HasBaseService()
-        {
-            var origName = "MyServiceA";
-            var newName = "MyServiceB";
-            var span = _tracer.StartSpan(nameof(SetTag_Double), serviceName: origName);
-            span.ServiceName = newName;
-
-            span.ServiceName.Should().Be(newName);
-            span.GetTag(Tags.BaseService).Should().Be(origName);
-        }
-
-        [Fact]
-        public void ServiceOverride_WhenNotSet_HasNoBaseService()
-        {
-            var origName = "MyServiceA";
-            var span = _tracer.StartSpan(nameof(SetTag_Double), serviceName: origName);
-
-            span.ServiceName.Should().Be(origName);
-            span.GetTag(Tags.BaseService).Should().BeNull();
-        }
-
-        [Fact]
-        public void ServiceOverride_WhenSetSame_HasNoBaseService()
-        {
-            var origName = "MyServiceA";
-            var span = _tracer.StartSpan(nameof(SetTag_Double), serviceName: origName);
-            span.ServiceName = origName;
-
-            span.ServiceName.Should().Be(origName);
-            span.GetTag(Tags.BaseService).Should().BeNull();
-        }
-
-        [Fact]
-        public void ServiceOverride_WhenSetSameWithDifferentCase_HasNoBaseService()
-        {
-            var origName = "MyServiceA";
-            var newName = origName.ToUpper();
-            var span = _tracer.StartSpan(nameof(SetTag_Double), serviceName: origName);
-            span.ServiceName = newName;
-
-            span.ServiceName.Should().Be(newName); // ServiceName should change although _dd.base_service has not been added
-            span.GetTag(Tags.BaseService).Should().BeNull();
-        }
-
-        [Fact]
-        public void ServiceOverride_WhenSetTwice_HasBaseService()
-        {
-            var origName = "MyServiceA";
-            var newName = "MyServiceC";
-            var span = _tracer.StartSpan(nameof(SetTag_Double), serviceName: origName);
-            span.ServiceName = "MyServiceB";
-            span.ServiceName = newName;
-
-            span.ServiceName.Should().Be(newName);
-            span.GetTag(Tags.BaseService).Should().Be(origName);
-        }
     }
 }