Skip to content

Commit

Permalink
Merge pull request #3352 from aws/muhamoth/DOTNET-7519-Observability-…
Browse files Browse the repository at this point in the history 
…Create-tracing-spans

Create tracing spans to cover the SDK operations
  • Loading branch information
@muhammad-othman
muhammad-othman authored Jun 27, 2024
2 parents a692e9e + 8569b06 commit c614f0b
Show file tree
Hide file tree
Showing 9 changed files with 245 additions and 43 deletions.
3 changes: 3 additions & 0 deletions sdk/src/Core/Amazon.Runtime/Pipeline/Handlers/CompressionHandler.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@
using Amazon.Util;
using Amazon.Runtime.Internal.Util;
using Amazon.Runtime.Internal.Compression;
using Amazon.Runtime.Telemetry.Tracing;
using Amazon.Runtime.Telemetry;

namespace Amazon.Runtime.Internal
{
Expand Down Expand Up @@ -105,6 +107,7 @@ protected virtual void PreInvoke(IExecutionContext executionContext)
if (input.Length >= minCompressionSize)
{
executionContext.RequestContext.Metrics.AddProperty(Metric.UncompressedRequestSize, input.Length);
using (var traceSpan = TracingUtilities.CreateSpan(executionContext.RequestContext, TelemetryConstants.RequestCompressionSpanName))
using (executionContext.RequestContext.Metrics.StartEvent(Metric.RequestCompressionTime))
{
request.Content = compressionAlgorithm.Compress(input);
Expand Down
4 changes: 4 additions & 0 deletions sdk/src/Core/Amazon.Runtime/Pipeline/Handlers/CredentialsRetriever.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@

using Amazon.Runtime.Internal.Auth;
using Amazon.Runtime.Internal.Util;
using Amazon.Runtime.Telemetry;
using Amazon.Runtime.Telemetry.Tracing;
using Amazon.Util;
using System;
using System.IO;
Expand Down Expand Up @@ -51,6 +53,7 @@ protected virtual void PreInvoke(IExecutionContext executionContext)
ImmutableCredentials ic = null;
if (Credentials != null && !(Credentials is AnonymousAWSCredentials) && !(executionContext.RequestContext.Signer is BearerTokenSigner))
{
using (var traceSpan = TracingUtilities.CreateSpan(executionContext.RequestContext, TelemetryConstants.CredentialsRetrievalSpanName))
using(executionContext.RequestContext.Metrics.StartEvent(Metric.CredentialsRequestTime))
{
ic = Credentials.GetCredentials();
Expand Down Expand Up @@ -87,6 +90,7 @@ public override async System.Threading.Tasks.Task<T> InvokeAsync<T>(IExecutionCo
ImmutableCredentials ic = null;
if (Credentials != null && !(Credentials is AnonymousAWSCredentials))
{
using (var traceSpan = TracingUtilities.CreateSpan(executionContext.RequestContext, TelemetryConstants.CredentialsRetrievalSpanName))
using(executionContext.RequestContext.Metrics.StartEvent(Metric.CredentialsRequestTime))
{
ic = await Credentials.GetCredentialsAsync().ConfigureAwait(false);
Expand Down
13 changes: 2 additions & 11 deletions sdk/src/Core/Amazon.Runtime/Pipeline/Handlers/EndpointDiscoveryHandler.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
using System;
using System.Collections.Generic;
using System.Net;
using Amazon.Util;

namespace Amazon.Runtime.Internal
{
Expand Down Expand Up @@ -178,23 +179,13 @@ private static IEnumerable<DiscoveryEndpointBase> ProcessEndpointDiscovery(IRequ
var operationName = string.Empty;
if (endpointDiscoveryData.Identifiers != null && endpointDiscoveryData.Identifiers.Count > 0)
{
operationName = OperationNameFromRequestName(requestContext.RequestName);
operationName = AWSSDKUtils.ExtractOperationName(requestContext.RequestName);
}
return options.EndpointOperation(new EndpointOperationContext(requestContext.ImmutableCredentials.AccessKey, operationName, endpointDiscoveryData, evictCacheKey, evictUri));
}

return null;
}

private static string OperationNameFromRequestName(string requestName)
{
if (requestName.EndsWith("Request", StringComparison.Ordinal))
{
return requestName.Substring(0, requestName.Length - 7);
}

return requestName;
}

private static bool IsInvalidEndpointException(Exception exception)
{
Expand Down
32 changes: 30 additions & 2 deletions sdk/src/Core/Amazon.Runtime/Pipeline/Handlers/MetricsHandler.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@
using Amazon.Runtime.Internal.Util;
using System.Globalization;
using System;
using Amazon.Runtime.Telemetry.Tracing;
using Amazon.Runtime.Telemetry;

namespace Amazon.Runtime.Internal
{
Expand All @@ -34,15 +36,28 @@ public class MetricsHandler : PipelineHandler
public override void InvokeSync(IExecutionContext executionContext)
{
executionContext.RequestContext.Metrics.AddProperty(Metric.AsyncCall, false);

var operationName = AWSSDKUtils.ExtractOperationName(executionContext.RequestContext.RequestName);
var spanName = $"{executionContext.RequestContext.ServiceMetaData.ServiceId}.{operationName}";
var span = TracingUtilities.CreateSpan(executionContext.RequestContext, spanName);

try
{
executionContext.RequestContext.Metrics.StartEvent(Metric.ClientExecuteTime);
base.InvokeSync(executionContext);

span.SetAttribute(TelemetryConstants.RequestIdAttributeKey, executionContext.ResponseContext.Response.ResponseMetadata.RequestId);
}
catch(Exception ex)
{
span.SetExceptionAttributes(ex);
throw;
}
finally
{
executionContext.RequestContext.Metrics.StopEvent(Metric.ClientExecuteTime);
this.LogMetrics(executionContext);
span.Dispose();
}
}

Expand All @@ -59,16 +74,29 @@ public override void InvokeSync(IExecutionContext executionContext)
public override async System.Threading.Tasks.Task<T> InvokeAsync<T>(IExecutionContext executionContext)
{
executionContext.RequestContext.Metrics.AddProperty(Metric.AsyncCall, true);

var operationName = AWSSDKUtils.ExtractOperationName(executionContext.RequestContext.RequestName);
var spanName = $"{executionContext.RequestContext.ServiceMetaData.ServiceId}.{operationName}";
var span = TracingUtilities.CreateSpan(executionContext.RequestContext, spanName);

try
{
executionContext.RequestContext.Metrics.StartEvent(Metric.ClientExecuteTime);
var response = await base.InvokeAsync<T>(executionContext).ConfigureAwait(false);
return response;
var response = await base.InvokeAsync<T>(executionContext).ConfigureAwait(false);

span.SetAttribute(TelemetryConstants.RequestIdAttributeKey, executionContext.ResponseContext.Response.ResponseMetadata.RequestId);
return response;
}
catch (Exception ex)
{
span.SetExceptionAttributes(ex);
throw;
}
finally
{
executionContext.RequestContext.Metrics.StopEvent(Metric.ClientExecuteTime);
this.LogMetrics(executionContext);
span.Dispose();
}
}

Expand Down
82 changes: 52 additions & 30 deletions sdk/src/Core/Amazon.Runtime/Pipeline/HttpHandler/HttpHandler.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@
using Amazon.Runtime.Internal.Auth;
using Amazon.Runtime.Internal.Transform;
using Amazon.Runtime.Internal.Util;
using Amazon.Runtime.Telemetry;
using Amazon.Runtime.Telemetry.Tracing;
using Amazon.Util;
using Amazon.Util.Internal;
using System;
Expand Down Expand Up @@ -68,6 +70,7 @@ public override void InvokeSync(IExecutionContext executionContext)
httpRequest.SetRequestHeaders(wrappedRequest.Headers);

using (executionContext.RequestContext.Metrics.StartEvent(Metric.HttpRequestTime))
using (var traceSpan = TracingUtilities.CreateSpan(executionContext.RequestContext, TelemetryConstants.HTTPRequestSpanName))
{
// Send request body if present.
if (wrappedRequest.HasRequestBody())
Expand All @@ -77,14 +80,16 @@ public override void InvokeSync(IExecutionContext executionContext)
var requestContent = httpRequest.GetRequestContent();
WriteContentToRequestBody(requestContent, httpRequest, executionContext.RequestContext);
}
catch
catch(Exception ex)
{
CompleteFailedRequest(httpRequest);
traceSpan.SetExceptionAttributes(ex);
throw;
}
}

executionContext.ResponseContext.HttpResponse = httpRequest.GetResponse();
SetTraceSpanHttpAttributes(traceSpan, httpRequest.Method, wrappedRequest, executionContext.ResponseContext.HttpResponse);
}
}
finally
Expand Down Expand Up @@ -130,8 +135,22 @@ private static void CompleteFailedRequest(IHttpRequest<TRequestContent> httpRequ
}
catch { }
}

#if AWS_ASYNC_API

private static void SetTraceSpanHttpAttributes(TraceSpan traceSpan, string method, IRequest request, IWebResponseData response)
{
traceSpan.SetAttribute(TelemetryConstants.HTTPMethodAttributeKey, method);
traceSpan.SetAttribute(TelemetryConstants.HTTPStatusCodeAttributeKey, ((int)response.StatusCode));

var contentLengthHeader = request.GetHeaderValue(HeaderKeys.ContentLengthHeader);
if (!string.IsNullOrEmpty(contentLengthHeader) && long.TryParse(contentLengthHeader, out var contentLength))
{
traceSpan.SetAttribute(TelemetryConstants.HTTPRequestContentLengthAttributeKey, contentLength);
}

traceSpan.SetAttribute(TelemetryConstants.HTTPResponseContentLengthAttributeKey, response.ContentLength);
}

#if AWS_ASYNC_API

/// <summary>
/// Issues an HTTP request for the current request context.
Expand All @@ -149,42 +168,45 @@ public override async System.Threading.Tasks.Task<T> InvokeAsync<T>(IExecutionCo
IRequest wrappedRequest = executionContext.RequestContext.Request;
httpRequest = CreateWebRequest(executionContext.RequestContext);
httpRequest.SetRequestHeaders(wrappedRequest.Headers);

using(executionContext.RequestContext.Metrics.StartEvent(Metric.HttpRequestTime))
{
// Send request body if present.
if (wrappedRequest.HasRequestBody())

using (executionContext.RequestContext.Metrics.StartEvent(Metric.HttpRequestTime))
using (var traceSpan = TracingUtilities.CreateSpan(executionContext.RequestContext, TelemetryConstants.HTTPRequestSpanName))
{
System.Runtime.ExceptionServices.ExceptionDispatchInfo edi = null;
try
// Send request body if present.
if (wrappedRequest.HasRequestBody())
{
// In .NET Framework, there needs to be a cancellation token in this method since GetRequestStreamAsync
// does not accept a cancellation token. A workaround is used. This isn't necessary in .NET Standard
// where the stream is a property of the request.
System.Runtime.ExceptionServices.ExceptionDispatchInfo edi = null;
try
{
// In .NET Framework, there needs to be a cancellation token in this method since GetRequestStreamAsync
// does not accept a cancellation token. A workaround is used. This isn't necessary in .NET Standard
// where the stream is a property of the request.
#if BCL45
var requestContent = await httpRequest.GetRequestContentAsync(executionContext.RequestContext.CancellationToken).ConfigureAwait(false);
await WriteContentToRequestBodyAsync(requestContent, httpRequest, executionContext.RequestContext).ConfigureAwait(false);
#else
var requestContent = await httpRequest.GetRequestContentAsync().ConfigureAwait(false);
WriteContentToRequestBody(requestContent, httpRequest, executionContext.RequestContext);
var requestContent = await httpRequest.GetRequestContentAsync().ConfigureAwait(false);
WriteContentToRequestBody(requestContent, httpRequest, executionContext.RequestContext);
#endif
}
catch (Exception e)
{
traceSpan.SetExceptionAttributes(e);
edi = System.Runtime.ExceptionServices.ExceptionDispatchInfo.Capture(e);
}

if (edi != null)
{
await CompleteFailedRequest(executionContext, httpRequest).ConfigureAwait(false);

edi.Throw();
}
}
catch(Exception e)
{
edi = System.Runtime.ExceptionServices.ExceptionDispatchInfo.Capture(e);
}

if (edi != null)
{
await CompleteFailedRequest(executionContext, httpRequest).ConfigureAwait(false);

edi.Throw();
}
}

var response = await httpRequest.GetResponseAsync(executionContext.RequestContext.CancellationToken).
ConfigureAwait(false);
executionContext.ResponseContext.HttpResponse = response;
var response = await httpRequest.GetResponseAsync(executionContext.RequestContext.CancellationToken).
ConfigureAwait(false);
executionContext.ResponseContext.HttpResponse = response;
SetTraceSpanHttpAttributes(traceSpan, httpRequest.Method, wrappedRequest, executionContext.ResponseContext.HttpResponse);
}
// The response is not unmarshalled yet.
return null;
Expand Down
5 changes: 5 additions & 0 deletions sdk/src/Core/Amazon.Runtime/Telemetry/DefaultTelemetryProvider.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
* permissions and limitations under the License.
*/

using System;
using Amazon.Runtime.Telemetry.Metrics;
using Amazon.Runtime.Telemetry.Metrics.NoOp;
using Amazon.Runtime.Telemetry.Tracing;
Expand Down Expand Up @@ -56,6 +57,8 @@ public DefaultTelemetryProvider()
/// <param name="tracerProvider">The tracer provider to register.</param>
public override void RegisterTracerProvider(TracerProvider tracerProvider)
{
if (tracerProvider == null)
throw new ArgumentNullException(nameof(tracerProvider));
TracerProvider = tracerProvider;
}

Expand All @@ -67,6 +70,8 @@ public override void RegisterTracerProvider(TracerProvider tracerProvider)
/// <param name="meterProvider">The meter provider to register.</param>
public override void RegisterMeterProvider(MeterProvider meterProvider)
{
if (meterProvider == null)
throw new ArgumentNullException(nameof(meterProvider));
MeterProvider = meterProvider;
}
}
Expand Down
50 changes: 50 additions & 0 deletions sdk/src/Core/Amazon.Runtime/Telemetry/TelemetryConstants.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
/*
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License").
* You may not use this file except in compliance with the License.
* A copy of the License is located at
*
* http://aws.amazon.com/apache2.0
*
* or in the "license" file accompanying this file. This file is distributed
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing
* permissions and limitations under the License.
*/

namespace Amazon.Runtime.Telemetry
{
/// <summary>
/// Contains constants used for telemetry within the SDK.
/// </summary>
public static class TelemetryConstants
{
/// <summary>
/// The prefix used for telemetry scopes within the SDK.
/// </summary>
public const string TelemetryScopePrefix = "AWSSDK";

public const string HTTPRequestSpanName = "HttpRequest";
public const string RequestCompressionSpanName = "RequestCompression";
public const string CredentialsRetrievalSpanName = "CredentialsRetrieval";

public const string SystemAttributeKey = "rpc.system";
public const string SystemAttributeValue = "aws-api";
public const string MethodAttributeKey = "rpc.method";
public const string ServiceAttributeKey = "rpc.service";

public const string RequestIdAttributeKey = "aws.request_id";

public const string HTTPStatusCodeAttributeKey = "http.status_code";
public const string HTTPRequestContentLengthAttributeKey = "http.request_content_length";
public const string HTTPResponseContentLengthAttributeKey = "http.response_content_length";
public const string HTTPMethodAttributeKey = "http.method";

public const string ExceptionMessageAttributeKey = "exception.message";
public const string ExceptionStackTraceAttributeKey = "exception.stacktrace";
public const string ExceptionTypeAttributeKey = "exception.type";
public const string ErrorAttributeKey = "error";
public const string AWSErrorCodeAttributeKey = "aws.error_code";
}
}
Loading

0 comments on commit c614f0b

Please sign in to comment.