Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

OpenTelemetry Metrics #1456

Merged
merged 3 commits into from
Nov 3, 2023
Merged

OpenTelemetry Metrics #1456

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
038895f
Active requests
AndersAbel Oct 4, 2023
9fc199a
Metrics everywhere there's an event
AndersAbel Oct 24, 2023
719c7ca
Use constants
AndersAbel Nov 1, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 7 additions & 5 deletions Directory.Build.targets
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,12 +76,14 @@

<!-- open telemetry -->
<PackageReference Update="OpenTelemetry" Version="1.6.0" />
<PackageReference Update="OpenTelemetry.Exporter.Console" Version="1.3.0" />
<PackageReference Update="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.3.0" />
<PackageReference Update="OpenTelemetry.Extensions.Hosting" Version="1.0.0-rc9" />
<PackageReference Update="OpenTelemetry.Instrumentation.AspNetCore" Version="1.0.0-rc9" />
<PackageReference Update="OpenTelemetry.Instrumentation.Http" Version="1.0.0-rc9" />
<PackageReference Update="OpenTelemetry.Exporter.Console" Version="1.6.0" />
<PackageReference Update="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.6.0" />
<PackageReference Update="OpenTelemetry.Exporter.Prometheus.AspNetCore" Version="1.6.0-rc.1" />
<PackageReference Update="OpenTelemetry.Extensions.Hosting" Version="1.6.0" />
<PackageReference Update="OpenTelemetry.Instrumentation.AspNetCore" Version="1.5.1-beta.1" />
<PackageReference Update="OpenTelemetry.Instrumentation.Http" Version="1.5.1-beta.1" />
<PackageReference Update="OpenTelemetry.Instrumentation.SqlClient" Version="1.0.0-rc9" />

</ItemGroup>

<Target Name="SetAssemblyVersion" AfterTargets="MinVer">
Expand Down
1 change: 1 addition & 0 deletions hosts/main/Host.Main.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
<PackageReference Include="OpenTelemetry" />
<PackageReference Include="OpenTelemetry.Exporter.Console" />
<PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" />
<PackageReference Include="OpenTelemetry.Exporter.Prometheus.AspNetCore" />
<PackageReference Include="OpenTelemetry.Extensions.Hosting" />
<PackageReference Include="OpenTelemetry.Instrumentation.AspNetCore" />
<PackageReference Include="OpenTelemetry.Instrumentation.Http" />
Expand Down
39 changes: 12 additions & 27 deletions hosts/main/HostingExtensions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@
using IdentityServerHost.Extensions;
using Microsoft.AspNetCore.Mvc.RazorPages;
using Microsoft.IdentityModel.Tokens;
using OpenTelemetry.Metrics;
using OpenTelemetry.Resources;
using Serilog;
using Serilog.Events;

Expand Down Expand Up @@ -39,34 +41,14 @@ internal static WebApplication ConfigureServices(this WebApplicationBuilder buil
return Task.FromResult(principal);
});

var openTelemetry = builder.Services.AddOpenTelemetry();

// var apiKey = builder.Configuration["HoneyCombApiKey"];
// var dataset = "IdentityServerDev";
//
// builder.Services.AddOpenTelemetryTracing(builder =>
// {
// builder
// .AddSource(IdentityServerConstants.Tracing.Basic)
// .AddSource(IdentityServerConstants.Tracing.Cache)
// .AddSource(IdentityServerConstants.Tracing.Services)
// .AddSource(IdentityServerConstants.Tracing.Stores)
// .AddSource(IdentityServerConstants.Tracing.Validation)
//
// .SetResourceBuilder(
// ResourceBuilder.CreateDefault()
// .AddService("IdentityServerHost.Main"))
//
// //.SetSampler(new AlwaysOnSampler())
// .AddHttpClientInstrumentation()
// .AddAspNetCoreInstrumentation()
// .AddSqlClientInstrumentation()
// //.AddConsoleExporter()
// .AddOtlpExporter(option =>
// {
// option.Endpoint = new Uri("https://api.honeycomb.io");
// option.Headers = $"x-honeycomb-team={apiKey},x-honeycomb-dataset={dataset}";
// });
// });
openTelemetry.ConfigureResource(r => r
.AddService(builder.Environment.ApplicationName));

openTelemetry.WithMetrics(m => m
.AddMeter(Telemetry.ServiceName)
.AddPrometheusExporter());

return builder.Build();
}
Expand Down Expand Up @@ -169,6 +151,9 @@ internal static WebApplication ConfigurePipeline(this WebApplication app)
app.MapDynamicClientRegistration()
.AllowAnonymous();

// Map /metrics that displays Otel data in human readable form.
app.UseOpenTelemetryPrometheusScrapingEndpoint();

return app;
}
}
2 changes: 1 addition & 1 deletion src/EntityFramework.Storage/Duende.IdentityServer.EntityFramework.Storage.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
</ItemGroup>

<ItemGroup>
<Compile Include="..\Tracing\Tracing.cs">
<Compile Include="..\Telemetry\Tracing.cs">
<Link>Tracing.cs</Link>
</Compile>
</ItemGroup>
Expand Down
5 changes: 4 additions & 1 deletion src/IdentityServer/Duende.IdentityServer.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,12 @@
</ItemGroup>

<ItemGroup>
<Compile Include="..\Tracing\Tracing.cs">
<Compile Include="..\Telemetry\Tracing.cs">
<Link>Tracing.cs</Link>
</Compile>
<Compile Include="..\Telemetry\Telemetry.cs">
<Link>Telemetry.cs</Link>
</Compile>
</ItemGroup>

</Project>
4 changes: 3 additions & 1 deletion src/IdentityServer/Endpoints/AuthorizeEndpointBase.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ internal async Task<IEndpointResult> ProcessAuthorizeRequestAsync(NameValueColle

if (consent != null && consent.Data == null)
{
return await CreateErrorResultAsync("consent message is missing data");
return await CreateErrorResultAsync("consent message is missing data", result.ValidatedRequest);
}
}

Expand Down Expand Up @@ -225,6 +225,7 @@ private void LogTokens(AuthorizeResponse response)

private Task RaiseFailureEventAsync(ValidatedAuthorizeRequest request, string error, string errorDescription)
{
Telemetry.Metrics.TokenIssuedFailure(request.ClientId, request.GrantType, error);
return _events.RaiseAsync(new TokenIssuedFailureEvent(request, error, errorDescription));
}

Expand All @@ -233,6 +234,7 @@ private Task RaiseResponseEventAsync(AuthorizeResponse response)
if (!response.IsError)
{
LogTokens(response);
Telemetry.Metrics.TokenIssued(response.Request.ClientId, response.Request.GrantType);
return _events.RaiseAsync(new TokenIssuedSuccessEvent(response));
}

Expand Down
7 changes: 6 additions & 1 deletion src/IdentityServer/Endpoints/BackchannelAuthenticationEndpoint.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,10 @@ private async Task<IEndpointResult> ProcessAuthenticationRequestAsync(HttpContex
var clientResult = await _clientValidator.ValidateAsync(context);
if (clientResult.IsError)
{
return Error(clientResult.Error ?? OidcConstants.BackchannelAuthenticationRequestErrors.InvalidClient);
var error = clientResult.Error ?? OidcConstants.BackchannelAuthenticationRequestErrors.InvalidClient;
Telemetry.Metrics.BackChannelAuthenticationFailureCounter
.Add(1, new("client", clientResult.Client?.ClientId), new("error", error));
return Error(error);
}

// validate request
Expand All @@ -86,6 +89,7 @@ private async Task<IEndpointResult> ProcessAuthenticationRequestAsync(HttpContex
if (requestResult.IsError)
{
await _events.RaiseAsync(new BackchannelAuthenticationFailureEvent(requestResult));
Telemetry.Metrics.BackChannelAuthenticationFailure(clientResult.Client?.ClientId, requestResult.Error);
return Error(requestResult.Error, requestResult.ErrorDescription);
}

Expand All @@ -94,6 +98,7 @@ private async Task<IEndpointResult> ProcessAuthenticationRequestAsync(HttpContex
var response = await _responseGenerator.ProcessAsync(requestResult);

await _events.RaiseAsync(new BackchannelAuthenticationSuccessEvent(requestResult));
Telemetry.Metrics.BackChannelAuthentication(clientResult.Client.ClientId);
LogResponse(response, requestResult);

// return result
Expand Down
9 changes: 8 additions & 1 deletion src/IdentityServer/Endpoints/DeviceAuthorizationEndpoint.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,12 @@ private async Task<IEndpointResult> ProcessDeviceAuthorizationRequestAsync(HttpC

// validate client
var clientResult = await _clientValidator.ValidateAsync(context);
if (clientResult.IsError) return Error(clientResult.Error ?? OidcConstants.TokenErrors.InvalidClient);
if (clientResult.IsError)
{
var error = clientResult.Error ?? OidcConstants.TokenErrors.InvalidClient;
Telemetry.Metrics.DeviceAuthenticationFailure(clientResult.Client?.ClientId, error);
return Error(error);
}

// validate request
var form = (await context.Request.ReadFormAsync()).AsNameValueCollection();
Expand All @@ -92,6 +97,7 @@ private async Task<IEndpointResult> ProcessDeviceAuthorizationRequestAsync(HttpC
if (requestResult.IsError)
{
await _events.RaiseAsync(new DeviceAuthorizationFailureEvent(requestResult));
Telemetry.Metrics.DeviceAuthenticationFailure(clientResult.Client.ClientId, requestResult.Error);
return Error(requestResult.Error, requestResult.ErrorDescription);
}

Expand All @@ -100,6 +106,7 @@ private async Task<IEndpointResult> ProcessDeviceAuthorizationRequestAsync(HttpC
var response = await _responseGenerator.ProcessAsync(requestResult, _urls.BaseUrl);

await _events.RaiseAsync(new DeviceAuthorizationSuccessEvent(response, requestResult));
Telemetry.Metrics.DeviceAuthentication(clientResult.Client.ClientId);

// return result
_logger.LogDebug("Device authorize request success.");
Expand Down
7 changes: 4 additions & 3 deletions src/IdentityServer/Endpoints/IntrospectionEndpoint.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,8 +128,9 @@ private async Task<IEndpointResult> ProcessIntrospectionRequestAsync(HttpContext
if (body == null)
{
_logger.LogError("Malformed request body. aborting.");
await _events.RaiseAsync(new TokenIntrospectionFailureEvent(callerName, "Malformed request body"));

const string error = "Malformed request body";
await _events.RaiseAsync(new TokenIntrospectionFailureEvent(callerName, error));
Telemetry.Metrics.IntrospectionFailure(callerName, error);
return new StatusCodeResult(HttpStatusCode.BadRequest);
}

Expand All @@ -146,7 +147,7 @@ private async Task<IEndpointResult> ProcessIntrospectionRequestAsync(HttpContext
{
LogFailure(validationResult.Error, callerName);
await _events.RaiseAsync(new TokenIntrospectionFailureEvent(callerName, validationResult.Error));

Telemetry.Metrics.IntrospectionFailure(callerName, validationResult.Error);
return new BadRequestResult(validationResult.Error);
}

Expand Down
8 changes: 7 additions & 1 deletion src/IdentityServer/Endpoints/TokenEndpoint.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,9 @@ private async Task<IEndpointResult> ProcessTokenRequestAsync(HttpContext context
var clientResult = await _clientValidator.ValidateAsync(context);
if (clientResult.IsError)
{
return Error(clientResult.Error ?? OidcConstants.TokenErrors.InvalidClient);
var errorMsg = clientResult.Error ?? OidcConstants.TokenErrors.InvalidClient;
Telemetry.Metrics.TokenIssuedFailure(clientResult.Client?.ClientId, null, errorMsg);
return Error(errorMsg);
}

// validate request
Expand All @@ -111,13 +113,16 @@ private async Task<IEndpointResult> ProcessTokenRequestAsync(HttpContext context
var error = await TryReadProofTokens(context, requestContext);
if (error != null)
{
Telemetry.Metrics.TokenIssuedFailure(clientResult.Client.ClientId, null, error.Response.Error);
return error;
}

var requestResult = await _requestValidator.ValidateRequestAsync(requestContext);
if (requestResult.IsError)
{
await _events.RaiseAsync(new TokenIssuedFailureEvent(requestResult));
Telemetry.Metrics.TokenIssuedFailure(
clientResult.Client.ClientId, requestResult.ValidatedRequest?.GrantType, requestResult.Error);
var err = Error(requestResult.Error, requestResult.ErrorDescription, requestResult.CustomResponse);
err.Response.DPoPNonce = requestResult.DPoPNonce;
return err;
Expand All @@ -128,6 +133,7 @@ private async Task<IEndpointResult> ProcessTokenRequestAsync(HttpContext context
var response = await _responseGenerator.ProcessAsync(requestResult);

await _events.RaiseAsync(new TokenIssuedSuccessEvent(response, requestResult));
Telemetry.Metrics.TokenIssued(clientResult.Client.ClientId, requestResult.ValidatedRequest.GrantType);
LogTokens(response, requestResult);

// return result
Expand Down
6 changes: 5 additions & 1 deletion src/IdentityServer/Endpoints/TokenRevocationEndpoint.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,9 @@ private async Task<IEndpointResult> ProcessRevocationRequestAsync(HttpContext co
var clientValidationResult = await _clientValidator.ValidateAsync(context);
if (clientValidationResult.IsError)
{
return new TokenRevocationErrorResult(clientValidationResult.Error ?? OidcConstants.TokenErrors.InvalidClient);
var error = clientValidationResult.Error ?? OidcConstants.TokenErrors.InvalidClient;
Telemetry.Metrics.RevocationFailure(clientValidationResult.Client?.ClientId, error);
return new TokenRevocationErrorResult(error);
}

_logger.LogTrace("Client validation successful");
Expand All @@ -107,6 +109,7 @@ private async Task<IEndpointResult> ProcessRevocationRequestAsync(HttpContext co

if (requestValidationResult.IsError)
{
Telemetry.Metrics.RevocationFailure(clientValidationResult.Client.ClientId, requestValidationResult.Error);
return new TokenRevocationErrorResult(requestValidationResult.Error);
}

Expand All @@ -116,6 +119,7 @@ private async Task<IEndpointResult> ProcessRevocationRequestAsync(HttpContext co
if (response.Success)
{
_logger.LogInformation("Token revocation complete");
Telemetry.Metrics.Revocation(clientValidationResult.Client.ClientId);
await _events.RaiseAsync(new TokenRevokedSuccessEvent(requestValidationResult, requestValidationResult.Client));
}
else
Expand Down
4 changes: 4 additions & 0 deletions src/IdentityServer/Hosting/DynamicProviders/Store/ValidatingIdentityProviderStore.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,15 +60,19 @@ public async Task<IdentityProvider> GetBySchemeAsync(string scheme)
if (context.IsValid)
{
_logger.LogDebug("IdentityProvider validation for scheme {scheme} succeeded.", scheme);
Telemetry.Metrics.DynamicIdentityProviderValidation(scheme);
return idp;
}

_logger.LogError("Invalid IdentityProvider configuration for scheme {scheme}: {errorMessage}", scheme, context.ErrorMessage);
Telemetry.Metrics.DynamicIdentityProviderValidationFailure(scheme, context.ErrorMessage);
await _events.RaiseAsync(new InvalidIdentityProviderConfiguration(idp, context.ErrorMessage));

return null;
}

Telemetry.Metrics.DynamicIdentityProviderValidationFailure(scheme, "Scheme not found");

return null;
}
}
34 changes: 23 additions & 11 deletions src/IdentityServer/Hosting/IdentityServerMiddleware.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@
using Duende.IdentityServer.Models;
using System.Linq;
using Duende.IdentityServer.Configuration;
using Microsoft.AspNetCore.WebUtilities;
using System.Collections.Generic;

namespace Duende.IdentityServer.Hosting;

Expand Down Expand Up @@ -89,28 +91,38 @@ public async Task Invoke(
if (endpoint != null)
{
var endpointType = endpoint.GetType().FullName;

using var activity = Tracing.BasicActivitySource.StartActivity("IdentityServerProtocolRequest");
activity?.SetTag(Tracing.Properties.EndpointType, endpointType);
var requestPath = context.Request.Path.ToString();

IdentityServerLicenseValidator.Instance.ValidateIssuer(await issuerNameService.GetCurrentAsync());
Telemetry.Metrics.IncreaseActiveRequests(endpointType, requestPath);
try
{
using var activity = Tracing.BasicActivitySource.StartActivity("IdentityServerProtocolRequest");
activity?.SetTag(Tracing.Properties.EndpointType, endpointType);

IdentityServerLicenseValidator.Instance.ValidateIssuer(await issuerNameService.GetCurrentAsync());

_logger.LogInformation("Invoking IdentityServer endpoint: {endpointType} for {url}", endpointType, requestPath);

_logger.LogInformation("Invoking IdentityServer endpoint: {endpointType} for {url}", endpointType, context.Request.Path.ToString());
var result = await endpoint.ProcessAsync(context);

var result = await endpoint.ProcessAsync(context);
if (result != null)
{
_logger.LogTrace("Invoking result: {type}", result.GetType().FullName);
await result.ExecuteAsync(context);
}

if (result != null)
return;
}
finally
{
_logger.LogTrace("Invoking result: {type}", result.GetType().FullName);
await result.ExecuteAsync(context);
Telemetry.Metrics.DecreaseActiveRequests(endpointType, requestPath);
}

return;
}
}
catch (Exception ex) when (options.Logging.UnhandledExceptionLoggingFilter?.Invoke(context, ex) is not false)
{
await events.RaiseAsync(new UnhandledExceptionEvent(ex));
Telemetry.Metrics.UnHandledException(ex);
_logger.LogCritical(ex, "Unhandled exception: {exception}", ex.Message);

throw;
Expand Down
9 changes: 8 additions & 1 deletion src/IdentityServer/ResponseHandling/Default/IntrospectionResponseGenerator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,10 +61,13 @@ public virtual async Task<Dictionary<string, object>> ProcessAsync(Introspection
{ "active", false }
};

var callerName = validationResult.Api?.Name ?? validationResult.Client.ClientId;

// token is invalid
if (validationResult.IsActive == false)
{
Logger.LogDebug("Creating introspection response for inactive token.");
Telemetry.Metrics.Introspection(callerName, false);
await Events.RaiseAsync(new TokenIntrospectionSuccessEvent(validationResult));

return response;
Expand Down Expand Up @@ -124,7 +127,11 @@ protected virtual async Task<bool> AreExpectedScopesPresentAsync(IntrospectionRe
{
// no scopes for this API are found in the token
Logger.LogError("Expected scope {scopes} is missing in token", apiScopes);
await Events.RaiseAsync(new TokenIntrospectionFailureEvent(validationResult.Api.Name, "Expected scopes are missing", validationResult.Token, apiScopes, tokenScopes.Select(s => s.Value)));

const string errorMessage = "Expected scopes are missing";
var callerName = validationResult.Api?.Name ?? validationResult.Client.ClientId;
Telemetry.Metrics.IntrospectionFailure(callerName, errorMessage);
await Events.RaiseAsync(new TokenIntrospectionFailureEvent(validationResult.Api.Name, errorMessage, validationResult.Token, apiScopes, tokenScopes.Select(s => s.Value)));
}

return result;
Expand Down
Loading