Layer enrichment & redaction on top of HttpLogging

src/Libraries/Microsoft.AspNetCore.Telemetry.Middleware/Logging/HttpLoggingServiceExtensions.cs

@@ -4,6 +4,7 @@
 using System;
 using System.Diagnostics.CodeAnalysis;
 using Microsoft.AspNetCore.Builder;
+using Microsoft.AspNetCore.HttpLogging;
 using Microsoft.AspNetCore.Telemetry.Http.Logging;
 using Microsoft.AspNetCore.Telemetry.Internal;
 using Microsoft.Extensions.Configuration;
@@ -22,6 +23,17 @@ namespace Microsoft.AspNetCore.Telemetry;
 /// </summary>
 public static class HttpLoggingServiceExtensions
 {
+    public static IServiceCollection AddHttpLoggingRedaction(IServiceCollection services, Action<LoggingRedactionOptions> configureRedaction)
+    {
+        _ = Throw.IfNull(services);
+        _ = Throw.IfNull(configureRedaction);
+
+        _ = services.Configure(configureRedaction)
+            .AddSingleton<IHttpLoggingInterceptor, HttpRedactionHandler>();
+
+        return services;
+    }
+
     /// <summary>
     /// Adds components for incoming HTTP requests logging into <see cref="IServiceCollection"/>.
     /// </summary>

src/Libraries/Microsoft.AspNetCore.Telemetry.Middleware/Logging/Internal/HeaderReader.cs

@@ -5,6 +5,7 @@
 using System.Collections.Generic;
 using System.Linq;
 using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.HttpLogging;
 using Microsoft.Extensions.Compliance.Classification;
 using Microsoft.Extensions.Compliance.Redaction;
 
@@ -46,4 +47,22 @@ public void Read(IHeaderDictionary headers, List<KeyValuePair<string, string>> l
             }
         }
     }
+
+    public void Read(IHeaderDictionary headers, HttpLoggingContext logContext)
+    {
+        if (headers.Count == 0)
+        {
+            return;
+        }
+
+        foreach (var header in _headers)
+        {
+            if (headers.TryGetValue(header.Key, out var headerValue))
+            {
+                var provider = _redactorProvider.GetRedactor(header.Value);
+                var redacted = provider.Redact(headerValue.ToString());
+                logContext.Add(header.Key, redacted);
+            }
+        }
+    }
 }

src/Libraries/Microsoft.AspNetCore.Telemetry.Middleware/Logging/Internal/HttpRedactionHandler.cs

@@ -0,0 +1,203 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Buffers;
+using System.Collections.Frozen;
+using System.Collections.Generic;
+using System.Linq;
+using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.HttpLogging;
+using Microsoft.AspNetCore.Routing;
+using Microsoft.AspNetCore.Telemetry.Internal;
+using Microsoft.Extensions.Compliance.Classification;
+using Microsoft.Extensions.Compliance.Redaction;
+using Microsoft.Extensions.Http.Telemetry;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using Microsoft.Extensions.Telemetry.Internal;
+using Microsoft.Extensions.Telemetry.Logging;
+
+namespace Microsoft.AspNetCore.Telemetry.Http.Logging;
+
+internal sealed class HttpRedactionHandler : IHttpLoggingInterceptor
+{
+    // These three fields are "internal" solely for testing purposes:
+    internal TimeProvider TimeProvider = TimeProvider.System;
+
+    private readonly IncomingPathLoggingMode _requestPathLogMode;
+    private readonly HttpRouteParameterRedactionMode _parameterRedactionMode;
+    private readonly ILogger<HttpRedactionHandler> _logger;
+    private readonly IHttpRouteParser _httpRouteParser;
+    private readonly IHttpRouteFormatter _httpRouteFormatter;
+    private readonly IIncomingHttpRouteUtility _httpRouteUtility;
+    private readonly HeaderReader _requestHeadersReader;
+    private readonly HeaderReader _responseHeadersReader;
+    private readonly string[] _excludePathStartsWith;
+    private readonly IHttpLogEnricher[] _enrichers;
+    private readonly FrozenDictionary<string, DataClassification> _parametersToRedactMap;
+
+    public HttpRedactionHandler(
+        IOptions<LoggingRedactionOptions> options,
+        ILogger<HttpRedactionHandler> logger,
+        IEnumerable<IHttpLogEnricher> httpLogEnrichers,
+        IHttpRouteParser httpRouteParser,
+        IHttpRouteFormatter httpRouteFormatter,
+        IRedactorProvider redactorProvider,
+        IIncomingHttpRouteUtility httpRouteUtility)
+    {
+        var optionsValue = options.Value;
+        _logger = logger;
+        _httpRouteParser = httpRouteParser;
+        _httpRouteFormatter = httpRouteFormatter;
+        _httpRouteUtility = httpRouteUtility;
+
+        _parametersToRedactMap = optionsValue.RouteParameterDataClasses.ToFrozenDictionary(StringComparer.Ordinal, optimizeForReading: true);
+
+        _requestPathLogMode = EnsureRequestPathLoggingModeIsValid(optionsValue.RequestPathLoggingMode);
+        _parameterRedactionMode = optionsValue.RequestPathParameterRedactionMode;
+
+        _requestHeadersReader = new(optionsValue.RequestHeadersDataClasses, redactorProvider);
+        _responseHeadersReader = new(optionsValue.ResponseHeadersDataClasses, redactorProvider);
+
+        _excludePathStartsWith = optionsValue.ExcludePathStartsWith.ToArray();
+
+        _enrichers = httpLogEnrichers.ToArray();
+    }
+
+    public void OnRequest(HttpLoggingContext logContext)
+    {
+        var context = logContext.HttpContext;
+        var request = context.Request;
+        if (ShouldExcludePath(context.Request.Path))
+        {
+            logContext.LoggingFields = HttpLoggingFields.None;
+        }
+
+        // Don't enrich if we're not going to log any part of the request
+        if ((HttpLoggingFields.Request & logContext.LoggingFields) == HttpLoggingFields.None)
+        {
+            return;
+        }
+
+        // TODO: Should we put a state filed on logContext?
+        context.Items["RequestStartTimestamp"] = TimeProvider.GetTimestamp();
+
+        if (logContext.LoggingFields.HasFlag(HttpLoggingFields.RequestPath))
+        {
+            string path = TelemetryConstants.Unknown;
+
+            if (_parameterRedactionMode != HttpRouteParameterRedactionMode.None)
+            {
+                var endpoint = context.GetEndpoint() as RouteEndpoint;
+
+                if (endpoint?.RoutePattern.RawText != null)
+                {
+                    var httpRoute = endpoint.RoutePattern.RawText;
+                    var paramsToRedact = _httpRouteUtility.GetSensitiveParameters(httpRoute, request, _parametersToRedactMap);
+
+                    var routeSegments = _httpRouteParser.ParseRoute(httpRoute);
+
+                    if (_requestPathLogMode == IncomingPathLoggingMode.Formatted)
+                    {
+                        path = _httpRouteFormatter.Format(in routeSegments, request.Path, _parameterRedactionMode, paramsToRedact);
+                    }
+                    else
+                    {
+                        // Case when logging mode is IncomingPathLoggingMode.Structured
+                        path = httpRoute;
+                        var routeParams = ArrayPool<HttpRouteParameter>.Shared.Rent(routeSegments.ParameterCount);
+
+                        // Setting this value right away to be able to return it back to pool in a callee's "finally" block:
+                        if (_httpRouteParser.TryExtractParameters(request.Path, in routeSegments, _parameterRedactionMode, paramsToRedact, ref routeParams))
+                        {
+                            foreach (var param in routeParams)
+                            {
+                                logContext.Add(param.Name, param.Value);
+                            }
+                        }
+                    }
+                }
+            }
+            else if (request.Path.HasValue)
+            {
+                path = request.Path.Value!;
+            }
+
+            logContext.Add("path", path);
+
+            // We've handled the path, turn off the default logging
+            logContext.LoggingFields &= ~HttpLoggingFields.RequestPath;
+        }
+
+        if (logContext.LoggingFields.HasFlag(HttpLoggingFields.RequestHeaders))
+        {
+            // TODO: HttpLoggingOptions.Request/ResponseHeaders are ignored which could be confusing.
+            // Do we try to reconcile that with LoggingRedactionOptions.RequestHeadersDataClasses?
+            _requestHeadersReader.Read(context.Request.Headers, logContext);
+
+            // We've handled the request headers, turn off the default logging
+            logContext.LoggingFields &= ~HttpLoggingFields.RequestHeaders;
+        }
+    }
+
+    public void OnResponse(HttpLoggingContext logContext)
+    {
+        // Don't enrich if we're not going to log any part of the response
+        if ((HttpLoggingFields.Response & logContext.LoggingFields) == HttpLoggingFields.None)
+        {
+            return;
+        }
+
+        var context = logContext.HttpContext;
+
+        if (logContext.LoggingFields.HasFlag(HttpLoggingFields.ResponseHeaders))
+        {
+            _responseHeadersReader.Read(context.Response.Headers, logContext);
+
+            // We've handled the response headers, turn off the default logging
+            logContext.LoggingFields &= ~HttpLoggingFields.ResponseHeaders;
+        }
+
+        if (_enrichers.Length == 0)
+        {
+            var enrichmentBag =  LogMethodHelper.GetHelper();
+            foreach (var enricher in _enrichers)
+            {
+                enricher.Enrich(enrichmentBag, context.Request, context.Response);
+            }
+
+            foreach (var (key, value) in enrichmentBag)
+            {
+                logContext.Add(key, value);
+            }
+        }
+
+        // Catching duration at the end:
+        var startTime = (long)context.Items["RequestStartTimestamp"]!;
+        var duration = (long)TimeProvider.GetElapsedTime(startTime, TimeProvider.GetTimestamp()).TotalMilliseconds;
+        logContext.Add("duration", duration);
+
+        // TODO: What about the exception case?
+    }
+
+    private static IncomingPathLoggingMode EnsureRequestPathLoggingModeIsValid(IncomingPathLoggingMode mode)
+        => mode switch
+        {
+            IncomingPathLoggingMode.Structured or IncomingPathLoggingMode.Formatted => mode,
+            _ => throw new InvalidOperationException($"Unsupported value '{mode}' for enum type '{nameof(IncomingPathLoggingMode)}'"),
+        };
+
+    private bool ShouldExcludePath(string path)
+    {
+        foreach (var excludedPath in _excludePathStartsWith)
+        {
+            if (path.StartsWith(excludedPath, StringComparison.OrdinalIgnoreCase))
+            {
+                return true;
+            }
+        }
+
+        return false;
+    }
+}

src/Libraries/Microsoft.AspNetCore.Telemetry.Middleware/Logging/LoggingRedactionOptions.cs

@@ -0,0 +1,100 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Collections.Generic;
+using System.ComponentModel.DataAnnotations;
+using System.Diagnostics.CodeAnalysis;
+using Microsoft.Extensions.Compliance.Classification;
+using Microsoft.Extensions.Http.Telemetry;
+
+namespace Microsoft.AspNetCore.Telemetry;
+
+/// <summary>
+/// Top-level model for redacting incoming HTTP requests and their corresponding responses.
+/// </summary>
+public class LoggingRedactionOptions
+{
+    private const IncomingPathLoggingMode DefaultRequestPathLoggingMode = IncomingPathLoggingMode.Formatted;
+    private const HttpRouteParameterRedactionMode DefaultPathParameterRedactionMode = HttpRouteParameterRedactionMode.Strict;
+
+    /// <summary>
+    /// Gets or sets a strategy how request path should be logged.
+    /// </summary>
+    /// <remarks>
+    /// Make sure you add redactors to ensure that sensitive information doesn't find its way into your log records.
+    /// Default set to <see cref="IncomingPathLoggingMode.Formatted"/>.
+    /// This option only applies when the <see cref="RequestPathParameterRedactionMode"/>
+    /// option is not set to <see cref="HttpRouteParameterRedactionMode.None"/>.
+    /// </remarks>
+    public IncomingPathLoggingMode RequestPathLoggingMode { get; set; } = DefaultRequestPathLoggingMode;
+
+    /// <summary>
+    /// Gets or sets a value indicating how request path parameter should be redacted.
+    /// </summary>
+    /// <remarks>
+    /// Default set to <see cref="HttpRouteParameterRedactionMode.Strict"/>.
+    /// </remarks>
+    [Experimental]
+    public HttpRouteParameterRedactionMode RequestPathParameterRedactionMode { get; set; } = DefaultPathParameterRedactionMode;
+
+    /// <summary>
+    /// Gets or sets a map between HTTP path parameters and their data classification.
+    /// </summary>
+    /// <remarks>
+    /// Default set to an empty dictionary.
+    /// If a parameter within a controller's action is not annotated with a data classification attribute and
+    /// it's not found in this map, it will be redacted as if it was <see cref="DataClassification.Unknown"/>.
+    /// If you don't want a parameter to be redacted, mark it as <see cref="DataClassification.None"/>.
+    /// </remarks>
+    [Required]
+    [SuppressMessage("Usage", "CA2227:Collection properties should be read only", Justification = "Options pattern.")]
+    public IDictionary<string, DataClassification> RouteParameterDataClasses { get; set; } = new Dictionary<string, DataClassification>();
+
+    /// <summary>
+    /// Gets or sets a map between request headers to be logged and their data classification.
+    /// </summary>
+    /// <remarks>
+    /// Default set to an empty dictionary.
+    /// That means that no request header will be logged by default.
+    /// </remarks>
+    [Required]
+    [SuppressMessage("Usage", "CA2227:Collection properties should be read only",
+        Justification = "Options pattern.")]
+    public IDictionary<string, DataClassification> RequestHeadersDataClasses { get; set; } = new Dictionary<string, DataClassification>();
+
+    /// <summary>
+    /// Gets or sets a map between response headers to be logged and their data classification.
+    /// </summary>
+    /// <remarks>
+    /// Default set to an empty dictionary.
+    /// That means that no response header will be logged by default.
+    /// </remarks>
+    [Required]
+    [SuppressMessage("Usage", "CA2227:Collection properties should be read only",
+        Justification = "Options pattern.")]
+    public IDictionary<string, DataClassification> ResponseHeadersDataClasses { get; set; } = new Dictionary<string, DataClassification>();
+
+    /// <summary>
+    /// Gets or sets the set of HTTP paths that should be excluded from logging.
+    /// </summary>
+    /// <remarks>
+    /// Any path added to the set will not be logged.
+    /// Paths are case insensitive.
+    /// Default set to an empty <see cref="HashSet{T}"/>.
+    /// </remarks>
+    /// <example>
+    /// A typical set of HTTP paths would be:
+    /// <code>
+    /// ExcludePathStartsWith = new HashSet&lt;string&gt;
+    /// {
+    ///     "/probe/live",
+    ///     "/probe/ready"
+    /// };
+    /// </code>
+    /// </example>
+    [Experimental]
+    [Required]
+    [SuppressMessage("Usage", "CA2227:Collection properties should be read only",
+        Justification = "Options pattern.")]
+    public ISet<string> ExcludePathStartsWith { get; set; } = new HashSet<string>();
+}

src/Libraries/Microsoft.AspNetCore.Telemetry.Middleware/Microsoft.AspNetCore.Telemetry.Middleware.csproj

@@ -7,7 +7,7 @@
   </PropertyGroup>
 
   <PropertyGroup>
-    <TargetFrameworks>$(NetCoreTargetFrameworks)</TargetFrameworks>
+    <TargetFrameworks>net8.0</TargetFrameworks>
     <UseOptionsValidationGenerator>true</UseOptionsValidationGenerator>
     <UseLoggingGenerator>true</UseLoggingGenerator>
     <UseMeteringGenerator>true</UseMeteringGenerator>

test/Libraries/Microsoft.AspNetCore.Telemetry.Middleware.Tests/Microsoft.AspNetCore.Telemetry.Middleware.Tests.csproj

@@ -5,7 +5,7 @@
   </PropertyGroup>
 
   <PropertyGroup>
-    <TargetFrameworks>$(NetCoreTargetFrameworks)</TargetFrameworks>
+    <TargetFrameworks>net8.0</TargetFrameworks>
     <InjectSharedEmptyCollections>true</InjectSharedEmptyCollections>
   </PropertyGroup>