Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Stop health checks running until the underlying resource enters the running state. #5601

Merged
merged 8 commits into from
Sep 10, 2024
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
"Default": "Information",
"Microsoft.AspNetCore": "Warning",
"Aspire.Hosting.Dcp": "Warning"

mitchdenny marked this conversation as resolved.
Show resolved Hide resolved
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
"Default": "Information",
"Microsoft.AspNetCore": "Warning",
"Aspire.Hosting.Dcp": "Warning"

mitchdenny marked this conversation as resolved.
Show resolved Hide resolved
}
}
}
1 change: 1 addition & 0 deletions playground/TestShop/TestShop.AppHost/appsettings.json
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
"Default": "Information",
"Microsoft.AspNetCore": "Warning",
"Aspire.Hosting.Dcp": "Warning"

mitchdenny marked this conversation as resolved.
Show resolved Hide resolved
}
}
}
9 changes: 9 additions & 0 deletions playground/nats/Nats.AppHost/appsettings.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
{
"Logging": {
"LogLevel": {
"Default": "Information",
"Microsoft.AspNetCore": "Warning",
"Aspire.Hosting.Dcp": "Warning"
}
}
}
1 change: 1 addition & 0 deletions playground/python/Python.AppHost/appsettings.json
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
"Default": "Information",
"Microsoft.AspNetCore": "Warning",
"Aspire.Hosting.Dcp": "Warning"

mitchdenny marked this conversation as resolved.
Show resolved Hide resolved
}
}
}
16 changes: 14 additions & 2 deletions src/Aspire.Hosting/DistributedApplicationBuilder.cs
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,10 @@ public DistributedApplicationBuilder(DistributedApplicationOptions options)
_innerBuilder.Logging.AddFilter("Aspire.Hosting.Dashboard", LogLevel.Error);
_innerBuilder.Logging.AddFilter("Grpc.AspNetCore.Server.ServerCallHandler", LogLevel.Error);

// This is to reduce log noise when we activate health checks for resources which may not yet be
// fully initialized. For example a database which is not yet created.
_innerBuilder.Logging.AddFilter("Microsoft.Extensions.Diagnostics.HealthChecks.DefaultHealthCheckService", LogLevel.None);

// This is so that we can see certificate errors in the resource server in the console logs.
// See: https://github.com/dotnet/aspire/issues/2914
_innerBuilder.Logging.AddFilter("Microsoft.AspNetCore.Server.Kestrel.Core.KestrelServer", LogLevel.Warning);
Expand Down Expand Up @@ -162,11 +166,19 @@ public DistributedApplicationBuilder(DistributedApplicationOptions options)
_innerBuilder.Services.AddHostedService<DistributedApplicationRunner>();
_innerBuilder.Services.AddSingleton(options);
_innerBuilder.Services.AddSingleton<ResourceNotificationService>();
_innerBuilder.Services.AddSingleton<IHealthCheckPublisher, ResourceNotificationHealthCheckPublisher>();
_innerBuilder.Services.AddSingleton<ResourceLoggerService>();
_innerBuilder.Services.AddSingleton<IDistributedApplicationEventing>(Eventing);
_innerBuilder.Services.AddHealthChecks();

// Healthchecks
_innerBuilder.Services.AddSingleton<IHealthCheckPublisher, ResourceNotificationHealthCheckPublisher>();
_innerBuilder.Services.AddHostedService<ResourceHealthCheckScheduler>();
_innerBuilder.Services.AddSingleton<ResourceLoggerService>();
mitchdenny marked this conversation as resolved.
Show resolved Hide resolved
_innerBuilder.Services.Configure<HealthCheckPublisherOptions>(options =>
{
// Disable health checks from running!
options.Predicate = (check) => false;
});

if (ExecutionContext.IsRunMode)
{
// Dashboard
Expand Down
67 changes: 67 additions & 0 deletions src/Aspire.Hosting/Health/ResourceHealthCheckScheduler.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

using Aspire.Hosting.ApplicationModel;
using Microsoft.Extensions.Diagnostics.HealthChecks;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Options;

namespace Aspire.Hosting.Health;

internal class ResourceHealthCheckScheduler(IOptions<HealthCheckPublisherOptions> healthCheckPublisherOptions, ResourceNotificationService resourceNotificationService, DistributedApplicationModel model) : BackgroundService, IHostedLifecycleService
{
public Task StartedAsync(CancellationToken cancellationToken) => Task.CompletedTask;

public Task StartingAsync(CancellationToken cancellationToken)
{
// When we startup pre-populate the list of checks and set them
// all to false so we don't run any initially.
foreach (var resource in model.Resources)
{
UpdateCheckEnablement(resource, false);
}

healthCheckPublisherOptions.Value.Period = TimeSpan.FromSeconds(5);
healthCheckPublisherOptions.Value.Predicate = ShouldRunCheck;

return Task.CompletedTask;
}

private readonly Dictionary<string, bool> _checkEnablement = new();

private bool ShouldRunCheck(HealthCheckRegistration check)
{
// We don't run any health checks that aren't associated with a resource.
return _checkEnablement.TryGetValue(check.Name, out var enabled) ? enabled : false;
}

public Task StoppedAsync(CancellationToken cancellationToken) => Task.CompletedTask;

public Task StoppingAsync(CancellationToken cancellationToken) => Task.CompletedTask;

protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
var resourceEvents = resourceNotificationService.WatchAsync(stoppingToken);

await foreach (var resourceEvent in resourceEvents)
{
if (resourceEvent.Snapshot.State == KnownResourceStates.Running)
{
// Each time we receive an event that tells us that the resource is
// running we need to enable the health check annotation.
UpdateCheckEnablement(resourceEvent.Resource, true);
}
}
}

private void UpdateCheckEnablement(IResource resource, bool enabled)
{
if (resource.TryGetAnnotationsOfType<HealthCheckAnnotation>(out var annotations))
{
foreach (var annotation in annotations)
{
_checkEnablement[annotation.Key] = enabled;
}
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@ public async Task PublishAsync(HealthReport report, CancellationToken cancellati
{
if (resource.TryGetAnnotationsOfType<HealthCheckAnnotation>(out var annotations))
{
var resourceEntries = report.Entries.Where(e => annotations.Any(a => a.Key == e.Key));
var status = resourceEntries.All(e => e.Value.Status == HealthStatus.Healthy) ? HealthStatus.Healthy : HealthStatus.Unhealthy;
// Make sure every annotation is represented as health in the report, and if an entry is missing that means it is unhealthy.
var status = annotations.All(a => report.Entries.TryGetValue(a.Key, out var entry) && entry.Status == HealthStatus.Healthy) ? HealthStatus.Healthy : HealthStatus.Unhealthy;

await resourceNotificationService.PublishUpdateAsync(resource, s => s with
{
Expand Down
Loading