Skip to content

Commit

Permalink
Stop health checks running until the underlying resource enters the r…
Browse files Browse the repository at this point in the history
…unning state. (dotnet#5601)
  • Loading branch information
mitchdenny authored and radical committed Sep 11, 2024
1 parent 24b51a1 commit f0ab1b9
Show file tree
Hide file tree
Showing 5 changed files with 121 additions and 26 deletions.
9 changes: 9 additions & 0 deletions playground/nats/Nats.AppHost/appsettings.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
{
"Logging": {
"LogLevel": {
"Default": "Information",
"Microsoft.AspNetCore": "Warning",
"Aspire.Hosting.Dcp": "Warning"
}
}
}
39 changes: 16 additions & 23 deletions playground/waitfor/WaitForSandbox.AppHost/aspire-manifest.json
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,22 @@
"$schema": "https://json.schemastore.org/aspire-8.0.json",
"resources": {
"pg": {
"type": "azure.bicep.v0",
"connectionString": "{pg.secretOutputs.connectionString}",
"path": "pg.module.bicep",
"params": {
"keyVaultName": "",
"administratorLogin": "{pg-username.value}",
"administratorLoginPassword": "{pg-password.value}"
"type": "container.v0",
"connectionString": "Host={pg.bindings.tcp.host};Port={pg.bindings.tcp.port};Username=postgres;Password={pg-password.value}",
"image": "docker.io/library/postgres:16.4",
"env": {
"POSTGRES_HOST_AUTH_METHOD": "scram-sha-256",
"POSTGRES_INITDB_ARGS": "--auth-host=scram-sha-256 --auth-local=scram-sha-256",
"POSTGRES_USER": "postgres",
"POSTGRES_PASSWORD": "{pg-password.value}"
},
"bindings": {
"tcp": {
"scheme": "tcp",
"protocol": "tcp",
"transport": "tcp",
"targetPort": 5432
}
}
},
"db": {
Expand Down Expand Up @@ -65,22 +74,6 @@
}
}
},
"pg-username": {
"type": "parameter.v0",
"value": "{pg-username.inputs.value}",
"inputs": {
"value": {
"type": "string",
"default": {
"generate": {
"minLength": 10,
"numeric": false,
"special": false
}
}
}
}
},
"pg-password": {
"type": "parameter.v0",
"value": "{pg-password.inputs.value}",
Expand Down
36 changes: 35 additions & 1 deletion src/Aspire.Hosting/DistributedApplicationBuilder.cs
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,10 @@ public DistributedApplicationBuilder(DistributedApplicationOptions options)
_innerBuilder.Logging.AddFilter("Aspire.Hosting.Dashboard", LogLevel.Error);
_innerBuilder.Logging.AddFilter("Grpc.AspNetCore.Server.ServerCallHandler", LogLevel.Error);

// This is to reduce log noise when we activate health checks for resources which may not yet be
// fully initialized. For example a database which is not yet created.
_innerBuilder.Logging.AddFilter("Microsoft.Extensions.Diagnostics.HealthChecks.DefaultHealthCheckService", LogLevel.None);

// This is so that we can see certificate errors in the resource server in the console logs.
// See: https://github.com/dotnet/aspire/issues/2914
_innerBuilder.Logging.AddFilter("Microsoft.AspNetCore.Server.Kestrel.Core.KestrelServer", LogLevel.Warning);
Expand Down Expand Up @@ -162,11 +166,12 @@ public DistributedApplicationBuilder(DistributedApplicationOptions options)
_innerBuilder.Services.AddHostedService<DistributedApplicationRunner>();
_innerBuilder.Services.AddSingleton(options);
_innerBuilder.Services.AddSingleton<ResourceNotificationService>();
_innerBuilder.Services.AddSingleton<IHealthCheckPublisher, ResourceNotificationHealthCheckPublisher>();
_innerBuilder.Services.AddSingleton<ResourceLoggerService>();
_innerBuilder.Services.AddSingleton<IDistributedApplicationEventing>(Eventing);
_innerBuilder.Services.AddHealthChecks();

ConfigureHealthChecks();

if (ExecutionContext.IsRunMode)
{
// Dashboard
Expand Down Expand Up @@ -251,6 +256,35 @@ public DistributedApplicationBuilder(DistributedApplicationOptions options)
LogBuilderConstructed(this);
}

private void ConfigureHealthChecks()
{
_innerBuilder.Services.AddSingleton<IConfigureOptions<HealthCheckPublisherOptions>>(sp =>
{
return new ConfigureOptions<HealthCheckPublisherOptions>(options =>
{
if (ExecutionContext.IsRunMode)
{
// In run mode we route requests to the health check scheduler.
var hcs = sp.GetRequiredService<ResourceHealthCheckScheduler>();
options.Predicate = hcs.Predicate;
options.Period = TimeSpan.FromSeconds(5);
}
else
{
// In publish mode we don't run any checks.
options.Predicate = (check) => false;
}
});
});

if (ExecutionContext.IsRunMode)
{
_innerBuilder.Services.AddSingleton<IHealthCheckPublisher, ResourceNotificationHealthCheckPublisher>();
_innerBuilder.Services.AddSingleton<ResourceHealthCheckScheduler>();
_innerBuilder.Services.AddHostedService<ResourceHealthCheckScheduler>(sp => sp.GetRequiredService<ResourceHealthCheckScheduler>());
}
}

private void MapTransportOptionsFromCustomKeys(TransportOptions options)
{
if (Configuration.GetBool(KnownConfigNames.AllowUnsecuredTransport) is { } allowUnsecuredTransport)
Expand Down
59 changes: 59 additions & 0 deletions src/Aspire.Hosting/Health/ResourceHealthCheckScheduler.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

using Aspire.Hosting.ApplicationModel;
using Microsoft.Extensions.Diagnostics.HealthChecks;
using Microsoft.Extensions.Hosting;

namespace Aspire.Hosting.Health;

internal class ResourceHealthCheckScheduler : BackgroundService
{
private readonly ResourceNotificationService _resourceNotificationService;
private readonly DistributedApplicationModel _model;
private readonly Dictionary<string, bool> _checkEnablement = new();

public ResourceHealthCheckScheduler(ResourceNotificationService resourceNotificationService, DistributedApplicationModel model)
{
_resourceNotificationService = resourceNotificationService ?? throw new ArgumentNullException(nameof(resourceNotificationService));
_model = model ?? throw new ArgumentNullException(nameof(model));

foreach (var resource in model.Resources)
{
UpdateCheckEnablement(resource, false);
}

Predicate = (check) =>
{
return _checkEnablement.TryGetValue(check.Name, out var enabled) ? enabled : false;
};
}

public Func<HealthCheckRegistration, bool> Predicate { get; init; }

protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
var resourceEvents = _resourceNotificationService.WatchAsync(stoppingToken);

await foreach (var resourceEvent in resourceEvents)
{
if (resourceEvent.Snapshot.State == KnownResourceStates.Running)
{
// Each time we receive an event that tells us that the resource is
// running we need to enable the health check annotation.
UpdateCheckEnablement(resourceEvent.Resource, true);
}
}
}

private void UpdateCheckEnablement(IResource resource, bool enabled)
{
if (resource.TryGetAnnotationsOfType<HealthCheckAnnotation>(out var annotations))
{
foreach (var annotation in annotations)
{
_checkEnablement[annotation.Key] = enabled;
}
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@ public async Task PublishAsync(HealthReport report, CancellationToken cancellati
{
if (resource.TryGetAnnotationsOfType<HealthCheckAnnotation>(out var annotations))
{
var resourceEntries = report.Entries.Where(e => annotations.Any(a => a.Key == e.Key));
var status = resourceEntries.All(e => e.Value.Status == HealthStatus.Healthy) ? HealthStatus.Healthy : HealthStatus.Unhealthy;
// Make sure every annotation is represented as health in the report, and if an entry is missing that means it is unhealthy.
var status = annotations.All(a => report.Entries.TryGetValue(a.Key, out var entry) && entry.Status == HealthStatus.Healthy) ? HealthStatus.Healthy : HealthStatus.Unhealthy;

await resourceNotificationService.PublishUpdateAsync(resource, s => s with
{
Expand Down

0 comments on commit f0ab1b9

Please sign in to comment.