Stop health checks running until the underlying resource enters the r…

…unning state. (dotnet#5601)

playground/nats/Nats.AppHost/appsettings.json

@@ -0,0 +1,9 @@
+{
+  "Logging": {
+    "LogLevel": {
+      "Default": "Information",
+      "Microsoft.AspNetCore": "Warning",
+      "Aspire.Hosting.Dcp": "Warning"
+    }
+  }
+}

playground/waitfor/WaitForSandbox.AppHost/aspire-manifest.json

@@ -2,13 +2,22 @@
   "$schema": "https://json.schemastore.org/aspire-8.0.json",
   "resources": {
     "pg": {
-      "type": "azure.bicep.v0",
-      "connectionString": "{pg.secretOutputs.connectionString}",
-      "path": "pg.module.bicep",
-      "params": {
-        "keyVaultName": "",
-        "administratorLogin": "{pg-username.value}",
-        "administratorLoginPassword": "{pg-password.value}"
+      "type": "container.v0",
+      "connectionString": "Host={pg.bindings.tcp.host};Port={pg.bindings.tcp.port};Username=postgres;Password={pg-password.value}",
+      "image": "docker.io/library/postgres:16.4",
+      "env": {
+        "POSTGRES_HOST_AUTH_METHOD": "scram-sha-256",
+        "POSTGRES_INITDB_ARGS": "--auth-host=scram-sha-256 --auth-local=scram-sha-256",
+        "POSTGRES_USER": "postgres",
+        "POSTGRES_PASSWORD": "{pg-password.value}"
+      },
+      "bindings": {
+        "tcp": {
+          "scheme": "tcp",
+          "protocol": "tcp",
+          "transport": "tcp",
+          "targetPort": 5432
+        }
       }
     },
     "db": {
@@ -65,22 +74,6 @@
         }
       }
     },
-    "pg-username": {
-      "type": "parameter.v0",
-      "value": "{pg-username.inputs.value}",
-      "inputs": {
-        "value": {
-          "type": "string",
-          "default": {
-            "generate": {
-              "minLength": 10,
-              "numeric": false,
-              "special": false
-            }
-          }
-        }
-      }
-    },
     "pg-password": {
       "type": "parameter.v0",
       "value": "{pg-password.inputs.value}",

src/Aspire.Hosting/DistributedApplicationBuilder.cs

@@ -131,6 +131,10 @@ public DistributedApplicationBuilder(DistributedApplicationOptions options)
         _innerBuilder.Logging.AddFilter("Aspire.Hosting.Dashboard", LogLevel.Error);
         _innerBuilder.Logging.AddFilter("Grpc.AspNetCore.Server.ServerCallHandler", LogLevel.Error);
 
+        // This is to reduce log noise when we activate health checks for resources which may not yet be
+        // fully initialized. For example a database which is not yet created.
+        _innerBuilder.Logging.AddFilter("Microsoft.Extensions.Diagnostics.HealthChecks.DefaultHealthCheckService", LogLevel.None);
+
         // This is so that we can see certificate errors in the resource server in the console logs.
         // See: https://github.com/dotnet/aspire/issues/2914
         _innerBuilder.Logging.AddFilter("Microsoft.AspNetCore.Server.Kestrel.Core.KestrelServer", LogLevel.Warning);
@@ -162,11 +166,12 @@ public DistributedApplicationBuilder(DistributedApplicationOptions options)
         _innerBuilder.Services.AddHostedService<DistributedApplicationRunner>();
         _innerBuilder.Services.AddSingleton(options);
         _innerBuilder.Services.AddSingleton<ResourceNotificationService>();
-        _innerBuilder.Services.AddSingleton<IHealthCheckPublisher, ResourceNotificationHealthCheckPublisher>();
         _innerBuilder.Services.AddSingleton<ResourceLoggerService>();
         _innerBuilder.Services.AddSingleton<IDistributedApplicationEventing>(Eventing);
         _innerBuilder.Services.AddHealthChecks();
 
+        ConfigureHealthChecks();
+
         if (ExecutionContext.IsRunMode)
         {
             // Dashboard
@@ -251,6 +256,35 @@ public DistributedApplicationBuilder(DistributedApplicationOptions options)
         LogBuilderConstructed(this);
     }
 
+    private void ConfigureHealthChecks()
+    {
+        _innerBuilder.Services.AddSingleton<IConfigureOptions<HealthCheckPublisherOptions>>(sp =>
+        {
+            return new ConfigureOptions<HealthCheckPublisherOptions>(options =>
+            {
+                if (ExecutionContext.IsRunMode)
+                {
+                    // In run mode we route requests to the health check scheduler.
+                    var hcs = sp.GetRequiredService<ResourceHealthCheckScheduler>();
+                    options.Predicate = hcs.Predicate;
+                    options.Period = TimeSpan.FromSeconds(5);
+                }
+                else
+                {
+                    // In publish mode we don't run any checks.
+                    options.Predicate = (check) => false;
+                }
+            });
+        });
+
+        if (ExecutionContext.IsRunMode)
+        {
+            _innerBuilder.Services.AddSingleton<IHealthCheckPublisher, ResourceNotificationHealthCheckPublisher>();
+            _innerBuilder.Services.AddSingleton<ResourceHealthCheckScheduler>();
+            _innerBuilder.Services.AddHostedService<ResourceHealthCheckScheduler>(sp => sp.GetRequiredService<ResourceHealthCheckScheduler>());
+        }
+    }
+
     private void MapTransportOptionsFromCustomKeys(TransportOptions options)
     {
         if (Configuration.GetBool(KnownConfigNames.AllowUnsecuredTransport) is { } allowUnsecuredTransport)

src/Aspire.Hosting/Health/ResourceHealthCheckScheduler.cs

@@ -0,0 +1,59 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using Aspire.Hosting.ApplicationModel;
+using Microsoft.Extensions.Diagnostics.HealthChecks;
+using Microsoft.Extensions.Hosting;
+
+namespace Aspire.Hosting.Health;
+
+internal class ResourceHealthCheckScheduler : BackgroundService
+{
+    private readonly ResourceNotificationService _resourceNotificationService;
+    private readonly DistributedApplicationModel _model;
+    private readonly Dictionary<string, bool> _checkEnablement = new();
+
+    public ResourceHealthCheckScheduler(ResourceNotificationService resourceNotificationService, DistributedApplicationModel model)
+    {
+        _resourceNotificationService = resourceNotificationService ?? throw new ArgumentNullException(nameof(resourceNotificationService));
+        _model = model ?? throw new ArgumentNullException(nameof(model));
+
+        foreach (var resource in model.Resources)
+        {
+            UpdateCheckEnablement(resource, false);
+        }
+
+        Predicate = (check) =>
+        {
+            return _checkEnablement.TryGetValue(check.Name, out var enabled) ? enabled : false;
+        };
+    }
+
+    public Func<HealthCheckRegistration, bool> Predicate { get; init; }
+
+    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+    {
+        var resourceEvents = _resourceNotificationService.WatchAsync(stoppingToken);
+
+        await foreach (var resourceEvent in resourceEvents)
+        {
+            if (resourceEvent.Snapshot.State == KnownResourceStates.Running)
+            {
+                // Each time we receive an event that tells us that the resource is
+                // running we need to enable the health check annotation.
+                UpdateCheckEnablement(resourceEvent.Resource, true);
+            }
+        }
+    }
+
+    private void UpdateCheckEnablement(IResource resource, bool enabled)
+    {
+        if (resource.TryGetAnnotationsOfType<HealthCheckAnnotation>(out var annotations))
+        {
+            foreach (var annotation in annotations)
+            {
+                _checkEnablement[annotation.Key] = enabled;
+            }
+        }
+    }
+}

src/Aspire.Hosting/Health/ResourceNotificationHealthCheckPublisher.cs

@@ -14,8 +14,8 @@ public async Task PublishAsync(HealthReport report, CancellationToken cancellati
         {
             if (resource.TryGetAnnotationsOfType<HealthCheckAnnotation>(out var annotations))
             {
-                var resourceEntries = report.Entries.Where(e => annotations.Any(a => a.Key == e.Key));
-                var status = resourceEntries.All(e => e.Value.Status == HealthStatus.Healthy) ? HealthStatus.Healthy : HealthStatus.Unhealthy;
+                // Make sure every annotation is represented as health in the report, and if an entry is missing that means it is unhealthy.
+                var status = annotations.All(a => report.Entries.TryGetValue(a.Key, out var entry) && entry.Status == HealthStatus.Healthy) ? HealthStatus.Healthy : HealthStatus.Unhealthy;
 
                 await resourceNotificationService.PublishUpdateAsync(resource, s => s with
                 {