Fix parallel healthcheck (#1052)

sofastack · Jan 9, 2023 · b52393c · b52393c
1 parent b28980c
commit b52393c
Show file tree

Hide file tree

Showing 8 changed files with 120 additions and 22 deletions.
diff --git a/pom.xml b/pom.xml
@@ -37,7 +37,7 @@
     <description>SOFABoot Build</description>
 
     <properties>
-        <revision>3.16.2</revision>
+        <revision>3.16.3</revision>
         <sofa.boot.version>${revision}</sofa.boot.version>
         <!--maven plugin-->
         <maven.staging.plugin>1.6.7</maven.staging.plugin>

diff --git a/...althcheck-sofa-boot/src/main/java/com/alipay/sofa/healthcheck/HealthCheckerProcessor.java b/...althcheck-sofa-boot/src/main/java/com/alipay/sofa/healthcheck/HealthCheckerProcessor.java
@@ -176,6 +176,9 @@ public boolean readinessHealthCheck(Map<String, Health> healthMap) {
             } catch (InterruptedException e) {
                 logger.error(ErrorCode.convert("01-22005"), e);
             }
+            if (!finished) {
+                healthMap.put(SofaBootConstants.SOFABOOT_HEALTH_CHECK_TIMEOUT_KEY, Health.unknown().withDetail(SofaBootConstants.SOFABOOT_HEALTH_CHECK_TIMEOUT_KEY,SofaBootConstants.SOFABOOT_HEALTH_CHECK_TIMEOUT_MSG).build());
+            }
             result = finished && parallelResult.get();
         } else {
             result = readinessHealthCheckers.entrySet().stream()

diff --git a/...thcheck-sofa-boot/src/main/java/com/alipay/sofa/healthcheck/HealthIndicatorProcessor.java b/...thcheck-sofa-boot/src/main/java/com/alipay/sofa/healthcheck/HealthIndicatorProcessor.java
@@ -181,6 +181,9 @@ public boolean readinessHealthCheck(Map<String, Health> healthMap) {
             } catch (InterruptedException e) {
                 logger.error(ErrorCode.convert("01-21004"), e);
             }
+            if (!finished) {
+                healthMap.put(SofaBootConstants.SOFABOOT_HEALTH_CHECK_TIMEOUT_KEY, Health.unknown().withDetail(SofaBootConstants.SOFABOOT_HEALTH_CHECK_TIMEOUT_KEY,SofaBootConstants.SOFABOOT_HEALTH_CHECK_TIMEOUT_MSG).build());
+            }
             result = finished && parallelResult.get();
         } else {
             result = healthIndicators.entrySet().stream()

diff --git a/...althcheck-sofa-boot/src/main/java/com/alipay/sofa/healthcheck/ReadinessCheckListener.java b/...althcheck-sofa-boot/src/main/java/com/alipay/sofa/healthcheck/ReadinessCheckListener.java
@@ -238,29 +238,30 @@ public Health aggregateReadinessHealth() {
                         SofaBootConstants.SOFABOOT_HEALTH_CHECK_NOT_READY_MSG).build());
         } else {
             boolean healthCheckerStatus = getHealthCheckerStatus();
+            boolean healthIndicatorStatus = getHealthIndicatorStatus();
+            boolean afterReadinessCheckCallbackStatus = getHealthCallbackStatus();
             Map<String, Health> healthCheckerDetails = getHealthCheckerDetails();
             Map<String, Health> healthIndicatorDetails = getHealthIndicatorDetails();
-
-            boolean afterReadinessCheckCallbackStatus = getHealthCallbackStatus();
             Map<String, Health> afterReadinessCheckCallbackDetails = getHealthCallbackDetails();
 
             Health.Builder builder;
-            if (healthCheckerStatus && afterReadinessCheckCallbackStatus) {
-                builder = Health.up();
-            } else {
-                builder = Health.down();
-            }
+            builder = healthCheckerStatus ? Health.up():Health.down();
             if (!CollectionUtils.isEmpty(healthCheckerDetails)) {
-                builder = builder.withDetail("HealthChecker", healthCheckerDetails);
+                builder = builder.withDetails(healthCheckerDetails);
             }
-            if (!CollectionUtils.isEmpty(afterReadinessCheckCallbackDetails)) {
-                builder = builder.withDetail("ReadinessCheckCallback",
-                    afterReadinessCheckCallbackDetails);
+            healths.put("HealthCheckerInfo", builder.build());
+
+            builder = healthIndicatorStatus ? Health.up():Health.down();
+            if (!CollectionUtils.isEmpty(healthIndicatorDetails)) {
+                builder = builder.withDetails(healthIndicatorDetails);
             }
-            healths.put("SOFABootReadinessHealthCheckInfo", builder.build());
+            healths.put("HealthIndicatorInfo", builder.build());
 
-            // HealthIndicator
-            healths.putAll(healthIndicatorDetails);
+            builder = afterReadinessCheckCallbackStatus ? Health.up():Health.down();
+            if (!CollectionUtils.isEmpty(afterReadinessCheckCallbackDetails)) {
+                builder = builder.withDetails(afterReadinessCheckCallbackDetails);
+            }
+            healths.put("ReadinessCheckCallbackInfo", builder.build());
         }
         Status overallStatus = this.statusAggregator.getAggregateStatus(
                 healths.values().stream().map(Health::getStatus).collect(Collectors.toSet()));

diff --git a/...ot/src/test/java/com/alipay/sofa/healthcheck/test/HealthCheckerProcessorParallelTest.java b/...ot/src/test/java/com/alipay/sofa/healthcheck/test/HealthCheckerProcessorParallelTest.java
@@ -223,6 +223,21 @@ public void testComponentHealthCheckerFailedFirst() {
         }
     }
 
+    @Test
+    public void testHealthCheckerParallelTimeout() {
+        initApplicationContextTimeout(0, false, 4);
+
+        HashMap<String, Health> hashMap = new HashMap<>();
+        HealthCheckerProcessor healthCheckerProcessor = applicationContext
+            .getBean(HealthCheckerProcessor.class);
+        boolean result = healthCheckerProcessor.readinessHealthCheck(hashMap);
+        Assert.assertFalse(result);
+        Health timeoutHealth = hashMap.get(SofaBootConstants.SOFABOOT_HEALTH_CHECK_TIMEOUT_KEY);
+        Assert.assertEquals(Status.UNKNOWN, timeoutHealth.getStatus());
+        Assert.assertEquals(SofaBootConstants.SOFABOOT_HEALTH_CHECK_TIMEOUT_MSG, timeoutHealth
+            .getDetails().get(SofaBootConstants.SOFABOOT_HEALTH_CHECK_TIMEOUT_KEY));
+    }
+
     private void initApplicationContext(int count, boolean strict, int retryCount) {
         Map<String, Object> properties = new LinkedHashMap<>();
         properties.put("memory-health-checker.count", count);
@@ -237,4 +252,20 @@ private void initApplicationContext(int count, boolean strict, int retryCount) {
         springApplication.setWebApplicationType(WebApplicationType.NONE);
         applicationContext = springApplication.run();
     }
+
+    private void initApplicationContextTimeout(int count, boolean strict, int retryCount) {
+        Map<String, Object> properties = new LinkedHashMap<>();
+        properties.put("memory-health-checker.count", count);
+        properties.put("memory-health-checker.strict", strict);
+        properties.put("memory-health-checker.retry-count", retryCount);
+        properties.put("spring.application.name", "HealthCheckerProcessorTest");
+        properties.put(SofaBootConstants.SOFABOOT_SKIP_COMPONENT_HEALTH_CHECK, true);
+        properties.put("com.alipay.sofa.boot.health-check-parallel-timeout", "1");
+
+        SpringApplication springApplication = new SpringApplication(
+            HealthCheckerProcessorTestConfiguration.class);
+        springApplication.setDefaultProperties(properties);
+        springApplication.setWebApplicationType(WebApplicationType.NONE);
+        applicationContext = springApplication.run();
+    }
 }
diff --git a/...test/java/com/alipay/sofa/healthcheck/test/HealthIndicatorCheckProcessorParallelTest.java b/...test/java/com/alipay/sofa/healthcheck/test/HealthIndicatorCheckProcessorParallelTest.java
@@ -16,6 +16,7 @@
  */
 package com.alipay.sofa.healthcheck.test;
 
+import com.alipay.sofa.boot.constant.SofaBootConstants;
 import com.alipay.sofa.healthcheck.AfterReadinessCheckCallbackProcessor;
 import com.alipay.sofa.healthcheck.HealthCheckProperties;
 import com.alipay.sofa.healthcheck.HealthCheckerProcessor;
@@ -30,6 +31,7 @@
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.WebApplicationType;
 import org.springframework.boot.actuate.health.Health;
+import org.springframework.boot.actuate.health.Status;
 import org.springframework.boot.context.properties.EnableConfigurationProperties;
 import org.springframework.context.ApplicationContext;
 import org.springframework.context.annotation.Bean;
@@ -121,10 +123,39 @@ public void testCheckIndicatorFailed() {
         Assert.assertEquals(2, hashMap.size());
     }
 
+    @Test
+    public void testCheckIndicatorParallelTimeout() {
+        initApplicationContextTimeout();
+        ReadinessCheckListener readinessCheckListener = applicationContext
+            .getBean(ReadinessCheckListener.class);
+        Health health = readinessCheckListener.aggregateReadinessHealth();
+        Health healthIndicatorInfo = (Health) health.getDetails().get("HealthIndicatorInfo");
+        Health timeoutHealth = (Health) healthIndicatorInfo.getDetails().get(
+            SofaBootConstants.SOFABOOT_HEALTH_CHECK_TIMEOUT_KEY);
+        Assert.assertEquals(Status.DOWN, health.getStatus());
+        Assert.assertEquals(Status.UNKNOWN, timeoutHealth.getStatus());
+        Assert.assertEquals(SofaBootConstants.SOFABOOT_HEALTH_CHECK_TIMEOUT_MSG, timeoutHealth
+            .getDetails().get(SofaBootConstants.SOFABOOT_HEALTH_CHECK_TIMEOUT_KEY));
+    }
+
     private void initApplicationContext(boolean health) {
         Map<String, Object> properties = new LinkedHashMap<>();
         properties.put("disk-health-indicator.health", health);
         properties.put("spring.application.name", "HealthIndicatorCheckProcessorTest");
+        properties.put(SofaBootConstants.SOFABOOT_SKIP_HEALTH_INDICATOR_CHECK, "true");
+        SpringApplication springApplication = new SpringApplication(
+            HealthIndicatorConfiguration.class);
+        springApplication.setDefaultProperties(properties);
+        springApplication.setWebApplicationType(WebApplicationType.NONE);
+        applicationContext = springApplication.run();
+    }
+
+    private void initApplicationContextTimeout() {
+        Map<String, Object> properties = new LinkedHashMap<>();
+        properties.put("disk-health-indicator.health", true);
+        properties.put("spring.application.name", "HealthIndicatorCheckProcessorTest");
+        //        properties.put(SofaBootConstants.SOFABOOT_SKIP_HEALTH_INDICATOR_CHECK, "true");
+        properties.put("com.alipay.sofa.boot.health-check-parallel-timeout", "1");
         SpringApplication springApplication = new SpringApplication(
             HealthIndicatorConfiguration.class);
         springApplication.setDefaultProperties(properties);

diff --git a/...-sofa-boot/src/test/java/com/alipay/sofa/healthcheck/test/ReadinessCheckListenerTest.java b/...-sofa-boot/src/test/java/com/alipay/sofa/healthcheck/test/ReadinessCheckListenerTest.java
@@ -16,10 +16,19 @@
  */
 package com.alipay.sofa.healthcheck.test;
 
+import com.alipay.sofa.healthcheck.AfterReadinessCheckCallbackProcessor;
 import com.alipay.sofa.healthcheck.HealthCheckProperties;
+import com.alipay.sofa.healthcheck.HealthCheckerProcessor;
+import com.alipay.sofa.healthcheck.HealthIndicatorProcessor;
+import com.alipay.sofa.healthcheck.ReadinessCheckListener;
 import com.alipay.sofa.healthcheck.core.HealthCheckExecutor;
+import com.alipay.sofa.healthcheck.test.bean.DiskHealthIndicator;
+import com.alipay.sofa.healthcheck.test.bean.MemoryHealthChecker;
+import com.alipay.sofa.healthcheck.test.bean.MiddlewareHealthCheckCallback;
+import com.alipay.sofa.runtime.SofaRuntimeProperties;
 import com.alipay.sofa.runtime.configure.SofaRuntimeConfigurationProperties;
 import org.junit.Assert;
+import org.junit.BeforeClass;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.springframework.beans.BeansException;
@@ -40,13 +49,9 @@
 import org.springframework.test.context.TestPropertySource;
 import org.springframework.test.context.junit4.SpringRunner;
 
-import com.alipay.sofa.healthcheck.AfterReadinessCheckCallbackProcessor;
-import com.alipay.sofa.healthcheck.HealthCheckerProcessor;
-import com.alipay.sofa.healthcheck.HealthIndicatorProcessor;
-import com.alipay.sofa.healthcheck.ReadinessCheckListener;
-import com.alipay.sofa.healthcheck.test.bean.DiskHealthIndicator;
-import com.alipay.sofa.healthcheck.test.bean.MemoryHealthChecker;
-import com.alipay.sofa.healthcheck.test.bean.MiddlewareHealthCheckCallback;
+import java.lang.reflect.Field;
+import java.lang.reflect.Modifier;
+import java.util.concurrent.ConcurrentHashMap;
 
 /**
  * @author liangen
@@ -120,6 +125,20 @@ public HealthCheckExecutor healthCheckExecutor(HealthCheckProperties properties)
         }
     }
 
+    @BeforeClass
+    public static void beforeClass() {
+        try {
+            Field f1 = SofaRuntimeProperties.class.getDeclaredField("manualReadinessCallbackMap");
+            f1.setAccessible(true);
+            Field modifiers = f1.getClass().getDeclaredField("modifiers");
+            modifiers.setAccessible(true);
+            modifiers.setInt(f1, f1.getModifiers() & ~Modifier.FINAL);
+            f1.set(SofaRuntimeProperties.class, new ConcurrentHashMap<>());
+        } catch (Exception e) {
+            System.out.println(e);
+        }
+    }
+
     @Test
     public void testReadinessCheck() throws BeansException {
         ReadinessCheckListener readinessCheckListener = applicationContext

diff --git a/...boot-project/sofa-boot/src/main/java/com/alipay/sofa/boot/constant/SofaBootConstants.java b/...boot-project/sofa-boot/src/main/java/com/alipay/sofa/boot/constant/SofaBootConstants.java
@@ -200,11 +200,21 @@ public class SofaBootConstants {
      */
     public static final String  SOFABOOT_HEALTH_CHECK_NOT_READY_KEY                          = "HEALTH-CHECK-NOT-READY";
 
+    /**
+     * health check timeout key
+     */
+    public static final String  SOFABOOT_HEALTH_CHECK_TIMEOUT_KEY                            = "HEALTH-CHECK-TIMEOUT";
+
     /**
      * health check not ready result
      */
     public static final String  SOFABOOT_HEALTH_CHECK_NOT_READY_MSG                          = "App is still in startup process, please try later!";
 
+    /**
+     * health check timeout result
+     */
+    public static final String  SOFABOOT_HEALTH_CHECK_TIMEOUT_MSG                            = "Timeout when wait for readiness check result!";
+
     /** framework constants **/
     public static String        APPLICATION                                                  = "SOFABOOT-APPLICATION";
     public static String        PROCESSORS_OF_ROOT_APPLICATION_CONTEXT                       = "PROCESSORS_OF_ROOT_APPLICATION_CONTEXT";