From 44ea009c4a4641a963521aca43b63c4478a8becd Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Fri, 13 Sep 2024 21:44:36 +1000 Subject: [PATCH] log hot threads if node fails to close in time --- server/src/main/java/org/elasticsearch/node/Node.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java index 3302114b078a8..34c3bed953b0d 100644 --- a/server/src/main/java/org/elasticsearch/node/Node.java +++ b/server/src/main/java/org/elasticsearch/node/Node.java @@ -64,6 +64,7 @@ import org.elasticsearch.indices.store.IndicesStore; import org.elasticsearch.injection.guice.Injector; import org.elasticsearch.monitor.fs.FsHealthService; +import org.elasticsearch.monitor.jvm.HotThreads; import org.elasticsearch.monitor.jvm.JvmInfo; import org.elasticsearch.monitor.metrics.NodeMetrics; import org.elasticsearch.node.internal.TerminationHandler; @@ -92,6 +93,7 @@ import java.io.Closeable; import java.io.File; import java.io.IOException; +import java.io.StringWriter; import java.net.InetAddress; import java.net.InetSocketAddress; import java.nio.charset.Charset; @@ -111,7 +113,6 @@ import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collectors; - import javax.net.ssl.SNIHostName; import static org.elasticsearch.core.Strings.format; @@ -724,6 +725,13 @@ public synchronized boolean awaitClose(long timeout, TimeUnit timeUnit) throws I + "Something is leaking index readers or store references." ); } + } else { + try (var writer = new StringWriter()) { + new HotThreads().busiestThreads(9999).ignoreIdleThreads(false).detect(writer); + logger.info("--> hot threads failing close node:\n{}\n", writer.toString()); + } catch (Exception e) { + logger.info("--> fail to log hot threads failing close node [{}]", e.getMessage()); + } } return terminated; }