From d46444468c4c01c5df5553bc2efa1ca433940eb4 Mon Sep 17 00:00:00 2001 From: yyj8 <1012293987@qq.com> Date: Sat, 15 Jun 2024 11:41:54 +0800 Subject: [PATCH 1/2] [fix] [broker] fix broker log a full thread dump when a deadlock is detected in healthcheck every time --- .../org/apache/pulsar/broker/admin/impl/BrokersBase.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java index 9db17f76a8dbe..bd2aab3c45118 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java @@ -88,7 +88,7 @@ public class BrokersBase extends AdminResource { private static final Duration HEALTH_CHECK_READ_TIMEOUT = Duration.ofSeconds(58); private static final TimeoutException HEALTH_CHECK_TIMEOUT_EXCEPTION = FutureUtil.createTimeoutException("Timeout", BrokersBase.class, "healthCheckRecursiveReadNext(...)"); - private volatile long threadDumpLoggedTimestamp; + private static volatile long threadDumpLoggedTimestamp; @GET @Path("/{cluster}") @@ -395,8 +395,9 @@ private void checkDeadlockedThreads() { String threadNames = Arrays.stream(threadInfos) .map(threadInfo -> threadInfo.getThreadName() + "(tid=" + threadInfo.getThreadId() + ")").collect( Collectors.joining(", ")); - if (System.currentTimeMillis() - threadDumpLoggedTimestamp - > LOG_THREADDUMP_INTERVAL_WHEN_DEADLOCK_DETECTED) { + if ((System.currentTimeMillis() - threadDumpLoggedTimestamp + > LOG_THREADDUMP_INTERVAL_WHEN_DEADLOCK_DETECTED) || + threadDumpLoggedTimestamp == 0) { threadDumpLoggedTimestamp = System.currentTimeMillis(); LOG.error("Deadlocked threads detected. {}\n{}", threadNames, ThreadDumpUtil.buildThreadDiagnosticString()); From 31ed154fec6b5a0f4a5ffe78ae855e71b7271f0c Mon Sep 17 00:00:00 2001 From: yyj8 <1012293987@qq.com> Date: Tue, 18 Jun 2024 09:48:17 +0800 Subject: [PATCH 2/2] [fix] [broker] fix broker log a full thread dump when a deadlock is detected in healthcheck every time --- .../org/apache/pulsar/broker/admin/impl/BrokersBase.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java index bd2aab3c45118..4d0b598a8e4f1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java @@ -395,9 +395,8 @@ private void checkDeadlockedThreads() { String threadNames = Arrays.stream(threadInfos) .map(threadInfo -> threadInfo.getThreadName() + "(tid=" + threadInfo.getThreadId() + ")").collect( Collectors.joining(", ")); - if ((System.currentTimeMillis() - threadDumpLoggedTimestamp - > LOG_THREADDUMP_INTERVAL_WHEN_DEADLOCK_DETECTED) || - threadDumpLoggedTimestamp == 0) { + if (System.currentTimeMillis() - threadDumpLoggedTimestamp + > LOG_THREADDUMP_INTERVAL_WHEN_DEADLOCK_DETECTED) { threadDumpLoggedTimestamp = System.currentTimeMillis(); LOG.error("Deadlocked threads detected. {}\n{}", threadNames, ThreadDumpUtil.buildThreadDiagnosticString());