From 7842ed01643c12d2a7683ef6652fbdf20af075cf Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 5 May 2026 11:43:58 -0500 Subject: [PATCH 1/4] Reduce `WORKER_LOCK_MAX_RETRY_INTERVAL` to 5 seconds. Better to retry more quickly than have workers wait around. 5 seconds is still a reasonable gap in time to no overwhelm anything. This matters in cross-worker scenarios. When locks are on the same worker, when the locker holder releases, we signal to other locks that they should try reacquiring the lock immediately. But locks on other workers for the same keys only re-check based on their retry `_timeout_interval`. Updating to 5 seconds to match the previous intentions based on the [flawed code] (https://github.com/element-hq/synapse/blob/6100f6e4f7fb0c72f1ae2802683ebc811c0e3a77/synapse/handlers/worker_lock.py#L278). We can assume they were trying to have 5 seconds as the max value to retry. --- synapse/handlers/worker_lock.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/worker_lock.py b/synapse/handlers/worker_lock.py index 51be3b5084a..13130e78658 100644 --- a/synapse/handlers/worker_lock.py +++ b/synapse/handlers/worker_lock.py @@ -54,7 +54,18 @@ # will not disappear under our feet as long as we don't delete the room. NEW_EVENT_DURING_PURGE_LOCK_NAME = "new_event_during_purge_lock" -WORKER_LOCK_MAX_RETRY_INTERVAL = Duration(seconds=60) +WORKER_LOCK_MAX_RETRY_INTERVAL = Duration(seconds=5) +""" +The maximum wait time before retrying to acquire the lock. + +Better to retry more quickly than have workers wait around. 5 seconds is still a +reasonable gap in time to no overwhelm the CPU/Database. + +This matters most in cross-worker scenarios. When locks are on the same worker, when the +locker holder releases, we signal to other locks (with the same name/key) that they +should try reacquiring the lock immediately. But locks on other workers only re-check +based on their retry `_timeout_interval`. +""" WORKER_LOCK_EXCESSIVE_WAITING_WARN_DURATION = Duration(minutes=10) From 0286fcf66a4e55e8160bca27701d2ace2ad78e4a Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 5 May 2026 11:46:51 -0500 Subject: [PATCH 2/4] Add changelog --- changelog.d/19755.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/19755.misc diff --git a/changelog.d/19755.misc b/changelog.d/19755.misc new file mode 100644 index 00000000000..6ad478e531c --- /dev/null +++ b/changelog.d/19755.misc @@ -0,0 +1 @@ +Reduce `WORKER_LOCK_MAX_RETRY_INTERVAL` to 5 seconds to reduce idle time after lock is released. From ef6bddf91b1d01c52249690cb98e218416b3a536 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 5 May 2026 14:20:31 -0500 Subject: [PATCH 3/4] Fix typo --- synapse/handlers/worker_lock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/worker_lock.py b/synapse/handlers/worker_lock.py index 13130e78658..f12eff47b7f 100644 --- a/synapse/handlers/worker_lock.py +++ b/synapse/handlers/worker_lock.py @@ -59,7 +59,7 @@ The maximum wait time before retrying to acquire the lock. Better to retry more quickly than have workers wait around. 5 seconds is still a -reasonable gap in time to no overwhelm the CPU/Database. +reasonable gap in time to not overwhelm the CPU/Database. This matters most in cross-worker scenarios. When locks are on the same worker, when the locker holder releases, we signal to other locks (with the same name/key) that they From cceb21fd81c9e00fecd97853c7fc544833f00d52 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 5 May 2026 14:21:15 -0500 Subject: [PATCH 4/4] Fix lock typo --- synapse/handlers/worker_lock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/worker_lock.py b/synapse/handlers/worker_lock.py index f12eff47b7f..57792ea53cf 100644 --- a/synapse/handlers/worker_lock.py +++ b/synapse/handlers/worker_lock.py @@ -62,7 +62,7 @@ reasonable gap in time to not overwhelm the CPU/Database. This matters most in cross-worker scenarios. When locks are on the same worker, when the -locker holder releases, we signal to other locks (with the same name/key) that they +lock holder releases, we signal to other locks (with the same name/key) that they should try reacquiring the lock immediately. But locks on other workers only re-check based on their retry `_timeout_interval`. """