diff --git a/gunicorn/arbiter.py b/gunicorn/arbiter.py index c1fde1e68..8ebedf32c 100644 --- a/gunicorn/arbiter.py +++ b/gunicorn/arbiter.py @@ -590,10 +590,37 @@ def murder_workers(self): if not worker.aborted: self.log.critical("WORKER TIMEOUT (pid:%s)", pid) worker.aborted = True + worker.abort_time = time.monotonic() self.kill_worker(pid, signal.SIGABRT) + elif self._should_wait_for_coredump(pid, worker): + continue else: self.kill_worker(pid, signal.SIGKILL) + def _should_wait_for_coredump(self, pid: int, worker) -> bool: + """Return True if SIGKILL should be held because the worker is still coredumping.""" + if not self.cfg.worker_abort_timeout: + return False + elapsed = time.monotonic() - getattr(worker, 'abort_time', 0) + if elapsed > self.cfg.worker_abort_timeout: + return False + if self._is_coredumping(pid): + self.log.debug("Worker (pid:%d) is coredumping, waiting up to %ss", pid, self.cfg.worker_abort_timeout) + return True + return False + + @staticmethod + def _is_coredumping(pid: int) -> bool: + """Check whether a process is actively dumping core (Linux only, via /proc).""" + try: + with open(f"/proc/{pid}/status") as f: + for line in f: + if line.startswith("CoreDumping:"): + return line.split()[1] == "1" + except OSError: + pass + return False + def reap_workers(self): """\ Reap workers to avoid zombie processes diff --git a/gunicorn/config.py b/gunicorn/config.py index c6ab3777f..faf4986b6 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -854,6 +854,33 @@ class GracefulTimeout(Setting): """ +class WorkerAbortTimeout(Setting): + name = "worker_abort_timeout" + section = "Worker Processes" + cli = ["--worker-abort-timeout"] + meta = "INT" + validator = validate_pos_int + type = int + default = 0 + desc = """\ + Extra seconds to wait for a worker to finish coredumping after SIGABRT, + before sending SIGKILL. + + When a worker times out, gunicorn sends SIGABRT which may trigger a + core dump. On the next heartbeat check the worker is normally killed + with SIGKILL, which can truncate an in-progress core dump. + + Setting this to a positive value enables a Linux-specific check via + ``/proc//status``: if the worker is still dumping core and the + time since SIGABRT was sent is within this limit, gunicorn will skip + sending SIGKILL and check again on the next heartbeat. Once the core + dump finishes or this limit is exceeded, SIGKILL is sent as usual. + + The default value of 0 disables this check and preserves the original + behaviour. + """ + + class Keepalive(Setting): name = "keepalive" section = "Worker Processes" diff --git a/tests/test_arbiter.py b/tests/test_arbiter.py index 930f570a7..44135c4fd 100644 --- a/tests/test_arbiter.py +++ b/tests/test_arbiter.py @@ -553,6 +553,99 @@ def test_murder_workers_sends_sigkill_second(self): mock_kill.assert_called_once_with(42, signal.SIGKILL) + def test_murder_workers_records_abort_time(self): + """Verify abort_time is set on the worker when SIGABRT is sent.""" + arbiter = gunicorn.arbiter.Arbiter(DummyApplication()) + arbiter.timeout = 30 + + mock_worker = mock.Mock() + mock_worker.aborted = False + mock_worker.tmp.last_update.return_value = 0 + arbiter.WORKERS = {42: mock_worker} + + with mock.patch('time.monotonic', return_value=100), \ + mock.patch.object(arbiter, 'kill_worker'): + arbiter.murder_workers() + + assert mock_worker.abort_time == 100 + + def test_murder_workers_coredump_wait_skips_sigkill_while_dumping(self): + """SIGKILL is withheld while the worker is coredumping and within the extra timeout.""" + arbiter = gunicorn.arbiter.Arbiter(DummyApplication()) + arbiter.timeout = 30 + arbiter.cfg.set('worker_abort_timeout', 60) + + mock_worker = mock.Mock() + mock_worker.aborted = True + mock_worker.abort_time = 90 # SIGABRT sent 5 seconds ago (monotonic=95) + mock_worker.tmp.last_update.return_value = 0 + arbiter.WORKERS = {42: mock_worker} + + with mock.patch('time.monotonic', return_value=95), \ + mock.patch.object(arbiter, 'kill_worker') as mock_kill, \ + mock.patch.object(gunicorn.arbiter.Arbiter, '_is_coredumping', return_value=True): + arbiter.murder_workers() + + mock_kill.assert_not_called() + + def test_murder_workers_coredump_wait_sends_sigkill_when_not_dumping(self): + """SIGKILL is sent immediately when the worker is aborted but not coredumping.""" + arbiter = gunicorn.arbiter.Arbiter(DummyApplication()) + arbiter.timeout = 30 + arbiter.cfg.set('worker_abort_timeout', 60) + + mock_worker = mock.Mock() + mock_worker.aborted = True + mock_worker.abort_time = 90 + mock_worker.tmp.last_update.return_value = 0 + arbiter.WORKERS = {42: mock_worker} + + with mock.patch('time.monotonic', return_value=95), \ + mock.patch.object(arbiter, 'kill_worker') as mock_kill, \ + mock.patch.object(gunicorn.arbiter.Arbiter, '_is_coredumping', return_value=False): + arbiter.murder_workers() + + mock_kill.assert_called_once_with(42, signal.SIGKILL) + + def test_murder_workers_coredump_wait_sends_sigkill_after_extra_timeout(self): + """SIGKILL is sent once the extra timeout is exceeded, even if still coredumping.""" + arbiter = gunicorn.arbiter.Arbiter(DummyApplication()) + arbiter.timeout = 30 + arbiter.cfg.set('worker_abort_timeout', 60) + + mock_worker = mock.Mock() + mock_worker.aborted = True + mock_worker.abort_time = 0 # SIGABRT sent 200 seconds ago (monotonic=200) + mock_worker.tmp.last_update.return_value = 0 + arbiter.WORKERS = {42: mock_worker} + + with mock.patch('time.monotonic', return_value=200), \ + mock.patch.object(arbiter, 'kill_worker') as mock_kill, \ + mock.patch.object(gunicorn.arbiter.Arbiter, '_is_coredumping', return_value=True): + arbiter.murder_workers() + + mock_kill.assert_called_once_with(42, signal.SIGKILL) + + def test_murder_workers_coredump_wait_zero_disables_check(self): + """worker_abort_timeout=0 sends SIGKILL without checking /proc (default behaviour).""" + arbiter = gunicorn.arbiter.Arbiter(DummyApplication()) + arbiter.timeout = 30 + arbiter.cfg.set('worker_abort_timeout', 0) + + mock_worker = mock.Mock() + mock_worker.aborted = True + mock_worker.abort_time = 95 + mock_worker.tmp.last_update.return_value = 0 + arbiter.WORKERS = {42: mock_worker} + + with mock.patch('time.monotonic', return_value=96), \ + mock.patch.object(arbiter, 'kill_worker') as mock_kill, \ + mock.patch.object(gunicorn.arbiter.Arbiter, '_is_coredumping') as mock_coredump: + arbiter.murder_workers() + + mock_kill.assert_called_once_with(42, signal.SIGKILL) + mock_coredump.assert_not_called() + # ============================================================================ # Dirty Arbiter Orphan Cleanup Tests