From 187bbf62b49e40a6f820ad91baa544e5469644e8 Mon Sep 17 00:00:00 2001 From: Amr Gaber Date: Sun, 19 Apr 2026 00:15:53 -0500 Subject: [PATCH] chore(observability): remove temporary Sentry rate limiter The pool-exhaustion storm from 2026-04-18 has subsided; restoring full fidelity of TooManyConnectionsError reports so future incidents are visible again. Closes #78 --- app/observability/sentry.py | 42 ---------------- tests/test_sentry_rate_limit.py | 89 --------------------------------- 2 files changed, 131 deletions(-) delete mode 100644 tests/test_sentry_rate_limit.py diff --git a/app/observability/sentry.py b/app/observability/sentry.py index eadcd5f..18e9e58 100644 --- a/app/observability/sentry.py +++ b/app/observability/sentry.py @@ -4,51 +4,10 @@ StarletteIntegration can auto-instrument middleware. See app/main.py. """ -import time -from collections import deque -from threading import Lock - import sentry_sdk -from asyncpg.exceptions import TooManyConnectionsError from app.config import settings -# Rate-limit pool-exhaustion reports to Sentry. The first N events per -# window surface normally (storm visibility preserved); beyond that, -# events drop from Sentry but remain in structlog app logs. Per-process -# state, so aggregate Sentry volume scales with Cloud Run instance count. -_POOL_ERR_WINDOW_S = 60.0 -_POOL_ERR_MAX_PER_WINDOW = 5 -_pool_err_times: deque[float] = deque(maxlen=_POOL_ERR_MAX_PER_WINDOW) -_pool_err_lock = Lock() - - -def _contains_pool_exhaustion(exc: BaseException | None) -> bool: - if exc is None: - return False - if isinstance(exc, TooManyConnectionsError): - return True - if isinstance(exc, BaseExceptionGroup): - return any(_contains_pool_exhaustion(sub) for sub in exc.exceptions) - cause = exc.__cause__ if exc.__cause__ is not None else exc.__context__ - if cause is not None and cause is not exc: - return _contains_pool_exhaustion(cause) - return False - - -def _rate_limit_pool_exhaustion(event, hint): - exc_info = hint.get("exc_info") - if not exc_info or not _contains_pool_exhaustion(exc_info[1]): - return event - now = time.monotonic() - with _pool_err_lock: - while _pool_err_times and _pool_err_times[0] < now - _POOL_ERR_WINDOW_S: - _pool_err_times.popleft() - if len(_pool_err_times) >= _POOL_ERR_MAX_PER_WINDOW: - return None - _pool_err_times.append(now) - return event - def init_sentry() -> None: if not settings.sentry_dsn: @@ -68,5 +27,4 @@ def init_sentry() -> None: profile_session_sample_rate=1.0, profile_lifecycle="trace", enable_logs=True, - before_send=_rate_limit_pool_exhaustion, ) diff --git a/tests/test_sentry_rate_limit.py b/tests/test_sentry_rate_limit.py deleted file mode 100644 index 3b14baa..0000000 --- a/tests/test_sentry_rate_limit.py +++ /dev/null @@ -1,89 +0,0 @@ -"""Tests for the Sentry pool-exhaustion rate limiter in app.observability.sentry.""" - -from __future__ import annotations - -from unittest.mock import patch - -import pytest -from asyncpg.exceptions import TooManyConnectionsError - -from app.observability import sentry as sentry_mod - - -@pytest.fixture(autouse=True) -def _reset_rate_limiter(): - sentry_mod._pool_err_times.clear() - yield - sentry_mod._pool_err_times.clear() - - -def _event_hint_for(exc: BaseException) -> tuple[dict, dict]: - """Build the (event, hint) pair Sentry passes to before_send.""" - return {}, {"exc_info": (type(exc), exc, None)} - - -def test_unrelated_exception_passes_through(): - event, hint = _event_hint_for(ValueError("nope")) - assert sentry_mod._rate_limit_pool_exhaustion(event, hint) is event - assert len(sentry_mod._pool_err_times) == 0 - - -def test_missing_exc_info_passes_through(): - event = {} - assert sentry_mod._rate_limit_pool_exhaustion(event, {}) is event - - -def test_first_n_pool_events_pass_then_drop(): - for i in range(sentry_mod._POOL_ERR_MAX_PER_WINDOW): - event, hint = _event_hint_for(TooManyConnectionsError("exhausted")) - assert sentry_mod._rate_limit_pool_exhaustion(event, hint) is event, ( - f"event {i} should pass" - ) - event, hint = _event_hint_for(TooManyConnectionsError("exhausted")) - assert sentry_mod._rate_limit_pool_exhaustion(event, hint) is None - - -def test_dropped_events_do_not_count_against_quota(): - # Fill the window - for _ in range(sentry_mod._POOL_ERR_MAX_PER_WINDOW): - event, hint = _event_hint_for(TooManyConnectionsError("exhausted")) - sentry_mod._rate_limit_pool_exhaustion(event, hint) - # Dropped events must not extend the window — otherwise a sustained - # storm would permanently starve out new events even after the - # window rolls over. - for _ in range(10): - event, hint = _event_hint_for(TooManyConnectionsError("exhausted")) - sentry_mod._rate_limit_pool_exhaustion(event, hint) - assert len(sentry_mod._pool_err_times) == sentry_mod._POOL_ERR_MAX_PER_WINDOW - - -def test_cause_chain_is_walked(): - outer = RuntimeError("wrapped") - outer.__cause__ = TooManyConnectionsError("inner") - event, hint = _event_hint_for(outer) - assert sentry_mod._rate_limit_pool_exhaustion(event, hint) is event - assert len(sentry_mod._pool_err_times) == 1 - - -def test_exception_group_is_walked(): - group = BaseExceptionGroup("group", [TooManyConnectionsError("inner")]) - event, hint = _event_hint_for(group) - assert sentry_mod._rate_limit_pool_exhaustion(event, hint) is event - assert len(sentry_mod._pool_err_times) == 1 - - -def test_window_eviction_allows_new_events_after_expiry(): - with patch.object(sentry_mod.time, "monotonic", return_value=0.0): - for _ in range(sentry_mod._POOL_ERR_MAX_PER_WINDOW): - event, hint = _event_hint_for(TooManyConnectionsError("boom")) - sentry_mod._rate_limit_pool_exhaustion(event, hint) - event, hint = _event_hint_for(TooManyConnectionsError("boom")) - assert sentry_mod._rate_limit_pool_exhaustion(event, hint) is None - - with patch.object( - sentry_mod.time, - "monotonic", - return_value=sentry_mod._POOL_ERR_WINDOW_S + 1.0, - ): - event, hint = _event_hint_for(TooManyConnectionsError("boom")) - assert sentry_mod._rate_limit_pool_exhaustion(event, hint) is event