Suite 36 complete + Apex/Durham town coverage

backendServer/backend/settings/base.py

@@ -150,7 +150,23 @@
 # Upper bound on beat's sleep between ticks — it still wakes exactly on time for
 # actual due tasks (scrape 3:30am, ingest 4:00am, digest), this only caps how often
 # it re-polls the DB for schedule changes in between, to reduce Neon wake-ups.
+#
+# Trade-off: DatabaseScheduler keeps `PeriodicTask.last_run_at` in memory and only
+# flushes it to Postgres on a sync. With beat_sync_every left at its default (0),
+# the only other sync trigger is time-based (every 3 min, capped by this interval),
+# so last_run_at can visibly lag reality by up to 6h until CELERY_BEAT_SYNC_EVERY
+# (below) forces a flush after every task send instead. See docs/ingestion-monitoring.md
+# "Beat scheduler: last_run_at persistence lag" for the full incident writeup.
 CELERY_BEAT_MAX_LOOP_INTERVAL = 6 * 60 * 60
+# Force a DB sync after every single task send (celery/beat.py Scheduler.should_sync:
+# `self.sync_every_tasks and self._tasks_since_sync >= self.sync_every_tasks`), maps to
+# app.conf.beat_sync_every via Scheduler.__init__ (celery/beat.py:262-264). Without this,
+# a task firing within 180s of the prior sync (beat_sync_every default 0 disables the
+# task-count trigger, and the time-based trigger is `sync_every=180`) leaves
+# last_run_at stuck in memory until the next sync opportunity — up to 6h away given
+# CELERY_BEAT_MAX_LOOP_INTERVAL above. Costs one UPDATE per task fire; does not touch
+# the loop interval or Neon poll rate.
+CELERY_BEAT_SYNC_EVERY = 1
 CELERY_TIMEZONE = "UTC"
 
 # Headless-Chrome scraping is memory-heavy — pin it to its own queue drained by a

backendServer/devtools/monitoring.py

@@ -17,6 +17,7 @@
 from django.utils import timezone
 
 from broadcast.models import BroadcastSubmission, BroadcastTarget
+from events.models import Event
 from ingestion.models import EventSource, RawEvent, SourceRun, StagedEvent
 
 _STAGED_STATUSES = [choice[0] for choice in StagedEvent.STATUS_CHOICES]
@@ -251,10 +252,10 @@ def source_health(source_row, recent_runs, now):
 
     `source_row` is one row as produced by `_source_rows` (dict with `active`,
     `poll_interval_hours`, `last_polled_dt` and `created_at` — real datetimes,
-    not the ISO strings on the public row shape — `raw_count`, `funnel`).
-    `recent_runs` is a list of dicts (most-recent-first) with `status`,
-    `finished_at`, `error_message` for one source. `now` is a datetime compared
-    against `last_polled_dt` / `created_at`.
+    not the ISO strings on the public row shape — `raw_count`, `funnel`,
+    `published_all_time`). `recent_runs` is a list of dicts (most-recent-first)
+    with `status`, `finished_at`, `error_message` for one source. `now` is a
+    datetime compared against `last_polled_dt` / `created_at`.
 
     Returns {"level": "ok" | "unknown" | "warn" | "error" | "inactive",
     "reasons": [str, ...]}. Every applicable rule is evaluated and its reason
@@ -287,8 +288,20 @@ def source_health(source_row, recent_runs, now):
             reasons.append("polling but zero new raw events in window")
             level = _worse(level, "warn")
         elif source_row["funnel"]["published"] == 0:
-            reasons.append("raw events arriving but none published in window")
-            level = _worse(level, "warn")
+            # Suite 34+ sources can sit at zero *surviving StagedEvent
+            # anchors* windowed on raw_event__created_at forever — the
+            # StagedEvent that fed a live Event can be gone (pruned, or
+            # published before this window existed) while the Event itself
+            # is still live on the site. `published_all_time` (an all-time,
+            # un-windowed Event count keyed by source_name — see
+            # `_source_rows`) is the honest signal for "has this source ever
+            # actually gotten something published"; only warn when that is
+            # also zero, or a perfectly healthy source (Carrboro, The Plant
+            # NC) cries wolf forever just because its old anchors don't
+            # survive the window.
+            if source_row.get("published_all_time", 0) == 0:
+                reasons.append("raw events arriving but none published in window")
+                level = _worse(level, "warn")
 
     return {"level": level, "reasons": reasons}
 
@@ -311,6 +324,43 @@ def _raw_zero_note(raw_all_time, latest_raw_created_at_dt, now):
     )
 
 
+def _no_town_note(no_town_count: int) -> str | None:
+    """Compose the message for a nonzero `no_town` funnel cell.
+
+    Ticket 36.1's whole motivation: a source can sit at `raw > 0, published =
+    0` forever with no explanation, because its town is deliberately out of
+    coverage (e.g. Apex, Durham on prod). `skipped_no_town` is terminal-ish —
+    `publish_all_approved` only logs it once per row — so this note is the
+    monitor surfacing that same explanation instead of leaving the operator to
+    rediscover it in the logs. Mirrors `_raw_zero_note`: only rendered when
+    there is something to explain, never replacing the plain count.
+    """
+    if no_town_count == 0:
+        return None
+    return (
+        f"{no_town_count} skipped — town not in coverage "
+        f"(see `manage.py reopen_skipped_towns` once added)"
+    )
+
+
+def _published_note(published_all_time: int) -> str | None:
+    """Compose the message for a `published == 0` funnel cell.
+
+    Ticket 36.6's motivation: `published` is windowed on
+    `raw_event__created_at` via a *surviving* `StagedEvent` anchor — a source
+    published entirely before suite 34 can have zero surviving anchors in any
+    window forever, even though its events are live on the site right now.
+    `published_all_time` (an un-windowed, all-time `Event` count keyed by
+    `source_name` — see `_source_rows`) is the honest "has this ever actually
+    worked" signal `StagedEvent` can't provide once its anchors are gone.
+    Mirrors `_raw_zero_note` / `_no_town_note`: only rendered when there is
+    something to explain, never replacing the plain `0` count.
+    """
+    if published_all_time == 0:
+        return None
+    return f"0 in window; {published_all_time} events live all-time"
+
+
 def _source_rows(db, start, end, source_type_filter, runs_state=None):
     # Staleness is a wall-clock question ("has this source polled recently?"),
     # so it is measured against real `now`, never the window's `end`. In prod
@@ -369,6 +419,18 @@ def _source_rows(db, start, end, source_type_filter, runs_state=None):
     raw_all_time_counts = {row["source"]: row["n"] for row in all_time_rows}
     latest_raw_created_at = {row["source"]: row["latest"] for row in all_time_rows}
 
+    # All-time live Event count per source, keyed by `EventSource.name` since
+    # `Event` carries no source FK or creation timestamp — only a free-text
+    # `source_name` (see ticket 36.6). This cannot be windowed the way
+    # `raw`/`published` are; it is deliberately the un-windowed "has this
+    # source ever gotten anything published, ever" signal, mirroring
+    # `all_time_rows` above. One grouped query for every source in this call,
+    # not per-source — no N+1.
+    published_all_time_by_name = {
+        row["source_name"]: row["n"]
+        for row in Event.objects.using(db).values("source_name").annotate(n=Count("pk"))
+    }
+
     no_staged_counts = {
         row["source"]: row["n"]
         for row in RawEvent.objects.using(db)
@@ -423,6 +485,13 @@ def _source_rows(db, start, end, source_type_filter, runs_state=None):
             approved_unpublished=Count(
                 "id", filter=Q(status="approved", published_event__isnull=True)
             ),
+            # Terminal-ish: `publish_all_approved` couldn't resolve the row's
+            # town to a real `Town` (see ingestion/services.py). Was folded
+            # into `approved` before ticket 36.1 introduced this status, which
+            # silently hid these rows forever behind an `approved` count that
+            # never moved. Disjoint from every other bucket here, same as
+            # `approved_unpublished`.
+            skipped_no_town=Count("id", filter=Q(status="skipped_no_town")),
         )
     )
     funnel_staged_by_source = {row["raw_event__source_id"]: row for row in funnel_staged_qs}
@@ -462,6 +531,13 @@ def _source_rows(db, start, end, source_type_filter, runs_state=None):
             if raw_in_window == 0
             else None
         )
+        no_town_count = funnel_staged.get("skipped_no_town", 0)
+        no_town_note = _no_town_note(no_town_count)
+        published_in_window = funnel_staged.get("published", 0)
+        published_all_time = published_all_time_by_name.get(source["name"], 0)
+        # Only computed for a zero-in-window row — mirrors `raw_zero_note`:
+        # a healthy funnel has nothing to disambiguate.
+        published_note = _published_note(published_all_time) if published_in_window == 0 else None
         last_run = (
             {
                 "status": recent_runs[0]["status"],
@@ -491,13 +567,34 @@ def _source_rows(db, start, end, source_type_filter, runs_state=None):
             # "0", turning "8 muted zeros" into "1 stale source + 2 real
             # never-ingested sources" at a glance.
             "raw_zero_note": raw_zero_note,
+            # Only set when `no_town > 0` — see `_no_town_note`. Renders as a
+            # tooltip/subtext on the `no_town` funnel cell so a source stuck
+            # at `raw > 0, published = 0` because its town is out of coverage
+            # reads as "explained", not "broken".
+            "no_town_note": no_town_note,
+            # All-time, un-windowed count of live Events attributed to this
+            # source by `Event.source_name` — see the `published_all_time_by_name`
+            # query above. `Event` has no source FK or creation timestamp, so
+            # this cannot be windowed the way `raw_all_time` is; it is the
+            # honest "has this source ever gotten anything published, ever"
+            # signal for a source whose old StagedEvent anchors no longer
+            # survive the funnel window (ticket 36.6).
+            "published_all_time": published_all_time,
+            # Only set when `published == 0` in-window — see `_published_note`.
+            # Renders as a tooltip/subtext on the `published` funnel cell so a
+            # source that's actually healthy all-time doesn't read as broken
+            # just because its funnel window has nothing to show.
+            "published_note": published_note,
             "staged_by_status": staged_status_counts,
             "published_count": funnel_staged.get("published", 0),
             # Buckets are mutually exclusive — each raw event lands in exactly
             # one — so they sum to `raw`. `published` is every row carrying a
             # live Event (whether or not the sweep has flipped it to the
-            # terminal `published` status), and `approved` is the residual:
-            # approved but not yet published. See the annotations above.
+            # terminal `published` status), `approved` is the residual:
+            # approved but not yet published, and `no_town` is the terminal-ish
+            # `skipped_no_town` status (ticket 36.1) — a row whose town isn't
+            # in coverage, never re-attempted until `reopen_skipped_towns`
+            # runs. See the annotations above.
             "funnel": {
                 "raw": raw_in_window,
                 "unprocessed": unprocessed_counts.get(source_id, 0),
@@ -507,6 +604,7 @@ def _source_rows(db, start, end, source_type_filter, runs_state=None):
                 "held_for_review": funnel_staged.get("held_for_review", 0),
                 "rejected": staged_status_counts["rejected"],
                 "approved": funnel_staged.get("approved_unpublished", 0),
+                "no_town": no_town_count,
                 "published": funnel_staged.get("published", 0),
             },
             "last_run": last_run,

backendServer/devtools/templates/devtools/monitor.html

@@ -52,7 +52,9 @@
   .monitor td.num.zero { color: #b6bcc6; }
   /* Ticket 35.4: distinguishes "zero in this window" from "zero ever" inline,
      next to the muted 0 the funnel-cell convention already renders — see
-     `_raw_zero_note` in monitoring.py for the two message shapes. */
+     `_raw_zero_note` in monitoring.py for the two message shapes. Also used
+     by `_no_town_note` (no_town cell) and `_published_note` (published
+     cell, ticket 36.6) — same convention, different funnel columns. */
   .monitor .zero-note { font-weight: 400; font-size: 0.78rem; color: #92400e; white-space: nowrap; }
   .monitor .empty-note { color: #6b7280; font-size: 0.85rem; margin: 0.5rem 0 1rem; }
   .monitor .notice {
@@ -247,6 +249,7 @@ <h2>Collectors</h2>
         <th class="num">Held for review</th>
         <th class="num">Rejected</th>
         <th class="num">Approved</th>
+        <th class="num">No town</th>
         <th class="num">Published</th>
         <th></th>
       </tr>
@@ -274,10 +277,11 @@ <h2>Collectors</h2>
         <td class="num funnel-cell{% if not c.funnel.held_for_review %} zero{% endif %}">{{ c.funnel.held_for_review }}</td>
         <td class="num funnel-cell{% if not c.funnel.rejected %} zero{% endif %}">{{ c.funnel.rejected }}</td>
         <td class="num funnel-cell{% if not c.funnel.approved %} zero{% endif %}">{{ c.funnel.approved }}</td>
-        <td class="num funnel-cell{% if not c.funnel.published %} zero{% endif %}">{{ c.funnel.published }}</td>
+        <td class="num funnel-cell{% if not c.funnel.no_town %} zero{% endif %}"{% if c.no_town_note %} title="{{ c.no_town_note }}"{% endif %}>{{ c.funnel.no_town }}{% if c.no_town_note %}<span class="zero-note"> &mdash; {{ c.no_town_note }}</span>{% endif %}</td>
+        <td class="num funnel-cell{% if not c.funnel.published %} zero{% endif %}"{% if c.published_note %} title="{{ c.published_note }}"{% endif %}>{{ c.funnel.published }}{% if c.published_note %}<span class="zero-note"> &mdash; {{ c.published_note }}</span>{% endif %}</td>
         <td><button type="button" class="btn-probe" data-probe-source-id="{{ c.id }}">Probe</button></td>
       </tr>
-      <tr class="drilldown-row" data-for="collector:{{ c.id }}" style="display:none;"><td colspan="15"></td></tr>
+      <tr class="drilldown-row" data-for="collector:{{ c.id }}" style="display:none;"><td colspan="16"></td></tr>
       {% endfor %}
     </tbody>
   </table>
@@ -302,6 +306,7 @@ <h2>Broadcast — inbound (direct submissions)</h2>
         <th class="num">Held for review</th>
         <th class="num">Rejected</th>
         <th class="num">Approved</th>
+        <th class="num">No town</th>
         <th class="num">Published</th>
         <th></th>
       </tr>
@@ -325,10 +330,11 @@ <h2>Broadcast — inbound (direct submissions)</h2>
         <td class="num funnel-cell{% if not c.funnel.held_for_review %} zero{% endif %}">{{ c.funnel.held_for_review }}</td>
         <td class="num funnel-cell{% if not c.funnel.rejected %} zero{% endif %}">{{ c.funnel.rejected }}</td>
         <td class="num funnel-cell{% if not c.funnel.approved %} zero{% endif %}">{{ c.funnel.approved }}</td>
-        <td class="num funnel-cell{% if not c.funnel.published %} zero{% endif %}">{{ c.funnel.published }}</td>
+        <td class="num funnel-cell{% if not c.funnel.no_town %} zero{% endif %}"{% if c.no_town_note %} title="{{ c.no_town_note }}"{% endif %}>{{ c.funnel.no_town }}{% if c.no_town_note %}<span class="zero-note"> &mdash; {{ c.no_town_note }}</span>{% endif %}</td>
+        <td class="num funnel-cell{% if not c.funnel.published %} zero{% endif %}"{% if c.published_note %} title="{{ c.published_note }}"{% endif %}>{{ c.funnel.published }}{% if c.published_note %}<span class="zero-note"> &mdash; {{ c.published_note }}</span>{% endif %}</td>
         <td><button type="button" class="btn-probe" data-probe-source-id="{{ c.id }}">Probe</button></td>
       </tr>
-      <tr class="drilldown-row" data-for="inbound:{{ c.id }}" style="display:none;"><td colspan="14"></td></tr>
+      <tr class="drilldown-row" data-for="inbound:{{ c.id }}" style="display:none;"><td colspan="15"></td></tr>
       {% endfor %}
     </tbody>
   </table>