From 247cc86f7d04570eecacf76ea12d1509b2b3c1f8 Mon Sep 17 00:00:00 2001 From: Arya Venkatesan Date: Sat, 1 Aug 2026 01:34:08 -0400 Subject: [PATCH 1/8] feat: many calendar impls that aren't tested --- .claude/skills/handoff-report/SKILL.md | 325 +++++++++ backendServer/devtools/monitoring.py | 267 ++++++-- .../devtools/templates/devtools/monitor.html | 642 +++++++++++------- .../templates/devtools/playground.html | 22 +- .../devtools/tests/test_monitor_view_db.py | 109 +++ .../devtools/tests/test_monitoring_db.py | 211 +++++- .../devtools/tests/test_source_health_fast.py | 168 +++++ backendServer/devtools/views.py | 21 +- .../events/management/commands/seed_dev.py | 9 +- .../management/commands/seed_sources.py | 60 ++ .../ingestion/scraping/scrapers/__init__.py | 58 +- .../ingestion/scraping/scrapers/base.py | 6 + .../scraping/scrapers/carychamber.py | 123 ++++ .../scraping/scrapers/carysistercities.py | 118 ++++ .../scraping/scrapers/chapelhillarts.py | 99 +++ .../scraping/scrapers/chapelhillnc.py | 115 ++++ .../scraping/scrapers/chathamchamber.py | 122 ++++ .../scraping/scrapers/downtowncarync.py | 130 ++++ .../scraping/scrapers/downtownraleigh.py | 165 +++++ .../scrapers/eventbritemorrisville.py | 30 + .../scraping/scrapers/eventbritepittsboro.py | 26 + .../scraping/scrapers/eventbriteraleigh.py | 178 +++++ .../scraping/scrapers/morrisvillechamber.py | 129 ++++ .../scraping/scrapers/morrisvilleevents.py | 129 ++++ .../ingestion/scraping/scrapers/patch.py | 160 +++++ .../scraping/scrapers/patchdurham.py | 131 ++++ .../scraping/scrapers/patchmorrisville.py | 21 + .../scraping/scrapers/patchpittsboro.py | 19 + .../scraping/scrapers/patchraleigh.py | 16 + .../ingestion/scraping/scrapers/raleighnc.py | 97 +++ .../ingestion/scraping/scrapers/theplantnc.py | 1 + .../scraping/scrapers/thrivinginraleigh.py | 121 ++++ .../scraping/scrapers/triangleonthecheap.py | 123 ++++ .../scraping/scrapers/visitchapelhill.py | 1 + .../scraping/scrapers/visitpittsboro.py | 1 + .../scraping/scrapers/visitraleigh.py | 162 +++++ .../scraping/scrapers/visitraleigh_cary.py | 35 + backendServer/ingestion/services.py | 30 +- .../ingestion/tests/fixtures/carychamber.html | 70 ++ .../tests/fixtures/carysistercities.html | 150 ++++ .../tests/fixtures/chapelhillarts.html | 87 +++ .../tests/fixtures/chapelhillnc.html | 16 + .../tests/fixtures/chathamchamber.html | 203 ++++++ .../tests/fixtures/downtowncarync.html | 219 ++++++ .../tests/fixtures/downtownraleigh.html | 21 + .../tests/fixtures/eventbritemorrisville.html | 14 + .../tests/fixtures/eventbritepittsboro.html | 14 + .../tests/fixtures/eventbriteraleigh.html | 14 + .../tests/fixtures/morrisvillechamber.html | 116 ++++ .../tests/fixtures/morrisvilleevents.html | 129 ++++ .../ingestion/tests/fixtures/patchdurham.html | 12 + .../tests/fixtures/patchmorrisville.html | 12 + .../tests/fixtures/patchpittsboro.html | 12 + .../tests/fixtures/patchraleigh.html | 12 + .../ingestion/tests/fixtures/raleighnc.html | 196 ++++++ .../tests/fixtures/thrivinginraleigh.html | 177 +++++ .../tests/fixtures/triangleonthecheap.html | 63 ++ .../tests/fixtures/visitraleigh.html | 119 ++++ .../tests/fixtures/visitraleigh_cary.html | 124 ++++ .../tests/test_carychamber_extract_fast.py | 75 ++ .../test_carysistercities_extract_fast.py | 74 ++ .../tests/test_chapelhillarts_extract_fast.py | 66 ++ .../tests/test_chapelhillnc_extract_fast.py | 65 ++ .../tests/test_chathamchamber_extract_fast.py | 72 ++ .../ingestion/tests/test_direct_ingest_db.py | 146 ++++ .../tests/test_downtowncarync_extract_fast.py | 72 ++ .../test_downtownraleigh_extract_fast.py | 85 +++ ...test_eventbritemorrisville_extract_fast.py | 57 ++ .../test_eventbritepittsboro_extract_fast.py | 50 ++ .../test_eventbriteraleigh_extract_fast.py | 66 ++ .../test_morrisvillechamber_extract_fast.py | 67 ++ .../test_morrisvilleevents_extract_fast.py | 70 ++ .../tests/test_patchdurham_extract_fast.py | 62 ++ .../test_patchmorrisville_extract_fast.py | 64 ++ .../tests/test_patchpittsboro_extract_fast.py | 68 ++ .../tests/test_patchraleigh_extract_fast.py | 65 ++ .../tests/test_raleighnc_extract_fast.py | 65 ++ .../ingestion/tests/test_seed_sources_db.py | 70 ++ .../ingestion/tests/test_services_db.py | 2 + .../test_thrivinginraleigh_extract_fast.py | 79 +++ .../test_triangleonthecheap_extract_fast.py | 83 +++ .../test_visitraleigh_cary_extract_fast.py | 68 ++ .../tests/test_visitraleigh_extract_fast.py | 80 +++ human-docs/README.md | 15 + notion-sync/STATE.md | 4 +- 85 files changed, 7338 insertions(+), 312 deletions(-) create mode 100644 .claude/skills/handoff-report/SKILL.md create mode 100644 backendServer/ingestion/management/commands/seed_sources.py create mode 100644 backendServer/ingestion/scraping/scrapers/carychamber.py create mode 100644 backendServer/ingestion/scraping/scrapers/carysistercities.py create mode 100644 backendServer/ingestion/scraping/scrapers/chapelhillarts.py create mode 100644 backendServer/ingestion/scraping/scrapers/chapelhillnc.py create mode 100644 backendServer/ingestion/scraping/scrapers/chathamchamber.py create mode 100644 backendServer/ingestion/scraping/scrapers/downtowncarync.py create mode 100644 backendServer/ingestion/scraping/scrapers/downtownraleigh.py create mode 100644 backendServer/ingestion/scraping/scrapers/eventbritemorrisville.py create mode 100644 backendServer/ingestion/scraping/scrapers/eventbritepittsboro.py create mode 100644 backendServer/ingestion/scraping/scrapers/eventbriteraleigh.py create mode 100644 backendServer/ingestion/scraping/scrapers/morrisvillechamber.py create mode 100644 backendServer/ingestion/scraping/scrapers/morrisvilleevents.py create mode 100644 backendServer/ingestion/scraping/scrapers/patch.py create mode 100644 backendServer/ingestion/scraping/scrapers/patchdurham.py create mode 100644 backendServer/ingestion/scraping/scrapers/patchmorrisville.py create mode 100644 backendServer/ingestion/scraping/scrapers/patchpittsboro.py create mode 100644 backendServer/ingestion/scraping/scrapers/patchraleigh.py create mode 100644 backendServer/ingestion/scraping/scrapers/raleighnc.py create mode 100644 backendServer/ingestion/scraping/scrapers/thrivinginraleigh.py create mode 100644 backendServer/ingestion/scraping/scrapers/triangleonthecheap.py create mode 100644 backendServer/ingestion/scraping/scrapers/visitraleigh.py create mode 100644 backendServer/ingestion/scraping/scrapers/visitraleigh_cary.py create mode 100644 backendServer/ingestion/tests/fixtures/carychamber.html create mode 100644 backendServer/ingestion/tests/fixtures/carysistercities.html create mode 100644 backendServer/ingestion/tests/fixtures/chapelhillarts.html create mode 100644 backendServer/ingestion/tests/fixtures/chapelhillnc.html create mode 100644 backendServer/ingestion/tests/fixtures/chathamchamber.html create mode 100644 backendServer/ingestion/tests/fixtures/downtowncarync.html create mode 100644 backendServer/ingestion/tests/fixtures/downtownraleigh.html create mode 100644 backendServer/ingestion/tests/fixtures/eventbritemorrisville.html create mode 100644 backendServer/ingestion/tests/fixtures/eventbritepittsboro.html create mode 100644 backendServer/ingestion/tests/fixtures/eventbriteraleigh.html create mode 100644 backendServer/ingestion/tests/fixtures/morrisvillechamber.html create mode 100644 backendServer/ingestion/tests/fixtures/morrisvilleevents.html create mode 100644 backendServer/ingestion/tests/fixtures/patchdurham.html create mode 100644 backendServer/ingestion/tests/fixtures/patchmorrisville.html create mode 100644 backendServer/ingestion/tests/fixtures/patchpittsboro.html create mode 100644 backendServer/ingestion/tests/fixtures/patchraleigh.html create mode 100644 backendServer/ingestion/tests/fixtures/raleighnc.html create mode 100644 backendServer/ingestion/tests/fixtures/thrivinginraleigh.html create mode 100644 backendServer/ingestion/tests/fixtures/triangleonthecheap.html create mode 100644 backendServer/ingestion/tests/fixtures/visitraleigh.html create mode 100644 backendServer/ingestion/tests/fixtures/visitraleigh_cary.html create mode 100644 backendServer/ingestion/tests/test_carychamber_extract_fast.py create mode 100644 backendServer/ingestion/tests/test_carysistercities_extract_fast.py create mode 100644 backendServer/ingestion/tests/test_chapelhillarts_extract_fast.py create mode 100644 backendServer/ingestion/tests/test_chapelhillnc_extract_fast.py create mode 100644 backendServer/ingestion/tests/test_chathamchamber_extract_fast.py create mode 100644 backendServer/ingestion/tests/test_downtowncarync_extract_fast.py create mode 100644 backendServer/ingestion/tests/test_downtownraleigh_extract_fast.py create mode 100644 backendServer/ingestion/tests/test_eventbritemorrisville_extract_fast.py create mode 100644 backendServer/ingestion/tests/test_eventbritepittsboro_extract_fast.py create mode 100644 backendServer/ingestion/tests/test_eventbriteraleigh_extract_fast.py create mode 100644 backendServer/ingestion/tests/test_morrisvillechamber_extract_fast.py create mode 100644 backendServer/ingestion/tests/test_morrisvilleevents_extract_fast.py create mode 100644 backendServer/ingestion/tests/test_patchdurham_extract_fast.py create mode 100644 backendServer/ingestion/tests/test_patchmorrisville_extract_fast.py create mode 100644 backendServer/ingestion/tests/test_patchpittsboro_extract_fast.py create mode 100644 backendServer/ingestion/tests/test_patchraleigh_extract_fast.py create mode 100644 backendServer/ingestion/tests/test_raleighnc_extract_fast.py create mode 100644 backendServer/ingestion/tests/test_seed_sources_db.py create mode 100644 backendServer/ingestion/tests/test_thrivinginraleigh_extract_fast.py create mode 100644 backendServer/ingestion/tests/test_triangleonthecheap_extract_fast.py create mode 100644 backendServer/ingestion/tests/test_visitraleigh_cary_extract_fast.py create mode 100644 backendServer/ingestion/tests/test_visitraleigh_extract_fast.py create mode 100644 human-docs/README.md diff --git a/.claude/skills/handoff-report/SKILL.md b/.claude/skills/handoff-report/SKILL.md new file mode 100644 index 0000000..046dee4 --- /dev/null +++ b/.claude/skills/handoff-report/SKILL.md @@ -0,0 +1,325 @@ +--- +name: handoff-report +description: >- + Transfer working knowledge of an existing part of The Commons to someone who lacks it — + what it does and who depends on it, Mermaid diagrams of real behaviour, data-model/interface + tables, and the sharp edges that bite newcomers. Triggers: "how does X work", "document this + service", "onboard someone onto Y", "I'm handing this off", "explain this subsystem", + "write a handoff doc". +--- + +# Handoff Report + +## Overview + +A handoff report transfers working knowledge of a system that already exists to someone who +has to reason about it without you — a newcomer, an inheriting owner, a future maintainer of +this repo who wasn't around when it was built. + +The format is deliberately narrow: prose for why it exists, diagrams for how it behaves, +tables for surface area, and an explicit list of what will surprise you. Each carries what the +others can't. + +Two principles do most of the work: + +- **Describe behaviour, not intent.** Read the code and say what it does. A report that + documents what the system was supposed to do is worse than no report — the reader trusts it + and debugs against a fiction. +- **The report stands completely alone.** Every reference pointing outside the document is a + place the reader stops and doesn't come back. + +## When to use + +- Onboarding someone onto a service, subsystem, or domain in this repo (`ingestion/`, + `broadcast/`, the auth bridge, the devtools monitor, etc.). +- Handing off ownership — leaving the project, rotating, going on leave. (This is the sole-dev + → future-team case this skill exists for.) +- A neighbouring workstream needs to integrate and keeps asking how something works. +- Knowledge exists only in one session's context, one PR description, or one person's head. +- Post-hoc documentation of something built fast (a suite pushed in one sitting) and never + written down properly. + +**Not for:** point-in-time session status updates — this repo's `docs/handoff-suite-*.md` files +and `notion-sync/STATE.md` are that genre (what shipped, what's uncommitted, what to pick up +next). A handoff-report is durable subsystem knowledge, not a snapshot of where a session left +off. Also not for planning work that hasn't been built yet — use `/write-tickets` for that. + +## Where the finished report lands + +`docs/` is the system of record agents read on every task (per `CLAUDE.md`) — keep it lean and +agent-oriented, don't add human-onboarding material to it. A handoff report is written *for a +person*, so it lands in **`human-docs/`** instead: + +1. Write it to `human-docs/.md` (e.g. `human-docs/ingestion-pipeline.md` — descriptive + name, no suite number or date unless the report is itself dated point-in-time knowledge). +2. Add a row to the table in `human-docs/README.md` — Doc / Purpose / Written — so it's + discoverable. An undiscoverable handoff report is as good as not written. +3. If `docs/*.md` or a per-directory `AGENTS.md` already covers this ground for agents, don't + fork a duplicate — the handoff report can reference and go beyond it (more narrative context, + the sharp edges an agent doc wouldn't include), but state at the top which `docs/` file it + complements so the two don't drift into contradicting each other. + +If the report is a one-off session status note instead (not durable subsystem knowledge), it +follows the `docs/handoff-suite-N-*.md` pattern instead of this skill's structure — that's +agent/continuation-facing state, not human onboarding, and it stays in `docs/`. + +## Grounding — the part that makes it worth reading + +Everything in the report is a claim about a system that exists, so everything is checkable. +Check it. + +- **Read the source before writing the section.** Models in `models.py` for the data model, + `urls.py`/views for interfaces, the actual task/handler for a flow. Never reconstruct + behaviour from a ticket, a PR description, a commit message, or another doc — per + `CLAUDE.md`, if a doc contradicts the code, trust the code and flag the drift. +- **Point at the module, never the line.** Name the file, function, or component — + `ingestion/services.py`, "the `before_save` hook on `StagedEvent`" — so the reader knows + where to look. Never `services.py:88`. Line numbers are wrong within a week of anyone + touching the file, and a citation that's confidently wrong is worse than none: the reader + lands somewhere unrelated and stops trusting the rest. +- **Explain the idea, don't index the code.** The value is in what the module does and why, at + a level that survives refactoring. If a claim can only be supported by pointing at an exact + line, it's too fine-grained for a handoff report — the reader needs the shape, the code shows + the detail. +- **Say what you didn't verify.** "I couldn't find where this is cleaned up" is useful. A + confident guess in the same sentence style as verified facts is not. +- **Date it and name what it reflects** — a commit hash (`git rev-parse HEAD`), a release, or + just the date. A context-transfer doc is read long after it's written; the reader needs to + know how much drift to expect. +- **Prefer the system's own vocabulary.** If the code calls it a `StagedEvent`, don't call it a + "pending record" because it reads better. If it's a "source", don't call it a "feed" unless + the code does. + +## Structure + +Use the sections that carry weight; drop the rest and renumber contiguously. + +1. **What this is and who depends on it** — what the system does in plain terms, who the + callers are, what breaks for them if it's down. Two or three paragraphs, no diagrams. A + reader should be able to stop here and know whether they care. +2. **How it works** — a Mermaid diagram for every flow that earns one, not a representative + sample. Lead with the common case, then the paths people get wrong. Read path before write + path. See "Which flows get a diagram" below. +3. **Data model** — what the tables/collections are, what the non-obvious columns mean, what + nullable actually signifies. +4. **Interfaces** — endpoints, Celery tasks, management commands, webhooks. Who calls each one. +5. **Sharp edges** — the non-obvious behaviour that will bite. See below. +6. **Known gaps** — what's unresolved, undocumented, or actively suspicious, and what you'd + look at first. + +After each diagram, add a short "N things worth calling out" list for what the diagram can't +say: why it's shaped this way, what it costs, what it rules out, and where the obvious-looking +change is the wrong one. + +## Sharp edges + +This is the section that justifies the report, and the one only a current owner can write. It +is not a list of bugs — it's the knowledge that would otherwise be transferred by someone +losing a day. This repo's memory already holds candidates worth mining when writing about a +given subsystem (e.g. the `Event` PK being `uuid` not `id` breaking `Count("id")`, or +`INGEST_SHARD_COUNT` silently limiting a plain `ingest_events` run) — check for standing +gotchas before assuming you've found something new. + +Good entries: a field whose name lies about its contents; a sentinel value overloaded into a +normal column; ordering that looks incidental but isn't; a retry that's safe only because a +downstream call happens to be idempotent; a config that behaves differently in one environment +(dev vs. prod `DJANGO_ENV`, sharded vs. unsharded ingest); a "temporary" workaround load-bearing +enough that removing it breaks something distant. + +Each entry: what it is, why it's that way, what happens if someone "fixes" it. + +## Table shapes + +Describe what exists. Columns adapt to the domain — layer, owner, and consumer columns all +earn their place in different systems. + +| Table | Column | Meaning | Notes | +|---|---|---|---| +| `account` | `locked_until` | Null = not locked | Cleared on successful sign-in alongside the counter, in the sign-in handler (`auth.ts`). | + +| Endpoint | Auth | Called by | Description | +|---|---|---|---| +| `GET /v2/staff/thing` | `view_thing` | Admin settings screen | Returns the caller's groups only; 404s rather than 403s on a foreign id. | + +Name concrete values. "An expired sentinel" is not knowledge — `-infinity` is. If a column is +nullable and carries a magic value, state exactly what distinguishes null from the sentinel; +that ambiguity is a classic sharp edge. + +## If the report covers proposed changes instead + +Same structure, two swaps: add a New/Mod column to the tables so the reader can see what's +changing versus what's already there, and replace Sharp edges with a delivery sequence table +mapping each PR to the diagrams and rows it covers. Everything else — grounding, standalone +rule, diagram discipline — applies unchanged. + +## The standalone rule + +- No "companion to `.md`", no "per §3 of the plan", no local file paths as pointers. The + reader will not open them. Restate what they need inline, in a clause. (Naming a module is + different — that's a signpost for verifying, not a document the reader has to go read.) +- Number sections contiguously. Don't leave a §2→§4 gap advertising a missing piece. +- Every § reference must resolve inside the report. +- If the report is derived from a working doc (a design doc, a ticket thread), treat it as a + published copy. Regenerate when the source changes rather than patching, and expect the two + to diverge in structure. + +## Diagrams + +### Which flows get a diagram + +Diagram every flow that qualifies. Not one showcase diagram and prose for the rest — a +newcomer hits the uncommon paths too, and the retry, the failure, and the migration are exactly +the ones prose handles worst. + +A flow earns a diagram when any of these is true: + +- It crosses two or more components (view → DB → Celery task → worker). +- It branches in a way that changes the outcome (authorized vs. not, cache hit vs. miss, first + run vs. subsequent). +- Ordering matters and isn't obvious from the names. +- State persists between steps, or a step is only safe because an earlier one ran. +- You catch yourself writing "then… then… but if…". That sentence is a diagram you haven't + drawn yet. + +Skip it when the flow is one call and a return, when there's no branch and fewer than three +steps, or when the diagram would restate the sentence above it. Ten diagrams that each earn +their place is a good report; ten that include four trivial ones trains the reader to skip all +of them. + +Diagrams aren't confined to "How it works." A status lifecycle (`StagedEvent` states, a +`SourceRun` health level) belongs beside the data model; a decision tree explaining a sharp edge +belongs next to that sharp edge. Put the diagram where the question gets asked. + +### Don't draw the same diagram twice + +A redundant diagram costs more than a missing one. It pads the report, and it makes the reader +stop and work out whether two near-identical pictures differ in some way that matters. + +Before adding one, check it against what's already drawn: + +- **Same shape, different noun.** If "poll an ICS source" and "poll a scraper source" are the + same six steps with a different fetcher, draw it once and note what varies. +- **Happy path plus one branch.** A failure that diverges for a step or two is an `alt` block + inside the existing diagram, not a second diagram. +- **A subset.** If the read path is the write path with the last three steps removed, the write + diagram already showed it. +- **A table redrawn.** An `erDiagram` that restates the data-model table adds nothing. Keep + whichever one the reader will actually use, not both. + +Split when the shape diverges; merge when only the values do. A branch that changes which +participants are involved, or most of the steps, earns its own diagram. A branch that changes +one call or a label is an `alt`. + +### Picking the type + +Reaching for `sequenceDiagram` every time is the most common way these reports get harder to +read than they need to be. + +| What you're explaining | Type | +|---|---| +| Ordered interaction between components | `sequenceDiagram` | +| One entity moving through statuses over its lifetime | `stateDiagram-v2` | +| Branching logic, a decision tree, a dispatch table | `flowchart TD` | +| How tables/collections relate | `erDiagram` | +| What runs in parallel and what blocks | `flowchart LR` with subgraphs | + +If a "flow" is really "this record can be in one of six states and only some transitions are +legal" (e.g. `StagedEvent.status`), a sequence diagram will fight you the whole way. + +### Keeping them readable + +- One flow per diagram — when the flows genuinely differ in shape. A diagram covering both a + substantial read and a substantial write is two diagrams; one covering a happy path and its + one-step failure is still one. See "Don't draw the same diagram twice." +- Five participants is a lot. Middleware, context objects, and validators are not participants + — fold them into self-calls (`API->>API: check perms`). +- Watch nesting. `alt` inside `alt` inside `loop` renders wide and overflows a tracker's + comment column. Two levels is usually the limit. +- Autonumber so readers can say "step 7" instead of describing a line. + +### Mermaid syntax landmines + +Mermaid renders labels as HTML — which is why `
` works, and why these two break with +errors that don't point at the cause: + +- `;` is a statement separator. A semicolon in message or Note text truncates the statement; the + remainder parses as a new one and throws `Expecting 'SPACE', 'NEWLINE', 'create', 'box'...`. + Use a comma or dash. +- `<...>` is swallowed as an HTML tag. `col = ` silently loses the ``. Use + `(sentinel)`. + +Both are silent — one throws an error pointing nowhere near the cause, the other doesn't error +at all. Don't try to spot them by reading; run the two greps in "Before you publish." + +### Where it renders + +GitHub renders ```` ```mermaid ```` blocks natively — this matters here because handoff reports +land in `human-docs/` and get read on GitHub. VS Code's built-in Markdown preview does not — it +needs an extension, so don't promise it renders "everywhere." + +Don't build an HTML page or separate artifact just to make diagrams render; GitHub already +renders them in `human-docs/`. Build a standalone artifact only when the content is genuinely +interactive, and deliver it through its own channel rather than linking it from a report the +reader can't follow the link from. + +You usually can't verify rendering locally — `mmdc` is rarely installed. Say so and preview on +GitHub (or in the destination surface) before publishing. + +## Process + +1. **Establish who reads it and where it lands.** A newcomer needs the business context; an + integrating workstream needs the interfaces. It's landing in `human-docs/` per "Where the + finished report lands" above, not `docs/` — that decides emphasis, length, and structure. +2. **Read the system-of-record docs first**, per `CLAUDE.md`: root `AGENTS.md` → + `ARCHITECTURE.md` → `CODING_STYLE.md`, plus whichever per-directory `AGENTS.md` or + `docs/*.md` already touches this subsystem. Note where they're stale — flag drift rather + than propagating it. +3. **Read the code and note which modules matter as you go.** Models, urls/views, tasks. This + is most of the work, and skipping it is how reports become fiction. +4. **List every flow before drawing any of them.** Enumerate the paths through the system — + happy path, each failure, each branch, each background job — mark which qualify per "Which + flows get a diagram," then collapse the ones that share a shape. Deciding coverage up front + is what stops the report from diagramming the first flow and prosing the rest, and the + collapse pass is what stops the list from becoming twelve variations on one picture. +5. **Draft context → diagrams → tables → sharp edges → gaps.** Sharp edges accumulate while you + read; keep a running list from step 3. +6. **Reread as the newcomer.** Every unexplained noun is a gap. Every "obviously" is a sharp + edge you forgot to write down. +7. **Run the checks below**, then reread for the standalone rule — that one is invisible to + grep. +8. **Add the row to `human-docs/README.md`** before calling it done — an unlisted doc doesn't + count as written, per "Where the finished report lands." +9. **Have the current owner review before publishing.** They'll catch the confident-but-wrong + sentence, which is the one that does real damage. Don't commit or push it yourself unless + explicitly asked — surface it for review first. + +### Before you publish + +Two mechanical checks worth running literally, because both failures are silent and neither is +reliably caught by reading. Set `F` to the file. + +```bash +# 1. Semicolons inside a diagram — ';' ends the statement, and the rest of the +# line parses as a new one. Error message points nowhere near the cause. +awk '/^```mermaid/{b=1;next} /^```/{b=0} b{print NR": "$0}' "$F" | grep ';' + +# 2. Stray angle brackets — labels render as HTML, so is swallowed +# with no error at all. Strips
and arrows first; any hit is real. +awk '/^```mermaid/{b=1;next} /^```/{b=0} b{print NR": "$0}' "$F" \ + | sed -e 's|
||g' -e 's|[-=][-.=]*>>*||g' | grep '[<>]' +``` + +Both should print nothing. Then check by eye, in rough order of how much damage each does: + +1. **Does it stand alone?** Any pointer to another doc, any § that resolves nowhere, any + section-number gap. This is the failure that wastes the most reader time and no tool sees + it. +2. **Any line numbers?** `grep -nE '\.[a-z]+:[0-9]+' "$F"` — name the module instead. +3. **Do the diagrams cover the flows that matter, without duplicating each other?** Both + directions are failures. +4. **Are the sharp edges the real ones?** If they read like a changelog rather than hard-won + knowledge, the section isn't done. +5. **Is it in `human-docs/README.md`?** + +None of this tells you the report is correct — that's what owner review in step 9 is for, and +it's the check that matters most. diff --git a/backendServer/devtools/monitoring.py b/backendServer/devtools/monitoring.py index cd616d8..82c22b5 100644 --- a/backendServer/devtools/monitoring.py +++ b/backendServer/devtools/monitoring.py @@ -10,6 +10,7 @@ """ import os +from datetime import datetime, timedelta from django.conf import settings from django.db import OperationalError, ProgrammingError @@ -43,8 +44,13 @@ # Sort order for the health column: most severe first, `inactive` last since # those rows are intentionally excluded from alerting. `unknown` sits between # `warn` and `ok`: absence of signal is worth surfacing above a healthy row, -# but it is not evidence of a problem the way `warn` is. -_HEALTH_RANK = {"error": 0, "warn": 1, "unknown": 2, "ok": 3, "inactive": 4} +# but it is not evidence of a problem the way `warn` is. `push` ranks +# alongside `inactive` (same value, not just "also near the end") — a +# direct/push source is excluded from staleness alerting for the identical +# reason an inactive one is: nothing ever polls it, so "stale" doesn't apply. +# This dict is read by both `_worse()` and the results sort below; a level +# missing from it is a `KeyError` that takes the whole page down (ticket 40.2). +_HEALTH_RANK = {"error": 0, "warn": 1, "unknown": 2, "ok": 3, "inactive": 4, "push": 4} # How readable `ingestion_sourcerun` is on a target database. A three-state # value rather than a bool because the three failure causes need different @@ -251,18 +257,31 @@ def source_health(source_row, recent_runs, now): """Classify a source's health from plain data — no DB access. `source_row` is one row as produced by `_source_rows` (dict with `active`, - `poll_interval_hours`, `last_polled_dt` and `created_at` — real datetimes, - not the ISO strings on the public row shape — `raw_count`, `funnel`, - `published_all_time`). `recent_runs` is a list of dicts (most-recent-first) - with `status`, `finished_at`, `error_message` for one source. `now` is a - datetime compared against `last_polled_dt` / `created_at`. - - Returns {"level": "ok" | "unknown" | "warn" | "error" | "inactive", - "reasons": [str, ...]}. Every applicable rule is evaluated and its reason - collected; the most severe level (error > warn > unknown > ok) wins. - Inactive sources are excluded from alerting entirely and always report - "inactive", never "error". + `source_type`, `poll_interval_hours`, `last_polled_dt` and `created_at` — + real datetimes, not the ISO strings on the public row shape — `raw_count`, + `funnel`, `published_all_time`). `recent_runs` is a list of dicts + (most-recent-first) with `status`, `finished_at`, `error_message` for one + source. `now` is a datetime compared against `last_polled_dt` / + `created_at`. + + Returns {"level": "ok" | "unknown" | "warn" | "error" | "inactive" | + "push", "reasons": [str, ...]}. Every applicable rule is evaluated and its + reason collected; the most severe level (error > warn > unknown > ok) + wins. Inactive sources are excluded from alerting entirely and always + report "inactive", never "error". Direct/push sources (ticket 40.2) are + excluded the same way but report the distinct "push" level instead, since + `inactive` reads as "broken" for a source that is working exactly as + designed — pushed to by users, never polled. """ + # Direct-submission sources (ingestion/views.py:76-80) are deliberately + # created `active=False` because they're pushed to by users, not polled — + # flipping that to `active=True` would make the staleness rules below + # false-alarm on a source that was never supposed to be polled in the + # first place. Checked before the `active` gate so a direct source never + # falls into the `inactive` branch and reads as "broken". + if source_row.get("source_type") == "direct": + return {"level": "push", "reasons": ["direct/push source — not polled"]} + if not source_row["active"]: return {"level": "inactive", "reasons": []} @@ -431,6 +450,39 @@ def _source_rows(db, start, end, source_type_filter, runs_state=None): for row in Event.objects.using(db).values("source_name").annotate(n=Count("pk")) } + # Direct-submission sources never match `published_all_time_by_name` + # above, because the exact-match lookup on `EventSource.name` doesn't + # match what `Event.source_name` actually holds for these rows. Two + # different code paths stamp two different shapes: the per-submission + # path (ingestion/services.py:220-222) stamps a per-organizer string — + # f"Direct submission by {organizer}" (or "...by host" with no organizer) + # — while the bulk sweep path (ingestion/services.py:80-83) instead + # stamps the literal `EventSource.name` ("Direct Host Submission"). Ticket + # 40.1: count both, so the all-time safety net (36.6, above) isn't + # permanently dead for this source. One additional query, gated on + # whether this call has any direct sources at all — `collector_summary` + # excludes direct sources entirely (never pays it) and + # `broadcast_inbound_summary` pays it once per call, not once per source. + # If `ingestion/services.py`'s "Direct submission by" prefix ever changes, + # update it here too. + # + # Caveat worth naming: this is one total shared by every direct row, not a + # per-source breakdown — the prefix carries the *organizer*, not the source, + # so it cannot be attributed back to a specific `EventSource`. Correct today + # because `ingestion/views.py` get_or_creates exactly one direct source; if a + # second one is ever added, both rows would report this same combined count. + direct_source_names = {s["name"] for s in sources if s["source_type"] == "direct"} + published_all_time_direct = 0 + if direct_source_names: + published_all_time_direct = ( + Event.objects.using(db) + .filter( + Q(source_name__startswith="Direct submission by") + | Q(source_name__in=direct_source_names) + ) + .count() + ) + no_staged_counts = { row["source"]: row["n"] for row in RawEvent.objects.using(db) @@ -478,7 +530,20 @@ def _source_rows(db, start, end, source_type_filter, runs_state=None): # keying off status alone would report a genuinely-live event as # unpublished for as long as that gap lasts (on prod, indefinitely — # `auto_publish_safe_events` early-returns when nothing is pending). - published=Count("id", filter=Q(published_event__isnull=False)), + # + # 40.4: also scoped to status in ("approved", "published") — not + # just any non-null `published_event`. `ingest_direct_submission` + # now keeps `published_event` pointed at a previously-published + # Event on its duplicate/held/no_town early returns too, so it can + # re-point a terminal row at a live Event without orphaning it. + # Those rows are still correctly credited to their own bucket + # (`duplicate`/`held_for_review`/`no_town`); without this status + # filter they'd *also* count here, double-counting against `raw`. + # This keeps the original intent — "published means a live event + # this row is responsible for" — rather than "any non-null FK". + published=Count( + "id", filter=Q(published_event__isnull=False, status__in=["approved", "published"]) + ), # The residual approved bucket: approved but not yet published. # Disjoint from `published` above, which is what keeps the funnel # reconciliation exact. @@ -534,7 +599,14 @@ def _source_rows(db, start, end, source_type_filter, runs_state=None): no_town_count = funnel_staged.get("skipped_no_town", 0) no_town_note = _no_town_note(no_town_count) published_in_window = funnel_staged.get("published", 0) - published_all_time = published_all_time_by_name.get(source["name"], 0) + # Direct sources use the prefix-or-literal count computed once above + # (ticket 40.1) — the exact-match dict never has anything for them. + # Collector sources keep the original exact-match lookup, unchanged. + published_all_time = ( + published_all_time_direct + if source["source_type"] == "direct" + else published_all_time_by_name.get(source["name"], 0) + ) # Only computed for a zero-in-window row — mirrors `raw_zero_note`: # a healthy funnel has nothing to disambiguate. published_note = _published_note(published_all_time) if published_in_window == 0 else None @@ -574,11 +646,13 @@ def _source_rows(db, start, end, source_type_filter, runs_state=None): "no_town_note": no_town_note, # All-time, un-windowed count of live Events attributed to this # source by `Event.source_name` — see the `published_all_time_by_name` - # query above. `Event` has no source FK or creation timestamp, so - # this cannot be windowed the way `raw_all_time` is; it is the - # honest "has this source ever gotten anything published, ever" - # signal for a source whose old StagedEvent anchors no longer - # survive the funnel window (ticket 36.6). + # / `published_all_time_direct` queries above. `Event` has no + # source FK or creation timestamp, so this cannot be windowed the + # way `raw_all_time` is; it is the honest "has this source ever + # gotten anything published, ever" signal for a source whose old + # StagedEvent anchors no longer survive the funnel window (ticket + # 36.6), and for direct sources whose `source_name` never matches + # `EventSource.name` exactly (ticket 40.1). "published_all_time": published_all_time, # Only set when `published == 0` in-window — see `_published_note`. # Renders as a tooltip/subtext on the `published` funnel cell so a @@ -672,13 +746,100 @@ def broadcast_outbound_summary(db: str, start, end) -> dict: return {"by_status": by_status, "total": total, "targets_by_status": targets_by_status} -def _drilldown_source(db, key, start, end, limit): - raw_events = list( +def summarize_sources(rows: list[dict], now=None) -> dict: + """Aggregate already-built collector/inbound rows into the monitor page's + KPI tile numbers (ticket 40.6). `rows` is the concatenation of + `collector_summary()` + `broadcast_inbound_summary()`'s output — this + function issues no queries of its own, only reductions over fields those + rows already carry (`funnel`, `health.level`, `last_polled`, + `latest_raw_created_at`). `monitor()` already pays for those round trips, + some over a WAN to Neon on the `prod_readonly` path; adding a fresh query + here to answer a question the rows already know the answer to would + double that cost for no reason. + + `now` defaults to `timezone.now()`; pass it explicitly for deterministic + tests, mirroring `source_health`. `last_polled` / `latest_raw_created_at` + are ISO strings parsed with `datetime.fromisoformat` and compared as real + datetimes, never as strings: two rows' ISO strings can carry different + UTC offsets (e.g. a naive local run vs. a `+00:00` Postgres value), and a + lexicographic string comparison would silently pick the wrong "newest". + + Returns: + { + "funnel": {"raw": int, "published": int, "held_for_review": int, + "duplicate": int, "no_town": int}, + "health": {"error": int, "warn": int, "unknown": int, "ok": int, + "inactive": int, "push": int}, + "freshness": {"newest_raw_created_at": str | None, + "polled_last_24h": int}, + } + """ + if now is None: + now = timezone.now() + + funnel_totals = dict.fromkeys( + ("raw", "published", "held_for_review", "duplicate", "no_town"), 0 + ) + health_counts = dict.fromkeys(_HEALTH_RANK, 0) + newest_raw_dt = None + polled_last_24h = 0 + + for row in rows: + row_funnel = row.get("funnel") or {} + for key in funnel_totals: + funnel_totals[key] += row_funnel.get(key, 0) + + level = (row.get("health") or {}).get("level") + if level in health_counts: + health_counts[level] += 1 + + latest_raw = row.get("latest_raw_created_at") + if latest_raw: + latest_raw_dt = datetime.fromisoformat(latest_raw) + if newest_raw_dt is None or latest_raw_dt > newest_raw_dt: + newest_raw_dt = latest_raw_dt + + last_polled = row.get("last_polled") + if last_polled: + delta = now - datetime.fromisoformat(last_polled) + if timedelta(0) <= delta <= timedelta(hours=24): + polled_last_24h += 1 + + return { + "funnel": funnel_totals, + "health": health_counts, + "freshness": { + "newest_raw_created_at": newest_raw_dt.isoformat() if newest_raw_dt else None, + "polled_last_24h": polled_last_24h, + }, + } + + +def _drilldown_source(db, kind, key, start, end, limit, offset=0): + """Rows for one source (`key` is a numeric id), or — ticket 40.5 — every + source of `kind`'s type when `key` is `None`. The all-sources predicates + mirror `collector_summary` / `broadcast_inbound_summary` exactly + (`~Q(source_type="direct")` / `Q(source_type="direct")`), just applied via + the `source__` join since this queries `RawEvent`, not `EventSource`. + + `select_related("source")` alongside the existing `select_related("staged")` + keeps the all-sources mode from N+1ing on `source_name` — one row per raw + event either way, just a wider select. + """ + qs = ( RawEvent.objects.using(db) - .filter(source_id=key, created_at__gte=start, created_at__lt=end) - .select_related("staged") - .order_by("-created_at")[:limit] + .filter(created_at__gte=start, created_at__lt=end) + .select_related("staged", "source") ) + if key is not None: + qs = qs.filter(source_id=key) + elif kind == "inbound": + qs = qs.filter(Q(source__source_type="direct")) + else: + qs = qs.filter(~Q(source__source_type="direct")) + + total = qs.count() + raw_events = list(qs.order_by("-created_at")[offset : offset + limit]) rows = [] for raw in raw_events: staged = getattr(raw, "staged", None) @@ -690,19 +851,23 @@ def _drilldown_source(db, key, start, end, limit): "processed": raw.processed, "staged_status": staged.status if staged else None, "published": bool(staged and staged.published_event_id), + "source_name": raw.source.name, } ) - return rows + return rows, total -def _drilldown_outbound(db, key, start, end, limit): +def _drilldown_outbound(db, key, start, end, limit, offset=0): submissions_qs = BroadcastSubmission.objects.using(db).filter( created_at__gte=start, created_at__lt=end ) if key: submissions_qs = submissions_qs.filter(targets__site_key=key).distinct() - submissions = list(submissions_qs.order_by("-created_at").prefetch_related("targets")[:limit]) + total = submissions_qs.count() + submissions = list( + submissions_qs.order_by("-created_at").prefetch_related("targets")[offset : offset + limit] + ) rows = [] for submission in submissions: @@ -719,20 +884,28 @@ def _drilldown_outbound(db, key, start, end, limit): ], } ) - return rows + return rows, total -def _drilldown_runs(db, key, start, end, limit, runs_state=None): +def _drilldown_runs(db, key, start, end, limit, offset=0, runs_state=None): """Recent SourceRun rows for one source — "what has this source been doing", not windowed by `start`/`end` (a source's run history matters regardless of the funnel window currently selected). `start`/`end` are accepted for signature symmetry with the other drilldown helpers only. + + Run history is inherently per-source — there's no sensible "all sources" + fan-out for "what has this source been doing" — so a missing `key` + degrades to empty rather than querying every source's runs (ticket 40.5). """ + if key is None: + return [], 0 if runs_state is None: runs_state = resolve_source_runs_state(db) if runs_state != RUNS_AVAILABLE: - return [] - runs = SourceRun.objects.using(db).filter(source_id=key).order_by("-started_at")[:limit] + return [], 0 + qs = SourceRun.objects.using(db).filter(source_id=key).order_by("-started_at") + total = qs.count() + runs = qs[offset : offset + limit] return [ { "started_at": _iso(run.started_at), @@ -745,17 +918,29 @@ def _drilldown_runs(db, key, start, end, limit, runs_state=None): "error_message": run.error_message, } for run in runs - ] - - -def drilldown(db: str, kind: str, key, start, end, limit: int = 100, runs_state=None) -> list[dict]: + ], total + + +def drilldown( + db: str, kind: str, key, start, end, limit: int = 100, offset: int = 0, runs_state=None +) -> tuple[list[dict], int]: + """Returns `(rows, total)`, where `total` is the unpaginated count of rows + matching the same filters/window — the client renders "showing X-Y of N" + from it (ticket 40.5). `offset` is clamped defensively here (not just at + the view layer) since this is a public function other callers can reach + directly. Every early-return below must keep the `(list, int)` shape — + degrading to a bare `[]` is exactly the kind of mismatch that took the + whole monitor page down before (see `_HEALTH_RANK`'s docstring for the + same lesson learned the hard way). + """ + offset = max(offset, 0) if not _db_ok(db): - return [] + return [], 0 if kind in ("collector", "inbound"): - return _drilldown_source(db, key, start, end, limit) + return _drilldown_source(db, kind, key, start, end, limit, offset=offset) if kind == "outbound": - return _drilldown_outbound(db, key, start, end, limit) + return _drilldown_outbound(db, key, start, end, limit, offset=offset) if kind == "runs": - return _drilldown_runs(db, key, start, end, limit, runs_state=runs_state) - return [] + return _drilldown_runs(db, key, start, end, limit, offset=offset, runs_state=runs_state) + return [], 0 diff --git a/backendServer/devtools/templates/devtools/monitor.html b/backendServer/devtools/templates/devtools/monitor.html index 785071c..febd349 100644 --- a/backendServer/devtools/templates/devtools/monitor.html +++ b/backendServer/devtools/templates/devtools/monitor.html @@ -8,6 +8,7 @@ .monitor { max-width: 1100px; } .monitor h1 { font-size: 1.6rem; font-weight: 700; margin-bottom: 0.5rem; } .monitor h2 { font-size: 1.15rem; font-weight: 700; margin: 2rem 0 0.75rem; } + .monitor h3 { font-size: 0.95rem; font-weight: 700; margin: 1.25rem 0 0.5rem; } .monitor .description { color: #6b7280; margin: 0.5rem 0 1.5rem; line-height: 1.65; } .monitor .controls { @@ -45,8 +46,6 @@ .monitor table { width: 100%; border-collapse: collapse; font-size: 0.85rem; margin-bottom: 0.5rem; } .monitor th, .monitor td { border: 1px solid #e5e7eb; padding: 0.4rem 0.6rem; text-align: left; } .monitor th { background: #f3f4f6; font-weight: 700; } - .monitor tr.source-row { cursor: pointer; } - .monitor tr.source-row:hover { background: #f9fafb; } .monitor tr.source-row.inactive { color: #9ca3af; } .monitor td.num { text-align: right; font-variant-numeric: tabular-nums; } .monitor td.num.zero { color: #b6bcc6; } @@ -57,6 +56,7 @@ cell, ticket 36.6) — same convention, different funnel columns. */ .monitor .zero-note { font-weight: 400; font-size: 0.78rem; color: #92400e; white-space: nowrap; } .monitor .empty-note { color: #6b7280; font-size: 0.85rem; margin: 0.5rem 0 1rem; } + .monitor .loading { color: #6b7280; font-size: 0.8rem; } .monitor .notice { background: #fef3c7; border: 1px solid #fcd34d; @@ -69,14 +69,33 @@ } .monitor .notice code { background: rgba(146, 64, 14, 0.1); padding: 0.05rem 0.25rem; border-radius: 3px; } - .monitor .summary-stats { display: flex; gap: 1.5rem; flex-wrap: wrap; margin-bottom: 1rem; } - .monitor .stat { background: #f9fafb; border: 1px solid #e5e7eb; border-radius: 6px; padding: 0.5rem 0.9rem; font-size: 0.85rem; } - .monitor .stat strong { display: block; font-size: 1.15rem; } + /* Ticket 40.6: KPI tiles — everything on this page used to require a row + click and a tab click to see; these four groups are the "at a glance" + answer, computed server-side by `summarize_sources` with zero queries + of its own (see monitoring.py). */ + .monitor .tiles { display: flex; flex-direction: column; gap: 1.1rem; margin-bottom: 2rem; } + .monitor .tile-group h3 { + font-size: 0.72rem; + font-weight: 700; + text-transform: uppercase; + letter-spacing: 0.05em; + color: #6b7280; + margin: 0 0 0.5rem; + } + .monitor .tile-row { display: flex; gap: 0.6rem; flex-wrap: wrap; } + .monitor .tile { + background: #f9fafb; + border: 1px solid #e5e7eb; + border-radius: 6px; + padding: 0.5rem 0.9rem; + min-width: 5.5rem; + } + .monitor .tile .tile-value { display: block; font-size: 1.25rem; font-weight: 700; font-variant-numeric: tabular-nums; } + .monitor .tile .tile-value.zero { color: #b6bcc6; } + .monitor .tile .tile-label { font-size: 0.72rem; color: #6b7280; } + .monitor .tile.health-tile { display: flex; align-items: center; padding: 0.55rem 0.9rem; } + .monitor .tile.health-tile .badge { font-size: 0.85rem; padding: 0.15rem 0.5rem; margin-right: 0.4rem; } - .monitor .drilldown-row td { background: #fafafa; padding: 0; } - .monitor .drilldown-wrap { padding: 0.75rem 1rem; } - .monitor .drilldown-wrap .loading { color: #6b7280; font-size: 0.8rem; } - .monitor .drilldown-wrap table { margin-bottom: 0; } .monitor .badge { display: inline-block; padding: 0.1rem 0.4rem; border-radius: 3px; font-size: 0.75rem; background: #e5e7eb; } .monitor .badge.published, .monitor .badge.succeeded, .monitor .badge.done, .monitor .badge.approved, .monitor .badge.ok { background: #dcfce7; color: #166534; } .monitor .badge.rejected, .monitor .badge.failed, .monitor .badge.canceled, .monitor .badge.refused { background: #fee2e2; color: #991b1b; } @@ -90,23 +109,34 @@ badge of the four, so it never competes with error/warn for attention. */ .monitor .badge.health-unknown { background: #f3f4f6; color: #6b7280; } .monitor .badge.health-inactive { background: #e5e7eb; color: #6b7280; } - - .monitor .tabs { display: flex; gap: 0.25rem; border-bottom: 1px solid #e5e7eb; margin-bottom: 0.75rem; } - .monitor .tab-button { - padding: 0.35rem 0.75rem; + /* `push` (ticket 40.2): direct/inbound sources are pushed to, never + polled — same muted treatment as `inactive` so neither reads as broken. */ + .monitor .badge.health-push { background: #e5e7eb; color: #6b7280; } + + /* Ticket 40.6: persistent, paginated "Recent events" panel per section — + replaces the old click-a-row-to-expand drilldown as the primary path to + seeing events. */ + .monitor .events-panel { margin: 0.5rem 0 1.75rem; } + .monitor .events-panel table { margin-bottom: 0; } + .monitor .events-panel .pager { + display: flex; + align-items: center; + gap: 0.75rem; + margin-top: 0.5rem; font-size: 0.8rem; - font-weight: 600; - background: none; - border: 1px solid transparent; - border-bottom: none; - border-radius: 4px 4px 0 0; - cursor: pointer; color: #6b7280; } - .monitor .tab-button:hover { color: #111827; } - .monitor .tab-button.active { color: #111827; background: #fff; border-color: #e5e7eb; } - .monitor .tab-panel { display: none; } - .monitor .tab-panel.active { display: block; } + .monitor .events-panel .pager button { + padding: 0.25rem 0.7rem; + border: 1px solid #d1d5db; + border-radius: 4px; + background: #fff; + cursor: pointer; + font-size: 0.8rem; + color: #374151; + } + .monitor .events-panel .pager button:hover:not(:disabled) { background: #f9fafb; border-color: #9ca3af; } + .monitor .events-panel .pager button:disabled { color: #9ca3af; cursor: not-allowed; background: #f9fafb; } .monitor .btn-probe { font-size: 0.75rem; @@ -121,6 +151,64 @@ .monitor .btn-probe:hover { background: #f9fafb; border-color: #9ca3af; } .monitor .btn-probe:disabled { background: #f9fafb; color: #9ca3af; cursor: not-allowed; } + /* Probe modal: the probe is a real diagnostic (streams SSE from + /devtools/probe) kept reachable but out of the default page path — a + per-source button opens it here instead of an inline click-to-expand + row. */ + .monitor .modal-overlay { + display: none; + position: fixed; + inset: 0; + background: rgba(17, 24, 39, 0.5); + align-items: center; + justify-content: center; + z-index: 1000; + padding: 1.5rem; + } + .monitor .modal-overlay.open { display: flex; } + .monitor .modal { + background: #fff; + border-radius: 8px; + width: 100%; + max-width: 720px; + max-height: 85vh; + overflow-y: auto; + padding: 1rem 1.25rem 1.25rem; + box-shadow: 0 10px 40px rgba(0, 0, 0, 0.25); + } + .monitor .modal-header { display: flex; justify-content: space-between; align-items: center; margin-bottom: 0.75rem; } + .monitor .modal-header h3 { margin: 0; font-size: 1rem; } + .monitor .modal-close { + background: none; + border: none; + font-size: 1.4rem; + line-height: 1; + cursor: pointer; + color: #6b7280; + padding: 0 0.25rem; + } + .monitor .modal-close:hover { color: #111827; } + + /* Tabs inside the Probe modal (Probe | Runs). Scoped to the modal on + purpose: the dashboard itself is deliberately tab-free after the 40.6 + rebuild, and these styles should not invite tabs back onto the page. */ + .monitor .modal-tabs { display: flex; gap: 0.25rem; border-bottom: 1px solid #e5e7eb; margin-bottom: 0.75rem; } + .monitor .tab-button { + padding: 0.35rem 0.75rem; + font-size: 0.8rem; + font-weight: 600; + background: none; + border: 1px solid transparent; + border-bottom: none; + border-radius: 4px 4px 0 0; + cursor: pointer; + color: #6b7280; + } + .monitor .tab-button:hover { color: #111827; } + .monitor .tab-button.active { color: #111827; background: #fff; border-color: #e5e7eb; } + .monitor .tab-panel { display: none; } + .monitor .tab-panel.active { display: block; } + .monitor .probe-caveat { font-size: 0.75rem; color: #92400e; @@ -182,8 +270,9 @@

Ingestion & Broadcast Monitor

- Per-collector and per-broadcast event counts for the selected window. Click a row to see the - underlying raw events (or broadcast submissions). + Summary tiles, then per-collector and per-broadcast event counts, for the selected window — + everything below loads with zero clicks. Use a source's Probe button to dry-run its + fetch/parse/write path.

@@ -215,7 +304,8 @@

Ingestion & Broadcast Monitor

Run history unavailable: ingestion_sourcerun doesn’t exist here. This database is behind on migrations — it hasn’t had - 0014_sourcerun applied. The Runs tab is empty and health levels fall back to + 0014_sourcerun applied. The Runs tab (in a source’s Probe dialog) is + empty and health levels fall back to last-polled staleness only. Everything else on this page is accurate, and the probe still works — it never reads SourceRun.

@@ -225,12 +315,64 @@

Ingestion & Broadcast Monitor

The table exists, but this connection’s role can’t read it — ALTER DEFAULT PRIVILEGES doesn’t follow tables created by a different role, so a newly migrated table needs its grant re-run: - GRANT SELECT ON ingestion_sourcerun TO monitor_readonly;. The Runs tab is - empty and health levels fall back to last-polled staleness only. Everything else on this - page is accurate. + GRANT SELECT ON ingestion_sourcerun TO monitor_readonly;. The Runs tab (in a + source’s Probe dialog) is empty and health levels fall back to last-polled staleness + only. Everything else on this page is accurate.

{% endif %} +
+
+

Funnel (window)

+
+
{{ summary.funnel.raw }}Raw
+
{{ summary.funnel.published }}Published
+
{{ summary.funnel.held_for_review }}Held for review
+
{{ summary.funnel.duplicate }}Duplicate
+
{{ summary.funnel.no_town }}No town
+
+
+ +
+

Health rollup

+
+
{{ summary.health.error }}error
+
{{ summary.health.warn }}warn
+
{{ summary.health.unknown }}unknown
+
{{ summary.health.ok }}ok
+
{{ summary.health.inactive }}inactive
+
{{ summary.health.push }}push
+
+
+ +
+

Outbound broadcast

+
+
{{ outbound.total|default:0 }}Submissions
+ {% for status, count in outbound.by_status.items %} +
{{ count }}{{ status }}
+ {% endfor %} + {% for status, count in outbound.targets_by_status.items %} +
{{ count }}targets: {{ status }}
+ {% endfor %} +
+
+ +
+

Freshness

+
+
{{ summary.freshness.newest_raw_created_at|default:"—" }}Newest raw event
+
{{ summary.freshness.polled_last_24h }}Sources polled (24h)
+
+
+
+ +

+ Sources below are sorted worst-first by health (error → warn → unknown → ok + → inactive/push) — _source_rows already sorts them that way, so this + page doesn't re-sort client-side. Recent events for each section paginate beneath its table. +

+

Collectors

{% if collectors %} @@ -256,7 +398,7 @@

Collectors

{% for c in collectors %} - + @@ -279,9 +421,8 @@

Collectors

- + - {% endfor %}
{{ c.health.level }} {{ c.name }} {{ c.source_type }} {{ c.funnel.approved }} {{ c.funnel.no_town }}{% if c.no_town_note %} — {{ c.no_town_note }}{% endif %} {{ c.funnel.published }}{% if c.published_note %} — {{ c.published_note }}{% endif %}
@@ -289,6 +430,16 @@

Collectors

No collectors found for this window/database.

{% endif %} +
+

Recent events

+
Loading…
+
+ + Showing 0–0 of 0 + +
+
+

Broadcast — inbound (direct submissions)

{% if inbound %} @@ -313,7 +464,7 @@

Broadcast — inbound (direct submissions)

{% for c in inbound %} - + @@ -332,9 +483,8 @@

Broadcast — inbound (direct submissions)

- + - {% endfor %}
{{ c.health.level }} {{ c.name }} {{ c.active|yesno:"yes,no" }} {{ c.funnel.approved }} {{ c.funnel.no_town }}{% if c.no_town_note %} — {{ c.no_town_note }}{% endif %} {{ c.funnel.published }}{% if c.published_note %} — {{ c.published_note }}{% endif %}
@@ -342,26 +492,36 @@

Broadcast — inbound (direct submissions)

No direct/inbound submissions for this window/database.

{% endif %} +
+

Recent events

+
Loading…
+
+ + Showing 0–0 of 0 + +
+
+

Broadcast — outbound (syndication)

-
-
{{ outbound.total|default:0 }}submissions
- {% for status, count in outbound.by_status.items %} -
{{ count }}{{ status }}
- {% endfor %} +
+

Recent submissions

+
Loading…
+
+ + Showing 0–0 of 0 + +
-
- {% for status, count in outbound.targets_by_status.items %} -
{{ count }}targets: {{ status }}
- {% endfor %} +
+ + + + diff --git a/backendServer/ingestion/tests/fixtures/eventbritepittsboro.html b/backendServer/ingestion/tests/fixtures/eventbritepittsboro.html new file mode 100644 index 0000000..4ba7fcc --- /dev/null +++ b/backendServer/ingestion/tests/fixtures/eventbritepittsboro.html @@ -0,0 +1,14 @@ + + +Pittsboro, NC Events | Eventbrite + +
+

Trimmed real sample of eventbrite.com/d/nc--pittsboro/events/. + The visible DOM is elided; the scraper reads window.__SERVER_DATA__ below. + One event has been date-shifted to 2020 (id 9999999999) to exercise the + past-event filter; all other fields are unmodified real API data captured + 2026-07-31.

+
+ + + diff --git a/backendServer/ingestion/tests/fixtures/eventbriteraleigh.html b/backendServer/ingestion/tests/fixtures/eventbriteraleigh.html new file mode 100644 index 0000000..e4e87f5 --- /dev/null +++ b/backendServer/ingestion/tests/fixtures/eventbriteraleigh.html @@ -0,0 +1,14 @@ + + +Raleigh, NC Events | Eventbrite + +
+

Trimmed real sample of eventbrite.com/d/nc--raleigh/events/. + The visible DOM is elided; the scraper reads window.__SERVER_DATA__ below. + One event has been date-shifted to 2020 (id 9999999999) to exercise the + past-event filter; all other fields are unmodified real API data captured + 2026-07-31.

+
+ + + diff --git a/backendServer/ingestion/tests/fixtures/morrisvillechamber.html b/backendServer/ingestion/tests/fixtures/morrisvillechamber.html new file mode 100644 index 0000000..fae5cec --- /dev/null +++ b/backendServer/ingestion/tests/fixtures/morrisvillechamber.html @@ -0,0 +1,116 @@ +{ + "success": true, + "data": { + "memberEventsEnabled": true, + "events": [ + { + "activityKey": "EXCq5cP0tOLp7S7CEjtAWg2", + "admission": null, + "eventUrl": null, + "eventDetailUrl": "https://morrisvillechamber.org/event-detail?e=EXCq5cP0tOLp7S7CEjtAWg2", + "eventTypeCode": "CHMBR", + "eventName": "Bambino's Pizza Grand Opening Ribbon Cutting Celebration Bambino's Pizza", + "eventDescription": "Join us as we celebrate the Grand Opening and Ribbon Cutting of one of Morrisville's newest local restaurants, Bambino's Pizza!", + "eventFullDescription": "

Join us as we celebrate the Grand Opening and Ribbon Cutting of one of Morrisville's newest local restaurants, Bambino's Pizza!

", + "startDateTime": "2026-08-04T20:00:00", + "endDateTime": "2026-08-04T21:00:00", + "contactTypeCode": "In Person", + "address": { + "name": null, + "street1": "4129 Davis Drive", + "street2": null, + "city": "Morrisville", + "zip": "27560", + "stateCode": "NC" + }, + "noEndTime": false, + "noTimes": false + }, + { + "activityKey": "vJYvcnpYRey8nxDhxbNSVA2", + "admission": null, + "eventUrl": "https://meet.google.com/dzn-qgvq-ngr?authuser=0", + "eventDetailUrl": "https://morrisvillechamber.org/event-detail?e=vJYvcnpYRey8nxDhxbNSVA2", + "eventTypeCode": "CHMBR", + "eventName": "Launch Morrisville Info Session (Virtual)", + "eventDescription": "Join us for an information session for the Launch Morrisville program. This is a business development program designed to support aspiring entrepreneurs, startups, and early-stage business owners through education, mentorship, networking, and community connections.", + "eventFullDescription": "

READY TO GROW YOUR BUSINESS?

Join us for an information session for the Launch Morrisville program.

", + "startDateTime": "2026-08-05T16:00:00", + "endDateTime": "2026-08-05T17:00:00", + "contactTypeCode": "Online", + "address": null, + "noEndTime": false, + "noTimes": false + }, + { + "activityKey": "WRWibLYsKyDSawVZb8o7Qw2", + "admission": null, + "eventUrl": null, + "eventDetailUrl": "https://morrisvillechamber.org/event-detail?e=WRWibLYsKyDSawVZb8o7Qw2", + "eventTypeCode": "CHMBR", + "eventName": "Coffee & Connections - Hosted by Sonesta Select", + "eventDescription": "Start your morning with meaningful conversations and valuable new connections at Coffee & Connections, hosted by Sonesta Select in Morrisville! This fan-favorite Morrisville Chamber networking event is designed to help you build relationships, strengthen your network, and grow your business in a welcoming and engaging environment.", + "eventFullDescription": "

\u2615Coffee & Connections at Sonesta Select in Morrisville\u2615

", + "startDateTime": "2026-08-06T12:00:00", + "endDateTime": "2026-08-06T13:30:00", + "contactTypeCode": "In Person", + "address": { + "name": null, + "street1": "2001 Hospitality Court", + "street2": null, + "city": "Morrisville", + "zip": "27560", + "stateCode": "NC" + }, + "noEndTime": false, + "noTimes": false + }, + { + "activityKey": "BLFUedylzt7PeGE34K8ffA2", + "admission": null, + "eventUrl": null, + "eventDetailUrl": "https://morrisvillechamber.org/event-detail?e=BLFUedylzt7PeGE34K8ffA2", + "eventTypeCode": "CHMBR", + "eventName": "*CANCELLED* Cregger Showroom Ribbon-Cutting After Hours Social", + "eventDescription": "*THIS EVENT HAS BEEN CANCELLED UNTIL FURTHER NOTICE* Join the Morrisville Chamber of Commerce as we celebrate the grand opening of Cregger Showrooms with an official Ribbon Cutting Ceremony on Thursday, August 27, at 4:30 PM!", + "eventFullDescription": "

*CANCELLED UNTIL FURTHER NOTICE*

", + "startDateTime": "2026-08-27T20:30:00", + "endDateTime": "2026-08-27T22:00:00", + "contactTypeCode": "In Person", + "address": { + "name": null, + "street1": "1234 Cregger Way", + "street2": null, + "city": "Morrisville", + "zip": "27560", + "stateCode": "NC" + }, + "noEndTime": false, + "noTimes": false + }, + { + "activityKey": "u4YNPVqIRtMeMEEs7DjKtw2-2020", + "admission": "Free", + "eventUrl": null, + "eventDetailUrl": "https://morrisvillechamber.org/event-detail?e=u4YNPVqIRtMeMEEs7DjKtw2-2020", + "eventTypeCode": "CHMBR", + "eventName": "Wake Counseling & Meditation Ribbon-Cutting (back-dated for filter test)", + "eventDescription": "Join the Morrisville Chamber of Commerce as we celebrate the grand opening of Wake Counseling with an official Ribbon Cutting Ceremony.", + "eventFullDescription": "

Join the Morrisville Chamber of Commerce as we celebrate the grand opening of Wake Counseling.

", + "startDateTime": "2020-01-01T12:30:00", + "endDateTime": "2020-01-01T13:30:00", + "contactTypeCode": "In Person", + "address": { + "name": "Wake Counseling & Meditation", + "street1": "5920 South Miami Boulevard", + "street2": "Suite 204", + "city": "Morrisville", + "zip": "27560", + "stateCode": "NC" + }, + "noEndTime": false, + "noTimes": false + } + ] + } +} diff --git a/backendServer/ingestion/tests/fixtures/morrisvilleevents.html b/backendServer/ingestion/tests/fixtures/morrisvilleevents.html new file mode 100644 index 0000000..8e7a603 --- /dev/null +++ b/backendServer/ingestion/tests/fixtures/morrisvilleevents.html @@ -0,0 +1,129 @@ + + +Events listing | Town of Morrisville NC + +
+
+ +
+ +
+ +
+ +
+ +
+
+
+
+ + + +
+
+ + diff --git a/backendServer/ingestion/tests/fixtures/patchdurham.html b/backendServer/ingestion/tests/fixtures/patchdurham.html new file mode 100644 index 0000000..ec322f6 --- /dev/null +++ b/backendServer/ingestion/tests/fixtures/patchdurham.html @@ -0,0 +1,12 @@ + + +Durham, NC Events Calendar | Durham, NC Patch + +
+

Durham Events Calendar

+

Trimmed real sample of patch.com/north-carolina/durham-nc/calendar. + The visible DOM is elided; the scraper reads __NEXT_DATA__ below.

+
+ + + diff --git a/backendServer/ingestion/tests/fixtures/patchmorrisville.html b/backendServer/ingestion/tests/fixtures/patchmorrisville.html new file mode 100644 index 0000000..ae5ecb1 --- /dev/null +++ b/backendServer/ingestion/tests/fixtures/patchmorrisville.html @@ -0,0 +1,12 @@ + + +Morrisville, NC Events Calendar | Morrisville, NC Patch + +
+

Morrisville Events Calendar

+

Trimmed real sample of patch.com/north-carolina/morrisville-nc/calendar. + The visible DOM is elided; the scraper reads __NEXT_DATA__ below.

+
+ + + diff --git a/backendServer/ingestion/tests/fixtures/patchpittsboro.html b/backendServer/ingestion/tests/fixtures/patchpittsboro.html new file mode 100644 index 0000000..bb60df6 --- /dev/null +++ b/backendServer/ingestion/tests/fixtures/patchpittsboro.html @@ -0,0 +1,12 @@ + + +Pittsboro, NC Events Calendar | Pittsboro, NC Patch + +
+

Pittsboro Events Calendar

+

Trimmed real sample of patch.com/north-carolina/pittsboro-nc/calendar. + The visible DOM is elided; the scraper reads __NEXT_DATA__ below.

+
+ + + diff --git a/backendServer/ingestion/tests/fixtures/patchraleigh.html b/backendServer/ingestion/tests/fixtures/patchraleigh.html new file mode 100644 index 0000000..46fdf4e --- /dev/null +++ b/backendServer/ingestion/tests/fixtures/patchraleigh.html @@ -0,0 +1,12 @@ + + +Raleigh, NC Events Calendar | Raleigh, NC Patch + +
+

Raleigh Events Calendar

+

Trimmed real sample of patch.com/north-carolina/raleigh/calendar. + The visible DOM is elided; the scraper reads __NEXT_DATA__ below.

+
+ + + diff --git a/backendServer/ingestion/tests/fixtures/raleighnc.html b/backendServer/ingestion/tests/fixtures/raleighnc.html new file mode 100644 index 0000000..893413e --- /dev/null +++ b/backendServer/ingestion/tests/fixtures/raleighnc.html @@ -0,0 +1,196 @@ + + + + +
+

Events

+
+
+

Upcoming Events

+
+ +
+

2 + +Sunday + +August 2026 + +

+ +
+ +
+

4 + +Tuesday + +August 2026 + +

+ +
+ +
+

7 + +Friday + +August 2026 + +

+ +
+ +
+

1 + +Wednesday + +January 2026 + +

+ +
+
+ +
+
+

Ongoing Events

+
+
+ +
+
+
+ + diff --git a/backendServer/ingestion/tests/fixtures/thrivinginraleigh.html b/backendServer/ingestion/tests/fixtures/thrivinginraleigh.html new file mode 100644 index 0000000..0e34422 --- /dev/null +++ b/backendServer/ingestion/tests/fixtures/thrivinginraleigh.html @@ -0,0 +1,177 @@ + + + + +
+
+ +
+
+
+
+
Aug
+
22
+
+
+
+
+

Lazy Daze Arts & Crafts Festival

+
    +
  • + +
    • +
  • + +
    • +
  • + Cary Town Hall Campus + (map) +
    • +
  • + Google Calendar + ICS +
    • +
+
+

Join us for the 49th annual Lazy Daze Arts & Crafts Festival showcasing over 250 exceptional artists from across the country! Enjoy live entertainment on four stages, savor delicious offerings from 25+ food vendors.

+
+ View Event → +
+
+ +
+
+
+
+
Aug
+
15
+
+
+
+
+

12th Annual CaribMask Carnival

+
    +
  • + +
    • +
  • + + + + + +
    • +
  • + Downtown Raleigh + (map) +
    • +
  • + Google Calendar + ICS +
    • +
+
+

In 2012, RDACA (Raleigh-Durham AfroCaribbean Association) came together to promote diversity, equality and create a deeper appreciation for Afro-Caribbean culture among the citizens of the Raleigh/Durham Metropolitan area.

+
+ View Event → +
+
+ +
+
+
+
+
Aug
+
13
+
+
+
+
+

Live After 5

+
    +
  • + +
    • +
  • + + + + + +
    • +
  • + Moore Square + (map) +
    • +
  • + Google Calendar + ICS +
    • +
+
+

DRA and Southern Bank presents Live After 5 returns this summer on a new night, at a new time, in three new locations across Downtown Raleigh for an extended six-date run.

+
+ View Event → +
+
+ +
+ +
+ +
+
+
+
+
Jun
+
17
+
+
+
+
+

Pottery Painting

+
    +
  • + +
    • +
  • + + + + + +
    • +
  • + Cozy Panda + (map) +
    • +
  • + Google Calendar + ICS +
    • +
+
+

Cozy Panda is on the move! Pick your ceramic, paint your masterpiece and take it home the same day.

+
+ View Event → +
+
+ +
+
+ + diff --git a/backendServer/ingestion/tests/fixtures/triangleonthecheap.html b/backendServer/ingestion/tests/fixtures/triangleonthecheap.html new file mode 100644 index 0000000..290883b --- /dev/null +++ b/backendServer/ingestion/tests/fixtures/triangleonthecheap.html @@ -0,0 +1,63 @@ + + + + +
+ +

Today: Friday, July 31, 2026

+

Baskin Robbins: 31% off scoops

+

All Day | Discounted | Baskin-Robbins, multiple locations

+ +
+

Tasting at Ten -- free coffee tasting at Counter Culture Coffee

+

10:00 am | FREE | Counter Culture Coffee Headquarters and Training Center, Durham

+ +
+

Last Fridays Makers Market

+

5:30 pm to 8:30 pm | FREE | Old Courthouse–Hillsborough

+ +
+

Hillsborough Arts Council's Live on the Lawn

+

5:30 pm to 8:30 pm | FREE

+ +
+

Last Fridays Art Walk in Hillsborough

+

5:30 pm to 9:00 pm | FREE | Downtown Hillsborough

+ +
+
+

Tomorrow: Saturday, August 1, 2026

+

Museums on Us: Free admission to Marbles Kids Museum first weekend of month for Bank of America customers and employees

+

All Day | FREE | Marbles Kids Museum, Raleigh

+ +
+

National Pinball Day! Use the Pinball Map to find local pinball machines

+

All Day | Various locations

+ +
+
+

Thursday, January 1, 2026

+

Past Event (back-dated for filter test)

+

All Day | Discounted | Baskin-Robbins, multiple locations

+ +
+
+ +
+ + diff --git a/backendServer/ingestion/tests/fixtures/visitraleigh.html b/backendServer/ingestion/tests/fixtures/visitraleigh.html new file mode 100644 index 0000000..613f7e2 --- /dev/null +++ b/backendServer/ingestion/tests/fixtures/visitraleigh.html @@ -0,0 +1,119 @@ + +
+
+ +
+
+
+
+
+
+

The Rocky Horror Picture Show

+
+
+ +
+
+
+
+
+
+ +
+
+
+
+
+
+

Andrew Orolfo

+
+
+
    +
  • Dates vary between July 31, 2026 - August 1, 2026
    • +
  • Times: Fri., 9:15pm; Sat., 9pm
    • +
  • + Venue: + + Raleigh Improv + +
    • +
+
+
+
+
+
+
+ +
+
+
+
+
+
+

Beer, Bourbon & BBQ Festival 2026

+
+
+ +
+
+
+
+
+
+ + +
+
+
+
+
+
+

Past Event (back-dated for filter test)

+
+
+ +
+
+
+
+
+
+ +
+
diff --git a/backendServer/ingestion/tests/fixtures/visitraleigh_cary.html b/backendServer/ingestion/tests/fixtures/visitraleigh_cary.html new file mode 100644 index 0000000..a0b51a9 --- /dev/null +++ b/backendServer/ingestion/tests/fixtures/visitraleigh_cary.html @@ -0,0 +1,124 @@ + +
+
+ +
+
+
+
+
+
+

Comedy Show: Cyrus Steele + Joe Perrow

+
+
+ +
+
+
+
+
+
+ +
+
+
+
+
+
+

Country Grass

+
+
+ +
+
+
+
+
+
+ + +
+
+
+
+
+
+

Clue (High School Edition)

+
+
+
    +
  • Recurring weekly on Sunday, Friday, Saturday until August 9, 2026
    • +
  • Times: Fri.-Sat., 7:30pm; Sun., 3pm
    • +
  • + Venue: + + Raleigh Little Theatre + +
    • +
+
+
+
+
+
+
+ + +
+
+
+
+
+
+

Past Event (back-dated for filter test)

+
+
+ +
+
+
+
+
+
+ +
+
diff --git a/backendServer/ingestion/tests/test_carychamber_extract_fast.py b/backendServer/ingestion/tests/test_carychamber_extract_fast.py new file mode 100644 index 0000000..29127a7 --- /dev/null +++ b/backendServer/ingestion/tests/test_carychamber_extract_fast.py @@ -0,0 +1,75 @@ +from datetime import datetime +from pathlib import Path +from unittest import mock + +from django.test import SimpleTestCase, tag +from django.utils import timezone + +from ingestion.scraping.scrapers.carychamber import CarychamberScraper + +FIXTURE = Path(__file__).parent / "fixtures" / "carychamber.html" + +# Fixed "now" the fixture's events were built to be future-dated against +# (real data confirmed live 2026-07-31); the fixture also carries one +# back-dated 2020 event to exercise the past-event filter. +_FIXTURE_BUILD_TIME = timezone.make_aware(datetime(2026, 7, 31, 12, 0, 0)) + + +@tag("fast") +class CarychamberExtractTests(SimpleTestCase): + """Pure `extract()` against a saved real fixture — no DB, no browser, no network.""" + + def setUp(self): + self.html = FIXTURE.read_text(encoding="utf-8") + + @mock.patch("ingestion.scraping.scrapers.carychamber.timezone.now") + def test_extracts_future_events_and_drops_past(self, mock_now): + mock_now.return_value = _FIXTURE_BUILD_TIME + events = CarychamberScraper().extract(self.html) + + # Fixture carries 5 events: 4 future, 1 back-dated to 2020 that the + # past-event filter must drop. + self.assertEqual(len(events), 4) + self.assertNotIn("Past Event (back-dated for filter test)", [e.title for e in events]) + + @mock.patch("ingestion.scraping.scrapers.carychamber.timezone.now") + def test_start_and_end_built_from_date_plus_free_text_time(self, mock_now): + mock_now.return_value = _FIXTURE_BUILD_TIME + events = CarychamberScraper().extract(self.html) + + ambassador = next(e for e in events if e.title == "Ambassador Information Session") + # "12:00 Noon" is the site's own non-standard time string for noon. + self.assertEqual(ambassador.start.hour, 12) + self.assertEqual(ambassador.start.minute, 0) + self.assertEqual(ambassador.start.day, 5) + self.assertIsNotNone(ambassador.start.tzinfo) + self.assertEqual(ambassador.end.hour, 13) + self.assertEqual(ambassador.source_uid, "6241") + self.assertEqual( + ambassador.source_url, + "https://web.carychamber.com/events/eventdetail.aspx?eventid=6241", + ) + + @mock.patch("ingestion.scraping.scrapers.carychamber.timezone.now") + def test_description_is_html_unescaped_and_stripped(self, mock_now): + mock_now.return_value = _FIXTURE_BUILD_TIME + events = CarychamberScraper().extract(self.html) + + golf = next(e for e in events if e.title == "Education Golf Tournament") + self.assertNotIn("

", golf.description) + self.assertNotIn(" ", golf.description) + self.assertIn("beautiful day on the course", golf.description) + + @mock.patch("ingestion.scraping.scrapers.carychamber.timezone.now") + def test_titles_are_html_unescaped(self, mock_now): + mock_now.return_value = _FIXTURE_BUILD_TIME + events = CarychamberScraper().extract(self.html) + + golf_academy = next(e for e in events if "Golf Academy" in e.title) + self.assertEqual(golf_academy.title, "Ladies Lunch & Learn Golf Academy") + + def test_no_events_yields_empty_list(self): + self.assertEqual(CarychamberScraper().extract(''), []) + + def test_malformed_xml_yields_empty_list(self): + self.assertEqual(CarychamberScraper().extract("nope"), []) diff --git a/backendServer/ingestion/tests/test_carysistercities_extract_fast.py b/backendServer/ingestion/tests/test_carysistercities_extract_fast.py new file mode 100644 index 0000000..7d865d7 --- /dev/null +++ b/backendServer/ingestion/tests/test_carysistercities_extract_fast.py @@ -0,0 +1,74 @@ +from datetime import datetime +from pathlib import Path +from unittest import mock + +from django.test import SimpleTestCase, tag +from django.utils import timezone + +from ingestion.scraping.scrapers.carysistercities import CarysistercitiesScraper + +FIXTURE = Path(__file__).parent / "fixtures" / "carysistercities.html" + +# Fixed "now" the fixture's events were built to be future-dated against +# (real data confirmed live 2026-07-31); the fixture also carries one +# back-dated event (2026-07-25, before "now") to exercise the past-event +# filter. +_FIXTURE_BUILD_TIME = timezone.make_aware(datetime(2026, 7, 31, 12, 0, 0)) + + +@tag("fast") +class CarysistercitiesExtractTests(SimpleTestCase): + """Pure `extract()` against a saved real fixture — no DB, no browser, no network.""" + + def setUp(self): + self.html = FIXTURE.read_text(encoding="utf-8") + + @mock.patch("ingestion.scraping.scrapers.carysistercities.timezone.now") + def test_extracts_future_events_and_drops_past(self, mock_now): + mock_now.return_value = _FIXTURE_BUILD_TIME + events = CarysistercitiesScraper().extract(self.html) + + # Fixture carries 4 events: 3 upcoming, 1 back-dated past event that + # the filter must drop. + self.assertEqual(len(events), 3) + self.assertNotIn("CaryLIVE! The Suitcase Junket (Cary)", [e.title for e in events]) + + @mock.patch("ingestion.scraping.scrapers.carysistercities.timezone.now") + def test_google_calendar_dates_param_gives_utc_start_end(self, mock_now): + mock_now.return_value = _FIXTURE_BUILD_TIME + events = CarysistercitiesScraper().extract(self.html) + + final_friday = next(e for e in events if e.title == "Final Friday: Meet the Artists (Cary)") + self.assertEqual(final_friday.start.isoformat(), "2026-07-31T22:00:00+00:00") + self.assertEqual(final_friday.end.isoformat(), "2026-08-01T00:00:00+00:00") + self.assertIsNotNone(final_friday.start.tzinfo) + self.assertEqual( + final_friday.source_url, + "https://www.carysistercities.org/events-1/final-friday-meet-the-artists", + ) + self.assertEqual(final_friday.source_uid, final_friday.source_url) + self.assertIn("Final Fridays Art Crawl", final_friday.description) + # `description` is HTML on the wire; the extractor stores plain text. + self.assertNotIn("

", final_friday.description) + self.assertNotIn("", final_friday.description) + + @mock.patch("ingestion.scraping.scrapers.carysistercities.timezone.now") + def test_extracts_address_when_present(self, mock_now): + mock_now.return_value = _FIXTURE_BUILD_TIME + events = CarysistercitiesScraper().extract(self.html) + + lazy_daze = next(e for e in events if e.title == "Lazy Daze Beer Garden (CSC)") + self.assertEqual(lazy_daze.location, "Fred G. Bond Metro Park") + # The "(map)" link text must not leak into the venue name. + self.assertNotIn("(map)", lazy_daze.location) + + @mock.patch("ingestion.scraping.scrapers.carysistercities.timezone.now") + def test_missing_address_yields_empty_location(self, mock_now): + mock_now.return_value = _FIXTURE_BUILD_TIME + events = CarysistercitiesScraper().extract(self.html) + + asia_fest = next(e for e in events if e.title == "Asia Fest (CSC)") + self.assertEqual(asia_fest.location, "") + + def test_no_events_yields_empty_list(self): + self.assertEqual(CarysistercitiesScraper().extract("nope"), []) diff --git a/backendServer/ingestion/tests/test_chapelhillarts_extract_fast.py b/backendServer/ingestion/tests/test_chapelhillarts_extract_fast.py new file mode 100644 index 0000000..6ed06a0 --- /dev/null +++ b/backendServer/ingestion/tests/test_chapelhillarts_extract_fast.py @@ -0,0 +1,66 @@ +import json +from datetime import datetime +from pathlib import Path +from unittest import mock +from zoneinfo import ZoneInfo + +from django.test import SimpleTestCase, tag +from django.utils import timezone + +from ingestion.scraping.scrapers.chapelhillarts import ChapelhillartsScraper + +FIXTURE = Path(__file__).parent / "fixtures" / "chapelhillarts.html" +_LOCAL_TZ = ZoneInfo("America/New_York") + +# Fixed "now" the fixture's events (Jun 25 - Sep 25, 2026) were built to be +# future-dated against -- real data confirmed live via +# GET /wp-json/nmc-feeds/v1/events on 2026-07-31, back-dated here to before +# the earliest of those events since several had already passed by the +# capture date. The fixture also carries one back-dated 2020 event to +# exercise the past-event filter. +_FIXTURE_BUILD_TIME = timezone.make_aware(datetime(2026, 6, 1)) + + +@tag("fast") +class ChapelhillartsExtractTests(SimpleTestCase): + """Pure `extract()` against a saved real fixture -- no DB, no browser, no network.""" + + def setUp(self): + self.html = FIXTURE.read_text(encoding="utf-8") + + @mock.patch("ingestion.scraping.scrapers.chapelhillarts.timezone.now") + def test_extracts_future_events_from_json_feed(self, mock_now): + mock_now.return_value = _FIXTURE_BUILD_TIME + events = ChapelhillartsScraper().extract(self.html) + + # Fixture carries 6 items: 1 back-dated to 2020 (dropped by the + # past-event filter) and 5 real future events. + self.assertEqual(len(events), 5) + self.assertNotIn("Past Event (back-dated for filter test)", [e.title for e in events]) + + drone_talk = next(e for e in events if e.title == "Visiting Kalamkari Master Artist Talk") + self.assertEqual(drone_talk.start, datetime(2026, 7, 16, 17, 30, tzinfo=_LOCAL_TZ)) + self.assertIsNotNone(drone_talk.start.tzinfo) + self.assertEqual(drone_talk.source_uid, "8451") + self.assertEqual( + drone_talk.source_url, + "https://www.chapelhillarts.org/calendar/visiting-kalamkari-master-artist-talk/", + ) + # The feed's `end` field isn't valid ISO8601, so it's dropped rather + # than misparsed. + self.assertIsNone(drone_talk.end) + + def test_no_start_or_title_events_are_skipped(self): + html = json.dumps( + [ + {"id": 1, "title": "", "start": "2099-01-01T00:00:00", "url": "x"}, + {"id": 2, "start": "", "title": "No start"}, + ] + ) + self.assertEqual(ChapelhillartsScraper().extract(html), []) + + def test_malformed_json_yields_no_events(self): + self.assertEqual(ChapelhillartsScraper().extract("nope"), []) + + def test_non_list_json_yields_no_events(self): + self.assertEqual(ChapelhillartsScraper().extract('{"error": "not found"}'), []) diff --git a/backendServer/ingestion/tests/test_chapelhillnc_extract_fast.py b/backendServer/ingestion/tests/test_chapelhillnc_extract_fast.py new file mode 100644 index 0000000..e949a7e --- /dev/null +++ b/backendServer/ingestion/tests/test_chapelhillnc_extract_fast.py @@ -0,0 +1,65 @@ +from datetime import datetime +from pathlib import Path +from unittest import mock +from zoneinfo import ZoneInfo + +from django.test import SimpleTestCase, tag +from django.utils import timezone + +from ingestion.scraping.scrapers.chapelhillnc import ChapelhillncScraper + +_LOCAL_TZ = ZoneInfo("America/New_York") + +FIXTURE = Path(__file__).parent / "fixtures" / "chapelhillnc.html" + +# Fixed "now" the fixture's events (July 2-29, 2026) were built to be +# future-dated against -- real data confirmed live on the site 2026-07-31, +# captured from a month-grid render whose events had already mostly passed +# by that date, so "now" is pinned to the start of that same month instead. +# The fixture also carries one back-dated 2020 event to exercise the +# past-event filter. +_FIXTURE_BUILD_TIME = timezone.make_aware(datetime(2026, 7, 1)) + + +@tag("fast") +class ChapelhillncExtractTests(SimpleTestCase): + """Pure `extract()` against a saved real fixture -- no DB, no browser, no network.""" + + def setUp(self): + self.html = FIXTURE.read_text(encoding="utf-8") + + @mock.patch("ingestion.scraping.scrapers.chapelhillnc.timezone.now") + def test_extracts_future_events_from_calendar_grid(self, mock_now): + mock_now.return_value = _FIXTURE_BUILD_TIME + events = ChapelhillncScraper().extract(self.html) + + # Fixture carries 6 day cells: 1 back-dated to 2020 (dropped by the + # past-event filter) and 5 real future events. + self.assertEqual(len(events), 5) + self.assertNotIn("Past Event (back-dated for filter test)", [e.title for e in events]) + + drone_show = next(e for e in events if e.title == "Chapel Hill July 4th Drone Show") + self.assertEqual(drone_show.start, datetime(2026, 7, 4, tzinfo=_LOCAL_TZ)) + self.assertIsNotNone(drone_show.start.tzinfo) + self.assertEqual(drone_show.source_uid, "62ab25af-7708-4f73-80c3-53ad722a6312:2026-07-04") + # The DOM carries no start time, location, description, or permalink -- + # only day-level dates and a stable item id -- so these stay blank + # rather than being invented. + self.assertEqual(drone_show.location, "") + self.assertEqual(drone_show.description, "") + self.assertEqual(drone_show.source_url, "") + + @mock.patch("ingestion.scraping.scrapers.chapelhillnc.timezone.now") + def test_recurring_series_gets_per_occurrence_uid(self, mock_now): + mock_now.return_value = _FIXTURE_BUILD_TIME + events = ChapelhillncScraper().extract(self.html) + + food_distribution = next(e for e in events if e.title == "Food Distribution") + # Same `data-item-id` as the site's other "Food Distribution" occurrence + # would reuse, disambiguated by date so each occurrence is unique. + self.assertEqual( + food_distribution.source_uid, "a5ff3873-c5e9-4f09-9449-47ca0db32b15:2026-07-08" + ) + + def test_no_calendar_cells_yields_no_events(self): + self.assertEqual(ChapelhillncScraper().extract("nope"), []) diff --git a/backendServer/ingestion/tests/test_chathamchamber_extract_fast.py b/backendServer/ingestion/tests/test_chathamchamber_extract_fast.py new file mode 100644 index 0000000..00386f0 --- /dev/null +++ b/backendServer/ingestion/tests/test_chathamchamber_extract_fast.py @@ -0,0 +1,72 @@ +from datetime import datetime +from pathlib import Path +from unittest import mock + +from django.test import SimpleTestCase, tag +from django.utils import timezone + +from ingestion.scraping.scrapers.chathamchamber import ChathamchamberScraper + +FIXTURE = Path(__file__).parent / "fixtures" / "chathamchamber.html" + +# Fixed "now" the fixture's events were built to be future-dated against +# (real data confirmed live 2026-07-31); the fixture also carries one +# card whose dates were edited to 2020 to exercise the past-event filter. +_FIXTURE_BUILD_TIME = timezone.make_aware(datetime(2026, 7, 31)) + + +@tag("fast") +class ChathamchamberExtractTests(SimpleTestCase): + """Pure `extract()` against a saved real fixture -- no DB, no browser, no network.""" + + def setUp(self): + self.html = FIXTURE.read_text(encoding="utf-8") + + @mock.patch("ingestion.scraping.scrapers.chathamchamber.timezone.now") + def test_extracts_future_and_ongoing_events_from_microdata(self, mock_now): + mock_now.return_value = _FIXTURE_BUILD_TIME + events = ChathamchamberScraper().extract(self.html) + + # Fixture carries 5 cards: 4 real (one an ongoing series whose + # startDate is already past but endDate is future) + 1 back-dated + # to 2020 that the past-event filter must drop. + self.assertEqual(len(events), 4) + self.assertNotIn("Past Event (back-dated for filter test)", [e.title for e in events]) + + film_series = next(e for e in events if e.title.startswith("Goldston America 250")) + self.assertEqual(film_series.start.isoformat(), "2026-01-17T16:00:00-05:00") + self.assertEqual(film_series.end.isoformat(), "2026-12-22T18:00:00-05:00") + self.assertIsNotNone(film_series.start.tzinfo) + self.assertEqual( + film_series.source_url, + "https://business.chathamchambernc.org/chatham-community-events/" + "Details/goldston-america-250-film-series-1591589", + ) + self.assertEqual(film_series.source_uid, film_series.source_url) + + walking_tour = next(e for e in events if e.title.startswith("Historical Walking Tour")) + self.assertEqual(walking_tour.start.isoformat(), "2026-08-09T13:00:00-04:00") + self.assertEqual(walking_tour.end.isoformat(), "2026-08-09T14:30:00-04:00") + self.assertEqual(walking_tour.description, "") + + @mock.patch("ingestion.scraping.scrapers.chathamchamber.timezone.now") + def test_description_is_html_unescaped_when_present(self, mock_now): + mock_now.return_value = _FIXTURE_BUILD_TIME + events = ChathamchamberScraper().extract(self.html) + + race_amity = next(e for e in events if e.title == "Race Amity Day") + self.assertIn("Mindful Musical Walk", race_amity.description) + self.assertNotIn("nope"), []) diff --git a/backendServer/ingestion/tests/test_direct_ingest_db.py b/backendServer/ingestion/tests/test_direct_ingest_db.py index 5bc832f..91b2cc7 100644 --- a/backendServer/ingestion/tests/test_direct_ingest_db.py +++ b/backendServer/ingestion/tests/test_direct_ingest_db.py @@ -105,6 +105,8 @@ def test_over_threshold_holds_event(self): self.assertEqual(Event.objects.count(), 0) staged = StagedEvent.objects.get() self.assertEqual(staged.status, "pending") + # No prior Event exists on a first submission — must stay unset. + self.assertIsNone(staged.published_event) def test_blank_organizer_falls_back_to_host(self): """No organizer name on the raw row → generic 'by host' attribution.""" @@ -126,3 +128,147 @@ def test_anonymous_submission_no_user_id(self): self.assertFalse(event.is_verified) staged = StagedEvent.objects.get() self.assertIsNone(staged.submitted_by) + + +@tag("db") +class DirectIngestOrphanPreventionTests(TestCase): + """Ticket 40.4: a resubmitted-and-edited direct submission that gets + gated (duplicate / over-safety-threshold / no-town) must not strand the + Event a *prior* call to `ingest_direct_submission` already published. + `events.Event` has no soft-delete or published flag — its existence is + its publication — so the fix is to keep the terminal `StagedEvent` + pointed at that prior Event (`published_event`) rather than touch the + Event at all. Production audit (40.3) found 4 of 4 live direct-submission + Events orphaned this way. + """ + + STD_PAYLOAD = { + "title": "Test Concert", + "description": "A great show at the Cradle.", + "location_name": "Cat's Cradle", + "town": "Carrboro", + "tags": ["live-music"], + "price": 10, + } + + def setUp(self): + self.source = EventSource.objects.create( + name="Direct Feed", + source_type="ics", + url="https://feed.test/direct.ics", + ) + self.raw = RawEvent.objects.create( + source=self.source, + raw_title="Raw Concert", + raw_description="A show", + raw_location="Cat's Cradle, Carrboro, NC", + raw_start=datetime(2099, 6, 1, 18, 0, tzinfo=UTC), + source_url="", # avoids fetch_page_text network call + source_uid="direct-uid-orphan", + raw_organizer="The MAKRS Society", + ) + self.town = make_town(slug="carrboro") + self.user = make_user(user_type="BUSINESS") + + def _gemini_mocks(self, std_payload, score): + fake_std = mock.Mock() + fake_std.models.generate_content.return_value = mock.Mock(text=json.dumps(std_payload)) + fake_scorer = mock.Mock() + fake_scorer.models.generate_content.return_value = mock.Mock( + text=json.dumps({"score": score, "notes": ""}) + ) + return mock.patch( + "ingestion.standardizer.genai.Client", + side_effect=[fake_std, fake_scorer], + ) + + def test_resubmission_that_becomes_a_duplicate_keeps_prior_event_published(self): + with self._gemini_mocks(self.STD_PAYLOAD, 0.0): + first_event = ingest_direct_submission(self.raw.id, self.user.id) + self.assertEqual(Event.objects.count(), 1) + + # A genuine duplicate elsewhere in the system. find_duplicate only + # matches against a lower pk (see its docstring), and this row is + # created before the resubmission below, so it qualifies. + other = StagedEvent.objects.create( + title=self.STD_PAYLOAD["title"], + description="d", + location_name=self.STD_PAYLOAD["location_name"], + town=self.STD_PAYLOAD["town"], + start_datetime=self.raw.raw_start, + status="published", + ) + + with self._gemini_mocks(self.STD_PAYLOAD, 0.0): + result = ingest_direct_submission(self.raw.id, self.user.id) + + self.assertIsNone(result) + # The prior Event stays live and moderatable — not orphaned, not deleted. + self.assertEqual(Event.objects.count(), 1) + self.assertTrue(Event.objects.filter(pk=first_event.pk).exists()) + + staged = StagedEvent.objects.get(raw_event=self.raw) + self.assertEqual(staged.status, "duplicate") + self.assertEqual(staged.duplicate_of, other) + self.assertEqual(staged.published_event_id, first_event.pk) + + def test_resubmission_over_safety_threshold_keeps_prior_event_published(self): + with self._gemini_mocks(self.STD_PAYLOAD, 0.0): + first_event = ingest_direct_submission(self.raw.id, self.user.id) + self.assertEqual(Event.objects.count(), 1) + + with self._gemini_mocks(self.STD_PAYLOAD, 0.9): + result = ingest_direct_submission(self.raw.id, self.user.id) + + self.assertIsNone(result) + self.assertEqual(Event.objects.count(), 1) + self.assertTrue(Event.objects.filter(pk=first_event.pk).exists()) + + staged = StagedEvent.objects.get(raw_event=self.raw) + # Held for review, same terminal value the automated pipeline uses + # (see auto_publish_safe_events / monitoring.py's held_for_review + # bucket) — not silently left at whatever standardize_event defaulted to. + self.assertEqual(staged.status, "pending") + self.assertEqual(staged.published_event_id, first_event.pk) + + def test_resubmission_with_no_matching_town_keeps_prior_event_published(self): + with self._gemini_mocks(self.STD_PAYLOAD, 0.0): + first_event = ingest_direct_submission(self.raw.id, self.user.id) + self.assertEqual(Event.objects.count(), 1) + + out_of_coverage_payload = {**self.STD_PAYLOAD, "town": "Greensboro"} + with self._gemini_mocks(out_of_coverage_payload, 0.0): + result = ingest_direct_submission(self.raw.id, self.user.id) + + self.assertIsNone(result) + self.assertEqual(Event.objects.count(), 1) + staged = StagedEvent.objects.get(raw_event=self.raw) + self.assertEqual(staged.status, "skipped_no_town") + self.assertEqual(staged.published_event_id, first_event.pk) + + # The gated resubmission's content must never have touched the Event — + # only a fully successful re-publish is allowed to mutate it. + first_event.refresh_from_db() + self.assertEqual(first_event.town.slug, "carrboro") + + def test_first_submission_duplicate_has_no_published_event(self): + """No prior_event exists on a first-time duplicate — published_event + must stay None, not resurrect a reference to a nonexistent Event.""" + other = StagedEvent.objects.create( + title=self.STD_PAYLOAD["title"], + description="d", + location_name=self.STD_PAYLOAD["location_name"], + town=self.STD_PAYLOAD["town"], + start_datetime=self.raw.raw_start, + status="published", + ) + + with self._gemini_mocks(self.STD_PAYLOAD, 0.0): + result = ingest_direct_submission(self.raw.id, self.user.id) + + self.assertIsNone(result) + self.assertFalse(Event.objects.exists()) + staged = StagedEvent.objects.get(raw_event=self.raw) + self.assertEqual(staged.status, "duplicate") + self.assertEqual(staged.duplicate_of, other) + self.assertIsNone(staged.published_event) diff --git a/backendServer/ingestion/tests/test_downtowncarync_extract_fast.py b/backendServer/ingestion/tests/test_downtowncarync_extract_fast.py new file mode 100644 index 0000000..be20d9d --- /dev/null +++ b/backendServer/ingestion/tests/test_downtowncarync_extract_fast.py @@ -0,0 +1,72 @@ +from datetime import datetime +from pathlib import Path +from unittest import mock + +from django.test import SimpleTestCase, tag +from django.utils import timezone + +from ingestion.scraping.scrapers.downtowncarync import DowntowncarynScraper + +FIXTURE = Path(__file__).parent / "fixtures" / "downtowncarync.html" + +# Fixed "now" the fixture's events were built to be future-dated against +# (real data confirmed live 2026-07-31, mid-afternoon UTC). The fixture keeps +# two back-dated (2026-07-03) events and one same-day (2026-07-31, 7pm ET, +# still future relative to this "now") plus one clearly-future (2026-08-02) +# event to exercise the past-event filter. +_FIXTURE_BUILD_TIME = timezone.make_aware(datetime(2026, 7, 31, 12, 0)) + + +@tag("fast") +class DowntowncarynExtractTests(SimpleTestCase): + """Pure `extract()` against a saved real fixture -- no DB, no browser, no network.""" + + def setUp(self): + self.html = FIXTURE.read_text(encoding="utf-8") + + @mock.patch("ingestion.scraping.scrapers.downtowncarync.timezone.now") + def test_extracts_future_events_from_month_grid(self, mock_now): + mock_now.return_value = _FIXTURE_BUILD_TIME + events = DowntowncarynScraper().extract(self.html) + + # Fixture carries 5 events: 2 back-dated to July 3, 1 later today + # (July 31, 7pm), and 2 in the future (Aug 2). + self.assertEqual(len(events), 2) + titles = [e.title for e in events] + self.assertNotIn("July 3rd Celebration", titles) + self.assertNotIn( + "Cary Town Band presents America’s 250th Anniversary Patriotic Celebration", + titles, + ) + + dance = next(e for e in events if e.title == "Discover Dance: DISHOOM") + self.assertEqual(dance.start.isoformat(), "2026-07-31T19:00:00-04:00") + self.assertIsNotNone(dance.start.tzinfo) + self.assertEqual(dance.end.isoformat(), "2026-07-31T22:30:00-04:00") + self.assertEqual( + dance.source_url, "https://downtowncarync.org/event/discover-dance-dishoom-2/" + ) + self.assertEqual(dance.source_uid, dance.source_url) + self.assertEqual(dance.location, "") + self.assertIn("DISHOOM returns to Downtown Cary Park", dance.description) + + artists = next(e for e in events if e.title == "Meet the Artists: Fine Arts League of Cary") + self.assertEqual(artists.start.isoformat(), "2026-08-02T14:00:00-04:00") + + @mock.patch("ingestion.scraping.scrapers.downtowncarync.timezone.now") + def test_titles_are_html_unescaped(self, mock_now): + # Use a "now" before all fixture events so the back-dated (but + # entity-escaped) title survives the past-event filter. + mock_now.return_value = timezone.make_aware(datetime(2000, 1, 1)) + events = DowntowncarynScraper().extract(self.html) + + band = next(e for e in events if "Cary Town Band" in e.title) + self.assertEqual( + band.title, + "Cary Town Band presents America’s 250th Anniversary Patriotic Celebration", + ) + self.assertNotIn("’", band.description) + self.assertIn("America’s Semi-Quincentennial", band.description) + + def test_no_matching_articles_yields_no_events(self): + self.assertEqual(DowntowncarynScraper().extract("nope"), []) diff --git a/backendServer/ingestion/tests/test_downtownraleigh_extract_fast.py b/backendServer/ingestion/tests/test_downtownraleigh_extract_fast.py new file mode 100644 index 0000000..5788a74 --- /dev/null +++ b/backendServer/ingestion/tests/test_downtownraleigh_extract_fast.py @@ -0,0 +1,85 @@ +from datetime import datetime +from pathlib import Path +from unittest import mock + +from django.test import SimpleTestCase, tag +from django.utils import timezone + +from ingestion.scraping.scrapers.downtownraleigh import DowntownraleighScraper + +FIXTURE = Path(__file__).parent / "fixtures" / "downtownraleigh.html" + +# Fixed "now" the fixture's events were built to be future-dated against (real +# data confirmed live 2026-07-31); the fixture also carries one back-dated +# "Jan 1" row (same header year, earlier month) to exercise the past-event +# filter. +_FIXTURE_BUILD_TIME = timezone.make_aware(datetime(2026, 7, 31)) + + +@tag("fast") +class DowntownraleighExtractTests(SimpleTestCase): + """Pure `extract()` against a saved real fixture -- no DB, no browser, no network.""" + + def setUp(self): + self.html = FIXTURE.read_text(encoding="utf-8") + + @mock.patch("ingestion.scraping.scrapers.downtownraleigh.timezone.now") + def test_extracts_future_events_grouped_by_day(self, mock_now): + mock_now.return_value = _FIXTURE_BUILD_TIME + events = DowntownraleighScraper().extract(self.html) + + # Fixture carries 7 events: 1 back-dated (Jan 1) that the filter must + # drop, 3 on Jul 31, 3 on Aug 1. The "Ongoing" bucket has no day/month + # anchor and is skipped entirely regardless of date. + self.assertEqual(len(events), 6) + self.assertNotIn("Past Event (back-dated for filter test)", [e.title for e in events]) + self.assertNotIn( + "Staying Alive Special Exhibition", [e.title for e in events] + ) + + @mock.patch("ingestion.scraping.scrapers.downtownraleigh.timezone.now") + def test_time_range_parses_start_hour(self, mock_now): + mock_now.return_value = _FIXTURE_BUILD_TIME + events = DowntownraleighScraper().extract(self.html) + + karaoke = next(e for e in events if e.title == "Crank Karaoke @ Crank Arm Brewing") + self.assertEqual(karaoke.start.isoformat(), "2026-07-31T20:00:00-04:00") + self.assertIsNotNone(karaoke.start.tzinfo) + self.assertIsNone(karaoke.end) + self.assertEqual(karaoke.location, "Crank Arm Brewing, Raleigh, NC") + self.assertEqual( + karaoke.source_url, + "https://downtownraleigh.org/do/crank-karaoke--crank-arm-brewing", + ) + self.assertEqual( + karaoke.source_uid, "/do/crank-karaoke--crank-arm-brewing#2026-07-31" + ) + + @mock.patch("ingestion.scraping.scrapers.downtownraleigh.timezone.now") + def test_range_without_leading_meridiem_uses_trailing_one(self, mock_now): + mock_now.return_value = _FIXTURE_BUILD_TIME + events = DowntownraleighScraper().extract(self.html) + + reset = next(e for e in events if e.title == "Friday Reset for the Weekend") + # "1-3pm" -> starts at 1pm, not 3pm. + self.assertEqual(reset.start.hour, 13) + + @mock.patch("ingestion.scraping.scrapers.downtownraleigh.timezone.now") + def test_single_time_token_uppercase(self, mock_now): + mock_now.return_value = _FIXTURE_BUILD_TIME + events = DowntownraleighScraper().extract(self.html) + + golf = next(e for e in events if e.title == "National Disc Golf Day") + self.assertEqual(golf.start.hour, 11) + + @mock.patch("ingestion.scraping.scrapers.downtownraleigh.timezone.now") + def test_no_time_text_defaults_to_midnight(self, mock_now): + mock_now.return_value = _FIXTURE_BUILD_TIME + events = DowntownraleighScraper().extract(self.html) + + cozy = next(e for e in events if e.title.startswith("Cozy Saturday")) + self.assertEqual((cozy.start.hour, cozy.start.minute), (0, 0)) + self.assertEqual(cozy.location, "Haymaker, Raleigh, NC") + + def test_no_rows_yields_no_events(self): + self.assertEqual(DowntownraleighScraper().extract("nope"), []) diff --git a/backendServer/ingestion/tests/test_eventbritemorrisville_extract_fast.py b/backendServer/ingestion/tests/test_eventbritemorrisville_extract_fast.py new file mode 100644 index 0000000..d3f0255 --- /dev/null +++ b/backendServer/ingestion/tests/test_eventbritemorrisville_extract_fast.py @@ -0,0 +1,57 @@ +from datetime import datetime +from pathlib import Path +from unittest import mock + +from django.test import SimpleTestCase, tag +from django.utils import timezone + +from ingestion.scraping.scrapers.eventbritemorrisville import EventbritemorrisvilleScraper + +FIXTURE = Path(__file__).parent / "fixtures" / "eventbritemorrisville.html" + +# Fixed "now" the fixture's events were built to be future-dated against +# (real data confirmed live 2026-07-31); the fixture also carries one +# back-dated 2020 event to exercise the past-event filter. +_FIXTURE_BUILD_TIME = timezone.make_aware(datetime(2026, 7, 31)) + + +@tag("fast") +class EventbritemorrisvilleExtractTests(SimpleTestCase): + """Pure `extract()` against a saved real fixture — no DB, no browser, no network. + + Delegates to the shared `extract_eventbrite_events` helper tested more + thoroughly in `test_eventbriteraleigh_extract_fast.py`. This fixture is + also the receipt for the quality warning in + `eventbritemorrisville.py`'s docstring: none of the "Popular" bucket + events on the real Morrisville discovery page (captured 2026-07-31) are + actually Morrisville-located — extraction still works correctly, it + just doesn't return Morrisville events. + """ + + def setUp(self): + self.html = FIXTURE.read_text(encoding="utf-8") + + @mock.patch("ingestion.scraping.scrapers.eventbriteraleigh.timezone.now") + def test_extracts_future_events_none_of_which_are_morrisville(self, mock_now): + mock_now.return_value = _FIXTURE_BUILD_TIME + events = EventbritemorrisvilleScraper().extract(self.html) + + # Fixture carries 5 events: 3 future, 1 online (dropped), 1 + # back-dated to 2020 (dropped by the past-event filter). + self.assertEqual(len(events), 3) + # Real spillover: the Morrisville discovery page's "Popular" bucket + # returned only Raleigh-located events on this capture. + self.assertTrue(all("Raleigh" in e.location for e in events)) + self.assertTrue(all("Morrisville" not in e.location for e in events)) + + jersey = next(e for e in events if e.title.startswith("The 4th Annual Jersey Fest")) + self.assertEqual(jersey.start.isoformat(), "2026-08-01T18:00:00-04:00") + self.assertEqual(jersey.source_uid, "1992735047169") + self.assertEqual( + jersey.location, "The London Bridge Pub, 110 East Hargett Street, Raleigh, NC" + ) + + def test_no_server_data_yields_no_events(self): + self.assertEqual( + EventbritemorrisvilleScraper().extract("nope"), [] + ) diff --git a/backendServer/ingestion/tests/test_eventbritepittsboro_extract_fast.py b/backendServer/ingestion/tests/test_eventbritepittsboro_extract_fast.py new file mode 100644 index 0000000..50f6aae --- /dev/null +++ b/backendServer/ingestion/tests/test_eventbritepittsboro_extract_fast.py @@ -0,0 +1,50 @@ +from datetime import datetime +from pathlib import Path +from unittest import mock + +from django.test import SimpleTestCase, tag +from django.utils import timezone + +from ingestion.scraping.scrapers.eventbritepittsboro import EventbritepittsboroScraper + +FIXTURE = Path(__file__).parent / "fixtures" / "eventbritepittsboro.html" + +# Fixed "now" the fixture's events were built to be future-dated against +# (real data confirmed live 2026-07-31); the fixture also carries one +# back-dated 2020 event to exercise the past-event filter. +_FIXTURE_BUILD_TIME = timezone.make_aware(datetime(2026, 7, 31)) + + +@tag("fast") +class EventbritepittsboroExtractTests(SimpleTestCase): + """Pure `extract()` against a saved real fixture — no DB, no browser, no network. + + Delegates to the shared `extract_eventbrite_events` helper tested more + thoroughly in `test_eventbriteraleigh_extract_fast.py`; this test just + confirms the Pittsboro-specific fixture wires through correctly. + """ + + def setUp(self): + self.html = FIXTURE.read_text(encoding="utf-8") + + @mock.patch("ingestion.scraping.scrapers.eventbriteraleigh.timezone.now") + def test_extracts_future_local_events(self, mock_now): + mock_now.return_value = _FIXTURE_BUILD_TIME + events = EventbritepittsboroScraper().extract(self.html) + + # Fixture carries 5 events: 3 future local, 1 online (dropped), 1 + # back-dated to 2020 (dropped by the past-event filter). + self.assertEqual(len(events), 3) + + trivia = next(e for e in events if e.title == "Doherty's Pub Trivia w/ Patrick W") + self.assertEqual(trivia.start.isoformat(), "2026-08-04T19:00:00-04:00") + self.assertIsNotNone(trivia.start.tzinfo) + self.assertEqual(trivia.source_uid, "1989861724985") + self.assertEqual( + trivia.location, + "Doherty's Irish Pub & Restaurant, 56 Sanford Road, Pittsboro, NC", + ) + self.assertIn("Pittsboro, NC", trivia.description) + + def test_no_server_data_yields_no_events(self): + self.assertEqual(EventbritepittsboroScraper().extract("nope"), []) diff --git a/backendServer/ingestion/tests/test_eventbriteraleigh_extract_fast.py b/backendServer/ingestion/tests/test_eventbriteraleigh_extract_fast.py new file mode 100644 index 0000000..37d5dd8 --- /dev/null +++ b/backendServer/ingestion/tests/test_eventbriteraleigh_extract_fast.py @@ -0,0 +1,66 @@ +from datetime import datetime +from pathlib import Path +from unittest import mock + +from django.test import SimpleTestCase, tag +from django.utils import timezone + +from ingestion.scraping.scrapers.eventbriteraleigh import EventbriteraleighScraper + +FIXTURE = Path(__file__).parent / "fixtures" / "eventbriteraleigh.html" + +# Fixed "now" the fixture's events were built to be future-dated against +# (real data confirmed live 2026-07-31); the fixture also carries one +# back-dated 2020 event to exercise the past-event filter. +_FIXTURE_BUILD_TIME = timezone.make_aware(datetime(2026, 7, 31)) + + +@tag("fast") +class EventbriteraleighExtractTests(SimpleTestCase): + """Pure `extract()` against a saved real fixture — no DB, no browser, no network.""" + + def setUp(self): + self.html = FIXTURE.read_text(encoding="utf-8") + + @mock.patch("ingestion.scraping.scrapers.eventbriteraleigh.timezone.now") + def test_extracts_future_local_events_from_server_data(self, mock_now): + mock_now.return_value = _FIXTURE_BUILD_TIME + events = EventbriteraleighScraper().extract(self.html) + + # Fixture carries 5 events: 3 future local, 1 online (dropped), 1 + # back-dated to 2020 (dropped by the past-event filter). + self.assertEqual(len(events), 3) + titles = [e.title for e in events] + self.assertNotIn( + "LITERACY. NOW. A National Parent-Led Conference to End the Literacy Crisis", titles + ) + self.assertNotIn( + "250 aka Finesse2tymes takes over CLUB LOVE (back-dated for filter test)", titles + ) + + club = next(e for e in events if e.title == "250 aka Finesse2tymes takes over CLUB LOVE") + self.assertEqual(club.start.isoformat(), "2026-07-31T22:00:00-04:00") + self.assertIsNotNone(club.start.tzinfo) + self.assertEqual(club.end.isoformat(), "2026-08-01T02:00:00-04:00") + self.assertEqual(club.source_uid, "1995343344657") + self.assertEqual( + club.source_url, + "https://www.eventbrite.com/e/250-aka-finesse2tymes-takes-over-club-love-tickets-1995343344657", + ) + # Venue name plus street address, so the standardizer can infer the town. + self.assertEqual(club.location, "4400 Craftsman Dr, 4400 Craftsman Drive, Raleigh, NC") + self.assertIn("free for everybody", club.description) + + @mock.patch("ingestion.scraping.scrapers.eventbriteraleigh.timezone.now") + def test_online_events_are_dropped(self, mock_now): + mock_now.return_value = _FIXTURE_BUILD_TIME + events = EventbriteraleighScraper().extract(self.html) + + self.assertTrue(all("LITERACY" not in e.title for e in events)) + + def test_no_server_data_yields_no_events(self): + self.assertEqual(EventbriteraleighScraper().extract("nope"), []) + + def test_malformed_server_data_yields_no_events(self): + html = "" + self.assertEqual(EventbriteraleighScraper().extract(html), []) diff --git a/backendServer/ingestion/tests/test_morrisvillechamber_extract_fast.py b/backendServer/ingestion/tests/test_morrisvillechamber_extract_fast.py new file mode 100644 index 0000000..6f93602 --- /dev/null +++ b/backendServer/ingestion/tests/test_morrisvillechamber_extract_fast.py @@ -0,0 +1,67 @@ +from datetime import datetime +from pathlib import Path +from unittest import mock + +from django.test import SimpleTestCase, tag +from django.utils import timezone + +from ingestion.scraping.scrapers.morrisvillechamber import MorrisvillechamberScraper + +FIXTURE = Path(__file__).parent / "fixtures" / "morrisvillechamber.html" + +# Fixed "now" the fixture's events were built to be future-dated against +# (real data confirmed live 2026-07-31); the fixture also carries one +# back-dated 2020 event to exercise the past-event filter, and one event +# whose title the chamber marked *CANCELLED* to exercise that filter. +_FIXTURE_BUILD_TIME = timezone.make_aware(datetime(2026, 7, 31)) + + +@tag("fast") +class MorrisvillechamberExtractTests(SimpleTestCase): + """Pure `extract()` against a saved real fixture — no DB, no browser, no network.""" + + def setUp(self): + self.html = FIXTURE.read_text(encoding="utf-8") + + @mock.patch("ingestion.scraping.scrapers.morrisvillechamber.timezone.now") + def test_extracts_future_uncancelled_events(self, mock_now): + mock_now.return_value = _FIXTURE_BUILD_TIME + events = MorrisvillechamberScraper().extract(self.html) + + # Fixture carries 5 events: 3 plain future events, 1 future event + # marked *CANCELLED* (dropped), 1 back-dated to 2020 (dropped). + self.assertEqual(len(events), 3) + titles = [e.title for e in events] + self.assertNotIn("*CANCELLED* Cregger Showroom Ribbon-Cutting After Hours Social", titles) + self.assertFalse(any("back-dated for filter test" in t for t in titles)) + + @mock.patch("ingestion.scraping.scrapers.morrisvillechamber.timezone.now") + def test_in_person_event_fields(self, mock_now): + mock_now.return_value = _FIXTURE_BUILD_TIME + events = MorrisvillechamberScraper().extract(self.html) + + pizza = next(e for e in events if e.title.startswith("Bambino's Pizza")) + self.assertEqual(pizza.start.isoformat(), "2026-08-04T20:00:00-04:00") + self.assertIsNotNone(pizza.start.tzinfo) + self.assertEqual(pizza.end.isoformat(), "2026-08-04T21:00:00-04:00") + self.assertEqual( + pizza.source_url, + "https://morrisvillechamber.org/event-detail?e=EXCq5cP0tOLp7S7CEjtAWg2", + ) + self.assertEqual(pizza.source_uid, "EXCq5cP0tOLp7S7CEjtAWg2") + self.assertEqual(pizza.location, "4129 Davis Drive, Morrisville, NC, 27560") + self.assertIn("Grand Opening and Ribbon Cutting", pizza.description) + + @mock.patch("ingestion.scraping.scrapers.morrisvillechamber.timezone.now") + def test_online_event_has_no_street_address(self, mock_now): + mock_now.return_value = _FIXTURE_BUILD_TIME + events = MorrisvillechamberScraper().extract(self.html) + + virtual = next(e for e in events if "Virtual" in e.title) + self.assertEqual(virtual.location, "Online") + + def test_invalid_json_yields_no_events(self): + self.assertEqual(MorrisvillechamberScraper().extract("not json"), []) + + def test_no_events_key_yields_no_events(self): + self.assertEqual(MorrisvillechamberScraper().extract('{"data": {}}'), []) diff --git a/backendServer/ingestion/tests/test_morrisvilleevents_extract_fast.py b/backendServer/ingestion/tests/test_morrisvilleevents_extract_fast.py new file mode 100644 index 0000000..de9cad7 --- /dev/null +++ b/backendServer/ingestion/tests/test_morrisvilleevents_extract_fast.py @@ -0,0 +1,70 @@ +from datetime import datetime +from pathlib import Path +from unittest import mock + +from django.test import SimpleTestCase, tag +from django.utils import timezone + +from ingestion.scraping.scrapers.morrisvilleevents import MorrisvilleeventsScraper + +FIXTURE = Path(__file__).parent / "fixtures" / "morrisvilleevents.html" + +# Fixed "now" the fixture's events were built to be future-dated against +# (real data confirmed live 2026-07-31); the fixture also carries one +# back-dated 2020 event to exercise the past-event filter. +_FIXTURE_BUILD_TIME = timezone.make_aware(datetime(2026, 7, 31)) + + +@tag("fast") +class MorrisvilleeventsExtractTests(SimpleTestCase): + """Pure `extract()` against a saved real fixture — no DB, no browser, no network.""" + + def setUp(self): + self.html = FIXTURE.read_text(encoding="utf-8") + + @mock.patch("ingestion.scraping.scrapers.morrisvilleevents.timezone.now") + def test_extracts_future_events_and_drops_past(self, mock_now): + mock_now.return_value = _FIXTURE_BUILD_TIME + events = MorrisvilleeventsScraper().extract(self.html) + + # Fixture carries 5 events: 4 future, 1 back-dated to 2020 that the + # past-event filter must drop. + self.assertEqual(len(events), 4) + self.assertNotIn("Founders Day (back-dated for filter test)", [e.title for e in events]) + + @mock.patch("ingestion.scraping.scrapers.morrisvilleevents.timezone.now") + def test_event_fields(self, mock_now): + mock_now.return_value = _FIXTURE_BUILD_TIME + events = MorrisvilleeventsScraper().extract(self.html) + + psac = next(e for e in events if e.title == "Public Safety Advisory Committee Meeting") + self.assertEqual( + psac.start.isoformat(), + datetime(2026, 8, 4, tzinfo=psac.start.tzinfo).isoformat(), + ) + self.assertIsNotNone(psac.start.tzinfo) + self.assertIsNone(psac.end) + self.assertEqual( + psac.source_url, + "https://www.morrisvillenc.gov/Events-directory/" + "Public-Safety-Advisory-Committee-Meeting", + ) + self.assertEqual(psac.source_uid, psac.source_url) + self.assertIn( + "health, safety, and welfare", + psac.description, + ) + self.assertEqual(psac.location, "Fire Station No. 1, 200 Town Hall Drive, 27560") + + @mock.patch("ingestion.scraping.scrapers.morrisvilleevents.timezone.now") + def test_address_trailing_comma_stripped_when_no_zip(self, mock_now): + mock_now.return_value = _FIXTURE_BUILD_TIME + events = MorrisvilleeventsScraper().extract(self.html) + + music = next(e for e in events if e.title == "Music in the Park") + # Fixture's address has no zip: "Indian Creek Trailhead, 101 Town + # Hall Drive,  " -- trailing comma/whitespace must be trimmed. + self.assertEqual(music.location, "Indian Creek Trailhead, 101 Town Hall Drive") + + def test_no_items_yields_no_events(self): + self.assertEqual(MorrisvilleeventsScraper().extract("nope"), []) diff --git a/backendServer/ingestion/tests/test_patchdurham_extract_fast.py b/backendServer/ingestion/tests/test_patchdurham_extract_fast.py new file mode 100644 index 0000000..fa929d5 --- /dev/null +++ b/backendServer/ingestion/tests/test_patchdurham_extract_fast.py @@ -0,0 +1,62 @@ +from datetime import datetime +from pathlib import Path +from unittest import mock + +from django.test import SimpleTestCase, tag +from django.utils import timezone + +from ingestion.scraping.scrapers.patchdurham import PatchdurhamScraper + +FIXTURE = Path(__file__).parent / "fixtures" / "patchdurham.html" + +# Fixed "now" the fixture's events were built to be future-dated against +# (real data confirmed live 2026-07-31); the fixture also carries one +# back-dated 2020 event to exercise the past-event filter. +_FIXTURE_BUILD_TIME = timezone.make_aware(datetime(2026, 7, 31)) + + +@tag("fast") +class PatchdurhamExtractTests(SimpleTestCase): + """Pure `extract()` against a saved real fixture — no DB, no browser, no network.""" + + def setUp(self): + self.html = FIXTURE.read_text(encoding="utf-8") + + @mock.patch("ingestion.scraping.scrapers.patchdurham.timezone.now") + def test_extracts_future_events_from_next_data(self, mock_now): + mock_now.return_value = _FIXTURE_BUILD_TIME + events = PatchdurhamScraper().extract(self.html) + + # Fixture carries 4 events across day buckets: 3 future, 1 back-dated + # to 2020 that the past-event filter must drop. + self.assertEqual(len(events), 3) + self.assertNotIn("Past Event (back-dated for filter test)", [e.title for e in events]) + + concert = next(e for e in events if e.title.startswith("Beer Garden Concert Series")) + self.assertEqual(concert.start.isoformat(), "2026-07-31T22:00:00+00:00") + self.assertIsNotNone(concert.start.tzinfo) + self.assertIsNone(concert.end) + self.assertEqual(concert.source_uid, "20fdd37c-7c6b-4b89-9d06-e09553ee9838") + self.assertEqual( + concert.source_url, + "https://patch.com/north-carolina/durham-nc/calendar/event/20260731/" + "20fdd37c-7c6b-4b89-9d06-e09553ee9838/" + "beer-garden-concert-series-friday-night-edition-with-the-hourglass-kids", + ) + # Venue name plus street address, so the standardizer can infer the town. + self.assertEqual(concert.location, "The Glass Jug Beer Lab - RTP, 5410 NC-55, Durham, NC") + # `body` is HTML on the wire; the extractor stores plain text. + self.assertNotIn("

", concert.description) + self.assertNotIn("", concert.description) + self.assertIn("Glass Jug RTP", concert.description) + + @mock.patch("ingestion.scraping.scrapers.patchdurham.timezone.now") + def test_titles_are_html_unescaped(self, mock_now): + mock_now.return_value = _FIXTURE_BUILD_TIME + events = PatchdurhamScraper().extract(self.html) + + soap = next(e for e in events if "Soap Making" in e.title) + self.assertEqual(soap.title, "Sip & Create: Soap Making Workshop") + + def test_no_next_data_yields_no_events(self): + self.assertEqual(PatchdurhamScraper().extract("nope"), []) diff --git a/backendServer/ingestion/tests/test_patchmorrisville_extract_fast.py b/backendServer/ingestion/tests/test_patchmorrisville_extract_fast.py new file mode 100644 index 0000000..ead7be7 --- /dev/null +++ b/backendServer/ingestion/tests/test_patchmorrisville_extract_fast.py @@ -0,0 +1,64 @@ +from datetime import datetime +from pathlib import Path +from unittest import mock + +from django.test import SimpleTestCase, tag +from django.utils import timezone + +from ingestion.scraping.scrapers.patchmorrisville import PatchmorrisvilleScraper + +FIXTURE = Path(__file__).parent / "fixtures" / "patchmorrisville.html" + +# Fixed "now" the fixture's events were built to be future-dated against +# (real data confirmed live 2026-07-31); the fixture also carries one +# synthetic back-dated event to exercise the past-event filter. +_FIXTURE_BUILD_TIME = timezone.make_aware(datetime(2026, 7, 31)) + + +@tag("fast") +class PatchmorrisvilleExtractTests(SimpleTestCase): + """Pure `extract()` against a saved real fixture -- no DB, no browser, no network. + + Like Pittsboro, Morrisville's calendar carries only `patchAmFreeEvent` + nodes -- fetched from https://patch.com/north-carolina/morrisville-nc/calendar, + not the assigned section-landing URL (which has no `allEvents` at all). + """ + + def setUp(self): + self.html = FIXTURE.read_text(encoding="utf-8") + + @mock.patch("ingestion.scraping.scrapers.patch.timezone.now") + def test_extracts_future_events_from_next_data(self, mock_now): + mock_now.return_value = _FIXTURE_BUILD_TIME + events = PatchmorrisvilleScraper().extract(self.html) + + # Fixture carries 4 events across day buckets: 3 future, 1 back-dated + # to 2020 that the past-event filter must drop. + self.assertEqual(len(events), 3) + self.assertNotIn("Past Event (back-dated for filter test)", [e.title for e in events]) + + rock_climb = next(e for e in events if "Homeschool Rock Climb" in e.title) + self.assertEqual(rock_climb.start.isoformat(), "2026-08-27T10:00:00-04:00") + self.assertIsNotNone(rock_climb.start.tzinfo) + self.assertIsNone(rock_climb.end) + self.assertEqual(rock_climb.source_uid, "10717592") + self.assertEqual( + rock_climb.source_url, + "https://raleighfamilyadventure.com/event/homeschool-rock-climb-at-trc-morrisville/2026-08-27/", + ) + self.assertEqual( + rock_climb.location, + "Triangle Rock Club - Morrisville, 102 Pheasant Wood Ct, Morrisville, NC", + ) + self.assertEqual(rock_climb.description, "") + + @mock.patch("ingestion.scraping.scrapers.patch.timezone.now") + def test_evening_event_parses_pm_time(self, mock_now): + mock_now.return_value = _FIXTURE_BUILD_TIME + events = PatchmorrisvilleScraper().extract(self.html) + + parents_night = next(e for e in events if "Parents" in e.title) + self.assertEqual(parents_night.start.isoformat(), "2026-08-21T18:00:00-04:00") + + def test_no_next_data_yields_no_events(self): + self.assertEqual(PatchmorrisvilleScraper().extract("nope"), []) diff --git a/backendServer/ingestion/tests/test_patchpittsboro_extract_fast.py b/backendServer/ingestion/tests/test_patchpittsboro_extract_fast.py new file mode 100644 index 0000000..9666dde --- /dev/null +++ b/backendServer/ingestion/tests/test_patchpittsboro_extract_fast.py @@ -0,0 +1,68 @@ +from datetime import datetime +from pathlib import Path +from unittest import mock + +from django.test import SimpleTestCase, tag +from django.utils import timezone + +from ingestion.scraping.scrapers.patchpittsboro import PatchpittsboroScraper + +FIXTURE = Path(__file__).parent / "fixtures" / "patchpittsboro.html" + +# Fixed "now" the fixture's events were built to be future-dated against +# (real data confirmed live 2026-07-31); the fixture also carries one +# synthetic back-dated event to exercise the past-event filter. +_FIXTURE_BUILD_TIME = timezone.make_aware(datetime(2026, 7, 31)) + + +@tag("fast") +class PatchpittsboroExtractTests(SimpleTestCase): + """Pure `extract()` against a saved real fixture -- no DB, no browser, no network. + + Pittsboro's calendar carries only `patchAmFreeEvent` nodes (no + Patch-authored `event` nodes at all), so this exercises the aggregator + node shape: date + separate 12-hour time string, no body/description, + `externalUrl` as the source link. + """ + + def setUp(self): + self.html = FIXTURE.read_text(encoding="utf-8") + + @mock.patch("ingestion.scraping.scrapers.patch.timezone.now") + def test_extracts_future_events_from_next_data(self, mock_now): + mock_now.return_value = _FIXTURE_BUILD_TIME + events = PatchpittsboroScraper().extract(self.html) + + # Fixture carries 4 events across day buckets: 3 future, 1 back-dated + # to 2020 that the past-event filter must drop. + self.assertEqual(len(events), 3) + self.assertNotIn("Past Event (back-dated for filter test)", [e.title for e in events]) + + okra = next(e for e in events if "Okra Jamboree" in e.title) + self.assertEqual(okra.start.isoformat(), "2026-08-02T12:00:00-04:00") + self.assertIsNotNone(okra.start.tzinfo) + self.assertIsNone(okra.end) + self.assertEqual(okra.source_uid, "10837944") + self.assertEqual( + okra.source_url, + "https://thetriangleweekender.com/event/third-annual-okra-jamboree/", + ) + # `patchAmAddressStr` already reads "Venue, Street, City"; the region + # gets appended since it isn't part of that string. + self.assertEqual(okra.location, "The Plant, 220 Lorax Ln, Pittsboro, NC") + # No body/summary field exists on this node shape. + self.assertEqual(okra.description, "") + + @mock.patch("ingestion.scraping.scrapers.patch.timezone.now") + def test_titles_are_html_unescaped(self, mock_now): + mock_now.return_value = _FIXTURE_BUILD_TIME + events = PatchpittsboroScraper().extract(self.html) + + dance = next(e for e in events if "Inclusive Dance" in e.title) + self.assertEqual( + dance.title, "ALL-Together! Friends Group – Inclusive Dance and Activities" + ) + self.assertEqual(dance.start.isoformat(), "2026-08-06T11:00:00-04:00") + + def test_no_next_data_yields_no_events(self): + self.assertEqual(PatchpittsboroScraper().extract("nope"), []) diff --git a/backendServer/ingestion/tests/test_patchraleigh_extract_fast.py b/backendServer/ingestion/tests/test_patchraleigh_extract_fast.py new file mode 100644 index 0000000..fc58f2c --- /dev/null +++ b/backendServer/ingestion/tests/test_patchraleigh_extract_fast.py @@ -0,0 +1,65 @@ +from datetime import datetime +from pathlib import Path +from unittest import mock + +from django.test import SimpleTestCase, tag +from django.utils import timezone + +from ingestion.scraping.scrapers.patchraleigh import PatchraleighScraper + +FIXTURE = Path(__file__).parent / "fixtures" / "patchraleigh.html" + +# Fixed "now" the fixture's events were built to be future-dated against +# (real data confirmed live 2026-07-31); the fixture also carries one +# synthetic back-dated event to exercise the past-event filter. +_FIXTURE_BUILD_TIME = timezone.make_aware(datetime(2026, 7, 31)) + + +@tag("fast") +class PatchraleighExtractTests(SimpleTestCase): + """Pure `extract()` against a saved real fixture -- no DB, no browser, no network.""" + + def setUp(self): + self.html = FIXTURE.read_text(encoding="utf-8") + + @mock.patch("ingestion.scraping.scrapers.patch.timezone.now") + def test_extracts_future_events_from_next_data(self, mock_now): + mock_now.return_value = _FIXTURE_BUILD_TIME + events = PatchraleighScraper().extract(self.html) + + # Fixture carries 4 events across day buckets: 3 future, 1 back-dated + # to 2020 that the past-event filter must drop. + self.assertEqual(len(events), 3) + self.assertNotIn("Past Event (back-dated for filter test)", [e.title for e in events]) + + concert = next(e for e in events if e.title.startswith("Beer Garden Concert Series")) + self.assertEqual(concert.start.isoformat(), "2026-07-31T22:00:00+00:00") + self.assertIsNotNone(concert.start.tzinfo) + self.assertIsNone(concert.end) + self.assertEqual(concert.source_uid, "c65e76f8-3165-4bab-a398-76a4c6d8fbe6") + self.assertEqual( + concert.source_url, + "https://patch.com/north-carolina/raleigh/calendar/event/20260731/" + "c65e76f8-3165-4bab-a398-76a4c6d8fbe6/" + "beer-garden-concert-series-friday-night-edition-with-the-hourglass-kids", + ) + self.assertEqual(concert.location, "The Glass Jug Beer Lab - RTP, 5410 NC-55, Durham, NC") + self.assertNotIn("

", concert.description) + self.assertNotIn("", concert.description) + + @mock.patch("ingestion.scraping.scrapers.patch.timezone.now") + def test_address_with_only_a_name_field_falls_back_gracefully(self, mock_now): + mock_now.return_value = _FIXTURE_BUILD_TIME + events = PatchraleighScraper().extract(self.html) + + soap = next(e for e in events if "Soap Making" in e.title) + # streetAddress/city/region are all empty on this real record; only + # the concatenated `name` field carries the full address (with the + # site's real non-breaking spaces between segments, preserved as-is). + self.assertEqual( + soap.location, + "The Glass Jug Downtown\xa0545 Foster Street, Suite 10\xa0Durham, NC 27701", + ) + + def test_no_next_data_yields_no_events(self): + self.assertEqual(PatchraleighScraper().extract("nope"), []) diff --git a/backendServer/ingestion/tests/test_raleighnc_extract_fast.py b/backendServer/ingestion/tests/test_raleighnc_extract_fast.py new file mode 100644 index 0000000..6939bf2 --- /dev/null +++ b/backendServer/ingestion/tests/test_raleighnc_extract_fast.py @@ -0,0 +1,65 @@ +from datetime import datetime +from pathlib import Path +from unittest import mock + +from django.test import SimpleTestCase, tag +from django.utils import timezone + +from ingestion.scraping.scrapers.raleighnc import RaleighncScraper + +FIXTURE = Path(__file__).parent / "fixtures" / "raleighnc.html" + +# Fixed "now" the fixture's events were built to be future-dated against +# (real data confirmed live 2026-07-31); the fixture also carries one +# back-dated (Jan 1, 2026) teaser -- a copy of a real one with only its +# title and `datetime` edited -- to exercise the past-event filter. +_FIXTURE_BUILD_TIME = timezone.make_aware(datetime(2026, 7, 31)) + + +@tag("fast") +class RaleighncExtractTests(SimpleTestCase): + """Pure `extract()` against a saved real fixture -- no DB, no browser, no network.""" + + def setUp(self): + self.html = FIXTURE.read_text(encoding="utf-8") + + @mock.patch("ingestion.scraping.scrapers.raleighnc.timezone.now") + def test_extracts_future_events_from_teaser_list(self, mock_now): + mock_now.return_value = _FIXTURE_BUILD_TIME + events = RaleighncScraper().extract(self.html) + + # Fixture carries 5 "Upcoming Events" teasers: 4 future, 1 back-dated + # to Jan 1 that the past-event filter must drop. The "Ongoing Events" + # sidebar article (no `

", carnival.description) + + @mock.patch("ingestion.scraping.scrapers.thrivinginraleigh.timezone.now") + def test_address_deduplicated_when_map_link_repeats_venue_name(self, mock_now): + mock_now.return_value = _FIXTURE_BUILD_TIME + events = ThrivinginraleighScraper().extract(self.html) + + live_after_5 = next(e for e in events if e.title.strip() == "Live After 5") + # The maps.google.com link for this event is just "?q=Moore Square" -- + # identical to the venue text -- so location must not repeat itself. + self.assertEqual(live_after_5.location, "Moore Square") + + def test_no_events_yields_no_events(self): + self.assertEqual(ThrivinginraleighScraper().extract("nope"), []) diff --git a/backendServer/ingestion/tests/test_triangleonthecheap_extract_fast.py b/backendServer/ingestion/tests/test_triangleonthecheap_extract_fast.py new file mode 100644 index 0000000..7ab6c74 --- /dev/null +++ b/backendServer/ingestion/tests/test_triangleonthecheap_extract_fast.py @@ -0,0 +1,83 @@ +from datetime import datetime +from pathlib import Path +from unittest import mock + +from django.test import SimpleTestCase, tag +from django.utils import timezone + +from ingestion.scraping.scrapers.triangleonthecheap import TriangleonthecheapScraper + +FIXTURE = Path(__file__).parent / "fixtures" / "triangleonthecheap.html" + +# Fixed "now" the fixture's events were built to be future-dated against +# (real data confirmed live 2026-07-31); the fixture also carries one +# back-dated (Jan 1, 2026) day bucket -- a copy of a real event with only +# its header date and title edited -- to exercise the past-event filter. +_FIXTURE_BUILD_TIME = timezone.make_aware(datetime(2026, 7, 31)) + + +@tag("fast") +class TriangleonthecheapExtractTests(SimpleTestCase): + """Pure `extract()` against a saved real fixture -- no DB, no browser, no network.""" + + def setUp(self): + self.html = FIXTURE.read_text(encoding="utf-8") + + @mock.patch("ingestion.scraping.scrapers.triangleonthecheap.timezone.now") + def test_extracts_future_events_grouped_by_day_header(self, mock_now): + mock_now.return_value = _FIXTURE_BUILD_TIME + events = TriangleonthecheapScraper().extract(self.html) + + # Fixture carries 8 events: 5 under "Today" + 2 under "Tomorrow" (7 + # future) and 1 back-dated to Jan 1 that the filter must drop. + self.assertEqual(len(events), 7) + self.assertNotIn("Past Event (back-dated for filter test)", [e.title for e in events]) + + @mock.patch("ingestion.scraping.scrapers.triangleonthecheap.timezone.now") + def test_time_range_parses_start_hour_with_explicit_meridiem(self, mock_now): + mock_now.return_value = _FIXTURE_BUILD_TIME + events = TriangleonthecheapScraper().extract(self.html) + + tasting = next(e for e in events if e.title.startswith("Tasting at Ten")) + self.assertEqual(tasting.start.isoformat(), "2026-07-31T10:00:00-04:00") + self.assertIsNotNone(tasting.start.tzinfo) + self.assertEqual( + tasting.location, "Counter Culture Coffee Headquarters and Training Center, Durham" + ) + self.assertEqual( + tasting.source_url, + "https://triangleonthecheap.com/tasting-ten-free-tasting-tour-counter-culture-coffee-hq/", + ) + + @mock.patch("ingestion.scraping.scrapers.triangleonthecheap.timezone.now") + def test_all_day_defaults_to_midnight(self, mock_now): + mock_now.return_value = _FIXTURE_BUILD_TIME + events = TriangleonthecheapScraper().extract(self.html) + + baskin = next(e for e in events if e.title.startswith("Baskin Robbins")) + self.assertEqual((baskin.start.hour, baskin.start.minute), (0, 0)) + self.assertEqual(baskin.location, "Baskin-Robbins, multiple locations") + + @mock.patch("ingestion.scraping.scrapers.triangleonthecheap.timezone.now") + def test_duplicate_permalink_stays_unique_per_title_same_day(self, mock_now): + mock_now.return_value = _FIXTURE_BUILD_TIME + events = TriangleonthecheapScraper().extract(self.html) + + hillsborough = [ + e + for e in events + if e.source_url == "https://triangleonthecheap.com/hillsborough-last-fridays-art-walk/" + ] + # Three distinct listings share one permalink on the same date -- + # source_uid must disambiguate all three. + self.assertEqual(len(hillsborough), 3) + self.assertEqual(len({e.source_uid for e in hillsborough}), 3) + + live_on_lawn = next( + e for e in hillsborough if e.title == "Hillsborough Arts Council's Live on the Lawn" + ) + # 2-segment meta ("time | price", no venue segment) -> no location. + self.assertEqual(live_on_lawn.location, "") + + def test_no_day_headers_yields_no_events(self): + self.assertEqual(TriangleonthecheapScraper().extract("nope"), []) diff --git a/backendServer/ingestion/tests/test_visitraleigh_cary_extract_fast.py b/backendServer/ingestion/tests/test_visitraleigh_cary_extract_fast.py new file mode 100644 index 0000000..156336c --- /dev/null +++ b/backendServer/ingestion/tests/test_visitraleigh_cary_extract_fast.py @@ -0,0 +1,68 @@ +from datetime import datetime +from pathlib import Path +from unittest import mock + +from django.test import SimpleTestCase, tag +from django.utils import timezone + +from ingestion.scraping.scrapers.visitraleigh_cary import VisitraleighCaryScraper + +FIXTURE = Path(__file__).parent / "fixtures" / "visitraleigh_cary.html" + +# Fixed "now" the fixture's events were built to be future-dated against +# (real data confirmed live 2026-07-31); the fixture also carries one +# back-dated 2020 event to exercise the past-event filter. +_FIXTURE_BUILD_TIME = timezone.make_aware(datetime(2026, 7, 31)) + + +@tag("fast") +class VisitraleighCaryExtractTests(SimpleTestCase): + """Pure `extract()` against a saved real fixture -- no DB, no browser, no network. + + Exercises the same `_extract_events` helper as `visitraleigh.py` via the + Cary-scoped subclass, so this focuses on cases that module's own test + doesn't cover (the "Recurring ... until " no-start-date case) + rather than re-testing shared parsing logic end to end. + """ + + def setUp(self): + self.html = FIXTURE.read_text(encoding="utf-8") + + @mock.patch("ingestion.scraping.scrapers.visitraleigh.timezone.now") + def test_extracts_future_events(self, mock_now): + mock_now.return_value = _FIXTURE_BUILD_TIME + events = VisitraleighCaryScraper().extract(self.html) + + # Fixture carries 4 events: 3 future, 1 back-dated to 2020 that the + # past-event filter must drop. + self.assertEqual(len(events), 3) + self.assertNotIn("Past Event (back-dated for filter test)", [e.title for e in events]) + + @mock.patch("ingestion.scraping.scrapers.visitraleigh.timezone.now") + def test_recurring_listing_falls_back_to_until_date(self, mock_now): + mock_now.return_value = _FIXTURE_BUILD_TIME + events = VisitraleighCaryScraper().extract(self.html) + + clue = next(e for e in events if e.title == "Clue (High School Edition)") + # "Recurring weekly on Sunday, Friday, Saturday until August 9, 2026" + # has only one parseable date, so it becomes `start` with no `end`. + self.assertEqual(clue.start.isoformat(), "2026-08-09T19:30:00-04:00") + self.assertIsNone(clue.end) + self.assertEqual(clue.source_uid, "105919") + self.assertEqual(clue.location, "Raleigh Little Theatre") + + @mock.patch("ingestion.scraping.scrapers.visitraleigh.timezone.now") + def test_source_url_and_name(self, mock_now): + mock_now.return_value = _FIXTURE_BUILD_TIME + events = VisitraleighCaryScraper().extract(self.html) + + comedy = next(e for e in events if e.title.startswith("Comedy Show")) + self.assertEqual( + comedy.source_url, + "https://www.visitraleigh.com/event/comedy-show%3a-cyrus-steele-%2b-joe-perrow/108510/", + ) + self.assertEqual(VisitraleighCaryScraper.name, "Visit Raleigh (Cary)") + self.assertEqual(VisitraleighCaryScraper.key, "visitraleigh_cary") + + def test_no_recid_items_yields_no_events(self): + self.assertEqual(VisitraleighCaryScraper().extract("nope"), []) diff --git a/backendServer/ingestion/tests/test_visitraleigh_extract_fast.py b/backendServer/ingestion/tests/test_visitraleigh_extract_fast.py new file mode 100644 index 0000000..f10e5ef --- /dev/null +++ b/backendServer/ingestion/tests/test_visitraleigh_extract_fast.py @@ -0,0 +1,80 @@ +from datetime import datetime +from pathlib import Path +from unittest import mock + +from django.test import SimpleTestCase, tag +from django.utils import timezone + +from ingestion.scraping.scrapers.visitraleigh import VisitraleighScraper + +FIXTURE = Path(__file__).parent / "fixtures" / "visitraleigh.html" + +# Fixed "now" the fixture's events were built to be future-dated against +# (real data confirmed live 2026-07-31); the fixture also carries one +# back-dated 2020 event to exercise the past-event filter. +_FIXTURE_BUILD_TIME = timezone.make_aware(datetime(2026, 7, 31)) + + +@tag("fast") +class VisitraleighExtractTests(SimpleTestCase): + """Pure `extract()` against a saved real fixture -- no DB, no browser, no network.""" + + def setUp(self): + self.html = FIXTURE.read_text(encoding="utf-8") + + @mock.patch("ingestion.scraping.scrapers.visitraleigh.timezone.now") + def test_extracts_future_events(self, mock_now): + mock_now.return_value = _FIXTURE_BUILD_TIME + events = VisitraleighScraper().extract(self.html) + + # Fixture carries 4 events: 3 future, 1 back-dated to 2020 that the + # past-event filter must drop. + self.assertEqual(len(events), 3) + self.assertNotIn("Past Event (back-dated for filter test)", [e.title for e in events]) + + @mock.patch("ingestion.scraping.scrapers.visitraleigh.timezone.now") + def test_single_date_and_time(self, mock_now): + mock_now.return_value = _FIXTURE_BUILD_TIME + events = VisitraleighScraper().extract(self.html) + + rocky = next(e for e in events if e.title == "The Rocky Horror Picture Show") + self.assertEqual(rocky.start.isoformat(), "2026-07-31T20:30:00-04:00") + self.assertIsNotNone(rocky.start.tzinfo) + self.assertIsNone(rocky.end) + self.assertEqual(rocky.source_uid, "109424") + self.assertEqual( + rocky.source_url, + "https://www.visitraleigh.com/event/the-rocky-horror-picture-show/109424/", + ) + self.assertEqual(rocky.location, "The Rialto Theatre") + + @mock.patch("ingestion.scraping.scrapers.visitraleigh.timezone.now") + def test_date_range_with_vary_prefix(self, mock_now): + mock_now.return_value = _FIXTURE_BUILD_TIME + events = VisitraleighScraper().extract(self.html) + + andrew = next(e for e in events if e.title == "Andrew Orolfo") + # "Dates vary between July 31, 2026 - August 1, 2026", first time + # token "9:15pm" applied to the start date. + self.assertEqual(andrew.start.isoformat(), "2026-07-31T21:15:00-04:00") + self.assertEqual(andrew.end.isoformat(), "2026-08-01T00:00:00-04:00") + + @mock.patch("ingestion.scraping.scrapers.visitraleigh.timezone.now") + def test_date_range_without_vary_prefix(self, mock_now): + mock_now.return_value = _FIXTURE_BUILD_TIME + events = VisitraleighScraper().extract(self.html) + + bbb = next(e for e in events if e.title.startswith("Beer, Bourbon")) + self.assertEqual(bbb.start.isoformat(), "2026-07-31T18:00:00-04:00") + self.assertEqual(bbb.end.isoformat(), "2026-08-01T00:00:00-04:00") + self.assertEqual(bbb.location, "Koka Booth Amphitheatre") + + def test_no_recid_items_yields_no_events(self): + self.assertEqual(VisitraleighScraper().extract("nope"), []) + + def test_unrendered_template_placeholder_is_skipped(self): + html = ( + '

' + '

{{title}}

' + ) + self.assertEqual(VisitraleighScraper().extract(html), []) diff --git a/human-docs/README.md b/human-docs/README.md new file mode 100644 index 0000000..95a5182 --- /dev/null +++ b/human-docs/README.md @@ -0,0 +1,15 @@ +# Human Docs + +This folder is for people, not agents. `docs/` is the system of record that Claude reads before +every task (`CLAUDE.md` → `AGENTS.md` → `ARCHITECTURE.md` → `CODING_STYLE.md` → `docs/*.md`) — +keep that tree lean and agent-oriented. Anything written primarily for a human reader — an +inheriting owner, a new teammate, a neighbouring team — lands here instead, so it doesn't dilute +what agents load on every task. + +Written mainly by `/handoff-report`. See that skill for structure, grounding rules, and the +publish checklist. + +## Index + +| Doc | Purpose | Written | +|---|---|---| diff --git a/notion-sync/STATE.md b/notion-sync/STATE.md index 9ef3f4d..282b405 100644 --- a/notion-sync/STATE.md +++ b/notion-sync/STATE.md @@ -6,7 +6,7 @@ The ledger mirrors what *should* be on the Notion board so the desktop app can r --- -**Next suite number:** `39` +**Next suite number:** `41` ## Suite ledger @@ -34,6 +34,8 @@ per-ticket status lives on each ticket subpage (see OUTBOX preamble). | 36 | Ingestion funnel dead-ends (out-of-coverage limbo, town-less events, missed sends, beat bookkeeping) | Needs QA | 36.1–36.4, 36.6–36.7 (36.5 closed won't-fix; 36.8 investigation → benign, no code) | _(pending)_ | | 37 | Central auth reintegration — standalone portal + fix the live JWT bridge | Needs QA | 37.1–37.8, 37.10, 37.11 built+Needs QA (dev E2E 6/6); 37.9 Needs QA (prod cutover executed 2026-07-30, live JWT bridge verified); completes 29 + unbuilt half of 30 | _(pending)_ | | 38 | Password-required accounts + decoupled newsletter | Open | 38.A1–38.A4, 38.B1–38.B4, 38.D1 (planned, not built) | _(pending)_ | +| 39 | Password-reset flow wiring (unblock passwordless rollover) + auth model drift | Open | 39.1–39.2 (planned, not built) | _(pending)_ | +| 40 | Monitor dashboard rebuild + direct-submission attribution | Needs QA | 40.1–40.6 all built (40.3 = read-only prod audit, no code; confirmed 4/4 live direct Events orphaned → 40.4 shipped) | _(pending)_ | |"ingest-events-daily,\nweekly/monthly digest"| QDefault + Beat -->|scrape-sources-daily| QScrape + Beat -->|broadcast-orphan-recovery| QBroadcast + HTTPCron --> QDefault + HTTPPublish --> QDefault + HTTPSubmit --> QDefault + HTTPBroadcast -->|"transaction.on_commit\n(process_broadcast_queue)"| QBroadcast + + QDefault --> WDefault + QScrape --> WScrape + QBroadcast --> WBroadcast +``` + +**Five things worth calling out:** + +1. **The default queue is memory-light and network-bound** (LLM calls, Brevo email, plain + HTTP fetches), so it runs at `--concurrency=2` on the same 1-OCPU/6GB VM everything + else shares. `scrape` and `broadcast` each get their own `MemoryMax=2G`-capped, `-c 1` + worker specifically because both drive headless Chromium (Playwright), which is memory- + heavy enough that letting it share the default worker risked taking down gunicorn and + Next.js alongside it. +2. **The `broadcast` queue's `-c 1` is not a tuning knob — it's a correctness + requirement.** `recover_broadcast_orphans` assumes that any `BroadcastSubmission` still + in `status="running"` at the moment it runs is necessarily orphaned (the worker that + claimed it must have died mid-job). That assumption only holds if `process_broadcast_queue` + can never be mid-drain on a *second* worker at the same time — i.e., if there is + structurally only one broadcast worker process. Scaling `broadcast-worker.service` up + (more replicas, or dropping the `-c 1`) is the obvious-looking fix for "broadcast feels + slow" and it is wrong: it reintroduces exactly the race the single-worker constraint + exists to prevent. +3. **`process_broadcast_queue` is not on a beat schedule at all.** Every submission, + retry, and promote-to-real action calls `transaction.on_commit(process_broadcast_queue.delay)` + (`broadcast/services.py`) — the queue gets drained on demand, right after the DB row + that created the work actually commits, not on a poll loop. `recover_broadcast_orphans` + *is* on a beat schedule (every 6 hours) precisely because on-demand dispatch has no + mechanism to notice a submission that got stranded when nobody triggered a new one. +4. **Scrape and ingest are two separate beat schedules on two separate queues, not one + pipeline.** `run_ingestion_pipeline` (the `ingest-events-daily` task, on the default + queue) only polls `ics`-type sources. `scrape_all_sources_task` (on the `scrape` queue, + fired 30 minutes earlier so scraped rows land before the standardizer runs) polls + `scraper`- and `http`-type sources. They are deliberately not chained — a prior version + of `run_ingestion_pipeline`'s docstring claimed it mirrored the CLI command end-to-end, + which was false, and two `http`-type sources went unpolled for weeks before anyone + noticed. `python manage.py ingest_events` with no flags is the one entrypoint that + actually covers both legs in one pass. +5. **"default" is a convenience name, not a Celery setting.** Nothing in + `backend/settings/base.py` sets `CELERY_TASK_DEFAULT_QUEUE` — the queue everything lands + on by default is Celery's own built-in `celery` queue. `CELERY_TASK_ROUTES` only lists + the two exceptions (`scrape`, `broadcast`); everything else falls through to that + implicit queue, which `commons-default`'s worker drains because it starts with no `-Q` + flag at all. + +### 2.2 Where a schedule comes from, and how to change one + +Beat schedules are **not defined in a settings file you can grep for the current cron +times.** `django-celery-beat`'s `DatabaseScheduler` reads `CrontabSchedule`/`PeriodicTask` +rows out of Postgres, and Postgres — not any `.py` file — is the live truth. Code only +gets involved once, to seed the row the first time: + +```mermaid +flowchart TD + A["Write the @shared_task\nin the owning app's tasks.py"] --> B["Write a data migration:\nCrontabSchedule.objects.get_or_create(...)\nPeriodicTask.objects.update_or_create(name=..., task='app.tasks.func')"] + B --> C["Migration runs once,\non whichever DB applies it"] + C --> D["Row now lives in Postgres\n(django_celery_beat_periodictask /\n_crontabschedule)"] + D --> E["DatabaseScheduler polls Postgres\nand picks it up automatically —\nno beat restart needed"] + E --> F{"Need to change the time,\nenable/disable, or force\na one-off run later?"} + F -->|yes| G["Edit the PeriodicTask / CrontabSchedule\nlive in the django-unfold admin\n('Periodic Tasks')"] + G --> H["Takes effect on beat's next tick —\nno deploy needed"] + F -->|"no, need a permanent\nfresh-install default too"| I["Also edit the seed migration —\nit only affects DBs that haven't\napplied it yet"] +``` + +**Three things worth calling out:** + +1. **The migration is a one-time seed, not a sync.** It runs `get_or_create`/`update_or_create` + once, on first apply. Editing the migration file after it has already run on a given + database (dev, prod) does nothing to that database's existing row — you have to either + edit the row live (admin) or write a follow-up migration, the same way + `newsletter/migrations/0002_repoint_digest_beat.py` did to repoint the digest tasks' + dotted path after they moved apps (see § Interfaces below). +2. **To see the schedule that's actually running right now**, don't read a settings file — + open the django-unfold admin's "Periodic Tasks" page (or `PeriodicTask.objects.all()` + in a shell), on the database you actually care about (dev vs. prod are different + Postgres instances with potentially different live edits). +3. **A schedule change made in the admin is real and permanent for that database**, but + invisible to anyone reading the seed migrations in the repo. If prod's actual fire time + for a task ever needs to match what's in git, someone has to check both places. + +### 2.3 One beat tick, and why `last_run_at` can lie + +This is the flow behind the sharpest edge in this whole layer, so it earns its own +diagram. `django-celery-beat`'s `DatabaseScheduler` keeps `PeriodicTask.last_run_at` in +memory while it runs, and only writes it back to Postgres when Celery's own +`Scheduler._do_sync()` fires: + +```mermaid +sequenceDiagram + autonumber + participant Beat as celerybeat process + participant Mem as Beat's in-memory schedule + participant Broker as Redis DB 0 + participant Worker as commons-default worker + participant PG as Postgres (PeriodicTask row) + + Beat->>Mem: wake at the scheduled time (e.g. 04:00 ET) + Mem->>Broker: send task message (ingest-events-daily) + Mem->>Mem: update last_run_at in memory + Broker->>Worker: deliver message + Worker->>Worker: run_ingestion_pipeline() + Beat->>PG: sync last_run_at (forced after every task send,\nCELERY_BEAT_SYNC_EVERY = 1) +``` + +**Four things worth calling out:** + +1. **`CELERY_BEAT_SYNC_EVERY = 1` (in `backend/settings/base.py`) is why step 6 happens + right after every task send instead of on a timer.** Without it, the only sync triggers + are `CELERY_BEAT_MAX_LOOP_INTERVAL` (currently `6 * 60 * 60`, i.e. six hours — this + setting caps how long beat may sleep between waking to re-poll Postgres for schedule + *changes*, not how often it fires due tasks) and a 3-minute time-based trigger baked + into Celery itself. A task that fires shortly after the last sync could sit unflushed + in memory for up to six hours. +2. **This is not hypothetical — it's a confirmed 2026-07-29 prod incident** + (`docs/ingestion-monitoring.md`, "Beat scheduler: last_run_at persistence lag"). Before + `CELERY_BEAT_SYNC_EVERY = 1` was added, `broadcast-orphan-recovery` was firing exactly + on schedule every six hours, but the healthcheck kept reporting it `STALE` because + `last_run_at` in Postgres was still showing the *previous* run — the write was buffered + in memory and only flushed when celerybeat happened to restart. The task was fine; the + bookkeeping column was lying. +3. **Never widen a staleness window to paper over this.** The tempting fix when a + monitoring check false-alarms is to make the window bigger. That hides the false alarm + but also means a *real* multi-hour outage now takes that much longer to surface. The + actual fix is making the persisted value trustworthy (`CELERY_BEAT_SYNC_EVERY = 1`), not + giving the check more slack to be wrong in. +4. **If `CELERY_BEAT_SYNC_EVERY` is ever unset or set back to a falsy value, this lag + comes back exactly as it was** — `backendServer/events/management/commands/healthcheck.py`'s + `crontab_grace_seconds()` knows this and automatically widens its own grace window to + match `CELERY_BEAT_MAX_LOOP_INTERVAL` when that happens, so the healthcheck won't + immediately start crying wolf again — but the underlying lag itself is back, silently. + +### 2.4 Diagnosing "the job didn't run" + +`manage.py healthcheck` (run hourly on the VM by `healthcheck.timer` → `healthcheck.service`, +and manually via `bash deploy/healthcheck.sh`) is the tool for this. Its periodic-task +check doesn't use a single hand-tuned staleness window for every task — a window wide +enough to never false-alarm on a *weekly* task would let a missed *daily* task ride along +silently for over a week. Instead, wherever a task has a crontab, it asks the crontab +itself what the next expected fire time was and compares that to now: + +```mermaid +flowchart TD + A["For each enabled PeriodicTask"] --> B{"last_run_at is\nnull?"} + B -->|yes| FAIL1["FAIL — enabled, never run yet"] + B -->|no| C{"Task has a\ncrontab schedule?"} + C -->|no, interval or none| D{"Task name has an entry\nin DEFAULT_STALENESS_HOURS?"} + D -->|no| WARN1["WARN — no staleness window\nconfigured for this task"] + D -->|yes| E{"now minus last_run_at\nexceeds the configured window?"} + E -->|yes| FAIL2["FAIL — STALE"] + E -->|no| OK1["OK"] + C -->|yes| F["Derive expected next fire\nfrom the crontab itself\n(remaining_estimate, tz-corrected)"] + F --> G{"Expected fire\nalready passed?"} + G -->|no| OK2["OK — reports the derived next-fire time"] + G -->|yes| H{"Overdue by more than\ncrontab_grace_seconds()?\n(~5 min if BEAT_SYNC_EVERY is set,\nelse ~7h)"} + H -->|no| WARN2["WARN — overdue but within grace,\ncould be persistence lag, not a real miss"] + H -->|yes| FAIL3["FAIL — MISSED"] +``` + +**Three things worth calling out:** + +1. **`DEFAULT_STALENESS_HOURS` (in `healthcheck.py`) plays a narrower role than its name + suggests.** It's not the primary freshness check for the four crontab-backed seeded + tasks (`ingest-events-daily`, `scrape-sources-daily`, `weekly-digest-sunday`, + `broadcast-orphan-recovery`) — those are judged against their own crontab's expected + fire time. It's used for two narrower things: flagging a seeded task that's missing + from the schedule entirely (a FAIL, regardless of staleness), and as the only signal + available for a hypothetical interval-scheduled or schedule-less task, which has no + crontab to derive an expected time from. +2. **The grace window in step H exists because of § 2.3's persistence lag, and shrinks + automatically once that lag is fixed.** With `CELERY_BEAT_SYNC_EVERY = 1` (current + prod config), the grace is ~5 minutes — just enough for scheduler jitter and the task's + own run time. If that setting is ever lost, the grace widens itself back out to match + `CELERY_BEAT_MAX_LOOP_INTERVAL` so a `FAIL` doesn't start firing on every ordinary sync + delay. +3. **A `WARN` here is not "probably fine" — it means "cannot yet distinguish a real miss + from bookkeeping lag."** Don't dismiss it as noise; if a `WARN` on the same task + persists across multiple healthcheck runs (an hour or more apart), that's no longer + plausibly persistence lag and is worth treating as a real miss. + +--- + +## 3. Data model + +There's no domain data model here in the usual sense — the state this layer owns is +scheduling and caching metadata, not business data: + +| Table (owner) | Key fields | What it means | +|---|---|---| +| `django_celery_beat_periodictask` (`django_celery_beat`) | `name` (unique, human label like `ingest-events-daily`), `task` (dotted Python path, e.g. `ingestion.tasks.run_ingestion_pipeline`), `crontab` FK, `enabled`, `last_run_at`, `total_run_count` | One row per scheduled job. `task` is a plain string, not a real import reference — nothing breaks at write time if it points at a function that no longer exists; it just fails loudly the next time beat tries to send it. This is exactly why moving a task to a new app requires a *data migration* to repoint this column (§ 2.2), not just moving the Python code. | +| `django_celery_beat_crontabschedule` (`django_celery_beat`) | `minute`/`hour`/`day_of_week`/`day_of_month`/`month_of_year`, `timezone` | The crontab a `PeriodicTask` points at. `timezone` is stored per-row (`America/New_York` for the ingest/scrape/digest tasks, `UTC` for broadcast orphan recovery) — beat uses `TzAwareCrontab` (see § 5) to honor it. | +| Redis DB 0 keys (broker/result backend) | Celery-internal — task messages, results | Not meant to be read directly; `redis-cli -n 0 KEYS '*'` works for debugging but there's no app-level schema here worth documenting. | +| Redis DB 1 keys (`events/cache.py`) | `events:list:version` (an integer, bumped on every `Event`/write); `events:list:v{N}:{sha256-prefix-of-sorted-query-params}` (TTL 60s); `events:towns`, `events:categories` (plain keys, TTL 1h) | The read-endpoint cache. **The version-in-the-key trick exists because Django's stdlib `RedisCache` backend has no `delete_pattern`** — there's no way to invalidate "every cached event-list page" by pattern, so instead every write bumps a version counter and old-version keys just age out via TTL instead of ever being explicitly deleted. `Town`/`Category` writes clear their own plain keys directly instead, since those aren't parameterized by query string. | + +Invalidation is signal-driven, not task-driven: `events/signals.py`, registered in +`EventsConfig.ready()`, listens for `post_save`/`post_delete` on `Event`, `Town`, and +`Category` and calls the corresponding `events/cache.py` invalidation function inline, +synchronously, in the request/transaction that made the write — there's no Celery task +involved in cache invalidation at all. + +--- + +## 4. Interfaces + +### Periodic (beat-scheduled) tasks — the live schedule as seeded + +| `PeriodicTask.name` | Task (dotted path) | What it does | Queue | Cadence | Worker that drains it | +|---|---|---|---|---|---| +| `ingest-events-daily` | `ingestion.tasks.run_ingestion_pipeline` | Full ICS-leg pipeline: cleanup → poll ICS sources → Gemini standardize → dedup → safety-score → auto-publish | default | 04:00 daily, `America/New_York` | `commons-default` (`celery.service`) | +| `scrape-sources-daily` | `ingestion.tasks.scrape_all_sources_task` | Polls `scraper`- and `http`-type sources (fires 30 min before the daily ingest so rows land in time) | scrape | 03:30 daily, `America/New_York` | `commons-scrape` (`scrape-worker.service`) | +| `weekly-digest-sunday` | `newsletter.tasks.fan_out_weekly_digest` | Queues one `send_one_digest` per WEEKLY subscriber/user-profile | default | Sundays 18:00, `America/New_York` | `commons-default` | +| `monthly-digest-first` | `newsletter.tasks.fan_out_monthly_digest` | Queues one `send_one_digest` per MONTHLY subscriber/user-profile | default | 1st of month, 18:00, `America/New_York` | `commons-default` | +| `broadcast-orphan-recovery` | `broadcast.tasks.recover_broadcast_orphans` | Re-queues `BroadcastSubmission`/`BroadcastTarget` rows stranded by a crashed worker | broadcast | every 6 hours on the hour, `UTC` | `commons-broadcast` (`broadcast-worker.service`) | + +The two digest rows are the ones with history worth knowing: both were originally seeded +(by `events/migrations/0015_seed_digest_beat.py` and `0020_seed_monthly_digest_beat.py`) +pointing at `events.tasks.fan_out_weekly_digest`/`fan_out_monthly_digest`. When the digest +engine moved to the `newsletter` app, a follow-up data migration +(`newsletter/migrations/0002_repoint_digest_beat.py`) updated the `task` string on those +*same* rows to `newsletter.tasks.fan_out_weekly_digest`/`fan_out_monthly_digest` — the +`PeriodicTask.name` and its `CrontabSchedule` never changed, only the dotted path it +dispatches to. `docs/redis-celery-handoff.md` referred to the pre-move path; that's been +corrected as part of this pass. + +### On-demand tasks (not on any beat schedule) + +| Task | Queue | Triggered by | +|---|---|---| +| `ingestion.tasks.publish_all_approved_task` | default | `POST /api/events/publish-approved` (API key), and the admin's "Publish approved events" docs page | +| `ingestion.tasks.ingest_direct_submission_task` | default | `POST /api/events/direct-submit` — the broadcast SPA's host-submission flow | +| `newsletter.tasks.send_one_digest` | default | Called once per recipient by `fan_out_weekly_digest`/`fan_out_monthly_digest` — never called directly from a view | +| `broadcast.tasks.process_broadcast_queue` | broadcast | `transaction.on_commit(...)` inside `broadcast/services.py`, after `POST /broadcast/submit`, `/retry`, or `/submit-real` commits | +| `events.tasks.ping` | default | Nothing in the running app calls this — it exists purely as a smoke-test task (`events/tests/test_tasks_fast.py`), not part of the healthcheck's worker-liveness probe (that uses Celery's own `control.ping()` RPC, a different mechanism with the same name) | + +### Local dev commands + +```bash +# alongside runserver +uv run celery -A backend worker -l info # default queue +uv run celery -A backend beat -l info # scheduler +uv run celery -A backend worker -Q scrape -c 1 -l info # scrape queue +uv run celery -A backend worker -Q broadcast -c 1 -l info # broadcast queue +``` + +The test suite doesn't need any of this running — `backend/settings/test.py` sets +`CELERY_TASK_ALWAYS_EAGER = True`, so every `.delay()` call executes inline, synchronously, +in the test process, and exceptions propagate immediately instead of vanishing into a +worker that isn't there. + +### Adding a new periodic task, correctly + +1. Write the `@shared_task` function in the owning app's `tasks.py`. `autodiscover_tasks()` + (called from `backend/celery.py`) finds it automatically — no registration step. +2. If it needs to stay off the default queue (heavy browser work, or anything that + shouldn't share a process with digests/ingestion), add an entry to `CELERY_TASK_ROUTES` + in `backend/settings/base.py`, and give it a dedicated worker unit if it's going to run + in prod unattended (follow the pattern in `deploy/scrape-worker.service`). +3. Write a data migration in the owning app that does the `CrontabSchedule.get_or_create` + / `PeriodicTask.update_or_create` seed — copy the shape of e.g. + `ingestion/migrations/0012_seed_scrape_beat.py`. Pick the timezone deliberately: + `America/New_York` for anything user-facing/business-hours-sensitive, `UTC` for + internal crash-recovery-style jobs (that's the actual distinction the two existing + choices encode, not an arbitrary pick). +4. Run the migration wherever the task needs to actually be live (dev DB, then prod on + deploy). Nothing else needs restarting for the schedule itself to take effect — beat + picks up new/changed rows on its own poll of Postgres. +5. If the task ever moves to a different app afterward, its dotted `task` path has to be + repointed with a *second* data migration (see the digest fan-out example in § Interfaces + above) — moving the Python file alone silently orphans the schedule, since + `PeriodicTask.task` is just a string beat doesn't validate until send time. +6. Add it to `DEFAULT_STALENESS_HOURS` in `events/management/commands/healthcheck.py` only + if it has no crontab (interval-based) — crontab-backed tasks get their freshness judged + automatically from the crontab itself (§ 2.4). + +--- + +## 5. Sharp edges + +**`CELERY_BEAT_MAX_LOOP_INTERVAL` (6 hours) and `CELERY_BEAT_SYNC_EVERY` (1) work together, +and tuning one without the other silently reopens a monitoring blind spot.** Both live in +`backend/settings/base.py`. `CELERY_BEAT_MAX_LOOP_INTERVAL = 6 * 60 * 60` caps how long +beat may sleep before re-polling Postgres for schedule *changes* — a deliberate trade-off +to reduce Neon wake-ups, and worth keeping. `CELERY_BEAT_SYNC_EVERY = 1` is the fix for the +side effect that setting has on `last_run_at` bookkeeping: without it, a fired task's +`last_run_at` can sit unflushed in memory for up to that same six hours (§ 2.3), which is +exactly what happened in the confirmed 2026-07-29 prod incident. Never change a +staleness/monitoring window that reads `last_run_at` without checking both of these +settings' current values first — as of this commit they're `6 * 60 * 60` and `1` +respectively. + +**`TzAwareCrontab.remaining_estimate()` has a timezone bug that `is_due()` on the same +class doesn't.** `TzAwareCrontab` (from `django_celery_beat.tzcrontab`, the class every +seeded crontab actually uses) overrides `is_due(last_run_at)` to convert its argument into +the schedule's own timezone (`last_run_at.astimezone(self.tz)`) before delegating to the +base Celery `crontab` class — that conversion is what makes an `America/New_York` schedule +actually fire at the right wall-clock time. It does **not** override +`remaining_estimate()`, which is inherited unmodified from `celery.schedules.crontab` and +compares whatever `datetime` it's handed directly against the crontab's fields, with no +conversion at all. In real beat operation this is invisible — beat only ever calls +`is_due()`, and its own `nowfunc()` already returns time in the schedule's tz, so both +sides of every comparison beat makes are already in the same zone. It becomes a real bug +the moment anything calls `remaining_estimate()` directly with a UTC datetime, which is +exactly what `manage.py healthcheck`'s freshness check does (§ 2.4) — without an explicit +`.astimezone(schedule.tz)` conversion on both operands first, `"0 4 * * *"`/ +`America/New_York` gets silently read as "04:00 UTC," landing the expected-fire estimate +four to five hours off **every single day**, not just around a DST transition. The +healthcheck code already carries this conversion (see the comment in +`_crontab_freshness` in `healthcheck.py`) — the edge to remember is that it's a property of +*that call site*, not of `TzAwareCrontab` in general: any new code that calls +`remaining_estimate()` directly (rather than going through beat's own `is_due()` machinery) +has to redo the same conversion by hand, or it will be wrong in exactly this way. + +**Beat schedules are database rows, not code — the file you'd grep for the current cron +time can be lying.** A seed migration only writes its `CrontabSchedule`/`PeriodicTask` once, +on first apply, and django-unfold admin edits made afterward change the row live without +ever touching a migration file. Reading `ingestion/migrations/0007_seed_ingest_beat.py` +tells you what the schedule *was* the day that migration was written and applied — it does +not tell you what's actually firing in prod right now if anyone has since edited it in the +admin. § 2.2 covers how to check the live truth. + +--- + +## 6. Known gaps and doc drift + +- **`ARCHITECTURE.md`'s Broadcast section is stale**, per § 1 above — it says broadcast + "does not use Celery," which was true before the queue-drain and orphan-recovery logic + moved into `broadcast/tasks.py`'s `@shared_task` functions. The Async section further + down the same file is accurate; only the Broadcast section wasn't updated to match. +- **`docs/redis-celery-handoff.md`'s "Testing" section says the suite runs under dev + settings with `CELERY_TASK_ALWAYS_EAGER = False`.** That's backwards from what's actually + configured: the suite runs under `backend.settings.test` (which inherits from `dev.py` + but then explicitly sets `CELERY_TASK_ALWAYS_EAGER = True`), so tasks run inline by + default and the `@override_settings` pattern shown there is only needed if a test + deliberately wants *non*-eager behavior, not the reverse. This wasn't in scope to correct + on that file for this pass (only its stale digest task-path references were) — flagging + it here so it doesn't get propagated further. +- **This doc did not independently verify `docs/redis-celery-handoff.md`'s prod + provisioning commands** (Redis install/config, systemd `cp`/`enable` sequence) — they + read as plausible and internally consistent with the service files in `deploy/`, but + weren't re-run against a live VM for this pass. Treat that doc as authoritative for the + exact commands; this doc is about the shape of the system, not a runbook. +- **No consumer of `events.tasks.ping` exists outside its own test.** It's either a leftover + smoke-test scaffold from before `manage.py healthcheck`'s `control.ping()`-based worker + check existed, or an intentionally-kept minimal example task — nothing in the codebase + says which, and it wasn't worth guessing at here. diff --git a/human-docs/auth.md b/human-docs/auth.md new file mode 100644 index 0000000..94ae72f --- /dev/null +++ b/human-docs/auth.md @@ -0,0 +1,353 @@ +# The Auth Bridge — Better Auth and Django + +*Reflects commit `5fe7a45`, 2026-08-01. Written by reading `backendServer/accounts/` (models, +admin, urls, views, migrations), `backendServer/backend/jwt_auth.py`, +`backendServer/backend/permissions.py`, `backendServer/backend/settings/` (`__init__.py`, +`base.py`, `dev.py`, `prod.py`), `theCommonsWeb/src/lib/auth.ts`, `auth-schema.ts`, +`auth-client.ts`, `db.ts`, `theCommonsWeb/src/hooks/useAuth.tsx`, the portal route group +(`src/app/(portal)/`), `src/app/reset-password/`, `src/lib/redirect-allowlist.ts`, and +`broadcastWeb/src/App.tsx`. Complements `ARCHITECTURE.md`'s Authentication section, which is +also current as of this commit; the two shouldn't drift, and if they ever do, trust the code. +Companion docs referenced below: `overview.md` (system map), `data-model.md` (full schema), +`deploy-ops.md` (VM/nginx/systemd), `frontend.md` (Next.js app structure).* + +The single fact that matters more than any other in this codebase: **Django does not own user +accounts.** There is no `django.contrib.auth.User` for app users, no Django login view, no +Django-issued session. Identity lives entirely inside the Next.js app (`theCommonsWeb`), owned +by an embedded library called **Better Auth**. Django's job is narrower and more mechanical than +a newcomer usually assumes: it keeps a read-only mirror of Better Auth's tables so it can `JOIN` +against them, and it verifies a token that Better Auth issued. It never creates a user, never +checks a password, and never has the authority to invalidate a session. If you find yourself +about to add a login endpoint to Django, or wondering why `python manage.py createsuperuser` +doesn't create an account regular users can sign in with, that's the inversion this doc exists +to correct. + +## 1. What this is and who depends on it + +Two systems share one identity: **Better Auth**, running inside `theCommonsWeb`'s Next.js +process, and Django's `accounts` app, which mirrors Better Auth's tables just enough to join +against them and verify the tokens Better Auth mints. Every other backend app that needs to know +"who is this request from" — `events`, `newsletter`, `broadcast`, `ingestion` — depends on +`accounts` for that answer; none of them talk to Better Auth directly. + +Concretely, three things depend on this bridge working: + +- **The main site** (`theCommonsWeb`) — every authenticated page (`/profile`, `/dashboard`, + `/post`) needs both a live Better Auth session (for the UI) and a valid JWT (to call Django's + API for profile data, event ownership, and business listings). +- **The broadcast SPA** (`broadcastWeb`) — a separate Vite app on its own subdomain that + authenticates against the *same* Better Auth instance via a shared cookie, so a user who signs + in on the main site is already signed in there, and vice versa. +- **Django itself** — `BearerTokenAuthentication` (`backend/permissions.py`) is the + authentication class nearly every non-public DRF view in this repo declares. If JWT + verification breaks, every one of those views starts rejecting real users, and the failure + mode looks nothing like "auth is down" — see Sharp edge 1. + +If this bridge is down, users can typically still sign in on the frontend (Better Auth doesn't +depend on Django), but every page that calls the Django API for personalized data — profile, +dashboard, event submission, business listings, all of `broadcast/` — starts failing or serving +generic/anonymous responses. + +## 2. How it works + +### 2.1 Signing in or creating an account + +Every place in the product that needs a user to authenticate — the main site's header, the +broadcast SPA's "Sign in" button, the old `/auth` legacy routes — does the same thing: a full +browser navigation to the portal (`/signin` or `/join`, on `theCommonsWeb`, at the shared auth +origin) with a `?redirect_to=` query param. Sign-up and sign-in converge on the +same downstream steps once Better Auth accepts the credentials, so this is one diagram with an +`alt` for the branch that differs — account creation also fires a database hook that inserts a +matching Django-side profile row. + +```mermaid +sequenceDiagram + autonumber + participant Browser + participant PortalUI as Portal UI (JoinForm / SignInForm, via useAuth) + participant BetterAuth as Better Auth (theCommonsWeb, /api/auth/*) + participant PG as Postgres + participant Django + + Browser->>PortalUI: GET /join or /signin?redirect_to=... + alt Creating an account + PortalUI->>BetterAuth: authClient.signUp.email(email, password, user_type) + BetterAuth->>PG: INSERT neon_auth.user, neon_auth.account (provider=credential) + BetterAuth->>PG: databaseHooks.user.create.after - INSERT public.events_userprofile + Note over BetterAuth,PG: best-effort - a failed mirror insert is caught and logged,
never rolled back, since Better Auth already committed the login-usable account + else Signing in + PortalUI->>BetterAuth: authClient.signIn.email(email, password) + BetterAuth->>PG: look up neon_auth.account (provider=credential), verify password + end + BetterAuth-->>Browser: Set-Cookie session, cross-subdomain when BETTER_AUTH_COOKIE_DOMAIN is set + PortalUI->>BetterAuth: GET /api/auth/token (cookie sent) + BetterAuth-->>PortalUI: short-lived JWT (jwt() plugin - sub=user id, email claim) + PortalUI->>Django: GET /events/me/profile, Authorization Bearer JWT + Django-->>PortalUI: profile JSON (verification internals: next diagram) + PortalUI->>Browser: window.location.href = resolveRedirect(redirect_to) + Note over Browser: full navigation, not a client route change - lands back on the calling app,
whose own useAuth re-resolves session + JWT + profile independently on mount +``` + +**Five things worth calling out about this picture:** + +1. **The JWT is fetched and re-fetched from scratch, never stored.** `useAuth.tsx` keeps the JWT + only in React state — nothing is written to `localStorage` or a non-httpOnly cookie. Every + fresh page load (including the one after the portal's full navigation back) calls + `authClient.getSession()` then `/api/auth/token` again. That's simple and safe against XSS + token theft, but it means a slow or failing `/api/auth/token` call blocks every authenticated + page load, not just sign-in. +2. **The account-creation side effect can silently fail without blocking signup.** The + `databaseHooks.user.create.after` hook in `theCommonsWeb/src/lib/auth.ts` inserts into + `public.events_userprofile` wrapped in a try/catch specifically so a schema hiccup on the + Django side can never roll back the `neon_auth.user`/`account` rows Better Auth just + committed — a user must never be left with a login that immediately breaks. The trade-off is + a `UserProfile`-less `BetterAuthUser` if that insert ever does fail, which most `accounts` + views assume can't happen (`me`, `businesses`, etc. all `.filter(user_id=...).first()` and + 404 rather than crash, so this degrades gracefully, but there's no reconciliation job that + backfills the missing row). +3. **The physical table name is a historical fossil, not a bug.** That `INSERT INTO + public.events_userprofile` targets a table whose Django model is `accounts.UserProfile` + today — the app moved in a state-only migration (§5, Sharp edge 5) but the table's physical + name never changed, and this raw-SQL insert (written before the move, unchanged since) + still says `events_userprofile` on purpose. +4. **Cross-subdomain cookies are conditional, not a repo-wide constant.** `BETTER_AUTH_COOKIE_DOMAIN` + is only set to `.thecommons.town` in production; when it's unset (local dev), Better Auth's + default `SameSite=Lax`, host-scoped cookie applies instead, so a session on `localhost:3000` + won't be visible to a broadcast dev server on `localhost:5173` — you sign in separately in + dev, and that's expected, not broken. In prod, the cookie is scoped to the whole + `.thecommons.town` domain with `SameSite=None; Secure`, which is what lets someone who signs + in on the apex site show up already signed in on `broadcast.thecommons.town` — and what lets + `broadcastWeb`'s "Sign in" button (`App.tsx`) just navigate to + `${VITE_BETTER_AUTH_URL}/signin?redirect_to=` and rely on the shared cookie to + bring the user right back in. +5. **Password sign-up is the only path today.** `emailAndPassword: { enabled: true, autoSignIn: + true }` in `auth.ts` is the entire signup surface — `signUp.email` creates the credential + account and signs the user in in the same call. Google sign-in exists in the codebase + (`socialProviders.google`) but is commented out; there's no passwordless flow left either + (see Sharp edge 4). + +### 2.2 Django verifying a Bearer JWT against JWKS + +This is the other half of the bridge, and the one that actually enforces anything: every +protected DRF view in this repo uses `BearerTokenAuthentication` +(`backend/permissions.py`), which accepts either the shared `THE_COMMONS_API_KEY` (no user +attached — for server-to-server calls like the ingestion pipeline) or a Better Auth JWT, +verified statelessly against Better Auth's published JWKS endpoint. Django never calls back into +Next.js to ask "is this session valid" — it downloads Better Auth's public signing keys once, +caches them in-process, and verifies the JWT's signature locally. + +```mermaid +sequenceDiagram + autonumber + participant Client + participant View as DRF view + participant Auth as BearerTokenAuthentication + participant JWTAuth as jwt_auth.verify_better_auth_jwt + participant JWKS as Better Auth JWKS endpoint + participant DB as neon_auth.user (Postgres) + + Client->>View: Authorization Bearer token + View->>Auth: authenticate(request) + alt token equals THE_COMMONS_API_KEY + Auth-->>View: (None, None) - authorized, no user attached + else Better Auth JWT + Auth->>JWTAuth: verify_better_auth_jwt(token) + alt JWKS client cached and under 600s old + Note over JWTAuth: reuse the in-process PyJWKClient, no network call + else cache cold, expired, or never fetched + JWTAuth->>JWKS: GET jwks.json, browser User-Agent header set explicitly + JWKS-->>JWTAuth: signing keys, cache refreshed + Note over JWTAuth: on fetch failure, reuse the last-known client for up to
1h stale-grace before returning None (fail closed) + end + JWTAuth->>JWTAuth: jwt.decode against the resolved signing key (EdDSA/RS256/ES256) + alt no client, or decode fails + JWTAuth-->>Auth: None + Auth-->>View: raise AuthenticationFailed - Invalid token + else decode succeeds + JWTAuth-->>Auth: claims (sub, email, ...) + Auth->>DB: BetterAuthUser.objects.get(id=claims.sub) + DB-->>Auth: user row + Auth-->>View: (user, claims) + end + end +``` + +**Three things worth calling out about this picture:** + +1. **The browser User-Agent header on the JWKS fetch is load-bearing, not decoration** — see + Sharp edge 1. It looks like the kind of line a cleanup pass removes. +2. **The stale-grace fallback exists so a Next.js deploy or blip doesn't cascade into Django + auth.** For up to an hour after the last successful JWKS fetch, a failing refresh reuses the + old signing keys rather than rejecting every request. Past that window, verification fails + closed (`None`), which surfaces to callers as `AuthenticationFailed`. +3. **`broadcast/` calls `verify_better_auth_jwt` directly**, skipping the `BetterAuthUser` ORM + lookup step entirely (`broadcast/access.py`) — by design, to keep `broadcast/` from importing + anything outside itself. It gets back claims, not a Django user object, and resolves an + access tier from the email claim instead. + +## 3. Data model + +`accounts` owns two kinds of tables: five **Better Auth mirrors** it never writes to, and two +**Django-owned profile tables** that hang off them. + +| Model | Schema / table | Managed by Django | What it's for | +|---|---|---|---| +| `BetterAuthUser` | `neon_auth.user` | No (`managed = False`) | The account record — id, email, name. Hardcodes `is_authenticated = True` / `is_anonymous = False` as class attributes so DRF's permission classes treat it as a real authenticated user without a database round trip for that check. | +| `BetterAuthSession` | `neon_auth.session` | No | Better Auth's server-side session record. Django never reads this for request auth — auth is JWT-based, not session-based, on the Django side. | +| `BetterAuthAccount` | `neon_auth.account` | No | One row per sign-in method (`provider_id = 'credential'` for password auth). Holds the hashed password for credential accounts. `user_id` is a `UUIDField` — see Sharp edge 3. | +| `BetterAuthVerification` | `neon_auth.verification` | No | Better Auth's internal token bookkeeping (password-reset tokens, etc). | +| `BetterAuthJwks` | `neon_auth.jwks` | No | The signing keypair(s) Better Auth's `jwt()` plugin uses. Django never reads this table directly — it fetches the public half over HTTP from the JWKS endpoint instead (§2.2), not from the database. | +| `UserProfile` | `public.events_userprofile` | Yes | `user_type` (LOCAL/BUSINESS/VENUE), `primary_city`, `address`, `email_preference`, tag interests. `OneToOneField` to `BetterAuthUser` with `db_constraint=False` (no DB-level FK across the schema boundary onto an unmanaged table). | +| `BusinessProfile` | `public.events_businessprofile` | Yes | Business-account listing data — name, description, contact info, service area, publish state. Also `OneToOneField` to `BetterAuthUser`, `db_constraint=False`. | + +Every `neon_auth` model's `db_table` uses a double-quote trick to cross a schema boundary Django +doesn't natively support in this form — `db_table = 'neon_auth"."user'` — so that when Django +wraps it in its own quoting (`"` + value + `"`), the emitted SQL comes out as +`FROM "neon_auth"."user"`, a valid Postgres cross-schema reference. It reads like a stray quote +character; it isn't one. + +The full event/newsletter/ingestion/broadcast schema (and how those tables relate to the ones +here) is `data-model.md`'s job, not this doc's — this table only covers the identity slice. + +## 4. Interfaces + +| Method | Path | Auth | Notes | +|---|---|---|---| +| GET/PATCH | `/auth/me` | Bearer JWT | Read/update the caller's own `UserProfile` — email preference, city, address, tags, and (once, LOCAL to BUSINESS/VENUE only) account type. PATCH also syncs a `NewsletterSubscriber` row to match `email_preference`. | +| GET/POST | `/businesses` | Bearer JWT | Browse published businesses (VENUE accounts only) / create a listing (BUSINESS accounts only). | +| GET | `/businesses/me` | Bearer JWT | Caller's own business listing. | +| GET/PATCH/DELETE | `/businesses/` | Bearer JWT | Business listing CRUD, owner-only for write. | +| GET/POST | `/api/auth/*` | — | Not a Django route — Better Auth's own catch-all handler in `theCommonsWeb` (`src/app/api/auth/[...all]/route.ts`). This is where `/api/auth/token` (JWT mint), `/api/auth/jwks` (public keys), and the password/reset-password endpoints actually live. | + +Every `accounts` view above uses `BearerTokenAuthentication` + DRF's `IsAuthenticated` — there is +no Django session auth and no CSRF token involved, by design (see §1). A request with no +`Authorization` header, an expired JWT, or a JWT whose `sub` doesn't resolve to a +`BetterAuthUser` all end up as `401`; a valid user hitting an endpoint gated on the wrong +`user_type` (e.g. a LOCAL account calling `POST /businesses`) gets `403`, not `401` — worth +knowing when a bug report says "I'm logged in but I get an error." + +## 5. Sharp edges + +**1. The JWKS fetch needs a browser-like User-Agent, or every JWT verification fails.** +Cloudflare sits in front of the auth origin and 403s the default `python-urllib` User-Agent that +`PyJWKClient` sends by default. `backend/jwt_auth.py` sets an explicit +`_JWKS_HEADERS = {"User-Agent": "Mozilla/5.0 (compatible; TheCommons/1.0)"}` on both the +pre-flight `requests.get` and the `PyJWKClient` itself specifically to route around this. If +someone "cleans up" that header as unnecessary boilerplate, the JWKS fetch starts getting 403'd +by Cloudflare, `_get_jwks_client` returns `None` (once any stale-grace window expires), +`verify_better_auth_jwt` returns `None` for every token, and every Bearer-JWT-protected endpoint +in the app starts returning 401/403 for real, legitimately-signed-in users. This is exactly what +happened once in production — broadcast's tier-2 endpoints started blanket-403ing, and the +symptom (auth suddenly broken everywhere) looked nothing like the cause (one HTTP header). + +**2. `DJANGO_ENV` has to be exactly `prod` for the production settings to load — and the +failure mode for getting this wrong is still partly silent.** `backend/settings/__init__.py` +resolves `DJANGO_ENV` to `dev` or `prod` before anything else loads: + +```mermaid +flowchart TD + A[DJANGO_ENV read from the environment] --> B{Value} + B -->|unset or blank| C[dev.py loads - DEBUG True, ALLOWED_HOSTS is localhost only] + B -->|prod, case-insensitive, trimmed| D[prod.py loads - DEBUG False, ALLOWED_HOSTS from DJANGO_ALLOWED_HOSTS] + B -->|anything else, e.g. production| E[ImproperlyConfigured raised at import time, process fails to start] + C --> F[If this happens on the public host: every request 400s with DisallowedHost, site looks like it has no events] +``` + +A *typo'd* value now fails loudly (`ImproperlyConfigured` at process startup) — that part used +to be silent and has since been hardened, with a regression test +(`events/tests/test_config_fast.py::SelectSettingsEnvTests`) locking the behavior in. What's +*still* true, and still by design (every local dev setup relies on it), is that an **unset** +`DJANGO_ENV` silently resolves to `dev`. If the production VM's `.env` is ever missing that line +entirely, the app comes up as `dev.py` — `DEBUG=True`, `ALLOWED_HOSTS=["localhost", "127.0.0.1"]` +— and every request to the real public host gets rejected with `DisallowedHost` (HTTP 400), +which the frontend renders as "no events." `manage.py healthcheck --require-prod` now exists +specifically to catch this after the fact (it fails loud if `DEBUG` is `True` or `ALLOWED_HOSTS` +is localhost-only) and is wired into the periodic health check, but it's a detector, not a +guard — it doesn't stop a misconfigured deploy from going live, it just makes the misconfig +visible fast instead of only when someone notices the site is empty. + +**3. `BetterAuthAccount.user_id` is a `UUIDField`, matching the real column — this used to be +wrong.** The live `neon_auth.account."userId"` column is `uuid`. The Django mirror model +originally declared it as a `TextField`, which worked fine for simple lookups but broke any ORM +join across it — `operator does not exist: uuid = text` — because Postgres won't implicitly +compare those types. This was fixed by changing the field to `UUIDField(db_column="userId")`, +and there's now a standing regression test +(`BetterAuthAccountUserIdFieldTests` in `events/tests/test_config_fast.py`) asserting the field +type matches. If someone reverts that field back to `TextField` — plausible, since a lone +`get()`-by-id lookup on this model still works fine with either type and gives no visible signal +anything's wrong — any code doing a cross-model anti-join (exactly the shape the passwordless +account rollover, Sharp edge 4, needs) breaks again at query time. Because this model is +`managed = False`, nothing about the database enforces the Django field type matches reality; +only the test does. + +**4. Password-required accounts and the passwordless rollover: wiring is done, the actual send +looks outstanding.** Before password requirements landed, an account could exist with no +password at all — sign-in was just "enter your email." That flow is gone: `SignInForm.tsx` +requires both fields, and there is no other way to reach a session without one. Anyone whose +account predates that change is locked out with no password to type. The fix that's since landed +threads all the way through: `theCommonsWeb/src/lib/auth.ts` now wires +`emailAndPassword.sendResetPassword` (sends the reset email directly via Brevo's REST API from +the Next.js server), `/forgot-password` calls `authClient.requestPasswordReset` for real instead +of showing a stub message, and a new public `/reset-password` page consumes the token via +`authClient.resetPassword`. The identification side is also built: +`events.management.commands.rollover_passwordless_accounts` finds affected users with a raw-SQL +anti-join against `neon_auth` (deliberately raw SQL, not the ORM — the `neon_auth` mirror tables +aren't built in the test database, so there's no way to exercise an ORM version of this query in +CI) and, with `--send`, emails each one a rollover notice. What isn't confirmed from the repo: +whether that command has actually been run with `--send` against production yet. Its own +docstring, and the runbook it points to (`docs/suite-38-passwordless-rollover.md`), still +describe the reset link as a dead end and the whole effort as blocked — that prose predates the +wiring above and is now stale; don't trust it over the code in `auth.ts` and +`ForgotPasswordForm.tsx`/`ResetPasswordForm.tsx`. Before ever passing `--send`, verify the +current prerequisites for real (an unset `BREVO_API_KEY` in the frontend's production +environment makes `/forgot-password` report success while sending nothing), and treat the +management command's dry-run output as the actual list of who's still stranded. + +**5. `neon_auth` is never migrated by Django, and the app-extraction migration that moved +`UserProfile`/`BusinessProfile` into `accounts` is deliberately state-only.** Every `neon_auth` +mirror model carries `managed = False` for exactly one reason: Better Auth owns those tables +completely, and a Django `migrate` must never touch them. If that flag were ever dropped or +flipped — "cleaning up" what looks like a stray `False` — the next `makemigrations` would try to +generate real `CreateModel`/`AlterField` operations against tables that already exist with a +different owner, and `migrate` would then attempt DDL against `neon_auth.user`, +`neon_auth.account`, and the rest, which at best fails outright and at worst silently diverges +Django's idea of the schema from the one Better Auth is actually writing to — breaking +authentication for the whole app, not just the mirror. Related, and worth reading correctly: the +migration that moved `UserProfile`/`BusinessProfile` (and all five mirrors) into the `accounts` +app, `accounts/migrations/0001_initial.py`, is built entirely from +`migrations.SeparateDatabaseAndState` with `database_operations=[]` — it *looks* like it creates +seven tables (there's a `CreateModel` for every one of them, mirrors included), but zero DDL +actually runs. It only changes which Django app the ORM believes owns each model; every +`db_table` (`events_userprofile`, `events_businessprofile`, and every `neon_auth.*` mirror) stays +physically exactly where it was. A newcomer skimming that migration file and assuming it built +those tables would be wrong in a way that matters if they ever try to reason about what +`migrate` did or didn't touch. + +## 6. Known gaps + +- **The passwordless-rollover command and its guardrail test still live under `events/`, not + `accounts/`**, even though they exist entirely to work around a bug in `accounts.models`. + `events/management/commands/rollover_passwordless_accounts.py`'s own docstring still says + `BetterAuthAccount.user_id (events/models.py)` — that model has lived in `accounts/models.py` + since the app extraction (§5). Neither file was moved when `accounts` was carved out of + `events`; functionally harmless (the import path in the code itself is correct — only the + comment and the file's location are stale), but worth relocating alongside any future touch of + either file. +- **Whether the passwordless-rollover email has actually been sent to production users is not + verifiable from the repository.** See Sharp edge 4 — the wiring is confirmed done, the send is + not confirmed either way. +- **No email verification.** Signup doesn't gate on a verified email for MVP (`emailAndPassword` + has no `requireEmailVerification`, and there's no verify-email route in + `theCommonsWeb/src/app`). Confirmed by absence rather than by finding an explicit "no + verification" flag — worth a second look if this doc is read much later than its date above. +- **Google sign-in is fully wired in Better Auth's config but commented out** + (`socialProviders.google` in `auth.ts`), and the client-side popup flow that used to call it + was deleted along with the pre-portal auth UI. Re-enabling it needs a new post-OAuth + account-type step built into the portal (it used to bypass the LOCAL/BUSINESS/VENUE choice + entirely) — not just uncommenting the block. +- **Production nginx routing for the auth origin was not verified for this doc.** This repo's + deploy tooling was mid-change while this doc was written (a Docker-based nginx config was + present but not yet part of a committed state), so the exact current routing for + `auth.thecommons.town` is `deploy-ops.md`'s claim to verify, not this one's — everything above + about cookies, trusted origins, and the JWKS URL is grounded in application code and env-var + wiring, not in reading the live nginx config. diff --git a/human-docs/broadcast.md b/human-docs/broadcast.md new file mode 100644 index 0000000..95b4eaa --- /dev/null +++ b/human-docs/broadcast.md @@ -0,0 +1,346 @@ +# Broadcast — Pushing Events Out + +Written 2026-08-01 against commit `5fe7a45`. This is the human-facing companion to +[`docs/broadcast.md`](../docs/broadcast.md), which stays the system of record — the exact +model fields, every API endpoint's rate limit, the full adapter list, environment variables, +and management commands all live there in more detail than belongs here. Read this first for +orientation, then go to `docs/broadcast.md` for anything precise. Where the two disagree, the +code (and `docs/broadcast.md`, which was re-verified against it) wins — this doc calls out the +one place that matters below. + +Audience: someone inheriting this codebase who has never touched the broadcast subsystem and +needs to understand what it is before the deep-dive doc makes sense. + +--- + +## 1. What this is and who depends on it + +Ingestion (see `ingestion.md`) pulls events *in* — it polls other towns' calendars and pulls +their events onto The Commons. Broadcast does the reverse: it takes a single event and pushes +it *out* onto several other towns' calendar websites, so a partner organization — a chamber of +commerce, a venue, an event host — can list one event once and have it land on The Commons plus +half a dozen third-party sites, instead of retyping the same event into six different forms by +hand. + +The people who use it are not residents browsing the public site. They're partner +organizations and event hosts, working through a separate console (`broadcastWeb`) gated +behind an access code or a login. If broadcast is down, nothing on the main site +(`theCommonsWeb`) breaks — it's an entirely separate flow with its own frontend, its own +backend app, and its own database rows. What breaks is a partner's ability to distribute an +event listing; the event itself can still be posted directly to The Commons through the normal +submission form. + +Architecturally, broadcast is deliberately walled off from the rest of the backend. The +`broadcast` Django app is not allowed to import anything from `events` or `ingestion` — a real +test (`broadcast/tests/test_isolation.py`) walks every `.py` file in the app with Python's +`ast` module and fails the suite if it finds an import rooted at either package. This isn't +incidental tidiness. Broadcast operates on its own denormalized copy of an event's fields +(title, datetimes, venue, locality tags, category tags — all just columns on +`BroadcastSubmission`, not a foreign key to `events.Event`), on its own locality/category +vocabulary (`broadcast/routing.py` defines its own list of towns and categories rather than +reading `events.Town`/`events.Category`), and its own access-control system entirely separate +from Better Auth accounts. The one place broadcast reaches back toward the rest of the site is +one-directional and lives on the other side of the wall: when a host submits an event through +the broadcast console, the SPA also fires a request at an ingestion endpoint +(`POST /api/events/direct-submit`) that runs the event through the same LLM standardization +pipeline as a scraped event. That bridge code lives entirely inside `ingestion/` — `broadcast/` +still imports nothing from `ingestion/` or `events/`. If someone "fixes" the isolation by +having `broadcast/` reach into `events.Town` directly to reuse a vocabulary, the isolation test +fails immediately, but the deeper cost is that a future change to `events.Town` (renaming a +slug, deleting a town) would silently start breaking broadcast's routing logic in a way nobody +watching the `events` app would think to check for. + +--- + +## 2. Who operates it, and how + +Three pieces work together: + +- **`broadcastWeb`** (`broadcastWeb/`) — a Vite + React single-page app, separate from + `theCommonsWeb` entirely (different build, different deploy, different test suite). This is + where a partner fills out an event, picks which calendars to send it to, and watches the + status of each site. +- **`broadcastExtension`** (`broadcastExtension/`) — a Chrome MV3 browser extension that the + SPA talks to. It's the thing that actually opens each third-party calendar's submission form + and fills it in. +- **`broadcast`** (`backendServer/broadcast/`) — the Django app underneath both. It decides + which calendars a given event is eligible for, builds the per-site "recipe" the extension + fills in, gates all of this behind an access-tier system, and also contains a second, + currently-unused submission path described in §4. + +### Signing in and getting access + +There's no separate broadcast account system — signing in uses the same Better Auth identity +as the main site (see `auth.md`), via the shared cookie domain across `thecommons.town` +subdomains. But being logged in isn't enough by itself; broadcast additionally gates every +feature behind a **tier** (0, 1, or 2), resolved per request from either a Bearer JWT or an +access-code header. Tier 0 (logged in, no grant, or not logged in at all) can't fill or +broadcast anything. Tier 1 can fill and broadcast. Tier 2 additionally unlocks the AI-autofill +helper that turns pasted free text into a draft event. + +There are two independent ways to get above tier 0, and they're deliberately kept apart: + +- A **trial code**, entered anonymously with no account required. It's always tier 2, and + instead of being metered by number of uses it just expires — three days by default. This is + the "hand someone a code so they can try it right now" path. +- An **upgrade code**, which only works for a logged-in user (`POST /broadcast/redeem`) and + permanently sets that account's tier. Whoever enters the code last wins — there's no + downgrade protection, which is a deliberate choice so sales/support can just say "enter this + code" without worrying about someone accidentally reverting a customer's access. + +The one text field in the SPA does double duty depending on login state — logged out it reads +"Access Code" and resolves a trial code anonymously; logged in it relabels to "Upgrade Account" +and permanently redeems an upgrade code against that account. `broadcast/access.py` is where +this resolution actually happens; `docs/broadcast.md`'s Access control section has the full +tier table and every metering rule. + +--- + +## 3. How an event actually gets onto another town's calendar + +This is the live flow — the one a partner uses today. + +```mermaid +sequenceDiagram + autonumber + participant Partner + participant SPA as broadcastWeb (SPA) + participant API as Django (broadcast app) + participant Ext as broadcastExtension + participant Site as Third-party calendar site + + Partner->>SPA: Fill out event form + SPA->>API: POST /broadcast/preview (event data) + API-->>SPA: eligible sites / excluded sites + reason + SPA-->>API: POST /api/events/direct-submit (fire-and-forget) + Note over API: ingestion bridge — standardizes the event\ninto the normal pipeline in parallel, non-blocking + + Partner->>SPA: Select sites, click "Autofill with extension" + loop per selected site + SPA->>API: POST /broadcast/direct-recipe (event data, site_key) + API-->>SPA: recipe JSON (fields, selectors, values — no DB row created) + SPA->>Ext: sendFill(extensionId, recipe) + Ext->>Site: open new tab at recipe.url, fill every field but the submit button + Ext-->>SPA: fill acknowledged + end + Partner->>Site: Review prefilled form, solve captcha, click Submit +``` + +**Five things this diagram can't say on its own:** + +1. **This is the primary, and today the only reachable, submission path.** The current + `broadcastWeb` UI has no button that creates a job for the server-side headless path + described in §4 — the code that would call `POST /broadcast/submit` isn't wired to anything + in `App.tsx` anymore. A partner's event never becomes a `BroadcastSubmission`/ + `BroadcastTarget` database row under this flow; `direct-recipe` is explicitly a read-through + endpoint that computes a recipe and creates nothing. +2. **The recipe request and the actual autofill are two separate round trips per site**, + because the recipe is computed fresh from whatever's currently in the form (no server-side + state to go stale) and the extension is a different origin the SPA can only reach by + messaging. +3. **The extension never clicks Submit.** That's not a missing feature — it's the whole point. + Every recipe-driven site has either a captcha, a terms checkbox, or both, and the design + intentionally leaves the final click (and any captcha-solving) to the human who can see the + filled-in form and vouch for it. +4. **The direct-submit call in step 3 runs in parallel with everything else and can fail + silently without affecting the visible flow.** It exists so a broadcast submission also + becomes a real event on The Commons itself, going through the identical LLM + standardize/dedupe/safety-score pipeline described in `ingestion.md` — but a failure there + never blocks or surfaces as an error in the preview step. +5. **Not every eligible-looking site gets this treatment.** Four Tier-1 sites + (`fun4raleighkids`, `chapelboro`, `explore_pittsboro`, `shop_pittsboro`) require a login + before any form appears, or are otherwise not deterministically fillable — their adapters + carry no recipe at all, and `SitePicker` in the SPA greys them out with "coming soon" instead + of letting a partner select them. + +### The adapter pattern + +Each third-party calendar has its own **adapter** — a small Python class in +`broadcast/adapters/` (`abc11_community.py`, `triangle_on_the_cheap.py`, `visit_raleigh.py`, +and so on) that knows that one site's form: its field selectors, what format each field expects +a date or a price in, and which locality/category combination the site is even willing to +accept (`routing.py` uses this to compute the eligible/excluded split in step 2 of the diagram +above). An adapter is intentionally dumb by design — it never invents content, never calls an +LLM at request time, and never solves a captcha; if a required field can't be resolved from the +event data, or the site presents a login wall or a captcha, the result is "needs manual +attention," not an invented guess. + +What makes an adapter usable by the extension flow is a second, declarative layer on top of the +same field definitions: `RecipeField` objects (selector, type, how to resolve a value from the +event) that get serialized straight into the recipe JSON the extension consumes. The two layers +share the same field/selector definitions on purpose, so the imperative fill logic (used by the +disabled headless path in §4) and the recipe JSON (used by the live extension path) can't drift +apart into filling a form two different ways. Six adapters currently expose a full recipe — +`abc11_community`, `triangle_on_the_cheap`, `triangle_weekender`, `visit_raleigh`, +`chatham_arts`, `chatham_chamber` — and those six are exactly the sites listed in the +extension's `host_permissions`. The remaining Tier-1 adapters exist (so routing can still +report them as "eligible, but not automatable") but return no recipe. + +--- + +## 4. The other path: server-side headless submission (disabled today) + +The `broadcast` app also contains a complete, independent second way of doing this, built +first and still fully present in the code: a database-backed job queue, a Playwright-driven +headless Chromium worker, and the same adapters' *imperative* `fill_and_submit` methods (as +opposed to their declarative `recipe()` output). `POST /broadcast/submit` creates a +`BroadcastSubmission` plus one `BroadcastTarget` row per selected site, a Celery task drains +the queue, and a worker process launches a real (usually headless) browser per target and +drives the form itself — no human in the loop unless an adapter reports `needs_manual`. + +```mermaid +flowchart TD + Submit["POST /broadcast/submit\n(NOT called by the current SPA UI)"] --> Rows["BroadcastSubmission + one BroadcastTarget\nper site created, status=queued"] + Rows --> Dispatch["transaction.on_commit ->\nprocess_broadcast_queue.delay (Celery)"] + Dispatch --> Claim["worker.claim_next:\nSELECT FOR UPDATE SKIP LOCKED\noldest queued submission -> status=running"] + Claim --> Loop{"For each pending target,\nin site_key order"} + Loop --> Launch["Launch one Chromium session\n(no ORM calls inside sync_playwright)"] + Launch --> Fill["adapter.fill_and_submit(page, event, ctx)"] + Fill --> Outcome{"Result?"} + Outcome -->|"required field\nresolves fine"| Succeed["status=succeeded\nexternal_url recorded"] + Outcome -->|"captcha / login wall /\nmissing field"| Manual["status=needs_manual\n(legacy recipe fetch still\nworks via GET .../manual/site_key)"] + Outcome -->|"adapter raised"| Fail["status=failed, error recorded"] + Succeed --> Next{"More targets?"} + Manual --> Next + Fail --> Next + Next -->|yes| Loop + Next -->|no| Done["submission.status = done or failed"] +``` + +**This path is real, tested, and currently unreachable from the SPA.** `broadcastWeb`'s +`App.tsx` still has handlers for retrying a job, promoting a dry-run job to a real submission, +and canceling a job — but nothing in the current UI ever calls `POST /broadcast/submit` to +*create* one, so those handlers only matter for a job that already exists. It's worth stating +plainly because two of the repo's own root-level docs disagree about this: `ARCHITECTURE.md`'s +Broadcast section narrates this headless path as if it were the live flow, and separately +claims broadcast "does not use Celery." Neither is accurate as of this commit — trust +`docs/broadcast.md` and the code instead, both of which agree the extension flow in §3 is +primary and this one is dispatched over a real Celery queue (see §5). + +Why keep a whole second, disabled path around instead of deleting it? Because it's the honest +long-term answer for sites without a usable recipe, or for a future bulk/unattended mode — the +architecture supports both a human-supervised and a fully automated submission without +duplicating the per-site form logic, since both paths are adapter methods on the same class. + +--- + +## 5. The worker, and why its concurrency is fixed at one + +Both the headless path's queue-drain and its crash-recovery sweep run as real Celery tasks +(`broadcast.tasks.process_broadcast_queue` and `broadcast.tasks.recover_broadcast_orphans`), +routed to a dedicated `broadcast` Celery queue via `CELERY_TASK_ROUTES` in +`backend/settings/base.py`. In production, exactly one worker process drains that queue — the +`broadcast-worker` systemd unit runs `celery -A backend worker -Q broadcast -c 1`, concurrency +pinned to one. + +That `-c 1` is load-bearing, not a conservative default someone forgot to tune up. The +crash-recovery task assumes that any `BroadcastSubmission` still marked `running` when it runs +must have been orphaned by a worker that died mid-drain — a safe assumption only if there is +never more than one worker that could have been mid-drain in the first place. Scale that worker +to two or more processes and this assumption breaks: a submission actually being processed by +worker B could get its `running` status matched by orphan-recovery's sweep and get re-queued +out from under worker B, which would then finish and write results for a submission that a +second, freshly-spawned attempt is now also racing to process. The obvious-looking fix for "the +queue is slow" — bump the worker's concurrency — is exactly the wrong move here; the right fix +is a second single-concurrency worker on its own separate queue, not a wider pool on this one. + +This queue is not polled on an interval. `services._dispatch_worker()` calls +`transaction.on_commit(process_broadcast_queue.delay)` right when a submission is created, +retried, or promoted from dry-run to real — so a job starts draining the instant its database +transaction commits, not on the next tick of a fixed poll loop. A separate, coarser safety net — +`recover_broadcast_orphans`, seeded as a periodic task running every six hours — exists purely +to catch a submission stranded by a worker that crashed mid-drain between one partner's session +and the next; it is not how ordinary progress happens. Day-to-day stalled-target recovery is +actually driven by the SPA's own polling loop, which flags a target that's been sitting +`queued`/`in_progress` too long and asks the backend to re-queue just that target, capped at a +couple of automatic retries before surfacing a "stuck" state to the partner. `async-jobs.md` +covers the full Celery queue layout (the ingestion and digest queues too) in depth; this +section is scoped to what's specific to broadcast. + +--- + +## 6. Access codes, briefly + +Access codes are managed entirely through the database — there is no environment-variable code +list to update on deploy. Codes are generated from the Django admin (self-serve, shows the raw +code once, right after creation, then never again) or from a management command. `data-model.md` +has the full field-by-field reference for `AccessCode`, `AccessCodeUse`, and +`AccessCodeRedemption`; the one thing worth internalizing here is that **trial** and +**upgrade** codes are two genuinely separate pools distinguished by `AccessCode.kind`, not two +labels on the same mechanism — a trial code is only ever checked against the anonymous +header/body path, an upgrade code only ever against the logged-in redeem endpoint, and neither +code type will ever validate against the other's path. There's also a small evergreen layer on +top (`SalesCodeSlot`) that keeps exactly one always-current, always-visible code per tier +showing in the admin, specifically so a salesperson never has to run a CLI command to hand +someone a working code. + +--- + +## 7. Sharp edges + +1. **Isolation from `events`/`ingestion` is enforced by a real test, and it's easy to break + without noticing.** `broadcast/tests/test_isolation.py` parses every `.py` file in the app + with Python's `ast` module and fails if any file imports from `events` or `ingestion` at any + depth — not just direct top-level imports. Reaching for `events.Town` to avoid duplicating + broadcast's own locality list, or importing an `ingestion` helper to reuse some parsing + logic, will fail CI immediately if it's caught — and if it somehow isn't (a new test file, + a refactor that moves code around), the real cost lands later, as a change to `events` or + `ingestion` silently breaking broadcast in a way nobody reviewing that change would think to + check. + +2. **No Django ORM call may happen while a `sync_playwright()` block is open.** `runner.py`'s + docstring explains why: Playwright keeps its own event loop alive on the thread it runs on, + and Django's async-aware connection handling can silently start using a second database + connection inside that block instead of the one the surrounding code expects. The fix in + place is structural — `run_submission` fetches everything it needs into a plain + `CanonicalEvent` object *before* opening a browser session, and the browser is opened and + closed once per target, with every database write happening in between sessions rather than + inside one. Any change that tries to look something up from the database mid-fill (to avoid + an extra round trip, say) reintroduces this specific hazard, and it's the kind of bug that + only shows up under real concurrent load, not in a quick manual test. + +3. **The `broadcast` Celery queue's single-worker constraint is a correctness requirement, not + a performance knob** — covered in full in §5. Worth repeating here as its own edge because + it's the one most likely to look like an easy fix to someone trying to speed up a slow + broadcast queue. + +4. **The Chrome extension only autofills hosts explicitly listed in `manifest.json`'s + `host_permissions`.** Today that list is exactly the six recipe-enabled adapters' domains + plus `api.thecommons.town` (needed so the extension's service worker, which isn't bound by a + page's CORS policy, can fetch an event's self-hosted image before attaching it to a form — + see `BroadcastImage` in `data-model.md`). Add a seventh recipe-enabled adapter without adding + its domain to `host_permissions`, and nothing throws an error anywhere: the extension simply + never runs its content script on that tab, no banner appears, no field gets filled, and the + most likely diagnosis from an operator's chair is "the extension is broken" rather than "the + manifest is missing an entry." Adding a `host_permissions` entry also isn't free once the + extension is actually published to the Chrome Web Store — it triggers a mandatory re-review, + and existing installs won't pick up the new permission until each user individually + re-approves it, so there's a real rollout gap between shipping the fix and it actually + reaching partners already using the extension. + +5. **The two on-record disabled/dormant framings mean different things and it's worth not + conflating them.** The extension is "dormant" in the sense that its content script never + runs during ordinary browsing — it only activates when the SPA explicitly messages it, which + is a deliberate security property (visit any of the target calendar sites directly and + nothing happens). The headless Playwright path in §4 is "disabled" in a completely different + sense — its code is complete and tested, but nothing in the current SPA UI can trigger it. + Neither means "unfinished" or "safe to delete." + +--- + +## 8. Not independently verified + +- Whether the extension has actually been submitted to and approved on the Chrome Web Store, or + is still distributed as a load-unpacked developer build to partners. `broadcastWeb/.env` has + a real-looking published extension ID alongside a dev-unpacked one, and the extension's own + README documents the Web Store submission process as something the account owner runs + manually — this doc did not confirm whether that submission has actually happened. +- The actual production behavior of the headless path end-to-end (a real Chromium run against a + live third-party form) — its code was read, not exercised, for this doc. + +--- + +See also: `data-model.md` for the full `broadcast` model field reference, `async-jobs.md` for +the complete Celery queue and beat-schedule layout, `deploy-ops.md` for the `broadcast-worker` +systemd unit and how it's deployed, and `ingestion.md` for the pipeline the direct-submit +bridge in §3 feeds into. diff --git a/human-docs/containerization.md b/human-docs/containerization.md new file mode 100644 index 0000000..ed105e2 --- /dev/null +++ b/human-docs/containerization.md @@ -0,0 +1,305 @@ +# Containerization — status + +> **PROVISIONAL — written 2026-08-01, against commit `5fe7a45`.** Every file +> this doc describes was **uncommitted** in the working tree at the time of +> writing (tracked internally as Suite 42, status "Needs QA"): `docker-compose.yml`, +> `docker-compose.override.yml`, `backendServer/Dockerfile`, `Dockerfile.frontend`, +> both `.dockerignore` files, `deploy/nginx/` (`Dockerfile` + `thecommons.conf`), +> `docs/adr/0001-containerization.md`, and a Docker-first rewrite of `DEPLOY.md`. +> Treat anything here as accurate only up to that commit and that uncommitted +> snapshot — if the working tree has moved since, re-read the source files +> before trusting a claim in this doc over the code. +> +> **Built and verified locally end-to-end. Not deployed.** The production VM +> has no Docker Engine installed and still runs the systemd units in `deploy/` +> — nothing described below is live. This doc is a placeholder: once the +> cutover actually lands, it should either be promoted into a proper handoff +> report or folded into `human-docs/deploy-ops.md` (see the closing section). +> `human-docs/deploy-ops.md` covers what's running today, in production, right +> now — this doc covers what's coming. Read that one first if you're trying to +> fix something in prod this minute. + +## What was containerized, and why + +The Commons runs today as seven long-lived processes hand-wired as individual +systemd units on a single Oracle Cloud VM, plus a hand-edited nginx config +with no local-dev equivalent at all. `docs/adr/0001-containerization.md` is +the actual design record for the work described here — this section +summarizes it; read the ADR for the full reasoning and the incidents that +motivated it. + +Four decisions anchor the design. First, **nginx moves into a container and +becomes the single ingress** for all three subdomains, replacing the +host-installed nginx package entirely. The Cloudflare origin cert stays a +read-only bind mount rather than something baked into an image layer — a +rotated cert shouldn't require a rebuild, and a private key has no business +inside something that could be pulled or inspected off the VM. The +consequence worth remembering: gunicorn moves from a unix socket to plain +TCP, because a socket path can't cross a container boundary the way nginx and +gunicorn share one today. That's a strictly weaker isolation posture — any +container on the compose network can now reach port 8000 — accepted as the +standard shape for a containerized nginx-plus-app pair, not an oversight. + +Second, **Redis moves into a container** (`redis:7-alpine`), which is also +the only piece of async infrastructure that currently has zero local-dev +story — it's apt-install-only today. Nothing about the DB-0/DB-1 broker/cache +split changes: it was never hardcoded, it falls straight out of `REDIS_URL` +and `REDIS_CACHE_URL`, so pointing both at a containerized Redis with the +same `/0` and `/1` suffixes preserves it untouched. `requirepass` is kept, +sourced the same way it is today, and the container gets a named volume +because DB 0 holds in-flight Celery task state that an unpersisted restart +would silently drop. + +Third, **Postgres stays external, on Neon, in every environment** — it was +never a candidate for the compose stack. Neon already provides the actual +restore mechanism (branching, point-in-time recovery); a containerized +Postgres alongside that would just be a second, weaker source of truth for +local dev only. The one consequence: the CI pipeline's pre-migrate `pg_dump` +safety net, which today depends on a host-installed `postgresql-client`, +moves to a throwaway `postgres:18-alpine` container invoked for the duration +of the dump and discarded. + +Fourth, **build artifacts get baked into images; only real, unrecoverable +state gets a volume.** `collectstatic` output and the compiled broadcast SPA +bundle are both regenerated from source on every build, so baking them in is +strictly safer than a volume — a volume here would let a stale build survive +a fresh deploy. Client-uploaded media, the pre-migrate backup dumps, and the +Playwright debug screenshots/downloads are all volumes, because losing any of +them on a restart destroys something no rebuild can recreate. + +Worth noting separately: containerizing this stack also closes the exact +failure class that took the async stack down for eight days starting +2026-07-21 (`docs/prod-incident-2026-07-21-scheduler-outage.md`) — a +snap-packaged `uv run` spawning its child inside a systemd user-session scope +that a post-deploy SSH logout tore down. There's no snap, no `logind`, no +per-user systemd slice inside a container's PID namespace for a lost SSH +session to tear down, so that mitigation (exec the venv binary directly, plus +`loginctl enable-linger`) becomes moot by construction rather than by fix. +One piece of it is still carried forward for an unrelated reason: every +compose service execs its real binary directly rather than through a wrapper +shell, so the container's PID 1 receives `SIGTERM` on `docker stop` and shuts +down cleanly. + +## Service topology + +The diagram below shows what starts immediately, what waits on what, and +why — not every dependency edge is drawn individually where five services +share an identical gate; see the callouts underneath. + +```mermaid +flowchart LR + subgraph immediate["No dependencies - start immediately"] + redis[redis] + migrate[migrate - one-shot, exits 0] + nextjs[nextjs] + spabuild[broadcast-spa-build - one-shot, exits 0] + end + + subgraph gated["Wait for redis healthy AND migrate exited 0"] + backend[backend] + celery[celery] + celerybeat[celerybeat] + bworker[broadcast-worker] + sworker[scrape-worker] + end + + subgraph ingress["Waits for backend, nextjs, broadcast-spa-build to have started"] + nginx[nginx] + end + + redis -- healthy --> gated + migrate -- exit 0 --> gated + backend --> nginx + nextjs --> nginx + spabuild --> nginx +``` + +Four things the diagram can't say on its own: + +1. **`migrate` doesn't wait on Redis, deliberately.** It only ever talks to + Neon over `DATABASE_URL`, so it's grouped with `redis` in the "starts + immediately" tier rather than gated behind it — the compose file's own + comment calls this out ("not a Celery service, so it has no reason to wait + on redis"). +2. **All five gated services share the identical two-part gate**, even though + only the Celery ones use Redis as a broker in the traditional sense. + `backend` (gunicorn) is gated the same way because Django will 500 on + nearly every route without a completed migration — not because gunicorn + itself needs Redis to boot. (It does read `REDIS_CACHE_URL` for the cache + backend, but nothing in this dependency graph enforces that the cache + actually round-trips before `backend` starts serving.) +3. **`nginx`'s dependency is the weakest of the three tiers.** Compose's + default `depends_on` condition — used here with no explicit condition — is + "has started," not "is healthy" or "is ready to accept traffic." `nginx` + can come up and start proxying to a `backend` or `nextjs` container that + is still mid-initialization, unlike the `redis`/`migrate` gate ahead of + it, which explicitly waits on health and exit status. +4. **`broadcast-spa-build` and `migrate` are both `restart: "no"` one-shots + that are supposed to exit 0 immediately.** `docker compose ps` showing + them `Exited (0)` is correct behavior, not a crash — a newcomer running + `ps` for the first time and seeing two "stopped" containers among a list + of "running" ones could easily read that as a partial failure. + +### Compose service to systemd unit + +| Compose service | Replaces | Notes | +|---|---|---| +| `redis` | `redis-server.service` (apt-installed; no unit file was ever tracked in this repo) | `requirepass` moves from `/etc/redis/redis.conf` into the compose `command:` block, sourced from `REDIS_PASSWORD` in `backendServer/.env`. | +| `migrate` | New — previously a manual `manage.py migrate` step run by hand during deploy | Runs once per deploy as an exiting container instead of a step in a deploy script; also seeds the `django_celery_beat` schedule tables. | +| `backend` | `gunicorn.service` (its unit file, like `nextjs.service`, was apparently never checked into `deploy/` — only referenced by name in `DEPLOY.md` and the ADR) | Switches from a unix socket to TCP `8000`, internal to the compose network only. | +| `celery` | `deploy/celery.service` | Same `ExecStart` command, minus the exec-direct/linger workaround — see the ADR's historical note above. | +| `celerybeat` | `deploy/celerybeat.service` | Same command; still exactly one process, `DatabaseScheduler` keeps the schedule in Postgres. | +| `broadcast-worker` | `deploy/broadcast-worker.service` | Same command, `-c 1` concurrency preserved (mandatory — `recover_orphans()` assumes a single worker). | +| `scrape-worker` | `deploy/scrape-worker.service` | Same command, `-c 1` to keep headless-Chromium memory off the default worker. | +| `nextjs` | `nextjs.service` (also not tracked under `deploy/`) | Unchanged behavior: `node server.js` on port 3000, internal only. | +| `broadcast-spa-build` | New — previously a manual `pnpm run build` step whose `dist/` nginx served directly off the VM filesystem | Build-only helper; never runs as a long-lived process (see diagram callout 4). | +| `nginx` | The host-installed nginx package plus the hand-edited `/etc/nginx/sites-available/thecommons` (previously only tracked as a snippet, `deploy/nginx-broadcast.conf.snippet`) | Full config now lives at `deploy/nginx/thecommons.conf` and is baked into the image; the TLS cert stays a bind mount, never baked in. | +| — | `deploy/healthcheck.service` + `.timer` | **Not replaced.** Still a host-level systemd timer; `deploy/healthcheck.sh` now shells out to `docker compose ... exec` to reach the containers, and degrades to a `WARN` rather than crashing when Docker isn't installed on the host at all. | + +Two things stood out while building this table, worth flagging rather than +silently smoothing over: `docs/adr/0001-containerization.md` describes "seven +long-lived systemd units," but only four of them (`celery`, `celerybeat`, +`broadcast-worker`, `scrape-worker`) actually have a tracked unit file under +`deploy/` — `gunicorn.service`, `nextjs.service`, and `redis-server.service` +exist only by reference in `DEPLOY.md` and the ADR, not as files in this +repo. Not a contradiction, just a gap in what's checked in versus what runs +on the box. + +## Sharp edges already known + +**The override file is a loaded footgun on the VM.** `docker-compose.yml` +alone is the production config. A bare `docker compose` invocation with no +`-f` flag auto-loads `docker-compose.override.yml` from the same directory — +that is Compose's default two-file-merge behavior, not a bug — and that +override is explicitly local-dev-only: plain HTTP nginx with no TLS cert at +all, bind mounts remapped to repo-relative `.local-dev/` paths instead of the +real `/home/ubuntu/broadcast/...` directories, and `DJANGO_ENV` forced to +`dev`. Running that combination against the VM would not fail loudly — nginx +would happily bind port 80 in plain HTTP, Django would boot under dev +settings, and the Redis URLs would point at an unauthenticated connection +string that doesn't carry the real `REDIS_PASSWORD` even though the actual +Redis container still expects it (since `.env`'s `REDIS_PASSWORD` is +untouched by the override). The result is a mix of wrong TLS, wrong +hostnames, and a Redis auth mismatch, discovered only once someone notices +the site is serving unencrypted or Celery has gone silent. Every production +command — in `DEPLOY.md`, in CI, run by hand — passes `-f +docker-compose.yml` explicitly for exactly this reason. + +**The `.dockerignore` at the repo root exists as much for secrets as for +build speed.** The repo root holds a real private key (`oraclevps.key`) and +`broadcastWeb/` holds a committed certificate. The file's own header records +an empirically verified quirk of this Docker Desktop/buildkit version: a +pattern with no leading `**/` only matches at the exact context-root path, +even with no slash in the pattern — a bare `*.pem` did not exclude a nested +`broadcastWeb/commons-broadcast.pem` in a real test build, only `**/*.pem` +did. That's stricter than plain `.gitignore` semantics, where a no-slash +pattern matches at any depth, and it means every secret-adjacent pattern in +that file deliberately carries the `**/` form rather than the shorter one +someone might "simplify" it down to later. + +**`WORKDIR` plus `COPY --chown` does not chown a pre-existing directory.** +`backendServer/Dockerfile` sets `WORKDIR /app` as root before switching to a +non-root user, and `COPY --from=builder --chown=app:app /app /app` only sets +ownership on the files it copies in — the `/app` directory itself, created +earlier by `WORKDIR`, stays root-owned unless something says otherwise. The +Dockerfile carries an explicit `chown app:app /app` specifically to avoid +`collectstatic` failing with `EACCES` trying to create +`staticfiles_build/` under a directory it can't write to — this is prior +history in this repo, not a hypothetical, and the fix is already in place, +but the trap reappears instantly if a future edit reorders those two lines. + +**A Docker Desktop proxy can corrupt large apt fetches as "Hash Sum +mismatch."** Both stages of `backendServer/Dockerfile` that touch apt write +`Acquire::Retries "3"` and disable HTTP pipelining before installing +anything, specifically because the Playwright stage's `--with-deps` fetch is +large enough to trip this locally, and the failure reads like package +corruption on a different package each run rather than what it actually is — +a proxy mangling a large or pipelined response. Anyone debugging an +inexplicable, non-reproducible apt failure inside these images should look +here before assuming a broken mirror. + +**Building `nginx` in isolation doesn't work.** `deploy/nginx/Dockerfile` +consumes two Buildx named build contexts — `backend-static` and +`broadcast-spa`, bound in `docker-compose.yml` to the `backend` and +`broadcast-spa-build` services respectively — to pull in `collectstatic` +output and the compiled broadcast bundle. A standalone `docker build -f +deploy/nginx/Dockerfile deploy/nginx` has no way to resolve those names and +fails; `docker compose build nginx` (verified to build the two named-context +services first even when only `nginx` is requested) is the only supported +path. + +**Frontend build args are a separate mechanism from runtime env, and mixing +them up produces a build that "succeeds" with the wrong values baked in.** +`nextjs` and `broadcast-spa-build` inline their `NEXT_PUBLIC_*`/`VITE_*` +values at build time via Compose variable interpolation, which only reads +the shell environment or a root-level `.env` next to `docker-compose.yml` — +it cannot read `theCommonsWeb/.env.local` or `broadcastWeb/.env` directly, +even though those are the files that actually hold the real values. Every +build arg has a safe placeholder default so `docker compose build` always +succeeds with zero setup, which is exactly the trap: a real prod build that +skips sourcing the real files and re-exporting them under the +`NEXTJS_BUILD_*`/`BROADCAST_BUILD_*` names will build cleanly and ship a +broadcast bundle silently misrouting every API call. CI already greps the +compiled bundle for a `thecommons.town` origin as a guard, but that guard +only exists in the CI job — a manual build on the VM has no equivalent +safety net. + +## What's left before this is real + +The three blockers already known going in: `backendServer/.env`'s +`REDIS_URL` and `REDIS_CACHE_URL` still point at `localhost`/`127.0.0.1`, +which inside a container is the container itself — Celery would silently +connect to nothing and no task would ever run, while everything synchronous +stayed green, the same failure shape as the 2026-07-21 outage. Both need to +repoint at the `redis` compose service with the real password embedded. +`DJANGO_ALLOWED_HOSTS` must be present in that same file — `prod.py` does a +bare `os.environ["DJANGO_ALLOWED_HOSTS"]` with no default, so its absence +crashes every backend/Celery container at boot, not just one. And the VM +itself needs one-time prep — Docker Engine and the Compose v2 plugin +installed, and the deploy user added to the `docker` group — before the +first automated container deploy can run at all. + +Beyond those three, reading the actual compose file and CI workflow surfaced +more: + +- **The CI deploy job is already wired for containers, and the VM isn't + ready for it.** `.github/workflows/ci.yml`'s `deploy` job — gated on + `needs: [backend, frontend-commons, frontend-broadcast]` and triggered only + on a push to `main` — already runs `docker compose -f docker-compose.yml + build` and `up -d` over SSH. That means the very next merge to `main` after + this branch lands will fail that step outright (`docker: command not + found`) unless the VM prep above happens first. This fails loudly rather + than silently — the deploy job errors out — but it's still a live landmine + worth flagging before anyone merges, not something to discover from a red + CI run. +- The persistent host directories the compose file bind-mounts into + (media, broadcast screenshots/downloads, the backups directory) need to + exist and be writable by uid 1000 before the first `up`, matching the + images' non-root `app` user. +- The Cloudflare origin cert needs to already be in place at + `/etc/ssl/cloudflare/thecommons.town.{pem,key}` for the container nginx to + bind 443 at all — this isn't new (the same cert the host nginx uses today), + but the container won't start without it being reachable at that exact + path. +- The actual cutover moment — stopping the host nginx and starting the + container one — is a single step where two processes briefly cannot both + hold ports 80/443, and it's the one step in the whole plan where a mistake + produces an immediate, visible outage rather than a quiet misconfiguration. + It has not been rehearsed against the real VM; everything verified so far + has been local-only. +- Nothing here has been run against the actual Oracle VM's resource limits (1 + OCPU / 6 GB) — six long-running containers plus two Playwright-capable + images is a meaningfully different memory footprint than seven bare + systemd processes, and that has not been measured, only asserted via + `mem_limit` guesses on the Celery workers. + +## When to delete this doc + +This file is provisional by design and should not accumulate history. Once +the VM cutover has actually happened and been verified — containers running +in production, the old systemd units retired, the blockers above resolved — +do one of two things, not both: either promote this into a proper handoff +report under `human-docs/` with its own verified sharp edges from real +production operation, or fold whatever's still true into +`human-docs/deploy-ops.md` and delete this file outright. Don't leave it +sitting alongside a working `deploy-ops.md` describing the same system twice. diff --git a/human-docs/data-model.md b/human-docs/data-model.md new file mode 100644 index 0000000..f5c477c --- /dev/null +++ b/human-docs/data-model.md @@ -0,0 +1,453 @@ +# Data Model Reference + +*Reflects commit `5fe7a45`, 2026-08-01. Grounded in each app's `models.py` and `migrations/` +(`accounts`, `events`, `newsletter`, `ingestion`, `broadcast`), plus `theCommonsWeb/src/lib/auth-schema.ts` +for the Better Auth side. If anything here contradicts the code, trust the code — a couple of +places where `ARCHITECTURE.md` §Data Models has drifted are called out in §9.* + +This is the reference doc, not the narrative one — for the story of how an event moves through +the system, read `overview.md` first. This doc exists so you can keep a tab open and look up +"what does this column actually mean" without re-reading a `models.py`. + +## 1. The short version + +Five Django apps in `backendServer/` own the `public` Postgres schema between them — +`events`, `ingestion`, `accounts`, `newsletter`, `broadcast` — plus five more models that +`accounts` merely *mirrors*, read-only, out of a `neon_auth` schema that belongs to Better Auth +(the Next.js identity provider, not Django). A recent refactor (Suite 41) moved several models +between apps without moving their physical tables — §8 explains exactly what that means and why +a table name and its owning Django app no longer match for three of these models. `Town` and +`Category` are real SQL tables, not hardcoded choices — nothing in the codebase should treat them +as enums, and the ingestion pipeline silently parks (not drops) any event whose town slug doesn't +match a row. + +## 2. How the models relate + +```mermaid +erDiagram + Town ||--o{ Event : "town (SET_NULL)" + Event }o--o{ Tag : tags + Event }o--o{ Category : categories + BetterAuthUser ||--o{ Event : "created_by (SET_NULL)" + BetterAuthUser ||--o| UserProfile : "user (1:1)" + BetterAuthUser ||--o| BusinessProfile : "user (1:1)" + BetterAuthUser ||--o{ BetterAuthSession : sessions + BetterAuthUser ||--o{ BetterAuthAccount : accounts + UserProfile }o--o{ Tag : tags + BusinessProfile }o--o{ Tag : tags + BusinessProfile }o--o{ Town : service_area + + EventSource ||--o{ RawEvent : raw_events + EventSource ||--o{ SourceRun : runs + RawEvent ||--o| StagedEvent : "raw_event (1:1, nullable)" + StagedEvent ||--o| StagedEvent : "duplicate_of (self, SET_NULL)" + StagedEvent }o--o| Event : "published_event (SET_NULL)" + BetterAuthUser ||--o{ StagedEvent : "submitted_by (SET_NULL)" + + BroadcastSubmission ||--o{ BroadcastTarget : targets + AccessCode ||--o{ AccessCodeUse : uses + AccessCode ||--o{ AccessCodeRedemption : redemptions + AccessCode ||--o| SalesCodeSlot : "sales_slot (1:1, PROTECT)" +``` + +Two things this diagram deliberately leaves out, because they aren't foreign keys: + +- **`NewsletterSubscriber` has no relationship line to `UserProfile`.** There isn't one in the + database — see §5 and the sharp edge in §10. +- **`broadcast`'s models never point at `events`, `accounts`, or `ingestion` models**, and vice + versa. That's not an omission — `broadcast/routing.py` is contractually forbidden from + importing `events`, and the whole app operates on its own denormalized copy of an event's + fields (`BroadcastSubmission`'s title/datetime/venue/etc. columns) rather than a foreign key + into `Event`. `BroadcastAccess` and `BroadcastImage` are standalone rows keyed by a plain + string (email, client_label) for the same reason — no FK anywhere in the picture. + +## 3. `events` app + +Owns the genuine event/taxonomy models — the public-facing data. All four tables are physically +in `public.events_*`, and the app label matches the table prefix here (unlike `accounts` and +`newsletter` — see §8). + +### `Tag` + +| Field | Meaning | +|---|---| +| `name` | Unique, free-text. Lowercased by convention at write time (`ingestion.services` does `tag_name.strip().lower()` before `get_or_create`), not enforced by the field itself. | + +Reverse-related from `Event.tags`, `UserProfile.tags`, `BusinessProfile.tags` — one shared tag +vocabulary across events, personal interests, and business listings. + +### `Town` + +| Field | Meaning | +|---|---| +| `slug` | Unique, e.g. `carrboro`. This is the join key the ingestion pipeline computes from the LLM's free-text town guess (`town.lower().replace(" ", "-")`) — see §10. | +| `name` | Display name, e.g. `Carrboro`. | + +A real SQL table, not an enum — seeded by data migrations (`events/migrations/0016_seed_chatham_towns.py`, `0018_seed_apex_durham_towns.py`), editable in the admin. Adding a new town is a data change, not a code change. + +### `Category` + +| Field | Meaning | +|---|---| +| `slug` | Unique (`SlugField`). | +| `display_name` | Human label. | + +Same story as `Town` — a real table (`Meta.verbose_name_plural = "categories"` is the only +non-default option), not hardcoded. Unlike `Town`, an unmatched category slug does **not** stall +an event's publication — see §10. + +### `Event` + +| Field | Meaning | +|---|---| +| `uuid` | **Primary key.** Not `id` — see §10, the single most-hit trap in this codebase. | +| `title`, `venue`, `description`, `price`, `photo`, `link` | Plain content fields; `price` is nullable (no price listed, not "$0"). | +| `date` | Indexed (`db_index=True`) — this is the field every list/window query filters and sorts on. | +| `town` | FK → `Town`, `SET_NULL` — an event survives its town row being deleted, just loses the association. | +| `tags`, `categories` | M2M → `Tag`, `Category`. | +| `is_verified` | `True` only when the submitter was an authenticated user with `user_type="BUSINESS"` at publish time (`ingestion.services`) — not a manual admin toggle, not related to safety scoring. | +| `source_name` | Free text describing provenance: the originating `EventSource.name` for scraped events, `"Community Submission"` for anonymous `/events/create` posts, `"Direct submission by {organizer}"` / `"Direct submission by host"` for the broadcast direct-submit path. | +| `created_by` | FK → `accounts.BetterAuthUser`, `SET_NULL`, `db_constraint=False` (see §8 for why FKs into the mirrors never carry a DB-level constraint). Null for pipeline-ingested events — this is how you tell "someone submitted this" from "the scraper found this." | + +There is no soft-delete or unpublish concept: an `Event` row existing *is* what "published" +means. Only the owner (`created_by`) can hard-delete their own event via the API. + +## 4. `ingestion` app + +The pipeline that produces `Event` rows from external sources. Full flow narrative lives in +`overview.md` §3 and `ingestion.md`; this section is field-level reference. + +### `EventSource` + +| Field | Meaning | +|---|---| +| `source_type` | One of `ics`, `scraper`, `http`, `email`, `direct`. `direct` doesn't get polled at all — it exists so direct-submit's synthetic `RawEvent`s have a `source` FK to point at, same as every other row. | +| `active` | Poll loop skips inactive sources. | +| `last_polled` | Null = never successfully polled yet. | +| `poll_interval_hours` | Minimum gap between polls for this source, default 24. | +| `prompt_suffix` | Extra text appended to the Gemini standardization prompt for this source specifically — a per-source LLM tuning knob. | +| `scraper_key` | Looks up the corresponding Python scraper module for `source_type="scraper"`; blank for other types. | + +### `SourceRun` + +| Field | Meaning | +|---|---| +| `status` | `ok` / `failed` / `refused` / `skipped` — one row per poll attempt, for observability (this table isn't in `ARCHITECTURE.md`'s Data Models section — see §9). | +| `trigger` | `scheduled` / `probe` / `manual` — how the run was kicked off. | +| `items_fetched` / `items_new` / `items_duplicate` | Counts from that single run. | +| `finished_at` | Null while the run is still in progress (or if it crashed hard enough not to reach the finally block). | +| `error_class`, `error_message`, `traceback` | Populated only on `status="failed"`. | + +Ordered `-started_at` by default; indexed on `(source, -started_at)` for the "recent runs for +this source" query the admin/monitoring views use. + +### `RawEvent` + +| Field | Meaning | +|---|---| +| `source` | FK → `EventSource`. | +| `raw_*` fields | Exactly what was scraped/submitted, before any LLM cleanup. | +| `source_uid` | The feed's own per-item identifier (an ICS `UID`) for scraped sources; the client-generated `draft_id` string for direct-submit rows. `unique_together=(source, source_uid)` is what makes direct-submit idempotent — resubmitting the same `draft_id` upserts instead of creating a duplicate row. | +| `raw_organizer` | Only populated by direct host submissions; drives the `"Direct submission by {name}"` attribution on the eventual `Event.source_name`. Blank for everything else. | +| `processed` | Flips to `True` once `standardizer.py` has consumed it into a `StagedEvent`. | + +### `StagedEvent` + +| Field | Meaning | +|---|---| +| `raw_event` | OneToOne → `RawEvent`, **nullable** — null for events entered via the plain `/events/create` form, since those skip the raw/standardize step entirely and go straight to a pending `StagedEvent`. | +| `town`, `category` | Plain strings (an LLM's best guess, or a user's typed value), **not foreign keys** — see §10 for what happens when they don't match a real row. | +| `tags` | `JSONField`, a list of tag-name strings — converted to real `Tag` rows only at publish time via `get_or_create`. | +| `status` | `pending` / `approved` / `rejected` / `duplicate` / `published` / `skipped_no_town`. See the state diagram below — `published` is terminal but the row is **not deleted**, because it's part of the deduplicator's matching corpus (`deduplicator.CANDIDATE_STATUSES` includes `pending`, `approved`, `duplicate`, `published`, and `skipped_no_town` — everything except `rejected`). Rows are eventually reaped by `cleanup_old_events` once `start_datetime` is in the past. | +| `safety_score` | **Nullable, and the null-vs-value distinction matters.** `null` = not yet scored by Gemini. A non-null value with `status="pending"` means it *was* scored and came back above `SAFETY_SCORE_THRESHOLD` (default `0.3`) — held for manual review, not rejected. `<= SAFETY_SCORE_THRESHOLD` triggers auto-approval instead. There is no separate "held for review" status value; it's this null/non-null-plus-pending combination. | +| `duplicate_of` | Self-FK, `SET_NULL`. Set alongside `status="duplicate"` when the deduplicator (`thefuzz`) matches this row against an existing `StagedEvent`. | +| `published_event` | FK → `events.Event`, `SET_NULL`. Normally non-null only once `status="approved"` flips to `"published"` — **except** on a direct-submission re-edit that lands on `duplicate` or gets re-parked `pending`/`skipped_no_town`: in that case `published_event` still points at the *previously* published `Event`, because `Event` has no unpublish mechanism and the row must not orphan a live listing just because its latest edit didn't clear the gate. Don't assume `published_event != null` implies `status` is `approved` or `published`. | +| `submitted_by` | FK → `accounts.BetterAuthUser`, `SET_NULL`, `db_constraint=False`. Null for pipeline-ingested and anonymous direct-submit events. | + +```mermaid +stateDiagram-v2 + [*] --> pending: standardized (poll, /events/create, or direct-submit) + pending --> duplicate: dedup match found + pending --> skipped_no_town: town slug unmatched + pending --> approved: score at or below threshold (auto) or manual approval + pending --> rejected: manual rejection + approved --> published: publish_all_approved() + published --> [*]: cleanup_old_events (after start_datetime passes) +``` + +One thing the state diagram can't say: **`duplicate`, `skipped_no_town`, and even `pending` +are not necessarily dead ends** for a row created via direct-submit re-editing — the row can +carry a `published_event` pointer even while parked in one of those statuses, per the +`published_event` row above. The diagram shows the primary transitions; that pointer is a side +channel that survives them. + +## 5. `accounts` app + +Owns identity: the five read-only Better Auth mirrors, plus the two profile models that hang +off a user. A "business" is modeled as a *kind of user profile*, not a separate top-level +concept. + +### The Better Auth mirrors (`neon_auth` schema, `managed = False`) + +**Better Auth, running inside the `theCommonsWeb` Next.js app, owns writes to these five +tables — Django never migrates them.** `theCommonsWeb/src/lib/auth-schema.ts` (Drizzle) is the +actual source of truth for their shape; the Django models below exist purely so the ORM can join +against them. Each model's `db_table` uses a deliberate double-quote trick — +`'neon_auth"."user'` — so Django emits a valid cross-schema reference (`FROM "neon_auth"."user"`) +without Django having first-class multi-schema support. + +| Model | Table | Key fields | Notes | +|---|---|---|---| +| `BetterAuthUser` | `neon_auth.user` | `id` (UUID PK), `email` (unique), `email_verified`, `user_type` | Hardcodes `is_authenticated = True` / `is_anonymous = False` as class attributes so DRF permission classes treat an instance as a real authenticated user. | +| `BetterAuthSession` | `neon_auth.session` | `id` (text PK), `token` (unique), `expires_at`, `user_id` | `user_id` is a plain `TextField`, not a UUID FK column — see the sharp edge below. | +| `BetterAuthAccount` | `neon_auth.account` | `id` (text PK), `provider_id`, `user_id` (UUID), `password` | One row per sign-in method per user. For the `credential` provider (email+password), `password` holds the hashed credential — null for OAuth-provider rows. | +| `BetterAuthVerification` | `neon_auth.verification` | `identifier`, `value`, `expires_at` | Email-verification / password-reset tokens. | +| `BetterAuthJwks` | `neon_auth.jwks` | `public_key`, `private_key` | The signing keyset Django's JWKS client fetches to verify JWTs — see `auth.md`. | + +Every FK from a Django-managed model into one of these mirrors is declared with +`db_constraint=False` — there is no database-level foreign key against an unmanaged table, only +an application-level one. `BetterAuthUser.id` is a UUID, but `BetterAuthAccount.user_id` and +`BetterAuthSession.user_id` are plain `TextField`/`UUIDField` columns rather than declared FKs to +`BetterAuthUser.id` at the Django level (the join happens through matching values, mirroring +however Better Auth itself models it in Postgres). + +### `UserProfile` + +| Field | Meaning | +|---|---| +| `user` | OneToOne → `BetterAuthUser`, `db_constraint=False`. | +| `uuid` | A separate identifier from `user.id` — used in URLs/serializers where you don't want to expose the Better Auth user id directly. | +| `user_type` | `LOCAL` / `BUSINESS` / `VENUE`. Drives `Event.is_verified` at publish time (see §3) and gates the business-listing endpoints. | +| `primary_city`, `address` | Free text, both blank-allowed. | +| `email_preference` | `WEEKLY` / `MONTHLY` / `NEVER`. Writing this via `PATCH /auth/me` is what triggers the `NewsletterSubscriber` sync described in §10. | +| `tags` | M2M → `events.Tag` — this user's interest tags, read by `newsletter._build_recipients` to filter their digest. | + +`db_table = "events_userprofile"` — the physical table name still says `events`, because this +model *moved apps* without moving tables. See §8. + +### `BusinessProfile` + +| Field | Meaning | +|---|---| +| `user` | OneToOne → `BetterAuthUser`. | +| `business_name`, `description`, `contact_email`, `contact_phone` | Listing content. | +| `is_published` | Gates visibility on the public `/businesses` list — an unpublished listing is only visible to its owner. | +| `tags` | M2M → `events.Tag`. | +| `service_area` | M2M → `events.Town` — which towns this business serves; drives filtering on the business directory. | + +`db_table = "events_businessprofile"` — same historical-name situation as `UserProfile`. + +## 6. `newsletter` app + +### `NewsletterSubscriber` + +| Field | Meaning | +|---|---| +| `email` | Unique. The only identity a subscriber needs — **no FK to `UserProfile` or `BetterAuthUser`.** Anonymous subscribers (no account) and account holders alike get a row here; the two are correlated only by matching `email` string, case-insensitively, at read time in `newsletter._build_recipients` (see §10). | +| `frequency` | `WEEKLY` / `MONTHLY`. | +| `is_active` | `False` = unsubscribed. A `PATCH /newsletter/manage` with `frequency=NEVER` sets this rather than deleting the row, preserving history and the `manage_token`. | +| `manage_token` | `UUIDField`, unique, unguessable — the entire authentication mechanism for the login-free manage/unsubscribe link (`/newsletter/manage?token=...`). Anyone holding the token can read or change that one subscription; nothing else. | +| `subscribed_at` | Set once on creation (`auto_now_add`), not touched on re-subscribe. | + +`db_table = "events_newslettersubscriber"` — moved apps, table name unchanged. See §8. + +## 7. `broadcast` app + +Pushes a published event out to third-party town calendars. Its models never reference +`events`, `accounts`, or `ingestion` models — see §2. Full subsystem detail is in `broadcast.md`; +this is field-level reference for the models only. + +### `BroadcastSubmission` + +| Field | Meaning | +|---|---| +| `id` | UUID PK. | +| `client_label` | Identifies which partner/operator created this — not a FK to any user model, just a string. | +| Denormalized event fields (`title`, `start_datetime`, `venue_name`, `address_line1`, `locality` JSON, `categories` JSON, `event_url`, `price`, `organizer_name`, `contact_email`, …) | A frozen snapshot of the event's data at submission time — deliberately not a FK into `events.Event`, per the isolation contract. | +| `status` | `queued` / `running` / `done` / `failed` / `canceled`. | + +### `BroadcastTarget` + +| Field | Meaning | +|---|---| +| `submission` | FK → `BroadcastSubmission`. `UniqueConstraint(submission, site_key)` — one target row per site per submission. | +| `site_key` | Which third-party calendar adapter this target is for. | +| `status` | `pending` / `in_progress` / `succeeded` / `failed` / `needs_manual` / `skipped`. | +| `dry_run` | `True` = this target ran in preview/test mode, never actually submitted to the third-party site. | +| `screenshot_path` | Blank until a run captures one; gated behind the screenshots endpoint. | + +### `BroadcastAccess` + +| Field | Meaning | +|---|---| +| `email` | Unique, lowercased on save. | +| `tier` | `0` / `1` / `2`, default `0`. The *permanent* tier for a logged-in identity, set by redeeming an `AccessCode` of `kind="upgrade"` — resolved by `broadcast/access.py` off the JWT's email claim. | + +### `AccessCode` + +| Field | Meaning | +|---|---| +| `kind` | `trial` (anonymous, always forced to `tier=2` in `save()`, time-boxed via `expires_at` rather than metered) or `upgrade` (redeemed by a logged-in user against `POST /broadcast/redeem`, permanently sets that user's `BroadcastAccess.tier`). Two independent code pools — a trial code is never resolved through the JWT path, an upgrade code never through the anonymous header path. | +| `code` | Plaintext, nullable. **Null specifically means the code was created before this field existed** (added in migration `0008_accesscode_code`) — those older codes only ever had a hash, and there is no way to recover their plaintext. Non-null on every code created since, so operators can copy it after generation. | +| `code_hash` | SHA-256 hex of the code, unique. All validation compares against this — `code` is a convenience, never read for auth checks. | +| `max_uses` | **Null = unlimited.** Default `3` when set. Distinct from `0`, which would mean "already exhausted." | +| `is_active` | Manual kill switch, independent of `expires_at`/`max_uses`. | +| `expires_at` | Null = no expiry. | + +### `AccessCodeUse` + +| Field | Meaning | +|---|---| +| `access_code`, `draft_id` | `unique_together` — meters a **trial** code's anonymous preview usage per draft, so the same in-progress draft doesn't consume multiple uses on retry. | + +### `AccessCodeRedemption` + +| Field | Meaning | +|---|---| +| `access_code`, `email` | `unique_together` — one row per email that has redeemed an **upgrade** code; this is what `max_uses` counts against. `email` lowercased on save. | + +### `BroadcastImage` + +| Field | Meaning | +|---|---| +| `image` | Stored **re-encoded**, never as received — third-party share links often lack CORS headers or aren't direct image URLs, so uploads are self-hosted rather than linked. | +| `client_label` | Same free-text partner identifier as `BroadcastSubmission.client_label` — no FK between them. | + +### `SalesCodeSlot` + +| Field | Meaning | +|---|---| +| `slot` | One of `trial` / `tier1` / `tier2`, unique — exactly one evergreen row per slot. | +| `access_code` | OneToOne → `AccessCode`, `on_delete=PROTECT` — the code currently live in that slot; rotating the slot creates a new `AccessCode` and repoints this rather than mutating the old one. | +| `raw_code` | Plaintext, unlike `AccessCode.code_hash` elsewhere — exists so a salesperson can open the admin and see a live, copyable code with no CLI step, a deliberate convenience-over-defense-in-depth tradeoff scoped to this one low-stakes flow. | + +## 8. The Suite 41 model moves — what actually happened + +`UserProfile` and `BusinessProfile` moved from `events` to `accounts`; `NewsletterSubscriber` +moved from `events` to `newsletter`. Both moves used +`migrations.SeparateDatabaseAndState` with an **empty `database_operations` list** — meaning +**zero DDL ran**. `events/migrations/0021_move_identity_to_accounts.py` and +`0022_move_newsletter_to_newsletter.py` only delete the models from Django's *state* (so the ORM +stops thinking `events` owns them); the paired `accounts/migrations/0001_initial.py` and +`newsletter/migrations/0001_initial.py` re-create the same models in the new app's state, with +`db_table` pinned to the original name (`events_userprofile`, `events_businessprofile`, +`events_newslettersubscriber`). The physical Postgres tables never moved, were never renamed, +and never lost a row. + +The practical consequence: **the Django app label and the actual Postgres table name disagree** +for these three models. If you're debugging with `psql` directly, or reading a raw SQL log, you +will see `events_userprofile` — even though the model now lives in `accounts/models.py` and +migrates under the `accounts` app label. This is not a bug or a leftover to clean up; renaming +the table would be a real (and riskier) migration for zero functional benefit, so it was left +alone on purpose. + +`neon_auth.*` was never touched by any of this — those mirrors were already `managed=False` +before the move and remain so. + +A companion data migration, `newsletter/migrations/0002_repoint_digest_beat.py`, updates the +existing `django_celery_beat.PeriodicTask` rows' `task` dotted-path from +`events.tasks.fan_out_weekly_digest` / `fan_out_monthly_digest` to +`newsletter.tasks.fan_out_weekly_digest` / `fan_out_monthly_digest` — the beat schedule rows +themselves (created earlier by `events/migrations/0015_seed_digest_beat.py` and +`0020_seed_monthly_digest_beat.py`) were left in place and only repointed, not recreated. + +## 9. Doc drift found while writing this + +`ARCHITECTURE.md` §Data Models is mostly accurate post-Suite-41, but has fallen behind the +current models in a few places: + +- **`SourceRun` is missing entirely** from the `ingestion` app table in `ARCHITECTURE.md` — it's + a real, migrated model (`ingestion/migrations/0014_sourcerun.py`) used for per-poll + observability, documented in full in §4 above. +- **`broadcast`'s model list in `ARCHITECTURE.md` stops at `AccessCodeUse`** — it's missing + `AccessCodeRedemption`, `BroadcastImage`, and `SalesCodeSlot`, all three of which exist and are + migrated (`0006_accesscode_kind_accesscoderedemption.py`, `0010_broadcastimage.py`, + `0007_salescodeslot.py`). It also doesn't mention `AccessCode.kind` (trial vs. upgrade), which + is central to how the two code pools behave differently — see §7. +- **`StagedEvent.status`'s choice list in `ARCHITECTURE.md`** omits `published` and + `skipped_no_town`, both of which are real, reachable statuses (§4's state diagram). +- **`EventSource.source_type`'s choices in `ARCHITECTURE.md`** are described as "ics/scraper/email/direct" — the model also has `http`, and `prompt_suffix`/`scraper_key` fields aren't mentioned at all. + +None of this changes any relationship or field meaning documented elsewhere in +`ARCHITECTURE.md` — it's a coverage gap (models added after the doc was last updated), not a +factual disagreement. + +## 10. Sharp edges + +1. **`events.Event`'s primary key is `uuid`, not `id`.** There is no `id` field on `Event` at + all. `Event.objects.values("id")`, `Count("id")`, or any code that assumes a Django default + auto PK will raise `FieldError: Cannot resolve keyword 'id'`. Use `Count("pk")` or reference + `.uuid` explicitly. This is the single most-hit trap in this codebase — every other model + here uses Django's default `id` PK, which is exactly what makes `Event` easy to get wrong by + habit. + +2. **`neon_auth` mirror models must never be migrated by Django.** They're `managed=False` + specifically so `python manage.py migrate` never generates DDL for them — Better Auth + (running inside `theCommonsWeb`) owns their schema exclusively via Drizzle. If a future model + change to `BetterAuthUser`/etc. accidentally flips `managed` or drops the `Meta`, the next + migration would try to create or alter tables that already exist and are owned by a different + codebase. + +3. **`Town` and `Category` are SQL rows, not enums.** Don't hardcode a Python list of towns or + categories anywhere — new ones are added as data (seed migrations or the admin), not code. + The ingestion pipeline enforces this at the boundary: a `StagedEvent.town` string that doesn't + slugify to an existing `Town.slug` sends the row to `status="skipped_no_town"` rather than + publishing with a null/wrong town (see §4's state diagram). `Category` is looser — an + unmatched category slug doesn't block publication at all; `publish_all_approved` just skips + attaching a category and the event goes live uncategorized. + +4. **`StagedEvent.safety_score` null and `StagedEvent.safety_score` non-null-but-`pending` mean + different things.** Null = the Gemini safety scorer hasn't run on this row yet. A non-null + value with `status` still `"pending"` means it *was* scored and came back above + `SAFETY_SCORE_THRESHOLD` (default `0.3`) — held for a human to approve or reject by hand, + not an error state and not "unscored." + +5. **`StagedEvent.published_event` being non-null does not imply `status` is `"approved"` or + `"published"`.** On a direct-submission re-edit that lands on `duplicate` or gets re-parked + at `pending`/`skipped_no_town`, `published_event` is deliberately left pointing at whatever + `Event` a *previous* submission under the same `draft_id` already published — because `Event` + has no unpublish/soft-delete concept, and orphaning a live listing just because a later edit + didn't clear the safety/dedup/town gate would be worse than leaving the stale content live. + Check `status`, not just whether `published_event` is set, before assuming a row represents + the live content. + +6. **`NewsletterSubscriber` has no foreign key to `UserProfile` or `BetterAuthUser`.** The two + are joined only by a case-insensitive string match on `email`, done at read time inside + `newsletter._build_recipients` (`accounts` writes the `NewsletterSubscriber` row when a + user's `email_preference` changes; `newsletter` reads `accounts.UserProfile.tags` back out by + matching email). If a user changes their Better Auth email without the corresponding + `NewsletterSubscriber.email` being updated to match, digest tag-filtering for that address + silently stops working — there's no constraint that would catch the mismatch. + +7. **`accounts` ↔ `newsletter` is a deliberate two-way coupling, not a boundary bug.** + `accounts.views` (the `/auth/me` PATCH handler) writes/updates a `NewsletterSubscriber` row + whenever `email_preference` changes; `newsletter.email_service._build_recipients` reads + `accounts.UserProfile` back out for tag-filtered digests. Each app's `test_isolation_fast.py` + forbids reaching into `ingestion`/`broadcast`, but explicitly permits this pair reaching into + each other and into `events`. Don't "fix" this into a one-way dependency — both directions are + load-bearing. + +8. **The `accounts`/`newsletter` `db_table` values don't match their app labels.** + `UserProfile`/`BusinessProfile` physically live in `events_userprofile`/`events_businessprofile`; + `NewsletterSubscriber` lives in `events_newslettersubscriber`. All three migrated apps + *without* their tables — see §8. Grepping for `accounts_userprofile` in the database will + find nothing. + +9. **`AccessCode.max_uses = null` means unlimited, not zero.** A code with `max_uses=0` would + read as already-exhausted; `null` is the sentinel for "don't meter this code at all." The two + are easy to conflate when writing a query against this field. + +10. **FKs into the `neon_auth` mirrors always carry `db_constraint=False`.** `Event.created_by`, + `StagedEvent.submitted_by`, `UserProfile.user`, `BusinessProfile.user` all point at + `BetterAuthUser` with no database-level foreign key — only an application-level one. Postgres + will not stop you from inserting a value that doesn't correspond to a real `neon_auth.user` + row; referential integrity here is enforced by the application, not the schema. + +## 11. Not independently verified + +- Whether any code outside `ingestion.services` and `newsletter.email_service` also reads or + writes `StagedEvent.published_event` or `NewsletterSubscriber` directly — this doc is grounded + in the read/write paths found while writing it, not an exhaustive grep of every call site. + `ingestion.md` and `newsletter.md` are the deeper references for those flows. +- The exact query patterns the frontend (`theCommonsWeb`) issues against these tables — this doc + covers the Django-side model shape only; `frontend.md` is where that belongs. diff --git a/human-docs/deploy-ops.md b/human-docs/deploy-ops.md new file mode 100644 index 0000000..5a309f0 --- /dev/null +++ b/human-docs/deploy-ops.md @@ -0,0 +1,357 @@ +# Deployment & Operations + +*Written 2026-08-01 against commit `5fe7a45`. Complements [`DEPLOY.md`](../DEPLOY.md), +which stays the operational source of truth for step-by-step deploy/setup commands — this +doc is the mental model: what's actually running, why it's arranged this way, and what +fails how. Sibling docs: [`overview.md`](overview.md) (whole-system map), +[`async-jobs.md`](async-jobs.md) (Redis/Celery queue and beat-schedule detail), +[`auth.md`](auth.md) (the Better Auth bridge this doc's nginx section resolves a question +for), [`containerization.md`](containerization.md) (the Docker cutover this doc explains is +pending), [`testing.md`](testing.md) (local dev setup).* + +## 1. What's running, and the central fact to hold onto + +**Read this first: two deployment stories exist in this repository right now, and only one +of them is live.** `DEPLOY.md` was rewritten during this same work session to describe a +fully containerized stack — Docker Compose, an `nginx` container, a `backend` container, +one container per Celery role — built and verified locally end-to-end. None of it has +touched the production VM. The VM has no Docker installed. Every service a reader would +SSH in and find today is the plain **systemd-unit deployment** this doc describes. If you +take one thing from this document, take this: **as of 2026-08-01, production is systemd, +not containers.** §9 covers the pending cutover and what changes when it happens. + +The Commons runs on a single Oracle Cloud VM (Ubuntu 24.04, ARM64, 1 OCPU / 6 GB, IP +`129.80.229.41`) behind nginx, with Cloudflare in front for DNS and TLS (proxied, Full +strict). Postgres lives off-box, managed by Neon — the VM never runs a database server. +Everything else — the Django API, the Next.js site, Redis, four Celery-family processes — +runs as systemd units on that one box. There is no load balancer, no second VM, no +managed container platform. If this VM is down, the whole product is down: the public +site, the API, the broadcast operator console, ingestion, digests, everything. Anyone +touching production infrastructure, chasing a 2am page, or trying to understand why an +email didn't send depends on the picture in this document. + +## 2. How it works + +### Request routing + +nginx is the single ingress. It terminates TLS using a Cloudflare origin certificate +(`/etc/ssl/cloudflare/thecommons.town.{pem,key}`) and fans requests out to whichever +backend owns that subdomain — a plain Django app (gunicorn) for the API, a Node process +for the main site, and static files for everything else. + +```mermaid +flowchart TD + Client[Browser / API client] --> CF[Cloudflare edge - DNS + TLS proxy] + CF --> Nginx[nginx on the VM - terminates TLS again, Full strict] + + Nginx -->|thecommons.town| NextJS[Next.js process, port 3000] + Nginx -->|www.thecommons.town| Redirect1[301 to apex] + Nginx -->|auth.thecommons.town| NextJS + Nginx -->|api.thecommons.town| Gunicorn[gunicorn via Unix socket - run/gunicorn/gunicorn.sock] + Nginx -->|api.thecommons.town/static/| StaticFiles[backendServer/staticfiles - collectstatic output] + Nginx -->|api.thecommons.town/media/| MediaFiles[MEDIA_ROOT on disk - never touches Django] + Nginx -->|broadcast.thecommons.town| BroadcastSPA[static broadcastWeb build - dist/] +``` + +**Three things worth calling out.** First, `auth.thecommons.town` and the apex both land +on the *same* Next.js process — Better Auth lives inside `theCommonsWeb`, not a separate +service, so the "auth origin" is a routing decision, not a different deployable. Second, +`api.thecommons.town` reaches gunicorn over a **Unix socket** +(`unix:/run/gunicorn/gunicorn.sock`), not TCP — this matters for one sharp edge below +(django-ratelimit's IP key) and is the reason the containerized rewrite in `DEPLOY.md` +switches to TCP instead: a socket path doesn't cross a container boundary cleanly. Third, +`/media/` is nginx reading a directory directly; Django is never in that request path in +production — see §5. + +**On the `auth.thecommons.town` nginx routing question:** an earlier documentation pass +(`auth.md`) explicitly could not verify this and deferred it here. It's resolved: the +cutover runbook (`docs/runbook-auth-cutover.md`) records the exact server block added to +the VM's nginx config — `server_name auth.thecommons.town` with `proxy_pass +http://127.0.0.1:3000` and the standard `X-Real-IP`/`X-Forwarded-*` headers, TLS from the +same wildcard Cloudflare origin cert as every other subdomain — and an execution record +dated 2026-07-30 confirming it was applied and smoke-tested live (`curl +https://auth.thecommons.town/api/auth/jwks` returned 200 with a real JWKS body). One honest +caveat: that server block lives in a **hand-edited file directly on the VM** +(`/etc/nginx/sites-available/thecommons`), which is not itself checked into this +repository — only the *runbook instructions* for editing it are. The broadcast subdomain's +block is the one nginx fragment actually tracked in git +(`deploy/nginx-broadcast.conf.snippet`), meant to be pasted into that same live file. So +the routing is real, live, and verified — just not something `git grep` alone will ever +show you; you have to read the runbook or SSH in. + +### How a deploy happens + +Every push to `main` runs CI (`.github/workflows/ci.yml`): a `lint` job, then `backend` +(Django tests, Postgres 16 service container, `--tag=fast` then `--tag=db`), +`frontend-commons` and `frontend-broadcast` (pnpm build as the type-check gate, plus +`test:fast`/`test:db`) all run in parallel. Only if all three test jobs are green does a +gated `deploy` job SSH into the VM and touch anything — a failing test on `main` blocks +deployment outright, there is no way around that gate from the workflow file. + +```mermaid +sequenceDiagram + autonumber + participant GH as GitHub Actions + participant VM as Oracle VM + participant PG as Neon Postgres + + GH->>GH: lint, backend tests, frontend-commons tests, frontend-broadcast tests (parallel) + Note over GH: deploy job only starts if all three test jobs pass + GH->>VM: SSH in (appleboy/ssh-action, host key pinned via fingerprint) + VM->>VM: git pull origin main + VM->>VM: uv sync (backendServer) + VM->>VM: manage.py migrate --check + alt migrations pending + VM->>PG: pg_dump (gzip, timestamped) to /home/ubuntu/backups + VM->>VM: prune to 5 newest dumps + VM->>PG: manage.py migrate --noinput + else nothing pending + VM->>VM: skip migrate entirely + end + VM->>VM: manage.py collectstatic --noinput + VM->>VM: pnpm build (theCommonsWeb, then broadcastWeb) + VM->>VM: grep built broadcastWeb bundle for a real thecommons.town API origin + VM->>VM: sudo systemctl restart gunicorn nextjs celery celerybeat broadcast-worker scrape-worker + VM->>VM: systemctl is-active on all six (must all report active) + GH->>VM: second SSH step - post-deploy smoke test + VM->>VM: curl the three public domains, expect 200 + VM->>VM: POST an invalid broadcast request, expect 403 not 500 (Unix-socket REMOTE_ADDR regression check) + VM->>VM: GET /auth/me with no credentials, expect 401/403 not 500 +``` + +**Four things worth calling out.** First, the migration guard is genuinely conditional — +`migrate --check` exits non-zero only when there's real unapplied work, so most deploys +skip the dump-and-migrate branch entirely; a `pg_dump` is never skipped when a migration +*is* about to run, and the guard hard-fails the whole deploy if `pg_dump` isn't installed +rather than silently proceeding without a backup. Second, the broadcastWeb bundle grep +exists because a malformed `VITE_BROADCAST_API_BASE_URL` builds cleanly and only fails at +runtime, as every API call silently misroutes — this catches that class of bug before the +build goes live, not after. Third, `systemctl is-active` passing is necessary but not +sufficient — a crashing view or a misrouted SPA both restart clean and report `active`, +which is exactly why there's a separate smoke-test step hitting real URLs afterward, not +just a process-liveness check. Fourth, the smoke test's `403` check on a broadcast endpoint +is a deliberate regression probe: nginx talking to gunicorn over a Unix socket used to +leave `REMOTE_ADDR` empty, which crashed `django-ratelimit`'s IP-based rate limiting with +an unhandled 500 on every request to a rate-limited broadcast view — a `500` here means +that bug is back, a `403` means the request was correctly rejected before it ever became a +ratelimit crash. + +There is no separate `gunicorn.service` or `nextjs.service` file in this repository's +`deploy/` directory, and none exists anywhere in git history — those two units were set up +by hand directly on the VM and were never checked in, unlike the four Celery-family units +and the healthcheck unit, which are. If you need their exact unit-file contents, SSH in and +read `/etc/systemd/system/gunicorn.service` / `nextjs.service` directly, or see the last +commit of `DEPLOY.md` before its Docker rewrite (`git show 053d65b:DEPLOY.md`) for a +recorded copy of what they contained as of late July. + +## 3. The systemd units + +| Unit | What it runs | Drains / serves | How to check it | +|---|---|---|---| +| `gunicorn` | Django via a Unix socket, 3 sync workers | `api.thecommons.town` (proxied by nginx) | `systemctl status gunicorn`; not tracked in `deploy/` — hand-configured on the VM | +| `nextjs` | `node`/`npm run start` for `theCommonsWeb`, port 3000 | `thecommons.town` and `auth.thecommons.town` (both proxy to the same process) | `systemctl status nextjs`; not tracked in `deploy/` — hand-configured on the VM | +| `redis-server` | Standard `apt`-installed Redis, `/etc/redis/redis.conf` | DB 0 = Celery broker/results, DB 1 = Django cache | `systemctl status redis-server`; `redis-cli -a ping` | +| `celery` (`deploy/celery.service`) | Default worker, `.venv/bin/celery -A backend worker -n commons-default@%h --concurrency=2` | Everything not explicitly routed elsewhere — digest sends, misc tasks | `systemctl status celery`; `manage.py healthcheck`'s `celery_worker` probe | +| `celerybeat` (`deploy/celerybeat.service`) | Scheduler, `django_celery_beat`'s `DatabaseScheduler` — exactly one process, never scale this | Fires `ingest-events-daily` (04:00 ET), `weekly-digest-sunday`/`monthly-digest` (18:00 ET), `broadcast-orphan-recovery` | `systemctl status celerybeat`; `manage.py healthcheck`'s per-task `beat:` freshness probes — the check this doc's §8 incident is really about | +| `broadcast-worker` (`deploy/broadcast-worker.service`) | Playwright form-filler, `celery -A backend worker -Q broadcast -c 1` | The dedicated `broadcast` queue only — `-c 1` is load-bearing, not tuning: orphan recovery assumes a single worker | `systemctl status broadcast-worker` | +| `scrape-worker` (`deploy/scrape-worker.service`) | Headless-Chromium ingestion scraper, `celery -A backend worker -Q scrape -c 1` | The dedicated `scrape` queue, kept off the default worker so Chromium memory can't starve digests/ingestion | `systemctl status scrape-worker` | +| `healthcheck.timer` / `.service` (`deploy/healthcheck.*`) | Hourly `bash deploy/healthcheck.sh`, itself running `manage.py healthcheck --require-prod` | Nothing — read-only report | `systemctl list-timers healthcheck.timer`; `journalctl -u healthcheck.service -n 50` | + +`celery`, `celerybeat`, `broadcast-worker`, and `scrape-worker` all `Require=` and +`After=redis-server.service` and set `Restart=always` — the restart policy is +belt-and-suspenders, explained in §4, not the primary fix for anything. `deploy/`'s +`nginx-broadcast.conf.snippet` is not a systemd unit; it's an nginx server-block fragment +meant to be appended by hand into the VM's single live config file. + +## 4. The `uv run` vs. venv-binary sharp edge — a real outage, not a style rule + +Every long-lived unit in `deploy/` execs `/home/ubuntu/thecommons/backendServer/.venv/bin/celery` +directly. That specific phrasing — the venv binary, not `uv run celery`, and not a wrapper +shell script — is load-bearing, and the reason is a real production incident recorded in +full at `docs/prod-incident-2026-07-21-scheduler-outage.md`. + +```mermaid +flowchart TD + Deploy[Deploy finishes over SSH] --> Logout[SSH session ends] + Logout --> Teardown[logind tears down user-1001.slice - Linger was off] + + Teardown --> SnapPath{Unit's ExecStart} + SnapPath -->|snap uv run celery ...| SnapChild[Child process lives inside a transient snap.astral-uv scope UNDER the user slice] + SnapChild --> SnapDeath[Slice teardown kills it - clean exit, status=0/SUCCESS] + SnapDeath --> NoRestart[Restart=on-failure correctly declines to restart a clean exit] + NoRestart --> Dead[celery / celerybeat / both workers silently dead] + + SnapPath -->|.venv/bin/celery ...| VenvChild[Child process is the unit's own cgroup - never enters a user-manager scope] + VenvChild --> Survives[Slice teardown has nothing to do with this process] + Survives --> Alive[Process keeps running through the next login/logout cycle] +``` + +**What actually happened, concretely:** all four Celery-family units ran through +`/snap/bin/uv run celery …`. Snap's `uv` spawns its child inside a transient +`snap.astral-uv.uv-*.scope`, parented under `user@1001.service` — the *user* session +manager, not the systemd unit's own cgroup. With account lingering off, the moment the +deploying SSH session ended, `logind` tore down `user-1001.slice` and every snap scope +under it, taking `celery`/`celerybeat`/`broadcast-worker`/`scrape-worker` down with it — +**a clean exit**, `status=0/SUCCESS`, which `Restart=on-failure` correctly declined to +restart (it's not a failure by that policy's definition). `gunicorn` and `nextjs`, which +never touched snap, stayed up the entire time on the same VM through the same deploys. The +async stack was fully dead for 8 days before anyone noticed, because the site itself kept +serving pages — nothing about "the site is up" implied "background jobs are running." + +**What breaks if someone "simplifies" a unit back to `uv run`:** exactly this, again. The +fix — execing `.venv/bin/celery` directly — removes the snap-scope mechanism entirely, +which is the actual fix; `loginctl enable-linger ubuntu` and `Restart=always` are +defense-in-depth layered on top, not substitutes for it. A reviewer who sees `uv run` as +"more consistent with the rest of the deploy tooling" and reverts a unit to it silently +reopens this exact failure mode — it will not show up in `systemctl status` right after the +change, only after the next SSH session that started the deploy ends. + +**One deliberate exception, and it is not a contradiction:** `healthcheck.service` still +uses `/snap/bin/uv` (`Environment=UV_BIN=/snap/bin/uv`, `ExecStart=... bash +deploy/healthcheck.sh`) and that is fine. It's `Type=oneshot` — the process starts, runs +the health report to completion in a few seconds, and exits on its own, well before any SSH +session it happened to be triggered near could tear down. The failure mode above only bites +a process still running at the moment a user slice gets torn down; a oneshot that's already +finished has nothing left to kill. Don't read the healthcheck unit's `uv` line as +permission to relax the rule anywhere else — it's a narrow exception with a specific reason, +not evidence the rule is soft. + +## 5. Environment selection: `DJANGO_ENV` and how failure got narrower + +Django settings are resolved by `backend/settings/__init__.py` via a function, +`select_settings_env`, reading the `DJANGO_ENV` environment variable — not by +`DJANGO_SETTINGS_MODULE` pointing at `prod.py` directly, the way Django docs usually show +it. + +```mermaid +flowchart TD + Start[DJANGO_ENV read from environment] --> Empty{Unset or blank/whitespace?} + Empty -->|yes| Dev1[Resolve to dev - dev.py loads] + Empty -->|no| Known{Value, lowercased+stripped, is 'dev' or 'prod'?} + Known -->|prod| Prod[Resolve to prod - prod.py loads] + Known -->|dev| Dev2[Resolve to dev - dev.py loads] + Known -->|anything else, e.g. 'production'| Crash[ImproperlyConfigured raised at import time - process refuses to start] +``` + +This is a deliberately narrowed failure mode, and the history matters. The original +incident (June 2026, referenced directly in the module's own docstring) was `DJANGO_ENV` +simply **missing** on the VM: the app silently served `dev.py`, whose `ALLOWED_HOSTS` is +localhost-only, so every real request to `api.thecommons.town` came back `DisallowedHost` +(HTTP 400) — which the frontend rendered indistinguishably from "there are just no events +right now." That silent-unset-defaults-to-dev behavior is **still current and still +deliberate** — every local laptop relies on `DJANGO_ENV` being absent and getting `dev.py` +for free, and changing that would break local dev for everyone. What changed is the *other* +failure shape: a **typo'd but non-empty** value (`DJANGO_ENV=production`, a stray `PRD`, +anything not exactly `dev` or `prod` after trimming and lowercasing) now raises +`ImproperlyConfigured` immediately at import time instead of quietly falling back to +`dev.py` — the process won't boot at all, which is loud and fast rather than silent and +slow. `events/tests/test_config_fast.py` pins this exact behavior as a regression test +(unset/blank → `dev`; `prod`/`PROD `/`Dev` all normalize correctly; `production`, +`staging`, `PRD`, `true` all raise). + +The remaining gap — `DJANGO_ENV` unset in prod specifically, which the hard-error change +does nothing for, since unset is still valid input — is caught by `manage.py healthcheck +--require-prod`, run hourly via `healthcheck.timer`. That command checks `settings.DEBUG` +and whether `ALLOWED_HOSTS` is anything other than localhost-only, and reports a `FAIL` if +either looks like dev settings leaked into what's supposed to be prod. **Read `--require-prod` +correctly: it is a detector, not a guard.** It can tell you, up to an hour later, that +production is quietly running on dev settings; it cannot stop that from happening, and it +does not run on every request or every deploy — only once an hour, on the health-check +timer's own schedule. A misconfigured `.env` on the VM still means real downtime for up to +that long before anyone is told. + +## 6. Media: why it lives outside the checkout + +`MEDIA_ROOT` (client-uploaded event images) is set in production to +`/home/ubuntu/broadcast/media` — a path that sits next to the git checkout +(`/home/ubuntu/thecommons`), not inside it. `backend/settings/base.py`'s own comment on +`MEDIA_ROOT` states the reason directly: it defaults to a path *inside* the checkout for +local dev, but production overrides it in `.env` specifically so a `git pull` during deploy +can never touch uploaded files. A deploy that ran `git clean` or reset the working tree +inside the checkout would have no way to reach these files at all — they're simply not +under that directory. + +The second half of the same design: nginx serves `/media/` directly as a plain file alias, +and Django is never in that request path in production. The comment in `base.py` says this +outright ("Served by nginx in prod, never by Django"), and the pre-Docker `DEPLOY.md` +revision that documented the live nginx config confirms it as a sibling `location /media/` +block to the existing `/static/` alias, pointing at the same `MEDIA_ROOT` path. The reason +is the ordinary one for serving static assets from the ingress instead of the app server: +nginx does it faster and without spinning up a Python worker to stream a file back to +disk. There's a real cost worth knowing about, not a bug: uploaded images are kept +indefinitely — no pruning job exists anywhere in this repo — so `MEDIA_ROOT` grows without +bound. At roughly 1–3 MB per event this is currently negligible against the VM's block +volume, but it's a number worth keeping an eye on, not a problem to "fix" by inventing a +retention policy nobody asked for yet. + +## 7. Dev/prod database isolation + +Every developer's local `DATABASE_URL` should point at a **Neon branch**, not the +production database — Neon branches are copy-on-write snapshots with their own connection +string, so a branch can be migrated, seeded, and reset freely without ever touching prod +rows. `docs/dev-db-isolation.md` is the full design doc; the shape that matters here is: +the production VM's `.env` keeps the real `DATABASE_URL` pointed at Neon's main branch, and +`DJANGO_ENV` is what decides which settings module (and therefore which behavioral +guardrails) apply — it does not, by itself, decide which database gets used. Nothing in +Django enforces that a `dev`-settings process can't be pointed at the prod `DATABASE_URL`; +the isolation is a matter of which connection string ends up in which `.env` file, and +that's a human discipline, not a code guarantee. + +One extension of this worth knowing: `backend/settings/dev.py` supports an optional second +database alias, `prod_readonly`, populated only when `PROD_DATABASE_URL` is set — this lets +local devtools (the ingestion/broadcast monitor) inspect real production data without +routing writes through it and without merging prod into the primary `default` alias. It's +only actually safe if the credentials behind `PROD_DATABASE_URL` come from a Postgres role +that is read-only at the database level (a `monitor_readonly` role with `SELECT`-only +grants, per the checklist in `docs/dev-db-isolation.md`) — Django does not enforce +read-only-ness itself; a read-write DSN in that variable would happily let devtools write +to prod. If you ever set this variable locally, verify the role actually rejects writes (an +`INSERT`/`CREATE TABLE` against the `prod_readonly` connection should error with `permission +denied`) before trusting it. + +## 8. Historical incident, still worth knowing + +`docs/prod-incident-2026-07-21-scheduler-outage.md` is the full forensic record behind §4's +sharp edge — worth reading in full if you're the one debugging a "the site works but nothing +in the background is happening" report, because that is exactly the symptom this incident +produced: `gunicorn` and `nextjs` stayed up the entire 8 days, so nothing looked wrong from +the outside, while `celery`/`celerybeat`/`broadcast-worker`/`scrape-worker` were all dead. +The one lasting change from that incident that has nothing to do with the `uv run` fix: a +stale or never-fired beat schedule is now a hard `FAIL` in `manage.py healthcheck`, not a +`WARN` — a scheduler that stopped firing is treated as an outage, not a suggestion, because +that distinction is what would have caught this incident in hours instead of the 8 days it +actually took (the outage was only found by chance, during unrelated forensics against +`/devtools/monitor`, not by any monitoring that existed at the time). + +## 9. The pending cutover: containers are built, not live + +A parallel effort in this same working tree has built a complete Docker Compose +replacement for everything in this document — one container per service, described in +`docker-compose.yml`, `backendServer/Dockerfile`, `Dockerfile.frontend`, and +`deploy/nginx/`, with the full rationale in `docs/adr/0001-containerization.md`. It has been +verified locally end-to-end. **None of it is live.** The Oracle VM does not have Docker +installed, the `ubuntu` user isn't in a `docker` group, the persistent bind-mount +directories the compose file expects don't exist on the box, and the seven systemd units +this document describes have not been touched. `DEPLOY.md` was rewritten during this same +session to describe the containerized stack as the deploy target — read it as a plan for +the next cutover, not as a description of what answers a request to +`api.thecommons.town` right now. `containerization.md` (a sibling human doc, written +alongside this one) covers what changes once that cutover happens — new service names, +TCP instead of a Unix socket for gunicorn, images instead of a checked-out venv, and how +the exact sharp edges in §4–§6 above either disappear or get re-solved a different way +inside a container. Until someone runs that cutover on the actual VM, treat every fact in +§1–§7 of this document as the operative reality, and treat `DEPLOY.md`'s Docker +instructions as a runbook waiting for its day one, not a record of today. + +## 10. Known gaps + +No push notification exists for a failed health check — `systemctl --failed` and +`journalctl -u healthcheck.service` are the only read paths today; nothing pages anyone. +There is no automated rollback if a deploy's smoke test fails after the systemd restarts +already happened — the units are already running the new code by the time the smoke test +runs, so a failing smoke test currently means "go SSH in and diagnose," not "the previous +version is automatically restored." The exact live contents of `gunicorn.service` and +`nextjs.service` are not verifiable from this repository at all, for the reason noted in +§2 — they were never committed; anyone needing their precise current flags should SSH in +and read them directly rather than trust any doc's transcription, including this one's +citation of an old `DEPLOY.md` revision. diff --git a/human-docs/design-system.md b/human-docs/design-system.md new file mode 100644 index 0000000..1496680 --- /dev/null +++ b/human-docs/design-system.md @@ -0,0 +1,147 @@ +# Design System + +*Reflects commit `5fe7a45`, 2026-08-01. Every token, class, and component below was read out of `theCommonsWeb/src/app/globals.css` and `theCommonsWeb/src/components/ui/` directly — not inferred from prose. Where `CODING_STYLE.md` (the repo's canonical style statement, which this doc complements and does not replace) disagrees with the stylesheet, both are stated and the disagreement is called out. For the Next.js routing/data-fetching layer these components sit inside, see `frontend.md`; for the product as a whole, see `overview.md`.* + +## 1. What this is, in one paragraph + +The Commons is styled to look like a broadsheet newspaper's classifieds page had a baby with early Craigslist: serif type, cream newsprint, black ink, hairline and thick column rules doing the job cards and shadows do everywhere else, and a bias toward packing information in rather than giving it room to breathe. This isn't a retro skin bolted onto a normal SaaS layout — it's the whole vocabulary. A contributor who reaches for a rounded card with a soft shadow because that's what every other product looks like is not making a small stylistic choice; they're building the wrong product. The reason it matters: The Commons is a *local* events bulletin for three small towns, not a venture-backed platform, and it wants to read like a community notice board someone would trust a neighbor posted to — not like a pitch deck. Density, rules, and serifs are load-bearing for that trust, not decoration. + +## 2. Tokens + +All color and font values are CSS custom properties declared once, on `:root`, in `globals.css`. **Nothing else defines colors or fonts** — Tailwind v4 is configured with a bare `@import "tailwindcss";` at the top of that same file and no `@theme` block, no `tailwind.config.js`/`.ts` anywhere in the repo. Components consume the tokens either as Tailwind arbitrary values (`bg-[var(--color-bg)]`) or the newer Tailwind v4 shorthand (`border-(--color-border)`) — both forms are in active use side by side; neither is preferred over the other in the current code. + +| Token | Value | What it's for | +|---|---|---| +| `--color-bg` | `#f4f1eb` | Page background — the "newsprint cream." Default surface for everything. | +| `--color-bg-alt` | `#eae6dd` | A shade darker than `--color-bg`. Hover states on rows/cards, secondary surfaces (the digest box border-panel, dropdown item hover). | +| `--color-text` | `#1a1a1a` | Near-black ink. Body text, and (deliberately) also the default link color — links are not blue here. | +| `--color-text-muted` | `#555555` | Secondary text: bylines, metadata lines, captions, timestamps. | +| `--color-link` | `#1a1a1a` | Same value as `--color-text` — links read as ink, not as a distinct color, until hovered. | +| `--color-link-hover` | `#8b0000` | Dark red. Link hover state. | +| `--color-border` | `#1a1a1a` | Primary rule color — same near-black as text. Used for thick rules, card outlines, the hard "print" shadow (see §5). | +| `--color-border-light` | `#c8c3b8` | Hairline rule color. Dividers, `
`, subordinate borders (e.g., the sidebar's column rule). | +| `--color-accent` | `#8b0000` | Dark red. Same value as `--color-link-hover`. Used sparingly: active/selected states, the "Verified" stamp, kicker labels, the accent rule under section nameplates. Not a general-purpose brand color — it means "selected" or "emphasis," and overusing it dilutes that. | +| `--font-headline` | `Georgia, "Times New Roman", Times, serif` | Headings (`h1`–`h6`, applied globally in `globals.css`), and anywhere a component sets `fontFamily: 'var(--font-headline)'` inline for a display-sized headline (e.g. the masthead `

` in `Header.tsx`, the section nameplate in `EventFeed.tsx`, the footer watermark). | +| `--font-body` | `Georgia, "Times New Roman", Times, serif` | Body copy. Identical value to `--font-headline` today — there is exactly one serif stack in this system, split into two token names for future flexibility, not because they currently differ. | +| `--font-sans` | `-apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif` | The *only* sanctioned escape from serif — reserved for small-caps UI chrome that needs to read as a control rather than as editorial copy: `Button.tsx`'s base class, the "Verified" stamp's inline style. Don't reach for it for anything a reader would read as content. | +| `--focus-ring` | `2px solid #1a1a1a` | Keyboard focus outline, applied globally via `:focus-visible`. | +| `--focus-ring-offset` | `2px` | Offset for the above. | + +**Georgia is real, and it's loaded for free.** There is no `next/font` call anywhere in `src/app/layout.tsx` or elsewhere in the tree, no `@font-face`, no font files in an `assets/` directory (there isn't one). Georgia is a system font on essentially every OS that ships a browser; the stack falls through to Times New Roman / Times / generic serif if it's somehow missing. This is a deliberate performance and simplicity choice, not an oversight — it means zero font network requests, ever. + +**CODING_STYLE.md drift:** its `--font-sans` snippet says `system-ui, ...` — the real value in `globals.css` starts with `-apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif` and never mentions `system-ui` at all. Trust the CSS. Everything else in `CODING_STYLE.md`'s token table matches the stylesheet exactly. + +**A token that's referenced but doesn't exist:** `TimeWindowSelector.tsx` and `SectionSelector.tsx` both reference `var(--color-ink)` (e.g. `text-[var(--color-ink)]`). There is no `--color-ink` custom property defined anywhere in `globals.css` or any other stylesheet in the repo — it isn't a synonym that resolves elsewhere. An undefined CSS custom property used without a fallback makes the property using it invalid at computed-value time, which for `color` means it falls back to the inherited value rather than doing anything the author intended. In practice this makes the inactive state of the time-window and section dropdowns render in whatever color they'd inherit rather than the near-black ink the rest of the system uses. This is a live bug, not a stylistic choice — the fix is renaming both usages to `--color-text`. + +## 3. Type scale + +There is no declared type scale (no `@theme` font-size tokens, no Tailwind config extending `fontSize`). What exists is the Tailwind v4 default scale (`text-xs` through `text-6xl`) plus a lot of arbitrary pixel values and `clamp()` expressions for display headlines that need to be fluid. Reading across `Header.tsx`, `EventFeed.tsx`, `EventRow.tsx`, `Sidebar.tsx`, `Footer.tsx`, and `MiniCalendar.tsx`, the sizes actually in use settle into these bands: + +| Use | Typical value | Example | +|---|---|---| +| Masthead / site title | `clamp(2.75rem, 8vw, 6rem)`, `--font-headline`, `font-black` | `Header.tsx`'s `

` | +| Section nameplate / featured headline | `clamp(1.6rem, 3.2vw, 3.25rem)` (varies by context), `--font-headline` | `SectionNameplate` and `FeaturedCard` in `EventFeed.tsx` | +| Card headline (non-featured) | `text-base` (1rem) to `text-2xl`/`text-3xl` for featured rows, `font-bold` | `EventRow.tsx` | +| Body copy | default `15px` set on `body` in `globals.css`, `line-height: 1.6` | site-wide default | +| Metadata / byline line | `text-xs` (0.75rem) or `text-[10px]`/`text-[11px]` | venue/time lines, footer copyright | +| Kicker / eyebrow labels | `text-[9px]`–`text-[10px]`, `uppercase`, `tracking-[0.18em]`–`tracking-[0.25em]`, `font-black` | "Featured Event," "Towns:," section labels throughout `Sidebar.tsx` and `Footer.tsx` | +| Calendar grid cells | `text-[8px]`–`text-[9px]` | `MiniCalendar.tsx` | +| Drop cap first letter | `3.2rem`, `--font-headline`, `font-weight: 700` | `.drop-cap::first-letter` in `globals.css` | + +The pattern to copy: headlines are large and fluid via `clamp()` set inline (`style={{ fontSize: '...' }}`), everything else is small, uppercase, and letter-spaced when it's a label rather than content. There is no `text-4xl`/`text-5xl`/`text-6xl` Tailwind class in use anywhere — display sizes are handled by `clamp()`, not the static scale, because they need to shrink on mobile without a breakpoint ladder. + +## 4. Spacing and layout + +No custom spacing scale — Tailwind's default spacing scale (the `p-1`, `px-4`, `gap-6`, etc. system) is used directly, no `@theme` override. Two layout constants recur: + +- **Content max-width `960px`** — `PageLayout.tsx`'s `
` (`max-w-[960px]`), the reading column. +- **Chrome max-width `1200px`** (written as Tailwind's `max-w-300`, i.e. `300 × 4px = 1200px`) — `Header.tsx`, `Footer.tsx`, `TopBar.tsx`, `TagsBar.tsx` all use this wider band for the masthead and nav strips that span above/below the reading column. + +The sidebar/content split (`PageLayout.tsx`) is a 4-column CSS grid, sidebar taking 1 of 4 columns on large screens (`lg:grid-cols-4`, `lg:col-span-1` / `lg:col-span-3`), stacking to a single column below `lg`. The sidebar is separated from content by a `border-r border-[var(--color-border-light)]` hairline rule, not a gap-only whitespace split — this is the density-over-whitespace principle showing up structurally, not just typographically. + +Density in practice: `Sidebar.tsx` stacks a dozen-plus distinct blocks (post button, date, calendar, view toggle, social link, tag filters, clear-filters, count, digest box) separated only by `
` hairlines with no card wrapper around any of them. That's the intended texture — a long, rule-divided column, not a stack of padded cards. + +## 5. Rules, borders, and the one shadow that's allowed + +This is the section that replaces "cards with shadows for elevation." The system has exactly four separation devices, and reaching for anything outside this list should be treated as a smell. + +| Device | CSS | Where it shows up | +|---|---|---| +| Hairline rule | `border-*-[var(--color-border-light)]`, 1px | Dividers between list rows, sidebar `
`, footer link-column separators | +| Standard rule | `border-*-[var(--color-border)]`, 1–2px | Card/panel outlines (`EventRow.tsx` non-featured, `Modal.tsx`), the header's rule under the tagline | +| Thick rule | `.rule-thick` (`border-top: 3px solid`) / `.rule-double` (`border-top: 3px double`), both against `--color-border` | Section breaks that need more visual weight than a standard rule; declared as utility classes in `globals.css` but not currently called from any component in `src/components/` — available, underused | +| Hard "print" shadow | `shadow-[3px_3px_0px_var(--color-border)]` — a flat, zero-blur, fully-opaque offset box, not a soft/blurred elevation shadow | `Modal.tsx`, the featured variant of `EventRow.tsx` | + +That fourth row is the one to read carefully, because "no drop shadows" (the banned-list rule in §7) sounds like it should ban this too, and it doesn't — this is a hard-edged, zero-blur, fully-opaque offset that reads as a printed card stacked on newsprint (think a paper cutout with a solid ink-black edge behind it), not a soft ambient shadow implying elevation off the page. The distinguishing test: if the shadow has any blur radius, or any transparency, it's the banned kind. `shadow-[3px_3px_0px_var(--color-border)]` is the *only* shadow value used anywhere in `theCommonsWeb/src`, and it's used in exactly two places. Don't invent a second shadow value — reuse this one if a component genuinely needs the "stacked card" effect, and default to a hairline or standard rule (row 1 or 2) for everything else. + +**Decision procedure:** reaching for a shadow to create separation between a block and its background? First ask whether a rule (row 1–3) does the job — it almost always does, since most separation here is "this is a distinct row/section," not "this is floating above the page." If the block genuinely needs to read as a card sitting on top of the page (a modal, a featured/pinned item), use the exact hard-shadow value above, never a new blurred one. + +## 6. Component inventory — `src/components/ui/` + +Eight components, seven exported from `index.ts` (`Banner.tsx` exists in the directory but is not re-exported — import it directly from `../ui/Banner` if needed, or add it to the barrel if this omission isn't intentional; nothing else in the tree currently imports it that way, so it's unclear whether the omission was deliberate). + +| Component | Purpose | Notable behavior | +|---|---|---| +| `Button` | The only button styling in the system | Three variants: `primary` (filled ink-black, inverts to accent-red on hover), `secondary` (outlined, transparent, bg-alt on hover — the default), `link` (looks like an inline text link, no border/padding). Two sizes (`sm`, `md`). Uppercase, letter-spaced, bold, `--font-sans` — buttons are chrome, not editorial content, hence the one place `--font-sans` is baked into a component rather than opted into. | +| `Badge` | Small inline tag/label pill — but square, not pill-shaped | `active` boolean toggles between an inverted (filled ink, cream text) and outlined (hairline border, muted text) treatment. No `border-radius` at all. | +| `Banner` | Dismissible strip, optionally sticky with scroll-direction hide/reveal | Two variants (`default`, `accent`) that only change the border color. Not in the `ui` barrel export — see above. | +| `Input` | Labeled text input | Label is a separate `