From c82cce4ee433b92c1fe65af47472b5abea7cc3c0 Mon Sep 17 00:00:00 2001
From: Arya Venkatesan
Date: Mon, 3 Aug 2026 16:46:01 -0400
Subject: [PATCH 1/7] =?UTF-8?q?feat(broadcast):=20suite=2046=20=E2=80=94?=
=?UTF-8?q?=20client-feedback=20round=202=20(46.1-46.10)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Root-caused three tickets live before building; their stated premises
were wrong.
46.1 (P0): categories/tags shipped empty for two independent reasons,
neither the one the ticket blamed. The select2 search never filters, so
the dropdown always renders the full 62-term vocabulary and the 1200ms
sleep waited on a query that never fires. More importantly our search
terms did not exist on the destinations: only 4 of 11 Triangle Weekender
terms and 0 of 16 ABC11 terms were real labels. Rewrote both maps against
the live vocabularies and added exact->prefix ranked matching. Rejected
the ticket's proposed length guard, which would have blocked
"Food" -> "Food & Drink"; anchoring keeps "Market" out of "Art Market and
Exhibition" without penalizing length. Unmatched terms now surface in the
review banner instead of being dropped silently.
46.9: #eventStartDate-label is a Fluent UI , not a
)}
+ {locked && (
+
+ Locked while you’re reviewing this event — click “Make changes”
+ below to edit it again.
+
+ )}
{tier >= 2 && !isDraftEmpty(draft) && (
Generating will replace the event details below.
@@ -992,6 +1040,12 @@ export default function App() {
{!isDraftEmpty(draft) && (
Draft auto-saved on this device — cleared when you start over.
)}
+ {locked && (
+
+ Locked while you’re reviewing this event — click “Make changes”
+ below to edit any field.
+
+ )}
({
this.status = status;
}
},
+ SessionExpiredError: class SessionExpiredError extends Error {},
+ setTokenRefreshListener: vi.fn(),
authHeaders: () => ({}),
getAccess: getAccessMock,
previewBroadcast: vi.fn(async () => ({ eligible: [], excluded: [] })),
@@ -370,3 +372,103 @@ describe("T8: hard reset", () => {
});
});
});
+
+describe("T9: reviewed-state lock and contact-info edit affordance", () => {
+ const draftFixture = {
+ draft_id: "d1",
+ title: "Test Event",
+ description: "Test description",
+ start_datetime: "2026-01-01T10:00",
+ end_datetime: "",
+ all_day: false,
+ venue_name: "Venue",
+ address_line1: "1 Main St",
+ state: "NC",
+ zip: "27701",
+ locality: ["durham"],
+ categories: ["music"],
+ event_url: "",
+ ticket_url: "",
+ price: "",
+ is_free: true,
+ image_url: "",
+ organizer_name: "Acme Org",
+ contact_email: "acme@example.com",
+ contact_phone: "919-555-0100",
+ };
+
+ const renderWithPreview = async () => {
+ localStorage.setItem(
+ "broadcast:draft:v2",
+ JSON.stringify({
+ draft: draftFixture,
+ preview: { eligible: [], excluded: [] },
+ selected: [],
+ }),
+ );
+ // The initial draft state spreads the persisted session's sticky contact
+ // fields *after* the persisted draft (see App.tsx), so the session copy
+ // must agree with the draft fixture or it silently blanks these fields.
+ localStorage.setItem(
+ "broadcast:session:v2",
+ JSON.stringify({
+ organizer_name: draftFixture.organizer_name,
+ contact_email: draftFixture.contact_email,
+ contact_phone: draftFixture.contact_phone,
+ }),
+ );
+ useSessionMock.mockReturnValue({
+ data: { user: { email: "operator@thecommons.town" } },
+ isPending: false,
+ });
+ fetchJwtMock.mockResolvedValue("fake.jwt.token");
+ getAccessMock.mockResolvedValue({ tier: 2, is_trial: false, uses_remaining: null });
+ vi.resetModules();
+ const { default: FreshApp } = await import("../App");
+ return render();
+ };
+
+ it("dims the AI Autofill and Event form sections and names Make changes as the unlock", async () => {
+ await renderWithPreview();
+ await waitFor(() => {
+ expect(screen.getByText(/Acme Org/)).toBeInTheDocument();
+ });
+
+ const hints = screen.getAllByText(/Locked while.*reviewing this event/i);
+ expect(hints.length).toBe(2);
+ for (const hint of hints) {
+ expect(hint.textContent).toMatch(/Make changes/);
+ }
+
+ expect(screen.getByPlaceholderText(/Paste an event description/i)).toBeDisabled();
+ });
+
+ it("shows an Edit control on the collapsed contact summary that re-expands the fields in place", async () => {
+ await renderWithPreview();
+ await waitFor(() => {
+ expect(screen.getByText(/Acme Org/)).toBeInTheDocument();
+ });
+
+ // No affordance visible yet besides Edit — fields aren't rendered.
+ expect(screen.queryByLabelText(/Organizer \/ Organization Name/i)).toBeNull();
+
+ // clearDraft/clearSession aren't reset between tests in this file (unlike
+ // the other mocks in beforeEach), so snapshot call counts here rather than
+ // asserting "never called" across the whole suite run.
+ const clearDraftCallsBefore = vi.mocked(clearDraft).mock.calls.length;
+ const clearSessionCallsBefore = vi.mocked(clearSession).mock.calls.length;
+
+ const editButton = screen.getByRole("button", { name: /^Edit$/i });
+ editButton.click();
+
+ const nameInput = await screen.findByLabelText(/Organizer \/ Organization Name/i);
+ expect(nameInput).toHaveValue("Acme Org");
+ expect(nameInput).toBeEnabled();
+ expect(screen.getByLabelText(/Contact Email/i)).toHaveValue("acme@example.com");
+
+ // Reusing "Edit" must not touch the reset/sign-out path.
+ expect(vi.mocked(clearDraft).mock.calls.length).toBe(clearDraftCallsBefore);
+ expect(vi.mocked(clearSession).mock.calls.length).toBe(clearSessionCallsBefore);
+ expect(signOutMock).not.toHaveBeenCalled();
+ });
+});
diff --git a/broadcastWeb/src/services/__tests__/broadcastApiAuthRetry.fast.test.ts b/broadcastWeb/src/services/__tests__/broadcastApiAuthRetry.fast.test.ts
new file mode 100644
index 0000000..9169d2d
--- /dev/null
+++ b/broadcastWeb/src/services/__tests__/broadcastApiAuthRetry.fast.test.ts
@@ -0,0 +1,212 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+import type { EventDraft } from "../../models/broadcastModels";
+
+// vi.hoisted ensures this mock fn reference is available inside the vi.mock
+// factory below (which is hoisted above regular imports).
+const { fetchJwtMock } = vi.hoisted(() => ({
+ fetchJwtMock: vi.fn(async (): Promise => null),
+}));
+
+vi.mock("../../lib/authClient", () => ({
+ fetchJwt: fetchJwtMock,
+}));
+
+const EVENT: EventDraft = {
+ draft_id: "draft-uuid-123",
+ title: "Test Event",
+ description: "A description",
+ start_datetime: "2026-10-17T16:00:00.000Z",
+ all_day: false,
+ venue_name: "Some Venue",
+ address_line1: "1 Main St",
+ state: "NC",
+ zip: "27701",
+ locality: ["durham"],
+ categories: ["music"],
+ is_free: true,
+};
+
+const jsonResponse = (
+ body: unknown,
+ { ok = true, status = 200 }: { ok?: boolean; status?: number } = {},
+) => ({ ok, status, json: () => Promise.resolve(body) });
+
+const authHeader = (mock: ReturnType, callIndex: number): string | undefined => {
+ const [, init] = mock.mock.calls[callIndex] as [string, RequestInit];
+ return (init.headers as Record)["Authorization"];
+};
+
+const fetchMock = vi.fn();
+
+beforeEach(() => {
+ fetchMock.mockReset();
+ fetchJwtMock.mockReset();
+ vi.stubGlobal("fetch", fetchMock);
+ vi.spyOn(console, "error").mockImplementation(() => {});
+});
+
+afterEach(() => {
+ vi.unstubAllGlobals();
+ vi.restoreAllMocks();
+});
+
+describe("authFetch retry-on-401/403", () => {
+ it("transparently refreshes an expired JWT and retries the request once", async () => {
+ const { previewBroadcast } = await import("../broadcastApi");
+ const result = { eligible: [{ site_key: "a", name: "A" }], excluded: [] };
+
+ fetchMock
+ .mockResolvedValueOnce(jsonResponse({ detail: "invalid token" }, { ok: false, status: 403 }))
+ .mockResolvedValueOnce(jsonResponse(result));
+ fetchJwtMock.mockResolvedValueOnce("fresh.jwt.token");
+
+ await expect(previewBroadcast({ jwt: "stale.jwt.token" }, EVENT)).resolves.toEqual(result);
+
+ expect(fetchMock).toHaveBeenCalledTimes(2);
+ expect(fetchJwtMock).toHaveBeenCalledTimes(1);
+ expect(authHeader(fetchMock, 0)).toBe("Bearer stale.jwt.token");
+ expect(authHeader(fetchMock, 1)).toBe("Bearer fresh.jwt.token");
+ });
+
+ it("does not retry more than once when the reminted token is also rejected", async () => {
+ const { previewBroadcast } = await import("../broadcastApi");
+
+ fetchMock
+ .mockResolvedValueOnce(jsonResponse({ detail: "invalid token" }, { ok: false, status: 401 }))
+ .mockResolvedValueOnce(jsonResponse({ detail: "still invalid" }, { ok: false, status: 401 }));
+ fetchJwtMock.mockResolvedValueOnce("fresh.jwt.token");
+
+ await expect(previewBroadcast({ jwt: "stale.jwt.token" }, EVENT)).rejects.toMatchObject({
+ status: 401,
+ });
+
+ expect(fetchMock).toHaveBeenCalledTimes(2);
+ expect(fetchJwtMock).toHaveBeenCalledTimes(1);
+ });
+
+ it("surfaces a SessionExpiredError (not a raw ApiError) when the session itself is dead", async () => {
+ const { previewBroadcast, SessionExpiredError, ApiError } = await import("../broadcastApi");
+
+ // 401 is unambiguous (this backend never uses it for a business denial —
+ // only 403 is overloaded that way), so a failed refresh on a 401 is
+ // reported as a session problem.
+ fetchMock.mockResolvedValueOnce(jsonResponse({}, { ok: false, status: 401 }));
+ fetchJwtMock.mockResolvedValueOnce(null);
+
+ const error = await previewBroadcast({ jwt: "stale.jwt.token" }, EVENT).catch((e) => e);
+
+ expect(error).toBeInstanceOf(SessionExpiredError);
+ expect(error).not.toBeInstanceOf(ApiError);
+ expect(fetchMock).toHaveBeenCalledTimes(1);
+ });
+
+ it("does not retry and preserves the original ApiError when a 403's refresh returns the SAME token", async () => {
+ const { previewBroadcast, ApiError } = await import("../broadcastApi");
+
+ fetchMock.mockResolvedValueOnce(
+ jsonResponse(
+ { detail: "This access code has expired — please contact support." },
+ { ok: false, status: 403 },
+ ),
+ );
+ fetchJwtMock.mockResolvedValueOnce("stale.jwt.token");
+
+ const error = await previewBroadcast({ jwt: "stale.jwt.token" }, EVENT).catch((e) => e);
+
+ expect(error).toBeInstanceOf(ApiError);
+ expect(error).toMatchObject({
+ status: 403,
+ message: "This access code has expired — please contact support.",
+ });
+ expect(fetchMock).toHaveBeenCalledTimes(1);
+ expect(fetchJwtMock).toHaveBeenCalledTimes(1);
+ });
+
+ it("does not retry and preserves the original ApiError when a 403's refresh itself fails (null)", async () => {
+ const { previewBroadcast, ApiError, SessionExpiredError } = await import("../broadcastApi");
+
+ fetchMock.mockResolvedValueOnce(
+ jsonResponse(
+ { detail: "This access code has expired — please contact support." },
+ { ok: false, status: 403 },
+ ),
+ );
+ fetchJwtMock.mockResolvedValueOnce(null);
+
+ const error = await previewBroadcast({ jwt: "stale.jwt.token" }, EVENT).catch((e) => e);
+
+ expect(error).toBeInstanceOf(ApiError);
+ expect(error).not.toBeInstanceOf(SessionExpiredError);
+ expect(error).toMatchObject({
+ status: 403,
+ message: "This access code has expired — please contact support.",
+ });
+ expect(fetchMock).toHaveBeenCalledTimes(1);
+ });
+
+ it("does not attempt a refresh for access-code auth (no JWT to expire)", async () => {
+ const { previewBroadcast } = await import("../broadcastApi");
+
+ fetchMock.mockResolvedValueOnce(jsonResponse({ detail: "bad code" }, { ok: false, status: 403 }));
+
+ await expect(previewBroadcast({ accessCode: "CODE" }, EVENT)).rejects.toMatchObject({
+ status: 403,
+ });
+
+ expect(fetchMock).toHaveBeenCalledTimes(1);
+ expect(fetchJwtMock).not.toHaveBeenCalled();
+ });
+
+ it("calls the registered token-refresh listener with the reminted JWT", async () => {
+ const { previewBroadcast, setTokenRefreshListener } = await import("../broadcastApi");
+ const listener = vi.fn();
+ setTokenRefreshListener(listener);
+
+ fetchMock
+ .mockResolvedValueOnce(jsonResponse({}, { ok: false, status: 401 }))
+ .mockResolvedValueOnce(jsonResponse({ eligible: [], excluded: [] }));
+ fetchJwtMock.mockResolvedValueOnce("fresh.jwt.token");
+
+ await previewBroadcast({ jwt: "stale.jwt.token" }, EVENT);
+
+ expect(listener).toHaveBeenCalledWith("fresh.jwt.token");
+ setTokenRefreshListener(null);
+ });
+
+ it("single-flight: concurrent 401s share exactly one refresh call", async () => {
+ const { previewBroadcast, getJob } = await import("../broadcastApi");
+
+ let resolveRefresh!: (token: string) => void;
+ fetchJwtMock.mockImplementationOnce(
+ () => new Promise((resolve) => (resolveRefresh = resolve)),
+ );
+
+ // Every call in this test 401s until the reminted token shows up in the
+ // Authorization header, so each of the three concurrent requests below
+ // resolves on its own retry rather than needing per-call mock sequencing.
+ fetchMock.mockImplementation((_url: string, init?: RequestInit) => {
+ const authz = (init?.headers as Record | undefined)?.["Authorization"];
+ if (authz === "Bearer fresh.jwt.token") {
+ return Promise.resolve(jsonResponse({ eligible: [], excluded: [] }));
+ }
+ return Promise.resolve(jsonResponse({}, { ok: false, status: 401 }));
+ });
+
+ const auth = { jwt: "stale.jwt.token" };
+ const p1 = previewBroadcast(auth, EVENT);
+ const p2 = previewBroadcast(auth, EVENT);
+ const p3 = getJob(auth, "job-1");
+
+ // Let all three requests make their initial (401'd) fetch and call into
+ // refreshJwt() before the in-flight refresh resolves.
+ await Promise.resolve();
+ await Promise.resolve();
+ await Promise.resolve();
+
+ resolveRefresh("fresh.jwt.token");
+
+ await expect(Promise.all([p1, p2, p3])).resolves.toBeDefined();
+ expect(fetchJwtMock).toHaveBeenCalledTimes(1);
+ });
+});
diff --git a/broadcastWeb/src/services/broadcastApi.ts b/broadcastWeb/src/services/broadcastApi.ts
index 49e91a9..1956ef3 100644
--- a/broadcastWeb/src/services/broadcastApi.ts
+++ b/broadcastWeb/src/services/broadcastApi.ts
@@ -1,7 +1,15 @@
// Mirrors the house service pattern (theCommonsWeb/src/services/eventService.ts):
// plain fetch per call, response.ok checks, no shared client wrapper.
// Auth is passed per request via ApiAuth (Bearer JWT wins over access-code header).
+//
+// The SPA mints its JWT once on mount and holds it in memory for the whole
+// session (see App.tsx), so it eventually expires mid-session — the backend
+// then returns 401/403 (backendServer/broadcast/access.py deliberately does
+// NOT fall through to the access-code path for a present-but-invalid JWT).
+// authFetch() below transparently remints the token and retries once, so an
+// expired token never surfaces as a user-visible error.
+import { fetchJwt } from "../lib/authClient";
import type { EventDraft, JobDetail, PreviewResult, Recipe } from "../models/broadcastModels";
const RAW_BASE =
@@ -31,6 +39,18 @@ export class ApiError extends Error {
}
}
+// Distinguishable from ApiError so callers can branch on it (e.g. prompt the
+// user to sign in again) instead of rendering a raw backend error string.
+// Thrown only when a JWT-authenticated request fails auth and the follow-up
+// fetchJwt() also comes back empty — i.e. the Better Auth session itself is
+// gone, not just the in-memory token.
+export class SessionExpiredError extends Error {
+ constructor() {
+ super("Your session has expired. Please sign in again.");
+ this.name = "SessionExpiredError";
+ }
+}
+
// Bearer JWT wins over the access-code header; neither = no auth header.
export type ApiAuth = { jwt?: string; accessCode?: string };
@@ -40,6 +60,73 @@ export function authHeaders(auth: ApiAuth): Record {
return {};
}
+// App.tsx mints the JWT once on mount and holds it in state; when authFetch
+// remints a token behind the scenes, this lets App.tsx learn about it so the
+// *next* request doesn't immediately have to repeat the refresh dance. Purely
+// an optional hook — nothing here depends on it being registered.
+type TokenRefreshListener = (jwt: string) => void;
+let tokenRefreshListener: TokenRefreshListener | null = null;
+export function setTokenRefreshListener(listener: TokenRefreshListener | null): void {
+ tokenRefreshListener = listener;
+}
+
+// Single-flight: concurrent requests that all 401/403 together share one
+// in-flight fetchJwt() call instead of each minting their own token.
+let inFlightRefresh: Promise | null = null;
+function refreshJwt(): Promise {
+ if (!inFlightRefresh) {
+ inFlightRefresh = fetchJwt().finally(() => {
+ inFlightRefresh = null;
+ });
+ }
+ return inFlightRefresh;
+}
+
+// Wraps every authenticated fetch. On a 401/403 for a JWT-bearing request,
+// remints the token once and retries the same request; any other outcome
+// (access-code auth, non-auth-error status) passes through untouched. Retried
+// at most once — a failure on the retry is surfaced as-is by the caller.
+//
+// A 403 is ambiguous: the backend uses it both for a stale/invalid JWT (a
+// refresh fixes this) and for a legitimate business denial like an expired
+// access code (no refresh will ever fix this, and its `detail` message is
+// the whole point — see backendServer/broadcast/views.py:369-381). The
+// response body can't be trusted to tell those apart, but the token can:
+// - refresh comes back with a DIFFERENT token → the token really was
+// stale → retry once with the new token.
+// - refresh comes back with the SAME token we just sent → the token was
+// never the problem → leave the original response's error (status +
+// backend detail) to surface as-is instead of masking it.
+// - refresh comes back null → the Better Auth session itself is gone.
+// For a 401 that's unambiguous (401 never carries business meaning on
+// this backend — grep confirms it's never emitted), so it's reported as
+// a session problem. A 403 is inherently ambiguous, though, and a failed
+// refresh doesn't tell us *why* it failed — it never proves the 403 was
+// a token issue, so the original response (with whatever business
+// detail it carries) is left to surface rather than guessing.
+async function authFetch(url: string, init: RequestInit, auth: ApiAuth): Promise {
+ const response = await fetch(url, init);
+ if ((response.status !== 401 && response.status !== 403) || !auth.jwt) {
+ return response;
+ }
+ const sentJwt = auth.jwt;
+ const newJwt = await refreshJwt();
+ if (newJwt === null) {
+ if (response.status === 401) {
+ throw new SessionExpiredError();
+ }
+ return response;
+ }
+ if (newJwt === sentJwt) {
+ return response;
+ }
+ tokenRefreshListener?.(newJwt);
+ return fetch(url, {
+ ...init,
+ headers: { ...(init.headers as Record), Authorization: `Bearer ${newJwt}` },
+ });
+}
+
const messageFor = (status: number, body: unknown): string => {
// Prefer the backend's own `detail` (e.g. an expired/exhausted access code)
// so the specific message reaches the user instead of a generic 403 string.
@@ -54,11 +141,15 @@ const messageFor = (status: number, body: unknown): string => {
};
async function post(path: string, auth: ApiAuth, payload: object): Promise {
- const response = await fetch(`${API_BASE}${path}`, {
- method: "POST",
- headers: { "Content-Type": "application/json", ...authHeaders(auth) },
- body: JSON.stringify(payload),
- });
+ const response = await authFetch(
+ `${API_BASE}${path}`,
+ {
+ method: "POST",
+ headers: { "Content-Type": "application/json", ...authHeaders(auth) },
+ body: JSON.stringify(payload),
+ },
+ auth,
+ );
const body = await response.json().catch(() => ({}));
if (!response.ok) {
console.error(`POST ${path} failed:`, response.status, body);
@@ -70,9 +161,7 @@ async function post(path: string, auth: ApiAuth, payload: object): Promise
export const getAccess = async (
auth: ApiAuth,
): Promise<{ tier: 0 | 1 | 2; is_trial: boolean; uses_remaining: number | null }> => {
- const response = await fetch(`${API_BASE}/broadcast/access`, {
- headers: authHeaders(auth),
- });
+ const response = await authFetch(`${API_BASE}/broadcast/access`, { headers: authHeaders(auth) }, auth);
const body = await response.json().catch(() => ({}));
if (!response.ok) {
console.error("GET /broadcast/access failed:", response.status, body);
@@ -110,9 +199,11 @@ export const getJob = async (
auth: ApiAuth,
jobId: string,
): Promise => {
- const response = await fetch(`${API_BASE}/broadcast/jobs/${jobId}`, {
- headers: authHeaders(auth),
- });
+ const response = await authFetch(
+ `${API_BASE}/broadcast/jobs/${jobId}`,
+ { headers: authHeaders(auth) },
+ auth,
+ );
const body = await response.json().catch(() => ({}));
if (!response.ok) {
console.error(`GET job ${jobId} failed:`, response.status);
@@ -162,11 +253,11 @@ export const uploadImage = async (
): Promise<{ url: string }> => {
const formData = new FormData();
formData.append("image", file);
- const response = await fetch(`${API_BASE}/broadcast/upload-image`, {
- method: "POST",
- headers: authHeaders(auth),
- body: formData,
- });
+ const response = await authFetch(
+ `${API_BASE}/broadcast/upload-image`,
+ { method: "POST", headers: authHeaders(auth), body: formData },
+ auth,
+ );
const body = await response.json().catch(() => ({}));
if (!response.ok) {
console.error("POST /broadcast/upload-image failed:", response.status, body);
@@ -220,9 +311,10 @@ export const getManualRecipe = async (
jobId: string,
siteKey: string,
): Promise => {
- const response = await fetch(
+ const response = await authFetch(
`${API_BASE}/broadcast/jobs/${jobId}/manual/${siteKey}`,
{ headers: authHeaders(auth) },
+ auth,
);
const body = await response.json().catch(() => ({}));
if (!response.ok) {
@@ -238,9 +330,11 @@ export const openScreenshot = async (
auth: ApiAuth,
screenshotPath: string,
): Promise => {
- const response = await fetch(`${API_BASE}${screenshotPath}`, {
- headers: authHeaders(auth),
- });
+ const response = await authFetch(
+ `${API_BASE}${screenshotPath}`,
+ { headers: authHeaders(auth) },
+ auth,
+ );
if (!response.ok) {
console.error("screenshot fetch failed:", response.status);
throw new ApiError(response.status, "Could not load the screenshot.");
diff --git a/docs/broadcast.md b/docs/broadcast.md
index 9b26305..6270417 100644
--- a/docs/broadcast.md
+++ b/docs/broadcast.md
@@ -145,13 +145,22 @@ Two independent code pools, distinguished by `AccessCode.kind`:
| **1** | `BroadcastAccess.tier=1` — via `set_broadcast_access` or a tier-1 UPGRADE code | ✓ | ✗ |
| **2** | `BroadcastAccess.tier=2` (dev-granted or UPGRADE code), or any valid TRIAL code | ✓ | ✓ |
+`resolve_access` (the API layer) will grant tier 2 to a bare TRIAL code with no login at all — that's true of the header/body auth path used by `curl` or any external caller. It is **not** true of the SPA: the SPA has no logged-out entry point to the access-code field (see Frontend below), so in practice every code — TRIAL or UPGRADE — is entered only after signing in. Handing someone a TRIAL code is not, by itself, enough to get them broadcasting through the UI; they also need an account.
+
Permission classes: `RequiresBroadcastTier1` (preview / submit / recipe / job endpoints), `RequiresBroadcastTier2` (ai-autofill only), `RequiresBroadcastLogin` (redeem only — no tier check, just a valid JWT; stamps `request.broadcast_email`). The tier classes stamp `request.broadcast_access` (`AccessResult`) and `request.broadcast_client_label` for downstream views.
**Metering:** TRIAL codes are metered **only at `POST /broadcast/preview`** — `AccessCodeUse.objects.get_or_create(code, draft_id)` (idempotent). Trial callers must include `draft_id` in the preview body; missing → 400. Re-submitting the same `draft_id` (edits) is free. UPGRADE codes are metered by distinct redeeming email (`AccessCodeRedemption`), checked in `redeem_upgrade_code`. Logged-in JWT sessions are never metered on preview.
**`client_label`** on `BroadcastSubmission`: email for JWT users; `AccessCode.label` for TRIAL-code users. Access codes are stored only in the database — there is no env-var code list. `GET /broadcast/access` lets the SPA query the caller's tier and remaining trial uses (JWT or TRIAL code); `POST /broadcast/redeem` is how a logged-in user applies an UPGRADE code.
-**Frontend (`broadcastWeb`):** one textfield does double duty, labeled by login state — logged out it's "Access Code" (`getAccess`, anonymous TRIAL resolution, persisted in localStorage); logged in it relabels to "Upgrade Account" (`redeemAccessCode` → `POST /broadcast/redeem`, permanent, nothing persisted client-side since the grant now lives server-side against the account).
+**Frontend (`broadcastWeb`):** access always requires signing in first — the access-code field only renders in step 2 of the "Access" section (`App.tsx`), which is gated behind step 1 ("Sign in"; `step2` is `"todo"` while `!signedIn`). There is no logged-out entry point to the field. The label is static regardless of sign-in state or tier — it never relabels. The live copy (verified on prod 2026-08-03):
+
+- heading: "Enter your access code"
+- link (to re-open the field after a code is already verified): "Have another code?"
+- placeholder: "Provided by The Commons"
+- button: "Verify"
+
+One code box, two code kinds under the hood: `handleVerifyCode` first tries the entered code as an UPGRADE code (`redeemAccessCode` → `POST /broadcast/redeem`, requires the caller's JWT, permanent grant on the account); if the backend 403s that, it retries the same code as a TRIAL code (`getAccess({ accessCode })`, anonymous per-request resolution at the API layer, but only ever invoked here from an already-authenticated session). Either way the user sees one Verify step — the SPA never surfaces which kind matched.
**Sign-in:** there is no embedded auth UI in the SPA (the former inline `AuthModal` component was removed). The "Sign in / Create account" button (`App.tsx`'s `handleSignIn`) does a full-page navigation to `${VITE_BETTER_AUTH_URL}/signin?redirect_to=` — i.e. the shared auth **portal** (`theCommonsWeb`'s `src/app/(portal)/`, served at `https://auth.thecommons.town` in prod / `http://localhost:3000` in dev; see [ARCHITECTURE.md#authentication](../ARCHITECTURE.md#authentication)). Because the session cookie is scoped to `.thecommons.town`, completing sign-in in the portal returns the browser to the exact broadcast URL it left, already authenticated. `broadcastWeb/src/lib/authClient.ts` still exists — it's the Better Auth client used to read the session, mint a JWT (`fetchJwt`), and call `signOut()`, not to render any sign-in form.
diff --git a/notion-sync/STATE.md b/notion-sync/STATE.md
index e79836c..1703b40 100644
--- a/notion-sync/STATE.md
+++ b/notion-sync/STATE.md
@@ -6,7 +6,7 @@ The ledger mirrors what *should* be on the Notion board so the desktop app can r
---
-**Next suite number:** `46`
+**Next suite number:** `47`
## Suite ledger
@@ -41,6 +41,7 @@ per-ticket status lives on each ticket subpage (see OUTBOX preamble).
| 43 | Human onboarding docs (`human-docs/` subsystem set) | Needs QA | 43.1–43.12 BUILT 2026-08-01 via `/orchestrate` from `human-docs/HANDOFF_PLAN.md`, one `/handoff-report` pass each. 12 docs + index/route-map updates; 2 stale agent-doc refs fixed (`docs/admin-backend.md`, `docs/redis-celery-handoff.md`). Docs only, no code. **Findings needing owner action:** CI's deploy job already runs `docker compose` on push to `main` but the VM has no Docker (next merge to main fails); `--color-ink` referenced in two components but never defined (silent CSS fallback); the eslint `no-restricted-imports` authClient guard is dead (alias vs. relative imports) | _(pending)_ |
| 44 | Post-Docker-cutover follow-ups (kernel reboot, stray migration, untested calendar work) | In Progress | 44.1–44.3 planned 2026-08-02. **44.2 RESOLVED** (PR #42): the stray migration was real — committed as `0023_seed_raleigh_cary_morrisville_towns` + `0024_seed_wake_forest_town`; unpushed migrations are a silent no-op at deploy since `migrate` only sees what `COPY . .` baked in. **44.3 in progress**: exercising the PR #41 calendar work found 3 real bugs (morrisvillechamber UTC double-offset, downtownraleigh bare-hour times, force_town overriding correct guesses), all fixed + tested in PR #42; still needs a live `SourceRun` check on prod sources 23–27. **44.1 untouched** (VM still needs the kernel reboot) | _(pending)_ |
| 45 | Ingestion correctness + prod egress (post-ingest audit) | Needs QA | 45.1-45.10 planned 2026-08-02; **9 of 10 BUILT 2026-08-03 via `/orchestrate`** (752 tests green, ruff clean, migrations 0019-0021 linear). **45.2 premise CONFIRMED, not speculative** - reproduced before building: a 9:00 PM ET event rendered `Thursday, August 6 - 1:00 AM` (wrong time AND weekday); fixed by `TIME_ZONE=America/New_York`, double-correction sweep found zero compensating sites. **45.5 SHIPPED TO PROD** and the ticket's guess was inverted: across all 765 raw events source id 8 has **0 unique** source_uids vs id 6's 9, so id 8 (not id 6) was the strict subset - deactivated on the VM, SourceRun history preserved. **45.1** root cause was purely bare call sites; the batch fns already resolved prompt_suffix correctly. New `ingestion/prompt_dispatch.py`. Note `RawEvent.source` is non-nullable, so the orphan concern was theoretical. **45.3** guard runs before the Gemini call (saves the API call; raw title is the more reliable signal) and `cancelled` was deliberately **excluded** from `CANDIDATE_STATUSES` - organizers un-cancel and resubmit, so anchoring would silently suppress the legitimate resubmission. **45.10** scoped internal-only per owner decision: `raw_start`->`raw_start_datetime` (16 files); `Event.date` untouched, no API break. **45.6 NOT BUILT - needs owner decision.** Research overturned the presumed fix: Eventbrite's public `events/search` API was discontinued Dec 2019 (org-owned events only), so an official integration covers **0 of 4** sources, not 3 of 4; and their WAF `captcha` plausibly enforces their own anti-scraping ToS, so funding a proxy is paying to keep violating it. Recommended: quarantine the 3 Eventbrite sources via 45.7 + free delisting request for Downtown Raleigh (id 20). $0/mo. Report in scratchpad `45.6-egress-options.md`. **Ops follow-ups (Neon data, ship by no migration):** set `default_town` on prod sources then run `reopen_skipped_towns` (18 stranded rows); set `blocked_reason` on the 4 blocked sources; run `probe_sources` in **scrape-worker** (no Chromium in `backend`) | _(pending)_ |
+| 46 | Broadcast client-feedback round 2 (extension taxonomy regression, https image URLs, JWT refresh, form affordances) | Needs QA | **46.1–46.10 ALL BUILT 2026-08-03 via `/orchestrate`** (backend 265 tests OK + ruff clean; broadcastWeb 92/92 + `pnpm build` clean). **Three ticket premises were wrong and were root-caused live before building.** **46.1:** the defect was NOT the extension's exact matching — (a) the select2 search never filters, so the dropdown always renders the full 62-term vocabulary and the `sleep(1200)` waited on a query that never fires; (b) the real killer is that our search terms don't exist on the destinations — only **4 of 11** TW terms and **0 of 16** ABC11 terms were real (typing `Music` on ABC11 returns literally "No options"; its vocabulary is 17 slash-compounds like `Theater / Concerts`). The ticket's proposed **length guard was harmful** and was rejected — it would have blocked `Food`→`Food & Drink`; replaced with **anchoring** (exact → prefix), which still keeps `Market` out of `Art Market and Exhibition`. Both maps rewritten to verified labels + `test_category_maps_fast.py` snapshots all 62/17 labels as the regression guard. Per owner decision, ABC11 slugs with no defensible match (`family-kids`, `literary`, `nightlife`, `wellness`, `film`) send **no category** rather than `Misc.`. **46.9:** `#eventStartDate-label` is **not a ``** — it's a Fluent UI `` whose id merely ends in `-label`; selector was always correct, and the retry logic the ticket asked for already shipped in suite 31 (`fillInputVerified`, `f9eae28`). Net change = hint/comment corrections only; the real bug (Trumba self-population, form loads pre-filled with today's date) is confirmed. **46.4:** `form-dim` already included `locked`, so dimming worked; only the "Make changes" hint was missing. **Regression caught at integration:** 46.3's blanket refresh-on-403 swallowed legitimate business errors, because `views.py:369-381` overloads 403 for both stale JWTs and access-code denials (`_ACCESS_DENIED_DETAIL`); now discriminated by whether the refresh returns a *different* token. Note the broadcast API **never emits a real 401**. **46.8:** the ⚠️ drag-drop concern is resolved — the Trumba uploader is a real visible `input[type=file][accept="image/*"]` at `.image-field-esf input[type=file]`, so programmatic fill works. **⚠️ MANUAL QA STILL OUTSTANDING** (buildless MV3, no test runner): live-form fill verification for **46.1, 46.8, 46.9** on both destinations — fill, do NOT submit. **Follow-ups needing their own tickets:** `human-docs/broadcast.md:120-121` has the identical access-code drift 46.10 just fixed in `docs/broadcast.md`; ABC11 alt-text has no upstream `CanonicalEvent` field (gap documented, content deliberately not invented); extension test infrastructure. _Original plan:_ 46.1–46.10 planned 2026-08-03 via `/write-tickets`. Triaged live against prod; most client reports had different root causes than reported. **46.1 is the P0 and is a regression, not an unfixed bug** — `broadcastExtension/content.js:310-314` documents that the earlier "unrelated tags" bug (client 535) was already fixed by switching to strict exact-label matching; that fix over-corrected so nothing matches, and unmatched terms are skipped *silently* → every event ships to every destination with zero categories/tags. Fix must be a ranked match (exact → prefix → token-set) with a length guard, not a revert. **Waves:** 1 = 46.1, 46.2, 46.3, 46.5, 46.9, 46.10 (parallel); 2 = 46.4, 46.7; 3 = 46.6; 4 = 46.8 (needs 46.2). ⚠️ **46.4, 46.6, 46.7 all edit `broadcastWeb/src/App.tsx` — serialize or give all three to one instance** (see the shared-worktree hazard in project memory). 46.2 is a genuine one-liner (`SECURE_PROXY_SSL_HEADER` in `prod.py`; nginx already sends `X-Forwarded-Proto`) but carries a real blast radius via `request.is_secure()`. Open ⚠️ decisions: 46.1 root-cause assumption (label-shape/timing vs. dropdown never opening — log first), 46.2 nothing depending on the http scheme, 46.5 low-vs-high value of a `5-10` range, 46.7 re-expand in place vs. relocating the organizer fields, 46.8 whether the drag-drop widget accepts a programmatic file set, 46.9 the real ABC11 input selector (needs `capture_broadcast_form`). **No extension test command exists** — `broadcastExtension/` is buildless MV3, so 46.1 and 46.9 acceptance is manual live-form verification (fill, never submit); a ticket for extension test infra was proposed but not written | _(pending)_ |
CF[Cloudflare edge - DNS + TLS proxy]
- CF --> Nginx[nginx on the VM - terminates TLS again, Full strict]
+ CF --> Nginx[nginx container - terminates TLS again, Full strict]
- Nginx -->|thecommons.town| NextJS[Next.js process, port 3000]
+ Nginx -->|thecommons.town| NextJS[nextjs container, port 3000]
Nginx -->|www.thecommons.town| Redirect1[301 to apex]
Nginx -->|auth.thecommons.town| NextJS
- Nginx -->|api.thecommons.town| Gunicorn[gunicorn via Unix socket - run/gunicorn/gunicorn.sock]
- Nginx -->|api.thecommons.town/static/| StaticFiles[backendServer/staticfiles - collectstatic output]
- Nginx -->|api.thecommons.town/media/| MediaFiles[MEDIA_ROOT on disk - never touches Django]
- Nginx -->|broadcast.thecommons.town| BroadcastSPA[static broadcastWeb build - dist/]
+ Nginx -->|api.thecommons.town| Gunicorn[backend container - gunicorn on TCP 8000, internal only]
+ Nginx -->|api.thecommons.town/static/| StaticFiles[baked into the nginx image via COPY --from the backend build stage]
+ Nginx -->|api.thecommons.town/media/| MediaFiles[MEDIA_ROOT bind-mounted read-only - never touches Django]
+ Nginx -->|broadcast.thecommons.town| BroadcastSPA[broadcast-spa-build stage output, COPY --from into the nginx image]
```
**Three things worth calling out.** First, `auth.thecommons.town` and the apex both land
-on the *same* Next.js process — Better Auth lives inside `theCommonsWeb`, not a separate
-service, so the "auth origin" is a routing decision, not a different deployable. Second,
-`api.thecommons.town` reaches gunicorn over a **Unix socket**
-(`unix:/run/gunicorn/gunicorn.sock`), not TCP — this matters for one sharp edge below
-(django-ratelimit's IP key) and is the reason the containerized rewrite in `DEPLOY.md`
-switches to TCP instead: a socket path doesn't cross a container boundary cleanly. Third,
-`/media/` is nginx reading a directory directly; Django is never in that request path in
-production — see §5.
+on the *same* `nextjs` container — Better Auth lives inside `theCommonsWeb`, not a
+separate service, so the "auth origin" is a routing decision, not a different deployable.
+Second, `api.thecommons.town` reaches gunicorn over **TCP** (`backend:8000`, exposed only
+on the compose network, never to the host) — this is a deliberate change from the old
+systemd deployment's Unix socket (`unix:/run/gunicorn/gunicorn.sock`), because a socket
+path doesn't cross a container boundary cleanly; the switch is why the historical
+`django-ratelimit`/`REMOTE_ADDR` sharp edge below is written up as a regression check
+rather than a live risk. Third, `/media/` is nginx reading a read-only bind mount
+directly; Django is never in that request path in production — see §6.
**On the `auth.thecommons.town` nginx routing question:** an earlier documentation pass
(`auth.md`) explicitly could not verify this and deferred it here. It's resolved: the
cutover runbook (`docs/runbook-auth-cutover.md`) records the exact server block added to
-the VM's nginx config — `server_name auth.thecommons.town` with `proxy_pass
-http://127.0.0.1:3000` and the standard `X-Real-IP`/`X-Forwarded-*` headers, TLS from the
-same wildcard Cloudflare origin cert as every other subdomain — and an execution record
-dated 2026-07-30 confirming it was applied and smoke-tested live (`curl
-https://auth.thecommons.town/api/auth/jwks` returned 200 with a real JWKS body). One honest
-caveat: that server block lives in a **hand-edited file directly on the VM**
-(`/etc/nginx/sites-available/thecommons`), which is not itself checked into this
-repository — only the *runbook instructions* for editing it are. The broadcast subdomain's
-block is the one nginx fragment actually tracked in git
-(`deploy/nginx-broadcast.conf.snippet`), meant to be pasted into that same live file. So
-the routing is real, live, and verified — just not something `git grep` alone will ever
-show you; you have to read the runbook or SSH in.
-
-### How a deploy happens
+the VM's (then hand-edited) nginx config — `server_name auth.thecommons.town` with
+`proxy_pass http://127.0.0.1:3000` and the standard `X-Real-IP`/`X-Forwarded-*` headers,
+TLS from the same wildcard Cloudflare origin cert as every other subdomain — and an
+execution record dated 2026-07-30 confirming it was applied and smoke-tested live (`curl
+https://auth.thecommons.town/api/auth/jwks` returned 200 with a real JWKS body). That
+routing decision carried forward unchanged into the containerized nginx config in
+`deploy/nginx/` — same subdomain, same target process, now reached over the compose
+network instead of `127.0.0.1`. The broadcast subdomain's block is the nginx fragment
+tracked in git as `deploy/nginx-broadcast.conf.snippet`, folded into the current
+`deploy/nginx/` config that ships inside the nginx image.
+
+#### How a deploy happens
Every push to `main` runs CI (`.github/workflows/ci.yml`): a `lint` job, then `backend`
(Django tests, Postgres 16 service container, `--tag=fast` then `--tag=db`),
@@ -99,76 +126,88 @@ sequenceDiagram
Note over GH: deploy job only starts if all three test jobs pass
GH->>VM: SSH in (appleboy/ssh-action, host key pinned via fingerprint)
VM->>VM: git pull origin main
- VM->>VM: uv sync (backendServer)
- VM->>VM: manage.py migrate --check
+ VM->>VM: docker compose -f docker-compose.yml build (real build args sourced from .env.local/.env, exported under *_BUILD_* names)
+ VM->>VM: grep built broadcast-spa-build image for a real thecommons.town API origin
+ VM->>VM: docker compose -f docker-compose.yml run --rm migrate manage.py migrate --check
alt migrations pending
- VM->>PG: pg_dump (gzip, timestamped) to /home/ubuntu/backups
+ VM->>PG: pg_dump (via postgres:18-alpine container, gzip, timestamped) to /home/ubuntu/backups
VM->>VM: prune to 5 newest dumps
- VM->>PG: manage.py migrate --noinput
+ VM->>PG: docker compose run --rm migrate manage.py migrate --noinput
else nothing pending
VM->>VM: skip migrate entirely
end
- VM->>VM: manage.py collectstatic --noinput
- VM->>VM: pnpm build (theCommonsWeb, then broadcastWeb)
- VM->>VM: grep built broadcastWeb bundle for a real thecommons.town API origin
- VM->>VM: sudo systemctl restart gunicorn nextjs celery celerybeat broadcast-worker scrape-worker
- VM->>VM: systemctl is-active on all six (must all report active)
+ VM->>VM: docker compose -f docker-compose.yml up -d (recreates every service from the images just built)
+ VM->>VM: docker compose ps --status running - assert redis, backend, celery, celerybeat, broadcast-worker, scrape-worker, nextjs, nginx all running
GH->>VM: second SSH step - post-deploy smoke test
VM->>VM: curl the three public domains, expect 200
- VM->>VM: POST an invalid broadcast request, expect 403 not 500 (Unix-socket REMOTE_ADDR regression check)
+ VM->>VM: POST an invalid broadcast request, expect 403 not 500 (Unix-socket-era REMOTE_ADDR regression check, still run post-cutover)
VM->>VM: GET /auth/me with no credentials, expect 401/403 not 500
```
-**Four things worth calling out.** First, the migration guard is genuinely conditional —
-`migrate --check` exits non-zero only when there's real unapplied work, so most deploys
-skip the dump-and-migrate branch entirely; a `pg_dump` is never skipped when a migration
-*is* about to run, and the guard hard-fails the whole deploy if `pg_dump` isn't installed
-rather than silently proceeding without a backup. Second, the broadcastWeb bundle grep
-exists because a malformed `VITE_BROADCAST_API_BASE_URL` builds cleanly and only fails at
-runtime, as every API call silently misroutes — this catches that class of bug before the
-build goes live, not after. Third, `systemctl is-active` passing is necessary but not
-sufficient — a crashing view or a misrouted SPA both restart clean and report `active`,
-which is exactly why there's a separate smoke-test step hitting real URLs afterward, not
-just a process-liveness check. Fourth, the smoke test's `403` check on a broadcast endpoint
-is a deliberate regression probe: nginx talking to gunicorn over a Unix socket used to
-leave `REMOTE_ADDR` empty, which crashed `django-ratelimit`'s IP-based rate limiting with
-an unhandled 500 on every request to a rate-limited broadcast view — a `500` here means
-that bug is back, a `403` means the request was correctly rejected before it ever became a
-ratelimit crash.
-
-There is no separate `gunicorn.service` or `nextjs.service` file in this repository's
-`deploy/` directory, and none exists anywhere in git history — those two units were set up
-by hand directly on the VM and were never checked in, unlike the four Celery-family units
-and the healthcheck unit, which are. If you need their exact unit-file contents, SSH in and
-read `/etc/systemd/system/gunicorn.service` / `nextjs.service` directly, or see the last
-commit of `DEPLOY.md` before its Docker rewrite (`git show 053d65b:DEPLOY.md`) for a
-recorded copy of what they contained as of late July.
-
-## 3. The systemd units
-
-| Unit | What it runs | Drains / serves | How to check it |
+**Four things worth calling out.** First, `collectstatic` and the frontend builds
+(`pnpm build` for both `theCommonsWeb` and `broadcastWeb`) are no longer separate deploy
+steps — they happen at Docker image build time (`backendServer/Dockerfile` bakes
+`staticfiles_build/static`; `Dockerfile.frontend` builds both frontends), so the deploy
+script itself only builds images, runs the guarded migration, and swaps containers.
+Second, the broadcastWeb bundle grep exists because a malformed
+`VITE_BROADCAST_API_BASE_URL` builds cleanly and only fails at runtime, as every API call
+silently misroutes — this catches that class of bug before the build goes live, not
+after. Third, the migration guard is genuinely conditional — `migrate --check` exits
+non-zero only when there's real unapplied work, so most deploys skip the dump-and-migrate
+branch entirely; a `pg_dump` (now run via a disposable `postgres:18-alpine` container,
+since the VM no longer has a host-level `postgresql-client`) is never skipped when a
+migration *is* about to run. Fourth, `docker compose ps --status running` replacing
+`systemctl is-active` is necessary but not sufficient — a crashing view or a misrouted SPA
+both restart clean and report running, which is exactly why there's a separate
+smoke-test step hitting real URLs afterward, not just a container-liveness check.
+
+The old `gunicorn.service`/`nextjs.service` units (never checked into `deploy/`, hand-set
+up on the VM) are gone along with the rest of the systemd app units — their process
+definitions now live in `docker-compose.yml`'s `backend` and `nextjs` services instead.
+If you need the exact historical unit-file contents from the systemd era, see the last
+commit of `DEPLOY.md` before its Docker rewrite (`git show 053d65b:DEPLOY.md`).
+
+### 3. The retired systemd units (historical reference)
+
+These units ran production until the 2026-08-02 Docker cutover (PR #41). They are no
+longer active on the VM — the table is kept for anyone debugging an old incident report,
+reading `docs/prod-incident-2026-07-21-scheduler-outage.md`, or comparing pre- and
+post-cutover behavior. **The one still-active exception is the last row.**
+
+| Unit | What it ran | Drained / served | Status today |
|---|---|---|---|
-| `gunicorn` | Django via a Unix socket, 3 sync workers | `api.thecommons.town` (proxied by nginx) | `systemctl status gunicorn`; not tracked in `deploy/` — hand-configured on the VM |
-| `nextjs` | `node`/`npm run start` for `theCommonsWeb`, port 3000 | `thecommons.town` and `auth.thecommons.town` (both proxy to the same process) | `systemctl status nextjs`; not tracked in `deploy/` — hand-configured on the VM |
-| `redis-server` | Standard `apt`-installed Redis, `/etc/redis/redis.conf` | DB 0 = Celery broker/results, DB 1 = Django cache | `systemctl status redis-server`; `redis-cli -a ping` |
-| `celery` (`deploy/celery.service`) | Default worker, `.venv/bin/celery -A backend worker -n commons-default@%h --concurrency=2` | Everything not explicitly routed elsewhere — digest sends, misc tasks | `systemctl status celery`; `manage.py healthcheck`'s `celery_worker` probe |
-| `celerybeat` (`deploy/celerybeat.service`) | Scheduler, `django_celery_beat`'s `DatabaseScheduler` — exactly one process, never scale this | Fires `ingest-events-daily` (04:00 ET), `weekly-digest-sunday`/`monthly-digest` (18:00 ET), `broadcast-orphan-recovery` | `systemctl status celerybeat`; `manage.py healthcheck`'s per-task `beat:` freshness probes — the check this doc's §8 incident is really about |
-| `broadcast-worker` (`deploy/broadcast-worker.service`) | Playwright form-filler, `celery -A backend worker -Q broadcast -c 1` | The dedicated `broadcast` queue only — `-c 1` is load-bearing, not tuning: orphan recovery assumes a single worker | `systemctl status broadcast-worker` |
-| `scrape-worker` (`deploy/scrape-worker.service`) | Headless-Chromium ingestion scraper, `celery -A backend worker -Q scrape -c 1` | The dedicated `scrape` queue, kept off the default worker so Chromium memory can't starve digests/ingestion | `systemctl status scrape-worker` |
-| `healthcheck.timer` / `.service` (`deploy/healthcheck.*`) | Hourly `bash deploy/healthcheck.sh`, itself running `manage.py healthcheck --require-prod` | Nothing — read-only report | `systemctl list-timers healthcheck.timer`; `journalctl -u healthcheck.service -n 50` |
-
-`celery`, `celerybeat`, `broadcast-worker`, and `scrape-worker` all `Require=` and
-`After=redis-server.service` and set `Restart=always` — the restart policy is
-belt-and-suspenders, explained in §4, not the primary fix for anything. `deploy/`'s
-`nginx-broadcast.conf.snippet` is not a systemd unit; it's an nginx server-block fragment
-meant to be appended by hand into the VM's single live config file.
-
-## 4. The `uv run` vs. venv-binary sharp edge — a real outage, not a style rule
-
-Every long-lived unit in `deploy/` execs `/home/ubuntu/thecommons/backendServer/.venv/bin/celery`
-directly. That specific phrasing — the venv binary, not `uv run celery`, and not a wrapper
-shell script — is load-bearing, and the reason is a real production incident recorded in
-full at `docs/prod-incident-2026-07-21-scheduler-outage.md`.
+| `gunicorn` | Django via a Unix socket, 3 sync workers | `api.thecommons.town` (proxied by nginx) | Retired — replaced by the `backend` container (TCP 8000) |
+| `nextjs` | `node`/`npm run start` for `theCommonsWeb`, port 3000 | `thecommons.town` and `auth.thecommons.town` (both proxied to the same process) | Retired — replaced by the `nextjs` container |
+| `redis-server` | Standard `apt`-installed Redis, `/etc/redis/redis.conf` | DB 0 = Celery broker/results, DB 1 = Django cache | Retired — replaced by the `redis` container (`redis:7-alpine`) |
+| `celery` (`deploy/celery.service`) | Default worker, `.venv/bin/celery -A backend worker -n commons-default@%h --concurrency=2` | Everything not explicitly routed elsewhere — digest sends, misc tasks | Retired — replaced by the `celery` container, same command |
+| `celerybeat` (`deploy/celerybeat.service`) | Scheduler, `django_celery_beat`'s `DatabaseScheduler` — exactly one process, never scale this | Fires `ingest-events-daily` (04:00 ET), `weekly-digest-sunday`/`monthly-digest` (18:00 ET), `broadcast-orphan-recovery` | Retired — replaced by the `celerybeat` container; still exactly one process |
+| `broadcast-worker` (`deploy/broadcast-worker.service`) | Playwright form-filler, `celery -A backend worker -Q broadcast -c 1` | The dedicated `broadcast` queue only — `-c 1` is load-bearing, not tuning: orphan recovery assumes a single worker | Retired — replaced by the `broadcast-worker` container, still `-c 1` |
+| `scrape-worker` (`deploy/scrape-worker.service`) | Headless-Chromium ingestion scraper, `celery -A backend worker -Q scrape -c 1` | The dedicated `scrape` queue, kept off the default worker so Chromium memory can't starve digests/ingestion | Retired — replaced by the `scrape-worker` container, still `-c 1` |
+| `healthcheck.timer` / `.service` (`deploy/healthcheck.*`) | Hourly `bash deploy/healthcheck.sh`, itself running `manage.py healthcheck --require-prod` | Nothing — read-only report | **Still live, host-level.** Never containerized — it shells into the running containers to run the same health command. |
+
+The `deploy/*.service` unit files (`celery.service`, `celerybeat.service`,
+`broadcast-worker.service`, `scrape-worker.service`, `healthcheck.service`,
+`healthcheck.timer`, `healthcheck.sh`) and `deploy/nginx-broadcast.conf.snippet` still
+physically exist in the repo's `deploy/` directory — nothing was deleted. "Retired" means
+retired on the production box, not removed from git history; they're kept as a rollback
+reference and because `healthcheck.service`/`.timer` are still genuinely in use.
+
+### 4. The `uv run` vs. venv-binary sharp edge — a real outage, not a style rule
+
+This is systemd-era history: it describes why the *retired* Celery-family units execed
+the venv binary directly rather than `uv run`. It no longer applies mechanically inside
+Docker (there's no snap-scope, no user-session teardown a container can be torn down by
+in the same way), but the incident is still the reason today's containerized Celery
+services (`celery`, `celerybeat`, `broadcast-worker`, `scrape-worker`) each run with
+`restart: unless-stopped` and no dependency on a login shell surviving — the underlying
+lesson (a "clean" silent exit is not the same as "still running") is exactly what that
+policy guards against in the new architecture too.
+
+Every long-lived systemd unit in `deploy/` execed
+`/home/ubuntu/thecommons/backendServer/.venv/bin/celery` directly. That specific phrasing
+— the venv binary, not `uv run celery`, and not a wrapper shell script — was
+load-bearing, and the reason is a real production incident recorded in full at
+`docs/prod-incident-2026-07-21-scheduler-outage.md`.
```mermaid
flowchart TD
@@ -198,30 +237,28 @@ never touched snap, stayed up the entire time on the same VM through the same de
async stack was fully dead for 8 days before anyone noticed, because the site itself kept
serving pages — nothing about "the site is up" implied "background jobs are running."
-**What breaks if someone "simplifies" a unit back to `uv run`:** exactly this, again. The
-fix — execing `.venv/bin/celery` directly — removes the snap-scope mechanism entirely,
-which is the actual fix; `loginctl enable-linger ubuntu` and `Restart=always` are
-defense-in-depth layered on top, not substitutes for it. A reviewer who sees `uv run` as
-"more consistent with the rest of the deploy tooling" and reverts a unit to it silently
-reopens this exact failure mode — it will not show up in `systemctl status` right after the
-change, only after the next SSH session that started the deploy ends.
+**What broke if someone "simplified" a unit back to `uv run`:** exactly this, again. The
+fix — execing `.venv/bin/celery` directly — removed the snap-scope mechanism entirely,
+which was the actual fix; `loginctl enable-linger ubuntu` and `Restart=always` were
+defense-in-depth layered on top, not substitutes for it.
-**One deliberate exception, and it is not a contradiction:** `healthcheck.service` still
+**One deliberate exception, and it was not a contradiction:** `healthcheck.service` still
uses `/snap/bin/uv` (`Environment=UV_BIN=/snap/bin/uv`, `ExecStart=... bash
-deploy/healthcheck.sh`) and that is fine. It's `Type=oneshot` — the process starts, runs
-the health report to completion in a few seconds, and exits on its own, well before any SSH
-session it happened to be triggered near could tear down. The failure mode above only bites
-a process still running at the moment a user slice gets torn down; a oneshot that's already
-finished has nothing left to kill. Don't read the healthcheck unit's `uv` line as
-permission to relax the rule anywhere else — it's a narrow exception with a specific reason,
-not evidence the rule is soft.
+deploy/healthcheck.sh`) and that is fine — this unit is still live today (§3). It's
+`Type=oneshot` — the process starts, runs the health report to completion in a few
+seconds, and exits on its own, well before any SSH session it happened to be triggered
+near could tear down. The failure mode above only bites a process still running at the
+moment a user slice gets torn down; a oneshot that's already finished has nothing left to
+kill.
-## 5. Environment selection: `DJANGO_ENV` and how failure got narrower
+### 5. Environment selection: `DJANGO_ENV` and how failure got narrower
Django settings are resolved by `backend/settings/__init__.py` via a function,
`select_settings_env`, reading the `DJANGO_ENV` environment variable — not by
`DJANGO_SETTINGS_MODULE` pointing at `prod.py` directly, the way Django docs usually show
-it.
+it. This resolution mechanism is unchanged by the Docker cutover — every prod-facing
+service in `docker-compose.yml` (`backend`, `celery`, `celerybeat`, `broadcast-worker`,
+`scrape-worker`, `migrate`) sets `environment: DJANGO_ENV: prod` explicitly.
```mermaid
flowchart TD
@@ -251,46 +288,48 @@ slow. `events/tests/test_config_fast.py` pins this exact behavior as a regressio
The remaining gap — `DJANGO_ENV` unset in prod specifically, which the hard-error change
does nothing for, since unset is still valid input — is caught by `manage.py healthcheck
---require-prod`, run hourly via `healthcheck.timer`. That command checks `settings.DEBUG`
-and whether `ALLOWED_HOSTS` is anything other than localhost-only, and reports a `FAIL` if
-either looks like dev settings leaked into what's supposed to be prod. **Read `--require-prod`
-correctly: it is a detector, not a guard.** It can tell you, up to an hour later, that
-production is quietly running on dev settings; it cannot stop that from happening, and it
-does not run on every request or every deploy — only once an hour, on the health-check
-timer's own schedule. A misconfigured `.env` on the VM still means real downtime for up to
-that long before anyone is told.
-
-## 6. Media: why it lives outside the checkout
+--require-prod`, still run hourly via the host-level `healthcheck.timer` (§3), which now
+shells into the containers rather than running natively on the host. That command checks
+`settings.DEBUG` and whether `ALLOWED_HOSTS` is anything other than localhost-only, and
+reports a `FAIL` if either looks like dev settings leaked into what's supposed to be prod.
+**Read `--require-prod` correctly: it is a detector, not a guard.** It can tell you, up to
+an hour later, that production is quietly running on dev settings; it cannot stop that
+from happening, and it does not run on every request or every deploy — only once an hour.
+A misconfigured `.env`/`env_file` on the VM still means real downtime for up to that long
+before anyone is told.
+
+### 6. Media: why it lives outside the checkout
`MEDIA_ROOT` (client-uploaded event images) is set in production to
`/home/ubuntu/broadcast/media` — a path that sits next to the git checkout
(`/home/ubuntu/thecommons`), not inside it. `backend/settings/base.py`'s own comment on
`MEDIA_ROOT` states the reason directly: it defaults to a path *inside* the checkout for
-local dev, but production overrides it in `.env` specifically so a `git pull` during deploy
-can never touch uploaded files. A deploy that ran `git clean` or reset the working tree
-inside the checkout would have no way to reach these files at all — they're simply not
-under that directory.
-
-The second half of the same design: nginx serves `/media/` directly as a plain file alias,
-and Django is never in that request path in production. The comment in `base.py` says this
-outright ("Served by nginx in prod, never by Django"), and the pre-Docker `DEPLOY.md`
-revision that documented the live nginx config confirms it as a sibling `location /media/`
-block to the existing `/static/` alias, pointing at the same `MEDIA_ROOT` path. The reason
-is the ordinary one for serving static assets from the ingress instead of the app server:
-nginx does it faster and without spinning up a Python worker to stream a file back to
-disk. There's a real cost worth knowing about, not a bug: uploaded images are kept
-indefinitely — no pruning job exists anywhere in this repo — so `MEDIA_ROOT` grows without
-bound. At roughly 1–3 MB per event this is currently negligible against the VM's block
-volume, but it's a number worth keeping an eye on, not a problem to "fix" by inventing a
-retention policy nobody asked for yet.
-
-## 7. Dev/prod database isolation
+local dev, but production overrides it in `.env` specifically so a `git pull` during
+deploy can never touch uploaded files. That reasoning carried straight into the Docker
+cutover: `docker-compose.yml`'s `backend` and `nginx` services both bind-mount the same
+absolute host path (`${MEDIA_ROOT_HOST:-/home/ubuntu/broadcast/media}`) rather than
+baking it into an image layer — deliberately the identical path prod's `.env` already
+used, so the cutover needed zero changes to that variable.
+
+The second half of the same design: nginx serves `/media/` directly as a read-only bind
+mount, and Django is never in that request path in production — the comment in `base.py`
+says this outright ("Served by nginx in prod, never by Django"). The reason is the
+ordinary one for serving static assets from the ingress instead of the app server: nginx
+does it faster and without spinning up a Python worker to stream a file back to disk.
+There's a real cost worth knowing about, not a bug: uploaded images are kept indefinitely
+— no pruning job exists anywhere in this repo — so `MEDIA_ROOT` grows without bound. At
+roughly 1–3 MB per event this is currently negligible against the VM's block volume, but
+it's a number worth keeping an eye on, not a problem to "fix" by inventing a retention
+policy nobody asked for yet.
+
+### 7. Dev/prod database isolation
Every developer's local `DATABASE_URL` should point at a **Neon branch**, not the
production database — Neon branches are copy-on-write snapshots with their own connection
string, so a branch can be migrated, seeded, and reset freely without ever touching prod
rows. `docs/dev-db-isolation.md` is the full design doc; the shape that matters here is:
-the production VM's `.env` keeps the real `DATABASE_URL` pointed at Neon's main branch, and
+the production VM's `backendServer/.env` (now read by every service via `env_file:` in
+`docker-compose.yml`) keeps the real `DATABASE_URL` pointed at Neon's main branch, and
`DJANGO_ENV` is what decides which settings module (and therefore which behavioral
guardrails) apply — it does not, by itself, decide which database gets used. Nothing in
Django enforces that a `dev`-settings process can't be pointed at the prod `DATABASE_URL`;
@@ -309,7 +348,7 @@ to prod. If you ever set this variable locally, verify the role actually rejects
`INSERT`/`CREATE TABLE` against the `prod_readonly` connection should error with `permission
denied`) before trusting it.
-## 8. Historical incident, still worth knowing
+### 8. Historical incident, still worth knowing
`docs/prod-incident-2026-07-21-scheduler-outage.md` is the full forensic record behind §4's
sharp edge — worth reading in full if you're the one debugging a "the site works but nothing
@@ -321,37 +360,50 @@ stale or never-fired beat schedule is now a hard `FAIL` in `manage.py healthchec
`WARN` — a scheduler that stopped firing is treated as an outage, not a suggestion, because
that distinction is what would have caught this incident in hours instead of the 8 days it
actually took (the outage was only found by chance, during unrelated forensics against
-`/devtools/monitor`, not by any monitoring that existed at the time).
-
-## 9. The pending cutover: containers are built, not live
-
-A parallel effort in this same working tree has built a complete Docker Compose
-replacement for everything in this document — one container per service, described in
-`docker-compose.yml`, `backendServer/Dockerfile`, `Dockerfile.frontend`, and
-`deploy/nginx/`, with the full rationale in `docs/adr/0001-containerization.md`. It has been
-verified locally end-to-end. **None of it is live.** The Oracle VM does not have Docker
-installed, the `ubuntu` user isn't in a `docker` group, the persistent bind-mount
-directories the compose file expects don't exist on the box, and the seven systemd units
-this document describes have not been touched. `DEPLOY.md` was rewritten during this same
-session to describe the containerized stack as the deploy target — read it as a plan for
-the next cutover, not as a description of what answers a request to
-`api.thecommons.town` right now. `containerization.md` (a sibling human doc, written
-alongside this one) covers what changes once that cutover happens — new service names,
-TCP instead of a Unix socket for gunicorn, images instead of a checked-out venv, and how
-the exact sharp edges in §4–§6 above either disappear or get re-solved a different way
-inside a container. Until someone runs that cutover on the actual VM, treat every fact in
-§1–§7 of this document as the operative reality, and treat `DEPLOY.md`'s Docker
-instructions as a runbook waiting for its day one, not a record of today.
-
-## 10. Known gaps
-
-No push notification exists for a failed health check — `systemctl --failed` and
-`journalctl -u healthcheck.service` are the only read paths today; nothing pages anyone.
-There is no automated rollback if a deploy's smoke test fails after the systemd restarts
-already happened — the units are already running the new code by the time the smoke test
-runs, so a failing smoke test currently means "go SSH in and diagnose," not "the previous
-version is automatically restored." The exact live contents of `gunicorn.service` and
-`nextjs.service` are not verifiable from this repository at all, for the reason noted in
-§2 — they were never committed; anyone needing their precise current flags should SSH in
-and read them directly rather than trust any doc's transcription, including this one's
-citation of an old `DEPLOY.md` revision.
+`/devtools/monitor`, not by any monitoring that existed at the time). This incident predates
+the Docker cutover and describes systemd-era behavior, but the healthcheck severity change
+it produced is still in effect today.
+
+### 9. The Docker cutover: done, not pending
+
+**This section previously described the cutover as pending. It shipped.** As of
+2026-08-02 (PR #41), the Docker Compose replacement described here is what runs in
+production. The Oracle VM has Docker Engine + the `docker compose` v2 plugin installed,
+the `ubuntu` deploy user is in the `docker` group (no `sudo` needed in the deploy path),
+the persistent bind-mount directories (`/home/ubuntu/backups`, `/home/ubuntu/broadcast/{media,screenshots,downloads}`)
+exist on the box, and the seven systemd units §3 describes have been stopped and are no
+longer part of the live deploy path (healthcheck excepted).
+
+The stack is described by `docker-compose.yml`, `backendServer/Dockerfile`,
+`Dockerfile.frontend`, and `deploy/nginx/`, with the full rationale in
+`docs/adr/0001-containerization.md`. `DEPLOY.md` is the current, accurate, Docker-first
+deploy runbook — read it as a description of what answers a request to
+`api.thecommons.town` right now, not a plan for someday. `containerization.md` (a sibling
+human doc) covers what changed in the cutover in more detail — new service names, TCP
+instead of a Unix socket for gunicorn, images instead of a checked-out venv, and how the
+sharp edges in §4–§6 above either disappeared or got re-solved a different way inside a
+container. Treat every fact in §1–§8 of this document as reflecting the current
+containerized reality except where a section explicitly marks something as historical
+(§3, §4, §8).
+
+One live-config trap worth restating here because it bit the cutover directly: a bare
+`docker compose` command (no `-f docker-compose.yml`) auto-loads
+`docker-compose.override.yml`, which is local-dev-only (plain HTTP nginx, no cert,
+repo-relative bind mounts, `DJANGO_ENV=dev`) — running that against the VM would silently
+deploy the dev config to prod. Every command in `DEPLOY.md` and in the CI deploy job
+passes `-f docker-compose.yml` explicitly for this reason.
+
+### 10. Known gaps
+
+No push notification exists for a failed health check — `docker compose ps`,
+`journalctl -u healthcheck.service` (still host-level, §3), and `docker logs `
+are the only read paths today; nothing pages anyone. There is no automated rollback if a
+deploy's smoke test fails after `docker compose up -d` already ran — the containers are
+already running the new images by the time the smoke test runs, so a failing smoke test
+currently means "go SSH in and diagnose," not "the previous version is automatically
+restored." The exact historical contents of the retired `gunicorn.service` and
+`nextjs.service` units are not verifiable from this repository at all, for the reason
+noted in §2 — they were never committed even before the cutover; anyone needing their
+precise legacy flags should consult the last pre-Docker `DEPLOY.md` revision
+(`git show 053d65b:DEPLOY.md`) rather than trust any doc's transcription, including this
+one's.
diff --git a/human-docs/design-system.md b/human-docs/design-system.md
index 1496680..67c02a3 100644
--- a/human-docs/design-system.md
+++ b/human-docs/design-system.md
@@ -1,12 +1,30 @@
# Design System
-*Reflects commit `5fe7a45`, 2026-08-01. Every token, class, and component below was read out of `theCommonsWeb/src/app/globals.css` and `theCommonsWeb/src/components/ui/` directly — not inferred from prose. Where `CODING_STYLE.md` (the repo's canonical style statement, which this doc complements and does not replace) disagrees with the stylesheet, both are stated and the disagreement is called out. For the Next.js routing/data-fetching layer these components sit inside, see `frontend.md`; for the product as a whole, see `overview.md`.*
+> **Last updated:** 2026-08-03, commit `d66b059`, branch `main`. Every token, class, and component in the Deep Dive was read out of `theCommonsWeb/src/app/globals.css` and `theCommonsWeb/src/components/ui/` directly — not inferred from prose. Where `CODING_STYLE.md` (the repo's canonical style statement, which this doc complements and does not replace) disagrees with the stylesheet, both are stated and the disagreement is called out. For the Next.js routing/data-fetching layer these components sit inside, see `frontend.md`; for the product as a whole, see `overview.md`.
-## 1. What this is, in one paragraph
+## Overview
+
+- **What it is:** The Commons is styled to look like a broadsheet newspaper's classifieds page crossed with early Craigslist — serif type, cream newsprint background, black ink, hairline/thick column rules doing the separation work that cards and shadows do in most modern products, and a bias toward density over whitespace. This is the whole visual vocabulary, not a retro skin on a normal SaaS layout.
+- **Why it's this way:** The Commons is a local events bulletin for three small towns, not a venture-backed platform. It wants to read like a community notice board a neighbor posted, not a pitch deck — so density, rules, and serifs are load-bearing for trust, not decoration.
+- **Who depends on it:** Every component under `theCommonsWeb/src/components/ui/` and the layout components (`Header.tsx`, `Sidebar.tsx`, `Footer.tsx`, `EventFeed.tsx`, etc.) consume the same small set of CSS custom properties defined once in `globals.css`. There is no Tailwind config file and no `@theme` block — Tailwind v4 is used with its bare default scale plus these tokens.
+- **The one or two facts that matter most:** (1) There are exactly four separation devices — hairline rule, standard rule, thick rule, and one specific hard-edged "print" shadow — and no soft/blurred shadows or `border-radius` anywhere except a handful of sub-20px decorative dots and skeleton blocks. (2) There's only one serif stack (Georgia) used for all headlines and body copy, and one sans-serif stack reserved strictly for button/chrome labels, never for content.
+- **Known live bug:** `--color-ink` is referenced in two components but never defined anywhere in the stylesheet — see Deep Dive §2 and §8.
+- **Where to jump for a given task:**
+ - Adding/changing a color or font → Deep Dive §2 (Tokens)
+ - Sizing text → Deep Dive §3 (Type scale)
+ - Laying out a page or section → Deep Dive §4 (Spacing and layout)
+ - Deciding between a border, a rule, or a shadow → Deep Dive §5
+ - Reusing or extending a `ui/` component → Deep Dive §6
+ - Checking whether something you're about to write is disallowed → Deep Dive §7 (Banned)
+ - Known inconsistencies and open gaps → Deep Dive §8
+
+## Deep Dive
+
+### 1. What this is, in one paragraph
The Commons is styled to look like a broadsheet newspaper's classifieds page had a baby with early Craigslist: serif type, cream newsprint, black ink, hairline and thick column rules doing the job cards and shadows do everywhere else, and a bias toward packing information in rather than giving it room to breathe. This isn't a retro skin bolted onto a normal SaaS layout — it's the whole vocabulary. A contributor who reaches for a rounded card with a soft shadow because that's what every other product looks like is not making a small stylistic choice; they're building the wrong product. The reason it matters: The Commons is a *local* events bulletin for three small towns, not a venture-backed platform, and it wants to read like a community notice board someone would trust a neighbor posted to — not like a pitch deck. Density, rules, and serifs are load-bearing for that trust, not decoration.
-## 2. Tokens
+### 2. Tokens
All color and font values are CSS custom properties declared once, on `:root`, in `globals.css`. **Nothing else defines colors or fonts** — Tailwind v4 is configured with a bare `@import "tailwindcss";` at the top of that same file and no `@theme` block, no `tailwind.config.js`/`.ts` anywhere in the repo. Components consume the tokens either as Tailwind arbitrary values (`bg-[var(--color-bg)]`) or the newer Tailwind v4 shorthand (`border-(--color-border)`) — both forms are in active use side by side; neither is preferred over the other in the current code.
@@ -33,7 +51,7 @@ All color and font values are CSS custom properties declared once, on `:root`, i
**A token that's referenced but doesn't exist:** `TimeWindowSelector.tsx` and `SectionSelector.tsx` both reference `var(--color-ink)` (e.g. `text-[var(--color-ink)]`). There is no `--color-ink` custom property defined anywhere in `globals.css` or any other stylesheet in the repo — it isn't a synonym that resolves elsewhere. An undefined CSS custom property used without a fallback makes the property using it invalid at computed-value time, which for `color` means it falls back to the inherited value rather than doing anything the author intended. In practice this makes the inactive state of the time-window and section dropdowns render in whatever color they'd inherit rather than the near-black ink the rest of the system uses. This is a live bug, not a stylistic choice — the fix is renaming both usages to `--color-text`.
-## 3. Type scale
+### 3. Type scale
There is no declared type scale (no `@theme` font-size tokens, no Tailwind config extending `fontSize`). What exists is the Tailwind v4 default scale (`text-xs` through `text-6xl`) plus a lot of arbitrary pixel values and `clamp()` expressions for display headlines that need to be fluid. Reading across `Header.tsx`, `EventFeed.tsx`, `EventRow.tsx`, `Sidebar.tsx`, `Footer.tsx`, and `MiniCalendar.tsx`, the sizes actually in use settle into these bands:
@@ -50,7 +68,7 @@ There is no declared type scale (no `@theme` font-size tokens, no Tailwind confi
The pattern to copy: headlines are large and fluid via `clamp()` set inline (`style={{ fontSize: '...' }}`), everything else is small, uppercase, and letter-spaced when it's a label rather than content. There is no `text-4xl`/`text-5xl`/`text-6xl` Tailwind class in use anywhere — display sizes are handled by `clamp()`, not the static scale, because they need to shrink on mobile without a breakpoint ladder.
-## 4. Spacing and layout
+### 4. Spacing and layout
No custom spacing scale — Tailwind's default spacing scale (the `p-1`, `px-4`, `gap-6`, etc. system) is used directly, no `@theme` override. Two layout constants recur:
@@ -61,7 +79,7 @@ The sidebar/content split (`PageLayout.tsx`) is a 4-column CSS grid, sidebar tak
Density in practice: `Sidebar.tsx` stacks a dozen-plus distinct blocks (post button, date, calendar, view toggle, social link, tag filters, clear-filters, count, digest box) separated only by `` hairlines with no card wrapper around any of them. That's the intended texture — a long, rule-divided column, not a stack of padded cards.
-## 5. Rules, borders, and the one shadow that's allowed
+### 5. Rules, borders, and the one shadow that's allowed
This is the section that replaces "cards with shadows for elevation." The system has exactly four separation devices, and reaching for anything outside this list should be treated as a smell.
@@ -76,7 +94,7 @@ That fourth row is the one to read carefully, because "no drop shadows" (the ban
**Decision procedure:** reaching for a shadow to create separation between a block and its background? First ask whether a rule (row 1–3) does the job — it almost always does, since most separation here is "this is a distinct row/section," not "this is floating above the page." If the block genuinely needs to read as a card sitting on top of the page (a modal, a featured/pinned item), use the exact hard-shadow value above, never a new blurred one.
-## 6. Component inventory — `src/components/ui/`
+### 6. Component inventory — `src/components/ui/`
Eight components, seven exported from `index.ts` (`Banner.tsx` exists in the directory but is not re-exported — import it directly from `../ui/Banner` if needed, or add it to the barrel if this omission isn't intentional; nothing else in the tree currently imports it that way, so it's unclear whether the omission was deliberate).
@@ -95,7 +113,7 @@ Eight components, seven exported from `index.ts` (`Banner.tsx` exists in the dir
**When to reuse vs. add a new component:** if what you need is a differently-colored button, a differently-sized input, or a badge with a third state, that's a prop on the existing component, not a new file — none of the existing `ui/` components take a `variant` prop that isn't exhaustive of what's actually used elsewhere in the app. Add a new component only when the *shape* is new (nothing here is a modal-with-a-form, a toast, a tooltip, a dropdown-menu-as-a-primitive — `TimeWindowSelector.tsx` and `SectionSelector.tsx` in `layout/` each hand-roll their own dropdown rather than sharing one, which is itself worth noticing as duplication if you're about to build a third one).
-## 7. Banned — enforceable in code review
+### 7. Banned — enforceable in code review
If a diff introduces any of the following, it doesn't match this system regardless of how good it looks in isolation:
@@ -137,7 +155,7 @@ If a diff introduces any of the following, it doesn't match this system regardle
```
-## 8. Known drift and gaps
+### 8. Known drift and gaps
- **`--color-ink` is used but never defined** (§2) — a live bug in `TimeWindowSelector.tsx` and `SectionSelector.tsx`, not a documented token. Fix by renaming both usages to `--color-text`.
- **`Banner.tsx` is not exported from `src/components/ui/index.ts`** — every other `ui/` component is. Unclear whether this is intentional; flagged rather than silently fixed since fixing it is a code change outside this doc's scope.
diff --git a/human-docs/frontend.md b/human-docs/frontend.md
index 93faa2e..98764fa 100644
--- a/human-docs/frontend.md
+++ b/human-docs/frontend.md
@@ -1,14 +1,39 @@
# The Main Site (theCommonsWeb)
-Reflects `theCommonsWeb/` at commit `5fe7a45` on `all-things-ingestion`, dated 2026-08-01. This
-complements `theCommonsWeb/AGENTS.md` (the agent-facing directory map, which stays terse) —
+> **Last updated:** 2026-08-03, commit `d66b059`, branch `main`
+
+## Overview
+
+- `theCommonsWeb` is the public-facing Next.js 16 site (App Router, React 19, Tailwind v4,
+ TanStack Query v5) that renders the event feed, calendar, post-an-event flow, profile/dashboard
+ pages, and the standalone sign-in/sign-up portal. If this app is down, the whole site is down —
+ there's no fallback rendering path.
+- It talks to two backends: the Django REST API (`backendServer/`) for events/profiles/business
+ data, and its own **Better Auth instance running inside this same Next.js process** for
+ identity. The one fact newcomers get backwards most: this app *is* the identity provider —
+ Django only mirrors Better Auth's `neon_auth` tables read-only and verifies the JWTs this app
+ issues.
+- The Django API has no server-rendered pages of its own; it's a pure JSON backend consumed by
+ this app (and separately by `broadcastWeb`, not covered here).
+- Where to go for a given task: home-feed rendering/prefetch → Deep Dive §2.1; how sign-in/sign-up
+ resolves a session and JWT → §2.2; the "fill out `/post` while signed out" flow → §2.3; the
+ route table → §3; hooks/services and query keys → §4; client-side auth gating → §5; visual
+ spec pointer → §6; known traps (npm vs pnpm, type-checking, stale docs, query-key drift) → §7;
+ open gaps the author didn't chase down → §8.
+- Related docs: `theCommonsWeb/AGENTS.md` (terser, agent-facing directory map), `auth.md` (Better
+ Auth internals, JWT/JWKS, cross-subdomain cookie), `design-system.md` (visual spec), `data-model.md`
+ (Django API shapes), `testing.md` (running the test suite).
+
+## Deep Dive
+
+This complements `theCommonsWeb/AGENTS.md` (the agent-facing directory map, which stays terse) —
this doc goes deeper for a human landing in the codebase for the first time. For the identity
bridge itself (Better Auth internals, JWT/JWKS verification, the cross-subdomain cookie), see
`auth.md`; this doc only covers how the React side consumes that bridge. For the visual spec,
see `design-system.md`. For the shapes the Django API returns, see `data-model.md`. For running
the test suite, see `testing.md`.
-## 1. What this is and who depends on it
+### 1. What this is and who depends on it
`theCommonsWeb` is the public-facing Next.js 16 site — the App Router, React 19, Tailwind v4,
TanStack Query v5 stack that renders the newspaper-style event feed, the calendar, the post-an-
@@ -28,9 +53,9 @@ site is down — there is no fallback rendering path. If only Django is down, th
serves its shell and cached data (see §2.1's "Django down" branch), which is a deliberate
design choice in the home page's server component.
-## 2. How it works
+### 2. How it works
-### 2.1 Rendering the home feed — server prefetch, hydration, client refetch
+#### 2.1 Rendering the home feed — server prefetch, hydration, client refetch
The home route (`/`) is unusual among the app's pages: `app/page.tsx` is an async **server**
component that prefetches three queries into a `QueryClient` before any HTML reaches the
@@ -100,7 +125,7 @@ call when `window` is undefined (i.e., on the server) and a memoized singleton i
sharing one instance across concurrent server requests would leak one visitor's prefetched data
into another's response.
-### 2.2 From a session cookie to `useAuth().user`
+#### 2.2 From a session cookie to `useAuth().user`
`useAuth` (`src/hooks/useAuth.tsx`) is the only sanctioned way for a component to know who's
signed in — components must not call `authClient` (the Better Auth client, `src/lib/auth-client.ts`)
@@ -168,7 +193,7 @@ Second, sign-in and sign-up both leave the actual profile fetch to the same code
then `fetchProfileFromDjango`), so a bug in the profile fetch shows up identically regardless of
which portal flow triggered it — there's no separate "new user" profile-loading code.
-### 2.3 Posting an event across the auth wall
+#### 2.3 Posting an event across the auth wall
`/post` is reachable, and its form fully usable, without being signed in — the auth requirement
only bites at submission. This produces a flow that spans two page loads and a full-origin
@@ -197,7 +222,7 @@ using a private window that clears storage on cross-origin navigation) between s
loses the draft silently. There's no warning for this; the user just lands back on an empty
form.
-## 3. Routes
+### 3. Routes
| Path | File | Type | Auth required? | Purpose |
|---|---|---|---|---|
@@ -224,7 +249,7 @@ route handlers, all of which exist and are live in the current tree. Worth fixin
the next time someone's in there; not fixed here per this doc's scope (docs only, and that file
belongs to the agent-facing tree).
-## 4. The data layer: hooks and services
+### 4. The data layer: hooks and services
Every network call to Django goes through `src/services/`, never directly from a component or
hook. Each service function reads `NEXT_PUBLIC_API_BASE_URL` (default `http://127.0.0.1:8000`)
@@ -258,7 +283,7 @@ by string literal; `EditEventModal`'s update mutation invalidates `['my-events']
do match, but a rename in one file silently stops invalidating the other. There is no shared
query-key constants module; string literals are the entire convention.
-## 5. Auth on the client
+### 5. Auth on the client
`useAuth` is the boundary — see §2.2 for how it resolves state, and `auth.md` for the Better
Auth configuration, the JWKS bridge to Django, and the cross-subdomain cookie setup in
@@ -277,7 +302,7 @@ client-rendered content is gated. All of the pages that need this today are clie
with no sensitive server-fetched data, so it hasn't mattered in practice, but it's not a
structural guarantee.
-## 6. Design system
+### 6. Design system
Tailwind v4 with the newsprint palette and typography as CSS custom properties in
`src/app/globals.css`, plus the `.rule-thick`/`.drop-cap`/`.skeleton-block` utilities used
@@ -285,7 +310,7 @@ throughout the pages above. The full enforceable spec — the banned list, the t
component conventions — lives in `design-system.md`; nothing in this doc should be treated as
the aesthetic source of truth.
-## 7. Sharp edges
+### 7. Sharp edges
**`npm install` is not blocked by anything mechanical — only by convention.** There is no
`packageManager` field in `package.json`, no `engines` field, and no `preinstall`/`only-allow`
@@ -372,7 +397,7 @@ and `theCommonsWeb/AGENTS.md` don't exist in the code.** The actual keys are `['
code has since diverged from — and it matters in practice because it's exactly the kind of key
someone would copy-paste from the docs into a new invalidation call and have it silently no-op.
-## 8. Known gaps
+### 8. Known gaps
I did not find a shared constants module for query keys (confirmed by grepping every
`queryKey:`/`invalidateQueries`/`setQueryData` call site in `src/`) — every key is a string
diff --git a/human-docs/ingestion.md b/human-docs/ingestion.md
index c4d5c82..37c7b77 100644
--- a/human-docs/ingestion.md
+++ b/human-docs/ingestion.md
@@ -1,19 +1,47 @@
# The Ingestion Pipeline
-Written 2026-08-01 against commit `5fe7a45`. This is the human-facing companion to
-[`docs/ingestion-pipeline.md`](../docs/ingestion-pipeline.md) (the agent-facing deep dive —
-stays the system of record for exact request/response shapes) and
-[`docs/safety-scoring.md`](../docs/safety-scoring.md) /
-[`docs/ingestion-monitoring.md`](../docs/ingestion-monitoring.md) (the `/devtools/monitor`
-dashboard and the `SourceRun` health model). Where this doc and those disagree, the code
-won this argument — see "Drift from `docs/ingestion-pipeline.md`" at the end.
+> **Last updated:** 2026-08-03, commit `d66b059`, branch `main`. This is the human-facing
+> companion to [`docs/ingestion-pipeline.md`](../docs/ingestion-pipeline.md) (the
+> agent-facing deep dive — stays the system of record for exact request/response shapes)
+> and [`docs/safety-scoring.md`](../docs/safety-scoring.md) /
+> [`docs/ingestion-monitoring.md`](../docs/ingestion-monitoring.md) (the `/devtools/monitor`
+> dashboard and the `SourceRun` health model). Where this doc and those disagree, the code
+> won this argument — see "Drift from `docs/ingestion-pipeline.md`" at the end.
Audience: someone inheriting this codebase who needs to either add a new town's event
calendar as a source, or figure out why a source has gone quiet.
----
-
-## 1. What this is and who depends on it
+## Overview
+
+- **What it is:** The Commons doesn't rely on people submitting events by hand. It polls a
+ list of town/venue/chamber-of-commerce websites on a schedule, pulls whatever raw event
+ data each site exposes, hands each event to Google Gemini to clean it up into a
+ consistent record, screens it for spam/abuse (also via Gemini), and — if it's clean —
+ publishes it as a live `Event` on thecommons.town. A human only gets pulled in for an
+ unrecognized town, a borderline safety score, or a probable duplicate.
+- **Who depends on it:** the public `events` app (everything on the site is either
+ pipeline-published or a direct host submission that went through the same code path),
+ and the `broadcast` subsystem, which pushes already-published `Event` rows out to other
+ towns' calendars (broadcast only ever reads `Event`, never `RawEvent`/`StagedEvent`).
+- **The one fact that matters most:** there are two separate ways an event gets in —
+ the **scheduled bulk pipeline** (nightly poll of every due source) and **direct host
+ submission** (a business submits one event synchronously through the broadcast SPA).
+ They share most of the same machinery (standardize → dedupe → safety-score → publish)
+ but differ in ordering and terminal status — see Deep Dive §2 for both.
+- **If ingestion silently stops**, nothing crashes or errors on the frontend — the site
+ just slowly stops getting new events, which is why the `/devtools/monitor` dashboard
+ (Deep Dive §4, and `docs/ingestion-monitoring.md`) exists.
+- **Where to go for what:**
+ - Adding a new source or classifying a URL as `ics`/`scraper`/`http` → Deep Dive §5.
+ - A source has gone quiet or you're debugging why an event never went live → Deep Dive
+ §6 (Sharp edges) and §7 (Known gaps).
+ - Understanding the data model (`EventSource`, `SourceRun`, `RawEvent`, `StagedEvent`,
+ `Event`) → Deep Dive §3.
+ - Wiring into an endpoint or CLI command → Deep Dive §4 (Interfaces).
+
+## Deep Dive
+
+### 1. What this is and who depends on it
The Commons doesn't ask anyone to submit events by hand (mostly). Instead it polls a list
of town/venue/chamber-of-commerce websites on a schedule, pulls whatever raw event data
@@ -42,9 +70,9 @@ covered below.
---
-## 2. How it works
+### 2. How it works
-### 2.1 The scheduled pipeline: source → live Event
+#### 2.1 The scheduled pipeline: source → live Event
This is the nightly batch flow — one pass touches every due source, then every
unprocessed row created by that pass. It's the same shape whether it's triggered by
@@ -113,7 +141,7 @@ flowchart TD
deduplicator's matching corpus, so a duplicate of an already-published event still has
something to match against. See "Publishing doesn't delete" under Sharp Edges.
-### 2.2 Direct host submission: one event, synchronously
+#### 2.2 Direct host submission: one event, synchronously
A business/venue operator using the broadcast SPA can submit an event straight into the
pipeline (`POST /api/events/direct-submit`), bypassing the poll step entirely. This is a
@@ -180,7 +208,7 @@ flowchart TD
---
-## 3. Data model
+### 3. Data model
| Model | Key fields | What it represents |
|---|---|---|
@@ -194,7 +222,7 @@ For the full cross-app data model (accounts, newsletter, broadcast included) see
[`data-model.md`](data-model.md) once it exists, or `ARCHITECTURE.md`'s "Data Models"
section today.
-### `StagedEvent.status` lifecycle
+#### `StagedEvent.status` lifecycle
```mermaid
stateDiagram-v2
@@ -228,7 +256,7 @@ went through the real publish flow. See Sharp Edges.
---
-## 4. Interfaces
+### 4. Interfaces
| Interface | Auth | Calls |
|---|---|---|
@@ -250,7 +278,7 @@ went through the real publish flow. See Sharp Edges.
---
-## 5. Adding a new source: classification
+### 5. Adding a new source: classification
Every new source has to be classified as one of `ics`, `scraper`, or `http` before any
code gets written. The repo has a slash command for this (`/source-creation`, or the
@@ -338,7 +366,7 @@ separately, by hand or via `/devtools/ingestion-playground`
---
-## 6. Sharp edges
+### 6. Sharp edges
**`events.Event`'s primary key is `uuid`, not `id`.** `Event` declares
`uuid = models.UUIDField(primary_key=True, ...)` and has no `id` column at all — calling
@@ -427,7 +455,7 @@ permanently look unattributed and unverified compared to one that went through
---
-## 7. Known gaps
+### 7. Known gaps
- **`EventSource.source_type` has an `"email"` choice with no importer behind it anywhere
in the codebase.** It's a defined model choice, nothing more — treat it as reserved/dead
@@ -448,7 +476,7 @@ permanently look unattributed and unverified compared to one that went through
of the shipped tool. Treat it as historical design context, not a current behavior
reference, until someone re-verifies it against `devtools/` directly.
-### Drift from `docs/ingestion-pipeline.md`
+#### Drift from `docs/ingestion-pipeline.md`
That doc is the agent-facing deep dive and stays authoritative for request/response
shapes, but as of this pass it has fallen behind the code in a few concrete ways worth
diff --git a/human-docs/newsletter.md b/human-docs/newsletter.md
index 579df10..a5c9065 100644
--- a/human-docs/newsletter.md
+++ b/human-docs/newsletter.md
@@ -1,16 +1,42 @@
# The Newsletter (`newsletter` app)
-*Reflects commit `5fe7a45`, 2026-08-01. Written by reading `backendServer/newsletter/` in
-full (models, views, urls, `email_service.py`, `tasks.py`, `admin.py`, the digest management
-commands, migrations, and the email templates under `newsletter/templates/email/`), the
-generic transport it calls out to (`backendServer/events/email_service.py`), the parts of
-`backendServer/accounts/` that touch subscribers, and the beat/Brevo settings in
-`backendServer/backend/settings/base.py`. Complements `overview.md` (§6, one-paragraph
-summary), `data-model.md` (the field-level `NewsletterSubscriber` table and the Suite 41
-migration history), and `auth.md` (the account-holder side of identity). If anything here
-disagrees with the code, trust the code.*
-
-## 1. What this is and who depends on it
+> **Last updated:** 2026-08-03, commit `d66b059`, branch `main`
+>
+> Originally written by reading `backendServer/newsletter/` in full (models, views, urls,
+> `email_service.py`, `tasks.py`, `admin.py`, the digest management commands, migrations, and
+> the email templates under `newsletter/templates/email/`), the generic transport it calls out
+> to (`backendServer/events/email_service.py`), the parts of `backendServer/accounts/` that
+> touch subscribers, and the beat/Brevo settings in `backendServer/backend/settings/base.py`.
+> Complements `overview.md` (§6, one-paragraph summary), `data-model.md` (the field-level
+> `NewsletterSubscriber` table and the Suite 41 migration history), and `auth.md` (the
+> account-holder side of identity). If anything here disagrees with the code, trust the code.
+
+## Overview
+
+- `newsletter` is one of six Django apps in `backendServer`. It owns one thing: a mailing list
+ (`NewsletterSubscriber`) of email addresses with a `WEEKLY`/`MONTHLY` frequency preference, plus
+ a Celery-driven engine that renders and sends a personalized digest of upcoming `Event` rows.
+ There's no login anywhere in this app — subscribing takes only an email, and managing or
+ cancelling uses an unguessable token in a link, never a password or session.
+- Two kinds of people share the same table: anonymous subscribers who just typed an email in, and
+ account holders (`accounts.UserProfile`) whose digest preference syncs into this same
+ `NewsletterSubscriber` row. There's no separate model for the two.
+- The two facts that matter most: (1) a fresh signup gets `email_preference='NEVER'` and **no**
+ `NewsletterSubscriber` row at all, so brand-new users are silently opted out until they change
+ a setting (see Deep Dive §5); and (2) sends are entirely driven by a Postgres-stored
+ `PeriodicTask` schedule that does **not** auto-follow code moves — moving a task without a data
+ migration breaks digests silently (Deep Dive §5).
+- Who depends on it: Celery beat fires weekly/monthly with no human in the loop, so failures are
+ silent until a subscriber complains; `accounts` writes into this app's table directly; and the
+ public site's subscribe form and profile settings page both call its endpoints.
+- Where to go for a task: adding/removing a subscriber → Deep Dive §2.1; "why didn't recipient X
+ get an email" → Deep Dive §2.2 and §5; how sends actually fire → Deep Dive §2.3; a manual/ops
+ resend or smoke test → Deep Dive §2.4; the field-level schema → Deep Dive §3; the endpoint/task
+ list → Deep Dive §4; known traps → Deep Dive §5; what's unverified → Deep Dive §6.
+
+## Deep Dive
+
+### 1. What this is and who depends on it
`newsletter` is one of six Django apps in `backendServer`. It owns exactly one thing: a mailing
list of email addresses, each with a `WEEKLY`/`MONTHLY` frequency preference, and a Celery-driven
@@ -32,7 +58,7 @@ asks why they stopped getting email (see §5). The `accounts` app depends on it
a row here is how a profile's email preference becomes a real mailing-list entry), and the
public site's subscribe form and profile settings page both call its two endpoints directly.
-## 2. How it works
+### 2. How it works
### 2.1 Getting onto (or off) the list
@@ -193,7 +219,7 @@ tag filtering, and passes no `manage_url` into the template context at all — s
back to the "reply to unsubscribe" text rather than a real manage link. It's a template/delivery
smoke test, not a preview of what any real subscriber receives.
-## 3. Data model
+### 3. Data model
`newsletter` owns exactly one model, `NewsletterSubscriber` — `email` (unique), `frequency`
(`WEEKLY`/`MONTHLY`), `is_active`, `subscribed_at`, and `manage_token` (a UUID, unique, the
@@ -201,7 +227,7 @@ entire authentication mechanism for the login-free manage link). The full field-
its `db_table` history, and the Suite 41 migration mechanics live in `data-model.md` §6 and §8 —
this doc doesn't restate them.
-## 4. Interfaces
+### 4. Interfaces
| Interface | Trigger / caller | Description |
|---|---|---|
@@ -216,7 +242,7 @@ this doc doesn't restate them.
| `python manage.py send_weekly_digest` | Manual/ops | Thin wrapper that calls `fan_out_weekly_digest.delay()`. |
| `python manage.py send_test_digest --email ` | Manual/ops | One-off delivery/template smoke test — see §2.4 for how it differs from a real digest. |
-## 5. Sharp edges
+### 5. Sharp edges
**Moving or renaming a Celery task does not update the beat schedule — the `PeriodicTask` row
has to be repointed by hand, in a data migration, or it fails silently on a weekly cadence.**
@@ -289,7 +315,7 @@ didn't match anything in the window. There is no distinct "no events found" emai
in this case (that copy only exists in the template's `{% else %}` branch, which fires for an
*anonymous* subscriber whose full, untagged event list happens to be empty — a different case).
-## 6. Known gaps
+### 6. Known gaps
I did not verify whether any frontend code (`theCommonsWeb`) reads or displays
`NewsletterSubscriber.subscribed_at`, or whether the `/newsletter/manage` landing page in the
diff --git a/human-docs/overview.md b/human-docs/overview.md
index 9b1da1e..62cb4c8 100644
--- a/human-docs/overview.md
+++ b/human-docs/overview.md
@@ -1,10 +1,34 @@
# Overview
-*Reflects commit `5fe7a45`, 2026-08-01. If anything here contradicts the code, trust the code — this doc was written by reading `backendServer/ingestion/`, `backendServer/events/models.py`, `backendServer/newsletter/`, `backendServer/broadcast/`, `AGENTS.md`, `ARCHITECTURE.md`, and `docs/broadcast.md`, and a few of the drifts found along the way are called out below.*
+> **Last updated:** 2026-08-03, commit `d66b059`, branch `main`. Written by reading `backendServer/ingestion/`, `backendServer/events/models.py`, `backendServer/newsletter/`, `backendServer/broadcast/`, `AGENTS.md`, `ARCHITECTURE.md`, and `docs/broadcast.md`. If anything here contradicts the code, trust the code.
+
+## Overview
+
+The Commons is a local events aggregator for three small North Carolina towns — Chapel Hill, Carrboro, and Pittsboro — built with a deliberate "digital newspaper" look (serif type, ink on cream, no gradients or pill buttons) rather than a typical startup aesthetic.
+
+It does three jobs under the hood:
+
+- **Ingests events automatically** from town/community calendar sources, cleans them up with an LLM (Gemini), screens them for safety, and auto-publishes the ones that pass.
+- **Accepts direct submissions** from residents (public site) and partner hosts (the "broadcast" console), the latter of which can also *push* one event out to other towns' calendars via browser automation.
+- **Runs a newsletter** — a token-based (no-login) mailing list with a weekly/monthly digest.
+
+The two audiences that matter most: **residents** browsing the public site (`theCommonsWeb`), and **event hosts/partners** using the broadcast console (`broadcastWeb`) to distribute one listing across several towns' calendars at once.
+
+The single most important architectural fact: **Better Auth (inside the `theCommonsWeb` Next.js app), not Django, is the identity source of truth** — Django only verifies a JWT, it never issues its own login/session. The second: `ingestion/` and `broadcast/` are two genuinely separate subsystems, walled off from each other by both convention and isolation tests.
+
+Where to jump in the Deep Dive below, by task:
+- Understanding the event lifecycle (scrape → standardize → dedupe → score → publish) → **§3**
+- The broadcast/syndication subsystem → **§5**
+- The newsletter/digest → **§6**
+- Background jobs, Celery/Redis layout → **§7**
+- The overall architecture diagram → **§8**
+- Known doc drift as of this writing → **§10**
+
+## Deep Dive
This is the map. Read it first, then follow the pointers — `docs/` is the agent-facing system of record and stays canonical for line-by-line detail; this doc exists to orient a human who has never opened the repo.
-## 1. What this is
+### 1. What this is
The Commons is a local events aggregator for three small North Carolina towns — Chapel Hill, Carrboro, and Pittsboro. It is not a startup product. The intended feel, enforced throughout the frontend's CSS tokens, is a **digital newspaper**: Georgia serif, ink on cream newsprint, column rules instead of cards and shadows, density over whitespace, no gradients, no pill buttons. That aesthetic choice shows up as a real constraint on the codebase — a new UI component that reaches for a shadow or a rounded badge is fighting the design system, not extending it.
@@ -12,7 +36,7 @@ Under the hood it does three jobs. First, it finds events other people posted el
The two audiences that matter: **residents**, who browse the public site (`theCommonsWeb`) for what's happening nearby, and **event hosts / partner organizations**, who use the broadcast console (`broadcastWeb`) to get one event listing distributed across several towns' calendars at once. A third, much smaller audience is whoever is running the ingestion pipeline day to day — the Django admin and a dev-only monitoring dashboard exist for exactly that.
-## 2. The monorepo, piece by piece
+### 2. The monorepo, piece by piece
```
thecommons/
@@ -36,7 +60,7 @@ thecommons/
**`docs/`** is not for people — it's the agent-facing system of record that Claude Code (and any future coding agent) reads before touching this repo. It stays canonical for exact endpoint lists, adapter registries, and settings values; this human-docs tree exists alongside it so a person doesn't have to read agent-oriented prose to get oriented.
-## 3. How a single event moves through the system
+### 3. How a single event moves through the system
An event's life has three possible starting points and one of four possible endings. The common path — a scraped event that gets auto-published — looks like this:
@@ -73,25 +97,25 @@ flowchart TD
The pipeline runs automatically once a day via Celery beat, and can be triggered manually via a cron-secret-protected endpoint or a management command. It's built from independently-named steps — poll, standardize, dedupe, score, publish — that mirror the diagram above one-to-one; the exact functions and a walkthrough of tuning the safety threshold live in `ingestion.md` and `safety-scoring.md`.
-## 4. Publishing and the public site
+### 4. Publishing and the public site
Once an `Event` row exists, it's just data — read by the public API (`GET /events/...`, cached in Redis and invalidated on writes) and rendered by `theCommonsWeb`'s home feed, calendar view, and event-detail pages. There's no separate "publish" step beyond the `StagedEvent → Event` promotion described above; an `Event` row existing *is* what "published" means, which is also why a published event can't currently be unpublished or soft-deleted — only its owner can hard-delete it. The frontend's data layer, routes, and component conventions are covered in `frontend.md`; the visual system in `design-system.md`.
-## 5. Broadcast — pushing an event the other direction
+### 5. Broadcast — pushing an event the other direction
Ingestion pulls events *in*. Broadcast pushes a single event *out* to other towns' community calendars — Chapel Hill/Carrboro/Pittsboro partners who want one listing to land on several third-party sites without re-typing it five times. It is a genuinely separate subsystem: its own models (`BroadcastSubmission`, `BroadcastTarget`), its own access-tier gating (a Bearer JWT or an access code, resolved to tier 0/1/2), and a hard isolation rule enforced by tests — `broadcast/` never imports from `events/` or `ingestion/`, operating instead on its own denormalized copy of an event's fields.
The primary path today is extension-driven, not fully headless: the operator SPA (`broadcastWeb`) requests a per-site "recipe" from the backend, hands it to the Chrome extension, and the extension opens each target site's form in a new tab and fills every field it can — a human still reviews the prefilled form, solves any captcha, and clicks Submit themselves. A second, fully server-side path exists (a Playwright-driven headless browser that claims queued jobs from a database queue and submits without a human in the loop) but is currently disabled in the SPA in favor of the extension flow. Neither path uses the Django ORM while a headless browser session is open — that's a hard rule, since Playwright and Django's connection pooling don't mix safely. `broadcast.md` is the single source of truth for the adapter list, access-code mechanics, and the worker's queue setup; this paragraph is deliberately the whole story here.
-## 6. The newsletter
+### 6. The newsletter
`newsletter/` is small and mostly decoupled from the rest of the system: an email address plus a frequency preference (`WEEKLY`/`MONTHLY`), no login required to subscribe or to manage that preference — an unguessable token in the manage link is the only credential. A scheduled Celery job resolves the current recipient list once a week and once a month, builds a personalized set of upcoming `Event` rows per recipient (tag-filtered for account holders, everything for anonymous subscribers), and sends one email per recipient through Brevo. `newsletter.md` covers the recipient-resolution logic and the digest templates in more depth.
-## 7. Everything that makes it run in the background
+### 7. Everything that makes it run in the background
Two kinds of async work happen off the request cycle. Most of it — the daily ingestion pipeline, the weekly/monthly digest fan-out — runs on **Celery**, backed by a single Redis instance split into two logical databases (one for the Celery broker and results, a separate one for Django's read-through cache). Broadcast's dispatch also runs on Celery now, but on its own dedicated queue drained by exactly one single-concurrency worker — deliberately, because the orphan-recovery logic assumes only one worker could ever be mid-drain at a time. `async-jobs.md` covers the queue layout, the beat schedule, and the sharp edges around it in full; `deploy-ops.md` covers how each of these processes is kept running in production (they're systemd units, not ad hoc scripts).
-## 8. Architecture at a glance
+### 8. Architecture at a glance
```mermaid
flowchart LR
@@ -145,11 +169,11 @@ flowchart LR
3. **One Postgres database, two owners.** `public` schema tables are Django's (migrated normally); `neon_auth` schema tables are Better Auth's mirrors, read-only from Django's side, never migrated by Django.
4. **The whole thing runs on one virtual machine.** There's no separate services cluster — gunicorn, the Next.js server, Redis, and every worker process share one Oracle Cloud VM behind nginx. That's a deliberate scale-appropriate choice, not a stopgap; `deploy-ops.md` covers what that means operationally.
-## 9. Where to go next
+### 9. Where to go next
Onboarding order, roughly foundational-first: `auth.md` (identity bridge) → `ingestion.md` (the pipeline in §3, in full) → `data-model.md` (every model and how they relate) → the rest as needed — `broadcast.md`, `newsletter.md`, `async-jobs.md`, `deploy-ops.md`, `frontend.md`, `design-system.md`, `testing.md`, `containerization.md`. `docs/` stays the deeper, agent-facing reference underneath all of them — when a human doc and `docs/` seem to disagree, `docs/` and the code win.
-## 10. Doc drift found while writing this
+### 10. Doc drift found while writing this
Three things in the root-level docs are stale relative to the code as of this commit, flagged here rather than silently worked around:
diff --git a/human-docs/testing.md b/human-docs/testing.md
index 05ba3e2..589ba92 100644
--- a/human-docs/testing.md
+++ b/human-docs/testing.md
@@ -1,18 +1,44 @@
# Testing & Local Development
-Written 2026-08-01 against commit `5fe7a45`. This is the human-facing walkthrough that
-complements [`backendServer/AGENTS.md`](../backendServer/AGENTS.md)'s Testing section, which
-stays the agent-facing reference for exact tags, settings names, and management-command flags.
-Where the two disagree, trust `backendServer/AGENTS.md` and the code underneath it — this doc
-exists to be *followed*, not to be the source of truth.
+> **Last updated:** 2026-08-03, commit `d66b059`, branch `main`
+
+This is the human-facing walkthrough that complements
+[`backendServer/AGENTS.md`](../backendServer/AGENTS.md)'s Testing section, which stays the
+agent-facing reference for exact tags, settings names, and management-command flags. Where the
+two disagree, trust `backendServer/AGENTS.md` and the code underneath it — this doc exists to be
+*followed*, not to be the source of truth.
Audience: someone with a fresh clone of this repo, general web-dev skill, and zero context on
The Commons specifically. Every command below was checked against the repo's actual scripts and
`--help` output at this commit, not assumed.
+## Overview
+
+- This doc gets a fresh clone of The Commons running locally and explains how its test suites
+ are organized. It covers three toolchains: `uv` for the Django backend, `pnpm` for two
+ independent frontends (`theCommonsWeb` and `broadcastWeb`), and a local Redis needed to *run*
+ the app (not to test it). It does not cover architecture or any subsystem in depth — see
+ [`overview.md`](overview.md) and [`async-jobs.md`](async-jobs.md) for that.
+- Both backend and frontend tests are split into two tiers by explicit tags, not filenames:
+ `fast` (no database) and `db` (needs Postgres, or `jsdom` on the frontend side). The backend's
+ `db` tier runs against a **real Postgres test database on Neon**, not SQLite and not an
+ ephemeral container — that's a locked decision.
+- The single biggest thing to know before running anything: the Neon test database is shared
+ infrastructure. If your `DATABASE_URL` points at the same Neon branch as any other terminal,
+ session, or agent running `--tag=db` tests at the same time, both runs can race to
+ create/drop/write the same literal database — and the dangerous failure mode is a **silent
+ false-green**, not a crash. Always run `--tag=db` (and full `manage.py test`) suites serially,
+ with `--noinput` and `pipefail`. Full detail in Deep Dive §4.
+- Quick map by task: first-time setup → §2; understanding backend test tiers/tags → §3; hit a
+ "database is being accessed by other users" error → §4 and §5; frontend Vitest tiers → §6; do
+ I need Redis running to test? (no) → §7; full command cheat sheet → §8; what CI actually runs →
+ §9; known doc drift → §10.
+
+## Deep Dive
+
---
-## 1. What this covers, and who it's for
+### 1. What this covers, and who it's for
Getting this repo running locally touches three separate toolchains — `uv` for the Django
backend, `pnpm` for two independent frontends (`theCommonsWeb` and `broadcastWeb`), and a local
@@ -34,9 +60,9 @@ audience).
---
-## 2. First run: clone to a running system
+### 2. First run: clone to a running system
-### 2.1 Backend
+#### 2.1 Backend
```bash
git clone thecommons && cd thecommons/backendServer
@@ -73,7 +99,7 @@ To also process background jobs (digests, ingestion, broadcast) rather than just
run a worker alongside `runserver` — see [`async-jobs.md`](async-jobs.md) for the full queue
topology and which worker command to use for which queue.
-### 2.2 Frontend — theCommonsWeb (the public site)
+#### 2.2 Frontend — theCommonsWeb (the public site)
```bash
cd theCommonsWeb
@@ -92,7 +118,7 @@ Auth and Django share one Postgres database) plus `BETTER_AUTH_SECRET`. Then:
pnpm dev
```
-### 2.3 Frontend — broadcastWeb (optional, partner-facing SPA)
+#### 2.3 Frontend — broadcastWeb (optional, partner-facing SPA)
```bash
cd broadcastWeb
@@ -104,7 +130,7 @@ pnpm dev
Only needed if you're working on the broadcast/syndication side; the public site and its tests
don't depend on it.
-### 2.4 Verify you have a green suite
+#### 2.4 Verify you have a green suite
Once the backend `.env` has a working `DATABASE_URL`, confirm the full picture before doing
anything else, in this order (why this order matters is §4):
@@ -125,7 +151,7 @@ loop. §8 has the full command table, including `broadcastWeb` and lint.
---
-## 3. Backend tests: tiers, tags, and the Postgres test database
+### 3. Backend tests: tiers, tags, and the Postgres test database
Backend tests always run under a dedicated settings module, **never** the same `dev`/`prod`
settings the app itself uses:
@@ -184,7 +210,7 @@ sense of scale, not independently re-measured for this doc (see §11).
---
-## 4. The shared Neon test database — read this before running two things at once
+### 4. The shared Neon test database — read this before running two things at once
This is the sharp edge most likely to waste your afternoon, so it gets its own section instead
of a bullet point.
@@ -250,7 +276,7 @@ human at the keyboard.
---
-## 5. Stale Neon sessions blocking teardown
+### 5. Stale Neon sessions blocking teardown
A related but distinct failure: even with no genuine concurrent run, a Neon test-DB session can
occasionally outlive the process that opened it (a killed test run, a crashed connection pool)
@@ -271,7 +297,7 @@ database between runs (Django still applies any new migrations to it first).
---
-## 6. Frontend tests: Vitest fast/db tiers
+### 6. Frontend tests: Vitest fast/db tiers
Both `theCommonsWeb` and `broadcastWeb` use the same two-project Vitest layout, deliberately
mirroring the backend's `fast`/`db` naming — but **the frontend `db` tier does not touch any
@@ -302,7 +328,7 @@ build` for broadcastWeb) is the type-check gate for both, and is what CI runs.
---
-## 7. Local Redis — required to run the app, not to run its tests
+### 7. Local Redis — required to run the app, not to run its tests
Worth stating plainly, because it's easy to assume otherwise coming from §2's setup:
`backend/settings/test.py` sets `CELERY_TASK_ALWAYS_EAGER = True` and unconditionally swaps
@@ -336,7 +362,7 @@ native process's `REDIS_URL` at.
---
-## 8. Command reference
+### 8. Command reference
| Command | Runs | Needs | Roughly |
|---|---|---|---|
@@ -360,7 +386,7 @@ matching files changed) — no test suite runs in pre-commit, only lint/format/t
---
-## 9. What CI runs
+### 9. What CI runs
`.github/workflows/ci.yml` triggers on every push to `main` and every pull request into `main`; a
newer push to the same branch cancels an in-flight run for it.
@@ -405,7 +431,7 @@ A few specifics worth knowing:
---
-## 10. Known gaps and doc drift
+### 10. Known gaps and doc drift
- **`docs/redis-celery-handoff.md`'s Testing section is stale** on which settings the suite runs
under and which way `CELERY_TASK_ALWAYS_EAGER` is set — covered in full in §7, and also
From 9a38379a4ad43ce6aa8b7e4d41cb6485c3aa3b02 Mon Sep 17 00:00:00 2001
From: Arya Venkatesan
Date: Mon, 3 Aug 2026 18:00:20 -0400
Subject: [PATCH 3/7] feat: suite 47 orchestration
---
backendServer/events/cache.py | 9 +
.../events/tests/test_facets_api_db.py | 107 +++++++++
backendServer/events/urls.py | 1 +
backendServer/events/views.py | 73 ++++++-
backendServer/ingestion/standardizer.py | 10 +-
.../tests/test_direct_submission_db.py | 2 +-
.../templates/docs/pipeline_docs.html | 4 +-
docs/ingestion-pipeline.md | 4 +-
theCommonsWeb/src/app/HomePageClient.tsx | 14 +-
.../src/components/layout/Sidebar.tsx | 206 ++++++++++++++----
.../src/components/layout/TagsBar.tsx | 43 ----
.../components/layout/TimeWindowSelector.tsx | 2 +-
.../src/components/layout/TopBar.tsx | 6 +-
theCommonsWeb/src/constants/tags.ts | 4 +-
theCommonsWeb/src/data/mockEvents.ts | 14 +-
.../src/hooks/__tests__/useEvents.db.test.tsx | 32 ++-
theCommonsWeb/src/hooks/useEvents.ts | 9 +-
theCommonsWeb/src/hooks/useFacets.ts | 13 ++
theCommonsWeb/src/services/eventService.ts | 18 ++
19 files changed, 448 insertions(+), 123 deletions(-)
create mode 100644 backendServer/events/tests/test_facets_api_db.py
delete mode 100644 theCommonsWeb/src/components/layout/TagsBar.tsx
create mode 100644 theCommonsWeb/src/hooks/useFacets.ts
diff --git a/backendServer/events/cache.py b/backendServer/events/cache.py
index bfc409f..d09020c 100644
--- a/backendServer/events/cache.py
+++ b/backendServer/events/cache.py
@@ -36,6 +36,15 @@ def events_list_key(query_params):
return f"events:list:v{_events_list_version()}:{digest}"
+def events_facets_key(query_params):
+ """Deterministic key for a facet-count request. Shares the list version counter
+ with `events_list_key` so `invalidate_events_list()` invalidates both."""
+ items = sorted((k, v) for k in query_params for v in query_params.getlist(k))
+ raw = "&".join(f"{k}={v}" for k, v in items)
+ digest = hashlib.sha256(raw.encode()).hexdigest()[:16]
+ return f"events:facets:v{_events_list_version()}:{digest}"
+
+
def invalidate_events_list():
"""Bump the list version so all cached event-list pages are bypassed."""
try:
diff --git a/backendServer/events/tests/test_facets_api_db.py b/backendServer/events/tests/test_facets_api_db.py
new file mode 100644
index 0000000..a9f7c33
--- /dev/null
+++ b/backendServer/events/tests/test_facets_api_db.py
@@ -0,0 +1,107 @@
+from datetime import timedelta
+
+from django.core.cache import cache
+from django.test import TestCase, tag
+from django.urls import reverse
+from django.utils import timezone
+
+from events.models import Category, Event, Tag
+
+from .factories import make_event, make_town
+
+PAGE_SIZE = 30
+
+
+@tag("db")
+class EventFacetsTests(TestCase):
+ def setUp(self):
+ cache.clear()
+ self.carrboro = make_town("carrboro", "Carrboro")
+ self.chapel_hill = make_town("chapel-hill", "Chapel Hill")
+
+ def test_counts_exceed_a_single_page(self):
+ # More than PAGE_SIZE events in one town: a client-side count over the
+ # paginated list endpoint would cap at 30. The facet endpoint must not.
+ total = PAGE_SIZE + 5
+ for i in range(total):
+ make_event(f"Show {i}", town=self.carrboro, days_offset=1)
+
+ resp = self.client.get(reverse("event-facets"))
+ self.assertEqual(resp.status_code, 200)
+ self.assertEqual(resp.data["towns"]["carrboro"], total)
+
+ def test_response_shape(self):
+ tag_free = Tag.objects.create(name="free")
+ event = make_event("Free Show", town=self.carrboro, days_offset=1)
+ event.tags.add(tag_free)
+
+ resp = self.client.get(reverse("event-facets"))
+ self.assertEqual(resp.status_code, 200)
+ self.assertEqual(set(resp.data.keys()), {"towns", "tags"})
+ self.assertEqual(resp.data["towns"], {"carrboro": 1})
+ self.assertEqual(resp.data["tags"], {"free": 1})
+
+ def test_zero_count_facets_are_omitted(self):
+ # chapel-hill and any unused tag should never appear.
+ make_event("Carrboro Show", town=self.carrboro, days_offset=1)
+
+ resp = self.client.get(reverse("event-facets"))
+ self.assertNotIn("chapel-hill", resp.data["towns"])
+ self.assertEqual(resp.data["tags"], {})
+
+ def test_event_with_multiple_tags_counts_once_per_tag(self):
+ evenings = Tag.objects.create(name="evenings")
+ free = Tag.objects.create(name="free")
+ event = make_event("Multi Tag Show", town=self.carrboro, days_offset=1)
+ event.tags.add(evenings, free)
+
+ resp = self.client.get(reverse("event-facets"))
+ self.assertEqual(resp.data["tags"], {"evenings": 1, "free": 1})
+
+ def test_window_past_narrows_counts(self):
+ make_event("Past Show", town=self.carrboro, days_offset=-3)
+ make_event("Future Show", town=self.chapel_hill, days_offset=3)
+
+ resp = self.client.get(reverse("event-facets"), {"window": "past"})
+ self.assertEqual(resp.status_code, 200)
+ self.assertEqual(resp.data["towns"], {"carrboro": 1})
+
+ def test_category_filter_narrows_counts_consistently_with_list(self):
+ music = Category.objects.create(slug="music", display_name="Music")
+ art = Category.objects.create(slug="art", display_name="Art")
+
+ music_event = make_event("Music Show", town=self.carrboro, days_offset=1)
+ music_event.categories.add(music)
+ art_event = make_event("Art Show", town=self.chapel_hill, days_offset=1)
+ art_event.categories.add(art)
+
+ facets_resp = self.client.get(reverse("event-facets"), {"category": "music"})
+ list_resp = self.client.get(reverse("events"), {"category": "music"})
+
+ self.assertEqual(facets_resp.data["towns"], {"carrboro": 1})
+ self.assertEqual(list_resp.data["count"], 1)
+
+ def test_facets_are_cached_and_invalidated_with_list(self):
+ make_event("First", town=self.carrboro, days_offset=1)
+ first = self.client.get(reverse("event-facets"))
+ self.assertEqual(first.data["towns"]["carrboro"], 1)
+
+ # bulk_create bypasses post_save, so the cache is NOT invalidated.
+ Event.objects.bulk_create(
+ [
+ Event(
+ title="Hidden",
+ town=self.carrboro,
+ date=timezone.now() + timedelta(days=1),
+ venue="V",
+ description="d",
+ ),
+ ]
+ )
+ cached = self.client.get(reverse("event-facets"))
+ self.assertEqual(cached.data["towns"]["carrboro"], 1, "second call should hit the cache")
+
+ # A normal create fires post_save -> invalidates both list and facets caches.
+ make_event("Third", town=self.carrboro, days_offset=1)
+ fresh = self.client.get(reverse("event-facets"))
+ self.assertEqual(fresh.data["towns"]["carrboro"], 3)
diff --git a/backendServer/events/urls.py b/backendServer/events/urls.py
index 3fb17c8..58d7e4b 100644
--- a/backendServer/events/urls.py
+++ b/backendServer/events/urls.py
@@ -6,6 +6,7 @@
path("", views.get_all, name="events"),
path("towns/", views.get_towns, name="towns"),
path("categories/", views.get_categories, name="categories"),
+ path("facets/", views.get_facets, name="event-facets"),
path("me/profile", views.get_my_profile, name="my-profile"),
path("me/events", views.get_my_events, name="my-events"),
path("staged/", views.manage_staged_event, name="manage-staged-event"),
diff --git a/backendServer/events/views.py b/backendServer/events/views.py
index ff44c88..e4159fe 100644
--- a/backendServer/events/views.py
+++ b/backendServer/events/views.py
@@ -1,6 +1,7 @@
from datetime import timedelta
from django.core.cache import cache
+from django.db.models import Count
from django.shortcuts import get_object_or_404
from django.utils import timezone
from django.utils.dateparse import parse_datetime
@@ -47,10 +48,9 @@ def get_categories(request):
return Response(data)
-@api_view(["GET"])
-def get_all(request): # noqa: C901 # query-param filtering; complexity is inherent
+def _filtered_events_queryset(request): # noqa: C901 # query-param filtering; complexity is inherent
"""
- List published events (paginated, page_size=30).
+ Shared window/category/date filtering for the events list and facet-count endpoints.
Query params (applied in priority order — after/before/include_past override window):
after ISO datetime — events on or after this datetime
@@ -61,21 +61,20 @@ def get_all(request): # noqa: C901 # query-param filtering; complexity is inhe
otherwise date >= now (fills page from all future events)
past: date < now
future: date > now + 90 days
- """
- cache_key = events_cache.events_list_key(request.query_params)
- cached = cache.get(cache_key)
- if cached is not None:
- return Response(cached)
+ Note: does NOT apply `.order_by()` — callers that care about ordering (e.g. get_all's
+ "past" window, which reverses to newest-first) must apply it themselves.
+ """
now = timezone.now()
ninety_days_out = now + timedelta(days=90)
- events = Event.objects.all().order_by("date")
+ events = Event.objects.all()
include_past = request.query_params.get("include_past", "").lower() == "true"
after_param = request.query_params.get("after")
before_param = request.query_params.get("before")
window = request.query_params.get("window", "").lower()
+ is_past_window = False # only set by the unqualified window=past branch below
# after/before/include_past are explicit overrides; window applies only when none are set
if after_param or before_param or include_past:
@@ -92,7 +91,8 @@ def get_all(request): # noqa: C901 # query-param filtering; complexity is inhe
events = events.filter(date__lte=before_dt)
else:
if window == "past":
- events = events.filter(date__lt=now).order_by("-date")
+ events = events.filter(date__lt=now)
+ is_past_window = True
elif window == "future":
events = events.filter(date__gt=ninety_days_out)
else: # 'default' or unset — 90-day cap unless fewer than PAGE_SIZE events exist there
@@ -106,6 +106,24 @@ def get_all(request): # noqa: C901 # query-param filtering; complexity is inhe
if category_param:
events = events.filter(categories__slug__in=category_param).distinct()
+ return events, is_past_window
+
+
+@api_view(["GET"])
+def get_all(request):
+ """
+ List published events (paginated, page_size=30).
+
+ See `_filtered_events_queryset` for the supported query params.
+ """
+ cache_key = events_cache.events_list_key(request.query_params)
+ cached = cache.get(cache_key)
+ if cached is not None:
+ return Response(cached)
+
+ events, is_past_window = _filtered_events_queryset(request)
+ events = events.order_by("-date") if is_past_window else events.order_by("date")
+
paginator = EventsPagination()
page = paginator.paginate_queryset(events, request)
serializer = EventSerializer(page, many=True)
@@ -114,6 +132,41 @@ def get_all(request): # noqa: C901 # query-param filtering; complexity is inhe
return Response(data)
+@api_view(["GET"])
+def get_facets(request):
+ """
+ Facet counts (towns, tags) over the full filtered event set — unpaginated.
+
+ Accepts the same window/category/date query params as `get_all`. Used by the
+ frontend sidebar so counts reflect the whole filtered result, not just the
+ current page.
+ """
+ cache_key = events_cache.events_facets_key(request.query_params)
+ cached = cache.get(cache_key)
+ if cached is not None:
+ return Response(cached)
+
+ events, _is_past_window = _filtered_events_queryset(request)
+
+ town_counts = (
+ events.exclude(town__isnull=True)
+ .values("town__slug")
+ .annotate(n=Count("pk", distinct=True))
+ )
+ tag_counts = (
+ events.exclude(tags__isnull=True)
+ .values("tags__name")
+ .annotate(n=Count("pk", distinct=True))
+ )
+
+ data = {
+ "towns": {row["town__slug"]: row["n"] for row in town_counts if row["town__slug"]},
+ "tags": {row["tags__name"]: row["n"] for row in tag_counts if row["tags__name"]},
+ }
+ cache.set(cache_key, data, events_cache.EVENTS_LIST_TTL)
+ return Response(data)
+
+
@api_view(["GET", "DELETE"])
@authentication_classes([BearerTokenAuthentication])
def get_one(request, event_id):
diff --git a/backendServer/ingestion/standardizer.py b/backendServer/ingestion/standardizer.py
index 9813b42..c76203a 100644
--- a/backendServer/ingestion/standardizer.py
+++ b/backendServer/ingestion/standardizer.py
@@ -15,15 +15,13 @@
logger = logging.getLogger(__name__)
VALID_TAGS = [
- "weekends-only",
- "evenings-only",
- "daytime-only",
+ "weekends",
+ "evenings",
+ "daytime",
"free",
"family-friendly",
"nature",
"small-business",
- "lgbtq-friendly",
- "speaks-spanish",
"wheelchair-accessible",
"live-music",
"food-and-drink",
@@ -58,7 +56,7 @@
Rules:
- Only use tags from the provided list. Choose all that apply.
- If the event is free or price is 0, include "free" in tags.
-- If the event time is evening (after 5pm), include "evenings-only". If daytime (before 5pm), include "daytime-only".
+- If the event time is evening (after 5pm), include "evenings". If daytime (before 5pm), include "daytime".
- Keep descriptions factual — don't invent details that aren't in the raw data.
- Respond with ONLY the JSON object. No markdown, no backticks, no explanation.
diff --git a/backendServer/ingestion/tests/test_direct_submission_db.py b/backendServer/ingestion/tests/test_direct_submission_db.py
index 29233cd..c35c2c8 100644
--- a/backendServer/ingestion/tests/test_direct_submission_db.py
+++ b/backendServer/ingestion/tests/test_direct_submission_db.py
@@ -49,7 +49,7 @@
"description": "An evening of live jazz music in Carrboro.",
"location_name": "Cat's Cradle",
"town": "Carrboro",
- "tags": ["live-music", "evenings-only"],
+ "tags": ["live-music", "evenings"],
"price": 0,
}
diff --git a/backendServer/templates/docs/pipeline_docs.html b/backendServer/templates/docs/pipeline_docs.html
index fafd23e..4b64f54 100644
--- a/backendServer/templates/docs/pipeline_docs.html
+++ b/backendServer/templates/docs/pipeline_docs.html
@@ -87,9 +87,9 @@