From 1096a5a630226da5a642803975ee166fb3f64ac9 Mon Sep 17 00:00:00 2001
From: ak2k <19240940+ak2k@users.noreply.github.com>
Date: Fri, 5 Jun 2026 18:34:20 -0400
Subject: [PATCH] feat: list recently generated letters on the tracking page
The /tracking page previously required typing a serial + ZIP from memory.
It now lists the most recently generated letters as clickable references,
each prefilling the form (and auto-connecting) on click.
- db.recent_tracked_imbs(): newest-first registry rows enriched with a
coarse status (delivered / in transit / awaiting first scan) from the
latest stored scan. Rows without a recipient_zip are omitted.
- Capture recipient name + company into tracked_imbs so the list can show
who each letter is for. Added via an idempotent additive migration
(ALTER TABLE ADD COLUMN guarded by PRAGMA table_info), so existing
on-disk DBs pick up the columns without a destructive rebuild; legacy
rows read NULL and render as "Unnamed recipient". The fields are
display-only and play no part in routing or IV-MTR keying.
- Gated by SHOW_RECENT_TRACKING (default on) + RECENT_TRACKING_LIMIT.
The site has no auth beyond the webhook IP gate, so the flag is a kill
switch for deployments where /tracking is broadly reachable.
Tests cover ordering, status derivation, the limit, null-zip omission,
name/company round-trip, the additive migration (incl. re-run no-op),
and the route's flag on/off + rendered name/company.
Also refresh .secrets.baseline to cover existing test-fixture dummies.
---
.env.example | 7 ++
.secrets.baseline | 80 ++++++++++++++++++++-
mailwatch/config.py | 10 +++
mailwatch/db.py | 111 ++++++++++++++++++++++++++++--
mailwatch/routes.py | 82 ++++++++++++++++++++--
mailwatch/static/style.css | 24 +++++++
mailwatch/templates/tracking.html | 48 +++++++++++++
tests/test_db.py | 107 ++++++++++++++++++++++++++++
tests/test_routes.py | 40 +++++++++++
9 files changed, 495 insertions(+), 14 deletions(-)
diff --git a/.env.example b/.env.example
index e0c2442..78db738 100644
--- a/.env.example
+++ b/.env.example
@@ -27,6 +27,13 @@ DB_PATH=./mailwatch.db # SQLite file path; use /var/lib/mailwatch/mail
# Rate limiting (USPS apis.usps.com default = 60 req/hr; request tier upgrade to 300)
RATE_LIMIT_PER_HOUR=50 # Client-side cap, leaves headroom under the USPS limit
+# Tracking page UX.
+# When true, /tracking lists recently generated letters (serial + recipient ZIP
+# + status) as clickable references. The list exposes recipient ZIPs/serials to
+# anyone who can reach the page — set false where /tracking is broadly reachable.
+SHOW_RECENT_TRACKING=true
+RECENT_TRACKING_LIMIT=20 # How many recent letters to surface (1–100)
+
# Webhook IP allowlist for /usps_feed (IV-MTR push feed source ranges).
# Comma-separated CIDRs. Verify current list at provisioning time.
USPS_FEED_CIDRS=56.0.0.0/8
diff --git a/.secrets.baseline b/.secrets.baseline
index 5e8f9d0..51c9bc4 100644
--- a/.secrets.baseline
+++ b/.secrets.baseline
@@ -90,6 +90,10 @@
{
"path": "detect_secrets.filters.allowlist.is_line_allowlisted"
},
+ {
+ "path": "detect_secrets.filters.common.is_baseline_file",
+ "filename": ".secrets.baseline"
+ },
{
"path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies",
"min_level": 2
@@ -122,6 +126,78 @@
"path": "detect_secrets.filters.heuristic.is_templated_secret"
}
],
- "results": {},
- "generated_at": "2026-04-21T18:31:33Z"
+ "results": {
+ "tests/test_app.py": [
+ {
+ "type": "Secret Keyword",
+ "filename": "tests/test_app.py",
+ "hashed_secret": "9fa974b13be42cbc6379d9841c52277c4399e3a4",
+ "is_verified": false,
+ "line_number": 25
+ },
+ {
+ "type": "Secret Keyword",
+ "filename": "tests/test_app.py",
+ "hashed_secret": "a0281cd072cea8e80e7866b05dc124815760b6c9",
+ "is_verified": false,
+ "line_number": 27
+ }
+ ],
+ "tests/test_config.py": [
+ {
+ "type": "Secret Keyword",
+ "filename": "tests/test_config.py",
+ "hashed_secret": "d4539be1ab3f5a453c9c0dc107f2d989b3d22168",
+ "is_verified": false,
+ "line_number": 25
+ },
+ {
+ "type": "Secret Keyword",
+ "filename": "tests/test_config.py",
+ "hashed_secret": "76816e3dd24434df5ee83ae03730fd126e9a65da",
+ "is_verified": false,
+ "line_number": 27
+ },
+ {
+ "type": "Secret Keyword",
+ "filename": "tests/test_config.py",
+ "hashed_secret": "f09ac38e6897794f539a8aa7e096732765faa337",
+ "is_verified": false,
+ "line_number": 240
+ }
+ ],
+ "tests/test_routes.py": [
+ {
+ "type": "Secret Keyword",
+ "filename": "tests/test_routes.py",
+ "hashed_secret": "9fa974b13be42cbc6379d9841c52277c4399e3a4",
+ "is_verified": false,
+ "line_number": 32
+ },
+ {
+ "type": "Secret Keyword",
+ "filename": "tests/test_routes.py",
+ "hashed_secret": "a0281cd072cea8e80e7866b05dc124815760b6c9",
+ "is_verified": false,
+ "line_number": 34
+ }
+ ],
+ "tests/test_usps_api.py": [
+ {
+ "type": "Secret Keyword",
+ "filename": "tests/test_usps_api.py",
+ "hashed_secret": "9fa974b13be42cbc6379d9841c52277c4399e3a4",
+ "is_verified": false,
+ "line_number": 54
+ },
+ {
+ "type": "Secret Keyword",
+ "filename": "tests/test_usps_api.py",
+ "hashed_secret": "a0281cd072cea8e80e7866b05dc124815760b6c9",
+ "is_verified": false,
+ "line_number": 56
+ }
+ ]
+ },
+ "generated_at": "2026-06-05T22:34:03Z"
}
diff --git a/mailwatch/config.py b/mailwatch/config.py
index c9c15c2..e8e3bbb 100644
--- a/mailwatch/config.py
+++ b/mailwatch/config.py
@@ -63,6 +63,16 @@ class Settings(BaseSettings):
# IV-MTR pull-poll daemon (read by `python -m mailwatch.poll`)
POLL_LOOKBACK_DAYS: int = Field(14, ge=1, le=365)
+ # Tracking page UX. When True, ``/tracking`` lists the most recently
+ # generated letters (serial + recipient ZIP + status) as clickable
+ # references so an operator need not retype the serial/ZIP from memory.
+ # The list exposes recipient ZIPs and serials to anyone who can reach
+ # the page (the site has no auth beyond the webhook IP gate), so set this
+ # to False on any deployment where /tracking is broadly reachable.
+ SHOW_RECENT_TRACKING: bool = Field(True)
+ # How many recent letters to surface in that list.
+ RECENT_TRACKING_LIMIT: int = Field(20, ge=1, le=100)
+
# IV-MTR push feed source IP allowlist.
# ``NoDecode`` disables pydantic-settings' default JSON-decoding for list
# fields so the string form from env (``"a,b,c"``) reaches the
diff --git a/mailwatch/db.py b/mailwatch/db.py
index e124aaa..305a954 100644
--- a/mailwatch/db.py
+++ b/mailwatch/db.py
@@ -37,10 +37,12 @@
);
CREATE TABLE IF NOT EXISTS tracked_imbs (
- imb TEXT PRIMARY KEY,
- serial INTEGER NOT NULL,
- recipient_zip TEXT,
- created_at INTEGER NOT NULL DEFAULT (unixepoch())
+ imb TEXT PRIMARY KEY,
+ serial INTEGER NOT NULL,
+ recipient_zip TEXT,
+ recipient_name TEXT,
+ recipient_company TEXT,
+ created_at INTEGER NOT NULL DEFAULT (unixepoch())
);
CREATE INDEX IF NOT EXISTS tracked_imbs_serial_idx ON tracked_imbs(serial);
@@ -96,8 +98,43 @@ def connect(path: str | Path) -> sqlite3.Connection:
def init_db(conn: sqlite3.Connection) -> None:
- """Create all tables and indexes if they don't exist. Idempotent."""
+ """Create all tables and indexes if they don't exist, then migrate.
+
+ Idempotent: safe to call on every startup against a fresh or existing DB.
+ """
conn.executescript(SCHEMA)
+ _migrate(conn)
+
+
+def _add_columns_if_missing(conn: sqlite3.Connection, table: str, columns: dict[str, str]) -> None:
+ """``ALTER TABLE ADD COLUMN`` for each column not already present.
+
+ ``columns`` maps column name to its SQL type declaration. Existing columns
+ are detected via ``PRAGMA table_info`` and skipped, so re-running is a
+ no-op. ``table``/column names come from internal callers only (never user
+ input) — ``PRAGMA`` and ``ALTER`` can't be parameterized, so they're
+ interpolated directly.
+ """
+ existing = {row[1] for row in conn.execute(f"PRAGMA table_info({table})")}
+ for name, decl in columns.items():
+ if name not in existing:
+ conn.execute(f"ALTER TABLE {table} ADD COLUMN {name} {decl}")
+
+
+def _migrate(conn: sqlite3.Connection) -> None:
+ """Apply idempotent additive migrations to an existing database.
+
+ ``CREATE TABLE IF NOT EXISTS`` only creates *missing* tables; it never adds
+ columns to a table that predates a schema change. New nullable columns are
+ therefore added here, so an older on-disk DB picks them up on next startup
+ without a destructive rebuild. Keep migrations additive and nullable —
+ Litestream replicates the resulting WAL frames like any other write.
+ """
+ _add_columns_if_missing(
+ conn,
+ "tracked_imbs",
+ {"recipient_name": "TEXT", "recipient_company": "TEXT"},
+ )
# --- app_state K/V -----------------------------------------------------------
@@ -177,6 +214,8 @@ def register_imb(
imb: str,
serial: int,
recipient_zip: str | None = None,
+ recipient_name: str | None = None,
+ recipient_company: str | None = None,
) -> bool:
"""Record a generated IMb so the poller can track it without a prior scan.
@@ -187,16 +226,74 @@ def register_imb(
:func:`get_pollable_imbs` bootstrap a freshly mailed letter that has no
scan event yet.
+ ``recipient_name`` / ``recipient_company`` are display-only metadata for
+ the tracking-page list (:func:`recent_tracked_imbs`); they play no part in
+ routing or IV-MTR keying.
+
Idempotent via ``INSERT OR IGNORE`` on the IMb primary key. Returns True
when a new row was inserted.
"""
cur = conn.execute(
- "INSERT OR IGNORE INTO tracked_imbs (imb, serial, recipient_zip) VALUES (?, ?, ?)",
- (imb, serial, recipient_zip),
+ "INSERT OR IGNORE INTO tracked_imbs "
+ "(imb, serial, recipient_zip, recipient_name, recipient_company) "
+ "VALUES (?, ?, ?, ?, ?)",
+ (imb, serial, recipient_zip, recipient_name, recipient_company),
)
return cur.rowcount > 0
+def recent_tracked_imbs(conn: sqlite3.Connection, limit: int = 20) -> list[dict[str, Any]]:
+ """Return the most recently registered letters for the tracking-page list.
+
+ Newest first (by ``created_at``, tiebreaking on ``serial`` so the order is
+ deterministic when several pieces in a batch share a timestamp). Each row
+ is enriched with a coarse ``status`` derived from its latest stored scan:
+
+ - ``"delivered"`` — latest scan carries a delivery code
+ (:data:`DELIVERED_SCAN_CODES`).
+ - ``"in_transit"`` — at least one scan exists but it isn't a delivery code.
+ - ``"awaiting"`` — no scan event has been recorded yet.
+
+ The status reflects only locally stored scans (what the poller/webhook have
+ persisted), not a live IV-MTR pull; the live pull happens when the operator
+ actually opens the letter via ``/track-ws``. Rows without a
+ ``recipient_zip`` are omitted — they can't form a usable tracking link.
+ ``recipient_name`` / ``recipient_company`` may be ``None`` for letters
+ registered before that metadata was captured.
+ """
+ rows = conn.execute(
+ "SELECT imb, serial, recipient_zip, recipient_name, recipient_company, created_at "
+ "FROM tracked_imbs "
+ "WHERE recipient_zip IS NOT NULL "
+ "ORDER BY created_at DESC, serial DESC LIMIT ?",
+ (limit,),
+ ).fetchall()
+ result: list[dict[str, Any]] = []
+ for imb, serial, recipient_zip, recipient_name, recipient_company, created_at in rows:
+ latest = conn.execute(
+ "SELECT event_json FROM scan_events WHERE imb = ? "
+ "ORDER BY created_at DESC, event_id DESC LIMIT 1",
+ (imb,),
+ ).fetchone()
+ if latest is None:
+ status = "awaiting"
+ elif _is_delivered_payload(latest[0]):
+ status = "delivered"
+ else:
+ status = "in_transit"
+ result.append(
+ {
+ "serial": serial,
+ "recipient_zip": recipient_zip,
+ "recipient_name": recipient_name,
+ "recipient_company": recipient_company,
+ "created_at": created_at,
+ "status": status,
+ }
+ )
+ return result
+
+
def get_imb_by_serial(conn: sqlite3.Connection, serial: int) -> str | None:
"""Return the full registered IMb for ``serial``, or None if unregistered.
diff --git a/mailwatch/routes.py b/mailwatch/routes.py
index 9138b17..731c05b 100644
--- a/mailwatch/routes.py
+++ b/mailwatch/routes.py
@@ -19,6 +19,7 @@
import logging
import sqlite3
from dataclasses import dataclass
+from datetime import UTC, datetime
from pathlib import Path
from typing import Annotated, Any, Literal
from urllib.parse import urlencode
@@ -210,14 +211,27 @@ async def _register_pieces(
trackings: list[str],
routing: str,
recipient_zip: str,
+ recipient_name: str | None = None,
+ recipient_company: str | None = None,
) -> None:
"""Persist each piece's full IMb (tracking + routing) to ``tracked_imbs``.
Lets the poller bootstrap a letter before its first scan and lets
/track-ws recover the exact encoded routing from the serial alone.
+ ``recipient_name`` / ``recipient_company`` are display-only metadata
+ surfaced on the tracking page's recent-letters list.
"""
for serial, tracking in zip(serials, trackings, strict=True):
- await _db_call(lock, db.register_imb, conn, f"{tracking}{routing}", serial, recipient_zip)
+ await _db_call(
+ lock,
+ db.register_imb,
+ conn,
+ f"{tracking}{routing}",
+ serial,
+ recipient_zip,
+ recipient_name,
+ recipient_company,
+ )
def _build_tracking(settings: Settings, serial: int) -> str:
@@ -281,6 +295,16 @@ def count(self) -> int:
def recipient_zip(self) -> str:
return str(self.recipient["zip"])
+ @property
+ def recipient_name(self) -> str | None:
+ name = self.recipient.get("name")
+ return str(name) if name else None
+
+ @property
+ def recipient_company(self) -> str | None:
+ company = self.recipient.get("company")
+ return str(company) if company else None
+
def sender_lines(self) -> list[str]:
"""Split the sender textarea into non-empty display lines."""
return [line.strip() for line in self.sender_address.splitlines() if line.strip()]
@@ -452,7 +476,16 @@ async def post_generate(
# Register the full queryable IMb (tracking + routing) so the poller can
# track this letter before its first scan and so /track-ws can recover the
# exact 11-digit routing from the serial alone (a typed ZIP can't).
- await _register_pieces(conn, lock, serials, trackings, routing, str(recipient["zip"]))
+ await _register_pieces(
+ conn,
+ lock,
+ serials,
+ trackings,
+ routing,
+ str(recipient["zip"]),
+ recipient_name=recipient.get("name"),
+ recipient_company=recipient.get("company"),
+ )
# Session holds the piece-of-mail identity (sender/recipient + the
# growing lists of serials/trackings). Output-format preferences
@@ -507,7 +540,14 @@ async def _ensure_batch_size(
piece.routing,
)
await _register_pieces(
- conn, lock, new_serials, new_trackings, piece.routing, piece.recipient_zip
+ conn,
+ lock,
+ new_serials,
+ new_trackings,
+ piece.routing,
+ piece.recipient_zip,
+ recipient_name=piece.recipient_name,
+ recipient_company=piece.recipient_company,
)
combined_serials = list(piece.serials) + new_serials
combined_trackings = list(piece.trackings) + new_trackings
@@ -828,17 +868,49 @@ async def post_validate_address(
)
+def _epoch_to_iso(epoch: int) -> str:
+ """Render a unix epoch (seconds, UTC) as an ISO-8601 ``Z`` string.
+
+ Used for the ``datetime`` attribute on recent-tracking rows; the page's
+ JavaScript localizes it for display, matching the scan-event formatting.
+ """
+ return datetime.fromtimestamp(epoch, tz=UTC).isoformat().replace("+00:00", "Z")
+
+
@router.get("/tracking", response_class=HTMLResponse)
async def get_tracking(
request: Request,
serial: str | None = None,
zip: str | None = None,
+ settings: Settings = Depends(get_settings_dep),
+ db_locked: tuple[sqlite3.Connection, asyncio.Lock] = Depends(get_db_locked),
) -> Response:
- """Render the tracking-form page with optional prefilled fields."""
+ """Render the tracking-form page with optional prefilled fields.
+
+ When ``SHOW_RECENT_TRACKING`` is enabled, the page also lists the most
+ recently generated letters (serial + recipient ZIP + coarse status) as
+ clickable references, so the operator can pick one instead of retyping a
+ serial and ZIP from memory.
+ """
+ recent: list[dict[str, Any]] = []
+ if settings.SHOW_RECENT_TRACKING:
+ conn, lock = db_locked
+ rows = await _db_call(lock, db.recent_tracked_imbs, conn, settings.RECENT_TRACKING_LIMIT)
+ recent = [
+ {
+ "serial": row["serial"],
+ "recipient_zip": row["recipient_zip"],
+ "recipient_name": row["recipient_name"],
+ "recipient_company": row["recipient_company"],
+ "status": row["status"],
+ "created_iso": _epoch_to_iso(row["created_at"]),
+ }
+ for row in rows
+ ]
return templates.TemplateResponse(
request,
"tracking.html",
- {"serial": serial, "recipient_zip": zip},
+ {"serial": serial, "recipient_zip": zip, "recent": recent},
)
diff --git a/mailwatch/static/style.css b/mailwatch/static/style.css
index cba57c6..cf94a49 100644
--- a/mailwatch/static/style.css
+++ b/mailwatch/static/style.css
@@ -151,6 +151,30 @@ button.primary:hover, .button-link.primary:hover {
.status-closed { color: var(--muted); }
.status-error { border-color: var(--error); color: var(--error); }
+.recent-list { list-style: none; padding: 0; margin: 0; }
+.recent-item { margin-bottom: 0.5rem; }
+.recent-link {
+ display: flex; flex-wrap: wrap; align-items: baseline; gap: 0.2rem 0.9rem;
+ padding: 0.6rem 0.75rem; border: 1px solid var(--border);
+ border-radius: var(--radius); background: var(--surface);
+ text-decoration: none; color: var(--fg);
+}
+.recent-link:hover { border-color: var(--accent); color: var(--fg); }
+.recent-who { display: flex; flex-wrap: wrap; align-items: baseline; gap: 0.2rem 0.5rem; }
+.recent-name { font-weight: 600; }
+.recent-company { color: var(--muted); font-size: 0.9rem; }
+.recent-meta { display: flex; flex-wrap: wrap; align-items: baseline; gap: 0.2rem 0.6rem; }
+.recent-serial { color: var(--muted); font-variant-numeric: tabular-nums; }
+.recent-zip { color: var(--muted); }
+.recent-time { color: var(--muted); font-variant-numeric: tabular-nums; font-size: 0.9rem; }
+.recent-status {
+ margin-left: auto; font-size: 0.85rem; font-weight: 500;
+ padding: 0.05rem 0.5rem; border: 1px solid var(--border); border-radius: 999px;
+}
+.recent-status-delivered { color: var(--ok); border-color: var(--ok); }
+.recent-status-in_transit { color: var(--accent); border-color: var(--accent); }
+.recent-status-awaiting { color: var(--muted); }
+
.scans { list-style: none; padding: 0; margin: 0; }
.scan {
padding: 0.6rem 0.75rem; border: 1px solid var(--border);
diff --git a/mailwatch/templates/tracking.html b/mailwatch/templates/tracking.html
index 2b25920..95e81da 100644
--- a/mailwatch/templates/tracking.html
+++ b/mailwatch/templates/tracking.html
@@ -31,6 +31,40 @@ Track a letter
+{% if recent %}
+
+ Recently generated
+
+ Pick a letter to load its serial and ZIP into the form above and start tracking.
+
+
+
+{% endif %}
+
Status
Idle. Submit the form above to connect.
@@ -44,6 +78,20 @@ Scan events
{% endblock %}
{% block scripts %}
+