diff --git a/mailwatch/cleanup.py b/mailwatch/cleanup.py index 47a481a..68e4737 100644 --- a/mailwatch/cleanup.py +++ b/mailwatch/cleanup.py @@ -4,11 +4,13 @@ Purges: * ``scan_events`` older than 60 days — telemetry, not truth data. -* ``serial_counters`` older than 48 hours — only the current day's row - is load-bearing; stale days can be reclaimed. * ``address_cache`` older than 1 year — USPS delivery-point data does drift, so let entries re-standardize annually. +Never purges ``serial_state`` (the global monotonic serial counter must +climb forever to keep IMbs unique) or ``tracked_imbs`` (one tiny row per +letter, feeds long-tail tracking lookups). + Does *not* run ``VACUUM`` (rewrites the DB file and invalidates the Litestream generation) or ``PRAGMA wal_checkpoint`` (races with Litestream's own WAL management — see :func:`mailwatch.db.purge_old`). diff --git a/mailwatch/db.py b/mailwatch/db.py index fa03721..e124aaa 100644 --- a/mailwatch/db.py +++ b/mailwatch/db.py @@ -31,11 +31,18 @@ updated_at INTEGER NOT NULL DEFAULT (unixepoch()) ); -CREATE TABLE IF NOT EXISTS serial_counters ( - day_bucket INTEGER PRIMARY KEY, - counter INTEGER NOT NULL DEFAULT 0, - updated_at INTEGER NOT NULL DEFAULT (unixepoch()) +CREATE TABLE IF NOT EXISTS serial_state ( + id INTEGER PRIMARY KEY CHECK (id = 0), + counter INTEGER NOT NULL +); + +CREATE TABLE IF NOT EXISTS tracked_imbs ( + imb TEXT PRIMARY KEY, + serial INTEGER NOT NULL, + recipient_zip TEXT, + created_at INTEGER NOT NULL DEFAULT (unixepoch()) ); +CREATE INDEX IF NOT EXISTS tracked_imbs_serial_idx ON tracked_imbs(serial); CREATE TABLE IF NOT EXISTS scan_events ( event_id TEXT PRIMARY KEY, @@ -54,6 +61,17 @@ ); """ +# Floor for the global monotonic serial counter. The serial field in an IMb is +# only 6 digits when the Mailer ID is 9 digits (and 9 digits when the MID is 6), +# so an IMb's tracking identity must stay unique for the full IV-MTR retention +# window (~months). Earlier builds reset the counter per UTC day and encoded no +# date into the IMb, so serial 1 on two different days produced *identical* +# barcodes. The fix is a single ever-increasing counter that is never reset or +# purged. The floor starts allocation safely above any low serial issued under +# the old per-day scheme (historical daily volume was a handful), so a fresh +# global counter can never collide with a letter already in the mail stream. +SERIAL_FLOOR: int = 1000 + def connect(path: str | Path) -> sqlite3.Connection: """Open a connection with sensible pragmas. @@ -109,45 +127,89 @@ def set_state(conn: sqlite3.Connection, key: str, value: bytes | str) -> None: # --- serial counters --------------------------------------------------------- -def next_serial(conn: sqlite3.Connection, day_bucket: int) -> int: - """Atomically increment and return the counter for ``day_bucket``. +def next_serial(conn: sqlite3.Connection, floor: int = SERIAL_FLOOR) -> int: + """Allocate the next single global monotonic serial. - Uses a single ``INSERT ... ON CONFLICT DO UPDATE ... RETURNING`` so the - read-modify-write is a single statement under SQLite's write lock — no - TOCTOU between concurrent callers. + Thin wrapper over :func:`next_serials` for the common single-letter case. """ - row = conn.execute( - "INSERT INTO serial_counters (day_bucket, counter) VALUES (?, 1) " - "ON CONFLICT(day_bucket) DO UPDATE SET " - "counter = counter + 1, updated_at = unixepoch() " - "RETURNING counter", - (day_bucket,), - ).fetchone() - counter: int = row[0] - return counter + return next_serials(conn, 1, floor=floor)[0] + +def next_serials(conn: sqlite3.Connection, count: int, floor: int = SERIAL_FLOOR) -> list[int]: + """Atomically allocate ``count`` consecutive global serials. -def next_serials(conn: sqlite3.Connection, day_bucket: int, count: int) -> list[int]: - """Atomically allocate ``count`` consecutive serials from ``day_bucket``. + A single ``serial_state`` row (``id = 0``) holds an ever-increasing + counter that is **never reset and never purged** — this is what makes + every minted IMb globally unique (see :data:`SERIAL_FLOOR`). The counter + is seeded to ``floor`` on first use, so the first serial returned is + ``floor + 1``. - One UPSERT bumps the counter by ``count`` under SQLite's write lock, - then returns the range the caller owns (``[first, ..., last]``). Two - concurrent callers never get overlapping ranges. + Allocation runs inside a ``BEGIN IMMEDIATE`` transaction so the + seed-then-increment is atomic under SQLite's write lock; two concurrent + callers therefore never receive overlapping ranges. """ if count < 1: raise ValueError(f"count must be >= 1, got {count}") - row = conn.execute( - "INSERT INTO serial_counters (day_bucket, counter) VALUES (?, ?) " - "ON CONFLICT(day_bucket) DO UPDATE SET " - "counter = counter + excluded.counter, updated_at = unixepoch() " - "RETURNING counter", - (day_bucket, count), - ).fetchone() + conn.execute("BEGIN IMMEDIATE") + try: + conn.execute( + "INSERT OR IGNORE INTO serial_state (id, counter) VALUES (0, ?)", + (floor,), + ) + row = conn.execute( + "UPDATE serial_state SET counter = counter + ? WHERE id = 0 RETURNING counter", + (count,), + ).fetchone() + conn.execute("COMMIT") + except BaseException: + conn.execute("ROLLBACK") + raise last: int = row[0] first = last - count + 1 return list(range(first, last + 1)) +# --- tracked IMb registry ---------------------------------------------------- + + +def register_imb( + conn: sqlite3.Connection, + imb: str, + serial: int, + recipient_zip: str | None = None, +) -> bool: + """Record a generated IMb so the poller can track it without a prior scan. + + The stored ``imb`` is the *full* queryable barcode (tracking digits + + routing code) — the exact string IV-MTR keys on. This is what lets + :func:`get_imb_by_serial` reconstruct the precise IMb for ``/track-ws`` + (a typed ZIP can't reproduce the encoded 11-digit routing) and lets + :func:`get_pollable_imbs` bootstrap a freshly mailed letter that has no + scan event yet. + + Idempotent via ``INSERT OR IGNORE`` on the IMb primary key. Returns True + when a new row was inserted. + """ + cur = conn.execute( + "INSERT OR IGNORE INTO tracked_imbs (imb, serial, recipient_zip) VALUES (?, ?, ?)", + (imb, serial, recipient_zip), + ) + return cur.rowcount > 0 + + +def get_imb_by_serial(conn: sqlite3.Connection, serial: int) -> str | None: + """Return the full registered IMb for ``serial``, or None if unregistered. + + Serials are globally unique (see :func:`next_serials`), so at most one row + matches; the ordering is a defensive tiebreak only. + """ + row = conn.execute( + "SELECT imb FROM tracked_imbs WHERE serial = ? ORDER BY created_at DESC LIMIT 1", + (serial,), + ).fetchone() + return row[0] if row else None + + # --- scan events ------------------------------------------------------------- @@ -247,6 +309,45 @@ def get_in_flight_imbs(conn: sqlite3.Connection, lookback_days: int = 14) -> lis return result +def get_pollable_imbs( + conn: sqlite3.Connection, + lookback_days: int = 14, + max_age_days: int = 45, +) -> list[str]: + """Return every IMb the poller should query this pass. + + The union of two sources: + + 1. **Scan-driven** — IMbs with a recent non-delivery scan + (:func:`get_in_flight_imbs`). Covers push-fed pieces and anything + seen only via the webhook. + 2. **Registry-driven** — IMbs from :func:`register_imb` minted within + ``max_age_days`` whose latest scan (if any) isn't a delivery code. + This is what lets a *brand-new* letter be polled before its first + scan exists — the scan-driven source alone can never bootstrap one. + + ``max_age_days`` bounds how long we keep polling a registered IMb that + never produces a delivery scan, so the poll set can't grow without + bound. Delivered pieces drop out of both sources. + """ + pollable = set(get_in_flight_imbs(conn, lookback_days)) + cutoff = int(time.time()) - max_age_days * 86400 + rows = conn.execute( + "SELECT imb FROM tracked_imbs WHERE created_at >= ?", + (cutoff,), + ).fetchall() + for (imb,) in rows: + latest = conn.execute( + "SELECT event_json FROM scan_events WHERE imb = ? " + "ORDER BY created_at DESC, event_id DESC LIMIT 1", + (imb,), + ).fetchone() + if latest is not None and _is_delivered_payload(latest[0]): + continue + pollable.add(imb) + return sorted(pollable) + + def _is_delivered_payload(event_json: bytes | str | None) -> bool: """Return True if the decoded scan payload indicates delivery.""" if not event_json: @@ -324,11 +425,15 @@ def cache_put(conn: sqlite3.Connection, input_hash: str, response_json: bytes | def purge_old( conn: sqlite3.Connection, scan_events_ttl_days: int = 60, - serial_counters_ttl_hours: int = 48, address_cache_ttl_days: int = 365, ) -> dict[str, int]: """Delete rows past their TTLs. + Note: ``serial_state`` is deliberately never purged — the global serial + counter must keep climbing forever to guarantee IMb uniqueness (see + :func:`next_serials`). ``tracked_imbs`` is likewise retained; it's tiny + (one row per letter) and feeds long-tail tracking lookups. + Intentionally does not run ``VACUUM`` (that would rewrite the DB and invalidate the Litestream generation) or ``PRAGMA wal_checkpoint`` of any mode. Litestream holds a read lock specifically to prevent @@ -345,16 +450,11 @@ def purge_old( "DELETE FROM scan_events WHERE created_at < unixepoch() - ? * 86400", (scan_events_ttl_days,), ) - serial_cur = conn.execute( - "DELETE FROM serial_counters WHERE updated_at < unixepoch() - ? * 3600", - (serial_counters_ttl_hours,), - ) cache_cur = conn.execute( "DELETE FROM address_cache WHERE cached_at < unixepoch() - ? * 86400", (address_cache_ttl_days,), ) return { "scan_events": scan_cur.rowcount, - "serial_counters": serial_cur.rowcount, "address_cache": cache_cur.rowcount, } diff --git a/mailwatch/models.py b/mailwatch/models.py index becad76..92d4dae 100644 --- a/mailwatch/models.py +++ b/mailwatch/models.py @@ -179,40 +179,58 @@ def full_zip(self) -> str: class TrackingScan(BaseModel): - """One scan event from the IV-MTR tracking endpoint.""" + """One scan event from the IV-MTR tracking endpoint. + + The live IV-MTR pull API returns snake_case keys (``scan_date_time``, + ``scan_event_code``, ``scan_facility_*``) and carries **no** per-scan + ``imb`` — the piece IMb appears once at the ``data`` level. Field aliases + map those wire names onto stable camelCase attribute names, and + ``populate_by_name`` keeps construction-by-field-name (tests, internal + code) and ``model_dump_json()`` round-tripping working. ``model_dump_json`` + emits field names, so stored ``event_json`` blobs use the camelCase keys + that the merge/delivery-gating code reads. + """ - model_config = ConfigDict(extra="ignore") + model_config = ConfigDict(extra="ignore", populate_by_name=True) - imb: str - scanDatetime: datetime - scanEventCode: str - scanFacilityCity: str | None = None - scanFacilityState: str | None = None - scanFacilityZip: str | None = None - machineName: str | None = None + scanDatetime: datetime = Field(alias="scan_date_time") + scanEventCode: str = Field(alias="scan_event_code") + scanFacilityCity: str | None = Field(default=None, alias="scan_facility_city") + scanFacilityState: str | None = Field(default=None, alias="scan_facility_state") + scanFacilityZip: str | None = Field(default=None, alias="scan_facility_zip") + machineName: str | None = Field(default=None, alias="machine_name") + mailPhase: str | None = Field(default=None, alias="mail_phase") + handlingEventType: str | None = Field(default=None, alias="handling_event_type") class TrackingData(BaseModel): - """The ``data`` payload on success.""" + """The ``data`` payload on success. + + Carries the piece-level ``imb`` plus the scan list; the many other + IV-MTR fields (``piece_id``, ``mail_class``, …) are ignored except + ``expected_delivery_date``, surfaced for ETA display. + """ model_config = ConfigDict(extra="ignore") imb: str scans: list[TrackingScan] = Field(default_factory=list) + expected_delivery_date: str | None = None class TrackingResponse(BaseModel): """Wrapper returned by the tracking endpoint. - On success ``data`` is populated; on error ``error`` holds a message. - Both can technically be present (USPS occasionally returns partial - results with a warning), so we don't mark them mutually exclusive. + On success ``data`` is populated and the wire field ``message`` is null. + On a miss IV-MTR returns ``{"message": "Barcode not found.", "data": null}`` + — ``message`` is aliased onto ``error`` so callers can distinguish "not + found / error" from a genuinely empty result. """ - model_config = ConfigDict(extra="ignore") + model_config = ConfigDict(extra="ignore", populate_by_name=True) data: TrackingData | None = None - error: str | None = None + error: str | None = Field(default=None, alias="message") # --------------------------------------------------------------------------- # @@ -226,6 +244,14 @@ class PushFeedEvent(BaseModel): ``handlingEventType`` filtering (we only care about ``"L"`` — letter scans — for mailwatch) happens in the route handler, not here. The model accepts all event types so we can log the raw push and analyse it later. + + UNVERIFIED AGAINST LIVE DATA: these field names predate any real push + delivery. The IV-MTR *pull* API was found to return snake_case keys + (``scan_date_time`` etc.) and ``handling_event_type`` values like ``"A"`` + rather than ``"L"`` — see :class:`TrackingScan`. The push feed likely + shares that convention, so this model (and the ``handlingEventType == "L"`` + filter in ``routes.post_usps_feed``) should be re-checked against the + first real captured webhook payload before relying on the push path. """ model_config = ConfigDict(extra="ignore") diff --git a/mailwatch/poll.py b/mailwatch/poll.py index 807ec96..3d8ccac 100644 --- a/mailwatch/poll.py +++ b/mailwatch/poll.py @@ -64,7 +64,7 @@ async def _poll_once( conn = db.connect(settings.DB_PATH) db.init_db(conn) try: - in_flight = db.get_in_flight_imbs(conn, lookback_days=lookback_days) + in_flight = db.get_pollable_imbs(conn, lookback_days=lookback_days) if not in_flight: return {"polled": 0, "new_events": 0, "errors": 0} diff --git a/mailwatch/routes.py b/mailwatch/routes.py index d156167..9138b17 100644 --- a/mailwatch/routes.py +++ b/mailwatch/routes.py @@ -20,7 +20,6 @@ import sqlite3 from dataclasses import dataclass from pathlib import Path -from time import time as _now from typing import Annotated, Any, Literal from urllib.parse import urlencode @@ -204,10 +203,21 @@ def _build_routing(zip_value: str, delivery_point: str | None) -> str: return digits -def _day_bucket(epoch_seconds: float | None = None) -> int: - """Return the UTC-day integer used as the ``serial_counters`` PK.""" - ts = epoch_seconds if epoch_seconds is not None else _now() - return int(ts // 86400) +async def _register_pieces( + conn: sqlite3.Connection, + lock: asyncio.Lock, + serials: list[int], + trackings: list[str], + routing: str, + recipient_zip: str, +) -> None: + """Persist each piece's full IMb (tracking + routing) to ``tracked_imbs``. + + Lets the poller bootstrap a letter before its first scan and lets + /track-ws recover the exact encoded routing from the serial alone. + """ + for serial, tracking in zip(serials, trackings, strict=True): + await _db_call(lock, db.register_imb, conn, f"{tracking}{routing}", serial, recipient_zip) def _build_tracking(settings: Settings, serial: int) -> str: @@ -421,14 +431,13 @@ async def post_generate( recipient, effective_dp = await _standardize_for_generate(form, new_api) conn, lock = locked - bucket = _day_bucket() # Mint a single serial — the common-case output. Batch size is an # output option on /preview; extra serials are minted on demand by # :func:`_ensure_batch_size` when the user picks count > 1. Minting # the first one here (rather than waiting for /preview) keeps # /preview's "session required" guard meaningful: hitting /preview # without a prior /generate still yields the empty-session 400. - serials: list[int] = await _db_call(lock, db.next_serials, conn, bucket, 1) + serials: list[int] = await _db_call(lock, db.next_serials, conn, 1) trackings: list[str] = [_build_tracking(settings, serials[0])] routing = _build_routing(str(recipient["zip"]), effective_dp) # Validate encoding end-to-end — raises ValueError on any out-of-range field. @@ -440,6 +449,11 @@ async def post_generate( routing, ) + # Register the full queryable IMb (tracking + routing) so the poller can + # track this letter before its first scan and so /track-ws can recover the + # exact 11-digit routing from the serial alone (a typed ZIP can't). + await _register_pieces(conn, lock, serials, trackings, routing, str(recipient["zip"])) + # Session holds the piece-of-mail identity (sender/recipient + the # growing lists of serials/trackings). Output-format preferences # including batch count ride on the URL, not the session. @@ -482,8 +496,7 @@ async def _ensure_batch_size( return piece extra = count - piece.count conn, lock = locked - bucket = _day_bucket() - new_serials: list[int] = await _db_call(lock, db.next_serials, conn, bucket, extra) + new_serials: list[int] = await _db_call(lock, db.next_serials, conn, extra) new_trackings = [_build_tracking(settings, s) for s in new_serials] for serial in new_serials: imb.encode( @@ -493,6 +506,9 @@ async def _ensure_batch_size( serial, piece.routing, ) + await _register_pieces( + conn, lock, new_serials, new_trackings, piece.routing, piece.recipient_zip + ) combined_serials = list(piece.serials) + new_serials combined_trackings = list(piece.trackings) + new_trackings request.session["serials"] = combined_serials @@ -859,9 +875,16 @@ async def track_ws(websocket: WebSocket) -> None: await websocket.send_json({"error": "invalid JSON"}) continue - routing = _clean_zip(parsed.receipt_zip) + # Prefer the exact IMb registered at /generate time — it carries + # the full encoded routing (ZIP+4 + 2-digit delivery point) that + # IV-MTR keys on. The typed ZIP can only rebuild a 5/9-digit + # routing, which would never match a piece USPS has under its + # 11-digit routing. Fall back to the reconstructed key for letters + # generated before the registry existed (or in another session). tracking = _build_tracking(settings, parsed.serial) - imb_key = f"{tracking}{routing}" + fallback_key = f"{tracking}{_clean_zip(parsed.receipt_zip)}" + stored_imb = await _db_call(lock, db.get_imb_by_serial, conn, parsed.serial) + imb_key = stored_imb or fallback_key merged: list[dict[str, Any]] = [] diff --git a/mailwatch/usps_api.py b/mailwatch/usps_api.py index 64a0769..215d4e0 100644 --- a/mailwatch/usps_api.py +++ b/mailwatch/usps_api.py @@ -286,13 +286,22 @@ async def _cached_tokens(self) -> tuple[str | None, str | None, float]: ) async def _store_tokens(self, parsed: IVMTRTokenResponse) -> None: - """Persist a freshly issued IV-MTR token trio.""" + """Persist a freshly issued IV-MTR access token (and refresh, if any). + + USPS omits ``refresh_token`` on a renewal response (only the initial + BSG authenticate returns one). ``app_state.value`` is ``NOT NULL``, so + writing a missing refresh token raised ``sqlite3.IntegrityError`` and + aborted the whole renewal. We therefore only overwrite the stored + refresh token when the response actually carries one — the existing + refresh token stays valid for the next renewal. + """ lifetime = max(1, int(parsed.expires_in)) expiry = time.time() + (lifetime / 2.0) await _run_db(self._db_lock, db.set_state, self._db, "iv_access_token", parsed.access_token) - await _run_db( - self._db_lock, db.set_state, self._db, "iv_refresh_token", parsed.refresh_token - ) + if parsed.refresh_token: + await _run_db( + self._db_lock, db.set_state, self._db, "iv_refresh_token", parsed.refresh_token + ) await _run_db(self._db_lock, db.set_state, self._db, "iv_token_expiry", f"{expiry:.3f}") async def _maybe_rate_limit(self) -> None: diff --git a/tests/test_cleanup.py b/tests/test_cleanup.py index c5799c2..f322ecc 100644 --- a/tests/test_cleanup.py +++ b/tests/test_cleanup.py @@ -31,7 +31,7 @@ def test_cleanup_returns_delete_counts(_settings_env: Path) -> None: """Running ``cleanup.main`` on a fresh DB creates tables and reports zeros.""" assert not _settings_env.exists() deleted = cleanup.main() - assert deleted == {"scan_events": 0, "serial_counters": 0, "address_cache": 0} + assert deleted == {"scan_events": 0, "address_cache": 0} assert _settings_env.exists() diff --git a/tests/test_db.py b/tests/test_db.py index e0ae637..bba7786 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -46,7 +46,13 @@ def test_init_db_is_idempotent(tmp_path: Path) -> None: row[0] for row in c.execute("SELECT name FROM sqlite_master WHERE type='table' ORDER BY name") } - assert {"app_state", "serial_counters", "scan_events", "address_cache"} <= tables + assert { + "app_state", + "serial_state", + "tracked_imbs", + "scan_events", + "address_cache", + } <= tables # --- app_state K/V ---------------------------------------------------------- @@ -72,21 +78,92 @@ def test_state_upsert_overwrites(conn: sqlite3.Connection) -> None: assert db.get_state(conn, "k") == b"second" -# --- serial counter --------------------------------------------------------- +# --- serial counter (global monotonic) -------------------------------------- -def test_next_serial_starts_at_one_and_increments(conn: sqlite3.Connection) -> None: - assert db.next_serial(conn, 7) == 1 - assert db.next_serial(conn, 7) == 2 - assert db.next_serial(conn, 7) == 3 +def test_next_serial_starts_above_floor_and_increments(conn: sqlite3.Connection) -> None: + # First serial is floor+1; the counter is global and never resets. + assert db.next_serial(conn) == db.SERIAL_FLOOR + 1 + assert db.next_serial(conn) == db.SERIAL_FLOOR + 2 + assert db.next_serial(conn) == db.SERIAL_FLOOR + 3 -def test_next_serial_is_per_bucket(conn: sqlite3.Connection) -> None: - assert db.next_serial(conn, 1) == 1 - assert db.next_serial(conn, 2) == 1 - assert db.next_serial(conn, 1) == 2 - assert db.next_serial(conn, 2) == 2 - assert db.next_serial(conn, 3) == 1 +def test_next_serial_is_globally_monotonic_not_per_day(conn: sqlite3.Connection) -> None: + # There is exactly one counter — no per-bucket reset. Two "days" worth of + # allocations keep climbing, so no two pieces ever share a serial (and thus + # an IMb). + first = db.next_serial(conn) + second = db.next_serial(conn) + third = db.next_serial(conn) + assert first < second < third + assert len({first, second, third}) == 3 + + +def test_next_serials_allocates_contiguous_range(conn: sqlite3.Connection) -> None: + batch = db.next_serials(conn, 4) + assert batch == [ + db.SERIAL_FLOOR + 1, + db.SERIAL_FLOOR + 2, + db.SERIAL_FLOOR + 3, + db.SERIAL_FLOOR + 4, + ] + # The next single serial continues past the batch — no overlap. + assert db.next_serial(conn) == db.SERIAL_FLOOR + 5 + + +def test_next_serials_respects_custom_floor(conn: sqlite3.Connection) -> None: + # The floor only seeds the very first allocation; later calls ignore it. + assert db.next_serials(conn, 1, floor=50_000) == [50_001] + assert db.next_serial(conn, floor=50_000) == 50_002 + + +def test_next_serials_rejects_non_positive_count(conn: sqlite3.Connection) -> None: + with pytest.raises(ValueError): + db.next_serials(conn, 0) + + +# --- tracked IMb registry --------------------------------------------------- + + +def test_register_imb_and_lookup_by_serial(conn: sqlite3.Connection) -> None: + full_imb = "0" * 31 + assert db.register_imb(conn, full_imb, 1001, "10009") is True + # Idempotent: re-registering the same IMb is a no-op. + assert db.register_imb(conn, full_imb, 1001, "10009") is False + assert db.get_imb_by_serial(conn, 1001) == full_imb + + +def test_get_imb_by_serial_missing_returns_none(conn: sqlite3.Connection) -> None: + assert db.get_imb_by_serial(conn, 9999) is None + + +def test_pollable_includes_registered_imb_without_scans(conn: sqlite3.Connection) -> None: + # A freshly registered letter with no scan event must still be pollable — + # this is the bootstrap case the scan-only query could never cover. + db.register_imb(conn, "imb-fresh", 1001, "10009") + assert db.get_pollable_imbs(conn) == ["imb-fresh"] + + +def test_pollable_excludes_delivered_registered_imb(conn: sqlite3.Connection) -> None: + db.register_imb(conn, "imb-done", 1001, "10009") + _insert_scan(conn, "e1", "imb-done", b'{"scanEventCode":"01"}', age_seconds=60) + assert db.get_pollable_imbs(conn) == [] + + +def test_pollable_excludes_registered_imb_past_max_age(conn: sqlite3.Connection) -> None: + # Registered 60 days ago, never delivered → outside the 45-day poll window. + conn.execute( + "INSERT INTO tracked_imbs (imb, serial, recipient_zip, created_at) " + "VALUES ('imb-stale', 1001, '10009', unixepoch() - 60 * 86400)" + ) + assert db.get_pollable_imbs(conn) == [] + + +def test_pollable_unions_scan_and_registry_sources(conn: sqlite3.Connection) -> None: + # Scan-only IMb (e.g. push-fed, never registered) + a registry-only IMb. + _insert_scan(conn, "e1", "imb-scan", b'{"scanEventCode":"SD"}', age_seconds=60) + db.register_imb(conn, "imb-reg", 1001, "10009") + assert set(db.get_pollable_imbs(conn)) == {"imb-scan", "imb-reg"} # --- scan events ------------------------------------------------------------ @@ -184,15 +261,6 @@ def test_purge_old_respects_ttls(conn: sqlite3.Connection) -> None: "INSERT INTO scan_events (event_id, imb, event_json, scan_datetime, created_at) " "VALUES ('stale', 'imb1', '{}', NULL, unixepoch() - 70 * 86400)" ) - # serial_counters: fresh + stale - conn.execute( - "INSERT INTO serial_counters (day_bucket, counter, updated_at) " - "VALUES (10, 5, unixepoch())" - ) - conn.execute( - "INSERT INTO serial_counters (day_bucket, counter, updated_at) " - "VALUES (11, 5, unixepoch() - 72 * 3600)" - ) # address_cache: fresh + stale (400 days > 365 default) conn.execute( "INSERT INTO address_cache (input_hash, response_json, cached_at) " @@ -204,21 +272,29 @@ def test_purge_old_respects_ttls(conn: sqlite3.Connection) -> None: ) deleted = db.purge_old(conn) - assert deleted == {"scan_events": 1, "serial_counters": 1, "address_cache": 1} + assert deleted == {"scan_events": 1, "address_cache": 1} # Fresh rows survive assert conn.execute("SELECT COUNT(*) FROM scan_events").fetchone()[0] == 1 - assert conn.execute("SELECT COUNT(*) FROM serial_counters").fetchone()[0] == 1 assert conn.execute("SELECT COUNT(*) FROM address_cache").fetchone()[0] == 1 # Specifically the stale ones went assert conn.execute("SELECT event_id FROM scan_events").fetchone()[0] == "fresh" - assert conn.execute("SELECT day_bucket FROM serial_counters").fetchone()[0] == 10 assert conn.execute("SELECT input_hash FROM address_cache").fetchone()[0] == "fresh" +def test_purge_old_never_touches_serial_state(conn: sqlite3.Connection) -> None: + # The global serial counter must survive cleanup forever — purging it would + # reset serials and risk reusing an IMb already in the mail stream. + db.next_serials(conn, 3) # seed + advance the counter + before = conn.execute("SELECT counter FROM serial_state WHERE id = 0").fetchone()[0] + db.purge_old(conn) + after = conn.execute("SELECT counter FROM serial_state WHERE id = 0").fetchone()[0] + assert after == before + + def test_purge_old_on_empty_tables(conn: sqlite3.Connection) -> None: deleted = db.purge_old(conn) - assert deleted == {"scan_events": 0, "serial_counters": 0, "address_cache": 0} + assert deleted == {"scan_events": 0, "address_cache": 0} # --- in-flight IMbs --------------------------------------------------------- diff --git a/tests/test_models.py b/tests/test_models.py index 14ee77d..05c80ec 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -256,10 +256,47 @@ def test_empty_scans_default(self) -> None: assert td.scans == [] def test_scan_rejects_missing_required(self) -> None: + # scanDatetime + scanEventCode are the only required fields; a payload + # missing the event code must raise (per-scan imb is not required — + # the live IV-MTR pull API doesn't send one). with pytest.raises(ValidationError): - TrackingScan.model_validate( - {"scanDatetime": "2026-04-20T00:00:00Z", "scanEventCode": "SPM"} - ) + TrackingScan.model_validate({"scan_date_time": "2026-04-20T00:00:00Z"}) + + def test_parses_live_snake_case_payload(self) -> None: + # The exact shape the live IV-MTR pull API returns: snake_case keys, + # no per-scan imb, message-not-error wrapper. + resp = TrackingResponse.model_validate( + { + "message": None, + "data": { + "imb": "9" * 31, + "expected_delivery_date": "2026-06-01", + "scans": [ + { + "scan_date_time": "2026-05-30T02:02:04", + "scan_event_code": "919", + "scan_facility_city": "NEW YORK", + "scan_facility_state": "NY", + "scan_facility_zip": "10199", + "handling_event_type": "A", + "machine_name": "DBCS-051", + } + ], + }, + } + ) + assert resp.error is None + assert resp.data is not None + assert resp.data.expected_delivery_date == "2026-06-01" + scan = resp.data.scans[0] + assert scan.scanEventCode == "919" + assert scan.scanFacilityCity == "NEW YORK" + assert scan.scanDatetime == datetime(2026, 5, 30, 2, 2, 4) + + def test_barcode_not_found_message_maps_to_error(self) -> None: + resp = TrackingResponse.model_validate({"message": "Barcode not found.", "data": None}) + assert resp.data is None + assert resp.error == "Barcode not found." # --------------------------------------------------------------------------- # diff --git a/tests/test_poll.py b/tests/test_poll.py index a91b918..fd51d91 100644 --- a/tests/test_poll.py +++ b/tests/test_poll.py @@ -181,6 +181,55 @@ async def test_single_imb_stores_new_scans(settings: Settings) -> None: assert scan_codes == {"SD", "DP"} +async def test_registered_imb_is_polled_before_first_scan(settings: Settings) -> None: + """A registry-only IMb (no scan_event yet) must be polled and ingested. + + This is the bootstrap path: the scan-only query could never discover a + freshly mailed letter, so the poller would never fetch its first scan. + """ + imb = "9" * 31 + conn = db.connect(settings.DB_PATH) + db.init_db(conn) + db.register_imb(conn, imb, 1001, "10009") # registered, zero scans + _prime_iv_token(conn) + conn.close() + + recorder = Recorder() + recorder.add( + "GET", + IV_TRACKING_URL, + lambda _r: httpx.Response( + 200, + json={ + "message": None, + "data": { + "imb": imb, + "scans": [ + { + "scan_date_time": "2026-05-30T02:02:04", + "scan_event_code": "919", + "scan_facility_city": "NEW YORK", + "scan_facility_state": "NY", + "scan_facility_zip": "10199", + } + ], + }, + }, + ), + ) + + transport = httpx.MockTransport(recorder) + async with httpx.AsyncClient(transport=transport) as http: + result = await poll._poll_once(settings, http_client=http) + + assert result == {"polled": 1, "new_events": 1, "errors": 0} + conn = db.connect(settings.DB_PATH) + events = db.get_scan_events(conn, imb) + conn.close() + assert len(events) == 1 + assert events[0]["event"]["scanEventCode"] == "919" + + async def test_idempotent_second_run_stores_nothing(settings: Settings) -> None: """Running twice against the same mocked payload is a no-op on run 2.""" imb = "1" * 20 diff --git a/tests/test_routes.py b/tests/test_routes.py index 0d493ca..b6db4c1 100644 --- a/tests/test_routes.py +++ b/tests/test_routes.py @@ -535,9 +535,9 @@ def test_preview_regen_keeps_same_serial( call_count = {"n": 0} real_next_serials = db.next_serials - def counting_next_serials(conn: Any, bucket: int, count: int) -> list[int]: + def counting_next_serials(conn: Any, count: int) -> list[int]: call_count["n"] += 1 - return real_next_serials(conn, bucket, count) + return real_next_serials(conn, count) monkeypatch.setattr(routes.db, "next_serials", counting_next_serials) @@ -772,6 +772,39 @@ def test_track_ws_merges_stored_events(client: TestClient, fake_tracking: Tracki assert "SD" in events +def test_generate_registers_full_imb(client: TestClient) -> None: + """/generate persists the full queryable IMb (tracking + 11-digit routing).""" + _submit_generate(client) + conn = client.app.state.db + rows = conn.execute("SELECT serial, imb FROM tracked_imbs").fetchall() + assert len(rows) == 1 + serial, registered_imb = rows[0] + # 20-digit tracking + 11-digit routing (ZIP+4 + delivery point) = 31 digits. + assert len(registered_imb) == 31 + assert registered_imb.endswith("20500000599") # 20500-0005 + DP 99 + + +def test_track_ws_prefers_registered_imb_over_typed_zip(client: TestClient) -> None: + """track-ws must query the registered IMb by serial, ignoring a wrong ZIP. + + The whole point of the registry: a typed 5/9-digit ZIP can't reproduce the + encoded 11-digit routing IV-MTR keys on, so the serial-keyed lookup wins. + """ + _submit_generate(client) + conn = client.app.state.db + serial, registered_imb = conn.execute("SELECT serial, imb FROM tracked_imbs").fetchone() + + captured = AsyncMock(return_value=TrackingResponse(data=None)) + client.app.state.ivmtr.get_tracking = captured # type: ignore[method-assign] + + with client.websocket_connect("/track-ws") as ws: + # Deliberately wrong ZIP — the registry lookup must override it. + ws.send_text(json.dumps({"serial": serial, "receipt_zip": "00000"})) + ws.receive_json() + + captured.assert_awaited_once_with(registered_imb) + + # --------------------------------------------------------------------------- # # POST /usps_feed # # --------------------------------------------------------------------------- # diff --git a/tests/test_usps_api.py b/tests/test_usps_api.py index 96e53b0..4f05ce1 100644 --- a/tests/test_usps_api.py +++ b/tests/test_usps_api.py @@ -532,3 +532,109 @@ async def test_iv_get_tracking_parses_response( assert result.data.imb == imb assert len(result.data.scans) == 1 assert result.data.scans[0].scanEventCode == "SD" + + +async def test_iv_refresh_without_refresh_token_keeps_existing( + conn: sqlite3.Connection, + config: Settings, +) -> None: + """A renewal response that omits ``refresh_token`` must not crash or wipe it. + + USPS returns a fresh ``access_token`` but no ``refresh_token`` on renewal. + ``app_state.value`` is NOT NULL, so persisting a missing refresh token used + to raise ``IntegrityError`` and abort renewal. The existing refresh token + must survive untouched. + """ + db.set_state(conn, "iv_access_token", "expired") + db.set_state(conn, "iv_refresh_token", "keep-me") + db.set_state(conn, "iv_token_expiry", "0") + + renewal_no_refresh = { + "access_token": "fresh-access", + "token_type": "Bearer", + "expires_in": 3600, + } + recorder = CallRecorder() + recorder.add("POST", IV_TOKEN_URL, [httpx.Response(200, json=renewal_no_refresh)]) + + async with _make_client(recorder) as http: + client = IVMTRClient(config, conn, http) + token = await client.get_access_token() + + assert token == "fresh-access" + assert db.get_state(conn, "iv_access_token") == b"fresh-access" + # The pre-existing refresh token is preserved, not nulled. + assert db.get_state(conn, "iv_refresh_token") == b"keep-me" + + +async def test_iv_get_tracking_parses_live_snake_case_schema( + conn: sqlite3.Connection, + config: Settings, + iv_token_body: dict[str, object], +) -> None: + """The live IV-MTR pull API returns snake_case keys with no per-scan imb. + + Regression fence for the model alias fix — a camelCase-only model raised 15 + validation errors on this exact shape, silently dropping every real scan. + """ + imb = "9" * 31 + live_body = { + "message": None, + "data": { + "piece_id": "abc-123", + "mail_class": "First Class", + "expected_delivery_date": "2026-06-01", + "imb": imb, + "scans": [ + { + "scan_date_time": "2026-05-30T02:02:04", + "scan_event_code": "919", + "scan_facility_city": "NEW YORK", + "scan_facility_state": "NY", + "scan_facility_zip": "10199", + "mail_phase": "Phase 3c- Destination Sequenced Carrier Sortation", + "handling_event_type": "A", + "machine_name": "DBCS-051", + } + ], + }, + } + recorder = CallRecorder() + recorder.add("POST", IV_AUTH_URL, [httpx.Response(200, json=iv_token_body)]) + recorder.add("GET", IV_TRACKING_URL, [httpx.Response(200, json=live_body)]) + + async with _make_client(recorder) as http: + client = IVMTRClient(config, conn, http) + result = await client.get_tracking(imb) + + assert result.error is None + assert result.data is not None + assert result.data.imb == imb + assert result.data.expected_delivery_date == "2026-06-01" + assert len(result.data.scans) == 1 + scan = result.data.scans[0] + assert scan.scanEventCode == "919" + assert scan.scanFacilityCity == "NEW YORK" + assert scan.scanDatetime.isoformat() == "2026-05-30T02:02:04" + + +async def test_iv_get_tracking_barcode_not_found_maps_to_error( + conn: sqlite3.Connection, + config: Settings, + iv_token_body: dict[str, object], +) -> None: + """``{"message": "Barcode not found.", "data": null}`` → error set, data None.""" + recorder = CallRecorder() + recorder.add("POST", IV_AUTH_URL, [httpx.Response(200, json=iv_token_body)]) + recorder.add( + "GET", + IV_TRACKING_URL, + [httpx.Response(200, json={"message": "Barcode not found.", "data": None})], + ) + + async with _make_client(recorder) as http: + client = IVMTRClient(config, conn, http) + result = await client.get_tracking("9" * 31) + + assert result.data is None + assert result.error == "Barcode not found."