Backup browser-driven address resolver (free USPS standby)

README.md

@@ -108,6 +108,46 @@ All configuration is read from environment variables (see `.env.example`). Secre
 
 USPS's default tier for new developer accounts is **60 req/hr per application** across all endpoints on `apis.usps.com` (addresses + tracking + labels). Request a tier upgrade to 300 req/hr by emailing `emailus.usps.com` with your CRID, app name, and mailer-use justification. IV-MTR tracking calls (`iv.usps.com`) are on a separate bucket.
 
+## Backup address resolver (when the Addresses API is unavailable)
+
+Address standardization normally goes through the official `apis.usps.com`
+Addresses API (`NewApiClient`). If that API is down — or after USPS's
+2026-07-12 move of the Addresses API to a paid, licensed tier — a standby
+resolver can keep `/generate` producing full ZIP+4 + 11-digit-routing barcodes
+for **$0**.
+
+It drives USPS's free consumer ZIP-lookup (`tools.usps.com`) with a real
+browser. That endpoint sits behind Akamai Bot Manager, which requires a live
+JS-sensor session, so plain HTTP clients are blocked; the resolver uses
+`nodriver` + Chromium, headful under `xvfb`. It writes results straight into
+the `address_cache` table — keyed identically to what `/generate` looks up — so
+**the service and its request path are unchanged**; they simply get a cache hit
+and skip the API.
+
+Linux-only (needs Chromium + Xvfb), in the optional `browser` dependency group:
+
+```sh
+# On the mailwatch host. Pass the SAME recipient fields you'd enter on /generate
+# (the cache key is the hash of that raw input):
+nix run .#resolve-address -- "475 L'Enfant Plaza SW" \
+    --city Washington --state DC --zip 20260
+#   --company / --address2  if the recipient has them
+#   --print-only            resolve without writing the cache
+#   --db <path>             target DB (default: $DB_PATH)
+```
+
+Verify the resolver still beats Akamai (live canary, opt-in):
+
+```sh
+nix run .#test -- -m integration
+```
+
+Caveats: this automates a free public tool, which is a gray area in USPS's
+terms of use — it's a low-volume fallback, not the default path. Keep volume
+low; results are cached for a year. The browser engine is isolated behind
+`mailwatch/web_lookup.py`, so it can be swapped (e.g. for Camoufox) without
+touching callers.
+
 ## License
 
 MIT. See [LICENSE](LICENSE).

flake.nix

@@ -154,6 +154,24 @@
             meta.description = "Run mailwatch's pytest suite with WeasyPrint libs on the path";
           };
         }
+        # `nix run .#resolve-address -- "<street>" --city C --state ST --zip 12345`
+        # Backup USPS address resolver (mailwatch.cli_resolve): drives the
+        # nix-provided Chromium headful under xvfb to seed address_cache when
+        # apis.usps.com is unavailable. Linux-only — needs Chromium + Xvfb.
+        // nixpkgs.lib.optionalAttrs pkgs.stdenv.hostPlatform.isLinux {
+          resolve-address = {
+            type = "app";
+            program = toString (
+              pkgs.writeShellScript "mailwatch-resolve-address" ''
+                ${libEnv}
+                export CHROME_BIN=${pkgs.chromium}/bin/chromium
+                exec ${pkgs.xvfb-run}/bin/xvfb-run -a \
+                  ${devEnv}/bin/python -m mailwatch.cli_resolve "$@"
+              ''
+            );
+            meta.description = "Backup USPS address resolver (Chromium+xvfb) — seeds address_cache";
+          };
+        }
       );
 
       checks = forAllSystems (
@@ -177,7 +195,9 @@
             export HOME=$TMPDIR
             export PYTEST_CACHE_DIR=$TMPDIR/.pytest_cache
             ${libEnv}
-            pytest
+            # Integration tests hit live USPS through a real browser — excluded
+            # from the sandboxed check (no network/display/Chromium here).
+            pytest -m "not integration"
             touch $out
           '';
           lint = pkgs.runCommand "mailwatch-lint" { nativeBuildInputs = [ devEnv ]; } ''

mailwatch/cli_resolve.py

@@ -0,0 +1,120 @@
+"""Standby CLI: resolve a USPS address via the backup browser lookup and seed
+``address_cache`` so mailwatch's ``/generate`` path serves it without the API.
+
+Use when ``apis.usps.com`` is unavailable. Pass the **same recipient fields you
+enter on the generate form** — the cache key is the hash of that raw input
+(``mailwatch.usps_api.NewApiClient.validate_address``), so the fields must match
+for the service to get a hit. The cached value is a standardized response
+identical in shape to the API's, so nothing else in mailwatch changes.
+
+    python -m mailwatch.cli_resolve "475 L'Enfant Plaza SW" \
+        --city Washington --state DC --zip 20260
+
+Runs Chromium headful; on a display-less server wrap it in ``xvfb-run`` (the
+``resolve-address`` flake app does this and points it at the nix Chromium).
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import sys
+from pathlib import Path
+
+from mailwatch import db
+from mailwatch.config import get_settings
+from mailwatch.models import AddressRequest, StandardizedAddressResponse
+from mailwatch.web_lookup import WebLookupError, resolve
+
+_ZIP5_LEN = 5
+
+
+def _clean_zip(raw: str) -> str:
+    """Digits-only, first 5 — mirrors ``routes._clean_zip(...)[:5]`` so the
+    cache key matches what ``/generate`` computes for the same input."""
+    return "".join(ch for ch in raw if ch.isdigit())[:_ZIP5_LEN]
+
+
+def _build_request(args: argparse.Namespace) -> AddressRequest:
+    """Construct the request identically to ``routes._standardize_for_generate``
+    so ``hash_address_dict`` produces the same cache key the service looks up."""
+    return AddressRequest(
+        firm=args.company or None,
+        streetAddress=args.street,
+        secondaryAddress=args.address2 or None,
+        city=args.city,
+        state=args.state.upper(),
+        ZIPCode=_clean_zip(args.zip),
+    )
+
+
+def _format_human(std: StandardizedAddressResponse) -> str:
+    dp = std.additionalInfo.deliveryPoint if std.additionalInfo else None
+    line2 = f"{std.address.city} {std.address.state} {std.full_zip}"
+    return (
+        f"  {std.address.streetAddress}\n"
+        f"  {line2}\n"
+        f"  ZIP+4: {std.full_zip}   delivery point: {dp or '(none)'}"
+    )
+
+
+async def _run(args: argparse.Namespace) -> int:
+    req = _build_request(args)
+    try:
+        candidates = await resolve(req, chrome_path=args.chrome)
+    except WebLookupError as exc:
+        print(f"lookup failed: {exc}", file=sys.stderr)
+        return 2
+
+    if not candidates:
+        print("no USPS match for that address", file=sys.stderr)
+        return 1
+    if len(candidates) > 1:
+        print(f"warning: {len(candidates)} candidates returned; using the first", file=sys.stderr)
+
+    chosen = candidates[0]
+    if args.json:
+        print(chosen.model_dump_json(indent=2))
+    else:
+        print(_format_human(chosen))
+
+    if args.print_only:
+        return 0
+
+    db_path = args.db or get_settings().DB_PATH
+    conn = db.connect(Path(db_path))
+    try:
+        db.init_db(conn)
+        key = db.hash_address_dict(req.model_dump(exclude_none=True))
+        db.cache_put(conn, key, chosen.model_dump_json())
+    finally:
+        conn.close()
+    print(f"seeded address_cache ({key[:12]}…) in {db_path}", file=sys.stderr)
+    return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="mailwatch.cli_resolve",
+        description="Backup USPS address resolver — seeds mailwatch's address_cache.",
+    )
+    parser.add_argument("street", help="Street address line (as entered on /generate)")
+    parser.add_argument("--city", required=True)
+    parser.add_argument("--state", required=True, help="2-letter state code")
+    parser.add_argument("--zip", required=True, help="5-digit ZIP (as entered on /generate)")
+    parser.add_argument("--company", default=None, help="Firm/company line, if any")
+    parser.add_argument("--address2", default=None, help="Secondary line (apt/suite), if any")
+    parser.add_argument("--db", default=None, help="SQLite path (default: settings DB_PATH)")
+    parser.add_argument("--chrome", default=None, help="Chromium binary (default: $CHROME_BIN)")
+    parser.add_argument(
+        "--json", action="store_true", help="Emit the standardized response as JSON"
+    )
+    parser.add_argument(
+        "--print-only", action="store_true", help="Resolve and print without writing the cache"
+    )
+    args = parser.parse_args(argv)
+    return asyncio.run(_run(args))
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

mailwatch/web_lookup.py

@@ -0,0 +1,174 @@
+"""Backup address resolver: USPS's free consumer ZIP-lookup via a real browser.
+
+Standby for when the paid/official Addresses API (``apis.usps.com``) is
+unavailable. Returns the same :class:`StandardizedAddressResponse` shape the
+API does, so a cached entry produced here is indistinguishable to the rest of
+mailwatch (see :mod:`mailwatch.cli_resolve`, which seeds ``address_cache``).
+
+Why a browser at all: the data endpoint
+``tools.usps.com/.../ziplookup/zipByAddress`` sits behind Akamai Bot Manager.
+A live browser running Akamai's JS sensor in-request is mandatory — plain HTTP
+clients (including TLS-fingerprint-impersonating ones carrying a
+browser-minted ``_abck`` cookie) are blocked on the data XHR. We drive Chromium
+via ``nodriver``, headful under a virtual display (``xvfb`` on servers).
+
+``nodriver`` is imported lazily inside the driver so this module — and its pure
+:func:`parse_address_list` core — imports without the optional ``browser``
+dependency group installed. The engine is isolated to :func:`_capture_lookup`;
+swapping it (e.g. for Camoufox) touches nothing else.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import json
+import logging
+import os
+from typing import Any
+
+from mailwatch.models import (
+    AdditionalInfo,
+    AddressInfo,
+    AddressRequest,
+    StandardizedAddressResponse,
+)
+
+logger = logging.getLogger(__name__)
+
+LOOKUP_URL = "https://tools.usps.com/zip-code-lookup.htm?byaddress"
+_DELIVERY_POINT_LEN = 2
+
+
+class WebLookupError(RuntimeError):
+    """The USPS web lookup failed, was blocked by Akamai, or found no match."""
+
+
+def _norm_dp(value: Any) -> str | None:
+    """Keep a delivery point only if it's the expected 2-digit form."""
+    if isinstance(value, str) and len(value) == _DELIVERY_POINT_LEN and value.isdigit():
+        return value
+    return None
+
+
+def parse_address_list(payload: dict[str, Any]) -> list[StandardizedAddressResponse]:
+    """Parse a ``zipByAddress`` JSON body into standardized-response models.
+
+    Pure function — the unit-testable core. Maps USPS's consumer wire fields
+    (``addressLine1``/``zip5``/``zip4``/``deliveryPoint``/...) onto the same
+    models :class:`~mailwatch.usps_api.NewApiClient` produces. Returns ``[]``
+    when the lookup reports anything other than success or carries no usable
+    entries (callers treat empty as "no match").
+    """
+    if payload.get("resultStatus") != "SUCCESS":
+        return []
+
+    results: list[StandardizedAddressResponse] = []
+    for entry in payload.get("addressList") or []:
+        zip5 = entry.get("zip5")
+        street = entry.get("addressLine1")
+        if not zip5 or not street:
+            continue
+        firm = entry.get("firm") or None
+        address = AddressInfo(
+            firm=firm,
+            streetAddress=street,
+            secondaryAddress=entry.get("addressLine2") or None,
+            city=entry.get("city", ""),
+            state=entry.get("state", ""),
+            ZIPCode=zip5,
+            ZIPPlus4=entry.get("zip4") or None,
+        )
+        additional = AdditionalInfo(
+            deliveryPoint=_norm_dp(entry.get("deliveryPoint")),
+            carrierRoute=entry.get("carrierRoute") or None,
+            DPVConfirmation=entry.get("dpvConfirmation") or None,
+        )
+        results.append(
+            StandardizedAddressResponse(firm=firm, address=address, additionalInfo=additional)
+        )
+    return results
+
+
+async def _capture_lookup(  # pragma: no cover - browser I/O, exercised by integration test
+    req: AddressRequest,
+    *,
+    chrome_path: str | None,
+    settle_s: float,
+) -> dict[str, Any]:
+    """Drive Chromium through the lookup form and return the captured JSON.
+
+    Raises :class:`WebLookupError` if Akamai serves the block page or no
+    ``zipByAddress`` response is observed.
+    """
+    # Lazy: nodriver lives in the optional `browser` group, so importing this
+    # module (and its pure parser) must not require it.
+    import nodriver as uc
+
+    captured: list[dict[str, Any]] = []
+    browser = await uc.start(
+        headless=False,
+        sandbox=False,
+        browser_executable_path=chrome_path,
+        browser_args=["--window-size=1280,900", "--disable-gpu"],
+    )
+    try:
+        tab = await browser.get(LOOKUP_URL)
+
+        async def on_response(event: Any) -> None:
+            if "ziplookup" not in event.response.url.lower():
+                return
+            with contextlib.suppress(Exception):
+                body, _ = await tab.send(uc.cdp.network.get_response_body(event.request_id))
+                captured.append(json.loads(body))
+
+        tab.add_handler(uc.cdp.network.ResponseReceived, on_response)
+        await tab.send(uc.cdp.network.enable())
+        await tab.sleep(3)
+
+        el = await tab.select("#tAddress")
+        await el.send_keys(req.streetAddress)
+        el = await tab.select("#tCity")
+        await el.send_keys(req.city)
+        await tab.evaluate(
+            "(() => { const s = document.querySelector('#tState');"
+            f" if (s) {{ s.value = {req.state!r};"
+            " s.dispatchEvent(new Event('change', {bubbles: true})); } })()"
+        )
+        button = await tab.select("#zip-by-address")
+        await button.click()
+        await tab.sleep(settle_s)
+
+        body_text = str(await tab.evaluate("document.body.innerText") or "")
+    finally:
+        browser.stop()
+
+    if captured:
+        return captured[-1]
+    if "unavailable" in body_text.lower():
+        raise WebLookupError("USPS returned the Akamai bot-block page")
+    raise WebLookupError("no zipByAddress response was captured")
+
+
+async def resolve(  # pragma: no cover - thin orchestration over browser I/O
+    req: AddressRequest,
+    *,
+    chrome_path: str | None = None,
+    settle_s: float = 7.0,
+    retries: int = 1,
+) -> list[StandardizedAddressResponse]:
+    """Resolve ``req`` against USPS's free web lookup; retry once on a block.
+
+    ``chrome_path`` defaults to ``$CHROME_BIN`` (set by the ``resolve-address``
+    flake app to the nix-provided Chromium). Returns the standardized
+    candidates (usually one for a deliverable address).
+    """
+    chrome = chrome_path or os.environ.get("CHROME_BIN")
+    last_exc: WebLookupError | None = None
+    for attempt in range(retries + 1):
+        try:
+            payload = await _capture_lookup(req, chrome_path=chrome, settle_s=settle_s)
+            return parse_address_list(payload)
+        except WebLookupError as exc:
+            last_exc = exc
+            logger.warning("USPS web lookup attempt %d failed: %s", attempt + 1, exc)
+    raise last_exc if last_exc else WebLookupError("lookup failed")