From 3682821a35777fac50002c85ca46742ee5e99f4f Mon Sep 17 00:00:00 2001 From: Tyler Pearson Date: Wed, 1 Jul 2026 22:00:49 -0700 Subject: [PATCH] Fail make fetch when any forest returns zero features --- pipeline/fetch_mvum.py | 19 +++++++++++++++++++ tests/test_fetch_guard.py | 18 ++++++++++++++++++ 2 files changed, 37 insertions(+) create mode 100644 tests/test_fetch_guard.py diff --git a/pipeline/fetch_mvum.py b/pipeline/fetch_mvum.py index 016ebed..41cd91c 100644 --- a/pipeline/fetch_mvum.py +++ b/pipeline/fetch_mvum.py @@ -94,10 +94,24 @@ def fetch_forest(client: httpx.Client, forest: str) -> list[dict]: return feats +def data_quality_failures(per_forest_counts: dict[str, int]) -> list[str]: + """Forests whose fetch returned zero features. + + The EDW service has been observed answering datacenter IPs with HTTP 200 + and EMPTY feature arrays (2026-07-02: all 17 forests, from a GitHub + runner). Every forest in CA_FORESTS has MVUM routes, so an empty result + is always a fetch anomaly, never real data — treat it like a failure so + `make fetch` (and the data-refresh workflow) stops before building + hollow tiles. + """ + return [forest for forest, n in per_forest_counts.items() if n == 0] + + def main() -> int: DATA_DIR.mkdir(exist_ok=True) statewide: list[dict] = [] failures: list[str] = [] + per_forest_counts: dict[str, int] = {} with httpx.Client(headers={"User-Agent": "ca-mvum/0.1 (build pipeline)"}) as client: for forest in CA_FORESTS: @@ -118,14 +132,19 @@ def main() -> int: print(f" ✓ {forest:<38} {len(feats):>6} features " f"({roads} road / {trails} trail) {flag}") statewide.extend(feats) + per_forest_counts[forest] = len(feats) (DATA_DIR / "ca-statewide.geojson").write_text( json.dumps({"type": "FeatureCollection", "features": statewide}) ) print(f"\nStatewide total: {len(statewide)} features across " f"{len(CA_FORESTS) - len(failures)}/{len(CA_FORESTS)} forests.") + empty = data_quality_failures(per_forest_counts) if failures: print(f"Failed forests: {', '.join(failures)}", file=sys.stderr) + if empty: + print(f"Empty forests (fetch anomaly): {', '.join(empty)}", file=sys.stderr) + if failures or empty: return 1 return 0 diff --git a/tests/test_fetch_guard.py b/tests/test_fetch_guard.py new file mode 100644 index 0000000..84d9ee7 --- /dev/null +++ b/tests/test_fetch_guard.py @@ -0,0 +1,18 @@ +from pipeline.fetch_mvum import data_quality_failures +from pipeline.forests import CA_FORESTS + + +def test_all_counts_positive_no_failures(): + counts = {forest: 10 for forest in CA_FORESTS} + assert data_quality_failures(counts) == [] + + +def test_one_forest_empty_is_flagged(): + counts = {forest: 10 for forest in CA_FORESTS} + counts["Inyo National Forest"] = 0 + assert data_quality_failures(counts) == ["Inyo National Forest"] + + +def test_all_forests_empty_are_all_flagged(): + counts = {forest: 0 for forest in CA_FORESTS} + assert data_quality_failures(counts) == list(CA_FORESTS)