Add endpoint probe, rename cost_probe, and add PR/probe interval throttling

llmproxy/server.py

@@ -2110,7 +2110,11 @@ def _build_models_list(providers: dict, config: dict, timeout: int, models_ttl:
 
     if models_ttl > 0:
         with _models_list_cache_lock:
-            _models_list_cache = (full_list, time.monotonic())
+            # When called as a startup warmup (only_if_empty=True), skip the write
+            # if something already populated the cache — a concurrent request or a
+            # cross-test daemon thread from a previous test beat us here.
+            if not only_if_empty or _models_list_cache is None:
+                _models_list_cache = (full_list, time.monotonic())
 
     return full_list
 

tests/test_admin_api.py

@@ -45,6 +45,8 @@ def _make_server(monkeypatch, config_path: Path, config: dict):
     importlib.reload(config_mod)
     from llmproxy import server as server_mod
     importlib.reload(server_mod)
+    monkeypatch.setattr(server_mod, "_run_startup_tasks_once", lambda *a, **k: None)
+    monkeypatch.setattr(server_mod, "_maybe_fire_interval_probes", lambda *a, **k: None)
     server_mod.app.config["TESTING"] = True
     return server_mod
 

tests/test_admin_auth.py

@@ -21,6 +21,8 @@ def _client(monkeypatch, tmp_path, admin_block=None):
     importlib.reload(config_mod)
     from llmproxy import server as server_mod
     importlib.reload(server_mod)
+    server_mod._run_startup_tasks_once = lambda *a, **k: None
+    server_mod._maybe_fire_interval_probes = lambda *a, **k: None
     server_mod.app.config["TESTING"] = True
     return server_mod.app.test_client()
 

tests/test_admin_masking.py

@@ -27,6 +27,8 @@ def client(monkeypatch, tmp_path):
     importlib.reload(config_mod)
     from llmproxy import server as server_mod
     importlib.reload(server_mod)
+    monkeypatch.setattr(server_mod, "_run_startup_tasks_once", lambda *a, **k: None)
+    monkeypatch.setattr(server_mod, "_maybe_fire_interval_probes", lambda *a, **k: None)
     server_mod.app.config["TESTING"] = True
     return server_mod.app.test_client(), cfg_path
 

tests/test_capabilities.py

@@ -16,6 +16,8 @@ def _load_server_with_config(monkeypatch, config_path: Path):
     importlib.reload(config_mod)
     from llmproxy import server as server_mod
     importlib.reload(server_mod)
+    monkeypatch.setattr(server_mod, "_run_startup_tasks_once", lambda *a, **k: None)
+    monkeypatch.setattr(server_mod, "_maybe_fire_interval_probes", lambda *a, **k: None)
     return server_mod
 
 

tests/test_cycling_robustness.py

@@ -22,6 +22,8 @@ def _load_server_with_config(monkeypatch, config_path: Path):
     importlib.reload(config_mod)
     from llmproxy import server as server_mod
     importlib.reload(server_mod)
+    monkeypatch.setattr(server_mod, "_run_startup_tasks_once", lambda *a, **k: None)
+    monkeypatch.setattr(server_mod, "_maybe_fire_interval_probes", lambda *a, **k: None)
     return server_mod
 
 

tests/test_dialects.py

@@ -66,6 +66,8 @@ def server(monkeypatch, tmp_path):
     importlib.reload(config_mod)
     from llmproxy import server as server_mod
     importlib.reload(server_mod)
+    monkeypatch.setattr(server_mod, "_run_startup_tasks_once", lambda *a, **k: None)
+    monkeypatch.setattr(server_mod, "_maybe_fire_interval_probes", lambda *a, **k: None)
     server_mod.app.config["TESTING"] = True
     return server_mod
 

tests/test_favorite_ordering.py

@@ -17,6 +17,8 @@ def _reload_server(monkeypatch, tmp_path):
     importlib.reload(config_mod)
     from llmproxy import server as server_mod
     importlib.reload(server_mod)
+    monkeypatch.setattr(server_mod, "_run_startup_tasks_once", lambda *a, **k: None)
+    monkeypatch.setattr(server_mod, "_maybe_fire_interval_probes", lambda *a, **k: None)
 
 
 def _fn():

tests/test_fusion.py

@@ -99,6 +99,8 @@ def _load_server(monkeypatch, config_path: Path):
     importlib.reload(config_mod)
     from llmproxy import server as server_mod
     importlib.reload(server_mod)
+    monkeypatch.setattr(server_mod, "_run_startup_tasks_once", lambda *a, **k: None)
+    monkeypatch.setattr(server_mod, "_maybe_fire_interval_probes", lambda *a, **k: None)
     return server_mod
 
 

tests/test_input_aware_routing.py

@@ -21,6 +21,8 @@ def _load_server(monkeypatch, config_path: Path):
     importlib.reload(config_mod)
     from llmproxy import server as server_mod
     importlib.reload(server_mod)
+    monkeypatch.setattr(server_mod, "_run_startup_tasks_once", lambda *a, **k: None)
+    monkeypatch.setattr(server_mod, "_maybe_fire_interval_probes", lambda *a, **k: None)
     return server_mod
 
 

tests/test_loadbalanced.py

@@ -23,6 +23,8 @@ def _load_server_with_config(monkeypatch, config_path: Path):
     importlib.reload(config_mod)
     from llmproxy import server as server_mod
     importlib.reload(server_mod)
+    monkeypatch.setattr(server_mod, "_run_startup_tasks_once", lambda *a, **k: None)
+    monkeypatch.setattr(server_mod, "_maybe_fire_interval_probes", lambda *a, **k: None)
     return server_mod
 
 

tests/test_local_provider_not_in_believed_free.py

@@ -34,6 +34,8 @@ def _load_server(monkeypatch, config_path: Path):
     importlib.reload(config_mod)
     from llmproxy import server as server_mod
     importlib.reload(server_mod)
+    monkeypatch.setattr(server_mod, "_run_startup_tasks_once", lambda *a, **k: None)
+    monkeypatch.setattr(server_mod, "_maybe_fire_interval_probes", lambda *a, **k: None)
     return server_mod
 
 

tests/test_per_provider_virtual.py

@@ -20,6 +20,8 @@ def _load_server_with_config(monkeypatch, config_path: Path):
     importlib.reload(config_mod)
     from llmproxy import server as server_mod
     importlib.reload(server_mod)
+    monkeypatch.setattr(server_mod, "_run_startup_tasks_once", lambda *a, **k: None)
+    monkeypatch.setattr(server_mod, "_maybe_fire_interval_probes", lambda *a, **k: None)
     return server_mod
 
 

tests/test_provider_env_runtime.py

@@ -43,6 +43,8 @@ def server_mod(monkeypatch, tmp_path):
     importlib.reload(config_mod)
     from llmproxy import server as server_mod
     importlib.reload(server_mod)
+    monkeypatch.setattr(server_mod, "_run_startup_tasks_once", lambda *a, **k: None)
+    monkeypatch.setattr(server_mod, "_maybe_fire_interval_probes", lambda *a, **k: None)
     return server_mod
 
 

tests/test_proxy_display_id_format.py

@@ -24,6 +24,12 @@ def server(monkeypatch, minimal_config):
     importlib.reload(config_mod)
     from llmproxy import server as server_mod
     importlib.reload(server_mod)
+    # Suppress background daemon threads that can race with cache-sensitive tests:
+    # the startup warmup (step 5 of _run_startup_tasks_once) and the per-request
+    # interval probe check both write to _models_list_cache, and a stale thread
+    # from a prior test can overwrite the cache after the test sets it to None.
+    monkeypatch.setattr(server_mod, "_run_startup_tasks_once", lambda *a, **k: None)
+    monkeypatch.setattr(server_mod, "_maybe_fire_interval_probes", lambda *a, **k: None)
     return server_mod
 
 

tests/test_server_routes.py

@@ -20,6 +20,8 @@ def _load_server_with_config(monkeypatch, config_path: Path):
     importlib.reload(config_mod)
     from llmproxy import server as server_mod
     importlib.reload(server_mod)
+    monkeypatch.setattr(server_mod, "_run_startup_tasks_once", lambda *a, **k: None)
+    monkeypatch.setattr(server_mod, "_maybe_fire_interval_probes", lambda *a, **k: None)
     return server_mod
 
 

tests/test_usage_endpoint.py

@@ -15,6 +15,8 @@ def _load_server_with_config(monkeypatch, config_path: Path):
     importlib.reload(config_mod)
     from llmproxy import server as server_mod
     importlib.reload(server_mod)
+    monkeypatch.setattr(server_mod, "_run_startup_tasks_once", lambda *a, **k: None)
+    monkeypatch.setattr(server_mod, "_maybe_fire_interval_probes", lambda *a, **k: None)
     return server_mod