From 7ae350588c017c5a82cc13218185575d70658417 Mon Sep 17 00:00:00 2001 From: Robert Lippmann Date: Thu, 12 Mar 2026 14:46:53 -0400 Subject: [PATCH 1/2] Polish demo runner regression wording and LLM error handling --- demos/README.md | 1 + demos/llm_client.py | 185 ++++++++++++++++++++++++++++++- demos/run_demo.py | 14 ++- tests/test_llm_client.py | 227 +++++++++++++++++++++++++++++++++++++++ tests/test_run_demo.py | 38 ++++++- 5 files changed, 451 insertions(+), 14 deletions(-) create mode 100644 tests/test_llm_client.py diff --git a/demos/README.md b/demos/README.md index 24e799f..1b52d92 100644 --- a/demos/README.md +++ b/demos/README.md @@ -5,6 +5,7 @@ Context Compiler decision/state API. They are illustrative manual demos, not benchmarks or CI tests. They demonstrate common LLM failure modes and how authoritative compiled state can improve reliability. +All demos force deterministic decoding so results are reproducible. ## Requirements diff --git a/demos/llm_client.py b/demos/llm_client.py index 2a1419b..0d1d9ab 100644 --- a/demos/llm_client.py +++ b/demos/llm_client.py @@ -1,7 +1,12 @@ """Small OpenAI-compatible chat client for demo scripts.""" import os +import re +import sys +import time from dataclasses import dataclass +from datetime import UTC, datetime +from email.utils import parsedate_to_datetime from importlib import import_module from typing import Any, Literal, TypedDict @@ -28,6 +33,109 @@ def __str__(self) -> str: return f"Missing demo configuration: {missing_text}" +class DemoLLMError(RuntimeError): + """Friendly provider/client error for demos.""" + + +_RETRY_DELAYS_SECONDS = (1, 2, 4) +MAX_DEMO_RETRY_AFTER_SECONDS = 5 + + +def _is_model_not_found(exc_text: str, exc_name: str) -> bool: + return ( + "notfound" in exc_name + or "model not found" in exc_text + or "does not exist" in exc_text + or "unknown model" in exc_text + ) + + +def _is_authentication_error(exc_text: str, exc_name: str) -> bool: + return ( + "authentication" in exc_name + or "invalid api key" in exc_text + or "unauthorized" in exc_text + or "401" in exc_text + ) + + +def _is_permission_error(exc_text: str, exc_name: str) -> bool: + return "permission" in exc_name or "access denied" in exc_text or "forbidden" in exc_text + + +def _is_rate_limit_error(exc_text: str, exc_name: str) -> bool: + return ( + "ratelimit" in exc_name + or "rate limit" in exc_text + or "quota" in exc_text + or "retrydelay" in exc_text + or "retry in " in exc_text + ) + + +def _is_timeout_error(exc_text: str, exc_name: str) -> bool: + return "timeout" in exc_name or "timed out" in exc_text + + +def _is_connection_error(exc_text: str, exc_name: str) -> bool: + return ( + "apiconnection" in exc_name + or "connection" in exc_text + or "unreachable" in exc_text + or "temporary failure" in exc_text + ) + + +def _retry_after_seconds(exc: Exception) -> int | None: + response = getattr(exc, "response", None) + if response is None: + return None + headers = getattr(response, "headers", None) + if headers is None: + return None + raw_value = headers.get("retry-after") + if raw_value is None: + raw_value = headers.get("Retry-After") + if raw_value is None: + return None + value = str(raw_value).strip() + if not value: + return None + if value.isdigit(): + return int(value) + try: + retry_after_time = parsedate_to_datetime(value) + except (TypeError, ValueError, IndexError): + return None + if retry_after_time.tzinfo is None: + retry_after_time = retry_after_time.replace(tzinfo=UTC) + now = datetime.now(UTC) + delta = (retry_after_time - now).total_seconds() + if delta <= 0: + return 0 + return int(delta) + + +def _retry_after_seconds_from_text(exc_text: str) -> int | None: + patterns = ( + r"retry in\s+([0-9]+(?:\.[0-9]+)?)s", + r"retrydelay\s*[:=]\s*['\"]?([0-9]+(?:\.[0-9]+)?)s['\"]?", + ) + lowered = exc_text.lower() + for pattern in patterns: + match = re.search(pattern, lowered, flags=re.IGNORECASE) + if match is None: + continue + try: + delay_value = float(match.group(1)) + except (TypeError, ValueError): + continue + if delay_value <= 0: + return 0 + return int(delay_value) if delay_value.is_integer() else int(delay_value) + 1 + return None + + def load_config() -> LLMConfig: """Load OpenAI-compatible configuration from environment variables.""" base_url = os.getenv("OPENAI_BASE_URL") @@ -72,11 +180,78 @@ def complete_messages( client = _build_openai_client(config) target_model = model or config.model - response = client.chat.completions.create( - model=target_model, - messages=messages, - temperature=temperature, - ) + for attempt in range(len(_RETRY_DELAYS_SECONDS) + 1): + try: + # Demos require deterministic decoding so PASS/FAIL results are reproducible. + response = client.chat.completions.create( + model=target_model, + messages=messages, + temperature=0, + top_p=1, + ) + break + except Exception as exc: + exc_text = str(exc).lower() + exc_name = exc.__class__.__name__.lower() + if _is_model_not_found(exc_text, exc_name): + raise DemoLLMError( + f"Model '{target_model}' was not found at the configured endpoint. " + "Check MODEL or OPENAI_BASE_URL." + ) from exc + if _is_authentication_error(exc_text, exc_name): + raise DemoLLMError("Authentication failed. Check OPENAI_API_KEY.") from exc + if _is_permission_error(exc_text, exc_name): + raise DemoLLMError( + f"Access to model '{target_model}' was denied by the configured provider." + ) from exc + + is_rate_limit = _is_rate_limit_error(exc_text, exc_name) + is_timeout = _is_timeout_error(exc_text, exc_name) + is_connection = _is_connection_error(exc_text, exc_name) + + if is_rate_limit or is_timeout or is_connection: + retry_after = _retry_after_seconds(exc) if is_rate_limit else None + if retry_after is None and is_rate_limit: + retry_after = _retry_after_seconds_from_text(str(exc)) + if retry_after is not None and retry_after > MAX_DEMO_RETRY_AFTER_SECONDS: + raise DemoLLMError( + f"LLM provider requested retry after {retry_after}s, " + "which exceeds the demo retry limit. " + "Try again later or switch providers." + ) from exc + if attempt < len(_RETRY_DELAYS_SECONDS): + delay = ( + retry_after if retry_after is not None else _RETRY_DELAYS_SECONDS[attempt] + ) + if is_rate_limit: + print( + f"[retry] LLM rate limit hit — retrying in {delay}s...", + file=sys.stderr, + ) + elif is_timeout: + print( + f"[retry] LLM timeout — retrying in {delay}s...", + file=sys.stderr, + ) + else: + print( + f"[retry] LLM connection error — retrying in {delay}s...", + file=sys.stderr, + ) + time.sleep(delay) + continue + if is_rate_limit: + raise DemoLLMError( + "LLM provider rate limit exceeded. Try again later or switch providers." + ) from exc + raise DemoLLMError( + "Could not reach the configured LLM endpoint after retries. " + "Check OPENAI_BASE_URL and network access." + ) from exc + + raise DemoLLMError( + f"LLM provider error while calling model '{target_model}': {exc}" + ) from exc content = response.choices[0].message.content if isinstance(content, str): return content.strip() diff --git a/demos/run_demo.py b/demos/run_demo.py index 633cf93..722e8e5 100644 --- a/demos/run_demo.py +++ b/demos/run_demo.py @@ -13,7 +13,7 @@ consume_last_info_report, consume_last_report, ) -from demos.llm_client import MissingDemoConfigError +from demos.llm_client import DemoLLMError, MissingDemoConfigError DEMO_FILES: dict[str, str] = { "1": "01_llm_constraint_drift.py", @@ -37,7 +37,7 @@ def _is_compiler_regression(result: DemoReport) -> bool: def _print_compiler_regression_warning() -> None: print() - print("⚠️ COMPILER REGRESSION") + print("⚠️ MEDIATED REGRESSION") print("baseline succeeded but compiler-mediated version failed") @@ -107,6 +107,9 @@ def main() -> None: except MissingDemoConfigError as exc: _print_config_error(exc) raise SystemExit(2) from exc + except DemoLLMError as exc: + print(str(exc)) + raise SystemExit(2) from exc if info_report is not None: informational_reports.append(info_report) @@ -141,9 +144,9 @@ def main() -> None: if compiler_regressions > 0: print() if compiler_regressions == 1: - print("*** 1 COMPILER REGRESSION DETECTED ***") + print("*** 1 MEDIATED REGRESSION DETECTED ***") else: - print(f"*** {compiler_regressions} COMPILER REGRESSIONS DETECTED ***") + print(f"*** {compiler_regressions} MEDIATED REGRESSIONS DETECTED ***") if informational_reports: print() print("Informational demo:") @@ -164,6 +167,9 @@ def main() -> None: except MissingDemoConfigError as exc: _print_config_error(exc) raise SystemExit(2) from exc + except DemoLLMError as exc: + print(str(exc)) + raise SystemExit(2) from exc if args.demo in SCORED_DEMOS and result is not None and _is_compiler_regression(result): _print_compiler_regression_warning() diff --git a/tests/test_llm_client.py b/tests/test_llm_client.py new file mode 100644 index 0000000..329138e --- /dev/null +++ b/tests/test_llm_client.py @@ -0,0 +1,227 @@ +import sys +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[1] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +import demos.llm_client as llm_client # noqa: E402 +from demos.llm_client import DemoLLMError, LLMConfig, complete_messages # noqa: E402 + + +def _fake_config() -> LLMConfig: + return LLMConfig(base_url="http://localhost:11434/v1", api_key="test-key", model="bad-model") + + +class _FakeCompletions: + def __init__(self, outcomes: list[object]) -> None: + self._outcomes = outcomes + self._index = 0 + + def create(self, **_kwargs: object) -> object: + if self._index >= len(self._outcomes): + raise RuntimeError("No more fake outcomes configured.") + outcome = self._outcomes[self._index] + self._index += 1 + if isinstance(outcome, Exception): + raise outcome + return outcome + + +class _FakeChat: + def __init__(self, outcomes: list[object]) -> None: + self.completions = _FakeCompletions(outcomes) + + +class _FakeClient: + def __init__(self, outcomes: list[object]) -> None: + self.chat = _FakeChat(outcomes) + + +class _FakeMessage: + def __init__(self, content: str) -> None: + self.content = content + + +class _FakeChoice: + def __init__(self, content: str) -> None: + self.message = _FakeMessage(content) + + +class _FakeResponse: + def __init__(self, content: str) -> None: + self.choices = [_FakeChoice(content)] + + +class _FakeRateLimitError(RuntimeError): + def __init__(self, message: str, retry_after: str | None = None) -> None: + super().__init__(message) + if retry_after is None: + self.response = None + return + self.response = type( + "Response", + (), + {"headers": {"retry-after": retry_after}}, + )() + + +def test_complete_messages_maps_model_not_found_error(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(llm_client, "load_config", _fake_config) + monkeypatch.setattr( + llm_client, + "_build_openai_client", + lambda _config: _FakeClient([RuntimeError("model not found")]), + ) + + with pytest.raises(DemoLLMError) as exc_info: + complete_messages([{"role": "user", "content": "hello"}]) + + assert str(exc_info.value) == ( + "Model 'bad-model' was not found at the configured endpoint. " + "Check MODEL or OPENAI_BASE_URL." + ) + + +def test_complete_messages_maps_authentication_error(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(llm_client, "load_config", _fake_config) + monkeypatch.setattr( + llm_client, + "_build_openai_client", + lambda _config: _FakeClient([RuntimeError("invalid api key")]), + ) + + with pytest.raises(DemoLLMError) as exc_info: + complete_messages([{"role": "user", "content": "hello"}]) + + assert str(exc_info.value) == "Authentication failed. Check OPENAI_API_KEY." + + +def test_complete_messages_retries_rate_limit_then_succeeds( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + monkeypatch.setattr(llm_client, "load_config", _fake_config) + monkeypatch.setattr( + llm_client, + "_build_openai_client", + lambda _config: _FakeClient( + [ + RuntimeError("rate limit exceeded"), + RuntimeError("rate limit exceeded"), + _FakeResponse("ok"), + ] + ), + ) + delays: list[int] = [] + monkeypatch.setattr(llm_client.time, "sleep", lambda seconds: delays.append(seconds)) + + result = complete_messages([{"role": "user", "content": "hello"}]) + stderr = capsys.readouterr().err + + assert result == "ok" + assert delays == [1, 2] + assert "[retry] LLM rate limit hit — retrying in 1s..." in stderr + assert "[retry] LLM rate limit hit — retrying in 2s..." in stderr + + +def test_complete_messages_rate_limit_exhausted_raises_friendly_error( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + monkeypatch.setattr(llm_client, "load_config", _fake_config) + monkeypatch.setattr( + llm_client, + "_build_openai_client", + lambda _config: _FakeClient( + [ + _FakeRateLimitError("rate limit exceeded"), + _FakeRateLimitError("rate limit exceeded"), + _FakeRateLimitError("rate limit exceeded"), + _FakeRateLimitError("rate limit exceeded"), + ] + ), + ) + delays: list[int] = [] + monkeypatch.setattr(llm_client.time, "sleep", lambda seconds: delays.append(seconds)) + + with pytest.raises(DemoLLMError) as exc_info: + complete_messages([{"role": "user", "content": "hello"}]) + stderr = capsys.readouterr().err + + assert delays == [1, 2, 4] + assert ( + str(exc_info.value) + == "LLM provider rate limit exceeded. Try again later or switch providers." + ) + assert "[retry] LLM rate limit hit — retrying in 1s..." in stderr + assert "[retry] LLM rate limit hit — retrying in 2s..." in stderr + assert "[retry] LLM rate limit hit — retrying in 4s..." in stderr + + +def test_complete_messages_long_retry_after_fails_fast( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + monkeypatch.setattr(llm_client, "load_config", _fake_config) + monkeypatch.setattr( + llm_client, + "_build_openai_client", + lambda _config: _FakeClient([_FakeRateLimitError("rate limit exceeded", retry_after="10")]), + ) + delays: list[int] = [] + monkeypatch.setattr(llm_client.time, "sleep", lambda seconds: delays.append(seconds)) + + with pytest.raises(DemoLLMError) as exc_info: + complete_messages([{"role": "user", "content": "hello"}]) + stderr = capsys.readouterr().err + + assert delays == [] + assert "[retry]" not in stderr + assert str(exc_info.value) == ( + "LLM provider requested retry after 10s, which exceeds the demo retry limit. " + "Try again later or switch providers." + ) + + +def test_complete_messages_uses_gemini_retry_in_text_delay( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + monkeypatch.setattr(llm_client, "load_config", _fake_config) + monkeypatch.setattr( + llm_client, + "_build_openai_client", + lambda _config: _FakeClient( + [RuntimeError("Please retry in 1.311529971s."), _FakeResponse("ok")] + ), + ) + delays: list[int] = [] + monkeypatch.setattr(llm_client.time, "sleep", lambda seconds: delays.append(seconds)) + + result = complete_messages([{"role": "user", "content": "hello"}]) + stderr = capsys.readouterr().err + + assert result == "ok" + assert delays == [2] + assert "[retry] LLM rate limit hit — retrying in 2s..." in stderr + + +def test_complete_messages_uses_gemini_retry_delay_field( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + monkeypatch.setattr(llm_client, "load_config", _fake_config) + monkeypatch.setattr( + llm_client, + "_build_openai_client", + lambda _config: _FakeClient( + [RuntimeError("rate limit exceeded, retryDelay: '1s'"), _FakeResponse("ok")] + ), + ) + delays: list[int] = [] + monkeypatch.setattr(llm_client.time, "sleep", lambda seconds: delays.append(seconds)) + + result = complete_messages([{"role": "user", "content": "hello"}]) + stderr = capsys.readouterr().err + + assert result == "ok" + assert delays == [1] + assert "[retry] LLM rate limit hit — retrying in 1s..." in stderr diff --git a/tests/test_run_demo.py b/tests/test_run_demo.py index 631416b..ff44e67 100644 --- a/tests/test_run_demo.py +++ b/tests/test_run_demo.py @@ -10,6 +10,7 @@ from demos import run_demo # noqa: E402 from demos.common import consume_last_info_report # noqa: E402 +from demos.llm_client import DemoLLMError # noqa: E402 def _demo_report(*, baseline_pass: bool, compiler_pass: bool) -> run_demo.DemoReport: @@ -59,10 +60,10 @@ def fake_run( output = capsys.readouterr().out assert "result:" in output - assert "⚠️ COMPILER REGRESSION" in output + assert "⚠️ MEDIATED REGRESSION" in output assert "baseline succeeded but compiler-mediated version failed" in output result_index = output.index("result:") - warning_index = output.index("⚠️ COMPILER REGRESSION") + warning_index = output.index("⚠️ MEDIATED REGRESSION") detail_index = output.index("baseline succeeded but compiler-mediated version failed") assert result_index < warning_index < detail_index @@ -99,7 +100,7 @@ def fake_run( assert "Baseline results: 1 passed, 0 failed" in output assert "Compiler results: 0 passed, 1 failed" in output - assert "*** 1 COMPILER REGRESSION DETECTED ***" in output + assert "*** 1 MEDIATED REGRESSION DETECTED ***" in output def test_runner_prints_plural_summary_regression_banner_in_all_mode( @@ -138,7 +139,7 @@ def fake_run( assert "Baseline results: 2 passed, 0 failed" in output assert "Compiler results: 0 passed, 2 failed" in output - assert "*** 2 COMPILER REGRESSIONS DETECTED ***" in output + assert "*** 2 MEDIATED REGRESSIONS DETECTED ***" in output def test_informational_demo_is_non_scored_in_all_mode( @@ -177,7 +178,34 @@ def fake_run( "06_context_compaction — context 137 → 37 chars (73% reduction); " "prompt 247 → 160 chars (35% reduction)" ) in output - assert "*** 1 COMPILER REGRESSION DETECTED ***" not in output + assert "*** 1 MEDIATED REGRESSION DETECTED ***" not in output + + +def test_runner_prints_friendly_demo_llm_error_in_single_mode( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + def fake_run( + path: Path, *, verbose: bool + ) -> tuple[run_demo.DemoReport | None, run_demo.InfoReport | None]: + raise DemoLLMError( + "Model 'bad-model' was not found at the configured endpoint. " + "Check MODEL or OPENAI_BASE_URL." + ) + + monkeypatch.setattr(run_demo, "DEMO_FILES", {"1": "fake_01.py"}) + monkeypatch.setattr(run_demo, "SCORED_DEMOS", {"1"}) + monkeypatch.setattr(run_demo, "_run", fake_run) + monkeypatch.setattr("sys.argv", ["run_demo", "1"]) + + with pytest.raises(SystemExit) as exc_info: + run_demo.main() + output = capsys.readouterr().out + + assert exc_info.value.code == 2 + assert ( + "Model 'bad-model' was not found at the configured endpoint. " + "Check MODEL or OPENAI_BASE_URL." + ) in output def test_all_mode_scored_none_result_counts_as_failures( From 612abb4c674a24f85cae104d14456f02a8472748 Mon Sep 17 00:00:00 2001 From: Robert Lippmann Date: Sat, 14 Mar 2026 00:00:43 -0400 Subject: [PATCH 2/2] Add demo LLM pacing CLI and provider throttling docs --- demos/README.md | 26 ++++++++++++++++++++++--- demos/llm_client.py | 16 ++++++++++++++- demos/run_demo.py | 33 ++++++++++++++++++++++++------- tests/test_llm_client.py | 18 +++++++++++++++++ tests/test_run_demo.py | 42 +++++++++++++++++++++++++++++++++------- 5 files changed, 117 insertions(+), 18 deletions(-) diff --git a/demos/README.md b/demos/README.md index 1b52d92..ed03134 100644 --- a/demos/README.md +++ b/demos/README.md @@ -15,13 +15,15 @@ Install demo dependencies: pip install -e .[demos] ``` -Environment variables: +Environment variables (OpenAI-compatible API): - `MODEL` (optional; default: `gpt-4.1-mini`) - `OPENAI_API_KEY` (required) -- `OPENAI_BASE_URL` (optional; use for OpenAI-compatible local servers) +- `OPENAI_BASE_URL` (optional; use for local or alternative endpoints) -Ollama example: +Example: locally hosted OpenAI-compatible endpoint (Ollama) + +Any locally hosted OpenAI-compatible endpoint will work (for example Ollama, LM Studio, or a llama.cpp server). ```bash export OPENAI_BASE_URL=http://localhost:11434/v1 @@ -56,6 +58,24 @@ Run all demos with detailed traces: uv run python -m demos.run_demo all --verbose ``` +Run demos with pacing for low-quota providers: + +```bash +uv run python -m demos.run_demo all --llm-delay 1.5 +``` + +### Provider throttling + +The demos make multiple LLM requests and may trigger rate limits on very low-quota hosted providers (especially free tiers). + +If you encounter throttling, you can slow requests using: + +```bash +uv run python -m demos.run_demo all --llm-delay 1.5 +``` + +Running against a local OpenAI-compatible endpoint avoids provider rate limits. + ## Output modes - `Default (concise)`: diff --git a/demos/llm_client.py b/demos/llm_client.py index 0d1d9ab..7a8a03c 100644 --- a/demos/llm_client.py +++ b/demos/llm_client.py @@ -39,6 +39,7 @@ class DemoLLMError(RuntimeError): _RETRY_DELAYS_SECONDS = (1, 2, 4) MAX_DEMO_RETRY_AFTER_SECONDS = 5 +DEFAULT_LLM_DELAY_SECONDS = 0.0 def _is_model_not_found(exc_text: str, exc_name: str) -> bool: @@ -136,6 +137,12 @@ def _retry_after_seconds_from_text(exc_text: str) -> int | None: return None +def _configured_delay_seconds(delay_seconds: float) -> float: + if delay_seconds > 0: + return delay_seconds + return DEFAULT_LLM_DELAY_SECONDS + + def load_config() -> LLMConfig: """Load OpenAI-compatible configuration from environment variables.""" base_url = os.getenv("OPENAI_BASE_URL") @@ -173,19 +180,26 @@ def _build_openai_client(config: LLMConfig) -> Any: def complete_messages( - messages: list[Message], *, model: str | None = None, temperature: float = 0.0 + messages: list[Message], + *, + model: str | None = None, + delay_seconds: float = 0, ) -> str: """Send exact message list to chat completions and return the text output.""" config = load_config() client = _build_openai_client(config) target_model = model or config.model + configured_delay = _configured_delay_seconds(delay_seconds) for attempt in range(len(_RETRY_DELAYS_SECONDS) + 1): try: + if configured_delay > 0: + time.sleep(configured_delay) # Demos require deterministic decoding so PASS/FAIL results are reproducible. response = client.chat.completions.create( model=target_model, messages=messages, + # Intentionally hard-coded for deterministic demo behavior. temperature=0, top_p=1, ) diff --git a/demos/run_demo.py b/demos/run_demo.py index 722e8e5..80a295a 100644 --- a/demos/run_demo.py +++ b/demos/run_demo.py @@ -6,6 +6,7 @@ import sys from pathlib import Path +import demos.llm_client as llm_client from demos.common import ( VERBOSE_ENV_VAR, DemoReport, @@ -13,7 +14,10 @@ consume_last_info_report, consume_last_report, ) -from demos.llm_client import DemoLLMError, MissingDemoConfigError +from demos.llm_client import ( + DemoLLMError, + MissingDemoConfigError, +) DEMO_FILES: dict[str, str] = { "1": "01_llm_constraint_drift.py", @@ -41,19 +45,24 @@ def _print_compiler_regression_warning() -> None: print("baseline succeeded but compiler-mediated version failed") -def _run(path: Path, *, verbose: bool) -> tuple[DemoReport | None, InfoReport | None]: +def _run( + path: Path, *, verbose: bool, llm_delay: float +) -> tuple[DemoReport | None, InfoReport | None]: if verbose: print(f"===== Running {_verbose_demo_label(path)} =====") - old_value = os.getenv(VERBOSE_ENV_VAR) + old_verbose = os.getenv(VERBOSE_ENV_VAR) + old_delay = llm_client.DEFAULT_LLM_DELAY_SECONDS os.environ[VERBOSE_ENV_VAR] = "1" if verbose else "0" + llm_client.DEFAULT_LLM_DELAY_SECONDS = llm_delay if llm_delay > 0 else 0.0 try: runpy.run_path(str(path), run_name="__main__") return consume_last_report(), consume_last_info_report() finally: - if old_value is None: + if old_verbose is None: os.environ.pop(VERBOSE_ENV_VAR, None) else: - os.environ[VERBOSE_ENV_VAR] = old_value + os.environ[VERBOSE_ENV_VAR] = old_verbose + llm_client.DEFAULT_LLM_DELAY_SECONDS = old_delay def _print_config_error(exc: MissingDemoConfigError) -> None: @@ -90,6 +99,12 @@ def main() -> None: action="store_true", help="Show detailed prompts, compiler decisions, and model output excerpts.", ) + parser.add_argument( + "--llm-delay", + type=float, + default=0, + help="Delay between LLM calls in seconds (useful for low-quota providers).", + ) args = parser.parse_args() if args.demo == "all": @@ -103,7 +118,9 @@ def main() -> None: if index > 0 and not args.verbose: print() try: - result, info_report = _run(root / DEMO_FILES[key], verbose=args.verbose) + result, info_report = _run( + root / DEMO_FILES[key], verbose=args.verbose, llm_delay=args.llm_delay + ) except MissingDemoConfigError as exc: _print_config_error(exc) raise SystemExit(2) from exc @@ -163,7 +180,9 @@ def main() -> None: return try: - result, _ = _run(root / DEMO_FILES[args.demo], verbose=args.verbose) + result, _ = _run( + root / DEMO_FILES[args.demo], verbose=args.verbose, llm_delay=args.llm_delay + ) except MissingDemoConfigError as exc: _print_config_error(exc) raise SystemExit(2) from exc diff --git a/tests/test_llm_client.py b/tests/test_llm_client.py index 329138e..927b56a 100644 --- a/tests/test_llm_client.py +++ b/tests/test_llm_client.py @@ -225,3 +225,21 @@ def test_complete_messages_uses_gemini_retry_delay_field( assert result == "ok" assert delays == [1] assert "[retry] LLM rate limit hit — retrying in 1s..." in stderr + + +def test_complete_messages_applies_delay_seconds_before_call( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(llm_client, "load_config", _fake_config) + monkeypatch.setattr( + llm_client, + "_build_openai_client", + lambda _config: _FakeClient([_FakeResponse("ok")]), + ) + delays: list[float] = [] + monkeypatch.setattr(llm_client.time, "sleep", lambda seconds: delays.append(seconds)) + + result = complete_messages([{"role": "user", "content": "hello"}], delay_seconds=1.5) + + assert result == "ok" + assert delays == [1.5] diff --git a/tests/test_run_demo.py b/tests/test_run_demo.py index ff44e67..0293443 100644 --- a/tests/test_run_demo.py +++ b/tests/test_run_demo.py @@ -40,9 +40,10 @@ def test_runner_prints_per_demo_compiler_regression_warning( monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] ) -> None: def fake_run( - path: Path, *, verbose: bool + path: Path, *, verbose: bool, llm_delay: float ) -> tuple[run_demo.DemoReport | None, run_demo.InfoReport | None]: assert not verbose + assert llm_delay == 0 print("01_fake — regression fixture") print("baseline: PASS") print("compiler: FAIL") @@ -72,9 +73,10 @@ def test_runner_prints_summary_regression_banner_in_all_mode( monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] ) -> None: def fake_run( - path: Path, *, verbose: bool + path: Path, *, verbose: bool, llm_delay: float ) -> tuple[run_demo.DemoReport | None, run_demo.InfoReport | None]: assert not verbose + assert llm_delay == 0 if path.name == "fake_01.py": print("01_fake — regression fixture") print("baseline: PASS") @@ -107,9 +109,10 @@ def test_runner_prints_plural_summary_regression_banner_in_all_mode( monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] ) -> None: def fake_run( - path: Path, *, verbose: bool + path: Path, *, verbose: bool, llm_delay: float ) -> tuple[run_demo.DemoReport | None, run_demo.InfoReport | None]: assert not verbose + assert llm_delay == 0 if path.name in {"fake_01.py", "fake_02.py"}: print(f"{path.stem} — regression fixture") print("baseline: PASS") @@ -146,9 +149,10 @@ def test_informational_demo_is_non_scored_in_all_mode( monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] ) -> None: def fake_run( - path: Path, *, verbose: bool + path: Path, *, verbose: bool, llm_delay: float ) -> tuple[run_demo.DemoReport | None, run_demo.InfoReport | None]: assert not verbose + assert llm_delay == 0 if path.name == "fake_01.py": print("01_fake — pass fixture") print("baseline: PASS") @@ -185,8 +189,9 @@ def test_runner_prints_friendly_demo_llm_error_in_single_mode( monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] ) -> None: def fake_run( - path: Path, *, verbose: bool + path: Path, *, verbose: bool, llm_delay: float ) -> tuple[run_demo.DemoReport | None, run_demo.InfoReport | None]: + assert llm_delay == 0 raise DemoLLMError( "Model 'bad-model' was not found at the configured endpoint. " "Check MODEL or OPENAI_BASE_URL." @@ -212,9 +217,10 @@ def test_all_mode_scored_none_result_counts_as_failures( monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] ) -> None: def fake_run( - path: Path, *, verbose: bool + path: Path, *, verbose: bool, llm_delay: float ) -> tuple[run_demo.DemoReport | None, run_demo.InfoReport | None]: assert not verbose + assert llm_delay == 0 if path.name == "fake_01.py": return None, None print("06_context_compaction — superseded directives eliminated") @@ -240,9 +246,10 @@ def test_all_mode_counts_baseline_fail_and_compiler_pass( monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] ) -> None: def fake_run( - path: Path, *, verbose: bool + path: Path, *, verbose: bool, llm_delay: float ) -> tuple[run_demo.DemoReport | None, run_demo.InfoReport | None]: assert not verbose + assert llm_delay == 0 if path.name == "fake_01.py": print("01_fake — mixed fixture") print("baseline: FAIL") @@ -283,3 +290,24 @@ def test_compaction_demo_reports_sane_metrics() -> None: assert report["baseline_prompt_length"] > report["compiled_prompt_length"] assert report["context_reduction_percent"] > 0 assert report["prompt_reduction_percent"] > 0 + + +def test_runner_passes_llm_delay_from_cli(monkeypatch: pytest.MonkeyPatch) -> None: + captured: dict[str, float] = {} + + def fake_run( + path: Path, *, verbose: bool, llm_delay: float + ) -> tuple[run_demo.DemoReport | None, run_demo.InfoReport | None]: + assert path.name == "fake_06.py" + assert not verbose + captured["llm_delay"] = llm_delay + return None, None + + monkeypatch.setattr(run_demo, "DEMO_FILES", {"6": "fake_06.py"}) + monkeypatch.setattr(run_demo, "SCORED_DEMOS", {"1", "2", "3", "4", "5"}) + monkeypatch.setattr(run_demo, "_run", fake_run) + monkeypatch.setattr("sys.argv", ["run_demo", "6", "--llm-delay", "1.25"]) + + run_demo.main() + + assert captured["llm_delay"] == 1.25