Add demo LLM pacing CLI and improve provider-throttling UXCodex/more demo cleanup

demos/README.md

@@ -5,6 +5,7 @@ Context Compiler decision/state API.
 They are illustrative manual demos, not benchmarks or CI tests.
 They demonstrate common LLM failure modes and how authoritative compiled state
 can improve reliability.
+All demos force deterministic decoding so results are reproducible.
 
 ## Requirements
 
@@ -14,13 +15,15 @@ Install demo dependencies:
 pip install -e .[demos]
 ```
 
-Environment variables:
+Environment variables (OpenAI-compatible API):
 
 - `MODEL` (optional; default: `gpt-4.1-mini`)
 - `OPENAI_API_KEY` (required)
-- `OPENAI_BASE_URL` (optional; use for OpenAI-compatible local servers)
+- `OPENAI_BASE_URL` (optional; use for local or alternative endpoints)
 
-Ollama example:
+Example: locally hosted OpenAI-compatible endpoint (Ollama)
+
+Any locally hosted OpenAI-compatible endpoint will work (for example Ollama, LM Studio, or a llama.cpp server).
 
 ```bash
 export OPENAI_BASE_URL=http://localhost:11434/v1
@@ -55,6 +58,24 @@ Run all demos with detailed traces:
 uv run python -m demos.run_demo all --verbose
 ```
 
+Run demos with pacing for low-quota providers:
+
+```bash
+uv run python -m demos.run_demo all --llm-delay 1.5
+```
+
+### Provider throttling
+
+The demos make multiple LLM requests and may trigger rate limits on very low-quota hosted providers (especially free tiers).
+
+If you encounter throttling, you can slow requests using:
+
+```bash
+uv run python -m demos.run_demo all --llm-delay 1.5
+```
+
+Running against a local OpenAI-compatible endpoint avoids provider rate limits.
+
 ## Output modes
 
 - `Default (concise)`:

demos/llm_client.py

@@ -1,7 +1,12 @@
 """Small OpenAI-compatible chat client for demo scripts."""
 
 import os
+import re
+import sys
+import time
 from dataclasses import dataclass
+from datetime import UTC, datetime
+from email.utils import parsedate_to_datetime
 from importlib import import_module
 from typing import Any, Literal, TypedDict
 
@@ -28,6 +33,116 @@ def __str__(self) -> str:
         return f"Missing demo configuration: {missing_text}"
 
 
+class DemoLLMError(RuntimeError):
+    """Friendly provider/client error for demos."""
+
+
+_RETRY_DELAYS_SECONDS = (1, 2, 4)
+MAX_DEMO_RETRY_AFTER_SECONDS = 5
+DEFAULT_LLM_DELAY_SECONDS = 0.0
+
+
+def _is_model_not_found(exc_text: str, exc_name: str) -> bool:
+    return (
+        "notfound" in exc_name
+        or "model not found" in exc_text
+        or "does not exist" in exc_text
+        or "unknown model" in exc_text
+    )
+
+
+def _is_authentication_error(exc_text: str, exc_name: str) -> bool:
+    return (
+        "authentication" in exc_name
+        or "invalid api key" in exc_text
+        or "unauthorized" in exc_text
+        or "401" in exc_text
+    )
+
+
+def _is_permission_error(exc_text: str, exc_name: str) -> bool:
+    return "permission" in exc_name or "access denied" in exc_text or "forbidden" in exc_text
+
+
+def _is_rate_limit_error(exc_text: str, exc_name: str) -> bool:
+    return (
+        "ratelimit" in exc_name
+        or "rate limit" in exc_text
+        or "quota" in exc_text
+        or "retrydelay" in exc_text
+        or "retry in " in exc_text
+    )
+
+
+def _is_timeout_error(exc_text: str, exc_name: str) -> bool:
+    return "timeout" in exc_name or "timed out" in exc_text
+
+
+def _is_connection_error(exc_text: str, exc_name: str) -> bool:
+    return (
+        "apiconnection" in exc_name
+        or "connection" in exc_text
+        or "unreachable" in exc_text
+        or "temporary failure" in exc_text
+    )
+
+
+def _retry_after_seconds(exc: Exception) -> int | None:
+    response = getattr(exc, "response", None)
+    if response is None:
+        return None
+    headers = getattr(response, "headers", None)
+    if headers is None:
+        return None
+    raw_value = headers.get("retry-after")
+    if raw_value is None:
+        raw_value = headers.get("Retry-After")
+    if raw_value is None:
+        return None
+    value = str(raw_value).strip()
+    if not value:
+        return None
+    if value.isdigit():
+        return int(value)
+    try:
+        retry_after_time = parsedate_to_datetime(value)
+    except (TypeError, ValueError, IndexError):
+        return None
+    if retry_after_time.tzinfo is None:
+        retry_after_time = retry_after_time.replace(tzinfo=UTC)
+    now = datetime.now(UTC)
+    delta = (retry_after_time - now).total_seconds()
+    if delta <= 0:
+        return 0
+    return int(delta)
+
+
+def _retry_after_seconds_from_text(exc_text: str) -> int | None:
+    patterns = (
+        r"retry in\s+([0-9]+(?:\.[0-9]+)?)s",
+        r"retrydelay\s*[:=]\s*['\"]?([0-9]+(?:\.[0-9]+)?)s['\"]?",
+    )
+    lowered = exc_text.lower()
+    for pattern in patterns:
+        match = re.search(pattern, lowered, flags=re.IGNORECASE)
+        if match is None:
+            continue
+        try:
+            delay_value = float(match.group(1))
+        except (TypeError, ValueError):
+            continue
+        if delay_value <= 0:
+            return 0
+        return int(delay_value) if delay_value.is_integer() else int(delay_value) + 1
+    return None
+
+
+def _configured_delay_seconds(delay_seconds: float) -> float:
+    if delay_seconds > 0:
+        return delay_seconds
+    return DEFAULT_LLM_DELAY_SECONDS
+
+
 def load_config() -> LLMConfig:
     """Load OpenAI-compatible configuration from environment variables."""
     base_url = os.getenv("OPENAI_BASE_URL")
@@ -65,18 +180,92 @@ def _build_openai_client(config: LLMConfig) -> Any:
 
 
 def complete_messages(
-    messages: list[Message], *, model: str | None = None, temperature: float = 0.0
+    messages: list[Message],
+    *,
+    model: str | None = None,
+    delay_seconds: float = 0,
 ) -> str:
     """Send exact message list to chat completions and return the text output."""
     config = load_config()
     client = _build_openai_client(config)
     target_model = model or config.model
+    configured_delay = _configured_delay_seconds(delay_seconds)
 
-    response = client.chat.completions.create(
-        model=target_model,
-        messages=messages,
-        temperature=temperature,
-    )
+    for attempt in range(len(_RETRY_DELAYS_SECONDS) + 1):
+        try:
+            if configured_delay > 0:
+                time.sleep(configured_delay)
+            # Demos require deterministic decoding so PASS/FAIL results are reproducible.
+            response = client.chat.completions.create(
+                model=target_model,
+                messages=messages,
+                # Intentionally hard-coded for deterministic demo behavior.
+                temperature=0,
+                top_p=1,
+            )
+            break
+        except Exception as exc:
+            exc_text = str(exc).lower()
+            exc_name = exc.__class__.__name__.lower()
+            if _is_model_not_found(exc_text, exc_name):
+                raise DemoLLMError(
+                    f"Model '{target_model}' was not found at the configured endpoint. "
+                    "Check MODEL or OPENAI_BASE_URL."
+                ) from exc
+            if _is_authentication_error(exc_text, exc_name):
+                raise DemoLLMError("Authentication failed. Check OPENAI_API_KEY.") from exc
+            if _is_permission_error(exc_text, exc_name):
+                raise DemoLLMError(
+                    f"Access to model '{target_model}' was denied by the configured provider."
+                ) from exc
+
+            is_rate_limit = _is_rate_limit_error(exc_text, exc_name)
+            is_timeout = _is_timeout_error(exc_text, exc_name)
+            is_connection = _is_connection_error(exc_text, exc_name)
+
+            if is_rate_limit or is_timeout or is_connection:
+                retry_after = _retry_after_seconds(exc) if is_rate_limit else None
+                if retry_after is None and is_rate_limit:
+                    retry_after = _retry_after_seconds_from_text(str(exc))
+                if retry_after is not None and retry_after > MAX_DEMO_RETRY_AFTER_SECONDS:
+                    raise DemoLLMError(
+                        f"LLM provider requested retry after {retry_after}s, "
+                        "which exceeds the demo retry limit. "
+                        "Try again later or switch providers."
+                    ) from exc
+                if attempt < len(_RETRY_DELAYS_SECONDS):
+                    delay = (
+                        retry_after if retry_after is not None else _RETRY_DELAYS_SECONDS[attempt]
+                    )
+                    if is_rate_limit:
+                        print(
+                            f"[retry] LLM rate limit hit — retrying in {delay}s...",
+                            file=sys.stderr,
+                        )
+                    elif is_timeout:
+                        print(
+                            f"[retry] LLM timeout — retrying in {delay}s...",
+                            file=sys.stderr,
+                        )
+                    else:
+                        print(
+                            f"[retry] LLM connection error — retrying in {delay}s...",
+                            file=sys.stderr,
+                        )
+                    time.sleep(delay)
+                    continue
+                if is_rate_limit:
+                    raise DemoLLMError(
+                        "LLM provider rate limit exceeded. Try again later or switch providers."
+                    ) from exc
+                raise DemoLLMError(
+                    "Could not reach the configured LLM endpoint after retries. "
+                    "Check OPENAI_BASE_URL and network access."
+                ) from exc
+
+            raise DemoLLMError(
+                f"LLM provider error while calling model '{target_model}': {exc}"
+            ) from exc
     content = response.choices[0].message.content
     if isinstance(content, str):
         return content.strip()

demos/run_demo.py

@@ -6,14 +6,18 @@
 import sys
 from pathlib import Path
 
+import demos.llm_client as llm_client
 from demos.common import (
     VERBOSE_ENV_VAR,
     DemoReport,
     InfoReport,
     consume_last_info_report,
     consume_last_report,
 )
-from demos.llm_client import MissingDemoConfigError
+from demos.llm_client import (
+    DemoLLMError,
+    MissingDemoConfigError,
+)
 
 DEMO_FILES: dict[str, str] = {
     "1": "01_llm_constraint_drift.py",
@@ -37,23 +41,28 @@ def _is_compiler_regression(result: DemoReport) -> bool:
 
 def _print_compiler_regression_warning() -> None:
     print()
-    print("⚠️ COMPILER REGRESSION")
+    print("⚠️ MEDIATED REGRESSION")
     print("baseline succeeded but compiler-mediated version failed")
 
 
-def _run(path: Path, *, verbose: bool) -> tuple[DemoReport | None, InfoReport | None]:
+def _run(
+    path: Path, *, verbose: bool, llm_delay: float
+) -> tuple[DemoReport | None, InfoReport | None]:
     if verbose:
         print(f"===== Running {_verbose_demo_label(path)} =====")
-    old_value = os.getenv(VERBOSE_ENV_VAR)
+    old_verbose = os.getenv(VERBOSE_ENV_VAR)
+    old_delay = llm_client.DEFAULT_LLM_DELAY_SECONDS
     os.environ[VERBOSE_ENV_VAR] = "1" if verbose else "0"
+    llm_client.DEFAULT_LLM_DELAY_SECONDS = llm_delay if llm_delay > 0 else 0.0
     try:
         runpy.run_path(str(path), run_name="__main__")
         return consume_last_report(), consume_last_info_report()
     finally:
-        if old_value is None:
+        if old_verbose is None:
             os.environ.pop(VERBOSE_ENV_VAR, None)
         else:
-            os.environ[VERBOSE_ENV_VAR] = old_value
+            os.environ[VERBOSE_ENV_VAR] = old_verbose
+        llm_client.DEFAULT_LLM_DELAY_SECONDS = old_delay
 
 
 def _print_config_error(exc: MissingDemoConfigError) -> None:
@@ -90,6 +99,12 @@ def main() -> None:
         action="store_true",
         help="Show detailed prompts, compiler decisions, and model output excerpts.",
     )
+    parser.add_argument(
+        "--llm-delay",
+        type=float,
+        default=0,
+        help="Delay between LLM calls in seconds (useful for low-quota providers).",
+    )
     args = parser.parse_args()
 
     if args.demo == "all":
@@ -103,10 +118,15 @@ def main() -> None:
             if index > 0 and not args.verbose:
                 print()
             try:
-                result, info_report = _run(root / DEMO_FILES[key], verbose=args.verbose)
+                result, info_report = _run(
+                    root / DEMO_FILES[key], verbose=args.verbose, llm_delay=args.llm_delay
+                )
             except MissingDemoConfigError as exc:
                 _print_config_error(exc)
                 raise SystemExit(2) from exc
+            except DemoLLMError as exc:
+                print(str(exc))
+                raise SystemExit(2) from exc
 
             if info_report is not None:
                 informational_reports.append(info_report)
@@ -141,9 +161,9 @@ def main() -> None:
         if compiler_regressions > 0:
             print()
             if compiler_regressions == 1:
-                print("*** 1 COMPILER REGRESSION DETECTED ***")
+                print("*** 1 MEDIATED REGRESSION DETECTED ***")
             else:
-                print(f"*** {compiler_regressions} COMPILER REGRESSIONS DETECTED ***")
+                print(f"*** {compiler_regressions} MEDIATED REGRESSIONS DETECTED ***")
         if informational_reports:
             print()
             print("Informational demo:")
@@ -160,10 +180,15 @@ def main() -> None:
         return
 
     try:
-        result, _ = _run(root / DEMO_FILES[args.demo], verbose=args.verbose)
+        result, _ = _run(
+            root / DEMO_FILES[args.demo], verbose=args.verbose, llm_delay=args.llm_delay
+        )
     except MissingDemoConfigError as exc:
         _print_config_error(exc)
         raise SystemExit(2) from exc
+    except DemoLLMError as exc:
+        print(str(exc))
+        raise SystemExit(2) from exc
     if args.demo in SCORED_DEMOS and result is not None and _is_compiler_regression(result):
         _print_compiler_regression_warning()