feat: add demo cli passthrough and docs polish

README.md

@@ -105,7 +105,7 @@ class Decision(TypedDict):
 Meaning:
 
 | kind        | host behavior                                 |
-|-------------|-----------------------------------------------|
+|:-----------:|-----------------------------------------------|
 | passthrough | forward user input to LLM                     |
 | update      | forward input with updated state              |
 | clarify     | show `prompt_to_user` and do not call the LLM |

demos/05_llm_prompt_drift.py

@@ -1,7 +1,9 @@
 """Demo 5: long transcript drift vs stable compiled state."""
 
+import argparse
 import re
 
+import demos.llm_client as llm_client
 from context_compiler import create_engine
 from demos.common import (
     build_baseline_messages,
@@ -29,6 +31,81 @@
 )
 _NEGATION_RE = re.compile(r"\b(no|without|avoid|exclude|instead of)\b", flags=re.IGNORECASE)
 
+_ORIGINAL_DIRECTIVE = "use vegetarian curry"
+_FINAL_PROMPT = (
+    "Now give me a dinner plan. First line must be DINNER_STYLE:<vegetarian|non-vegetarian>."
+)
+_DISTRACTOR_TOPICS = [
+    "travel photography",
+    "city walking routes",
+    "weekend train trips",
+    "mountain day hikes",
+    "pour-over coffee brewing",
+    "espresso dialing",
+    "architecture sketching",
+    "museum planning",
+    "weather map reading",
+    "atlas navigation",
+    "independent bookstores",
+    "historical nonfiction reading",
+    "film photography",
+    "macro photography",
+    "night sky viewing",
+    "rail station architecture",
+    "public transit maps",
+    "urban design tours",
+    "coastal trail planning",
+    "desert trail planning",
+    "baking crust hydration",
+    "pan sauce reduction",
+    "knife-skill practice",
+    "tea brewing",
+    "city museum circuits",
+]
+_DISTRACTOR_PROMPT_TEMPLATES = [
+    "Quick question on {topic}: which beginner book gives solid fundamentals?",
+    "For {topic}, what common pitfall surprises newcomers?",
+    "In {topic}, which metric helps compare two options fairly?",
+    "How would you plan a one-day itinerary around {topic}?",
+    "For {topic}, what gear checklist keeps things practical?",
+    "In {topic}, what weather factor changes decisions the most?",
+    "What map detail matters most when preparing for {topic}?",
+    "For {topic}, which habit improves consistency over months?",
+    "How can someone budget for {topic} without losing quality?",
+    "For {topic}, what tradeoff appears between speed and accuracy?",
+    "What museum exhibit style pairs well with interest in {topic}?",
+    "For {topic}, which train route offers the most scenic segments?",
+]
+
+
+def _build_master_distractor_sequence() -> list[str]:
+    sequence = [
+        # Keep these first two distractors byte-identical to the original demo.
+        "Also I like hiking and jazz.",
+        "What camera should I buy for travel?",
+    ]
+    for topic in _DISTRACTOR_TOPICS:
+        for template in _DISTRACTOR_PROMPT_TEMPLATES:
+            sequence.append(template.format(topic=topic))
+    return sequence
+
+
+_MASTER_DISTRACTOR_SEQUENCE = _build_master_distractor_sequence()
+if len(_MASTER_DISTRACTOR_SEQUENCE) < 240:
+    raise RuntimeError("Demo 5 distractor sequence must support at least 240 turns.")
+
+
+_LADDER_TURNS = [10, 30, 60, 120, 240]
+_DEFAULT_TURNS = 2
+
+
+_ORIGINAL_DEFAULT_TRANSCRIPT = [
+    _ORIGINAL_DIRECTIVE,
+    "Also I like hiking and jazz.",
+    "What camera should I buy for travel?",
+    _FINAL_PROMPT,
+]
+
 
 def _plan_lines(output: str) -> list[str]:
     lines = output.splitlines()
@@ -62,22 +139,64 @@ def plan_includes_non_vegetarian_item(output: str) -> bool:
     return False
 
 
-def main() -> None:
+def _validate_turns(turns: int) -> None:
+    max_turns = len(_MASTER_DISTRACTOR_SEQUENCE)
+    if turns < 0:
+        raise ValueError("turns must be at least 0.")
+    if turns > max_turns:
+        raise ValueError(f"turns must be <= {max_turns}.")
+
+
+def build_context_turns(turns: int) -> list[str]:
+    _validate_turns(turns)
+    return [_ORIGINAL_DIRECTIVE, *_MASTER_DISTRACTOR_SEQUENCE[:turns]]
+
+
+def build_user_inputs(turns: int) -> list[str]:
+    return [*build_context_turns(turns), _FINAL_PROMPT]
+
+
+def _parse_args(argv: list[str] | None = None) -> argparse.Namespace:
+    max_turns = len(_MASTER_DISTRACTOR_SEQUENCE)
+    parser = argparse.ArgumentParser(
+        description=(
+            "Run Demo 5 with deterministic distractor distance for prompt-drift stress testing."
+        )
+    )
+    parser.add_argument(
+        "--turns",
+        type=int,
+        default=_DEFAULT_TURNS,
+        help=(
+            "Number of distractor turns between the original directive and final prompt "
+            f"(0-{max_turns}). Supports stress-test ladder points: "
+            f"{', '.join(map(str, _LADDER_TURNS))}."
+        ),
+    )
+    parser.add_argument(
+        "--llm-delay",
+        type=float,
+        default=None,
+        help="Delay between LLM calls in seconds (overrides shared default when provided).",
+    )
+    args = parser.parse_args(argv)
+    _validate_turns(args.turns)
+    return args
+
+
+def _run_demo(turns: int = _DEFAULT_TURNS) -> None:
     engine = create_engine()
-    user_inputs = [
-        "use vegetarian curry",
-        "Also I like hiking and jazz.",
-        "What camera should I buy for travel?",
-        "Now give me a dinner plan. First line must be DINNER_STYLE:<vegetarian|non-vegetarian>.",
-    ]
+    user_inputs = build_user_inputs(turns)
+    if turns == _DEFAULT_TURNS and user_inputs != _ORIGINAL_DEFAULT_TRANSCRIPT:
+        raise RuntimeError("Demo 5 default transcript diverged from original behavior.")
     print_user_inputs(user_inputs)
 
     for index, user_input in enumerate(user_inputs, start=1):
         decision = engine.step(user_input)
         print_decision(f"turn {index}", decision, engine.state)
 
     baseline_messages = build_baseline_messages(
-        [user_inputs[0], user_inputs[1], user_inputs[2], user_inputs[3]],
+        user_inputs,
         baseline_system_prompt=(
             "Be a helpful assistant. Use the conversation context to provide a useful answer."
         ),
@@ -86,7 +205,7 @@ def main() -> None:
     baseline_output = complete_messages(baseline_messages)
     print_model_output("Baseline", baseline_output)
 
-    mediated_messages = build_mediated_messages(engine.state, user_inputs[3])
+    mediated_messages = build_mediated_messages(engine.state, user_inputs[-1])
     print_messages("compiler-mediated", mediated_messages)
     mediated_output = complete_messages(mediated_messages)
     print_model_output("Compiler-mediated", mediated_output)
@@ -136,5 +255,17 @@ def main() -> None:
     )
 
 
+def main(turns: int = _DEFAULT_TURNS, llm_delay: float | None = None) -> None:
+    old_delay = llm_client.DEFAULT_LLM_DELAY_SECONDS
+    if llm_delay is not None:
+        llm_client.DEFAULT_LLM_DELAY_SECONDS = llm_delay if llm_delay > 0 else 0.0
+    try:
+        _run_demo(turns=turns)
+    finally:
+        if llm_delay is not None:
+            llm_client.DEFAULT_LLM_DELAY_SECONDS = old_delay
+
+
 if __name__ == "__main__":
-    main()
+    cli_args = _parse_args()
+    main(turns=cli_args.turns, llm_delay=cli_args.llm_delay)

demos/README.md

@@ -1,23 +1,25 @@
 # LLM Demos
 
-These scripts compare baseline prompting vs compiler-mediated prompting using the
-Context Compiler decision/state API.
-They are illustrative manual demos, not benchmarks or CI tests.
-They demonstrate common LLM failure modes and how authoritative compiled state
-can improve reliability.
-They also illustrate how a host application interprets compiler `Decision` outputs to control when the LLM is called.
-All demos force deterministic decoding so results are reproducible.
+These scripts show common ways LLM conversations can go wrong.
+
+They compare normal prompting with an approach where the application tracks
+important instructions explicitly instead of relying only on the conversation
+history. The scripts are designed to produce consistent results so the
+behavior is easy to see.
 
 ## Demo overview
 
-| Demo | Demonstrates | Concept |
-| --- | --- | --- |
-| [01](./01_llm_ambiguity_block.py) | Ambiguous directive blocking | clarification gate |
-| [02](./02_llm_constraint_drift.py) | Constraint drift | persistent policy enforcement |
-| [03](./03_llm_correction_replacement.py) | Correction replacement | exclusive fact semantics |
-| [04](./04_llm_tool_governance.py) | Tool governance | host-side denylist |
-| [05](./05_llm_prompt_drift.py) | Prompt drift | long transcript failure |
-| [06](./06_context_compaction.py) | Context compaction | compiled state replacing transcript |
+| Demo | Behavior | Concept | Most visible with |
+| :--: | --- | :--: | --- |
+| [01](./01_llm_ambiguity_block.py) | Ambiguous directive blocking | clarification gate | small instruct models |
+| [02](./02_llm_constraint_drift.py) | Constraint drift | persistent policy enforcement | small or quantized models |
+| [03](./03_llm_correction_replacement.py) | Correction replacement | exclusive fact semantics | models that summarize conversation |
+| [04](./04_llm_tool_governance.py) | Tool governance | host-side denylist | general assistant models |
+| [05](./05_llm_prompt_drift.py) | Prompt drift | long transcript failure | weaker long-context models ([see Demo 5 note](#demo-5-stress-ladder-turns)) |
+| [06](./06_context_compaction.py) | Context compaction | compiled state replacing transcript | small or local models |
+
+Stronger frontier models may show these behaviors less often, but the same
+patterns still appear in real applications.
 
 ## Requirements
 
@@ -29,28 +31,28 @@ pip install -e .[demos]
 
 Environment variables (OpenAI-compatible API):
 
-- `MODEL` (optional; default: `gpt-4.1-mini`)
+- `MODEL` (optional)
 - `OPENAI_API_KEY` (required)
 - `OPENAI_BASE_URL` (optional; use for local or alternative endpoints)
 
-Example: locally hosted OpenAI-compatible endpoint (Ollama)
+Example: locally hosted OpenAI-compatible endpoint
 
-Any locally hosted OpenAI-compatible endpoint will work (for example Ollama, LM Studio, or a llama.cpp server).
+Any locally hosted OpenAI-compatible endpoint will work.
 
 ```bash
 export OPENAI_BASE_URL=http://localhost:11434/v1
 export OPENAI_API_KEY=ollama
-export MODEL=llama3.1:8b
+export MODEL=your_local_model_id
 ```
 
-OpenAI example:
+OpenAI-compatible hosted example:
 
 ```bash
 export OPENAI_API_KEY=your_key_here
-export MODEL=gpt-4.1-mini
+export MODEL=your_model_id
 ```
 
-## Run
+## Quick run
 
 Run a single demo:
 
@@ -70,15 +72,10 @@ Run all demos with detailed traces:
 uv run python -m demos.run_demo all --verbose
 ```
 
-Run demos with pacing for low-quota providers:
+## Provider throttling
 
-```bash
-uv run python -m demos.run_demo all --llm-delay 1.5
-```
-
-### Provider throttling
-
-The demos make multiple LLM requests and may trigger rate limits on very low-quota hosted providers (especially free tiers).
+The demos make multiple LLM requests and may trigger rate limits on very
+low-quota hosted providers (especially free tiers).
 
 If you encounter throttling, you can slow requests using:
 
@@ -118,3 +115,27 @@ Running against a local OpenAI-compatible endpoint avoids provider rate limits.
     - compiled context
     - baseline and compiled prompts
     - context and prompt size comparisons
+
+### Demo 5: stress ladder (`--turns`)
+
+For Demo 5, `--turns` controls how many distractor turns are inserted between
+the original directive and the final prompt.
+Longer runs are strict prefix extensions of shorter runs.
+
+Direct demo invocation:
+
+```bash
+uv run python demos/05_llm_prompt_drift.py --turns 10
+uv run python demos/05_llm_prompt_drift.py --turns 30
+uv run python demos/05_llm_prompt_drift.py --turns 60
+uv run python demos/05_llm_prompt_drift.py --turns 120
+uv run python demos/05_llm_prompt_drift.py --turns 240
+```
+
+Add `--llm-delay 1.25` if your provider throttles requests.
+
+Runner invocation (demo args after `--`):
+
+```bash
+uv run python -m demos.run_demo 5 --llm-delay 1.25 -- --turns 120
+```

demos/run_demo.py

@@ -46,18 +46,21 @@ def _print_compiler_regression_warning() -> None:
 
 
 def _run(
-    path: Path, *, verbose: bool, llm_delay: float
+    path: Path, *, verbose: bool, llm_delay: float, demo_args: list[str] | None = None
 ) -> tuple[DemoReport | None, InfoReport | None]:
     if verbose:
         print(f"===== Running {_verbose_demo_label(path)} =====")
     old_verbose = os.getenv(VERBOSE_ENV_VAR)
     old_delay = llm_client.DEFAULT_LLM_DELAY_SECONDS
+    old_argv = sys.argv[:]
     os.environ[VERBOSE_ENV_VAR] = "1" if verbose else "0"
     llm_client.DEFAULT_LLM_DELAY_SECONDS = llm_delay if llm_delay > 0 else 0.0
+    sys.argv = [str(path), *(demo_args or [])]
     try:
         runpy.run_path(str(path), run_name="__main__")
         return consume_last_report(), consume_last_info_report()
     finally:
+        sys.argv = old_argv
         if old_verbose is None:
             os.environ.pop(VERBOSE_ENV_VAR, None)
         else:
@@ -105,7 +108,14 @@ def main() -> None:
         default=0,
         help="Delay between LLM calls in seconds (useful for low-quota providers).",
     )
-    args = parser.parse_args()
+    argv = sys.argv[1:]
+    args, demo_args = parser.parse_known_args(argv)
+    if demo_args and demo_args[0] == "--":
+        demo_args = demo_args[1:]
+    if demo_args and "--" not in argv:
+        parser.error("demo-specific args must be passed after `--`")
+    if args.demo == "all" and demo_args:
+        parser.error("demo-specific args are only supported when running a single demo")
 
     if args.demo == "all":
         baseline_pass_count = 0
@@ -180,9 +190,13 @@ def main() -> None:
         return
 
     try:
-        result, _ = _run(
-            root / DEMO_FILES[args.demo], verbose=args.verbose, llm_delay=args.llm_delay
-        )
+        run_kwargs = {
+            "verbose": args.verbose,
+            "llm_delay": args.llm_delay,
+        }
+        if demo_args:
+            run_kwargs["demo_args"] = demo_args
+        result, _ = _run(root / DEMO_FILES[args.demo], **run_kwargs)
     except MissingDemoConfigError as exc:
         _print_config_error(exc)
         raise SystemExit(2) from exc