diff --git a/README.md b/README.md index 56c9746..2a73b33 100644 --- a/README.md +++ b/README.md @@ -105,7 +105,7 @@ class Decision(TypedDict): Meaning: | kind | host behavior | -|-------------|-----------------------------------------------| +|:-----------:|-----------------------------------------------| | passthrough | forward user input to LLM | | update | forward input with updated state | | clarify | show `prompt_to_user` and do not call the LLM | diff --git a/demos/05_llm_prompt_drift.py b/demos/05_llm_prompt_drift.py index 508fa4e..620748a 100644 --- a/demos/05_llm_prompt_drift.py +++ b/demos/05_llm_prompt_drift.py @@ -1,7 +1,9 @@ """Demo 5: long transcript drift vs stable compiled state.""" +import argparse import re +import demos.llm_client as llm_client from context_compiler import create_engine from demos.common import ( build_baseline_messages, @@ -29,6 +31,81 @@ ) _NEGATION_RE = re.compile(r"\b(no|without|avoid|exclude|instead of)\b", flags=re.IGNORECASE) +_ORIGINAL_DIRECTIVE = "use vegetarian curry" +_FINAL_PROMPT = ( + "Now give me a dinner plan. First line must be DINNER_STYLE:." +) +_DISTRACTOR_TOPICS = [ + "travel photography", + "city walking routes", + "weekend train trips", + "mountain day hikes", + "pour-over coffee brewing", + "espresso dialing", + "architecture sketching", + "museum planning", + "weather map reading", + "atlas navigation", + "independent bookstores", + "historical nonfiction reading", + "film photography", + "macro photography", + "night sky viewing", + "rail station architecture", + "public transit maps", + "urban design tours", + "coastal trail planning", + "desert trail planning", + "baking crust hydration", + "pan sauce reduction", + "knife-skill practice", + "tea brewing", + "city museum circuits", +] +_DISTRACTOR_PROMPT_TEMPLATES = [ + "Quick question on {topic}: which beginner book gives solid fundamentals?", + "For {topic}, what common pitfall surprises newcomers?", + "In {topic}, which metric helps compare two options fairly?", + "How would you plan a one-day itinerary around {topic}?", + "For {topic}, what gear checklist keeps things practical?", + "In {topic}, what weather factor changes decisions the most?", + "What map detail matters most when preparing for {topic}?", + "For {topic}, which habit improves consistency over months?", + "How can someone budget for {topic} without losing quality?", + "For {topic}, what tradeoff appears between speed and accuracy?", + "What museum exhibit style pairs well with interest in {topic}?", + "For {topic}, which train route offers the most scenic segments?", +] + + +def _build_master_distractor_sequence() -> list[str]: + sequence = [ + # Keep these first two distractors byte-identical to the original demo. + "Also I like hiking and jazz.", + "What camera should I buy for travel?", + ] + for topic in _DISTRACTOR_TOPICS: + for template in _DISTRACTOR_PROMPT_TEMPLATES: + sequence.append(template.format(topic=topic)) + return sequence + + +_MASTER_DISTRACTOR_SEQUENCE = _build_master_distractor_sequence() +if len(_MASTER_DISTRACTOR_SEQUENCE) < 240: + raise RuntimeError("Demo 5 distractor sequence must support at least 240 turns.") + + +_LADDER_TURNS = [10, 30, 60, 120, 240] +_DEFAULT_TURNS = 2 + + +_ORIGINAL_DEFAULT_TRANSCRIPT = [ + _ORIGINAL_DIRECTIVE, + "Also I like hiking and jazz.", + "What camera should I buy for travel?", + _FINAL_PROMPT, +] + def _plan_lines(output: str) -> list[str]: lines = output.splitlines() @@ -62,14 +139,56 @@ def plan_includes_non_vegetarian_item(output: str) -> bool: return False -def main() -> None: +def _validate_turns(turns: int) -> None: + max_turns = len(_MASTER_DISTRACTOR_SEQUENCE) + if turns < 0: + raise ValueError("turns must be at least 0.") + if turns > max_turns: + raise ValueError(f"turns must be <= {max_turns}.") + + +def build_context_turns(turns: int) -> list[str]: + _validate_turns(turns) + return [_ORIGINAL_DIRECTIVE, *_MASTER_DISTRACTOR_SEQUENCE[:turns]] + + +def build_user_inputs(turns: int) -> list[str]: + return [*build_context_turns(turns), _FINAL_PROMPT] + + +def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: + max_turns = len(_MASTER_DISTRACTOR_SEQUENCE) + parser = argparse.ArgumentParser( + description=( + "Run Demo 5 with deterministic distractor distance for prompt-drift stress testing." + ) + ) + parser.add_argument( + "--turns", + type=int, + default=_DEFAULT_TURNS, + help=( + "Number of distractor turns between the original directive and final prompt " + f"(0-{max_turns}). Supports stress-test ladder points: " + f"{', '.join(map(str, _LADDER_TURNS))}." + ), + ) + parser.add_argument( + "--llm-delay", + type=float, + default=None, + help="Delay between LLM calls in seconds (overrides shared default when provided).", + ) + args = parser.parse_args(argv) + _validate_turns(args.turns) + return args + + +def _run_demo(turns: int = _DEFAULT_TURNS) -> None: engine = create_engine() - user_inputs = [ - "use vegetarian curry", - "Also I like hiking and jazz.", - "What camera should I buy for travel?", - "Now give me a dinner plan. First line must be DINNER_STYLE:.", - ] + user_inputs = build_user_inputs(turns) + if turns == _DEFAULT_TURNS and user_inputs != _ORIGINAL_DEFAULT_TRANSCRIPT: + raise RuntimeError("Demo 5 default transcript diverged from original behavior.") print_user_inputs(user_inputs) for index, user_input in enumerate(user_inputs, start=1): @@ -77,7 +196,7 @@ def main() -> None: print_decision(f"turn {index}", decision, engine.state) baseline_messages = build_baseline_messages( - [user_inputs[0], user_inputs[1], user_inputs[2], user_inputs[3]], + user_inputs, baseline_system_prompt=( "Be a helpful assistant. Use the conversation context to provide a useful answer." ), @@ -86,7 +205,7 @@ def main() -> None: baseline_output = complete_messages(baseline_messages) print_model_output("Baseline", baseline_output) - mediated_messages = build_mediated_messages(engine.state, user_inputs[3]) + mediated_messages = build_mediated_messages(engine.state, user_inputs[-1]) print_messages("compiler-mediated", mediated_messages) mediated_output = complete_messages(mediated_messages) print_model_output("Compiler-mediated", mediated_output) @@ -136,5 +255,17 @@ def main() -> None: ) +def main(turns: int = _DEFAULT_TURNS, llm_delay: float | None = None) -> None: + old_delay = llm_client.DEFAULT_LLM_DELAY_SECONDS + if llm_delay is not None: + llm_client.DEFAULT_LLM_DELAY_SECONDS = llm_delay if llm_delay > 0 else 0.0 + try: + _run_demo(turns=turns) + finally: + if llm_delay is not None: + llm_client.DEFAULT_LLM_DELAY_SECONDS = old_delay + + if __name__ == "__main__": - main() + cli_args = _parse_args() + main(turns=cli_args.turns, llm_delay=cli_args.llm_delay) diff --git a/demos/README.md b/demos/README.md index f29572e..8f0711b 100644 --- a/demos/README.md +++ b/demos/README.md @@ -1,23 +1,25 @@ # LLM Demos -These scripts compare baseline prompting vs compiler-mediated prompting using the -Context Compiler decision/state API. -They are illustrative manual demos, not benchmarks or CI tests. -They demonstrate common LLM failure modes and how authoritative compiled state -can improve reliability. -They also illustrate how a host application interprets compiler `Decision` outputs to control when the LLM is called. -All demos force deterministic decoding so results are reproducible. +These scripts show common ways LLM conversations can go wrong. + +They compare normal prompting with an approach where the application tracks +important instructions explicitly instead of relying only on the conversation +history. The scripts are designed to produce consistent results so the +behavior is easy to see. ## Demo overview -| Demo | Demonstrates | Concept | -| --- | --- | --- | -| [01](./01_llm_ambiguity_block.py) | Ambiguous directive blocking | clarification gate | -| [02](./02_llm_constraint_drift.py) | Constraint drift | persistent policy enforcement | -| [03](./03_llm_correction_replacement.py) | Correction replacement | exclusive fact semantics | -| [04](./04_llm_tool_governance.py) | Tool governance | host-side denylist | -| [05](./05_llm_prompt_drift.py) | Prompt drift | long transcript failure | -| [06](./06_context_compaction.py) | Context compaction | compiled state replacing transcript | +| Demo | Behavior | Concept | Most visible with | +| :--: | --- | :--: | --- | +| [01](./01_llm_ambiguity_block.py) | Ambiguous directive blocking | clarification gate | small instruct models | +| [02](./02_llm_constraint_drift.py) | Constraint drift | persistent policy enforcement | small or quantized models | +| [03](./03_llm_correction_replacement.py) | Correction replacement | exclusive fact semantics | models that summarize conversation | +| [04](./04_llm_tool_governance.py) | Tool governance | host-side denylist | general assistant models | +| [05](./05_llm_prompt_drift.py) | Prompt drift | long transcript failure | weaker long-context models ([see Demo 5 note](#demo-5-stress-ladder-turns)) | +| [06](./06_context_compaction.py) | Context compaction | compiled state replacing transcript | small or local models | + +Stronger frontier models may show these behaviors less often, but the same +patterns still appear in real applications. ## Requirements @@ -29,28 +31,28 @@ pip install -e .[demos] Environment variables (OpenAI-compatible API): -- `MODEL` (optional; default: `gpt-4.1-mini`) +- `MODEL` (optional) - `OPENAI_API_KEY` (required) - `OPENAI_BASE_URL` (optional; use for local or alternative endpoints) -Example: locally hosted OpenAI-compatible endpoint (Ollama) +Example: locally hosted OpenAI-compatible endpoint -Any locally hosted OpenAI-compatible endpoint will work (for example Ollama, LM Studio, or a llama.cpp server). +Any locally hosted OpenAI-compatible endpoint will work. ```bash export OPENAI_BASE_URL=http://localhost:11434/v1 export OPENAI_API_KEY=ollama -export MODEL=llama3.1:8b +export MODEL=your_local_model_id ``` -OpenAI example: +OpenAI-compatible hosted example: ```bash export OPENAI_API_KEY=your_key_here -export MODEL=gpt-4.1-mini +export MODEL=your_model_id ``` -## Run +## Quick run Run a single demo: @@ -70,15 +72,10 @@ Run all demos with detailed traces: uv run python -m demos.run_demo all --verbose ``` -Run demos with pacing for low-quota providers: +## Provider throttling -```bash -uv run python -m demos.run_demo all --llm-delay 1.5 -``` - -### Provider throttling - -The demos make multiple LLM requests and may trigger rate limits on very low-quota hosted providers (especially free tiers). +The demos make multiple LLM requests and may trigger rate limits on very +low-quota hosted providers (especially free tiers). If you encounter throttling, you can slow requests using: @@ -118,3 +115,27 @@ Running against a local OpenAI-compatible endpoint avoids provider rate limits. - compiled context - baseline and compiled prompts - context and prompt size comparisons + +### Demo 5: stress ladder (`--turns`) + +For Demo 5, `--turns` controls how many distractor turns are inserted between +the original directive and the final prompt. +Longer runs are strict prefix extensions of shorter runs. + +Direct demo invocation: + +```bash +uv run python demos/05_llm_prompt_drift.py --turns 10 +uv run python demos/05_llm_prompt_drift.py --turns 30 +uv run python demos/05_llm_prompt_drift.py --turns 60 +uv run python demos/05_llm_prompt_drift.py --turns 120 +uv run python demos/05_llm_prompt_drift.py --turns 240 +``` + +Add `--llm-delay 1.25` if your provider throttles requests. + +Runner invocation (demo args after `--`): + +```bash +uv run python -m demos.run_demo 5 --llm-delay 1.25 -- --turns 120 +``` diff --git a/demos/run_demo.py b/demos/run_demo.py index 8e5096f..041af1e 100644 --- a/demos/run_demo.py +++ b/demos/run_demo.py @@ -46,18 +46,21 @@ def _print_compiler_regression_warning() -> None: def _run( - path: Path, *, verbose: bool, llm_delay: float + path: Path, *, verbose: bool, llm_delay: float, demo_args: list[str] | None = None ) -> tuple[DemoReport | None, InfoReport | None]: if verbose: print(f"===== Running {_verbose_demo_label(path)} =====") old_verbose = os.getenv(VERBOSE_ENV_VAR) old_delay = llm_client.DEFAULT_LLM_DELAY_SECONDS + old_argv = sys.argv[:] os.environ[VERBOSE_ENV_VAR] = "1" if verbose else "0" llm_client.DEFAULT_LLM_DELAY_SECONDS = llm_delay if llm_delay > 0 else 0.0 + sys.argv = [str(path), *(demo_args or [])] try: runpy.run_path(str(path), run_name="__main__") return consume_last_report(), consume_last_info_report() finally: + sys.argv = old_argv if old_verbose is None: os.environ.pop(VERBOSE_ENV_VAR, None) else: @@ -105,7 +108,14 @@ def main() -> None: default=0, help="Delay between LLM calls in seconds (useful for low-quota providers).", ) - args = parser.parse_args() + argv = sys.argv[1:] + args, demo_args = parser.parse_known_args(argv) + if demo_args and demo_args[0] == "--": + demo_args = demo_args[1:] + if demo_args and "--" not in argv: + parser.error("demo-specific args must be passed after `--`") + if args.demo == "all" and demo_args: + parser.error("demo-specific args are only supported when running a single demo") if args.demo == "all": baseline_pass_count = 0 @@ -180,9 +190,13 @@ def main() -> None: return try: - result, _ = _run( - root / DEMO_FILES[args.demo], verbose=args.verbose, llm_delay=args.llm_delay - ) + run_kwargs = { + "verbose": args.verbose, + "llm_delay": args.llm_delay, + } + if demo_args: + run_kwargs["demo_args"] = demo_args + result, _ = _run(root / DEMO_FILES[args.demo], **run_kwargs) except MissingDemoConfigError as exc: _print_config_error(exc) raise SystemExit(2) from exc diff --git a/tests/test_llm_demos.py b/tests/test_llm_demos.py index eccade9..9efd3b5 100644 --- a/tests/test_llm_demos.py +++ b/tests/test_llm_demos.py @@ -176,6 +176,47 @@ def test_demo_05_prompt_drift(monkeypatch: pytest.MonkeyPatch) -> None: assert "facts.focus.primary: vegetarian curry" in calls[1][0]["content"] +def test_demo_05_turns_support_ladder_and_keep_prompt_invariants() -> None: + module = _load_demo_module("05_llm_prompt_drift.py") + + default_inputs = module.build_user_inputs(module._DEFAULT_TURNS) + assert default_inputs == [ + "use vegetarian curry", + "Also I like hiking and jazz.", + "What camera should I buy for travel?", + "Now give me a dinner plan. First line must be DINNER_STYLE:.", + ] + assert module._MASTER_DISTRACTOR_SEQUENCE[:2] == [ + "Also I like hiking and jazz.", + "What camera should I buy for travel?", + ] + + ladder = [10, 30, 60, 120, 240] + for turns in ladder: + module._validate_turns(turns) + inputs = module.build_user_inputs(turns) + assert len(inputs) == turns + 2 + assert inputs[-1] == module._FINAL_PROMPT + + for short, long in zip(ladder, ladder[1:], strict=False): + short_context = module.build_context_turns(short) + long_context = module.build_context_turns(long) + assert long_context[: len(short_context)] == short_context + assert len(long_context) > len(short_context) + + turns_120 = module.build_user_inputs(120) + assert turns_120[1:121] == module._MASTER_DISTRACTOR_SEQUENCE[:120] + + +def test_demo_05_cli_parses_shared_llm_delay_and_turns() -> None: + module = _load_demo_module("05_llm_prompt_drift.py") + + args = module._parse_args(["--llm-delay", "1.25", "--turns", "120"]) + + assert args.llm_delay == 1.25 + assert args.turns == 120 + + def test_demo_02_negation_and_refusal_lines_do_not_count_as_violations() -> None: module = _load_demo_module("02_llm_constraint_drift.py") negated_recipe = "Ingredients:\n- no peanuts\n- coconut milk" diff --git a/tests/test_run_demo.py b/tests/test_run_demo.py index 0293443..b98c489 100644 --- a/tests/test_run_demo.py +++ b/tests/test_run_demo.py @@ -311,3 +311,35 @@ def fake_run( run_demo.main() assert captured["llm_delay"] == 1.25 + + +def test_runner_forwards_demo_specific_args_after_separator( + monkeypatch: pytest.MonkeyPatch, +) -> None: + captured: dict[str, object] = {} + + def fake_run( + path: Path, + *, + verbose: bool, + llm_delay: float, + demo_args: list[str] | None = None, + ) -> tuple[run_demo.DemoReport | None, run_demo.InfoReport | None]: + assert path.name == "fake_05.py" + assert not verbose + captured["llm_delay"] = llm_delay + captured["demo_args"] = demo_args + return _demo_report(baseline_pass=True, compiler_pass=True), None + + monkeypatch.setattr(run_demo, "DEMO_FILES", {"5": "fake_05.py"}) + monkeypatch.setattr(run_demo, "SCORED_DEMOS", {"5"}) + monkeypatch.setattr(run_demo, "_run", fake_run) + monkeypatch.setattr( + "sys.argv", + ["run_demo", "5", "--llm-delay", "1.25", "--", "--turns", "120"], + ) + + run_demo.main() + + assert captured["llm_delay"] == 1.25 + assert captured["demo_args"] == ["--turns", "120"]