From cf1e520da88e6de7858c5d2ae4990fd8348dedd6 Mon Sep 17 00:00:00 2001 From: Robert Lippmann Date: Wed, 18 Mar 2026 01:07:14 -0400 Subject: [PATCH] feat: add prompt engineering comparison demo --- demos/07_llm_prompt_engineering_comparison.py | 188 ++++++++++++++++++ demos/README.md | 3 +- demos/run_demo.py | 5 +- ...st_07_llm_prompt_engineering_comparison.py | 78 ++++++++ tests/test_llm_demos.py | 34 ++++ 5 files changed, 305 insertions(+), 3 deletions(-) create mode 100644 demos/07_llm_prompt_engineering_comparison.py create mode 100644 tests/test_07_llm_prompt_engineering_comparison.py diff --git a/demos/07_llm_prompt_engineering_comparison.py b/demos/07_llm_prompt_engineering_comparison.py new file mode 100644 index 0000000..eace211 --- /dev/null +++ b/demos/07_llm_prompt_engineering_comparison.py @@ -0,0 +1,188 @@ +"""Demo 7: prompt engineering and compiled state are complementary.""" + +import re + +from context_compiler import State, create_engine +from demos.common import ( + build_baseline_messages, + build_compiled_system_prompt, + extract_tag_value, + print_decision, + print_host_check, + print_messages, + print_model_output, + print_spec_report, + print_user_inputs, + yes_no, +) +from demos.llm_client import Message, complete_messages + +DEMO_NAME = "07_prompt_engineering_comparison — prompt engineering + authoritative state" +EXPECTED_FOCUS = "vegan curry" +FINAL_REQUEST = ( + "Give me a dinner plan. First line must be FOCUS_PRIMARY:. " + "Use the current selected focus and then provide a short shopping list." +) +USER_INPUTS = [ + "use vegan curry", + "Side note: I am planning a train trip and need camera advice later.", + "My coworkers mentioned chicken tikka and shrimp pasta in a brainstorm.", + "We also discussed weather apps and museum tickets for the weekend.", + ( + "Draft notes from another thread said beef stew, but those notes may be stale " + "and mixed with unrelated chatter." + ), + FINAL_REQUEST, +] + +WEAK_SYSTEM_PROMPT = "Be a helpful assistant." +STRONG_PROMPT_ENGINEERING_TEXT = ( + "You are a careful assistant.\n" + "Task: determine the user's current focus for this thread and answer the final request.\n" + "Rules:\n" + "1) Prioritize explicit user directives over brainstorm noise and side notes.\n" + "2) Keep the selected focus consistent across the response.\n" + "3) If multiple ideas appear, use the current selected focus instead of popularity.\n" + "4) First line must be exactly FOCUS_PRIMARY:." +) +_WHITESPACE_RE = re.compile(r"\s+") + + +def _normalize_text(value: str) -> str: + return _WHITESPACE_RE.sub(" ", value.strip().lower()) + + +def focus_matches_expected(output: str, expected_focus: str = EXPECTED_FOCUS) -> bool: + focus = extract_tag_value(output, "FOCUS_PRIMARY") + if focus is None: + return False + return _normalize_text(focus) == _normalize_text(expected_focus) + + +def build_weak_messages(user_inputs: list[str]) -> list[Message]: + return build_baseline_messages(user_inputs, baseline_system_prompt=WEAK_SYSTEM_PROMPT) + + +def build_strong_messages(user_inputs: list[str]) -> list[Message]: + return build_baseline_messages( + user_inputs, + baseline_system_prompt=STRONG_PROMPT_ENGINEERING_TEXT, + ) + + +def build_compiler_messages(state: State, user_inputs: list[str]) -> list[Message]: + compiled_prefix = build_compiled_system_prompt(state) + compiler_system_prompt = f"{compiled_prefix}\n{STRONG_PROMPT_ENGINEERING_TEXT}" + return build_baseline_messages(user_inputs, baseline_system_prompt=compiler_system_prompt) + + +def _actual_summary(*, weak_pass: bool, strong_pass: bool, compiler_pass: bool) -> str: + if not weak_pass and strong_pass and compiler_pass: + return ( + "basic prompting drifted, better prompting held the focus, and " + "prompting plus compiled state also held the focus" + ) + if weak_pass and strong_pass and compiler_pass: + return "all three paths held the focus in this run" + if not strong_pass and compiler_pass: + return ( + "better prompting alone drifted on focus, but prompting plus " + "compiled state held the authoritative focus" + ) + if strong_pass and not compiler_pass: + return "better prompting held focus, but prompting plus compiled state did not" + return "focus handling was inconsistent across paths" + + +def main() -> None: + engine = create_engine() + print_user_inputs(USER_INPUTS) + + for index, user_input in enumerate(USER_INPUTS, start=1): + decision = engine.step(user_input) + print_decision(f"turn {index}", decision, engine.state) + + weak_messages = build_weak_messages(USER_INPUTS) + print_messages("weak-baseline", weak_messages) + weak_output = complete_messages(weak_messages) + print_model_output("Weak baseline", weak_output) + + strong_messages = build_strong_messages(USER_INPUTS) + print_messages("strong-baseline", strong_messages) + strong_output = complete_messages(strong_messages) + print_model_output("Strong baseline", strong_output) + + compiler_messages = build_compiler_messages(engine.state, USER_INPUTS) + print_messages("compiler-mediated", compiler_messages) + compiler_output = complete_messages(compiler_messages) + print_model_output("Compiler-mediated", compiler_output) + + weak_focus = extract_tag_value(weak_output, "FOCUS_PRIMARY") + strong_focus = extract_tag_value(strong_output, "FOCUS_PRIMARY") + compiler_focus = extract_tag_value(compiler_output, "FOCUS_PRIMARY") + weak_pass = focus_matches_expected(weak_output) + strong_pass = focus_matches_expected(strong_output) + compiler_pass = focus_matches_expected(compiler_output) + + compiled_prefix = build_compiled_system_prompt(engine.state) + shared_prompt_text = compiler_messages[0]["content"].endswith(STRONG_PROMPT_ENGINEERING_TEXT) + compiler_augmented_only = ( + compiler_messages[1:] == strong_messages[1:] + and compiler_messages[0]["content"] + == f"{compiled_prefix}\n{STRONG_PROMPT_ENGINEERING_TEXT}" + ) + print_host_check( + "WEAK_MATCHES_EXPECTED_FOCUS", + f"{yes_no(weak_pass)}, focus_tag={weak_focus or 'MISSING'}", + context="weak-baseline", + ) + print_host_check( + "STRONG_MATCHES_EXPECTED_FOCUS", + f"{yes_no(strong_pass)}, focus_tag={strong_focus or 'MISSING'}", + context="strong-baseline", + ) + print_host_check( + "COMPILER_MATCHES_EXPECTED_FOCUS", + f"{yes_no(compiler_pass)}, focus_tag={compiler_focus or 'MISSING'}", + context="compiler-mediated", + ) + print_host_check( + "COMPILER_REUSES_STRONG_PROMPT_TEXT", + yes_no(shared_prompt_text), + context="compiler-mediated", + ) + print_host_check( + "COMPILER_ONLY_ADDS_COMPILED_STATE", + yes_no(compiler_augmented_only), + context="compiler-mediated", + ) + + demo_pass = ( + (not weak_pass) + and strong_pass + and compiler_pass + and shared_prompt_text + and compiler_augmented_only + ) + print_spec_report( + test_name=DEMO_NAME, + baseline_pass=strong_pass, + compiler_pass=compiler_pass, + expected=( + "prompting quality should help, and prompting plus compiled authoritative " + "state should be most reliable; compiler-mediated prompting should reuse " + "the same prompt text" + ), + actual=_actual_summary( + weak_pass=weak_pass, + strong_pass=strong_pass, + compiler_pass=compiler_pass, + ), + passed=demo_pass, + result_pass="prompting helps; authoritative compiled state adds reliability", + result_fail=("three-way complementarity claim not established in this run"), + ) + + +if __name__ == "__main__": + main() diff --git a/demos/README.md b/demos/README.md index 8f0711b..d0e5992 100644 --- a/demos/README.md +++ b/demos/README.md @@ -17,6 +17,7 @@ behavior is easy to see. | [04](./04_llm_tool_governance.py) | Tool governance | host-side denylist | general assistant models | | [05](./05_llm_prompt_drift.py) | Prompt drift | long transcript failure | weaker long-context models ([see Demo 5 note](#demo-5-stress-ladder-turns)) | | [06](./06_context_compaction.py) | Context compaction | compiled state replacing transcript | small or local models | +| [07](./07_llm_prompt_engineering_comparison.py) | Prompt engineering comparison | prompting vs compiled state | any model with long transcript sensitivity | Stronger frontier models may show these behaviors less often, but the same patterns still appear in real applications. @@ -89,7 +90,7 @@ Running against a local OpenAI-compatible endpoint avoids provider rate limits. - `Default (concise)`: - scenario name + description - - for evaluative demos (`01`–`05`): + - for evaluative demos (`01`–`05`, `07`): - `baseline: PASS|FAIL` - `compiler: PASS|FAIL` - expected behavior diff --git a/demos/run_demo.py b/demos/run_demo.py index 041af1e..2f24fdc 100644 --- a/demos/run_demo.py +++ b/demos/run_demo.py @@ -26,9 +26,10 @@ "4": "04_llm_tool_governance.py", "5": "05_llm_prompt_drift.py", "6": "06_context_compaction.py", + "7": "07_llm_prompt_engineering_comparison.py", } -SCORED_DEMOS = {"1", "2", "3", "4", "5"} +SCORED_DEMOS = {"1", "2", "3", "4", "5", "7"} def _verbose_demo_label(path: Path) -> str: @@ -95,7 +96,7 @@ def main() -> None: nargs="?", default="all", choices=["all", *DEMO_FILES.keys()], - help="Demo number (1-6) or all", + help="Demo number (1-7) or all", ) parser.add_argument( "--verbose", diff --git a/tests/test_07_llm_prompt_engineering_comparison.py b/tests/test_07_llm_prompt_engineering_comparison.py new file mode 100644 index 0000000..43684cf --- /dev/null +++ b/tests/test_07_llm_prompt_engineering_comparison.py @@ -0,0 +1,78 @@ +import importlib.util +import sys +from pathlib import Path +from types import ModuleType + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[1] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from demos.common import consume_last_report # noqa: E402 + + +def _load_demo_module(filename: str) -> ModuleType: + module_name = f"test_{filename[:-3]}" + module_path = REPO_ROOT / "demos" / filename + spec = importlib.util.spec_from_file_location(module_name, module_path) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_weak_prompt_has_less_guidance_than_strong_prompt() -> None: + module = _load_demo_module("07_llm_prompt_engineering_comparison.py") + + weak_prompt = module.WEAK_SYSTEM_PROMPT + strong_prompt = module.STRONG_PROMPT_ENGINEERING_TEXT + + assert "Rules:" not in weak_prompt + assert "Rules:" in strong_prompt + assert len(strong_prompt) > len(weak_prompt) + + +def test_compiler_path_reuses_same_strong_prompt_with_compiled_augmentation() -> None: + module = _load_demo_module("07_llm_prompt_engineering_comparison.py") + engine = module.create_engine() + for user_input in module.USER_INPUTS: + engine.step(user_input) + + strong_messages = module.build_strong_messages(module.USER_INPUTS) + compiler_messages = module.build_compiler_messages(engine.state, module.USER_INPUTS) + + compiled_prefix = module.build_compiled_system_prompt(engine.state) + assert strong_messages[1:] == compiler_messages[1:] + assert strong_messages[0]["content"] == module.STRONG_PROMPT_ENGINEERING_TEXT + assert compiler_messages[0]["content"] == ( + f"{compiled_prefix}\n{module.STRONG_PROMPT_ENGINEERING_TEXT}" + ) + assert compiler_messages[0]["content"].endswith(module.STRONG_PROMPT_ENGINEERING_TEXT) + + +def test_demo_07_report_semantics_fail_when_weak_not_worse( + monkeypatch: pytest.MonkeyPatch, +) -> None: + module = _load_demo_module("07_llm_prompt_engineering_comparison.py") + outputs = iter( + [ + ("FOCUS_PRIMARY: vegan curry\nShopping list:\n- tofu\n- coconut milk\n- curry paste"), + ("FOCUS_PRIMARY: vegan curry\nShopping list:\n- tofu\n- coconut milk\n- curry paste"), + ("FOCUS_PRIMARY: vegan curry\nShopping list:\n- tofu\n- coconut milk\n- curry paste"), + ] + ) + + def fake_complete_messages(_messages: list[dict[str, str]]) -> str: + return next(outputs) + + monkeypatch.setattr(module, "complete_messages", fake_complete_messages) + consume_last_report() + module.main() + report = consume_last_report() + assert report is not None + assert report["name"] == module.DEMO_NAME + assert report["baseline_pass"] is True + assert report["compiler_pass"] is True + assert report["demo_pass"] is False diff --git a/tests/test_llm_demos.py b/tests/test_llm_demos.py index 9efd3b5..4b88f30 100644 --- a/tests/test_llm_demos.py +++ b/tests/test_llm_demos.py @@ -176,6 +176,40 @@ def test_demo_05_prompt_drift(monkeypatch: pytest.MonkeyPatch) -> None: assert "facts.focus.primary: vegetarian curry" in calls[1][0]["content"] +def test_demo_07_prompt_engineering_comparison(monkeypatch: pytest.MonkeyPatch) -> None: + module = _load_demo_module("07_llm_prompt_engineering_comparison.py") + report, calls = _run_demo_with_mocked_llm( + module, + monkeypatch, + outputs=[ + ( + "FOCUS_PRIMARY: shrimp pasta\n" + "Shopping list:\n- shrimp\n- pasta\n- cream\n" + "Steps:\n1. Cook." + ), + ( + "FOCUS_PRIMARY: vegan curry\n" + "Shopping list:\n- tofu\n- coconut milk\n- curry paste\n" + "Steps:\n1. Cook." + ), + ( + "FOCUS_PRIMARY: vegan curry\n" + "Shopping list:\n- tofu\n- coconut milk\n- curry paste\n" + "Steps:\n1. Cook." + ), + ], + ) + + assert len(calls) == 3 + assert report["baseline_pass"] is True + assert report["compiler_pass"] is True + assert report["demo_pass"] is True + assert calls[1][0]["content"] == module.STRONG_PROMPT_ENGINEERING_TEXT + assert calls[2][0]["content"].endswith(module.STRONG_PROMPT_ENGINEERING_TEXT) + assert calls[1][1:] == calls[2][1:] + assert "facts.focus.primary: vegan curry" in calls[2][0]["content"] + + def test_demo_05_turns_support_ladder_and_keep_prompt_invariants() -> None: module = _load_demo_module("05_llm_prompt_drift.py")