feat: add prompt engineering comparison demo

demos/07_llm_prompt_engineering_comparison.py

@@ -0,0 +1,188 @@
+"""Demo 7: prompt engineering and compiled state are complementary."""
+
+import re
+
+from context_compiler import State, create_engine
+from demos.common import (
+    build_baseline_messages,
+    build_compiled_system_prompt,
+    extract_tag_value,
+    print_decision,
+    print_host_check,
+    print_messages,
+    print_model_output,
+    print_spec_report,
+    print_user_inputs,
+    yes_no,
+)
+from demos.llm_client import Message, complete_messages
+
+DEMO_NAME = "07_prompt_engineering_comparison — prompt engineering + authoritative state"
+EXPECTED_FOCUS = "vegan curry"
+FINAL_REQUEST = (
+    "Give me a dinner plan. First line must be FOCUS_PRIMARY:<value>. "
+    "Use the current selected focus and then provide a short shopping list."
+)
+USER_INPUTS = [
+    "use vegan curry",
+    "Side note: I am planning a train trip and need camera advice later.",
+    "My coworkers mentioned chicken tikka and shrimp pasta in a brainstorm.",
+    "We also discussed weather apps and museum tickets for the weekend.",
+    (
+        "Draft notes from another thread said beef stew, but those notes may be stale "
+        "and mixed with unrelated chatter."
+    ),
+    FINAL_REQUEST,
+]
+
+WEAK_SYSTEM_PROMPT = "Be a helpful assistant."
+STRONG_PROMPT_ENGINEERING_TEXT = (
+    "You are a careful assistant.\n"
+    "Task: determine the user's current focus for this thread and answer the final request.\n"
+    "Rules:\n"
+    "1) Prioritize explicit user directives over brainstorm noise and side notes.\n"
+    "2) Keep the selected focus consistent across the response.\n"
+    "3) If multiple ideas appear, use the current selected focus instead of popularity.\n"
+    "4) First line must be exactly FOCUS_PRIMARY:<value>."
+)
+_WHITESPACE_RE = re.compile(r"\s+")
+
+
+def _normalize_text(value: str) -> str:
+    return _WHITESPACE_RE.sub(" ", value.strip().lower())
+
+
+def focus_matches_expected(output: str, expected_focus: str = EXPECTED_FOCUS) -> bool:
+    focus = extract_tag_value(output, "FOCUS_PRIMARY")
+    if focus is None:
+        return False
+    return _normalize_text(focus) == _normalize_text(expected_focus)
+
+
+def build_weak_messages(user_inputs: list[str]) -> list[Message]:
+    return build_baseline_messages(user_inputs, baseline_system_prompt=WEAK_SYSTEM_PROMPT)
+
+
+def build_strong_messages(user_inputs: list[str]) -> list[Message]:
+    return build_baseline_messages(
+        user_inputs,
+        baseline_system_prompt=STRONG_PROMPT_ENGINEERING_TEXT,
+    )
+
+
+def build_compiler_messages(state: State, user_inputs: list[str]) -> list[Message]:
+    compiled_prefix = build_compiled_system_prompt(state)
+    compiler_system_prompt = f"{compiled_prefix}\n{STRONG_PROMPT_ENGINEERING_TEXT}"
+    return build_baseline_messages(user_inputs, baseline_system_prompt=compiler_system_prompt)
+
+
+def _actual_summary(*, weak_pass: bool, strong_pass: bool, compiler_pass: bool) -> str:
+    if not weak_pass and strong_pass and compiler_pass:
+        return (
+            "basic prompting drifted, better prompting held the focus, and "
+            "prompting plus compiled state also held the focus"
+        )
+    if weak_pass and strong_pass and compiler_pass:
+        return "all three paths held the focus in this run"
+    if not strong_pass and compiler_pass:
+        return (
+            "better prompting alone drifted on focus, but prompting plus "
+            "compiled state held the authoritative focus"
+        )
+    if strong_pass and not compiler_pass:
+        return "better prompting held focus, but prompting plus compiled state did not"
+    return "focus handling was inconsistent across paths"
+
+
+def main() -> None:
+    engine = create_engine()
+    print_user_inputs(USER_INPUTS)
+
+    for index, user_input in enumerate(USER_INPUTS, start=1):
+        decision = engine.step(user_input)
+        print_decision(f"turn {index}", decision, engine.state)
+
+    weak_messages = build_weak_messages(USER_INPUTS)
+    print_messages("weak-baseline", weak_messages)
+    weak_output = complete_messages(weak_messages)
+    print_model_output("Weak baseline", weak_output)
+
+    strong_messages = build_strong_messages(USER_INPUTS)
+    print_messages("strong-baseline", strong_messages)
+    strong_output = complete_messages(strong_messages)
+    print_model_output("Strong baseline", strong_output)
+
+    compiler_messages = build_compiler_messages(engine.state, USER_INPUTS)
+    print_messages("compiler-mediated", compiler_messages)
+    compiler_output = complete_messages(compiler_messages)
+    print_model_output("Compiler-mediated", compiler_output)
+
+    weak_focus = extract_tag_value(weak_output, "FOCUS_PRIMARY")
+    strong_focus = extract_tag_value(strong_output, "FOCUS_PRIMARY")
+    compiler_focus = extract_tag_value(compiler_output, "FOCUS_PRIMARY")
+    weak_pass = focus_matches_expected(weak_output)
+    strong_pass = focus_matches_expected(strong_output)
+    compiler_pass = focus_matches_expected(compiler_output)
+
+    compiled_prefix = build_compiled_system_prompt(engine.state)
+    shared_prompt_text = compiler_messages[0]["content"].endswith(STRONG_PROMPT_ENGINEERING_TEXT)
+    compiler_augmented_only = (
+        compiler_messages[1:] == strong_messages[1:]
+        and compiler_messages[0]["content"]
+        == f"{compiled_prefix}\n{STRONG_PROMPT_ENGINEERING_TEXT}"
+    )
+    print_host_check(
+        "WEAK_MATCHES_EXPECTED_FOCUS",
+        f"{yes_no(weak_pass)}, focus_tag={weak_focus or 'MISSING'}",
+        context="weak-baseline",
+    )
+    print_host_check(
+        "STRONG_MATCHES_EXPECTED_FOCUS",
+        f"{yes_no(strong_pass)}, focus_tag={strong_focus or 'MISSING'}",
+        context="strong-baseline",
+    )
+    print_host_check(
+        "COMPILER_MATCHES_EXPECTED_FOCUS",
+        f"{yes_no(compiler_pass)}, focus_tag={compiler_focus or 'MISSING'}",
+        context="compiler-mediated",
+    )
+    print_host_check(
+        "COMPILER_REUSES_STRONG_PROMPT_TEXT",
+        yes_no(shared_prompt_text),
+        context="compiler-mediated",
+    )
+    print_host_check(
+        "COMPILER_ONLY_ADDS_COMPILED_STATE",
+        yes_no(compiler_augmented_only),
+        context="compiler-mediated",
+    )
+
+    demo_pass = (
+        (not weak_pass)
+        and strong_pass
+        and compiler_pass
+        and shared_prompt_text
+        and compiler_augmented_only
+    )
+    print_spec_report(
+        test_name=DEMO_NAME,
+        baseline_pass=strong_pass,
+        compiler_pass=compiler_pass,
+        expected=(
+            "prompting quality should help, and prompting plus compiled authoritative "
+            "state should be most reliable; compiler-mediated prompting should reuse "
+            "the same prompt text"
+        ),
+        actual=_actual_summary(
+            weak_pass=weak_pass,
+            strong_pass=strong_pass,
+            compiler_pass=compiler_pass,
+        ),
+        passed=demo_pass,
+        result_pass="prompting helps; authoritative compiled state adds reliability",
+        result_fail=("three-way complementarity claim not established in this run"),
+    )
+
+
+if __name__ == "__main__":
+    main()

demos/README.md

@@ -17,6 +17,7 @@ behavior is easy to see.
 | [04](./04_llm_tool_governance.py) | Tool governance | host-side denylist | general assistant models |
 | [05](./05_llm_prompt_drift.py) | Prompt drift | long transcript failure | weaker long-context models ([see Demo 5 note](#demo-5-stress-ladder-turns)) |
 | [06](./06_context_compaction.py) | Context compaction | compiled state replacing transcript | small or local models |
+| [07](./07_llm_prompt_engineering_comparison.py) | Prompt engineering comparison | prompting vs compiled state | any model with long transcript sensitivity |
 
 Stronger frontier models may show these behaviors less often, but the same
 patterns still appear in real applications.
@@ -89,7 +90,7 @@ Running against a local OpenAI-compatible endpoint avoids provider rate limits.
 
 - `Default (concise)`:
   - scenario name + description
-  - for evaluative demos (`01`–`05`):
+  - for evaluative demos (`01`–`05`, `07`):
     - `baseline: PASS|FAIL`
     - `compiler: PASS|FAIL`
   - expected behavior

demos/run_demo.py

@@ -26,9 +26,10 @@
     "4": "04_llm_tool_governance.py",
     "5": "05_llm_prompt_drift.py",
     "6": "06_context_compaction.py",
+    "7": "07_llm_prompt_engineering_comparison.py",
 }
 
-SCORED_DEMOS = {"1", "2", "3", "4", "5"}
+SCORED_DEMOS = {"1", "2", "3", "4", "5", "7"}
 
 
 def _verbose_demo_label(path: Path) -> str:
@@ -95,7 +96,7 @@ def main() -> None:
         nargs="?",
         default="all",
         choices=["all", *DEMO_FILES.keys()],
-        help="Demo number (1-6) or all",
+        help="Demo number (1-7) or all",
     )
     parser.add_argument(
         "--verbose",

tests/test_07_llm_prompt_engineering_comparison.py

@@ -0,0 +1,78 @@
+import importlib.util
+import sys
+from pathlib import Path
+from types import ModuleType
+
+import pytest
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+if str(REPO_ROOT) not in sys.path:
+    sys.path.insert(0, str(REPO_ROOT))
+
+from demos.common import consume_last_report  # noqa: E402
+
+
+def _load_demo_module(filename: str) -> ModuleType:
+    module_name = f"test_{filename[:-3]}"
+    module_path = REPO_ROOT / "demos" / filename
+    spec = importlib.util.spec_from_file_location(module_name, module_path)
+    assert spec is not None
+    assert spec.loader is not None
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+def test_weak_prompt_has_less_guidance_than_strong_prompt() -> None:
+    module = _load_demo_module("07_llm_prompt_engineering_comparison.py")
+
+    weak_prompt = module.WEAK_SYSTEM_PROMPT
+    strong_prompt = module.STRONG_PROMPT_ENGINEERING_TEXT
+
+    assert "Rules:" not in weak_prompt
+    assert "Rules:" in strong_prompt
+    assert len(strong_prompt) > len(weak_prompt)
+
+
+def test_compiler_path_reuses_same_strong_prompt_with_compiled_augmentation() -> None:
+    module = _load_demo_module("07_llm_prompt_engineering_comparison.py")
+    engine = module.create_engine()
+    for user_input in module.USER_INPUTS:
+        engine.step(user_input)
+
+    strong_messages = module.build_strong_messages(module.USER_INPUTS)
+    compiler_messages = module.build_compiler_messages(engine.state, module.USER_INPUTS)
+
+    compiled_prefix = module.build_compiled_system_prompt(engine.state)
+    assert strong_messages[1:] == compiler_messages[1:]
+    assert strong_messages[0]["content"] == module.STRONG_PROMPT_ENGINEERING_TEXT
+    assert compiler_messages[0]["content"] == (
+        f"{compiled_prefix}\n{module.STRONG_PROMPT_ENGINEERING_TEXT}"
+    )
+    assert compiler_messages[0]["content"].endswith(module.STRONG_PROMPT_ENGINEERING_TEXT)
+
+
+def test_demo_07_report_semantics_fail_when_weak_not_worse(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    module = _load_demo_module("07_llm_prompt_engineering_comparison.py")
+    outputs = iter(
+        [
+            ("FOCUS_PRIMARY: vegan curry\nShopping list:\n- tofu\n- coconut milk\n- curry paste"),
+            ("FOCUS_PRIMARY: vegan curry\nShopping list:\n- tofu\n- coconut milk\n- curry paste"),
+            ("FOCUS_PRIMARY: vegan curry\nShopping list:\n- tofu\n- coconut milk\n- curry paste"),
+        ]
+    )
+
+    def fake_complete_messages(_messages: list[dict[str, str]]) -> str:
+        return next(outputs)
+
+    monkeypatch.setattr(module, "complete_messages", fake_complete_messages)
+    consume_last_report()
+    module.main()
+    report = consume_last_report()
+    assert report is not None
+    assert report["name"] == module.DEMO_NAME
+    assert report["baseline_pass"] is True
+    assert report["compiler_pass"] is True
+    assert report["demo_pass"] is False

tests/test_llm_demos.py

@@ -176,6 +176,40 @@ def test_demo_05_prompt_drift(monkeypatch: pytest.MonkeyPatch) -> None:
     assert "facts.focus.primary: vegetarian curry" in calls[1][0]["content"]
 
 
+def test_demo_07_prompt_engineering_comparison(monkeypatch: pytest.MonkeyPatch) -> None:
+    module = _load_demo_module("07_llm_prompt_engineering_comparison.py")
+    report, calls = _run_demo_with_mocked_llm(
+        module,
+        monkeypatch,
+        outputs=[
+            (
+                "FOCUS_PRIMARY: shrimp pasta\n"
+                "Shopping list:\n- shrimp\n- pasta\n- cream\n"
+                "Steps:\n1. Cook."
+            ),
+            (
+                "FOCUS_PRIMARY: vegan curry\n"
+                "Shopping list:\n- tofu\n- coconut milk\n- curry paste\n"
+                "Steps:\n1. Cook."
+            ),
+            (
+                "FOCUS_PRIMARY: vegan curry\n"
+                "Shopping list:\n- tofu\n- coconut milk\n- curry paste\n"
+                "Steps:\n1. Cook."
+            ),
+        ],
+    )
+
+    assert len(calls) == 3
+    assert report["baseline_pass"] is True
+    assert report["compiler_pass"] is True
+    assert report["demo_pass"] is True
+    assert calls[1][0]["content"] == module.STRONG_PROMPT_ENGINEERING_TEXT
+    assert calls[2][0]["content"].endswith(module.STRONG_PROMPT_ENGINEERING_TEXT)
+    assert calls[1][1:] == calls[2][1:]
+    assert "facts.focus.primary: vegan curry" in calls[2][0]["content"]
+
+
 def test_demo_05_turns_support_ladder_and_keep_prompt_invariants() -> None:
     module = _load_demo_module("05_llm_prompt_drift.py")