From 2422f883973522711754435dc8562184f6ae49fd Mon Sep 17 00:00:00 2001 From: Robert Lippmann Date: Mon, 25 May 2026 00:09:07 -0400 Subject: [PATCH 1/2] test: add state-transition semantic demos --- demos/08_llm_replacement_precondition.py | 137 +++++++++++++++++++ demos/09_llm_pending_clarification.py | 167 +++++++++++++++++++++++ demos/README.md | 6 +- demos/run_demo.py | 6 +- docs/demos-results.md | 6 + tests/test_demo_08_09_behavior.py | 162 ++++++++++++++++++++++ tests/test_run_demo.py | 2 + 7 files changed, 482 insertions(+), 4 deletions(-) create mode 100644 demos/08_llm_replacement_precondition.py create mode 100644 demos/09_llm_pending_clarification.py create mode 100644 tests/test_demo_08_09_behavior.py diff --git a/demos/08_llm_replacement_precondition.py b/demos/08_llm_replacement_precondition.py new file mode 100644 index 0000000..254c7ed --- /dev/null +++ b/demos/08_llm_replacement_precondition.py @@ -0,0 +1,137 @@ +"""Demo 8: replacement precondition is enforced by authoritative state.""" + +from context_compiler import DECISION_CLARIFY, State, create_engine +from demos.common import ( + build_baseline_messages, + build_reinjected_messages, + compact_user_turns, + print_decision, + print_host_check, + print_messages, + print_model_output, + print_spec_report, + print_user_inputs, + yes_no, +) +from demos.llm_client import complete_messages + +DEMO_NAME = "08_replacement_precondition — invalid replacement blocked" +USER_INPUT = "use podman instead of docker" + + +def _is_initial_authoritative_state(state: State) -> bool: + return state == {"premise": None, "policies": {}, "version": 2} + + +def main() -> None: + engine = create_engine() + user_inputs = [USER_INPUT] + print_user_inputs(user_inputs) + + decision = engine.step(USER_INPUT) + print_decision("turn 1", decision, engine.state) + + baseline_messages = build_baseline_messages( + [ + ( + "Analyze this input as state transition logic: 'use podman instead of docker'. " + "First line must be ACTION:." + ) + ], + baseline_system_prompt=( + "Be helpful and plausible. If an action seems ambiguous, make a reasonable guess." + ), + ) + print_messages("baseline", baseline_messages) + baseline_output = complete_messages(baseline_messages) + print_model_output("Baseline", baseline_output) + + _, reinjected_messages = build_reinjected_messages( + [ + ( + "Analyze this input as state transition logic: 'use podman instead of docker'. " + "First line must be ACTION:." + ) + ], + premise=None, + use_policies=[], + prohibit_policies=[], + ) + print_messages("reinjected-state", reinjected_messages) + reinjected_output = complete_messages(reinjected_messages) + print_model_output("Reinjected-state", reinjected_output) + + if decision["kind"] == DECISION_CLARIFY: + print_messages("compiler-mediated (full)", []) + mediated_output = f"[no call] clarification required: {decision['prompt_to_user']}" + print_model_output("Compiler-mediated (full)", mediated_output) + else: + print_messages("compiler-mediated (full)", []) + mediated_output = "[no call] expected clarify was not produced" + print_model_output("Compiler-mediated (full)", mediated_output) + + compacted_turns, compacted_state, compacted_prompt = compact_user_turns(user_inputs) + if compacted_prompt is not None: + print_messages("compiler-mediated + compact", []) + compact_output = f"[no call] clarification required: {compacted_prompt}" + print_model_output("Compiler-mediated + compact", compact_output) + else: + print_messages("compiler-mediated + compact", []) + compact_output = "[no call] expected clarify was not produced" + print_model_output("Compiler-mediated + compact", compact_output) + + state_unchanged = _is_initial_authoritative_state(engine.state) + compact_state_unchanged = _is_initial_authoritative_state(compacted_state) + no_pending = engine.has_pending_clarification() + compact_pending = compacted_prompt is not None + + baseline_has_authoritative_precondition = False + reinjected_has_authoritative_precondition = False + compiler_pass = decision["kind"] == DECISION_CLARIFY and state_unchanged and no_pending + compact_pass = compacted_prompt is not None and compact_state_unchanged and compact_pending + + print_host_check( + "BASELINE_AUTHORITATIVE_PRECONDITION", + yes_no(baseline_has_authoritative_precondition), + context="baseline", + ) + print_host_check( + "REINJECTED_AUTHORITATIVE_PRECONDITION", + yes_no(reinjected_has_authoritative_precondition), + context="reinjected-state", + ) + print_host_check( + "COMPILER_BLOCKED_INVALID_REPLACEMENT", + yes_no(decision["kind"] == DECISION_CLARIFY), + context="compiler-mediated", + ) + print_host_check( + "COMPILER_STATE_UNCHANGED", + yes_no(state_unchanged), + context="compiler-mediated", + ) + + print_spec_report( + test_name=DEMO_NAME, + baseline_pass=baseline_has_authoritative_precondition, + reinjected_state_pass=reinjected_has_authoritative_precondition, + compiler_pass=compiler_pass, + compiler_compact_pass=compact_pass, + expected=( + "invalid replacement should be blocked with clarification and no authoritative state " + "mutation" + ), + actual=( + "compiler blocked invalid replacement and preserved state; baseline and reinjected " + "paths have no authoritative replacement precondition" + if compiler_pass and compact_pass + else "compiler did not consistently enforce replacement precondition behavior" + ), + passed=compiler_pass and compact_pass, + result_pass="invalid replacement precondition enforced deterministically", + result_fail="invalid replacement precondition not enforced deterministically", + ) + + +if __name__ == "__main__": + main() diff --git a/demos/09_llm_pending_clarification.py b/demos/09_llm_pending_clarification.py new file mode 100644 index 0000000..f1076e3 --- /dev/null +++ b/demos/09_llm_pending_clarification.py @@ -0,0 +1,167 @@ +"""Demo 9: pending clarification requires confirmation-only continuation.""" + +from context_compiler import DECISION_CLARIFY, DECISION_UPDATE, State, create_engine +from demos.common import ( + build_baseline_messages, + build_reinjected_messages, + compact_user_turns, + print_decision, + print_host_check, + print_messages, + print_model_output, + print_spec_report, + print_user_inputs, + yes_no, +) +from demos.llm_client import complete_messages + +DEMO_NAME = "09_pending_clarification_continuation — confirmation-only state transition" +TURN_1 = "use podman instead of docker" +TURN_2 = "maybe" +TURN_3 = "yes" + + +def _has_podman_use(state: State) -> bool: + policies = state.get("policies") + if not isinstance(policies, dict): + return False + return policies.get("podman") == "use" + + +def _is_initial_authoritative_state(state: State) -> bool: + return state == {"premise": None, "policies": {}, "version": 2} + + +def main() -> None: + engine = create_engine() + user_inputs = [TURN_1, TURN_2, TURN_3] + print_user_inputs(user_inputs) + + first = engine.step(TURN_1) + print_decision("turn 1", first, engine.state) + pending_after_first = engine.has_pending_clarification() + state_unchanged_after_first = _is_initial_authoritative_state(engine.state) + + second = engine.step(TURN_2) + print_decision("turn 2", second, engine.state) + pending_after_second = engine.has_pending_clarification() + state_unchanged_after_second = _is_initial_authoritative_state(engine.state) + + third = engine.step(TURN_3) + print_decision("turn 3", third, engine.state) + pending_after_third = engine.has_pending_clarification() + + baseline_messages = build_baseline_messages( + [ + ( + "Conversation: user says 'use podman instead of docker', then 'maybe', then 'yes'. " + "First line must be STATE_MACHINE:." + ) + ], + baseline_system_prompt="Be helpful and produce a plausible interpretation.", + ) + print_messages("baseline", baseline_messages) + baseline_output = complete_messages(baseline_messages) + print_model_output("Baseline", baseline_output) + + _, reinjected_messages = build_reinjected_messages( + [ + ( + "Conversation: user says 'use podman instead of docker', then 'maybe', then 'yes'. " + "First line must be STATE_MACHINE:." + ) + ], + premise=None, + use_policies=[], + prohibit_policies=[], + ) + print_messages("reinjected-state", reinjected_messages) + reinjected_output = complete_messages(reinjected_messages) + print_model_output("Reinjected-state", reinjected_output) + + print_messages("compiler-mediated (full)", []) + mediated_output = "[no call] host-side state machine checked directly" + print_model_output("Compiler-mediated (full)", mediated_output) + + compacted_turns, compacted_state, compacted_prompt = compact_user_turns(user_inputs) + if compacted_prompt is not None: + print_messages("compiler-mediated + compact", []) + compact_output = f"[no call] clarification required: {compacted_prompt}" + print_model_output("Compiler-mediated + compact", compact_output) + else: + print_messages("compiler-mediated + compact", []) + compact_output = "[no call] compact replay resolved pending flow" + print_model_output("Compiler-mediated + compact", compact_output) + + blocked_initial_mutation = first["kind"] == DECISION_CLARIFY and state_unchanged_after_first + pending_preserved_on_nonconfirm = ( + second["kind"] == DECISION_CLARIFY + and pending_after_first + and pending_after_second + and state_unchanged_after_first + and state_unchanged_after_second + ) + confirmation_only_resolution = third["kind"] == DECISION_UPDATE and not pending_after_third + deterministic_final_state = _has_podman_use(engine.state) + + baseline_has_pending_state_machine = False + reinjected_has_pending_state_machine = False + + compiler_pass = ( + blocked_initial_mutation + and pending_preserved_on_nonconfirm + and confirmation_only_resolution + and deterministic_final_state + ) + + compact_pass = ( + compacted_prompt is not None + and compacted_turns == [TURN_1] + and _is_initial_authoritative_state(compacted_state) + ) + + print_host_check( + "BLOCKED_INITIAL_MUTATION", + yes_no(blocked_initial_mutation), + context="compiler-mediated", + ) + print_host_check( + "PENDING_PRESERVED_ON_NONCONFIRM", + yes_no(pending_preserved_on_nonconfirm), + context="compiler-mediated", + ) + print_host_check( + "CONFIRMATION_ONLY_RESOLUTION", + yes_no(confirmation_only_resolution), + context="compiler-mediated", + ) + print_host_check( + "FINAL_POLICY_PODMAN_USE", + yes_no(deterministic_final_state), + context="compiler-mediated", + ) + + print_spec_report( + test_name=DEMO_NAME, + baseline_pass=baseline_has_pending_state_machine, + reinjected_state_pass=reinjected_has_pending_state_machine, + compiler_pass=compiler_pass, + compiler_compact_pass=compact_pass, + expected=( + "replacement flow should require pending clarification, preserve pending on " + "non-confirmation, and apply only on confirmation" + ), + actual=( + "compiler enforced blocked mutation, pending continuation, and confirmation-only " + "resolution to final podman policy" + if compiler_pass and compact_pass + else "compiler did not consistently enforce pending clarification continuation" + ), + passed=compiler_pass and compact_pass, + result_pass="pending clarification continuation enforced deterministically", + result_fail="pending clarification continuation not enforced deterministically", + ) + + +if __name__ == "__main__": + main() diff --git a/demos/README.md b/demos/README.md index 16b35b2..bff4daf 100644 --- a/demos/README.md +++ b/demos/README.md @@ -26,6 +26,8 @@ Scored demos now compare four paths: | [05](./05_llm_prompt_drift_vs_state.py) | Prompt drift | long transcript failure | weaker long-context models ([see Demo 5 note](#demo-5-stress-ladder-turns)) | | [06](./06_llm_context_compaction.py) | Context compaction | saved compiler state replacing transcript context | small or local models | | [07](./07_llm_prompt_vs_state.py) | Prompt engineering comparison | prompting vs saved compiler state | any model with long transcript sensitivity | +| [08](./08_llm_replacement_precondition.py) | Replacement precondition | invalid replacement blocked without state mutation | any model | +| [09](./09_llm_pending_clarification.py) | Pending clarification continuation | confirmation-only resolution of suspended mutation | any model | Stronger frontier models may show these behaviors less often, but the same patterns still appear in real applications. @@ -142,7 +144,7 @@ is unavailable. The canonical cross-model results matrix is maintained in [docs/demos-results.md](../docs/demos-results.md). Notes: -- There are **6 scored demos** (`01`–`05`, `07`). `06_context_compaction` is informational and excluded from PASS/FAIL totals. +- There are **8 scored demos** (`01`–`05`, `07`, `08`, `09`). `06_context_compaction` is informational and excluded from PASS/FAIL totals. - Anthropic runs in this repo are executed through the `openai_compatible` provider path. - `PASS` means the demo-specific expected-behavior check for that path succeeded; `FAIL` means it did not. - `reinjected-state` can be enough for some persistence cases; this comparison is intended to show where deterministic state semantics add value. @@ -186,7 +188,7 @@ Running against a local OpenAI-compatible endpoint avoids provider rate limits. - `Default (concise)`: - scenario name + description - - for evaluative demos (`01`–`05`, `07`): + - for evaluative demos (`01`–`05`, `07`, `08`, `09`): - `baseline: PASS|FAIL` - `reinjected-state: PASS|FAIL` - `compiler: PASS|FAIL` diff --git a/demos/run_demo.py b/demos/run_demo.py index 9425874..ebb3957 100644 --- a/demos/run_demo.py +++ b/demos/run_demo.py @@ -27,9 +27,11 @@ "5": "05_llm_prompt_drift_vs_state.py", "6": "06_llm_context_compaction.py", "7": "07_llm_prompt_vs_state.py", + "8": "08_llm_replacement_precondition.py", + "9": "09_llm_pending_clarification.py", } -SCORED_DEMOS = {"1", "2", "3", "4", "5", "7"} +SCORED_DEMOS = {"1", "2", "3", "4", "5", "7", "8", "9"} def _preflight_all_mode(*, context_size: int | None = None) -> None: @@ -118,7 +120,7 @@ def main() -> None: nargs="?", default="all", choices=["all", *DEMO_FILES.keys()], - help="Demo number (1-7) or all", + help="Demo number (1-9) or all", ) parser.add_argument( "--verbose", diff --git a/docs/demos-results.md b/docs/demos-results.md index 90275e8..ab72b6c 100644 --- a/docs/demos-results.md +++ b/docs/demos-results.md @@ -9,6 +9,10 @@ Note: this published matrix predates the `reinjected-state` path added in the 0. - Scored demos: `01`, `02`, `03`, `04`, `05`, `07` (6 total) - Informational demo: `06_context_compaction` (excluded from PASS/FAIL totals) +Methodology note (2026-05): the demo suite now also includes scored state-transition +semantics demos `08` and `09`. The published matrix below predates those additions and +has not yet been fully rerun with the expanded scored set. + ## Results Matrix | Provider Path | Model | Baseline (P/F) | Compiler (P/F) | Compiler+Compact (P/F) | @@ -59,6 +63,8 @@ Scoring behavior uses post-audit oracle/checker logic in demos and shared helper - `demos/04_llm_tool_denylist_guardrail.py` - `demos/05_llm_prompt_drift_vs_state.py` - `demos/07_llm_prompt_vs_state.py` +- `demos/08_llm_replacement_precondition.py` +- `demos/09_llm_pending_clarification.py` - shared parsing/helpers in `demos/common.py` ### Run metadata diff --git a/tests/test_demo_08_09_behavior.py b/tests/test_demo_08_09_behavior.py new file mode 100644 index 0000000..5358399 --- /dev/null +++ b/tests/test_demo_08_09_behavior.py @@ -0,0 +1,162 @@ +import importlib.util +import sys +from collections.abc import Callable +from pathlib import Path +from types import ModuleType + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[1] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from demos.common import consume_last_report # noqa: E402 + + +def _load_demo_module(filename: str) -> ModuleType: + module_name = f"test_demo_behavior_{filename[:-3]}" + module_path = REPO_ROOT / "demos" / filename + spec = importlib.util.spec_from_file_location(module_name, module_path) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _sequenced_outputs(outputs: list[str]) -> Callable[[object], str]: + queue = list(outputs) + + def _fake_complete_messages(_messages: object) -> str: + if not queue: + raise AssertionError("No mocked LLM output remaining for this call.") + return queue.pop(0) + + return _fake_complete_messages + + +def test_demo_08_reports_invalid_replacement_block_and_unchanged_state( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + module = _load_demo_module("08_llm_replacement_precondition.py") + monkeypatch.setattr( + module, + "complete_messages", + _sequenced_outputs( + [ + "ACTION:proceed\nLooks fine.", + "ACTION:proceed\nSeems acceptable.", + ] + ), + ) + + module.main() + output = capsys.readouterr().out + report = consume_last_report() + + assert report is not None + assert report["name"].startswith("08_replacement_precondition") + assert report["baseline_pass"] is False + assert report["reinjected_state_pass"] is False + assert report["compiler_pass"] is True + assert report["compiler_compact_pass"] is True + assert report["demo_pass"] is True + assert "baseline: FAIL" in output + assert "reinjected-state: FAIL" in output + assert "compiler: PASS" in output + + +def test_demo_08_reinjected_path_does_not_instantiate_engine( + monkeypatch: pytest.MonkeyPatch, +) -> None: + module = _load_demo_module("08_llm_replacement_precondition.py") + + original_create_engine = module.create_engine + + class _EngineWrapper: + reinjected_seen = False + + def __init__(self, inner: object) -> None: + self._inner = inner + + def __getattr__(self, name: str) -> object: + return getattr(self._inner, name) + + def step(self, text: str) -> dict[str, object]: + if text == module.USER_INPUT: + _EngineWrapper.reinjected_seen = False + return self._inner.step(text) + + engine = _EngineWrapper(original_create_engine()) + + monkeypatch.setattr(module, "create_engine", lambda: engine) + + original_build_reinjected_messages = module.build_reinjected_messages + + def wrapped_build_reinjected_messages(*args: object, **kwargs: object): + _EngineWrapper.reinjected_seen = True + return original_build_reinjected_messages(*args, **kwargs) + + monkeypatch.setattr(module, "build_reinjected_messages", wrapped_build_reinjected_messages) + monkeypatch.setattr(module, "complete_messages", _sequenced_outputs(["x", "y"])) + + module.main() + report = consume_last_report() + + assert _EngineWrapper.reinjected_seen is True + assert report is not None + assert report["reinjected_state_pass"] is False + + +def test_demo_09_reports_pending_clarification_continuation( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + module = _load_demo_module("09_llm_pending_clarification.py") + monkeypatch.setattr( + module, + "complete_messages", + _sequenced_outputs( + [ + "STATE_MACHINE:plausible\nNarrative only.", + "STATE_MACHINE:plausible\nNarrative only.", + ] + ), + ) + + module.main() + output = capsys.readouterr().out + report = consume_last_report() + + assert report is not None + assert report["name"].startswith("09_pending_clarification_continuation") + assert report["baseline_pass"] is False + assert report["reinjected_state_pass"] is False + assert report["compiler_pass"] is True + assert report["compiler_compact_pass"] is True + assert report["demo_pass"] is True + assert "reinjected-state: FAIL" in output + assert "compiler: PASS" in output + + +def test_demo_09_reinjected_path_does_not_call_create_engine( + monkeypatch: pytest.MonkeyPatch, +) -> None: + module = _load_demo_module("09_llm_pending_clarification.py") + + original_create_engine = module.create_engine + create_engine_calls = 0 + + def wrapped_create_engine() -> object: + nonlocal create_engine_calls + create_engine_calls += 1 + return original_create_engine() + + monkeypatch.setattr(module, "create_engine", wrapped_create_engine) + monkeypatch.setattr(module, "complete_messages", _sequenced_outputs(["x", "y"])) + + module.main() + report = consume_last_report() + + assert create_engine_calls == 1 + assert report is not None + assert report["reinjected_state_pass"] is False diff --git a/tests/test_run_demo.py b/tests/test_run_demo.py index 20de305..0ec4397 100644 --- a/tests/test_run_demo.py +++ b/tests/test_run_demo.py @@ -57,6 +57,8 @@ def test_demo_file_mapping_uses_current_0_5_demo_filenames() -> None: "5": "05_llm_prompt_drift_vs_state.py", "6": "06_llm_context_compaction.py", "7": "07_llm_prompt_vs_state.py", + "8": "08_llm_replacement_precondition.py", + "9": "09_llm_pending_clarification.py", } demos_dir = REPO_ROOT / "demos" From 0b6a31a14a1d2198436e35d5fc2dde5b1b913660 Mon Sep 17 00:00:00 2001 From: Robert Lippmann Date: Mon, 25 May 2026 00:56:28 -0400 Subject: [PATCH 2/2] docs: clarify demo interpretation wording --- README.md | 6 ++++++ demos/README.md | 3 +++ docs/demos-results.md | 9 +++++++++ 3 files changed, 18 insertions(+) diff --git a/README.md b/README.md index 2873c02..6c8c113 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,12 @@ Yes, on the current scored demo set. - Scored checks (**6 demos per model**; Demo 6 excluded): baseline **26 / 42**, compiler **42 / 42**, compiler+compact **42 / 42**. - Across tested models, compiler-mediated paths pass all scored scenarios; baseline behavior is model-dependent. +Interpretation guide: +- Persistence/policy-following demos test whether instructions keep applying across turns. +- State-transition demos (`08`/`09`) test whether the host enforces state changes in a fixed, repeatable way. +- Demos `08`/`09` are capability checks, not general model-quality rankings. +- In those demos, baseline or reinjected-state can sound reasonable and still `FAIL` because required host-side transition checks are not present. + → [Full results and demo output](demos/README.md) Canonical matrix: [docs/demos-results.md](docs/demos-results.md) diff --git a/demos/README.md b/demos/README.md index bff4daf..14e63c5 100644 --- a/demos/README.md +++ b/demos/README.md @@ -148,6 +148,9 @@ Notes: - Anthropic runs in this repo are executed through the `openai_compatible` provider path. - `PASS` means the demo-specific expected-behavior check for that path succeeded; `FAIL` means it did not. - `reinjected-state` can be enough for some persistence cases; this comparison is intended to show where deterministic state semantics add value. +- Anti-overfitting guardrail: demos score host/state-transition invariants (for example blocked mutation, pending continuation, confirmation-only resolution), not model prose quality. Reinjected-state remains plain text injection only and does not include hidden compiler semantics. +- Interpretation: demos `01`-`05` and `07` focus on persistence and policy-following across turns, while demos `08`/`09` focus on fixed host-side state-transition rules. +- For demos `08`/`09`, similar outcomes across models are about architecture limits, not provider/model leaderboard ranking. ### Demo 05 example (prompt drift under longer context) diff --git a/docs/demos-results.md b/docs/demos-results.md index ab72b6c..e0478ac 100644 --- a/docs/demos-results.md +++ b/docs/demos-results.md @@ -67,6 +67,11 @@ Scoring behavior uses post-audit oracle/checker logic in demos and shared helper - `demos/09_llm_pending_clarification.py` - shared parsing/helpers in `demos/common.py` +Anti-overfitting note: scored checks are designed around deterministic host/state +invariants rather than preferred wording in model outputs. The `reinjected-state` +path is intentionally limited to plain state text injection and does not include +compiler precondition or pending-state semantics. + ### Run metadata - Date: 2026-05-06 @@ -78,6 +83,10 @@ Scoring behavior uses post-audit oracle/checker logic in demos and shared helper - Live demo runs are **evidence/smoke tests** across real model/provider behavior. - Deterministic test suites (unit/property tests) are the **regression authority** for oracle and engine contracts. +- Persistence demos and transition-semantics demos should be interpreted differently: +- `01`-`05`, `07`: persistence and policy-following under transcript pressure. +- `08`/`09`: host-side transition checks (for example, replacement preconditions and pending-confirmation handling). +- Demos `08`/`09` are not general LLM quality benchmarks. Baseline and reinjected-state can produce plausible text and still `FAIL` when those host-side transition checks are missing. ## Demo 05 Long-Transcript Stress (Exploratory Frontier Runs)