From e0b0438d1802386c38cb4d22677fc0082483e104 Mon Sep 17 00:00:00 2001 From: Robert Lippmann Date: Wed, 11 Mar 2026 16:13:49 -0400 Subject: [PATCH 1/2] demos: add baseline/compiler pass-fail reporting and improve demo output Add explicit baseline and compiler PASS/FAIL reporting to all LLM demos. Changes: - Print baseline and compiler results for each demo - Replace "*** PASS" with descriptive "result:" lines - Improve readability with spacing between demos - Simplify final summary to baseline/compiler totals - Introduce DemoReport structure for runner aggregation - Remove old demo outcome counters This makes the demos clearer for evaluating model behavior: baseline shows how the raw model performs, while compiler shows the mediated behavior. --- demos/01_llm_constraint_drift.py | 6 ++++ demos/02_llm_correction_replacement.py | 4 +++ demos/03_llm_ambiguity_block.py | 4 +++ demos/04_llm_tool_governance.py | 5 +++ demos/05_llm_prompt_drift.py | 5 +++ demos/README.md | 10 ++++-- demos/common.py | 50 ++++++++++++++++++++------ demos/run_demo.py | 35 ++++++++++++------ 8 files changed, 97 insertions(+), 22 deletions(-) diff --git a/demos/01_llm_constraint_drift.py b/demos/01_llm_constraint_drift.py index 8b80c00..187be9e 100644 --- a/demos/01_llm_constraint_drift.py +++ b/demos/01_llm_constraint_drift.py @@ -139,9 +139,13 @@ def main() -> None: yes_no(mediated_violation), context="compiler-mediated", ) + baseline_pass = not baseline_violation + compiler_pass = mediated_refusal and not mediated_violation passed = baseline_violation and mediated_refusal and not mediated_violation print_spec_report( test_name="01_constraint_drift — persistent prohibition", + baseline_pass=baseline_pass, + compiler_pass=compiler_pass, expected=( "compiler-mediated should refuse the prohibited request and offer a safe alternative" ), @@ -160,6 +164,8 @@ def main() -> None: ) ), passed=passed, + result_pass="prohibition enforced", + result_fail="prohibition not enforced", ) diff --git a/demos/02_llm_correction_replacement.py b/demos/02_llm_correction_replacement.py index 68dc6c6..f1ae02f 100644 --- a/demos/02_llm_correction_replacement.py +++ b/demos/02_llm_correction_replacement.py @@ -131,6 +131,8 @@ def main() -> None: ) print_spec_report( test_name="02_correction_replacement — latest value wins", + baseline_pass=baseline_respects, + compiler_pass=mediated_respects, expected="the corrected vegan preference should determine the final plan", actual=( "baseline still used the stale vegetarian value; " @@ -147,6 +149,8 @@ def main() -> None: ) ), passed=mediated_respects, + result_pass="corrected value determined the final plan", + result_fail="corrected value did not determine the final plan", ) diff --git a/demos/03_llm_ambiguity_block.py b/demos/03_llm_ambiguity_block.py index c3e6e54..e576c5e 100644 --- a/demos/03_llm_ambiguity_block.py +++ b/demos/03_llm_ambiguity_block.py @@ -64,6 +64,8 @@ def main() -> None: ) print_spec_report( test_name="03_ambiguity_block — host clarification gate", + baseline_pass=baseline_respects, + compiler_pass=mediated_respects, expected="host should block LLM call on ambiguous directive until clarification", actual=( "baseline answered instead of clarifying; compiler-mediated blocked the LLM call" @@ -75,6 +77,8 @@ def main() -> None: ) ), passed=mediated_respects, + result_pass="ambiguous directive blocked until clarification", + result_fail="ambiguous directive not blocked until clarification", ) diff --git a/demos/04_llm_tool_governance.py b/demos/04_llm_tool_governance.py index b17537f..8f3d62f 100644 --- a/demos/04_llm_tool_governance.py +++ b/demos/04_llm_tool_governance.py @@ -99,6 +99,7 @@ def main() -> None: print_tag_comparison("TOOL", baseline_output, mediated_output) baseline_tool = selected_tool(baseline_output) mediated_tool = selected_tool(mediated_output) + baseline_respects = baseline_tool is not None and baseline_tool not in prohibited mediated_respects = mediated_tool is not None and mediated_tool not in prohibited print_host_check("SELECTED_TOOL", baseline_tool or "MISSING", context="baseline") print_host_check( @@ -108,6 +109,8 @@ def main() -> None: ) print_spec_report( test_name="04_tool_governance — denylisted tool selection", + baseline_pass=baseline_respects, + compiler_pass=mediated_respects, expected="compiler-mediated should select an allowed tool and avoid the denylisted one", actual=( f"baseline selected {baseline_tool or 'no clear tool'}; " @@ -120,6 +123,8 @@ def main() -> None: ) ), passed=mediated_respects, + result_pass="denylisted tool avoided", + result_fail="denylisted tool not avoided", ) diff --git a/demos/05_llm_prompt_drift.py b/demos/05_llm_prompt_drift.py index 5d2d880..508fa4e 100644 --- a/demos/05_llm_prompt_drift.py +++ b/demos/05_llm_prompt_drift.py @@ -95,6 +95,7 @@ def main() -> None: mediated_style = extract_tag_value(mediated_output, "DINNER_STYLE") baseline_non_veg = plan_includes_non_vegetarian_item(baseline_output) mediated_non_veg = plan_includes_non_vegetarian_item(mediated_output) + baseline_respects = not baseline_non_veg mediated_respects = not mediated_non_veg print_host_check( "PLAN_INCLUDES_NON_VEGETARIAN", @@ -108,6 +109,8 @@ def main() -> None: ) print_spec_report( test_name="05_prompt_drift — preserve key dietary constraint", + baseline_pass=baseline_respects, + compiler_pass=mediated_respects, expected=( "compiler-mediated should preserve the vegetarian constraint in the final dinner plan" ), @@ -128,6 +131,8 @@ def main() -> None: ) ), passed=mediated_respects, + result_pass="vegetarian constraint preserved", + result_fail="vegetarian constraint not preserved", ) diff --git a/demos/README.md b/demos/README.md index c1315ce..f2c63f7 100644 --- a/demos/README.md +++ b/demos/README.md @@ -55,11 +55,17 @@ uv run python -m demos.run_demo all --verbose Output modes: -- default (concise): each demo prints only four lines +- default (concise): each demo prints the following: - scenario name + short description + - `baseline: PASS|FAIL` + - `compiler: PASS|FAIL` - expected behavior - actual outcome (plain English) - - `PASS` or `FAIL` + - `result: ...` (short deterministic description) + - a blank line between demos when running `all` + - final summary: + - `Baseline results: X passed, Y failed` + - `Compiler results: X passed, Y failed` - `--verbose`: prints detailed traces - user inputs - compiler decisions and state diff --git a/demos/common.py b/demos/common.py index 7cfbe48..7be5b9e 100644 --- a/demos/common.py +++ b/demos/common.py @@ -3,14 +3,25 @@ import json import os import re -from typing import Any +from typing import Any, TypedDict from context_compiler import Decision, State from context_compiler.const import FOCUS_PRIMARY, POLICY_PROHIBIT, STATE_FACTS, STATE_POLICIES from demos.llm_client import Message VERBOSE_ENV_VAR = "CONTEXT_COMPILER_DEMO_VERBOSE" -LAST_REPORT_PASSED: bool | None = None + + +class DemoReport(TypedDict): + name: str + expected: str + actual: str + baseline_pass: bool + compiler_pass: bool + demo_pass: bool + + +LAST_REPORT: DemoReport | None = None def canonical_json(obj: Any) -> str: @@ -90,21 +101,40 @@ def print_host_check(name: str, value: str, *, context: str) -> None: print(f"HOST_CHECK {name}: {value} ({context})") -def print_spec_report(*, test_name: str, expected: str, actual: str, passed: bool) -> None: - global LAST_REPORT_PASSED - LAST_REPORT_PASSED = passed +def print_spec_report( + *, + test_name: str, + baseline_pass: bool, + compiler_pass: bool, + expected: str, + actual: str, + passed: bool, + result_pass: str, + result_fail: str, +) -> None: + global LAST_REPORT + LAST_REPORT = { + "name": test_name, + "expected": expected, + "actual": actual, + "baseline_pass": baseline_pass, + "compiler_pass": compiler_pass, + "demo_pass": passed, + } print(test_name) + print(f"baseline: {'PASS' if baseline_pass else 'FAIL'}") + print(f"compiler: {'PASS' if compiler_pass else 'FAIL'}") print(f"expected: {expected}") print(f"actual: {actual}") - print("*** PASS" if passed else "*** FAIL") + print(f"result: {result_pass if passed else result_fail}") if is_verbose(): print() -def consume_last_report_passed() -> bool | None: - global LAST_REPORT_PASSED - value = LAST_REPORT_PASSED - LAST_REPORT_PASSED = None +def consume_last_report() -> DemoReport | None: + global LAST_REPORT + value = LAST_REPORT + LAST_REPORT = None return value diff --git a/demos/run_demo.py b/demos/run_demo.py index 561dd2c..b41a183 100644 --- a/demos/run_demo.py +++ b/demos/run_demo.py @@ -6,7 +6,7 @@ import sys from pathlib import Path -from demos.common import VERBOSE_ENV_VAR, consume_last_report_passed +from demos.common import VERBOSE_ENV_VAR, DemoReport, consume_last_report DEMO_FILES: dict[str, str] = { "1": "01_llm_constraint_drift.py", @@ -17,14 +17,14 @@ } -def _run(path: Path, *, verbose: bool) -> bool | None: +def _run(path: Path, *, verbose: bool) -> DemoReport | None: if verbose: print(f"===== Running {path.name} =====") old_value = os.getenv(VERBOSE_ENV_VAR) os.environ[VERBOSE_ENV_VAR] = "1" if verbose else "0" try: runpy.run_path(str(path), run_name="__main__") - return consume_last_report_passed() + return consume_last_report() finally: if old_value is None: os.environ.pop(VERBOSE_ENV_VAR, None) @@ -54,16 +54,31 @@ def main() -> None: args = parser.parse_args() if args.demo == "all": - passed = 0 - failed = 0 - for key in sorted(DEMO_FILES.keys()): + baseline_pass_count = 0 + baseline_fail_count = 0 + compiler_pass_count = 0 + compiler_fail_count = 0 + for index, key in enumerate(sorted(DEMO_FILES.keys())): + if index > 0: + print() result = _run(root / DEMO_FILES[key], verbose=args.verbose) - if result is True: - passed += 1 + if result is None: + baseline_fail_count += 1 + compiler_fail_count += 1 + continue + + if bool(result["baseline_pass"]): + baseline_pass_count += 1 + else: + baseline_fail_count += 1 + + if bool(result["compiler_pass"]): + compiler_pass_count += 1 else: - failed += 1 + compiler_fail_count += 1 print("=" * 60) - print(f"Context Compiler demos complete: {passed} passed, {failed} failed") + print(f"Baseline results: {baseline_pass_count} passed, {baseline_fail_count} failed") + print(f"Compiler results: {compiler_pass_count} passed, {compiler_fail_count} failed") return _run(root / DEMO_FILES[args.demo], verbose=args.verbose) From eb6b1581a9228876b2f307a95a9f01847102b705 Mon Sep 17 00:00:00 2001 From: Robert Lippmann Date: Thu, 12 Mar 2026 00:22:09 -0400 Subject: [PATCH 2/2] demos: handle missing LLM configuration gracefully --- demos/README.md | 6 +++--- demos/llm_client.py | 18 ++++++++++++++++-- demos/run_demo.py | 28 ++++++++++++++++++++++++++-- 3 files changed, 45 insertions(+), 7 deletions(-) diff --git a/demos/README.md b/demos/README.md index f2c63f7..a36e668 100644 --- a/demos/README.md +++ b/demos/README.md @@ -14,8 +14,8 @@ pip install -e .[demos] Environment variables: -- `MODEL` (required): model name -- `OPENAI_API_KEY` (required for OpenAI API) +- `MODEL` (optional; default: `gpt-4.1-mini`) +- `OPENAI_API_KEY` (required) - `OPENAI_BASE_URL` (optional; use for OpenAI-compatible local servers) Ollama example: @@ -23,7 +23,7 @@ Ollama example: ```bash export OPENAI_BASE_URL=http://localhost:11434/v1 export OPENAI_API_KEY=ollama -export MODEL=llama3.1 +export MODEL=llama3.1:8b ``` OpenAI example: diff --git a/demos/llm_client.py b/demos/llm_client.py index 97c3693..2a1419b 100644 --- a/demos/llm_client.py +++ b/demos/llm_client.py @@ -18,15 +18,29 @@ class LLMConfig: model: str +@dataclass(frozen=True) +class MissingDemoConfigError(RuntimeError): + missing: list[str] + base_url: str | None + + def __str__(self) -> str: + missing_text = ", ".join(self.missing) + return f"Missing demo configuration: {missing_text}" + + def load_config() -> LLMConfig: """Load OpenAI-compatible configuration from environment variables.""" base_url = os.getenv("OPENAI_BASE_URL") - api_key = os.getenv("OPENAI_API_KEY") or ("ollama" if base_url else "") + api_key = os.getenv("OPENAI_API_KEY") model = os.getenv("MODEL", "gpt-4.1-mini") + missing: list[str] = [] if not api_key: - raise RuntimeError("Set OPENAI_API_KEY (or OPENAI_BASE_URL for local Ollama).") + missing.append("OPENAI_API_KEY") + if missing: + raise MissingDemoConfigError(missing=missing, base_url=base_url) + assert api_key is not None return LLMConfig(base_url=base_url, api_key=api_key, model=model) diff --git a/demos/run_demo.py b/demos/run_demo.py index b41a183..b6cca88 100644 --- a/demos/run_demo.py +++ b/demos/run_demo.py @@ -7,6 +7,7 @@ from pathlib import Path from demos.common import VERBOSE_ENV_VAR, DemoReport, consume_last_report +from demos.llm_client import MissingDemoConfigError DEMO_FILES: dict[str, str] = { "1": "01_llm_constraint_drift.py", @@ -32,6 +33,21 @@ def _run(path: Path, *, verbose: bool) -> DemoReport | None: os.environ[VERBOSE_ENV_VAR] = old_value +def _print_config_error(exc: MissingDemoConfigError) -> None: + mode = "OpenAI-compatible endpoint" if exc.base_url else "OpenAI API" + print("Unable to run LLM demos: missing model configuration.") + print(f"Assumed mode: {mode}") + print(f"Missing variables: {', '.join(exc.missing)}") + print("Example setup:") + if exc.base_url: + print(" export OPENAI_BASE_URL=http://localhost:11434/v1") + print(" export OPENAI_API_KEY=ollama") + print(" export MODEL=llama3.1:8b") + else: + print(" export OPENAI_API_KEY=your_key_here") + print(" export MODEL=gpt-4.1-mini") + + def main() -> None: root = Path(__file__).resolve().parent project_root = root.parent @@ -61,7 +77,11 @@ def main() -> None: for index, key in enumerate(sorted(DEMO_FILES.keys())): if index > 0: print() - result = _run(root / DEMO_FILES[key], verbose=args.verbose) + try: + result = _run(root / DEMO_FILES[key], verbose=args.verbose) + except MissingDemoConfigError as exc: + _print_config_error(exc) + raise SystemExit(2) from exc if result is None: baseline_fail_count += 1 compiler_fail_count += 1 @@ -81,7 +101,11 @@ def main() -> None: print(f"Compiler results: {compiler_pass_count} passed, {compiler_fail_count} failed") return - _run(root / DEMO_FILES[args.demo], verbose=args.verbose) + try: + _run(root / DEMO_FILES[args.demo], verbose=args.verbose) + except MissingDemoConfigError as exc: + _print_config_error(exc) + raise SystemExit(2) from exc if __name__ == "__main__":