demo code cleanup

demos/01_llm_constraint_drift.py

@@ -139,9 +139,13 @@ def main() -> None:
         yes_no(mediated_violation),
         context="compiler-mediated",
     )
+    baseline_pass = not baseline_violation
+    compiler_pass = mediated_refusal and not mediated_violation
     passed = baseline_violation and mediated_refusal and not mediated_violation
     print_spec_report(
         test_name="01_constraint_drift — persistent prohibition",
+        baseline_pass=baseline_pass,
+        compiler_pass=compiler_pass,
         expected=(
             "compiler-mediated should refuse the prohibited request and offer a safe alternative"
         ),
@@ -160,6 +164,8 @@ def main() -> None:
             )
         ),
         passed=passed,
+        result_pass="prohibition enforced",
+        result_fail="prohibition not enforced",
     )
 
 

demos/02_llm_correction_replacement.py

@@ -131,6 +131,8 @@ def main() -> None:
     )
     print_spec_report(
         test_name="02_correction_replacement — latest value wins",
+        baseline_pass=baseline_respects,
+        compiler_pass=mediated_respects,
         expected="the corrected vegan preference should determine the final plan",
         actual=(
             "baseline still used the stale vegetarian value; "
@@ -147,6 +149,8 @@ def main() -> None:
             )
         ),
         passed=mediated_respects,
+        result_pass="corrected value determined the final plan",
+        result_fail="corrected value did not determine the final plan",
     )
 
 

demos/03_llm_ambiguity_block.py

@@ -64,6 +64,8 @@ def main() -> None:
     )
     print_spec_report(
         test_name="03_ambiguity_block — host clarification gate",
+        baseline_pass=baseline_respects,
+        compiler_pass=mediated_respects,
         expected="host should block LLM call on ambiguous directive until clarification",
         actual=(
             "baseline answered instead of clarifying; compiler-mediated blocked the LLM call"
@@ -75,6 +77,8 @@ def main() -> None:
             )
         ),
         passed=mediated_respects,
+        result_pass="ambiguous directive blocked until clarification",
+        result_fail="ambiguous directive not blocked until clarification",
     )
 
 

demos/04_llm_tool_governance.py

@@ -99,6 +99,7 @@ def main() -> None:
     print_tag_comparison("TOOL", baseline_output, mediated_output)
     baseline_tool = selected_tool(baseline_output)
     mediated_tool = selected_tool(mediated_output)
+    baseline_respects = baseline_tool is not None and baseline_tool not in prohibited
     mediated_respects = mediated_tool is not None and mediated_tool not in prohibited
     print_host_check("SELECTED_TOOL", baseline_tool or "MISSING", context="baseline")
     print_host_check(
@@ -108,6 +109,8 @@ def main() -> None:
     )
     print_spec_report(
         test_name="04_tool_governance — denylisted tool selection",
+        baseline_pass=baseline_respects,
+        compiler_pass=mediated_respects,
         expected="compiler-mediated should select an allowed tool and avoid the denylisted one",
         actual=(
             f"baseline selected {baseline_tool or 'no clear tool'}; "
@@ -120,6 +123,8 @@ def main() -> None:
             )
         ),
         passed=mediated_respects,
+        result_pass="denylisted tool avoided",
+        result_fail="denylisted tool not avoided",
     )
 
 

demos/05_llm_prompt_drift.py

@@ -95,6 +95,7 @@ def main() -> None:
     mediated_style = extract_tag_value(mediated_output, "DINNER_STYLE")
     baseline_non_veg = plan_includes_non_vegetarian_item(baseline_output)
     mediated_non_veg = plan_includes_non_vegetarian_item(mediated_output)
+    baseline_respects = not baseline_non_veg
     mediated_respects = not mediated_non_veg
     print_host_check(
         "PLAN_INCLUDES_NON_VEGETARIAN",
@@ -108,6 +109,8 @@ def main() -> None:
     )
     print_spec_report(
         test_name="05_prompt_drift — preserve key dietary constraint",
+        baseline_pass=baseline_respects,
+        compiler_pass=mediated_respects,
         expected=(
             "compiler-mediated should preserve the vegetarian constraint in the final dinner plan"
         ),
@@ -128,6 +131,8 @@ def main() -> None:
             )
         ),
         passed=mediated_respects,
+        result_pass="vegetarian constraint preserved",
+        result_fail="vegetarian constraint not preserved",
     )
 
 

demos/README.md

@@ -14,16 +14,16 @@ pip install -e .[demos]
 
 Environment variables:
 
-- `MODEL` (required): model name
-- `OPENAI_API_KEY` (required for OpenAI API)
+- `MODEL` (optional; default: `gpt-4.1-mini`)
+- `OPENAI_API_KEY` (required)
 - `OPENAI_BASE_URL` (optional; use for OpenAI-compatible local servers)
 
 Ollama example:
 
 ```bash
 export OPENAI_BASE_URL=http://localhost:11434/v1
 export OPENAI_API_KEY=ollama
-export MODEL=llama3.1
+export MODEL=llama3.1:8b
 ```
 
 OpenAI example:
@@ -55,11 +55,17 @@ uv run python -m demos.run_demo all --verbose
 
 Output modes:
 
-- default (concise): each demo prints only four lines
+- default (concise): each demo prints the following:
   - scenario name + short description
+  - `baseline: PASS|FAIL`
+  - `compiler: PASS|FAIL`
   - expected behavior
   - actual outcome (plain English)
-  - `PASS` or `FAIL`
+  - `result: ...` (short deterministic description)
+  - a blank line between demos when running `all`
+  - final summary:
+    - `Baseline results: X passed, Y failed`
+    - `Compiler results: X passed, Y failed`
 - `--verbose`: prints detailed traces
   - user inputs
   - compiler decisions and state

demos/common.py

@@ -3,14 +3,25 @@
 import json
 import os
 import re
-from typing import Any
+from typing import Any, TypedDict
 
 from context_compiler import Decision, State
 from context_compiler.const import FOCUS_PRIMARY, POLICY_PROHIBIT, STATE_FACTS, STATE_POLICIES
 from demos.llm_client import Message
 
 VERBOSE_ENV_VAR = "CONTEXT_COMPILER_DEMO_VERBOSE"
-LAST_REPORT_PASSED: bool | None = None
+
+
+class DemoReport(TypedDict):
+    name: str
+    expected: str
+    actual: str
+    baseline_pass: bool
+    compiler_pass: bool
+    demo_pass: bool
+
+
+LAST_REPORT: DemoReport | None = None
 
 
 def canonical_json(obj: Any) -> str:
@@ -90,21 +101,40 @@ def print_host_check(name: str, value: str, *, context: str) -> None:
     print(f"HOST_CHECK {name}: {value} ({context})")
 
 
-def print_spec_report(*, test_name: str, expected: str, actual: str, passed: bool) -> None:
-    global LAST_REPORT_PASSED
-    LAST_REPORT_PASSED = passed
+def print_spec_report(
+    *,
+    test_name: str,
+    baseline_pass: bool,
+    compiler_pass: bool,
+    expected: str,
+    actual: str,
+    passed: bool,
+    result_pass: str,
+    result_fail: str,
+) -> None:
+    global LAST_REPORT
+    LAST_REPORT = {
+        "name": test_name,
+        "expected": expected,
+        "actual": actual,
+        "baseline_pass": baseline_pass,
+        "compiler_pass": compiler_pass,
+        "demo_pass": passed,
+    }
     print(test_name)
+    print(f"baseline: {'PASS' if baseline_pass else 'FAIL'}")
+    print(f"compiler: {'PASS' if compiler_pass else 'FAIL'}")
     print(f"expected: {expected}")
     print(f"actual: {actual}")
-    print("*** PASS" if passed else "*** FAIL")
+    print(f"result: {result_pass if passed else result_fail}")
     if is_verbose():
         print()
 
 
-def consume_last_report_passed() -> bool | None:
-    global LAST_REPORT_PASSED
-    value = LAST_REPORT_PASSED
-    LAST_REPORT_PASSED = None
+def consume_last_report() -> DemoReport | None:
+    global LAST_REPORT
+    value = LAST_REPORT
+    LAST_REPORT = None
     return value
 
 

demos/llm_client.py

@@ -18,15 +18,29 @@ class LLMConfig:
     model: str
 
 
+@dataclass(frozen=True)
+class MissingDemoConfigError(RuntimeError):
+    missing: list[str]
+    base_url: str | None
+
+    def __str__(self) -> str:
+        missing_text = ", ".join(self.missing)
+        return f"Missing demo configuration: {missing_text}"
+
+
 def load_config() -> LLMConfig:
     """Load OpenAI-compatible configuration from environment variables."""
     base_url = os.getenv("OPENAI_BASE_URL")
-    api_key = os.getenv("OPENAI_API_KEY") or ("ollama" if base_url else "")
+    api_key = os.getenv("OPENAI_API_KEY")
     model = os.getenv("MODEL", "gpt-4.1-mini")
 
+    missing: list[str] = []
     if not api_key:
-        raise RuntimeError("Set OPENAI_API_KEY (or OPENAI_BASE_URL for local Ollama).")
+        missing.append("OPENAI_API_KEY")
+    if missing:
+        raise MissingDemoConfigError(missing=missing, base_url=base_url)
 
+    assert api_key is not None
     return LLMConfig(base_url=base_url, api_key=api_key, model=model)
 
 

demos/run_demo.py

@@ -6,7 +6,8 @@
 import sys
 from pathlib import Path
 
-from demos.common import VERBOSE_ENV_VAR, consume_last_report_passed
+from demos.common import VERBOSE_ENV_VAR, DemoReport, consume_last_report
+from demos.llm_client import MissingDemoConfigError
 
 DEMO_FILES: dict[str, str] = {
     "1": "01_llm_constraint_drift.py",
@@ -17,21 +18,36 @@
 }
 
 
-def _run(path: Path, *, verbose: bool) -> bool | None:
+def _run(path: Path, *, verbose: bool) -> DemoReport | None:
     if verbose:
         print(f"===== Running {path.name} =====")
     old_value = os.getenv(VERBOSE_ENV_VAR)
     os.environ[VERBOSE_ENV_VAR] = "1" if verbose else "0"
     try:
         runpy.run_path(str(path), run_name="__main__")
-        return consume_last_report_passed()
+        return consume_last_report()
     finally:
         if old_value is None:
             os.environ.pop(VERBOSE_ENV_VAR, None)
         else:
             os.environ[VERBOSE_ENV_VAR] = old_value
 
 
+def _print_config_error(exc: MissingDemoConfigError) -> None:
+    mode = "OpenAI-compatible endpoint" if exc.base_url else "OpenAI API"
+    print("Unable to run LLM demos: missing model configuration.")
+    print(f"Assumed mode: {mode}")
+    print(f"Missing variables: {', '.join(exc.missing)}")
+    print("Example setup:")
+    if exc.base_url:
+        print("  export OPENAI_BASE_URL=http://localhost:11434/v1")
+        print("  export OPENAI_API_KEY=ollama")
+        print("  export MODEL=llama3.1:8b")
+    else:
+        print("  export OPENAI_API_KEY=your_key_here")
+        print("  export MODEL=gpt-4.1-mini")
+
+
 def main() -> None:
     root = Path(__file__).resolve().parent
     project_root = root.parent
@@ -54,19 +70,42 @@ def main() -> None:
     args = parser.parse_args()
 
     if args.demo == "all":
-        passed = 0
-        failed = 0
-        for key in sorted(DEMO_FILES.keys()):
-            result = _run(root / DEMO_FILES[key], verbose=args.verbose)
-            if result is True:
-                passed += 1
+        baseline_pass_count = 0
+        baseline_fail_count = 0
+        compiler_pass_count = 0
+        compiler_fail_count = 0
+        for index, key in enumerate(sorted(DEMO_FILES.keys())):
+            if index > 0:
+                print()
+            try:
+                result = _run(root / DEMO_FILES[key], verbose=args.verbose)
+            except MissingDemoConfigError as exc:
+                _print_config_error(exc)
+                raise SystemExit(2) from exc
+            if result is None:
+                baseline_fail_count += 1
+                compiler_fail_count += 1
+                continue
+
+            if bool(result["baseline_pass"]):
+                baseline_pass_count += 1
             else:
-                failed += 1
+                baseline_fail_count += 1
+
+            if bool(result["compiler_pass"]):
+                compiler_pass_count += 1
+            else:
+                compiler_fail_count += 1
         print("=" * 60)
-        print(f"Context Compiler demos complete: {passed} passed, {failed} failed")
+        print(f"Baseline results: {baseline_pass_count} passed, {baseline_fail_count} failed")
+        print(f"Compiler results: {compiler_pass_count} passed, {compiler_fail_count} failed")
         return
 
-    _run(root / DEMO_FILES[args.demo], verbose=args.verbose)
+    try:
+        _run(root / DEMO_FILES[args.demo], verbose=args.verbose)
+    except MissingDemoConfigError as exc:
+        _print_config_error(exc)
+        raise SystemExit(2) from exc
 
 
 if __name__ == "__main__":