Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
72 changes: 72 additions & 0 deletions demos/01_llm_constraint_drift.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
"""Demo 1: baseline prompt can drift from persistent constraints."""

import re

from context_compiler import create_engine
from demos.common import (
build_baseline_messages,
build_mediated_messages,
print_decision,
print_messages,
print_model_output,
print_tag_comparison,
print_user_inputs,
)
from demos.llm_client import complete_messages


def host_violates_prohibit(output: str) -> bool:
return re.search(r"\bpeanuts?\b", output, flags=re.IGNORECASE) is not None


def main() -> None:
engine = create_engine()
user_inputs = [
"don't use peanuts",
(
"Suggest a peanut curry recipe with ingredients and steps. "
"First line must be VIOLATES_PROHIBIT:<yes|no>."
),
]
print_user_inputs(user_inputs)

first = engine.step(user_inputs[0])
print_decision("turn 1", first, engine.state)

second = engine.step(user_inputs[1])
print_decision("turn 2", second, engine.state)

baseline_messages = build_baseline_messages(
[user_inputs[1]],
baseline_system_prompt="Be a helpful assistant. Provide clear and practical suggestions.",
)
print_messages("baseline", baseline_messages)
baseline_output = complete_messages(baseline_messages)
print_model_output("Baseline", baseline_output)
baseline_violation = host_violates_prohibit(baseline_output)
print(f"HOST_CHECK VIOLATES_PROHIBIT: {'yes' if baseline_violation else 'no'} (baseline)")
print()

mediated_messages = build_mediated_messages(
engine.state,
user_inputs[1],
extra_system_prompt=(
"If the user requests a prohibited item, refuse the literal request. "
"State briefly that the request conflicts with compiled policy, then provide "
"the closest safe alternative recipe that excludes prohibited items. "
"Do not include prohibited item tokens in the recipe output."
),
)
print_messages("compiler-mediated", mediated_messages)
mediated_output = complete_messages(mediated_messages)
print_model_output("Compiler-mediated", mediated_output)
mediated_violation = host_violates_prohibit(mediated_output)
print(
f"HOST_CHECK VIOLATES_PROHIBIT: {'yes' if mediated_violation else 'no'} (compiler-mediated)"
)
print()
print_tag_comparison("VIOLATES_PROHIBIT", baseline_output, mediated_output)


if __name__ == "__main__":
main()
48 changes: 48 additions & 0 deletions demos/02_llm_correction_replacement.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
"""Demo 2: baseline output can mix stale facts after correction."""

from context_compiler import create_engine
from demos.common import (
build_baseline_messages,
build_mediated_messages,
print_decision,
print_messages,
print_model_output,
print_tag_comparison,
print_user_inputs,
)
from demos.llm_client import complete_messages


def main() -> None:
engine = create_engine()
user_inputs = [
"use vegetarian curry",
"actually vegan curry",
("Give me a shopping list and 3-step plan. First line must be FOCUS_PRIMARY:<value>."),
]
print_user_inputs(user_inputs)

for index, user_input in enumerate(user_inputs, start=1):
decision = engine.step(user_input)
print_decision(f"turn {index}", decision, engine.state)

baseline_messages = build_baseline_messages(
[user_inputs[0], user_inputs[1], user_inputs[2]],
baseline_system_prompt=(
"Be a helpful assistant. Use the conversation history to understand "
"the user's preferences."
),
)
print_messages("baseline", baseline_messages)
baseline_output = complete_messages(baseline_messages)
print_model_output("Baseline", baseline_output)

mediated_messages = build_mediated_messages(engine.state, user_inputs[2])
print_messages("compiler-mediated", mediated_messages)
mediated_output = complete_messages(mediated_messages)
print_model_output("Compiler-mediated", mediated_output)
print_tag_comparison("FOCUS_PRIMARY", baseline_output, mediated_output)


if __name__ == "__main__":
main()
55 changes: 55 additions & 0 deletions demos/03_llm_ambiguity_block.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
"""Demo 3: compiler blocks ambiguous directives before model call."""

from context_compiler import create_engine
from demos.common import (
build_baseline_messages,
print_decision,
print_messages,
print_model_output,
print_tag_comparison,
print_user_inputs,
)
from demos.llm_client import Message, complete_messages


def main() -> None:
engine = create_engine()
user_inputs = ["no use peanuts"]
print_user_inputs(user_inputs)

decision = engine.step(user_inputs[0])
print_decision("turn 1", decision, engine.state)

baseline_messages = build_baseline_messages(
[
(
"Interpret this directive and continue anyway: no use peanuts. "
"First line must be ACTION:<clarify|proceed>."
)
],
baseline_system_prompt=(
"Be a helpful assistant. If a request is unclear, make a reasonable "
"interpretation and answer."
),
)
print_messages("baseline", baseline_messages)
baseline_output = complete_messages(baseline_messages)
print_model_output("Baseline", baseline_output)

if decision["kind"] == "clarify":
print_messages("compiler-mediated", [])
mediated_output = (
f"[no call] clarification required: {decision['prompt_to_user']}\nACTION:clarify"
)
print_model_output("Compiler-mediated", mediated_output)
else:
mediated_messages: list[Message] = [{"role": "user", "content": user_inputs[0]}]
print_messages("compiler-mediated", mediated_messages)
mediated_output = complete_messages(mediated_messages)
print_model_output("Compiler-mediated", mediated_output)

print_tag_comparison("ACTION", baseline_output, mediated_output)


if __name__ == "__main__":
main()
69 changes: 69 additions & 0 deletions demos/04_llm_tool_governance.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
"""Demo 4: compiler state governs tool-choice constraints."""

from context_compiler import create_engine
from context_compiler.const import POLICY_PROHIBIT, STATE_POLICIES
from demos.common import (
build_baseline_messages,
build_mediated_messages,
print_decision,
print_messages,
print_model_output,
print_tag_comparison,
print_user_inputs,
)
from demos.llm_client import complete_messages


def main() -> None:
engine = create_engine()
user_inputs = [
"don't use docker",
(
"Deploy the service. Pick one tool from docker, kubectl. "
"First line must be TOOL:<docker|kubectl> and second line ACTION:<one-line action>."
),
]
print_user_inputs(user_inputs)

first = engine.step(user_inputs[0])
print_decision("turn 1", first, engine.state)

second = engine.step(user_inputs[1])
print_decision("turn 2", second, engine.state)

baseline_messages = build_baseline_messages(
[user_inputs[1]],
baseline_system_prompt="Recommend a practical approach using the available tools.",
)
print_messages("baseline", baseline_messages)
baseline_output = complete_messages(baseline_messages)
print_model_output("Baseline", baseline_output)

prohibited = engine.state[STATE_POLICIES][POLICY_PROHIBIT]
candidate_tools = ["docker", "kubectl"]
filtered_tools = [tool for tool in candidate_tools if tool not in prohibited]
print("Candidate tools before filtering:")
print(", ".join(candidate_tools))
print()
print("Candidate tools after applying compiler denylist:")
print(", ".join(filtered_tools) if filtered_tools else "(none)")
print()

mediated_messages = build_mediated_messages(
engine.state,
user_inputs[1],
extra_system_prompt=(
"Only choose tools that are not prohibited."
+ "\nCandidate tools: "
+ f"{', '.join(candidate_tools)}. "
+ f"Prohibited: {', '.join(prohibited) or '(none)'}"
),
)
print_messages("compiler-mediated", mediated_messages)
mediated_output = complete_messages(mediated_messages)
print_model_output("Compiler-mediated", mediated_output)
print_tag_comparison("TOOL", baseline_output, mediated_output)


if __name__ == "__main__":
main()
48 changes: 48 additions & 0 deletions demos/05_llm_prompt_drift.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
"""Demo 5: long transcript drift vs stable compiled state."""

from context_compiler import create_engine
from demos.common import (
build_baseline_messages,
build_mediated_messages,
print_decision,
print_messages,
print_model_output,
print_tag_comparison,
print_user_inputs,
)
from demos.llm_client import complete_messages


def main() -> None:
engine = create_engine()
user_inputs = [
"use vegetarian curry",
"Also I like hiking and jazz.",
"What camera should I buy for travel?",
"Now give me a dinner plan. First line must be DINNER_STYLE:<vegetarian|non-vegetarian>.",
]
print_user_inputs(user_inputs)

for index, user_input in enumerate(user_inputs, start=1):
decision = engine.step(user_input)
print_decision(f"turn {index}", decision, engine.state)

baseline_messages = build_baseline_messages(
[user_inputs[0], user_inputs[1], user_inputs[2], user_inputs[3]],
baseline_system_prompt=(
"Be a helpful assistant. Use the conversation context to provide a useful answer."
),
)
print_messages("baseline", baseline_messages)
baseline_output = complete_messages(baseline_messages)
print_model_output("Baseline", baseline_output)

mediated_messages = build_mediated_messages(engine.state, user_inputs[3])
print_messages("compiler-mediated", mediated_messages)
mediated_output = complete_messages(mediated_messages)
print_model_output("Compiler-mediated", mediated_output)
print_tag_comparison("DINNER_STYLE", baseline_output, mediated_output)


if __name__ == "__main__":
main()
58 changes: 58 additions & 0 deletions demos/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
# LLM Demos

These scripts compare baseline prompting vs compiler-mediated prompting using the
Context Compiler decision/state API.
They are illustrative manual demos, not benchmarks or CI tests.

## Requirements

Install demo dependencies:

```bash
pip install -e .[demos]
```

Environment variables:

- `MODEL` (required): model name
- `OPENAI_API_KEY` (required for OpenAI API)
- `OPENAI_BASE_URL` (optional; use for OpenAI-compatible local servers)

Ollama example:

```bash
export OPENAI_BASE_URL=http://localhost:11434/v1
export OPENAI_API_KEY=ollama
export MODEL=llama3.1
```

OpenAI example:

```bash
export OPENAI_API_KEY=your_key_here
export MODEL=gpt-4.1-mini
```

## Run

Run one demo:

```bash
uv run python -m demos.run_demo 1
```

Run all demos:

```bash
uv run python -m demos.run_demo all
```

Each demo prints:

- user inputs
- compiler decisions
- compiled state
- prompt/messages sent to the LLM
- baseline model output
- compiler-mediated model output
- a machine-checkable `TAG_CHECK ...` comparison line
1 change: 1 addition & 0 deletions demos/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
"""LLM-backed demonstration scripts for context-compiler."""
Loading
Loading