feat: add LiteLLM integrations and switch demos to LiteLLM transport

README.md

@@ -31,8 +31,6 @@ The idea is similar to a traditional compiler: user directives are translated in
 - Python 3.11+
 - `pip install context-compiler`
 - Dev/test: `uv sync --group dev` and `uv run pytest`
-- Examples: see [examples/README.md](examples/README.md)
-- Demonstrations: see [demos/README.md](demos/README.md)
 
 ---
 
@@ -248,9 +246,9 @@ Example:
 
 ## Examples
 
-Integration examples are available in the [examples/](examples/) directory.
-
-See [examples/README.md](examples/README.md) for walkthroughs.
+- [examples](examples/)
+- [demos](demos/)
+- [integrations](examples/integrations/)
 
 ---
 

demos/README.md

@@ -30,11 +30,18 @@ Install demo dependencies:
 pip install -e .[demos]
 ```
 
-Environment variables (OpenAI-compatible API):
+Environment variables (LiteLLM/OpenAI-compatible API):
 
 - `MODEL` (optional)
 - `OPENAI_API_KEY` (required)
 - `OPENAI_BASE_URL` (optional; use for local or alternative endpoints)
+Note: Demos require models that support deterministic decoding (`temperature=0`).
+Some newer models (e.g. `gpt-5`) do not support this and may error.
+
+LiteLLM model naming with Ollama:
+
+- Ollama native endpoint (`http://127.0.0.1:11434`): use `MODEL=ollama/llama3.1:8b`
+- OpenAI-compatible endpoint (`.../v1`): use `MODEL=openai/llama3.1:8b`
 
 Example: locally hosted OpenAI-compatible endpoint
 

demos/llm_client.py

@@ -1,4 +1,4 @@
-"""Small OpenAI-compatible chat client for demo scripts."""
+"""Small LiteLLM chat client for demo scripts."""
 
 import os
 import re
@@ -159,24 +159,35 @@ def load_config() -> LLMConfig:
     return LLMConfig(base_url=base_url, api_key=api_key, model=model)
 
 
-def _build_openai_client(config: LLMConfig) -> Any:
+def _litellm_completion(
+    *,
+    config: LLMConfig,
+    target_model: str,
+    messages: list[Message],
+) -> Any:
     try:
-        openai_module = import_module("openai")
+        litellm_module = import_module("litellm")
     except ModuleNotFoundError as exc:
         raise RuntimeError(
             "The demo scripts require the demos dependency group.\n"
             "Install it with:\n"
             "pip install -e .[demos]"
         ) from exc
 
-    client_cls = getattr(openai_module, "OpenAI", None)
-    if client_cls is None:
-        raise RuntimeError("Unsupported openai package: `OpenAI` client class not found.")
-
-    kwargs: dict[str, Any] = {"api_key": config.api_key}
+    completion_fn = getattr(litellm_module, "completion", None)
+    if completion_fn is None:
+        raise RuntimeError("Unsupported litellm package: `completion` function not found.")
+
+    kwargs: dict[str, Any] = {
+        "model": target_model,
+        "messages": messages,
+        # Demos require deterministic decoding so PASS/FAIL results are reproducible.
+        "temperature": 0,
+        "api_key": config.api_key,
+    }
     if config.base_url:
-        kwargs["base_url"] = config.base_url
-    return client_cls(**kwargs)
+        kwargs["api_base"] = config.base_url
+    return completion_fn(**kwargs)
 
 
 def complete_messages(
@@ -187,20 +198,17 @@ def complete_messages(
 ) -> str:
     """Send exact message list to chat completions and return the text output."""
     config = load_config()
-    client = _build_openai_client(config)
     target_model = model or config.model
     configured_delay = _configured_delay_seconds(delay_seconds)
 
     for attempt in range(len(_RETRY_DELAYS_SECONDS) + 1):
         try:
             if configured_delay > 0:
                 time.sleep(configured_delay)
-            # Demos require deterministic decoding so PASS/FAIL results are reproducible.
-            response = client.chat.completions.create(
-                model=target_model,
+            response = _litellm_completion(
+                config=config,
+                target_model=target_model,
                 messages=messages,
-                # Intentionally hard-coded for deterministic demo behavior.
-                temperature=0,
             )
             break
         except Exception as exc:
@@ -265,7 +273,26 @@ def complete_messages(
             raise DemoLLMError(
                 f"LLM provider error while calling model '{target_model}': {exc}"
             ) from exc
-    content = response.choices[0].message.content
+
+    choices = getattr(response, "choices", None)
+    if choices is None and isinstance(response, dict):
+        choices = response.get("choices")
+    if not isinstance(choices, list) or not choices:
+        return ""
+    first_choice = choices[0]
+
+    message_obj: object | None = None
+    if isinstance(first_choice, dict):
+        message_obj = first_choice.get("message")
+    else:
+        message_obj = getattr(first_choice, "message", None)
+
+    content: object | None = None
+    if isinstance(message_obj, dict):
+        content = message_obj.get("content")
+    else:
+        content = getattr(message_obj, "content", None)
+
     if isinstance(content, str):
         return content.strip()
     if content is None:

examples/integrations/README.md

@@ -0,0 +1,34 @@
+# Integrations
+
+These examples show how to integrate Context Compiler with external systems.
+
+## LiteLLM (SDK)
+
+Minimal example showing how to run Context Compiler before an LLM call with LiteLLM.
+
+### Requirements
+
+```shell
+pip install litellm
+export OPENAI_API_KEY=...
+```
+
+### Run
+
+MODEL uses the format `<provider>/<model>`, e.g. `openai/gpt-4o-mini`.
+
+```shell
+MODEL=openai/gpt-4o-mini python litellm_sdk.py
+```
+
+### Behavior
+
+- Context Compiler runs before any LLM call.
+- If clarification is required, no LLM call is made.
+- Otherwise, compiled state is injected into the prompt before calling the model.
+
+## LiteLLM Proxy
+
+Gateway-level integration using a LiteLLM pre-call hook.
+
+See: [LiteLLM Proxy README](litellm_proxy/README.md)

examples/integrations/litellm_proxy/README.md

@@ -0,0 +1,59 @@
+# LiteLLM Proxy (pre-call hook)
+
+This example shows how to run Context Compiler inside a LiteLLM proxy pre-call hook.
+The hook enforces deterministic state handling before any upstream model call.
+
+### Requirements
+
+```shell
+pip install 'litellm[proxy]'
+export OPENAI_API_KEY=...
+```
+
+### Run proxy
+
+```shell
+litellm --config config.example.yaml
+```
+
+The proxy runs on `http://localhost:4000` by default.
+The proxy loads the Context Compiler pre-call hook from `context_compiler_precall_hook.py`.
+
+### Make a request
+
+```python
+from openai import OpenAI
+
+client = OpenAI(
+    api_key="anything",
+    base_url="http://localhost:4000",
+)
+
+response = client.chat.completions.create(
+    model="gpt-4o-mini",
+    messages=[{"role": "user", "content": "don't use peanuts"}],
+)
+```
+
+Or with curl:
+
+```shell
+curl http://localhost:4000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer anything" \
+  -d '{
+    "model": "gpt-4o-mini",
+    "messages": [{"role": "user", "content": "don'\''t use peanuts"}]
+  }'
+```
+
+### Behavior
+
+- User messages are replayed through Context Compiler before the model call.
+- If clarification is required, the proxy returns the clarification text as the response instead of calling the model.
+- Otherwise, compiled state is injected into a system message.
+
+### Note
+
+- The callback path in `config.example.yaml` must be importable.
+  Run the proxy from the repo root or set `PYTHONPATH` accordingly.

examples/integrations/litellm_proxy/config.example.yaml

@@ -0,0 +1,11 @@
+model_list:
+  # `model_name` is the client-facing alias sent to the proxy.
+  # `litellm_params.model` is the upstream provider/model LiteLLM calls.
+  - model_name: gpt-4o-mini
+    litellm_params:
+      model: openai/gpt-4o-mini
+      api_key: os.environ/OPENAI_API_KEY
+
+litellm_settings:
+  callbacks:
+    - examples.integrations.litellm_proxy.context_compiler_precall_hook.proxy_handler_instance

examples/integrations/litellm_proxy/context_compiler_precall_hook.py

@@ -0,0 +1,92 @@
+"""Minimal LiteLLM Proxy pre-call hook example.
+
+Architecture:
+- Replay user transcript through Context Compiler before any model call.
+- If clarification is required, block upstream model call.
+- Otherwise inject compiled state guidance into a system message.
+"""
+
+from typing import Any, Literal
+
+from litellm.integrations.custom_logger import CustomLogger  # type: ignore[import-not-found]
+
+from context_compiler import State, compile_transcript, get_focus_value, get_prohibited_items
+
+
+def _render_compiled_state_contract(compiled_state: State) -> str:
+    prohibited = get_prohibited_items(compiled_state)
+    focus_primary = get_focus_value(compiled_state)
+
+    lines: list[str] = ["The following constraints are authoritative."]
+    if prohibited:
+        items = ", ".join(prohibited)
+        lines.append(f"Never recommend or use prohibited items: {items}.")
+    if focus_primary:
+        lines.append(
+            f"When the answer depends on the current subject or selected option, "
+            f"treat the current focus as: {focus_primary}."
+        )
+    lines.append("If the user message conflicts with these constraints, follow them exactly.")
+
+    return "Host policy contract:\n" + "\n".join(f"- {line}" for line in lines)
+
+
+def _extract_request_messages(data: dict[str, object]) -> list[dict[str, object]]:
+    raw_messages = data.get("messages")
+    if not isinstance(raw_messages, list):
+        return []
+    return [msg for msg in raw_messages if isinstance(msg, dict)]
+
+
+def _extract_user_transcript(messages: list[dict[str, object]]) -> list[dict[str, object]]:
+    transcript: list[dict[str, object]] = []
+    for message in messages:
+        role = message.get("role")
+        content = message.get("content")
+        if role == "user" and isinstance(content, str):
+            transcript.append({"role": "user", "content": content})
+    return transcript
+
+
+class ContextCompilerPreCallHook(CustomLogger):  # type: ignore[misc]
+    async def async_pre_call_hook(
+        self,
+        user_api_key_dict: Any,
+        cache: Any,
+        data: dict[str, object],
+        call_type: Literal[
+            "completion",
+            "text_completion",
+            "embeddings",
+            "image_generation",
+            "moderation",
+            "audio_transcription",
+        ],
+    ) -> dict[str, object] | str:
+        del user_api_key_dict, cache
+        if call_type not in {"completion", "text_completion"}:
+            return data
+
+        request_messages = _extract_request_messages(data)
+        user_transcript = _extract_user_transcript(request_messages)
+        replay_result = compile_transcript(user_transcript)
+
+        if replay_result["kind"] == "confirm":
+            # Intentional minimal blocking behavior: for completion/text completion
+            # calls, returning a string here is treated by LiteLLM as a rejected
+            # assistant response, so the upstream model call is not executed.
+            return replay_result["prompt_to_user"] or "Confirmation required."
+
+        compiled_state = replay_result["state"]
+        system_message: dict[str, object] = {
+            "role": "system",
+            "content": "You are a helpful assistant.\n"
+            + _render_compiled_state_contract(compiled_state),
+        }
+        # Prepend one compiler contract system message, then forward the original
+        # request messages unchanged. Existing system messages are preserved.
+        data["messages"] = [system_message, *request_messages]
+        return data
+
+
+proxy_handler_instance = ContextCompilerPreCallHook()