From 9d08c2910315036b26170bf36229caa74cbb16f0 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 13 Dec 2025 21:36:00 +0000 Subject: [PATCH 01/11] Initial plan From eed6353030ddc609b12247e88e74e7daaaa5a90d Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 13 Dec 2025 21:43:07 +0000 Subject: [PATCH 02/11] Implement LLM chat client and CLI interface Co-authored-by: SorraTheOrc <250240+SorraTheOrc@users.noreply.github.com> --- scripts/echoes_llm_chat.py | 313 ++++++++++++++++++++++++++ src/gengine/echoes/llm/chat_client.py | 143 ++++++++++++ tests/echoes/test_llm_chat_cli.py | 268 ++++++++++++++++++++++ 3 files changed, 724 insertions(+) create mode 100755 scripts/echoes_llm_chat.py create mode 100644 src/gengine/echoes/llm/chat_client.py create mode 100644 tests/echoes/test_llm_chat_cli.py diff --git a/scripts/echoes_llm_chat.py b/scripts/echoes_llm_chat.py new file mode 100755 index 00000000..60b19486 --- /dev/null +++ b/scripts/echoes_llm_chat.py @@ -0,0 +1,313 @@ +#!/usr/bin/env python3 +"""Interactive chat interface for Echoes LLM service. + +This script provides a developer-facing REPL for testing the LLM service +endpoints (/parse_intent and /narrate) with multi-turn history support. +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import sys +import time +from pathlib import Path +from typing import Any + +import httpx + +# Add src to path for direct script execution +sys.path.insert(0, str(Path(__file__).parent.parent / "src")) + +from gengine.echoes.llm.chat_client import LLMChatClient + + +class ChatSession: + """Manages an interactive chat session with the LLM service.""" + + def __init__( + self, + service_url: str, + mode: str = "parse", + history_limit: int = 10, + context_file: str | None = None, + ) -> None: + """Initialize chat session. + + Parameters + ---------- + service_url + Base URL of the LLM service + mode + Chat mode: "parse" or "narrate" + history_limit + Maximum number of history entries to keep + context_file + Optional JSON file with initial context + """ + self.service_url = service_url + self.mode = mode + self.history_limit = history_limit + self.history: list[dict[str, str]] = [] + self.additional_context: dict[str, Any] = {} + + # Load context file if provided + if context_file: + try: + with open(context_file, "r") as f: + self.additional_context = json.load(f) + print(f"✓ Loaded context from {context_file}") + except Exception as e: + print(f"⚠ Failed to load context file: {e}") + + def add_to_history(self, role: str, content: str) -> None: + """Add an entry to the conversation history.""" + self.history.append({"role": role, "content": content}) + # Trim history to limit + if len(self.history) > self.history_limit * 2: # *2 for user + assistant pairs + self.history = self.history[-self.history_limit * 2:] + + def clear_history(self) -> None: + """Clear the conversation history.""" + self.history.clear() + print("✓ History cleared") + + def save_transcript(self, path: str) -> None: + """Save conversation transcript to JSON file.""" + try: + transcript = { + "mode": self.mode, + "service_url": self.service_url, + "history": self.history, + "context": self.additional_context, + } + with open(path, "w") as f: + json.dump(transcript, f, indent=2) + print(f"✓ Transcript saved to {path}") + except Exception as e: + print(f"✗ Failed to save transcript: {e}") + + def build_context(self) -> dict[str, Any]: + """Build context payload including history.""" + context = dict(self.additional_context) + if self.history: + context["history"] = self.history + return context + + async def handle_parse_mode( + self, + client: LLMChatClient, + user_input: str, + ) -> None: + """Handle parse intent mode.""" + start_time = time.perf_counter() + try: + response = await client.parse_intent( + user_input, + self.build_context(), + ) + latency_ms = (time.perf_counter() - start_time) * 1000 + + # Display intents + print("\n📋 Intents:") + print(json.dumps(response.get("intents", []), indent=2)) + + # Display metadata + print(f"\n⏱ Latency: {latency_ms:.0f}ms") + if "confidence" in response and response["confidence"] is not None: + print(f"🎯 Confidence: {response['confidence']:.2f}") + + # Add to history + self.add_to_history("user", user_input) + self.add_to_history("assistant", json.dumps(response.get("intents", []))) + + except httpx.HTTPStatusError as e: + print(f"\n✗ HTTP Error {e.response.status_code}: {e.response.text}") + except Exception as e: + print(f"\n✗ Error: {e}") + + async def handle_narrate_mode( + self, + client: LLMChatClient, + user_input: str, + ) -> None: + """Handle narrate mode. + + In narrate mode, user input is interpreted as JSON events array. + """ + start_time = time.perf_counter() + try: + # Try to parse user input as JSON events + try: + events = json.loads(user_input) + if not isinstance(events, list): + events = [events] + except json.JSONDecodeError: + # If not JSON, create a simple event + events = [{"type": "user_input", "content": user_input}] + + response = await client.narrate( + events, + self.build_context(), + ) + latency_ms = (time.perf_counter() - start_time) * 1000 + + # Display narrative + print(f"\n📖 Narrative:") + print(response.get("narrative", "")) + + # Display metadata + print(f"\n⏱ Latency: {latency_ms:.0f}ms") + if "metadata" in response and response["metadata"]: + metadata = response["metadata"] + if "input_tokens" in metadata: + print(f"📊 Tokens: {metadata.get('input_tokens', 0)} in / {metadata.get('output_tokens', 0)} out") + + # Add to history + self.add_to_history("user", json.dumps(events)) + self.add_to_history("assistant", response.get("narrative", "")) + + except httpx.HTTPStatusError as e: + print(f"\n✗ HTTP Error {e.response.status_code}: {e.response.text}") + except Exception as e: + print(f"\n✗ Error: {e}") + + async def run(self) -> None: + """Run the interactive chat session.""" + print(f"Echoes LLM Chat Interface") + print(f"Service: {self.service_url}") + print(f"Mode: {self.mode}") + print(f"History limit: {self.history_limit}") + print(f"\nCommands: /clear, /save , /quit") + print(f"{'=' * 60}\n") + + async with LLMChatClient(self.service_url) as client: + # Health check + try: + health = await client.health_check() + print(f"✓ Connected to {health.get('provider', 'unknown')} provider") + if health.get("model"): + print(f" Model: {health['model']}") + print() + except Exception as e: + print(f"⚠ Warning: Health check failed: {e}\n") + + # Main REPL loop + while True: + try: + # Read user input + if self.mode == "parse": + prompt = "You: " + else: + prompt = "Events (JSON or text): " + + user_input = input(prompt).strip() + + if not user_input: + continue + + # Handle slash commands + if user_input.startswith("/"): + if user_input == "/quit": + print("Goodbye!") + break + elif user_input == "/clear": + self.clear_history() + elif user_input.startswith("/save "): + path = user_input[6:].strip() + self.save_transcript(path) + else: + print(f"Unknown command: {user_input}") + continue + + # Route to appropriate handler + if self.mode == "parse": + await self.handle_parse_mode(client, user_input) + else: + await self.handle_narrate_mode(client, user_input) + + print() # Blank line for readability + + except KeyboardInterrupt: + print("\n\nGoodbye!") + break + except EOFError: + print("\n\nGoodbye!") + break + + +def main() -> int: + """Main entry point.""" + parser = argparse.ArgumentParser( + description="Interactive chat interface for Echoes LLM service", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Connect to local service in parse mode + python scripts/echoes_llm_chat.py --service-url http://localhost:8001 + + # Connect in narrate mode + python scripts/echoes_llm_chat.py --service-url http://localhost:8001 --mode narrate + + # Use custom context and history limit + python scripts/echoes_llm_chat.py --service-url http://localhost:8001 \\ + --context-file context.json --history-limit 20 + +Environment variables: + ECHOES_LLM_PROVIDER, ECHOES_LLM_API_KEY, ECHOES_LLM_MODEL + (configure the service, not this client) + """, + ) + + parser.add_argument( + "--service-url", + default="http://localhost:8001", + help="Base URL of the LLM service (default: http://localhost:8001)", + ) + parser.add_argument( + "--mode", + choices=["parse", "narrate"], + default="parse", + help="Chat mode: parse (intent JSON) or narrate (story text)", + ) + parser.add_argument( + "--context-file", + help="JSON file with initial context", + ) + parser.add_argument( + "--history-limit", + type=int, + default=10, + help="Maximum number of history entries to keep (default: 10)", + ) + parser.add_argument( + "--export", + help="Export transcript to this file on exit (deprecated: use /save command)", + ) + + args = parser.parse_args() + + # Create and run session + session = ChatSession( + service_url=args.service_url, + mode=args.mode, + history_limit=args.history_limit, + context_file=args.context_file, + ) + + try: + asyncio.run(session.run()) + + # Export transcript if requested + if args.export: + session.save_transcript(args.export) + + return 0 + except Exception as e: + print(f"\n✗ Fatal error: {e}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/gengine/echoes/llm/chat_client.py b/src/gengine/echoes/llm/chat_client.py new file mode 100644 index 00000000..45fc0c7e --- /dev/null +++ b/src/gengine/echoes/llm/chat_client.py @@ -0,0 +1,143 @@ +"""HTTP client for LLM service chat interface.""" + +from __future__ import annotations + +from typing import Any + +import httpx + + +class LLMChatClient: + """HTTP client for interacting with the Echoes LLM service. + + Wraps httpx.AsyncClient and provides methods to hit /parse_intent + and /narrate endpoints. + """ + + def __init__( + self, + base_url: str, + timeout: float = 30.0, + headers: dict[str, str] | None = None, + ) -> None: + """Initialize the LLM chat client. + + Parameters + ---------- + base_url + Base URL of the LLM service (e.g., "http://localhost:8001") + timeout + Request timeout in seconds + headers + Optional HTTP headers (e.g., for API keys) + """ + self.base_url = base_url.rstrip("/") + self.timeout = timeout + self.headers = headers or {} + self._client: httpx.AsyncClient | None = None + + async def __aenter__(self) -> LLMChatClient: + """Enter async context manager.""" + self._client = httpx.AsyncClient( + base_url=self.base_url, + timeout=self.timeout, + headers=self.headers, + ) + return self + + async def __aexit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: + """Exit async context manager.""" + if self._client is not None: + await self._client.aclose() + + async def parse_intent( + self, + user_input: str, + context: dict[str, Any] | None = None, + ) -> dict[str, Any]: + """Call /parse_intent endpoint. + + Parameters + ---------- + user_input + Natural language input from user + context + Game state context (history, metadata, etc.) + + Returns + ------- + dict + Response JSON from the service + + Raises + ------ + httpx.HTTPStatusError + If the request fails + """ + if self._client is None: + raise RuntimeError("Client not initialized. Use 'async with' context.") + + payload = { + "user_input": user_input, + "context": context or {}, + } + + response = await self._client.post("/parse_intent", json=payload) + response.raise_for_status() + return response.json() + + async def narrate( + self, + events: list[dict[str, Any]], + context: dict[str, Any] | None = None, + ) -> dict[str, Any]: + """Call /narrate endpoint. + + Parameters + ---------- + events + Game events to narrate + context + Game state context (history, metadata, etc.) + + Returns + ------- + dict + Response JSON from the service + + Raises + ------ + httpx.HTTPStatusError + If the request fails + """ + if self._client is None: + raise RuntimeError("Client not initialized. Use 'async with' context.") + + payload = { + "events": events, + "context": context or {}, + } + + response = await self._client.post("/narrate", json=payload) + response.raise_for_status() + return response.json() + + async def health_check(self) -> dict[str, Any]: + """Call /healthz endpoint. + + Returns + ------- + dict + Health check response + + Raises + ------ + httpx.HTTPStatusError + If the request fails + """ + if self._client is None: + raise RuntimeError("Client not initialized. Use 'async with' context.") + + response = await self._client.get("/healthz") + response.raise_for_status() + return response.json() diff --git a/tests/echoes/test_llm_chat_cli.py b/tests/echoes/test_llm_chat_cli.py new file mode 100644 index 00000000..ca1c0796 --- /dev/null +++ b/tests/echoes/test_llm_chat_cli.py @@ -0,0 +1,268 @@ +"""Tests for LLM chat CLI and client.""" + +from __future__ import annotations + +import json +from unittest.mock import AsyncMock, Mock + +import httpx +import pytest + +from gengine.echoes.llm.chat_client import LLMChatClient + + +pytestmark = pytest.mark.anyio + + +class TestLLMChatClient: + """Tests for LLMChatClient.""" + + async def test_context_manager(self) -> None: + """Test that client can be used as async context manager.""" + async with LLMChatClient("http://localhost:8001") as client: + assert client._client is not None + # Client should be closed after exiting context + # (no direct way to check httpx.AsyncClient.is_closed, but we can verify it doesn't raise) + + async def test_parse_intent_request_format(self) -> None: + """Test that parse_intent formats requests correctly.""" + def handler(request: httpx.Request) -> httpx.Response: + assert request.url.path == "/parse_intent" + payload = json.loads(request.content) + assert payload["user_input"] == "test input" + assert payload["context"] == {"history": [{"role": "user", "content": "hi"}]} + + return httpx.Response( + 200, + json={ + "intents": [{"type": "test"}], + "raw_response": "test", + "confidence": 0.9, + }, + ) + + transport = httpx.MockTransport(handler) + async with httpx.AsyncClient( + base_url="http://localhost:8001", + transport=transport, + ) as mock_client: + client = LLMChatClient("http://localhost:8001") + client._client = mock_client + + result = await client.parse_intent( + "test input", + {"history": [{"role": "user", "content": "hi"}]}, + ) + + assert result["intents"] == [{"type": "test"}] + assert result["confidence"] == 0.9 + + async def test_narrate_request_format(self) -> None: + """Test that narrate formats requests correctly.""" + def handler(request: httpx.Request) -> httpx.Response: + assert request.url.path == "/narrate" + payload = json.loads(request.content) + assert payload["events"] == [{"type": "test_event"}] + assert payload["context"] == {"tick": 10} + + return httpx.Response( + 200, + json={ + "narrative": "Test narrative", + "raw_response": "test", + "metadata": {"input_tokens": 10, "output_tokens": 20}, + }, + ) + + transport = httpx.MockTransport(handler) + async with httpx.AsyncClient( + base_url="http://localhost:8001", + transport=transport, + ) as mock_client: + client = LLMChatClient("http://localhost:8001") + client._client = mock_client + + result = await client.narrate( + [{"type": "test_event"}], + {"tick": 10}, + ) + + assert result["narrative"] == "Test narrative" + assert result["metadata"]["input_tokens"] == 10 + + async def test_health_check(self) -> None: + """Test health check endpoint.""" + def handler(request: httpx.Request) -> httpx.Response: + assert request.url.path == "/healthz" + return httpx.Response( + 200, + json={ + "status": "ok", + "provider": "stub", + "model": "N/A", + }, + ) + + transport = httpx.MockTransport(handler) + async with httpx.AsyncClient( + base_url="http://localhost:8001", + transport=transport, + ) as mock_client: + client = LLMChatClient("http://localhost:8001") + client._client = mock_client + + result = await client.health_check() + assert result["status"] == "ok" + assert result["provider"] == "stub" + + async def test_http_error_handling(self) -> None: + """Test that HTTP errors are raised properly.""" + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 500, + json={"detail": "Internal server error"}, + ) + + transport = httpx.MockTransport(handler) + async with httpx.AsyncClient( + base_url="http://localhost:8001", + transport=transport, + ) as mock_client: + client = LLMChatClient("http://localhost:8001") + client._client = mock_client + + with pytest.raises(httpx.HTTPStatusError): + await client.parse_intent("test") + + async def test_client_not_initialized(self) -> None: + """Test that calling methods without context manager raises error.""" + client = LLMChatClient("http://localhost:8001") + + with pytest.raises(RuntimeError, match="Client not initialized"): + await client.parse_intent("test") + + with pytest.raises(RuntimeError, match="Client not initialized"): + await client.narrate([{"type": "test"}]) + + with pytest.raises(RuntimeError, match="Client not initialized"): + await client.health_check() + + async def test_custom_headers(self) -> None: + """Test that custom headers are passed through.""" + def handler(request: httpx.Request) -> httpx.Response: + assert request.headers.get("X-API-Key") == "test-key" + return httpx.Response(200, json={"status": "ok"}) + + transport = httpx.MockTransport(handler) + client = LLMChatClient( + "http://localhost:8001", + headers={"X-API-Key": "test-key"}, + ) + + async with httpx.AsyncClient( + base_url="http://localhost:8001", + transport=transport, + headers={"X-API-Key": "test-key"}, + ) as mock_client: + client._client = mock_client + result = await client.health_check() + assert result["status"] == "ok" + + +class TestChatSession: + """Tests for ChatSession (imported from scripts).""" + + def test_history_management(self) -> None: + """Test history add and clear operations.""" + # Import here to avoid issues with script path manipulation + import sys + from pathlib import Path + sys.path.insert(0, str(Path(__file__).parent.parent.parent / "scripts")) + + from echoes_llm_chat import ChatSession + + session = ChatSession("http://localhost:8001", history_limit=2) + + # Add entries + session.add_to_history("user", "hello") + session.add_to_history("assistant", "hi") + assert len(session.history) == 2 + + # Add more entries to trigger limit + session.add_to_history("user", "message 2") + session.add_to_history("assistant", "response 2") + session.add_to_history("user", "message 3") + session.add_to_history("assistant", "response 3") + + # Should only keep last 2*2 entries (user+assistant pairs) + assert len(session.history) == 4 + assert session.history[0]["content"] == "message 2" + + # Clear history + session.clear_history() + assert len(session.history) == 0 + + def test_context_building(self) -> None: + """Test context building with history.""" + import sys + from pathlib import Path + sys.path.insert(0, str(Path(__file__).parent.parent.parent / "scripts")) + + from echoes_llm_chat import ChatSession + + session = ChatSession("http://localhost:8001") + session.additional_context = {"tick": 10} + session.add_to_history("user", "hello") + session.add_to_history("assistant", "hi") + + context = session.build_context() + assert context["tick"] == 10 + assert len(context["history"]) == 2 + assert context["history"][0]["role"] == "user" + + def test_save_transcript(self, tmp_path) -> None: + """Test saving transcript to file.""" + import sys + from pathlib import Path + sys.path.insert(0, str(Path(__file__).parent.parent.parent / "scripts")) + + from echoes_llm_chat import ChatSession + + session = ChatSession("http://localhost:8001", mode="parse") + session.add_to_history("user", "test") + session.add_to_history("assistant", "response") + + transcript_path = tmp_path / "transcript.json" + session.save_transcript(str(transcript_path)) + + # Verify file was created and contains expected data + assert transcript_path.exists() + with open(transcript_path) as f: + data = json.load(f) + + assert data["mode"] == "parse" + assert data["service_url"] == "http://localhost:8001" + assert len(data["history"]) == 2 + + def test_context_file_loading(self, tmp_path) -> None: + """Test loading initial context from file.""" + import sys + from pathlib import Path + sys.path.insert(0, str(Path(__file__).parent.parent.parent / "scripts")) + + from echoes_llm_chat import ChatSession + + # Create a context file + context_file = tmp_path / "context.json" + context_data = {"tick": 5, "district": "industrial"} + with open(context_file, "w") as f: + json.dump(context_data, f) + + # Create session with context file + session = ChatSession( + "http://localhost:8001", + context_file=str(context_file), + ) + + assert session.additional_context["tick"] == 5 + assert session.additional_context["district"] == "industrial" From 43dbb68a9315d54d1eddfbc7795658fbe18774f8 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 13 Dec 2025 21:44:11 +0000 Subject: [PATCH 03/11] Add comprehensive documentation for LLM chat harness Co-authored-by: SorraTheOrc <250240+SorraTheOrc@users.noreply.github.com> --- README.md | 201 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 201 insertions(+) diff --git a/README.md b/README.md index 976a30df..1962232e 100644 --- a/README.md +++ b/README.md @@ -703,6 +703,207 @@ The gateway: The integration uses HTTP retry logic (2 retries by default) and handles LLM service health checks on session creation. This enables conversational gameplay where players use natural language instead of memorizing CLI commands. +## LLM Chat Harness + +The LLM chat harness (`scripts/echoes_llm_chat.py`) provides an interactive REPL for testing the LLM service endpoints (`/parse_intent` and `/narrate`) with multi-turn history support. This tool is useful for: + +- Testing prompt changes and observing model responses +- Debugging latency and token usage +- Running scripted demos against remote environments +- Validating provider configurations (stub, OpenAI, Anthropic, Foundry) + +### Prerequisites + +1. **LLM Service Running**: Start the service locally or point to a remote instance: + ```bash + # Start local service with stub provider (default) + uv run echoes-llm-service + + # Or with OpenAI provider + export ECHOES_LLM_PROVIDER=openai + export ECHOES_LLM_API_KEY=your-api-key + export ECHOES_LLM_MODEL=gpt-4 + uv run echoes-llm-service + ``` + +2. **Python Environment**: The chat client requires `httpx` (already in project dependencies). + +### Basic Usage + +**Parse Mode** (default) - Convert natural language to intents: +```bash +uv run python scripts/echoes_llm_chat.py --service-url http://localhost:8001 + +# Example session: +You: inspect the industrial district +📋 Intents: +[ + { + "type": "inspect", + "target": "district" + } +] +⏱ Latency: 45ms +🎯 Confidence: 0.95 +``` + +**Narrate Mode** - Generate narrative from game events: +```bash +uv run python scripts/echoes_llm_chat.py --service-url http://localhost:8001 --mode narrate + +# Example session: +Events (JSON or text): [{"type": "pollution_increase", "district": "industrial", "amount": 5}] +📖 Narrative: +The industrial district's pollution levels rose sharply as factory output increased... +⏱ Latency: 120ms +📊 Tokens: 45 in / 32 out +``` + +### Command-Line Options + +- `--service-url URL`: Base URL of the LLM service (default: `http://localhost:8001`) +- `--mode MODE`: Chat mode - `parse` (intent JSON) or `narrate` (story text) (default: `parse`) +- `--context-file FILE`: Load initial context from JSON file +- `--history-limit N`: Maximum conversation history entries to keep (default: `10`) +- `--export FILE`: (deprecated) Export transcript on exit; use `/save` command instead + +### Slash Commands + +- `/clear` - Clear conversation history +- `/save ` - Save transcript to JSON file +- `/quit` - Exit the chat interface + +### Multi-Turn History + +The client automatically maintains conversation history and sends it with each request in the `context.history` field: + +```json +{ + "user_input": "what happened next?", + "context": { + "history": [ + {"role": "user", "content": "inspect district"}, + {"role": "assistant", "content": "[{\"type\": \"inspect\", \"target\": \"district\"}]"} + ] + } +} +``` + +History is limited by `--history-limit` (default 10 exchanges) to prevent unbounded token growth. + +### Context Files + +Use `--context-file` to provide initial game state context: + +```bash +# Create context.json +cat > context.json << EOF +{ + "tick": 42, + "district": "industrial-tier", + "recent_events": ["pollution_spike", "faction_meeting"] +} +EOF + +uv run python scripts/echoes_llm_chat.py --context-file context.json +``` + +The context merges with conversation history in subsequent requests. + +### Transcript Export + +Save conversation logs for analysis or sharing: + +```bash +# During session +You: inspect district +/save my_session.json +✓ Transcript saved to my_session.json +``` + +Transcript format: +```json +{ + "mode": "parse", + "service_url": "http://localhost:8001", + "history": [ + {"role": "user", "content": "inspect district"}, + {"role": "assistant", "content": "[{\"type\": \"inspect\"}]"} + ], + "context": {"tick": 42} +} +``` + +**Note**: API keys are NOT included in exported transcripts for security. + +### Remote Endpoints + +Point the client at any running LLM service: + +```bash +# Staging environment +uv run python scripts/echoes_llm_chat.py --service-url https://echoes-llm-staging.example.com + +# Docker Compose stack +uv run python scripts/echoes_llm_chat.py --service-url http://localhost:8001 + +# Kubernetes port-forward +kubectl port-forward svc/echoes-llm-service 8001:8001 +uv run python scripts/echoes_llm_chat.py --service-url http://localhost:8001 +``` + +### Troubleshooting + +**Connection Refused / Timeout:** +- Verify service is running: `curl http://localhost:8001/healthz` +- Check Docker: `docker ps | grep llm` +- Check Kubernetes: `kubectl get pods -l app=echoes-llm-service` + +**HTTP 500 / Provider Errors:** +- Check service logs for authentication failures (OpenAI/Anthropic API keys) +- Verify provider configuration: `curl http://localhost:8001/healthz` (shows provider + model) +- Test with stub provider first (no API keys required) + +**TLS Certificate Errors (remote endpoints):** +- Use `http://` for local/dev environments +- Verify certificate chain for production endpoints +- Check firewall rules and ingress configuration + +**Empty Responses:** +- Stub provider returns deterministic responses for testing +- Real providers may fail if quota exceeded or model unavailable +- Check token limits and rate limiting in service logs + +### Provider Configuration + +The chat client connects to the service; provider configuration happens server-side via environment variables: + +```bash +# Stub provider (default, no API key needed) +export ECHOES_LLM_PROVIDER=stub +uv run echoes-llm-service + +# OpenAI +export ECHOES_LLM_PROVIDER=openai +export ECHOES_LLM_API_KEY=sk-... +export ECHOES_LLM_MODEL=gpt-4 +uv run echoes-llm-service + +# Anthropic +export ECHOES_LLM_PROVIDER=anthropic +export ECHOES_LLM_API_KEY=sk-ant-... +export ECHOES_LLM_MODEL=claude-3-sonnet-20240229 +uv run echoes-llm-service + +# Foundry Local (self-hosted) +export ECHOES_LLM_PROVIDER=foundry_local +export ECHOES_LLM_BASE_URL=http://foundry:8000 +export ECHOES_LLM_MODEL=your-model-name +uv run echoes-llm-service +``` + +See `src/gengine/echoes/llm/settings.py` for all available configuration options. + ## Headless Regression Driver `scripts/run_headless_sim.py` advances long simulations without interactive From b56d99516c58cfbad77f85a82d01cf9bfbcc70c7 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 13 Dec 2025 21:46:21 +0000 Subject: [PATCH 04/11] Improve test imports using importlib instead of sys.path manipulation Co-authored-by: SorraTheOrc <250240+SorraTheOrc@users.noreply.github.com> --- tests/echoes/test_llm_chat_cli.py | 42 +++++++++++++++---------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/tests/echoes/test_llm_chat_cli.py b/tests/echoes/test_llm_chat_cli.py index ca1c0796..ce94de49 100644 --- a/tests/echoes/test_llm_chat_cli.py +++ b/tests/echoes/test_llm_chat_cli.py @@ -2,7 +2,9 @@ from __future__ import annotations +import importlib.util import json +from pathlib import Path from unittest.mock import AsyncMock, Mock import httpx @@ -14,6 +16,17 @@ pytestmark = pytest.mark.anyio +def _import_chat_script(): + """Import the echoes_llm_chat script module.""" + script_path = Path(__file__).parent.parent.parent / "scripts" / "echoes_llm_chat.py" + spec = importlib.util.spec_from_file_location("echoes_llm_chat", script_path) + if spec is None or spec.loader is None: + raise ImportError(f"Could not load script from {script_path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + class TestLLMChatClient: """Tests for LLMChatClient.""" @@ -174,12 +187,8 @@ class TestChatSession: def test_history_management(self) -> None: """Test history add and clear operations.""" - # Import here to avoid issues with script path manipulation - import sys - from pathlib import Path - sys.path.insert(0, str(Path(__file__).parent.parent.parent / "scripts")) - - from echoes_llm_chat import ChatSession + chat_module = _import_chat_script() + ChatSession = chat_module.ChatSession session = ChatSession("http://localhost:8001", history_limit=2) @@ -204,11 +213,8 @@ def test_history_management(self) -> None: def test_context_building(self) -> None: """Test context building with history.""" - import sys - from pathlib import Path - sys.path.insert(0, str(Path(__file__).parent.parent.parent / "scripts")) - - from echoes_llm_chat import ChatSession + chat_module = _import_chat_script() + ChatSession = chat_module.ChatSession session = ChatSession("http://localhost:8001") session.additional_context = {"tick": 10} @@ -222,11 +228,8 @@ def test_context_building(self) -> None: def test_save_transcript(self, tmp_path) -> None: """Test saving transcript to file.""" - import sys - from pathlib import Path - sys.path.insert(0, str(Path(__file__).parent.parent.parent / "scripts")) - - from echoes_llm_chat import ChatSession + chat_module = _import_chat_script() + ChatSession = chat_module.ChatSession session = ChatSession("http://localhost:8001", mode="parse") session.add_to_history("user", "test") @@ -246,11 +249,8 @@ def test_save_transcript(self, tmp_path) -> None: def test_context_file_loading(self, tmp_path) -> None: """Test loading initial context from file.""" - import sys - from pathlib import Path - sys.path.insert(0, str(Path(__file__).parent.parent.parent / "scripts")) - - from echoes_llm_chat import ChatSession + chat_module = _import_chat_script() + ChatSession = chat_module.ChatSession # Create a context file context_file = tmp_path / "context.json" From aad66702217cc1da7f67fa4d2afc0fea62405095 Mon Sep 17 00:00:00 2001 From: GitHub Copilot Date: Sat, 13 Dec 2025 14:10:17 -0800 Subject: [PATCH 05/11] Fix linting errors in LLM chat CLI files --- scripts/echoes_llm_chat.py | 10 ++++++---- tests/echoes/test_llm_chat_cli.py | 9 +++++---- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/scripts/echoes_llm_chat.py b/scripts/echoes_llm_chat.py index 60b19486..532011f9 100755 --- a/scripts/echoes_llm_chat.py +++ b/scripts/echoes_llm_chat.py @@ -154,7 +154,7 @@ async def handle_narrate_mode( latency_ms = (time.perf_counter() - start_time) * 1000 # Display narrative - print(f"\n📖 Narrative:") + print("\n📖 Narrative:") print(response.get("narrative", "")) # Display metadata @@ -162,7 +162,9 @@ async def handle_narrate_mode( if "metadata" in response and response["metadata"]: metadata = response["metadata"] if "input_tokens" in metadata: - print(f"📊 Tokens: {metadata.get('input_tokens', 0)} in / {metadata.get('output_tokens', 0)} out") + in_tokens = metadata.get('input_tokens', 0) + out_tokens = metadata.get('output_tokens', 0) + print(f"📊 Tokens: {in_tokens} in / {out_tokens} out") # Add to history self.add_to_history("user", json.dumps(events)) @@ -175,11 +177,11 @@ async def handle_narrate_mode( async def run(self) -> None: """Run the interactive chat session.""" - print(f"Echoes LLM Chat Interface") + print("Echoes LLM Chat Interface") print(f"Service: {self.service_url}") print(f"Mode: {self.mode}") print(f"History limit: {self.history_limit}") - print(f"\nCommands: /clear, /save , /quit") + print("\nCommands: /clear, /save , /quit") print(f"{'=' * 60}\n") async with LLMChatClient(self.service_url) as client: diff --git a/tests/echoes/test_llm_chat_cli.py b/tests/echoes/test_llm_chat_cli.py index ce94de49..b6c8539f 100644 --- a/tests/echoes/test_llm_chat_cli.py +++ b/tests/echoes/test_llm_chat_cli.py @@ -5,14 +5,12 @@ import importlib.util import json from pathlib import Path -from unittest.mock import AsyncMock, Mock import httpx import pytest from gengine.echoes.llm.chat_client import LLMChatClient - pytestmark = pytest.mark.anyio @@ -35,7 +33,8 @@ async def test_context_manager(self) -> None: async with LLMChatClient("http://localhost:8001") as client: assert client._client is not None # Client should be closed after exiting context - # (no direct way to check httpx.AsyncClient.is_closed, but we can verify it doesn't raise) + # (no direct way to check httpx.AsyncClient.is_closed, + # but we can verify it doesn't raise) async def test_parse_intent_request_format(self) -> None: """Test that parse_intent formats requests correctly.""" @@ -43,7 +42,9 @@ def handler(request: httpx.Request) -> httpx.Response: assert request.url.path == "/parse_intent" payload = json.loads(request.content) assert payload["user_input"] == "test input" - assert payload["context"] == {"history": [{"role": "user", "content": "hi"}]} + assert payload["context"] == { + "history": [{"role": "user", "content": "hi"}] + } return httpx.Response( 200, From 5ca8ec1c85e535138156de7ed32ed2c693b19477 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 13 Dec 2025 22:37:03 +0000 Subject: [PATCH 06/11] Add auto-detection of service URL and /exit alias Co-authored-by: SorraTheOrc <250240+SorraTheOrc@users.noreply.github.com> --- README.md | 22 +++++++- scripts/echoes_llm_chat.py | 85 +++++++++++++++++++++++++++++-- tests/echoes/test_llm_chat_cli.py | 33 ++++++++++++ 3 files changed, 133 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 1962232e..7b6d5816 100644 --- a/README.md +++ b/README.md @@ -730,6 +730,16 @@ The LLM chat harness (`scripts/echoes_llm_chat.py`) provides an interactive REPL ### Basic Usage +**Auto-detect Service** (no arguments needed): +```bash +# Auto-detects service URL (tries Windows host if in WSL, then localhost) +uv run python scripts/echoes_llm_chat.py + +# Output: +Auto-detecting LLM service... +✓ Detected service at http://localhost:8001 +``` + **Parse Mode** (default) - Convert natural language to intents: ```bash uv run python scripts/echoes_llm_chat.py --service-url http://localhost:8001 @@ -761,7 +771,7 @@ The industrial district's pollution levels rose sharply as factory output increa ### Command-Line Options -- `--service-url URL`: Base URL of the LLM service (default: `http://localhost:8001`) +- `--service-url URL`: Base URL of the LLM service (default: auto-detect) - `--mode MODE`: Chat mode - `parse` (intent JSON) or `narrate` (story text) (default: `parse`) - `--context-file FILE`: Load initial context from JSON file - `--history-limit N`: Maximum conversation history entries to keep (default: `10`) @@ -771,7 +781,7 @@ The industrial district's pollution levels rose sharply as factory output increa - `/clear` - Clear conversation history - `/save ` - Save transcript to JSON file -- `/quit` - Exit the chat interface +- `/quit` or `/exit` - Exit the chat interface ### Multi-Turn History @@ -854,10 +864,18 @@ uv run python scripts/echoes_llm_chat.py --service-url http://localhost:8001 ### Troubleshooting +**Auto-detection Fails:** +- The client tries to detect the service on: + 1. Windows host IP (when running in WSL) - reads from `/etc/resolv.conf` + 2. `http://localhost:8001` +- If both fail, manually specify with `--service-url` +- Check if service is running: `curl http://localhost:8001/healthz` + **Connection Refused / Timeout:** - Verify service is running: `curl http://localhost:8001/healthz` - Check Docker: `docker ps | grep llm` - Check Kubernetes: `kubectl get pods -l app=echoes-llm-service` +- If in WSL and accessing Windows host, ensure Windows Firewall allows port 8001 **HTTP 500 / Provider Errors:** - Check service logs for authentication failures (OpenAI/Anthropic API keys) diff --git a/scripts/echoes_llm_chat.py b/scripts/echoes_llm_chat.py index 60b19486..80f718e2 100755 --- a/scripts/echoes_llm_chat.py +++ b/scripts/echoes_llm_chat.py @@ -10,6 +10,9 @@ import argparse import asyncio import json +import os +import re +import subprocess import sys import time from pathlib import Path @@ -23,6 +26,61 @@ from gengine.echoes.llm.chat_client import LLMChatClient +def detect_service_url() -> str | None: + """Auto-detect the LLM service URL. + + Tries the following in order: + 1. Windows host IP (when running in WSL) + 2. localhost + + Returns + ------- + str | None + The detected service URL, or None if not found + """ + urls_to_try = [] + + # Check if running in WSL and try Windows host IP + if os.path.exists("/proc/version"): + try: + with open("/proc/version", "r") as f: + if "microsoft" in f.read().lower() or "wsl" in f.read().lower(): + # Running in WSL, try to get Windows host IP + try: + result = subprocess.run( + ["cat", "/etc/resolv.conf"], + capture_output=True, + text=True, + timeout=2, + ) + # Look for nameserver line which points to Windows host + for line in result.stdout.split("\n"): + if line.strip().startswith("nameserver"): + match = re.search(r"nameserver\s+(\S+)", line) + if match: + host_ip = match.group(1) + urls_to_try.append(f"http://{host_ip}:8001") + break + except (subprocess.TimeoutExpired, FileNotFoundError, PermissionError): + pass + except (FileNotFoundError, PermissionError): + pass + + # Always try localhost as fallback + urls_to_try.append("http://localhost:8001") + + # Try each URL with a quick health check + for url in urls_to_try: + try: + response = httpx.get(f"{url}/healthz", timeout=2.0) + if response.status_code == 200: + return url + except (httpx.ConnectError, httpx.TimeoutException, httpx.HTTPError): + continue + + return None + + class ChatSession: """Manages an interactive chat session with the LLM service.""" @@ -179,7 +237,7 @@ async def run(self) -> None: print(f"Service: {self.service_url}") print(f"Mode: {self.mode}") print(f"History limit: {self.history_limit}") - print(f"\nCommands: /clear, /save , /quit") + print(f"\nCommands: /clear, /save , /quit, /exit") print(f"{'=' * 60}\n") async with LLMChatClient(self.service_url) as client: @@ -209,7 +267,7 @@ async def run(self) -> None: # Handle slash commands if user_input.startswith("/"): - if user_input == "/quit": + if user_input in ("/quit", "/exit"): print("Goodbye!") break elif user_input == "/clear": @@ -244,6 +302,9 @@ def main() -> int: formatter_class=argparse.RawDescriptionHelpFormatter, epilog=""" Examples: + # Auto-detect service (tries WSL Windows host, then localhost) + python scripts/echoes_llm_chat.py + # Connect to local service in parse mode python scripts/echoes_llm_chat.py --service-url http://localhost:8001 @@ -262,8 +323,8 @@ def main() -> int: parser.add_argument( "--service-url", - default="http://localhost:8001", - help="Base URL of the LLM service (default: http://localhost:8001)", + default=None, + help="Base URL of the LLM service (default: auto-detect)", ) parser.add_argument( "--mode", @@ -288,9 +349,23 @@ def main() -> int: args = parser.parse_args() + # Auto-detect service URL if not provided + service_url = args.service_url + if service_url is None: + print("Auto-detecting LLM service...") + service_url = detect_service_url() + if service_url is None: + print("\n✗ Error: Could not detect LLM service.", file=sys.stderr) + print(" Tried:", file=sys.stderr) + print(" - Windows host (if running in WSL)", file=sys.stderr) + print(" - http://localhost:8001", file=sys.stderr) + print("\n Please ensure the service is running or specify --service-url", file=sys.stderr) + return 1 + print(f"✓ Detected service at {service_url}\n") + # Create and run session session = ChatSession( - service_url=args.service_url, + service_url=service_url, mode=args.mode, history_limit=args.history_limit, context_file=args.context_file, diff --git a/tests/echoes/test_llm_chat_cli.py b/tests/echoes/test_llm_chat_cli.py index ce94de49..f191e14e 100644 --- a/tests/echoes/test_llm_chat_cli.py +++ b/tests/echoes/test_llm_chat_cli.py @@ -182,6 +182,39 @@ def handler(request: httpx.Request) -> httpx.Response: assert result["status"] == "ok" +class TestAutoDetection: + """Tests for service URL auto-detection.""" + + def test_detect_service_url_success(self) -> None: + """Test successful service detection.""" + import httpx + from unittest.mock import patch + + chat_module = _import_chat_script() + detect_service_url = chat_module.detect_service_url + + # Mock httpx.get to simulate successful connection + with patch("httpx.get") as mock_get: + mock_get.return_value.status_code = 200 + result = detect_service_url() + assert result is not None + assert "8001" in result + + def test_detect_service_url_failure(self) -> None: + """Test when no service is found.""" + import httpx + from unittest.mock import patch + + chat_module = _import_chat_script() + detect_service_url = chat_module.detect_service_url + + # Mock httpx.get to simulate connection failure + with patch("httpx.get") as mock_get: + mock_get.side_effect = httpx.ConnectError("Connection refused") + result = detect_service_url() + assert result is None + + class TestChatSession: """Tests for ChatSession (imported from scripts).""" From b4883aa07a2c992c5719d83f8a42332687aab6d4 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 13 Dec 2025 22:40:00 +0000 Subject: [PATCH 07/11] Fix code review issues: file read and import placement Co-authored-by: SorraTheOrc <250240+SorraTheOrc@users.noreply.github.com> --- scripts/echoes_llm_chat.py | 3 ++- tests/echoes/test_llm_chat_cli.py | 8 +------- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/scripts/echoes_llm_chat.py b/scripts/echoes_llm_chat.py index 80f718e2..cf5049a5 100755 --- a/scripts/echoes_llm_chat.py +++ b/scripts/echoes_llm_chat.py @@ -44,7 +44,8 @@ def detect_service_url() -> str | None: if os.path.exists("/proc/version"): try: with open("/proc/version", "r") as f: - if "microsoft" in f.read().lower() or "wsl" in f.read().lower(): + version_content = f.read().lower() + if "microsoft" in version_content or "wsl" in version_content: # Running in WSL, try to get Windows host IP try: result = subprocess.run( diff --git a/tests/echoes/test_llm_chat_cli.py b/tests/echoes/test_llm_chat_cli.py index f191e14e..f4bd1a08 100644 --- a/tests/echoes/test_llm_chat_cli.py +++ b/tests/echoes/test_llm_chat_cli.py @@ -5,7 +5,7 @@ import importlib.util import json from pathlib import Path -from unittest.mock import AsyncMock, Mock +from unittest.mock import AsyncMock, Mock, patch import httpx import pytest @@ -187,9 +187,6 @@ class TestAutoDetection: def test_detect_service_url_success(self) -> None: """Test successful service detection.""" - import httpx - from unittest.mock import patch - chat_module = _import_chat_script() detect_service_url = chat_module.detect_service_url @@ -202,9 +199,6 @@ def test_detect_service_url_success(self) -> None: def test_detect_service_url_failure(self) -> None: """Test when no service is found.""" - import httpx - from unittest.mock import patch - chat_module = _import_chat_script() detect_service_url = chat_module.detect_service_url From a933227b62304bcf10d63953e4bade6c984866e1 Mon Sep 17 00:00:00 2001 From: GitHub Copilot Date: Sat, 13 Dec 2025 15:06:20 -0800 Subject: [PATCH 08/11] Fix indentation error in chat CLI script --- scripts/echoes_llm_chat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/echoes_llm_chat.py b/scripts/echoes_llm_chat.py index 8c8a9350..a6666fdf 100755 --- a/scripts/echoes_llm_chat.py +++ b/scripts/echoes_llm_chat.py @@ -240,7 +240,7 @@ async def run(self) -> None: print(f"Service: {self.service_url}") print(f"Mode: {self.mode}") print(f"History limit: {self.history_limit}") - print("\nCommands: /clear, /save , /quit, /exit") + print("\nCommands: /clear, /save , /quit, /exit") print(f"{'=' * 60}\n") async with LLMChatClient(self.service_url) as client: From 91a33dc9f5a7c534652c2e78885af9111de0f551 Mon Sep 17 00:00:00 2001 From: GitHub Copilot Date: Sat, 13 Dec 2025 15:07:49 -0800 Subject: [PATCH 09/11] Fix LLMClient tests to use 127.0.0.1 to avoid WSL auto-discovery --- tests/echoes/test_gateway_llm_client.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/echoes/test_gateway_llm_client.py b/tests/echoes/test_gateway_llm_client.py index 4792db96..d0fcefac 100644 --- a/tests/echoes/test_gateway_llm_client.py +++ b/tests/echoes/test_gateway_llm_client.py @@ -13,8 +13,8 @@ class TestLLMClient: def test_initialization(self): """Test client initialization.""" - client = LLMClient("http://localhost:8001", timeout=10.0, max_retries=3) - assert client.base_url == "http://localhost:8001" + client = LLMClient("http://127.0.0.1:8001", timeout=10.0, max_retries=3) + assert client.base_url == "http://127.0.0.1:8001" assert client.timeout == 10.0 assert client.max_retries == 3 client.close() @@ -22,8 +22,8 @@ def test_initialization(self): @patch("httpx.Client") def test_context_manager(self, mock_client): """Test client works as context manager.""" - with LLMClient("http://localhost:8001") as client: - assert client.base_url == "http://localhost:8001" + with LLMClient("http://127.0.0.1:8001") as client: + assert client.base_url == "http://127.0.0.1:8001" @patch("httpx.Client.post") def test_parse_intent_success(self, mock_post): From d35376c2bf28aba53832386eb5a8322e594f087b Mon Sep 17 00:00:00 2001 From: GitHub Copilot Date: Sat, 13 Dec 2025 15:11:15 -0800 Subject: [PATCH 10/11] Add logging for auto-discovered Windows host IP in WSL --- src/gengine/echoes/gateway/llm_client.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/gengine/echoes/gateway/llm_client.py b/src/gengine/echoes/gateway/llm_client.py index 1d570484..40b4e777 100644 --- a/src/gengine/echoes/gateway/llm_client.py +++ b/src/gengine/echoes/gateway/llm_client.py @@ -46,6 +46,7 @@ def __init__( if line.startswith("default"): win_host_ip = line.split()[2] base_url = f"http://{win_host_ip}:8001" + print(f"Auto-discovered Windows host: {base_url}") break except Exception: pass From 03bb0c96981fd6a1d5cb43bb7896992d9b2d40ce Mon Sep 17 00:00:00 2001 From: GitHub Copilot Date: Sat, 13 Dec 2025 22:09:47 -0800 Subject: [PATCH 11/11] Update LLM client and chat script: unify WSL IP detection, improve logging, and update tracker --- .pm/tracker.md | 76 +++++++++++++++++++++--- gamedev-agent-thoughts.txt | 15 +++++ scripts/echoes_llm_chat.py | 27 ++++----- src/gengine/echoes/gateway/llm_client.py | 4 +- 4 files changed, 95 insertions(+), 27 deletions(-) diff --git a/.pm/tracker.md b/.pm/tracker.md index cd470392..28877c4e 100644 --- a/.pm/tracker.md +++ b/.pm/tracker.md @@ -1,6 +1,6 @@ # Project Task Tracker -**Last Updated:** 2025-12-08T07:42:16Z +**Last Updated:** 2025-12-14T05:53:29Z ## Quick Status Dashboard @@ -13,11 +13,15 @@ | 12.2.1 | Management Depth UI | complete | High | 12.1.1 | UI Team | 2025-12-07 | | 12.2.2 | Agent Roster Panel | complete | High | 12.2.1 | UI Team | 2025-12-07 | | 12.2.3 | Player Interactivity & UI Wiring | ✅ complete | High | 12.2.2 | UI Team | 2025-12-08 | +| 13.1.1 | Build Test Chat Interface | not-started | High | None | LLM Team | 2025-12-13 | +| 13.1.2 | Add RAG pipeline to LLM service | not-started | High | 13.1.1 | LLM Team | 2025-12-14 | **Active Tasks:** +- 🆕 **13.1.2** - Add RAG pipeline to LLM service (Issue #91) - **NOT STARTED** (created 2025-12-14) +- 🆕 **13.1.1** - Build Test Chat Interface (Issue #89) - **NOT STARTED** (created 2025-12-13) - ✅ **11.5.1** - CI Integration for Balance Validation - **COMPLETED** (merged 2025-12-05) - ✅ **10.1.9** - Comprehensive Scripts Test Coverage - **COMPLETED** (merged 2025-12-05) - ✅ **12.2.3** - Player Interactivity & UI Wiring (Issue #84) - **COMPLETED** (merged 2025-12-08) @@ -45,6 +49,8 @@ 5. Submit for code review and merge. - **Last Updated:** 2025-12-07 +5. **13.1.1** - Build Test Chat Interface (Issue #89) - Define CLI UX + httpx helper so LLM work can be validated manually. +6. **13.1.2** - Add RAG pipeline to LLM service (Issue #91) - Ground `/parse_intent` + `/narrate` with retrieved context per Semantic Kernel playbook. ## Comprehensive Project Status Report @@ -65,7 +71,7 @@ - Total tests: 1,042 (up from 849; +193 new tests) - Coverage: 91.37% overall (up from 90.95%), critical modules at 94-98%, scripts at 88.6% -- Open issues: 2 (Issue #70 - Designer Tooling, Issue #71 - Parameter Optimization; both low priority) +- Open issues: 4 (Issue #70 - Designer Tooling, Issue #71 - Parameter Optimization, Issue #89 - LLM Chat Harness, Issue #91 - LLM RAG pipeline) - Recent commits: 20+ commits in past 24 hours, excellent delivery pace - Repository hygiene: Excellent - clean issue backlog, well-documented - **Phase 11 Progress:** 4 of 6 milestones complete (11.1 Batch Sweeps, 11.2 Result Aggregation, 11.3 Analysis & Reporting, 11.5 CI Integration) @@ -77,6 +83,9 @@ **Recent Progress (since last update):** +- 🆕 **Task 13.1.2 (Add RAG pipeline to LLM service)** - Issue [#91](https://github.com/TheWizardsCode/GEngine/issues/91) opened 2025-12-14 outlining ingestion tooling, retrieval hooks, telemetry, and tests per Microsoft Semantic Kernel reference. +- 🆕 **Task 13.1.1 (Build Test Chat Interface)** - Issue [#89](https://github.com/TheWizardsCode/GEngine/issues/89) opened 2025-12-13 with CLI workflow, HTTP helper, docs, and test plan for exercising the LLM service manually. + - 🎉 **Task 12.1.1 (Terminal UI Core Implementation) COMPLETED** - Merged to main 2025-12-07 - Documented all Terminal UI views and keyboard controls in `docs/gengine/how_to_play_echoes.md` - Updated agent and faction view logic for GameState compatibility @@ -364,11 +373,13 @@ All tasks are either complete or unblocked and ready to start. | **Phase prioritization unclear** | Low | Resource allocation between Phase 11 completion vs. Phase 12 start | 🟡 Awaiting PM decision | | **UI implementation scope large** | Medium | Phase 12 has 5 substantial milestones; may need dedicated sprint | 📋 Planned, not yet started | | **Balance CI integration complexity** | Low | Task 11.5.1 requires careful baseline management and threshold tuning | 📋 Documented in task, ready to start | +| **LLM service lacks manual chat harness** | Medium | Hard to validate provider regressions without developer tooling | 🛠️ Task 13.1.1 / Issue #89 planned | +| **LLM intents lack RAG grounding** | Medium | Without retrieval context, LLM outputs may drift from canon | 🛠️ Task 13.1.2 / Issue #91 opened to implement RAG | ### 🔄 Monitoring - **Test Coverage:** Improved to 91.37% (up from 90.95%); scripts module at 88.6% -- **Issue Backlog:** Clean (1 open issue, just created) +- **Issue Backlog:** 4 open issues (#70, #71, #89, #91) with owners + next steps - **PR Queue:** Empty - excellent merge velocity - **Documentation Drift:** None detected - docs updated with each milestone @@ -465,14 +476,61 @@ The project has closely followed the implementation plan with excellent tracking | ID | Task | Status | Priority | Dependencies | Responsible | Updated | | -------: | ------------------------------------- | ----------- | -------- | ------------ | ---------------- | ---------- | -| 13.1.1 | Build Test Chat Interface (TinyLlama) | in-progress | High | None | LLM Team | 2025-12-09 | +| 13.1.1 | Build Test Chat Interface (echoes_llm_service) | not-started | High | None | LLM Team | 2025-12-13 | +| 13.1.2 | Add RAG pipeline to LLM service | not-started | High | 13.1.1 | LLM Team | 2025-12-14 | + +### 13.1.1 — Build Test Chat Interface (echoes_llm_service) + +- **Description:** Build a lightweight Python CLI chat harness that targets `echoes_llm_service` so engineers, PMs, and designers can manually exercise `/parse_intent` and `/narrate` against stub or real providers. +- **Acceptance Criteria:** + - `uv run python scripts/echoes_llm_chat.py --service-url http://localhost:8001` connects to the stub provider and supports interactive multi-turn chats. + - CLI supports `--mode parse|narrate`, optional context injection (`--context-file`), slash commands (`/clear`, `/save `, `/quit`), and `--history-limit`. + - Requests include prior turns when history is enabled; transcripts export to JSON; errors surface readable messages and set non-zero exit codes. + - README (or linked doc) describes setup, env vars (`ECHOES_LLM_*`), sample usage, and troubleshooting for local vs. remote endpoints. + - Automated tests cover HTTP payload formation, history management, export/reset commands, and error handling using mocked transports. +- **Priority:** High +- **Responsible:** LLM Team (owner TBD) +- **Dependencies:** None +- **Status:** not-started +- **Linked Issue:** [#89](https://github.com/TheWizardsCode/GEngine/issues/89) +- **Risks & Mitigations:** + - Risk: Provider regressions go undetected without manual harness; mitigation: deliver CLI defaults to stub provider and capture telemetry. + - Risk: Transcript exports could leak secrets; mitigation: redact sensitive env vars and allow configurable history trimming. +- **Testing Owner:** `test_agent` +- **Next Steps:** + 1. Finalize CLI spec/flags and align with UX expectations. + 2. Implement reusable HTTP helper (`src/gengine/echoes/llm/chat_client.py`) with telemetry extraction. + 3. Build the interactive script with prompt loop + slash commands + transcript export. + 4. Partner with `test_agent` to add `tests/echoes/test_llm_chat_cli.py` using mocked transports. + 5. Document usage and troubleshooting in README LLM section. +- **Last Updated:** 2025-12-14 -### 13.1.1 — Build Test Chat Interface (TinyLlama) +### 13.1.2 — Add RAG pipeline to LLM service -- **Description:** Implement a simple Python chat interface using TinyLlama-1.1B-Chat-v1.0-ONNX running on a Copilot+ PC with Snapdragon NPU. Use ONNX Runtime with QNNExecutionProvider for hardware acceleration. The interface should support conversational input/output and run locally. - - Environment subsystem (pollution, diffusion, biodiversity, stability) - - Narrative director with story seeds, pacing, lifecycle management - - LLM integration (OpenAI, Anthropic providers) with intent parsing +- **Description:** Adapt the Semantic Kernel + Foundry Local RAG approach (see [Microsoft TechCommunity article](https://techcommunity.microsoft.com/blog/educatordeveloperblog/building-enterprise-grade-local-rag-applications-with-semantic-kernel-and-foundr/4433945)) into our Python-based LLM service so `/parse_intent` and `/narrate` are grounded in curated Echoes documentation. +- **Acceptance Criteria:** + - `scripts/build_llm_knowledge_base.py` ingests configured corpora, chunkifies content, generates embeddings via the active provider, and writes a deterministic local index. + - Enabling `ECHOES_LLM_ENABLE_RAG=true` causes the LLM service to retrieve top-K snippets and append them (with citations) to provider prompts for both endpoints. + - Retrieval failures fall back gracefully while emitting actionable warnings/telemetry; Prometheus metrics expose `rag_hits`, `rag_latency`, and `rag_context_chars`. + - CLI tooling and docs explain how to rebuild the knowledge base, point at Foundry Local vs. cloud providers, and debug retrieval results. + - Automated tests cover chunking, embedding request formation, retrieval filtering, and endpoint wiring (owned by `test_agent`). +- **Priority:** High +- **Responsible:** LLM Team (owner TBD) +- **Dependencies:** 13.1.1 (chat harness useful for validation) +- **Status:** not-started +- **Linked Issue:** [#91](https://github.com/TheWizardsCode/GEngine/issues/91) +- **Risks & Mitigations:** + - Risk: Embedding provider differences complicate ingestion; mitigation: wrap calls via Semantic Kernel abstractions and document per-provider requirements. + - Risk: Index bloat/stale docs; mitigation: include hashing + `--clean` flag so rebuilds stay deterministic. + - Risk: Added latency; mitigation: configurable `rag_top_k`, caching, and visible metrics. +- **Testing Owner:** `test_agent` +- **Next Steps:** + 1. Implement ingestion/embedding script with configurable corpora and providers. + 2. Add retriever module + settings toggles inside LLM service, including graceful fallbacks. + 3. Wire prompts to include retrieved snippets + citation metadata for OpenAI/Anthropic/Foundry providers. + 4. Capture telemetry + debugging endpoints for retrieval context. + 5. Add docs + troubleshooting plus unit/integration tests. +- **Last Updated:** 2025-12-14 ### Phase 7: Player Experience ✅ COMPLETE (100%) diff --git a/gamedev-agent-thoughts.txt b/gamedev-agent-thoughts.txt index 5de51ef6..7f89ea5f 100644 --- a/gamedev-agent-thoughts.txt +++ b/gamedev-agent-thoughts.txt @@ -977,3 +977,18 @@ Created 2 new modules and enhanced 2 existing files: ## [GitHub Copilot] — 2025-12-12 10:05 - (n/a): Reviewed the README, Azure Foundry Local REST reference, and LLM service modules to outline the work required for wiring the LLM service to the Foundry Local API. +## [GitHub Copilot] — 2025-12-13 14:30 +- Merged PR #90 (LLM Chat Harness) into local branch. +- Resolved merge conflicts in `scripts/echoes_llm_chat.py` and `tests/echoes/test_llm_chat_cli.py`. +- Fixed indentation error in `scripts/echoes_llm_chat.py`. +- Fixed test failures in `tests/echoes/test_gateway_llm_client.py` by updating test expectations to use `127.0.0.1` instead of `localhost` to avoid WSL auto-discovery logic interference. +- Verified all relevant tests pass: `tests/echoes/test_llm_chat_cli.py` and `tests/echoes/test_gateway_llm_client.py`. +- Reviewed and confirmed completeness of PR documentation in README.md. + + +## [GitHub Copilot] — 2025-12-13 14:45 +- d35376c: Added logging to `LLMClient` to print the auto-discovered Windows host IP when running in WSL, aiding in debugging connection issues. Verified with existing tests. +## [tracker_agent] — 2025-12-14 05:53 +- Reviewed README plus latest repo state, then refreshed .pm/tracker.md header, dashboard rows, risk table, and monitoring stats to reflect new LLM tasks and issue counts. +- Captured detailed briefs for Tasks 13.1.1 and 13.1.2 (chat harness + RAG) with acceptance criteria, dependencies, test ownership, and next steps; added corresponding risk entries. +- Created GitHub Issue #91 for the RAG pipeline referencing the Microsoft Semantic Kernel article and Python adaptation plan. diff --git a/scripts/echoes_llm_chat.py b/scripts/echoes_llm_chat.py index a6666fdf..e3b52930 100755 --- a/scripts/echoes_llm_chat.py +++ b/scripts/echoes_llm_chat.py @@ -48,22 +48,16 @@ def detect_service_url() -> str | None: if "microsoft" in version_content or "wsl" in version_content: # Running in WSL, try to get Windows host IP try: - result = subprocess.run( - ["cat", "/etc/resolv.conf"], - capture_output=True, - text=True, - timeout=2, - ) - # Look for nameserver line which points to Windows host - for line in result.stdout.split("\n"): - if line.strip().startswith("nameserver"): - match = re.search(r"nameserver\s+(\S+)", line) - if match: - host_ip = match.group(1) - urls_to_try.append(f"http://{host_ip}:8001") - break - except (subprocess.TimeoutExpired, FileNotFoundError, PermissionError): - pass + result = subprocess.run(["ip", "route"], capture_output=True, text=True) + for line in result.stdout.splitlines(): + if line.startswith("default"): + win_host_ip = line.split()[2] + base_url = f"http://{win_host_ip}:8001" + print(f"Auto-discovered Windows host: {base_url}") + urls_to_try.append(base_url) + break + except Exception as e: + print(f"⚠ Failed to auto-discover Windows host IP: {e}") except (FileNotFoundError, PermissionError): pass @@ -73,6 +67,7 @@ def detect_service_url() -> str | None: # Try each URL with a quick health check for url in urls_to_try: try: + print("Trying LLM service URL:", url) response = httpx.get(f"{url}/healthz", timeout=2.0) if response.status_code == 200: return url diff --git a/src/gengine/echoes/gateway/llm_client.py b/src/gengine/echoes/gateway/llm_client.py index 40b4e777..00a10431 100644 --- a/src/gengine/echoes/gateway/llm_client.py +++ b/src/gengine/echoes/gateway/llm_client.py @@ -48,8 +48,8 @@ def __init__( base_url = f"http://{win_host_ip}:8001" print(f"Auto-discovered Windows host: {base_url}") break - except Exception: - pass + except Exception as e: + LOGGER.warning(f"Failed to auto-discover Windows host IP: {e}") if not base_url: raise RuntimeError("Could not determine LLM service URL. Set LLM_SERVICE_URL or provide base_url.") self.base_url = base_url.rstrip("/")