diff --git a/.gitignore b/.gitignore index c6125d8..31bebe5 100644 --- a/.gitignore +++ b/.gitignore @@ -40,4 +40,6 @@ htmlcov/ .pytest_cache/ # AI -.claude/ \ No newline at end of file +.claude/ +e2e/llm/transcripts/*.md + diff --git a/Makefile b/Makefile index 021dd18..acafa81 100644 --- a/Makefile +++ b/Makefile @@ -86,6 +86,12 @@ examples: pyparser pddlpy/pddl.py @echo "Running examples..." for f in examples/[0-9][0-9]_*.py ; do echo "=== $$f ===" ; $(PYTHON) $$f ; done +# LLM + solver end-to-end (#106): MANUAL, non-deterministic, never in CI. +# Needs OPENROUTER_API_KEY (or use `--dry-run` via the script directly). +MODEL?=qwen/qwen-2.5-72b-instruct +e2e-llm: + $(UV) run --with mcp,httpx python e2e/llm/harness.py --model $(MODEL) + # Test published package from TestPyPI using Docker testpublish: @echo "Testing pddlpy installation from TestPyPI..." @@ -117,5 +123,6 @@ help: @echo " demo - Run demo scripts" @echo " examples - Run the example showcase (examples/)" @echo " testpublish - Test package installation from TestPyPI (Docker)" + @echo " e2e-llm - Manual LLM+solver end-to-end (MODEL=..., needs OPENROUTER_API_KEY)" -.PHONY: all init testgrammar pyparser test coverage lint typecheck build clean demo testpublish help +.PHONY: all init testgrammar pyparser test coverage lint typecheck build clean demo testpublish e2e-llm help diff --git a/e2e/llm/README.md b/e2e/llm/README.md new file mode 100644 index 0000000..be7b92c --- /dev/null +++ b/e2e/llm/README.md @@ -0,0 +1,45 @@ +# LLM + solver end-to-end (#106) + +Manual, non-deterministic — **never wired into CI**. Reproduces +[docs/llm-interaction.md](../../docs/llm-interaction.md) with a *real* model: +the LLM gets the courier problem in natural language plus tools (a local +`write_file` and the pddlpy MCP tools `validate`/`ground`/`solve`/`parse`, +served by `uvx --from "pddlpy[mcp]" pddlpy-mcp` — the published package in a +transient environment). Each run captures a markdown transcript under +`transcripts/` and checks soundness: `validate` was used, `solve` returned the +7-toll optimum, and the final answer narrates it. + +## Small / open models — OpenRouter + +```bash +export OPENROUTER_API_KEY=... +make e2e-llm MODEL=qwen/qwen-2.5-72b-instruct +# equivalently: +uv run --with mcp,httpx python e2e/llm/harness.py --model qwen/qwen-2.5-72b-instruct +``` + +Exit codes: `0` sound, `1` finished but unsound (read the transcript), `2` +setup error. No key at hand? `--dry-run` replays the doc's scripted turns +through the entire pipeline (MCP server, tool dispatch, transcript, +soundness) without any API call. + +## Mid-tier models — CLI agents + +For models we reach through coding agents rather than raw APIs, drive the +same MCP server through the agent's own harness: + +```bash +./e2e/llm/claude_driver.sh # Claude Code: claude -p + --mcp-config +# codex exec works analogously with its MCP configuration +``` + +The driver prints where it saved the transcript; judge soundness with the +same criteria (validate used, cost 7.0, route narrated). + +## Iterating (issue steps 5–8) + +Repeat runs (vary model / round) until a transcript is sound *and reads +well*; then replace the simulated conversation in `docs/llm-interaction.md` +with that transcript (label it with model + date, and turn the "what is +simulated" box into "how this was captured"). Transcripts are gitignored — +commit only the winning one alongside the doc update. diff --git a/e2e/llm/claude_driver.sh b/e2e/llm/claude_driver.sh new file mode 100755 index 0000000..6debd10 --- /dev/null +++ b/e2e/llm/claude_driver.sh @@ -0,0 +1,34 @@ +#!/bin/sh +# Mid-tier driver (#106): let Claude Code solve the courier problem through +# the pddlpy MCP server. Requires the `claude` CLI. Manual — not CI. +set -eu + +HERE=$(cd "$(dirname "$0")" && pwd) +STAMP=$(date +%Y%m%d-%H%M%S) +OUT="$HERE/transcripts/claude-$STAMP.md" +MCPCFG=$(mktemp) +trap 'rm -f "$MCPCFG"' EXIT + +cat > "$MCPCFG" <<'EOF' +{"mcpServers": {"pddlpy": {"command": "uvx", "args": ["--from", "pddlpy[mcp]", "pddlpy-mcp"]}}} +EOF + +PROMPT='A courier is at the depot and must deliver a parcel to town, spending as +little as possible on tolls. The direct highway costs 10. There is a route +through the junction: 5 to get there, then 6 into town. And there is a +back-road chain: riverside (2), then the old bridge (2), then into town (3). +Roads are one-way. What is the cheapest route? + +Do not compute the plan yourself. Translate the problem to PDDL (domain + +problem files with :action-costs and a minimize metric), run the pddlpy +validate tool and fix any issue, run the solve tool with planner "ucs", and +answer in plain language based only on the solver output.' + +claude -p "$PROMPT" \ + --mcp-config "$MCPCFG" \ + --allowedTools "Write,mcp__pddlpy__validate,mcp__pddlpy__solve,mcp__pddlpy__parse,mcp__pddlpy__ground" \ + --output-format text | tee "$OUT" + +echo +echo "transcript: $OUT" +echo "judge soundness: validate used, solve cost 7.0, route narrated" diff --git a/e2e/llm/harness.py b/e2e/llm/harness.py new file mode 100644 index 0000000..5b0de4e --- /dev/null +++ b/e2e/llm/harness.py @@ -0,0 +1,274 @@ +#!/usr/bin/env python3 +"""LLM + solver end-to-end harness (#106). Manual, non-deterministic — NOT CI. + +Reproduces docs/llm-interaction.md with a REAL model: the LLM gets the courier +problem in natural language plus tools — a local ``write_file`` and the pddlpy +MCP server's ``validate``/``ground``/``solve``/``parse`` (served by +``uvx --from "pddlpy[mcp]" pddlpy-mcp``, i.e. the *published* package in a +transient environment, per the issue). The conversation is captured to a +markdown transcript; soundness checks assert the solver found the 7-toll plan +and the final answer mentions it. + +Usage: + export OPENROUTER_API_KEY=... + uv run --with mcp,httpx python e2e/llm/harness.py --model qwen/qwen-2.5-72b-instruct + uv run --with mcp,httpx python e2e/llm/harness.py --dry-run # no API key: replays + # the doc's scripted turns + # to exercise the pipeline + +Exit codes: 0 sound, 1 conversation finished but unsound, 2 harness/setup error. +""" +from __future__ import annotations + +import argparse +import asyncio +import datetime +import json +import os +import pathlib +import re +import sys +import tempfile + +import httpx +from mcp import ClientSession, StdioServerParameters +from mcp.client.stdio import stdio_client + +HERE = pathlib.Path(__file__).parent +OPENROUTER_URL = "https://openrouter.ai/api/v1/chat/completions" + +PROBLEM = """\ +A courier is at the depot and must deliver a parcel to town, spending as +little as possible on tolls. The direct highway costs 10. There's a route +through the junction: 5 to get there, then 6 into town. And there's a +back-road chain: riverside (2), then the old bridge (2), then into town (3). +Roads are one-way. What's the cheapest route?""" + +SYSTEM = """\ +You are a planning assistant with tools. Do not compute the plan yourself — +your arithmetic is not a proof. Instead: +1. Translate the user's problem into classical planning PDDL: a domain file + and a problem file with action costs (:action-costs, total-cost) and a + (:metric minimize (total-cost)). Use write_file to save them. +2. Call validate on the two files and fix any reported issue. +3. Call solve with planner "ucs" (cost-optimal). +4. Answer the user in plain language: the cheapest route and its total cost, + based ONLY on the solver's output. + +PDDL action-costs syntax reminder: declare `(:functions (total-cost) (toll +?a - place ?b - place))` in the domain; the action effect uses `(increase +(total-cost) (toll ?from ?to))`; the problem's :init sets `(= (toll a b) 5)` +and `(= (total-cost) 0)`; end the problem with +`(:metric minimize (total-cost))`. There is no `:cost` field.""" + + +def mcp_tools_to_openai(tools) -> list: + """Convert MCP tool declarations to OpenAI/OpenRouter function schemas.""" + out = [] + for t in tools: + out.append({ + "type": "function", + "function": { + "name": t.name, + "description": t.description or "", + "parameters": t.inputSchema, + }, + }) + return out + + +WRITE_FILE_TOOL = { + "type": "function", + "function": { + "name": "write_file", + "description": "Write a text file (e.g. domain.pddl / problem.pddl) into " + "the working directory. Returns the absolute path to pass " + "to the other tools.", + "parameters": { + "type": "object", + "properties": { + "filename": {"type": "string", "description": "Base name, e.g. domain.pddl"}, + "content": {"type": "string"}, + }, + "required": ["filename", "content"], + }, + }, +} + + +class Conversation: + """Message log + markdown rendering.""" + + def __init__(self) -> None: + self.messages: list = [{"role": "system", "content": SYSTEM}, + {"role": "user", "content": PROBLEM}] + self.tool_results: list = [] # (name, arguments, result_text) + + def to_markdown(self, model: str) -> str: + lines = [f"# LLM + pddlpy end-to-end transcript", + f"", f"- **model:** `{model}`", + f"- **captured:** {datetime.date.today().isoformat()}", ""] + for m in self.messages: + role = m["role"] + if role == "system": + lines += ["## System prompt", "", "```", m["content"], "```", ""] + elif role == "user": + lines += ["## User", "", m["content"], ""] + elif role == "assistant": + if m.get("content"): + lines += ["## Assistant", "", m["content"], ""] + for tc in m.get("tool_calls") or []: + fn = tc["function"] + lines += [f"### Tool call: `{fn['name']}`", "", "```json", + fn["arguments"], "```", ""] + elif role == "tool": + lines += ["### Tool result", "", "```json", m["content"], "```", ""] + return "\n".join(lines) + + +async def run_tool(session: ClientSession, workdir: pathlib.Path, + name: str, args: dict) -> str: + """Execute one tool call (local write_file or MCP) and return result text.""" + if name == "write_file": + base = os.path.basename(args["filename"]) + content = args["content"] + # Weaker models double-escape newlines inside JSON tool arguments; + # if the "file" is one line full of literal \n sequences, decode them. + if "\n" not in content and "\\n" in content: + content = content.replace("\\n", "\n").replace("\\t", "\t") + path = workdir / base + path.write_text(content, encoding="utf-8") + return json.dumps({"path": str(path)}) + result = await session.call_tool(name, args) + if result.structuredContent is not None: + return json.dumps(result.structuredContent.get("result", result.structuredContent)) + return "".join(c.text for c in result.content if getattr(c, "text", None)) + + +async def chat_round(client: httpx.AsyncClient, model: str, messages: list, + tools: list) -> dict: + r = await client.post( + OPENROUTER_URL, + headers={"Authorization": "Bearer %s" % os.environ["OPENROUTER_API_KEY"]}, + json={"model": model, "messages": messages, "tools": tools}, + timeout=180, + ) + r.raise_for_status() + return r.json()["choices"][0]["message"] + + +def dry_run_turns(workdir: pathlib.Path) -> list: + """Scripted assistant turns (the doc's simulation) to exercise the whole + pipeline — MCP server, tool dispatch, transcript, soundness — offline.""" + domain = (HERE / ".." / ".." / "examples" / "pddl" / "courier-domain.pddl").resolve() + problem = (HERE / ".." / ".." / "examples" / "pddl" / "courier-problem.pddl").resolve() + return [ + {"tool_calls": [ + ("write_file", {"filename": "domain.pddl", "content": domain.read_text()}), + ("write_file", {"filename": "problem.pddl", "content": problem.read_text()}), + ]}, + {"tool_calls": [ + ("validate", {"domain_file": str(workdir / "domain.pddl"), + "problem_file": str(workdir / "problem.pddl")}), + ]}, + {"tool_calls": [ + ("solve", {"domain_file": str(workdir / "domain.pddl"), + "problem_file": str(workdir / "problem.pddl"), + "planner": "ucs"}), + ]}, + {"content": "The cheapest delivery costs 7 tolls: depot -> riverside (2), " + "old bridge (2), then into town (3). The direct highway (10) " + "and the junction route (11) are both more expensive."}, + ] + + +def soundness(convo: Conversation) -> list: + """Step-7 checks: solver reached the optimum and the answer reports it.""" + failures = [] + solved = [r for (n, a, r) in convo.tool_results if n == "solve"] + if not any('"cost": 7.0' in r or "'cost': 7.0" in r for r in solved): + failures.append("no solve result with cost 7.0") + validated = [r for (n, a, r) in convo.tool_results if n == "validate"] + if not validated: + failures.append("validate was never called") + final = next((m.get("content") or "" for m in reversed(convo.messages) + if m["role"] == "assistant"), "") + if not re.search(r"\b7\b", final): + failures.append("final answer does not state the cost 7") + if not all(w in final.lower() for w in ("riverside", "bridge", "town")): + failures.append("final answer does not narrate the optimal route") + return failures + + +async def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--model", default="qwen/qwen-2.5-72b-instruct") + ap.add_argument("--server-cmd", default='uvx --from pddlpy[mcp] pddlpy-mcp', + help="MCP server command (default: published pddlpy via uvx)") + ap.add_argument("--max-turns", type=int, default=20) + ap.add_argument("--dry-run", action="store_true", + help="replay scripted turns; no API key needed") + args = ap.parse_args() + + if not args.dry_run and "OPENROUTER_API_KEY" not in os.environ: + print("OPENROUTER_API_KEY not set (use --dry-run to test the pipeline)", + file=sys.stderr) + return 2 + + workdir = pathlib.Path(tempfile.mkdtemp(prefix="pddlpy-e2e-")) + cmd, *cmd_args = args.server_cmd.split() + params = StdioServerParameters(command=cmd, args=cmd_args) + convo = Conversation() + + async with stdio_client(params) as (read, write): + async with ClientSession(read, write) as session: + await session.initialize() + mcp_tools = (await session.list_tools()).tools + tools = mcp_tools_to_openai(mcp_tools) + [WRITE_FILE_TOOL] + + scripted = dry_run_turns(workdir) if args.dry_run else None + async with httpx.AsyncClient() as http: + for turn in range(args.max_turns): + if scripted is not None: + if not scripted: + break + step = scripted.pop(0) + msg = {"role": "assistant", + "content": step.get("content"), + "tool_calls": [ + {"id": "call_%d_%d" % (turn, i), "type": "function", + "function": {"name": n, "arguments": json.dumps(a)}} + for i, (n, a) in enumerate(step.get("tool_calls", [])) + ] or None} + else: + msg = await chat_round(http, args.model, convo.messages, tools) + convo.messages.append(msg) + calls = msg.get("tool_calls") or [] + if not calls: + break + for tc in calls: + fn = tc["function"] + fargs = json.loads(fn["arguments"]) + result = await run_tool(session, workdir, fn["name"], fargs) + convo.tool_results.append((fn["name"], fargs, result)) + convo.messages.append({"role": "tool", + "tool_call_id": tc["id"], + "content": result}) + + label = "dry-run" if args.dry_run else args.model.replace("/", "_") + stamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") + out = HERE / "transcripts" / ("%s-%s.md" % (label, stamp)) + out.write_text(convo.to_markdown("dry-run" if args.dry_run else args.model), + encoding="utf-8") + print("transcript: %s" % out) + + failures = soundness(convo) + if failures: + print("UNSOUND:", "; ".join(failures)) + return 1 + print("SOUND: solver optimum reached and narrated") + return 0 + + +if __name__ == "__main__": + sys.exit(asyncio.run(main())) diff --git a/e2e/llm/transcripts/.gitkeep b/e2e/llm/transcripts/.gitkeep new file mode 100644 index 0000000..e69de29