From 6fa3f245158ef9486006df326e4b9f0041f0d8d1 Mon Sep 17 00:00:00 2001 From: hfoffani Date: Sun, 12 Jul 2026 22:23:42 +0200 Subject: [PATCH 1/3] #106: LLM+solver end-to-end harness (manual) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit e2e/llm/harness.py drives a real model (OpenRouter) against the pddlpy MCP server served from the PUBLISHED package (uvx --from pddlpy[mcp]): the model gets the courier problem in NL plus write_file + the MCP tools, and the run is judged sound iff validate was used, solve returned the 7-toll optimum, and the final answer narrates it. Each run captures a markdown transcript (gitignored; the winning one gets committed with the docs/llm-interaction.md replacement). --dry-run replays the doc's scripted turns through the full pipeline offline — verified green. claude_driver.sh covers mid-tier models via claude -p with an MCP config. make e2e-llm MODEL=... as the manual entry point; deliberately not in CI (non-deterministic). Co-Authored-By: Claude Fable 5 --- .claude/worktrees/agent-a276978947a194d50 | 1 + .claude/worktrees/agent-afe1dc4258a9dfcc5 | 1 + .gitignore | 2 +- Makefile | 9 +- e2e/llm/README.md | 45 +++ e2e/llm/claude_driver.sh | 34 +++ e2e/llm/harness.py | 263 ++++++++++++++++++ e2e/llm/transcripts/.gitkeep | 0 .../transcripts/dry-run-20260712-222235.md | 78 ++++++ 9 files changed, 431 insertions(+), 2 deletions(-) create mode 160000 .claude/worktrees/agent-a276978947a194d50 create mode 160000 .claude/worktrees/agent-afe1dc4258a9dfcc5 create mode 100644 e2e/llm/README.md create mode 100755 e2e/llm/claude_driver.sh create mode 100644 e2e/llm/harness.py create mode 100644 e2e/llm/transcripts/.gitkeep create mode 100644 e2e/llm/transcripts/dry-run-20260712-222235.md diff --git a/.claude/worktrees/agent-a276978947a194d50 b/.claude/worktrees/agent-a276978947a194d50 new file mode 160000 index 0000000..d2d2fd9 --- /dev/null +++ b/.claude/worktrees/agent-a276978947a194d50 @@ -0,0 +1 @@ +Subproject commit d2d2fd9509ea0372defa729a52af41419eaac284 diff --git a/.claude/worktrees/agent-afe1dc4258a9dfcc5 b/.claude/worktrees/agent-afe1dc4258a9dfcc5 new file mode 160000 index 0000000..d2d2fd9 --- /dev/null +++ b/.claude/worktrees/agent-afe1dc4258a9dfcc5 @@ -0,0 +1 @@ +Subproject commit d2d2fd9509ea0372defa729a52af41419eaac284 diff --git a/.gitignore b/.gitignore index c6125d8..45a250d 100644 --- a/.gitignore +++ b/.gitignore @@ -40,4 +40,4 @@ htmlcov/ .pytest_cache/ # AI -.claude/ \ No newline at end of file +.claude/e2e/llm/transcripts/*.md diff --git a/Makefile b/Makefile index 021dd18..acafa81 100644 --- a/Makefile +++ b/Makefile @@ -86,6 +86,12 @@ examples: pyparser pddlpy/pddl.py @echo "Running examples..." for f in examples/[0-9][0-9]_*.py ; do echo "=== $$f ===" ; $(PYTHON) $$f ; done +# LLM + solver end-to-end (#106): MANUAL, non-deterministic, never in CI. +# Needs OPENROUTER_API_KEY (or use `--dry-run` via the script directly). +MODEL?=qwen/qwen-2.5-72b-instruct +e2e-llm: + $(UV) run --with mcp,httpx python e2e/llm/harness.py --model $(MODEL) + # Test published package from TestPyPI using Docker testpublish: @echo "Testing pddlpy installation from TestPyPI..." @@ -117,5 +123,6 @@ help: @echo " demo - Run demo scripts" @echo " examples - Run the example showcase (examples/)" @echo " testpublish - Test package installation from TestPyPI (Docker)" + @echo " e2e-llm - Manual LLM+solver end-to-end (MODEL=..., needs OPENROUTER_API_KEY)" -.PHONY: all init testgrammar pyparser test coverage lint typecheck build clean demo testpublish help +.PHONY: all init testgrammar pyparser test coverage lint typecheck build clean demo testpublish e2e-llm help diff --git a/e2e/llm/README.md b/e2e/llm/README.md new file mode 100644 index 0000000..be7b92c --- /dev/null +++ b/e2e/llm/README.md @@ -0,0 +1,45 @@ +# LLM + solver end-to-end (#106) + +Manual, non-deterministic — **never wired into CI**. Reproduces +[docs/llm-interaction.md](../../docs/llm-interaction.md) with a *real* model: +the LLM gets the courier problem in natural language plus tools (a local +`write_file` and the pddlpy MCP tools `validate`/`ground`/`solve`/`parse`, +served by `uvx --from "pddlpy[mcp]" pddlpy-mcp` — the published package in a +transient environment). Each run captures a markdown transcript under +`transcripts/` and checks soundness: `validate` was used, `solve` returned the +7-toll optimum, and the final answer narrates it. + +## Small / open models — OpenRouter + +```bash +export OPENROUTER_API_KEY=... +make e2e-llm MODEL=qwen/qwen-2.5-72b-instruct +# equivalently: +uv run --with mcp,httpx python e2e/llm/harness.py --model qwen/qwen-2.5-72b-instruct +``` + +Exit codes: `0` sound, `1` finished but unsound (read the transcript), `2` +setup error. No key at hand? `--dry-run` replays the doc's scripted turns +through the entire pipeline (MCP server, tool dispatch, transcript, +soundness) without any API call. + +## Mid-tier models — CLI agents + +For models we reach through coding agents rather than raw APIs, drive the +same MCP server through the agent's own harness: + +```bash +./e2e/llm/claude_driver.sh # Claude Code: claude -p + --mcp-config +# codex exec works analogously with its MCP configuration +``` + +The driver prints where it saved the transcript; judge soundness with the +same criteria (validate used, cost 7.0, route narrated). + +## Iterating (issue steps 5–8) + +Repeat runs (vary model / round) until a transcript is sound *and reads +well*; then replace the simulated conversation in `docs/llm-interaction.md` +with that transcript (label it with model + date, and turn the "what is +simulated" box into "how this was captured"). Transcripts are gitignored — +commit only the winning one alongside the doc update. diff --git a/e2e/llm/claude_driver.sh b/e2e/llm/claude_driver.sh new file mode 100755 index 0000000..6debd10 --- /dev/null +++ b/e2e/llm/claude_driver.sh @@ -0,0 +1,34 @@ +#!/bin/sh +# Mid-tier driver (#106): let Claude Code solve the courier problem through +# the pddlpy MCP server. Requires the `claude` CLI. Manual — not CI. +set -eu + +HERE=$(cd "$(dirname "$0")" && pwd) +STAMP=$(date +%Y%m%d-%H%M%S) +OUT="$HERE/transcripts/claude-$STAMP.md" +MCPCFG=$(mktemp) +trap 'rm -f "$MCPCFG"' EXIT + +cat > "$MCPCFG" <<'EOF' +{"mcpServers": {"pddlpy": {"command": "uvx", "args": ["--from", "pddlpy[mcp]", "pddlpy-mcp"]}}} +EOF + +PROMPT='A courier is at the depot and must deliver a parcel to town, spending as +little as possible on tolls. The direct highway costs 10. There is a route +through the junction: 5 to get there, then 6 into town. And there is a +back-road chain: riverside (2), then the old bridge (2), then into town (3). +Roads are one-way. What is the cheapest route? + +Do not compute the plan yourself. Translate the problem to PDDL (domain + +problem files with :action-costs and a minimize metric), run the pddlpy +validate tool and fix any issue, run the solve tool with planner "ucs", and +answer in plain language based only on the solver output.' + +claude -p "$PROMPT" \ + --mcp-config "$MCPCFG" \ + --allowedTools "Write,mcp__pddlpy__validate,mcp__pddlpy__solve,mcp__pddlpy__parse,mcp__pddlpy__ground" \ + --output-format text | tee "$OUT" + +echo +echo "transcript: $OUT" +echo "judge soundness: validate used, solve cost 7.0, route narrated" diff --git a/e2e/llm/harness.py b/e2e/llm/harness.py new file mode 100644 index 0000000..84343fb --- /dev/null +++ b/e2e/llm/harness.py @@ -0,0 +1,263 @@ +#!/usr/bin/env python3 +"""LLM + solver end-to-end harness (#106). Manual, non-deterministic — NOT CI. + +Reproduces docs/llm-interaction.md with a REAL model: the LLM gets the courier +problem in natural language plus tools — a local ``write_file`` and the pddlpy +MCP server's ``validate``/``ground``/``solve``/``parse`` (served by +``uvx --from "pddlpy[mcp]" pddlpy-mcp``, i.e. the *published* package in a +transient environment, per the issue). The conversation is captured to a +markdown transcript; soundness checks assert the solver found the 7-toll plan +and the final answer mentions it. + +Usage: + export OPENROUTER_API_KEY=... + uv run --with mcp,httpx python e2e/llm/harness.py --model qwen/qwen-2.5-72b-instruct + uv run --with mcp,httpx python e2e/llm/harness.py --dry-run # no API key: replays + # the doc's scripted turns + # to exercise the pipeline + +Exit codes: 0 sound, 1 conversation finished but unsound, 2 harness/setup error. +""" +from __future__ import annotations + +import argparse +import asyncio +import datetime +import json +import os +import pathlib +import re +import sys +import tempfile + +import httpx +from mcp import ClientSession, StdioServerParameters +from mcp.client.stdio import stdio_client + +HERE = pathlib.Path(__file__).parent +OPENROUTER_URL = "https://openrouter.ai/api/v1/chat/completions" + +PROBLEM = """\ +A courier is at the depot and must deliver a parcel to town, spending as +little as possible on tolls. The direct highway costs 10. There's a route +through the junction: 5 to get there, then 6 into town. And there's a +back-road chain: riverside (2), then the old bridge (2), then into town (3). +Roads are one-way. What's the cheapest route?""" + +SYSTEM = """\ +You are a planning assistant with tools. Do not compute the plan yourself — +your arithmetic is not a proof. Instead: +1. Translate the user's problem into classical planning PDDL: a domain file + and a problem file with action costs (:action-costs, total-cost) and a + (:metric minimize (total-cost)). Use write_file to save them. +2. Call validate on the two files and fix any reported issue. +3. Call solve with planner "ucs" (cost-optimal). +4. Answer the user in plain language: the cheapest route and its total cost, + based ONLY on the solver's output.""" + + +def mcp_tools_to_openai(tools) -> list: + """Convert MCP tool declarations to OpenAI/OpenRouter function schemas.""" + out = [] + for t in tools: + out.append({ + "type": "function", + "function": { + "name": t.name, + "description": t.description or "", + "parameters": t.inputSchema, + }, + }) + return out + + +WRITE_FILE_TOOL = { + "type": "function", + "function": { + "name": "write_file", + "description": "Write a text file (e.g. domain.pddl / problem.pddl) into " + "the working directory. Returns the absolute path to pass " + "to the other tools.", + "parameters": { + "type": "object", + "properties": { + "filename": {"type": "string", "description": "Base name, e.g. domain.pddl"}, + "content": {"type": "string"}, + }, + "required": ["filename", "content"], + }, + }, +} + + +class Conversation: + """Message log + markdown rendering.""" + + def __init__(self) -> None: + self.messages: list = [{"role": "system", "content": SYSTEM}, + {"role": "user", "content": PROBLEM}] + self.tool_results: list = [] # (name, arguments, result_text) + + def to_markdown(self, model: str) -> str: + lines = [f"# LLM + pddlpy end-to-end transcript", + f"", f"- **model:** `{model}`", + f"- **captured:** {datetime.date.today().isoformat()}", ""] + for m in self.messages: + role = m["role"] + if role == "system": + lines += ["## System prompt", "", "```", m["content"], "```", ""] + elif role == "user": + lines += ["## User", "", m["content"], ""] + elif role == "assistant": + if m.get("content"): + lines += ["## Assistant", "", m["content"], ""] + for tc in m.get("tool_calls") or []: + fn = tc["function"] + lines += [f"### Tool call: `{fn['name']}`", "", "```json", + fn["arguments"], "```", ""] + elif role == "tool": + lines += ["### Tool result", "", "```json", m["content"], "```", ""] + return "\n".join(lines) + + +async def run_tool(session: ClientSession, workdir: pathlib.Path, + name: str, args: dict) -> str: + """Execute one tool call (local write_file or MCP) and return result text.""" + if name == "write_file": + base = os.path.basename(args["filename"]) + path = workdir / base + path.write_text(args["content"], encoding="utf-8") + return json.dumps({"path": str(path)}) + result = await session.call_tool(name, args) + if result.structuredContent is not None: + return json.dumps(result.structuredContent.get("result", result.structuredContent)) + return "".join(c.text for c in result.content if getattr(c, "text", None)) + + +async def chat_round(client: httpx.AsyncClient, model: str, messages: list, + tools: list) -> dict: + r = await client.post( + OPENROUTER_URL, + headers={"Authorization": "Bearer %s" % os.environ["OPENROUTER_API_KEY"]}, + json={"model": model, "messages": messages, "tools": tools}, + timeout=180, + ) + r.raise_for_status() + return r.json()["choices"][0]["message"] + + +def dry_run_turns(workdir: pathlib.Path) -> list: + """Scripted assistant turns (the doc's simulation) to exercise the whole + pipeline — MCP server, tool dispatch, transcript, soundness — offline.""" + domain = (HERE / ".." / ".." / "examples" / "pddl" / "courier-domain.pddl").resolve() + problem = (HERE / ".." / ".." / "examples" / "pddl" / "courier-problem.pddl").resolve() + return [ + {"tool_calls": [ + ("write_file", {"filename": "domain.pddl", "content": domain.read_text()}), + ("write_file", {"filename": "problem.pddl", "content": problem.read_text()}), + ]}, + {"tool_calls": [ + ("validate", {"domain_file": str(workdir / "domain.pddl"), + "problem_file": str(workdir / "problem.pddl")}), + ]}, + {"tool_calls": [ + ("solve", {"domain_file": str(workdir / "domain.pddl"), + "problem_file": str(workdir / "problem.pddl"), + "planner": "ucs"}), + ]}, + {"content": "The cheapest delivery costs 7 tolls: depot -> riverside (2), " + "old bridge (2), then into town (3). The direct highway (10) " + "and the junction route (11) are both more expensive."}, + ] + + +def soundness(convo: Conversation) -> list: + """Step-7 checks: solver reached the optimum and the answer reports it.""" + failures = [] + solved = [r for (n, a, r) in convo.tool_results if n == "solve"] + if not any('"cost": 7.0' in r or "'cost': 7.0" in r for r in solved): + failures.append("no solve result with cost 7.0") + validated = [r for (n, a, r) in convo.tool_results if n == "validate"] + if not validated: + failures.append("validate was never called") + final = next((m.get("content") or "" for m in reversed(convo.messages) + if m["role"] == "assistant"), "") + if not re.search(r"\b7\b", final): + failures.append("final answer does not state the cost 7") + if not all(w in final.lower() for w in ("riverside", "bridge", "town")): + failures.append("final answer does not narrate the optimal route") + return failures + + +async def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--model", default="qwen/qwen-2.5-72b-instruct") + ap.add_argument("--server-cmd", default='uvx --from pddlpy[mcp] pddlpy-mcp', + help="MCP server command (default: published pddlpy via uvx)") + ap.add_argument("--max-turns", type=int, default=12) + ap.add_argument("--dry-run", action="store_true", + help="replay scripted turns; no API key needed") + args = ap.parse_args() + + if not args.dry_run and "OPENROUTER_API_KEY" not in os.environ: + print("OPENROUTER_API_KEY not set (use --dry-run to test the pipeline)", + file=sys.stderr) + return 2 + + workdir = pathlib.Path(tempfile.mkdtemp(prefix="pddlpy-e2e-")) + cmd, *cmd_args = args.server_cmd.split() + params = StdioServerParameters(command=cmd, args=cmd_args) + convo = Conversation() + + async with stdio_client(params) as (read, write): + async with ClientSession(read, write) as session: + await session.initialize() + mcp_tools = (await session.list_tools()).tools + tools = mcp_tools_to_openai(mcp_tools) + [WRITE_FILE_TOOL] + + scripted = dry_run_turns(workdir) if args.dry_run else None + async with httpx.AsyncClient() as http: + for turn in range(args.max_turns): + if scripted is not None: + if not scripted: + break + step = scripted.pop(0) + msg = {"role": "assistant", + "content": step.get("content"), + "tool_calls": [ + {"id": "call_%d_%d" % (turn, i), "type": "function", + "function": {"name": n, "arguments": json.dumps(a)}} + for i, (n, a) in enumerate(step.get("tool_calls", [])) + ] or None} + else: + msg = await chat_round(http, args.model, convo.messages, tools) + convo.messages.append(msg) + calls = msg.get("tool_calls") or [] + if not calls: + break + for tc in calls: + fn = tc["function"] + fargs = json.loads(fn["arguments"]) + result = await run_tool(session, workdir, fn["name"], fargs) + convo.tool_results.append((fn["name"], fargs, result)) + convo.messages.append({"role": "tool", + "tool_call_id": tc["id"], + "content": result}) + + label = "dry-run" if args.dry_run else args.model.replace("/", "_") + stamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") + out = HERE / "transcripts" / ("%s-%s.md" % (label, stamp)) + out.write_text(convo.to_markdown("dry-run" if args.dry_run else args.model), + encoding="utf-8") + print("transcript: %s" % out) + + failures = soundness(convo) + if failures: + print("UNSOUND:", "; ".join(failures)) + return 1 + print("SOUND: solver optimum reached and narrated") + return 0 + + +if __name__ == "__main__": + sys.exit(asyncio.run(main())) diff --git a/e2e/llm/transcripts/.gitkeep b/e2e/llm/transcripts/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/e2e/llm/transcripts/dry-run-20260712-222235.md b/e2e/llm/transcripts/dry-run-20260712-222235.md new file mode 100644 index 0000000..2c041bf --- /dev/null +++ b/e2e/llm/transcripts/dry-run-20260712-222235.md @@ -0,0 +1,78 @@ +# LLM + pddlpy end-to-end transcript + +- **model:** `dry-run` +- **captured:** 2026-07-12 + +## System prompt + +``` +You are a planning assistant with tools. Do not compute the plan yourself — +your arithmetic is not a proof. Instead: +1. Translate the user's problem into classical planning PDDL: a domain file + and a problem file with action costs (:action-costs, total-cost) and a + (:metric minimize (total-cost)). Use write_file to save them. +2. Call validate on the two files and fix any reported issue. +3. Call solve with planner "ucs" (cost-optimal). +4. Answer the user in plain language: the cheapest route and its total cost, + based ONLY on the solver's output. +``` + +## User + +A courier is at the depot and must deliver a parcel to town, spending as +little as possible on tolls. The direct highway costs 10. There's a route +through the junction: 5 to get there, then 6 into town. And there's a +back-road chain: riverside (2), then the old bridge (2), then into town (3). +Roads are one-way. What's the cheapest route? + +### Tool call: `write_file` + +```json +{"filename": "domain.pddl", "content": ";; Courier domain for docs/llm-interaction.md (#99): moving between connected\n;; places accrues total-cost equal to the road toll. Deliberately tiny -- the\n;; point of the example is cost-OPTIMAL routing, where greedy answers fail.\n(define (domain courier)\n (:requirements :strips :typing :action-costs)\n (:types place)\n (:predicates\n (at ?p - place)\n (road ?from - place ?to - place))\n (:functions\n (total-cost)\n (toll ?from - place ?to - place))\n\n (:action drive\n :parameters (?from - place ?to - place)\n :precondition (and (at ?from) (road ?from ?to))\n :effect (and (not (at ?from))\n (at ?to)\n (increase (total-cost) (toll ?from ?to)))))\n"} +``` + +### Tool call: `write_file` + +```json +{"filename": "problem.pddl", "content": ";; The optimality trap (#99): the direct highway depot->town costs 10.\n;; The scenic detour depot->riverside->oldbridge->town costs 2+2+3 = 7.\n;; A decoy route depot->junction->town costs 5+6 = 11 (looks promising,\n;; is not). Cheapest plan: the three-hop detour, total 7.\n(define (problem courier-run)\n (:domain courier)\n (:objects depot riverside oldbridge junction town - place)\n (:init\n (at depot)\n (road depot town) (= (toll depot town) 10)\n (road depot junction) (= (toll depot junction) 5)\n (road junction town) (= (toll junction town) 6)\n (road depot riverside) (= (toll depot riverside) 2)\n (road riverside oldbridge) (= (toll riverside oldbridge) 2)\n (road oldbridge town) (= (toll oldbridge town) 3)\n (= (total-cost) 0))\n (:goal (at town))\n (:metric minimize (total-cost)))\n"} +``` + +### Tool result + +```json +{"path": "/var/folders/1_/4n5m6d194sl2wqmsl765w6j40000gn/T/pddlpy-e2e-vf6xzt9t/domain.pddl"} +``` + +### Tool result + +```json +{"path": "/var/folders/1_/4n5m6d194sl2wqmsl765w6j40000gn/T/pddlpy-e2e-vf6xzt9t/problem.pddl"} +``` + +### Tool call: `validate` + +```json +{"domain_file": "/var/folders/1_/4n5m6d194sl2wqmsl765w6j40000gn/T/pddlpy-e2e-vf6xzt9t/domain.pddl", "problem_file": "/var/folders/1_/4n5m6d194sl2wqmsl765w6j40000gn/T/pddlpy-e2e-vf6xzt9t/problem.pddl"} +``` + +### Tool result + +```json +{"valid": true, "issues": []} +``` + +### Tool call: `solve` + +```json +{"domain_file": "/var/folders/1_/4n5m6d194sl2wqmsl765w6j40000gn/T/pddlpy-e2e-vf6xzt9t/domain.pddl", "problem_file": "/var/folders/1_/4n5m6d194sl2wqmsl765w6j40000gn/T/pddlpy-e2e-vf6xzt9t/problem.pddl", "planner": "ucs"} +``` + +### Tool result + +```json +{"planner": "ucs", "solved": true, "cost": 7.0, "length": 3, "steps": [{"action": "drive", "args": {"?from": "depot", "?to": "riverside"}}, {"action": "drive", "args": {"?from": "riverside", "?to": "oldbridge"}}, {"action": "drive", "args": {"?from": "oldbridge", "?to": "town"}}]} +``` + +## Assistant + +The cheapest delivery costs 7 tolls: depot -> riverside (2), old bridge (2), then into town (3). The direct highway (10) and the junction route (11) are both more expensive. From d50211db57e487bb9377d00f672185b566898ead Mon Sep 17 00:00:00 2001 From: hfoffani Date: Sun, 12 Jul 2026 22:25:17 +0200 Subject: [PATCH 2/3] fix: repair .gitignore concatenation; untrack worktree gitlinks + transcript MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous commit appended the transcripts ignore pattern to a .gitignore lacking a trailing newline, fusing it with the '.claude/' line — which un-ignored both, letting 'git add -A' stage the agent worktree gitlinks and the dry-run transcript. Restore the two rules and drop the three paths from the index. Co-Authored-By: Claude Fable 5 --- .claude/worktrees/agent-a276978947a194d50 | 1 - .claude/worktrees/agent-afe1dc4258a9dfcc5 | 1 - .gitignore | 4 +- .../transcripts/dry-run-20260712-222235.md | 78 ------------------- 4 files changed, 3 insertions(+), 81 deletions(-) delete mode 160000 .claude/worktrees/agent-a276978947a194d50 delete mode 160000 .claude/worktrees/agent-afe1dc4258a9dfcc5 delete mode 100644 e2e/llm/transcripts/dry-run-20260712-222235.md diff --git a/.claude/worktrees/agent-a276978947a194d50 b/.claude/worktrees/agent-a276978947a194d50 deleted file mode 160000 index d2d2fd9..0000000 --- a/.claude/worktrees/agent-a276978947a194d50 +++ /dev/null @@ -1 +0,0 @@ -Subproject commit d2d2fd9509ea0372defa729a52af41419eaac284 diff --git a/.claude/worktrees/agent-afe1dc4258a9dfcc5 b/.claude/worktrees/agent-afe1dc4258a9dfcc5 deleted file mode 160000 index d2d2fd9..0000000 --- a/.claude/worktrees/agent-afe1dc4258a9dfcc5 +++ /dev/null @@ -1 +0,0 @@ -Subproject commit d2d2fd9509ea0372defa729a52af41419eaac284 diff --git a/.gitignore b/.gitignore index 45a250d..31bebe5 100644 --- a/.gitignore +++ b/.gitignore @@ -40,4 +40,6 @@ htmlcov/ .pytest_cache/ # AI -.claude/e2e/llm/transcripts/*.md +.claude/ +e2e/llm/transcripts/*.md + diff --git a/e2e/llm/transcripts/dry-run-20260712-222235.md b/e2e/llm/transcripts/dry-run-20260712-222235.md deleted file mode 100644 index 2c041bf..0000000 --- a/e2e/llm/transcripts/dry-run-20260712-222235.md +++ /dev/null @@ -1,78 +0,0 @@ -# LLM + pddlpy end-to-end transcript - -- **model:** `dry-run` -- **captured:** 2026-07-12 - -## System prompt - -``` -You are a planning assistant with tools. Do not compute the plan yourself — -your arithmetic is not a proof. Instead: -1. Translate the user's problem into classical planning PDDL: a domain file - and a problem file with action costs (:action-costs, total-cost) and a - (:metric minimize (total-cost)). Use write_file to save them. -2. Call validate on the two files and fix any reported issue. -3. Call solve with planner "ucs" (cost-optimal). -4. Answer the user in plain language: the cheapest route and its total cost, - based ONLY on the solver's output. -``` - -## User - -A courier is at the depot and must deliver a parcel to town, spending as -little as possible on tolls. The direct highway costs 10. There's a route -through the junction: 5 to get there, then 6 into town. And there's a -back-road chain: riverside (2), then the old bridge (2), then into town (3). -Roads are one-way. What's the cheapest route? - -### Tool call: `write_file` - -```json -{"filename": "domain.pddl", "content": ";; Courier domain for docs/llm-interaction.md (#99): moving between connected\n;; places accrues total-cost equal to the road toll. Deliberately tiny -- the\n;; point of the example is cost-OPTIMAL routing, where greedy answers fail.\n(define (domain courier)\n (:requirements :strips :typing :action-costs)\n (:types place)\n (:predicates\n (at ?p - place)\n (road ?from - place ?to - place))\n (:functions\n (total-cost)\n (toll ?from - place ?to - place))\n\n (:action drive\n :parameters (?from - place ?to - place)\n :precondition (and (at ?from) (road ?from ?to))\n :effect (and (not (at ?from))\n (at ?to)\n (increase (total-cost) (toll ?from ?to)))))\n"} -``` - -### Tool call: `write_file` - -```json -{"filename": "problem.pddl", "content": ";; The optimality trap (#99): the direct highway depot->town costs 10.\n;; The scenic detour depot->riverside->oldbridge->town costs 2+2+3 = 7.\n;; A decoy route depot->junction->town costs 5+6 = 11 (looks promising,\n;; is not). Cheapest plan: the three-hop detour, total 7.\n(define (problem courier-run)\n (:domain courier)\n (:objects depot riverside oldbridge junction town - place)\n (:init\n (at depot)\n (road depot town) (= (toll depot town) 10)\n (road depot junction) (= (toll depot junction) 5)\n (road junction town) (= (toll junction town) 6)\n (road depot riverside) (= (toll depot riverside) 2)\n (road riverside oldbridge) (= (toll riverside oldbridge) 2)\n (road oldbridge town) (= (toll oldbridge town) 3)\n (= (total-cost) 0))\n (:goal (at town))\n (:metric minimize (total-cost)))\n"} -``` - -### Tool result - -```json -{"path": "/var/folders/1_/4n5m6d194sl2wqmsl765w6j40000gn/T/pddlpy-e2e-vf6xzt9t/domain.pddl"} -``` - -### Tool result - -```json -{"path": "/var/folders/1_/4n5m6d194sl2wqmsl765w6j40000gn/T/pddlpy-e2e-vf6xzt9t/problem.pddl"} -``` - -### Tool call: `validate` - -```json -{"domain_file": "/var/folders/1_/4n5m6d194sl2wqmsl765w6j40000gn/T/pddlpy-e2e-vf6xzt9t/domain.pddl", "problem_file": "/var/folders/1_/4n5m6d194sl2wqmsl765w6j40000gn/T/pddlpy-e2e-vf6xzt9t/problem.pddl"} -``` - -### Tool result - -```json -{"valid": true, "issues": []} -``` - -### Tool call: `solve` - -```json -{"domain_file": "/var/folders/1_/4n5m6d194sl2wqmsl765w6j40000gn/T/pddlpy-e2e-vf6xzt9t/domain.pddl", "problem_file": "/var/folders/1_/4n5m6d194sl2wqmsl765w6j40000gn/T/pddlpy-e2e-vf6xzt9t/problem.pddl", "planner": "ucs"} -``` - -### Tool result - -```json -{"planner": "ucs", "solved": true, "cost": 7.0, "length": 3, "steps": [{"action": "drive", "args": {"?from": "depot", "?to": "riverside"}}, {"action": "drive", "args": {"?from": "riverside", "?to": "oldbridge"}}, {"action": "drive", "args": {"?from": "oldbridge", "?to": "town"}}]} -``` - -## Assistant - -The cheapest delivery costs 7 tolls: depot -> riverside (2), old bridge (2), then into town (3). The direct highway (10) and the junction route (11) are both more expensive. From 60b38efc6cda69b8f594f441bbcc4f9da0bd9f4a Mon Sep 17 00:00:00 2001 From: hfoffani Date: Mon, 13 Jul 2026 10:22:54 +0200 Subject: [PATCH 3/3] e2e: harden harness after round 1 (qwen-2.5-72b, unsound) Round 1 exposed two weak-model failure modes: double-escaped newlines in JSON tool arguments (PDDL written as one line of literal \n -> ANTLR token errors) and invented cost syntax (:cost fields, fantasy function blocks) that burned all turns in the validate loop. write_file now decodes literal \n/\t when the content has no real newlines, the system prompt gains a 4-line action-costs syntax reminder (the moral equivalent of the agent skill), and max turns go 12 -> 20. Co-Authored-By: Claude Fable 5 --- e2e/llm/harness.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/e2e/llm/harness.py b/e2e/llm/harness.py index 84343fb..5b0de4e 100644 --- a/e2e/llm/harness.py +++ b/e2e/llm/harness.py @@ -53,7 +53,13 @@ 2. Call validate on the two files and fix any reported issue. 3. Call solve with planner "ucs" (cost-optimal). 4. Answer the user in plain language: the cheapest route and its total cost, - based ONLY on the solver's output.""" + based ONLY on the solver's output. + +PDDL action-costs syntax reminder: declare `(:functions (total-cost) (toll +?a - place ?b - place))` in the domain; the action effect uses `(increase +(total-cost) (toll ?from ?to))`; the problem's :init sets `(= (toll a b) 5)` +and `(= (total-cost) 0)`; end the problem with +`(:metric minimize (total-cost))`. There is no `:cost` field.""" def mcp_tools_to_openai(tools) -> list: @@ -125,8 +131,13 @@ async def run_tool(session: ClientSession, workdir: pathlib.Path, """Execute one tool call (local write_file or MCP) and return result text.""" if name == "write_file": base = os.path.basename(args["filename"]) + content = args["content"] + # Weaker models double-escape newlines inside JSON tool arguments; + # if the "file" is one line full of literal \n sequences, decode them. + if "\n" not in content and "\\n" in content: + content = content.replace("\\n", "\n").replace("\\t", "\t") path = workdir / base - path.write_text(args["content"], encoding="utf-8") + path.write_text(content, encoding="utf-8") return json.dumps({"path": str(path)}) result = await session.call_tool(name, args) if result.structuredContent is not None: @@ -194,7 +205,7 @@ async def main() -> int: ap.add_argument("--model", default="qwen/qwen-2.5-72b-instruct") ap.add_argument("--server-cmd", default='uvx --from pddlpy[mcp] pddlpy-mcp', help="MCP server command (default: published pddlpy via uvx)") - ap.add_argument("--max-turns", type=int, default=12) + ap.add_argument("--max-turns", type=int, default=20) ap.add_argument("--dry-run", action="store_true", help="replay scripted turns; no API key needed") args = ap.parse_args()