From a8a527be2a3ff23dd4f8b06c810e66dcad10e60a Mon Sep 17 00:00:00 2001 From: Mohammed Anas Nathani Date: Wed, 22 Jul 2026 02:15:22 +0530 Subject: [PATCH 1/5] fix: make six CLI/orchestrator/llm errors actionable Append concrete next steps to config validation, pipeline stop, UNVERIFIED completion, budget exceeded, and claude -p failure paths. Keep diagnosis text, escape only at console.print sites, and add regression tests for the new actionable substrings. Closes #198 --- src/readme2demo/cli.py | 28 ++++++++++++++++++++++--- src/readme2demo/llm.py | 9 +++++---- src/readme2demo/orchestrator.py | 3 ++- tests/test_cli.py | 36 +++++++++++++++++++++++++++++++++ tests/test_llm_backend.py | 18 +++++++++++++++++ tests/test_orchestrator.py | 12 +++++++++++ 6 files changed, 98 insertions(+), 8 deletions(-) diff --git a/src/readme2demo/cli.py b/src/readme2demo/cli.py index 4800c46..b83e4af 100644 --- a/src/readme2demo/cli.py +++ b/src/readme2demo/cli.py @@ -17,6 +17,7 @@ import sys from pathlib import Path import json +from difflib import get_close_matches from typing import Any, Optional import typer @@ -141,14 +142,28 @@ def _load_config(config_file: Optional[Path], **overrides: Any) -> Config: location = ".".join(str(part) for part in error["loc"]) source = config_file or Path("readme2demo.toml") if error["type"] == "extra_forbidden": + valid_keys = sorted(Config.model_fields) + suggestion = get_close_matches(location, valid_keys, n=1, cutoff=0.6) + hint = ( + f" Did you mean '{escape(suggestion[0])}'?" + if suggestion + else f" Valid keys: {escape(', '.join(valid_keys))}." + ) console.print( f"[red]Unknown config key '{escape(location)}' in " - f"{escape(str(source))}.[/]" + f"{escape(str(source))}.{hint}[/]" ) else: + bad_input = error.get("input", None) + value_bit = ( + f" (got {escape(repr(bad_input))})" + if bad_input is not None + else "" + ) + key_bit = f" for '{escape(location)}'" if location else "" console.print( - f"[red]Invalid configuration in {escape(str(source))}: " - f"{escape(error['msg'])}.[/]" + f"[red]Invalid configuration in {escape(str(source))}{key_bit}: " + f"{escape(error['msg'])}{value_bit}.[/]" ) raise typer.Exit(2) from None @@ -615,6 +630,9 @@ def _drive(orch: Orchestrator) -> None: except PipelineError as e: console.print(f"[red]Pipeline stopped:[/] {escape(str(e))}") console.print(escape(summarize(orch.manifest))) + console.print( + f"[dim]Fix the cause, then: readme2demo resume {escape(str(orch.run_dir))}[/]" + ) raise typer.Exit(1) except Exception as e: # noqa: BLE001 — stage errors are already in the manifest console.print(f"[red]{type(e).__name__}:[/] {escape(str(e))}") @@ -637,6 +655,10 @@ def _drive(orch: Orchestrator) -> None: f"\n[bold yellow]⚠ Completed UNVERIFIED.[/] " f"See {escape(str(orch.run_dir))}/verify.log" ) + console.print( + f"[dim]Retry after fixes: readme2demo resume " + f"{escape(str(orch.run_dir))} --from-stage distill[/]" + ) if __name__ == "__main__": diff --git a/src/readme2demo/llm.py b/src/readme2demo/llm.py index a75c916..cae93c1 100644 --- a/src/readme2demo/llm.py +++ b/src/readme2demo/llm.py @@ -313,17 +313,18 @@ def _complete_cli(system: str, user: str, model: str) -> LLMResponse: timeout=_CLI_TIMEOUT_S, errors="replace", env=_sanitized_env(), ) except subprocess.TimeoutExpired: - raise LLMError(f"claude -p timed out after {_CLI_TIMEOUT_S}s") from None + raise LLMError(f"claude -p timed out after {_CLI_TIMEOUT_S}s. Check login with `claude -p hello`, or switch to `--llm-backend api` with ANTHROPIC_API_KEY set.") from None if proc.returncode != 0: raise LLMError( - f"claude -p failed ({proc.returncode}): {(proc.stderr or proc.stdout)[:500]}" + f"claude -p failed ({proc.returncode}): {(proc.stderr or proc.stdout)[:500]}." + f" Check login with `claude -p hello`, or switch to `--llm-backend api` with ANTHROPIC_API_KEY set." ) try: envelope = json.loads(proc.stdout) except json.JSONDecodeError as e: - raise LLMError(f"claude -p returned non-JSON output: {proc.stdout[:300]!r}") from e + raise LLMError(f"claude -p returned non-JSON output: {proc.stdout[:300]!r}. Check login with `claude -p hello`, or switch to `--llm-backend api` with ANTHROPIC_API_KEY set.") from e if envelope.get("is_error"): - raise LLMError(f"claude -p reported an error: {envelope.get('result', '')[:500]}") + raise LLMError(f"claude -p reported an error: {envelope.get('result', '')[:500]}. Check login with `claude -p hello`, or switch to `--llm-backend api` with ANTHROPIC_API_KEY set.") return LLMResponse( text=envelope.get("result", ""), cost_usd=float(envelope.get("total_cost_usd") or 0.0), diff --git a/src/readme2demo/orchestrator.py b/src/readme2demo/orchestrator.py index 10068e5..6ca3250 100644 --- a/src/readme2demo/orchestrator.py +++ b/src/readme2demo/orchestrator.py @@ -210,7 +210,8 @@ def _stage_normalize(self) -> None: ) if self.cfg.budget_usd and cost > self.cfg.budget_usd: raise PipelineError( - f"Agent cost ${cost:.2f} exceeded budget ${self.cfg.budget_usd:.2f}" + f"Agent cost ${cost:.2f} exceeded budget ${self.cfg.budget_usd:.2f}. " + "Raise --budget-usd (or budget_usd in config), then resume this run." ) def _stage_distill(self, feedback: str = "") -> None: diff --git a/tests/test_cli.py b/tests/test_cli.py index 27e7a62..792366a 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -652,3 +652,39 @@ def test_report_json_and_markdown_are_mutually_exclusive(tmp_path): # A usage error, not silent precedence: neither format was emitted. assert "Verified" not in result.output assert '"stages"' not in result.output + + +def test_invalid_config_type_names_key_and_value(tmp_path): + """Regression: type errors must name the offending key and value.""" + config_file = tmp_path / "readme2demo.toml" + config_file.write_text('max_turns = "sixty"\n', encoding="utf-8") + + result = runner.invoke(app, ["run", _URL, "--config", str(config_file)]) + + assert result.exit_code == 2 + assert "max_turns" in result.output + assert "sixty" in result.output + + +def test_unknown_config_key_suggests_nearest_match(tmp_path): + """Regression: unknown keys should suggest a nearest valid key name.""" + config_file = tmp_path / "readme2demo.toml" + config_file.write_text("max_turn = 99\n", encoding="utf-8") + + result = runner.invoke(app, ["run", _URL, "--config", str(config_file)]) + + assert result.exit_code == 2 + assert "Unknown config key 'max_turn'" in result.output + assert "max_turns" in result.output # nearest match / valid key + + +def test_unknown_config_key_suggestion_is_escaped_for_rich_markup(tmp_path): + """Regression: suggested key names with brackets must not break Rich markup.""" + config_file = tmp_path / "readme2demo.toml" + bad_key = "[bold]not_real[/bold]" + config_file.write_text(f'"{bad_key}" = 1\n', encoding="utf-8") + + result = runner.invoke(app, ["run", _URL, "--config", str(config_file)]) + + assert result.exit_code == 2 + assert bad_key in result.output diff --git a/tests/test_llm_backend.py b/tests/test_llm_backend.py index 3c1f2c0..5b465b0 100644 --- a/tests/test_llm_backend.py +++ b/tests/test_llm_backend.py @@ -975,3 +975,21 @@ def fake_run(cmd, **kw): render_mod.run_render(tmp_path, Config(allow_docker_socket=True)) assert "--group-add" in captured["cmd"] assert "999" in captured["cmd"] + + +def test_cli_timeout_error_includes_next_step_hint(monkeypatch): + """Regression: claude -p failures should hint login or --llm-backend api.""" + import subprocess + from readme2demo import llm as llm_mod + + def boom(*a, **k): + raise subprocess.TimeoutExpired(cmd=["claude", "-p"], timeout=1) + + monkeypatch.setattr(llm_mod.shutil, "which", lambda _: "/usr/bin/claude") + monkeypatch.setattr(llm_mod.subprocess, "run", boom) + try: + llm_mod._complete_cli("sys", "user", None) + assert False, "expected LLMError" + except llm_mod.LLMError as e: + text = str(e) + assert "--llm-backend api" in text or "claude -p hello" in text diff --git a/tests/test_orchestrator.py b/tests/test_orchestrator.py index 72ba618..a656983 100644 --- a/tests/test_orchestrator.py +++ b/tests/test_orchestrator.py @@ -390,3 +390,15 @@ def test_summarize_markdown_escapes_table_breaking_error_text(): def test_summarize_markdown_no_artifacts_omits_section(): md = summarize_markdown(make_report_manifest(verified=False, stages={}), []) assert "**Artifacts**" not in md + + +def test_budget_exceeded_message_mentions_flag(): + """Regression: budget stop should name --budget-usd and resumability.""" + # Construct the same message shape the orchestrator raises. + cost, budget = 6.0, 5.0 + msg = ( + f"Agent cost ${cost:.2f} exceeded budget ${budget:.2f}. " + "Raise --budget-usd (or budget_usd in config), then resume this run." + ) + assert "--budget-usd" in msg + assert "resume" in msg.lower() From 9e63408ab94785a925b231b8fa4766bf85a7d57f Mon Sep 17 00:00:00 2001 From: Mohammed Anas Nathani Date: Wed, 22 Jul 2026 03:09:15 +0530 Subject: [PATCH 2/5] fix: address review feedback on error-message tests Factor the claude -p next-step hint into a module constant, pass a str model arg in the timeout regression test, and exercise Orchestrator.run() for the budget-exceeded message instead of asserting a hand-built string. --- src/readme2demo/llm.py | 12 ++++++++---- tests/test_llm_backend.py | 2 +- tests/test_orchestrator.py | 40 +++++++++++++++++++++++++++++--------- 3 files changed, 40 insertions(+), 14 deletions(-) diff --git a/src/readme2demo/llm.py b/src/readme2demo/llm.py index cae93c1..7564331 100644 --- a/src/readme2demo/llm.py +++ b/src/readme2demo/llm.py @@ -88,6 +88,10 @@ class LLMError(RuntimeError): _backend: Backend = "auto" _CLI_TIMEOUT_S = 600 +_CLI_NEXT_STEP_HINT = ( + " Check login with `claude -p hello`, or switch to " + "`--llm-backend api` with ANTHROPIC_API_KEY set." +) @dataclass(frozen=True) @@ -313,18 +317,18 @@ def _complete_cli(system: str, user: str, model: str) -> LLMResponse: timeout=_CLI_TIMEOUT_S, errors="replace", env=_sanitized_env(), ) except subprocess.TimeoutExpired: - raise LLMError(f"claude -p timed out after {_CLI_TIMEOUT_S}s. Check login with `claude -p hello`, or switch to `--llm-backend api` with ANTHROPIC_API_KEY set.") from None + raise LLMError(f"claude -p timed out after {_CLI_TIMEOUT_S}s." + _CLI_NEXT_STEP_HINT) from None if proc.returncode != 0: raise LLMError( f"claude -p failed ({proc.returncode}): {(proc.stderr or proc.stdout)[:500]}." - f" Check login with `claude -p hello`, or switch to `--llm-backend api` with ANTHROPIC_API_KEY set." + + _CLI_NEXT_STEP_HINT ) try: envelope = json.loads(proc.stdout) except json.JSONDecodeError as e: - raise LLMError(f"claude -p returned non-JSON output: {proc.stdout[:300]!r}. Check login with `claude -p hello`, or switch to `--llm-backend api` with ANTHROPIC_API_KEY set.") from e + raise LLMError(f"claude -p returned non-JSON output: {proc.stdout[:300]!r}." + _CLI_NEXT_STEP_HINT) from e if envelope.get("is_error"): - raise LLMError(f"claude -p reported an error: {envelope.get('result', '')[:500]}. Check login with `claude -p hello`, or switch to `--llm-backend api` with ANTHROPIC_API_KEY set.") + raise LLMError(f"claude -p reported an error: {envelope.get('result', '')[:500]}." + _CLI_NEXT_STEP_HINT) return LLMResponse( text=envelope.get("result", ""), cost_usd=float(envelope.get("total_cost_usd") or 0.0), diff --git a/tests/test_llm_backend.py b/tests/test_llm_backend.py index 5b465b0..a9d107e 100644 --- a/tests/test_llm_backend.py +++ b/tests/test_llm_backend.py @@ -988,7 +988,7 @@ def boom(*a, **k): monkeypatch.setattr(llm_mod.shutil, "which", lambda _: "/usr/bin/claude") monkeypatch.setattr(llm_mod.subprocess, "run", boom) try: - llm_mod._complete_cli("sys", "user", None) + llm_mod._complete_cli("sys", "user", "") assert False, "expected LLMError" except llm_mod.LLMError as e: text = str(e) diff --git a/tests/test_orchestrator.py b/tests/test_orchestrator.py index a656983..2e70dab 100644 --- a/tests/test_orchestrator.py +++ b/tests/test_orchestrator.py @@ -392,13 +392,35 @@ def test_summarize_markdown_no_artifacts_omits_section(): assert "**Artifacts**" not in md -def test_budget_exceeded_message_mentions_flag(): +def test_budget_exceeded_message_mentions_flag(tmp_path: Path, monkeypatch): """Regression: budget stop should name --budget-usd and resumability.""" - # Construct the same message shape the orchestrator raises. - cost, budget = 6.0, 5.0 - msg = ( - f"Agent cost ${cost:.2f} exceeded budget ${budget:.2f}. " - "Raise --budget-usd (or budget_usd in config), then resume this run." - ) - assert "--budget-usd" in msg - assert "resume" in msg.lower() + from readme2demo import normalize as normalize_mod + from readme2demo.types import AgentResult, CommandLog + + cfg = Config(runs_dir=tmp_path, budget_usd=5.0) + orch = Orchestrator.new_run("https://github.com/x/y", cfg) + + def fake_ingest(repo_url, run_dir, model, **kwargs): + plan = make_plan() + (run_dir / "plan.json").write_text(plan.model_dump_json()) + return plan, "abc1234", 0.005 + + def fake_run_agent(run_dir, plan, engine, c): + (run_dir / "transcript.ndjson").write_text("") + return run_dir / "transcript.ndjson" + + def fake_normalize(path, engine, run_dir): + log = CommandLog( + engine="claude-code", + result=AgentResult(outcome="success", cost_usd=6.0), + ) + (run_dir / "command_log.json").write_text(log.model_dump_json()) + return log + + monkeypatch.setattr(ingest_mod, "ingest", fake_ingest) + monkeypatch.setattr("readme2demo.orchestrator.run_agent", fake_run_agent) + monkeypatch.setattr(normalize_mod, "normalize", fake_normalize) + + with pytest.raises(PipelineError, match="--budget-usd") as excinfo: + orch.run() + assert "resume" in str(excinfo.value).lower() From eaa5e7c3ea49b36bf981c5c145a18b37f3978259 Mon Sep 17 00:00:00 2001 From: Mohammed Anas Nathani Date: Thu, 23 Jul 2026 04:56:10 +0530 Subject: [PATCH 3/5] fix: address maintainer review on actionable errors - Budget stop now leads with budget_usd in readme2demo.toml (works with resume); --budget-usd only mentioned for a fresh run - Config key suggestions skip excluded/deprecated fields (vhs_image) - Truncate type-error value repr; strip claude -p stderr before append - Pin PipelineError resume hint and UNVERIFIED --from-stage distill via CliRunner; pin bracketed config *values* for Rich escape order - Widen claude -p tests to require full next-step hint; use pytest.raises --- src/readme2demo/cli.py | 7 ++-- src/readme2demo/llm.py | 18 +++++++--- src/readme2demo/orchestrator.py | 3 +- tests/test_cli.py | 61 ++++++++++++++++++++++++++++++--- tests/test_llm_backend.py | 35 ++++++++++++++----- tests/test_orchestrator.py | 10 ++++-- 6 files changed, 110 insertions(+), 24 deletions(-) diff --git a/src/readme2demo/cli.py b/src/readme2demo/cli.py index b83e4af..bb92f2c 100644 --- a/src/readme2demo/cli.py +++ b/src/readme2demo/cli.py @@ -142,7 +142,10 @@ def _load_config(config_file: Optional[Path], **overrides: Any) -> Config: location = ".".join(str(part) for part in error["loc"]) source = config_file or Path("readme2demo.toml") if error["type"] == "extra_forbidden": - valid_keys = sorted(Config.model_fields) + # Exclude deprecated no-op shims (e.g. vhs_image) from suggestions. + valid_keys = sorted( + k for k, f in Config.model_fields.items() if not f.exclude + ) suggestion = get_close_matches(location, valid_keys, n=1, cutoff=0.6) hint = ( f" Did you mean '{escape(suggestion[0])}'?" @@ -156,7 +159,7 @@ def _load_config(config_file: Optional[Path], **overrides: Any) -> Config: else: bad_input = error.get("input", None) value_bit = ( - f" (got {escape(repr(bad_input))})" + f" (got {escape(repr(bad_input)[:120])})" if bad_input is not None else "" ) diff --git a/src/readme2demo/llm.py b/src/readme2demo/llm.py index 7564331..2e1e84e 100644 --- a/src/readme2demo/llm.py +++ b/src/readme2demo/llm.py @@ -317,18 +317,26 @@ def _complete_cli(system: str, user: str, model: str) -> LLMResponse: timeout=_CLI_TIMEOUT_S, errors="replace", env=_sanitized_env(), ) except subprocess.TimeoutExpired: - raise LLMError(f"claude -p timed out after {_CLI_TIMEOUT_S}s." + _CLI_NEXT_STEP_HINT) from None + raise LLMError( + f"claude -p timed out after {_CLI_TIMEOUT_S}s." + _CLI_NEXT_STEP_HINT + ) from None if proc.returncode != 0: + detail = (proc.stderr or proc.stdout or "")[:500].strip() raise LLMError( - f"claude -p failed ({proc.returncode}): {(proc.stderr or proc.stdout)[:500]}." - + _CLI_NEXT_STEP_HINT + f"claude -p failed ({proc.returncode}): {detail}." + _CLI_NEXT_STEP_HINT ) try: envelope = json.loads(proc.stdout) except json.JSONDecodeError as e: - raise LLMError(f"claude -p returned non-JSON output: {proc.stdout[:300]!r}." + _CLI_NEXT_STEP_HINT) from e + raise LLMError( + f"claude -p returned non-JSON output: {proc.stdout[:300]!r}." + + _CLI_NEXT_STEP_HINT + ) from e if envelope.get("is_error"): - raise LLMError(f"claude -p reported an error: {envelope.get('result', '')[:500]}." + _CLI_NEXT_STEP_HINT) + detail = str(envelope.get("result", "") or "")[:500] + raise LLMError( + f"claude -p reported an error: {detail}." + _CLI_NEXT_STEP_HINT + ) return LLMResponse( text=envelope.get("result", ""), cost_usd=float(envelope.get("total_cost_usd") or 0.0), diff --git a/src/readme2demo/orchestrator.py b/src/readme2demo/orchestrator.py index 6ca3250..ede2a61 100644 --- a/src/readme2demo/orchestrator.py +++ b/src/readme2demo/orchestrator.py @@ -211,7 +211,8 @@ def _stage_normalize(self) -> None: if self.cfg.budget_usd and cost > self.cfg.budget_usd: raise PipelineError( f"Agent cost ${cost:.2f} exceeded budget ${self.cfg.budget_usd:.2f}. " - "Raise --budget-usd (or budget_usd in config), then resume this run." + "Raise budget_usd in readme2demo.toml (or --budget-usd on a fresh run), " + "then resume this run — the agent work is already saved." ) def _stage_distill(self, feedback: str = "") -> None: diff --git a/tests/test_cli.py b/tests/test_cli.py index 792366a..23f4be0 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -678,13 +678,64 @@ def test_unknown_config_key_suggests_nearest_match(tmp_path): assert "max_turns" in result.output # nearest match / valid key -def test_unknown_config_key_suggestion_is_escaped_for_rich_markup(tmp_path): - """Regression: suggested key names with brackets must not break Rich markup.""" +def test_config_type_error_value_with_brackets_is_escaped(tmp_path): + """Regression: bracketed config *values* must survive escape(repr(...)). + + Failure class 15: escape(repr(x)) is the safe order. This pins the new + value_bit path, not the unknown-key path already covered above. + """ config_file = tmp_path / "readme2demo.toml" - bad_key = "[bold]not_real[/bold]" - config_file.write_text(f'"{bad_key}" = 1\n', encoding="utf-8") + # max_turns expects int; a bracketed string hits the type-error branch. + config_file.write_text('max_turns = "[bold]sixty[/bold]"\n', encoding="utf-8") result = runner.invoke(app, ["run", _URL, "--config", str(config_file)]) assert result.exit_code == 2 - assert bad_key in result.output + assert "max_turns" in result.output + # Literal brackets from the value must appear (not be swallowed by Rich). + assert "[bold]sixty[/bold]" in result.output + assert "Traceback" not in result.output + + +def test_pipeline_error_prints_resume_hint(tmp_path, monkeypatch): + """Regression: PipelineError path must print an explicit resume command.""" + from readme2demo.orchestrator import Orchestrator, PipelineError + from readme2demo.manifest import Manifest + + run_dir = tmp_path / "r1" + Manifest.create(run_dir, _URL, "claude-code", "img").save() + + def boom(self): + raise PipelineError("synthetic stop for resume-hint test") + + monkeypatch.setattr(Orchestrator, "run", boom) + monkeypatch.setattr("readme2demo.cli._preflight", lambda cfg: None) + + result = runner.invoke(app, ["resume", str(run_dir)]) + assert result.exit_code == 1 + assert "Pipeline stopped" in result.output + assert "readme2demo resume" in result.output + # Rich may soft-wrap long absolute paths; compare collapsed whitespace. + collapsed = " ".join(result.output.split()) + assert str(run_dir) in collapsed or run_dir.name in result.output + + +def test_unverified_completion_prints_from_stage_distill(tmp_path, monkeypatch): + """Regression: UNVERIFIED completion must suggest resume --from-stage distill.""" + from readme2demo.orchestrator import Orchestrator + from readme2demo.manifest import Manifest + + run_dir = tmp_path / "r2" + m = Manifest.create(run_dir, _URL, "claude-code", "img") + m.verified = False + m.save() + + def fake_run(self): + return self.manifest + + monkeypatch.setattr(Orchestrator, "run", fake_run) + monkeypatch.setattr("readme2demo.cli._preflight", lambda cfg: None) + + result = runner.invoke(app, ["resume", str(run_dir)]) + assert "--from-stage distill" in result.output + assert "UNVERIFIED" in result.output or "unverified" in result.output.lower() diff --git a/tests/test_llm_backend.py b/tests/test_llm_backend.py index a9d107e..3b15159 100644 --- a/tests/test_llm_backend.py +++ b/tests/test_llm_backend.py @@ -551,7 +551,7 @@ def test_cli_backend_error_envelope(monkeypatch): cmd, 0, stdout=_fake_cli_envelope("rate limited", is_error=True), stderr="" ), ) - with pytest.raises(LLMError, match="reported an error"): + with pytest.raises(LLMError, match=r"reported an error.*--llm-backend api"): llm._complete_cli("s", "u", "m") @@ -561,7 +561,7 @@ def test_cli_backend_nonzero_exit(monkeypatch): "readme2demo.llm.subprocess.run", lambda cmd, **kw: subprocess.CompletedProcess(cmd, 1, stdout="", stderr="boom"), ) - with pytest.raises(LLMError, match="failed"): + with pytest.raises(LLMError, match=r"failed.*--llm-backend api"): llm._complete_cli("s", "u", "m") @@ -978,7 +978,7 @@ def fake_run(cmd, **kw): def test_cli_timeout_error_includes_next_step_hint(monkeypatch): - """Regression: claude -p failures should hint login or --llm-backend api.""" + """Regression: claude -p failures should hint login AND --llm-backend api.""" import subprocess from readme2demo import llm as llm_mod @@ -987,9 +987,28 @@ def boom(*a, **k): monkeypatch.setattr(llm_mod.shutil, "which", lambda _: "/usr/bin/claude") monkeypatch.setattr(llm_mod.subprocess, "run", boom) - try: + with pytest.raises( + llm_mod.LLMError, + match=r"timed out.*claude -p hello.*--llm-backend api", + ): + llm_mod._complete_cli("sys", "user", "") + + +def test_cli_nonjson_error_includes_next_step_hint(monkeypatch): + """Regression: non-JSON claude -p output carries the same next-step hint.""" + import subprocess + from readme2demo import llm as llm_mod + + monkeypatch.setattr(llm_mod.shutil, "which", lambda _: "/usr/bin/claude") + monkeypatch.setattr( + llm_mod.subprocess, + "run", + lambda *a, **k: subprocess.CompletedProcess( + ["claude"], 0, stdout="not-json{", stderr="" + ), + ) + with pytest.raises( + llm_mod.LLMError, + match=r"non-JSON.*--llm-backend api", + ): llm_mod._complete_cli("sys", "user", "") - assert False, "expected LLMError" - except llm_mod.LLMError as e: - text = str(e) - assert "--llm-backend api" in text or "claude -p hello" in text diff --git a/tests/test_orchestrator.py b/tests/test_orchestrator.py index 2e70dab..df5bb20 100644 --- a/tests/test_orchestrator.py +++ b/tests/test_orchestrator.py @@ -393,7 +393,7 @@ def test_summarize_markdown_no_artifacts_omits_section(): def test_budget_exceeded_message_mentions_flag(tmp_path: Path, monkeypatch): - """Regression: budget stop should name --budget-usd and resumability.""" + """Regression: budget stop names config remedy that works with resume.""" from readme2demo import normalize as normalize_mod from readme2demo.types import AgentResult, CommandLog @@ -421,6 +421,10 @@ def fake_normalize(path, engine, run_dir): monkeypatch.setattr("readme2demo.orchestrator.run_agent", fake_run_agent) monkeypatch.setattr(normalize_mod, "normalize", fake_normalize) - with pytest.raises(PipelineError, match="--budget-usd") as excinfo: + with pytest.raises(PipelineError, match="budget_usd") as excinfo: orch.run() - assert "resume" in str(excinfo.value).lower() + msg = str(excinfo.value) + assert "resume" in msg.lower() + assert "readme2demo.toml" in msg + # --budget-usd is only for a *fresh* run, not resume + assert "fresh run" in msg From b5f64e2dc4a6164e63499ce99fc97d5451f91153 Mon Sep 17 00:00:00 2001 From: Mohammed Anas Nathani Date: Fri, 24 Jul 2026 19:42:17 +0530 Subject: [PATCH 4/5] test: normalize Rich-wrapped CLI errors --- tests/test_cli.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 23f4be0..7c31f74 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -299,7 +299,9 @@ def test_run_rejects_missing_config_file_before_preflight(tmp_path): result = runner.invoke(app, ["run", _URL, "--config", str(missing)]) assert result.exit_code != 0 - assert "does not exist" in result.output + # Rich may soft-wrap long absolute paths; compare normalized output. + normalized = result.output.replace("│", " ") + assert "does not exist" in " ".join(normalized.split()) def test_run_reports_unknown_config_key_without_traceback(tmp_path): @@ -335,7 +337,9 @@ def test_resume_rejects_missing_run_dir(tmp_path): missing = tmp_path / "missing-run" result = runner.invoke(app, ["resume", str(missing)]) assert result.exit_code != 0 - assert "does not exist" in result.output + # Rich may soft-wrap long absolute paths; compare normalized output. + normalized = result.output.replace("│", " ") + assert "does not exist" in " ".join(normalized.split()) def test_resume_rejects_file_run_dir(tmp_path): From 274c1a2ef027ce7980f576feaa43a8f699d178af Mon Sep 17 00:00:00 2001 From: Mohammed Anas Nathani Date: Fri, 24 Jul 2026 21:18:28 +0530 Subject: [PATCH 5/5] test: compare collapsed whitespace in unverified completion hint --- tests/test_cli.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 7c31f74..731ab4d 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -741,5 +741,7 @@ def fake_run(self): monkeypatch.setattr("readme2demo.cli._preflight", lambda cfg: None) result = runner.invoke(app, ["resume", str(run_dir)]) - assert "--from-stage distill" in result.output - assert "UNVERIFIED" in result.output or "unverified" in result.output.lower() + # Rich may soft-wrap long absolute paths; compare collapsed whitespace. + collapsed = " ".join(result.output.split()) + assert "--from-stage distill" in collapsed + assert "UNVERIFIED" in collapsed or "unverified" in collapsed.lower()