From a9719809b306fddd2d79df2825bd193467ab5c95 Mon Sep 17 00:00:00 2001 From: philluiz2323 Date: Fri, 31 Jul 2026 11:12:13 -0700 Subject: [PATCH 1/2] feat(synthesize): file good answers back as page proposals kb.synthesize already produces citation-bearing prose traceable to approved claims, with a confidence grade - everything needed to file a page proposal was already computed, just not wired to propose_page. turning a good answer into a page meant manually copying the text and re-typing the cited claim ids into a separate kb.propose_page call. add an opt-in file_as_page parameter: when set, files the answer back as a PAGE proposal via the existing propose_page, citing the same claim ids the answer already cites. still fully gated - propose_page files a pending proposal, never approves anything, identical to every other write path in this codebase. skip filing (not an error, not a crash) when the answer is empty or cites nothing - an uncited "answer" is the kb saying it doesn't know, not knowledge worth filing - or when propose_page itself fails (e.g. a title collision): the synthesis already succeeded, and losing that to a secondary optional step would be worse than reporting why filing didn't happen. both outcomes surface via new page_proposal_id (None when skipped/failed) and page_proposal_skipped_reason result fields. extracted the existing no-llm path into _deterministic_synthesize so a new _maybe_file_page_proposal helper can wrap either backend's result the same way - the llm path already produces the identical {query, answer, claims, pages, gaps, _meta} shape. registered on mcp (kb_synthesize), jsonl (kb.synthesize), and cli (vouch synthesize --file-as-page [--page-title T] [--agent A]). opt-in and additive - file_as_page defaults to False, so every existing caller's result shape is unchanged. Closes #736 --- CHANGELOG.md | 16 +++++++ src/vouch/cli.py | 18 ++++++- src/vouch/jsonl_server.py | 4 ++ src/vouch/server.py | 10 ++++ src/vouch/synthesize.py | 81 +++++++++++++++++++++++++++++++- tests/test_synthesize.py | 98 ++++++++++++++++++++++++++++++++++++++- 6 files changed, 224 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b0113b4..b0aed972 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,22 @@ All notable changes to vouch are documented here. Format follows ## [Unreleased] ### Added +- **`kb.synthesize --file-as-page` — file a good answer back as a page + proposal** (roadmap 1.4, #736): `kb.synthesize` already produces + citation-bearing prose traceable to approved claims, but turning a good + answer into a page meant manually copying the text and re-typing the + cited claim ids into a separate `kb.propose_page` call. `file_as_page` + wires the two together — the answer's own claim ids become the new + page's `claims`, filed via the existing `propose_page`, still fully + gated: a pending proposal, never auto-approved. Skipped, not an error, + when the answer is empty or cites nothing (an uncited "answer" is the KB + saying it doesn't know, not knowledge worth filing) or when filing fails + (e.g. a title collision) — either way reported via the new + `page_proposal_id` / `page_proposal_skipped_reason` result fields. + Registered on MCP, JSONL, and CLI (`vouch synthesize --file-as-page + [--page-title T] [--agent A]`). Opt-in and additive — `file_as_page` + defaults to `False`, so every existing caller's result shape is + unchanged. - **bench: composite guards** (#616): `efficiency`, `consistency` and `canary` as bounded multipliers over the composite, plus a `bench_version` stamp on every report. Reported **beside** the composite, never folded into it — diff --git a/src/vouch/cli.py b/src/vouch/cli.py index 1f1aebc8..85ff2e73 100644 --- a/src/vouch/cli.py +++ b/src/vouch/cli.py @@ -3837,12 +3837,28 @@ def context_hook() -> None: help="Draft the answer with the configured compile.llm_cmd, grounded in " "pages and approved claims (citations still verified mechanically).", ) -def synthesize(query: str, depth: int, max_chars: int, use_llm: bool) -> None: +@click.option( + "--file-as-page", is_flag=True, + help="File the answer back as a page proposal — still gated by review, " + "never auto-approved. Skipped, not an error, if the answer is empty " + "or cites nothing.", +) +@click.option("--page-title", default=None, + help="Title for --file-as-page (default: 'Answer: ').") +@click.option("--agent", "agent", default=None, + help="Proposer identity for --file-as-page (default: VOUCH_AGENT/whoami).") +def synthesize( + query: str, depth: int, max_chars: int, use_llm: bool, + file_as_page: bool, page_title: str | None, agent: str | None, +) -> None: """Answer a query from the KB, with inline citations.""" store = _load_store() with _cli_errors(): result = synth.synthesize( store, query=query, depth=depth, max_chars=max_chars, llm=use_llm, + file_as_page=file_as_page, + proposed_by=(agent or _whoami()) if file_as_page else None, + page_title=page_title, ) _emit_json(result) diff --git a/src/vouch/jsonl_server.py b/src/vouch/jsonl_server.py index bdd4fd9f..0a97bd60 100644 --- a/src/vouch/jsonl_server.py +++ b/src/vouch/jsonl_server.py @@ -263,12 +263,16 @@ def _h_context(p: dict) -> dict: def _h_synthesize(p: dict) -> dict: + file_as_page = bool(p.get("file_as_page", False)) return synthesize( _store(), query=p["query"], depth=int(p.get("depth", 3)), max_chars=int(p.get("max_chars", 4000)), llm=bool(p.get("llm", False)), + file_as_page=file_as_page, + proposed_by=_agent() if file_as_page else None, + page_title=p.get("page_title"), ) diff --git a/src/vouch/server.py b/src/vouch/server.py index c2b3221a..bd4e391a 100644 --- a/src/vouch/server.py +++ b/src/vouch/server.py @@ -375,6 +375,8 @@ def kb_synthesize( depth: int = 3, max_chars: int = 4000, llm: bool = False, + file_as_page: bool = False, + page_title: str | None = None, ) -> dict[str, Any]: """Answer a query from the review-gated KB, with inline `[id]` citations, an explicit gaps block, and a synthesis_confidence grade. @@ -384,9 +386,17 @@ def kb_synthesize( claims only); `llm=True` drafts the answer with the deployment-configured LLM (compile.llm_cmd) grounded in pages and approved claims — citations are still verified mechanically, and the call is synchronous. + + `file_as_page=True` additionally files the answer back as a page + proposal — still gated by review, never auto-approved. Skipped (not an + error) when the answer is empty or cites nothing; see + `page_proposal_id` / `page_proposal_skipped_reason` in the result. """ return synthesize( _store(), query=query, depth=depth, max_chars=max_chars, llm=llm, + file_as_page=file_as_page, + proposed_by=_agent() if file_as_page else None, + page_title=page_title, ) diff --git a/src/vouch/synthesize.py b/src/vouch/synthesize.py index 20577b85..0417bc6b 100644 --- a/src/vouch/synthesize.py +++ b/src/vouch/synthesize.py @@ -26,6 +26,7 @@ from . import llm_draft from .context import build_context_pack from .models import Claim, ClaimStatus, Page, PageStatus +from .proposals import ProposalError, propose_page from .storage import ArtifactNotFoundError, KBStore Confidence = Literal["high", "medium", "low"] @@ -263,6 +264,9 @@ def synthesize( depth: int = 3, max_chars: int = 4000, llm: bool = False, + file_as_page: bool = False, + proposed_by: str | None = None, + page_title: str | None = None, ) -> dict[str, Any]: """Answer `query` from the review-gated KB, with inline citations. @@ -272,12 +276,87 @@ def synthesize( and `_meta.synthesis_confidence`. With `llm=True` the answer is drafted by the deployment-configured LLM grounded in pages and approved claims; every citation is still verified mechanically. + + `file_as_page=True` additionally files the answer back as a `PAGE` + proposal (`proposed_by` required) — gated by review like every other + write, never auto-approved. Skipped, not an error, when the answer is + empty or cites nothing: an uncited "answer" is the KB saying it doesn't + know, not knowledge worth filing. The result gains `page_proposal_id` + (`None` when skipped or filing failed) and, in either case, + `page_proposal_skipped_reason`. """ if llm: - return _llm_synthesize( + result = _llm_synthesize( + store, query=query, depth=depth, max_chars=max_chars, + ) + else: + result = _deterministic_synthesize( store, query=query, depth=depth, max_chars=max_chars, ) + return _maybe_file_page_proposal( + store, result, query=query, file_as_page=file_as_page, + proposed_by=proposed_by, page_title=page_title, + ) + + +def _maybe_file_page_proposal( + store: KBStore, + result: dict[str, Any], + *, + query: str, + file_as_page: bool, + proposed_by: str | None, + page_title: str | None, +) -> dict[str, Any]: + """Optionally file `result["answer"]` back as a PAGE proposal. + + Never approves anything — `propose_page` goes through the same review + gate as every other write. Filing is skipped (not an error) when the + answer is empty or cites no claims: an uncited "answer" is the KB saying + it doesn't know, not knowledge worth filing. A `propose_page` failure + (e.g. a title collision) degrades the same way — the synthesis itself + already succeeded, and losing that to a secondary, optional step would + be a worse failure than reporting why filing didn't happen. + """ + if not file_as_page: + return result + if proposed_by is None: + raise ValueError("file_as_page requires proposed_by") + + answer = result["answer"] + cited_claims: list[str] = list(result["claims"]) + if not answer or not cited_claims: + result["page_proposal_id"] = None + result["page_proposal_skipped_reason"] = "empty or uncited answer — nothing to file" + return result + + title = (page_title or f"Answer: {query}").strip() + body_lines = [f"# {title}", "", answer] + if result["gaps"]: + body_lines += ["", "## Gaps", *(f"- {g}" for g in result["gaps"])] + body = "\n".join(body_lines) + "\n" + + try: + pr = propose_page( + store, title=title, body=body, claim_ids=cited_claims, + proposed_by=proposed_by, + rationale=f"filed from kb.synthesize answer to: {query!r}", + ) + except (ProposalError, ArtifactNotFoundError) as e: + result["page_proposal_id"] = None + result["page_proposal_skipped_reason"] = str(e) + return result + + result["page_proposal_id"] = pr.id + result["page_proposal_skipped_reason"] = None + return result + +def _deterministic_synthesize( + store: KBStore, *, query: str, depth: int, max_chars: int, +) -> dict[str, Any]: + """The no-LLM path, extracted verbatim from `synthesize` so `file_as_page` + can wrap either backend's result the same way.""" pack = build_context_pack(store, query=query, limit=depth) items = pack["items"] if isinstance(pack, dict) else pack.items diff --git a/tests/test_synthesize.py b/tests/test_synthesize.py index f2dec904..95852df2 100644 --- a/tests/test_synthesize.py +++ b/tests/test_synthesize.py @@ -10,7 +10,7 @@ from vouch import capabilities, health, synthesize from vouch.jsonl_server import HANDLERS, handle_request -from vouch.models import Claim, ClaimStatus, Page, PageStatus +from vouch.models import Claim, ClaimStatus, Page, PageStatus, ProposalStatus from vouch.storage import KBStore _CITE = re.compile(r"\[([^\[\]]+)\]") @@ -206,3 +206,99 @@ def test_jsonl_synthesize_handler(store: KBStore, monkeypatch) -> None: assert set(resp["result"]["claims"]) <= set(ids) for cid in resp["result"]["claims"]: assert f"[{cid}]" in resp["result"]["answer"] + + +# --- file_as_page ------------------------------------------------------------ + + +def test_file_as_page_files_a_pending_proposal(store: KBStore) -> None: + ids = _auth_kb(store) + result = synthesize.synthesize( + store, query="auth tokens", depth=5, + file_as_page=True, proposed_by="agent-x", + ) + assert result["page_proposal_id"] is not None + assert result["page_proposal_skipped_reason"] is None + + pending = [p for p in store.list_proposals() if p.status == ProposalStatus.PENDING] + assert len(pending) == 1 + proposal = pending[0] + assert proposal.id == result["page_proposal_id"] + assert proposal.proposed_by == "agent-x" + assert set(proposal.payload["claims"]) == set(result["claims"]) + assert set(proposal.payload["claims"]) <= set(ids) + assert result["answer"] in proposal.payload["body"] + # never approved — still behind the review gate like every other write + assert proposal.status == ProposalStatus.PENDING + + +def test_file_as_page_uses_custom_title(store: KBStore) -> None: + _auth_kb(store) + result = synthesize.synthesize( + store, query="auth tokens", depth=5, + file_as_page=True, proposed_by="agent-x", page_title="Auth Q&A", + ) + proposal = store.get_proposal(result["page_proposal_id"]) + assert proposal.payload["title"] == "Auth Q&A" + + +def test_file_as_page_skipped_when_answer_is_empty(store: KBStore) -> None: + _auth_kb(store) + result = synthesize.synthesize( + store, query="kubernetes networking topology", + file_as_page=True, proposed_by="agent-x", + ) + assert result["answer"] == "" + assert result["page_proposal_id"] is None + assert "uncited" in result["page_proposal_skipped_reason"] or "empty" in ( + result["page_proposal_skipped_reason"] or "" + ) + assert store.list_proposals() == [] + + +def test_file_as_page_without_proposed_by_raises(store: KBStore) -> None: + _auth_kb(store) + with pytest.raises(ValueError, match="proposed_by"): + synthesize.synthesize(store, query="auth", file_as_page=True) + + +def test_file_as_page_defaults_to_false(store: KBStore) -> None: + """The existing no-arg call shape must keep working exactly as before — + file_as_page defaults off and the result carries no proposal keys.""" + _auth_kb(store) + result = synthesize.synthesize(store, query="auth", depth=5) + assert "page_proposal_id" not in result + assert "page_proposal_skipped_reason" not in result + assert store.list_proposals() == [] + + +def test_jsonl_synthesize_file_as_page(store: KBStore, monkeypatch) -> None: + _auth_kb(store) + monkeypatch.chdir(store.root) + resp = handle_request({ + "id": "s3", "method": "kb.synthesize", + "params": {"query": "auth tokens", "depth": 5, "file_as_page": True}, + }) + assert resp["ok"] + assert resp["result"]["page_proposal_id"] is not None + pending = [p for p in store.list_proposals() if p.status == ProposalStatus.PENDING] + assert len(pending) == 1 + + +def test_cli_synthesize_file_as_page(store: KBStore, monkeypatch, tmp_path: Path) -> None: + from click.testing import CliRunner + + from vouch.cli import cli + + _auth_kb(store) + monkeypatch.chdir(store.root) + runner = CliRunner() + result = runner.invoke( + cli, ["synthesize", "auth tokens", "--depth", "5", "--file-as-page", + "--agent", "cli-tester"], + ) + assert result.exit_code == 0, result.output + payload = json.loads(result.output) + assert payload["page_proposal_id"] is not None + proposal = store.get_proposal(payload["page_proposal_id"]) + assert proposal.proposed_by == "cli-tester" From b2c3d8d84239889ed4d9e70006786171034bd27d Mon Sep 17 00:00:00 2001 From: philluiz2323 Date: Fri, 31 Jul 2026 12:27:28 -0700 Subject: [PATCH 2/2] test(synthesize): cover the gaps-in-body and propose_page-failure paths diff-coverage flagged two branches in _maybe_file_page_proposal as untested: the gaps-section body append (only reachable when file_as_page=True AND the query is partially covered - the existing tests either fully cover or fully miss), and the except (ProposalError, ArtifactNotFoundError) skip path. add a partially-covered query (some cited claims, some uncovered terms) asserting the gaps land in the filed page body, and a monkeypatched propose_page failure asserting it degrades to a reported skip reason without losing the already-successful synthesis. --- tests/test_synthesize.py | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/tests/test_synthesize.py b/tests/test_synthesize.py index 95852df2..605e4ddf 100644 --- a/tests/test_synthesize.py +++ b/tests/test_synthesize.py @@ -232,6 +232,46 @@ def test_file_as_page_files_a_pending_proposal(store: KBStore) -> None: assert proposal.status == ProposalStatus.PENDING +def test_file_as_page_body_includes_gaps_section(store: KBStore) -> None: + """A partially-covered query (some cited claims, some uncovered terms) + must carry its gaps into the filed page body, not just the result dict.""" + _auth_kb(store) + result = synthesize.synthesize( + store, query="auth billing invoices", depth=5, + file_as_page=True, proposed_by="agent-x", + ) + assert result["answer"] != "" + assert result["gaps"] + proposal = store.get_proposal(result["page_proposal_id"]) + assert "## Gaps" in proposal.payload["body"] + for gap in result["gaps"]: + assert f"- {gap}" in proposal.payload["body"] + + +def test_file_as_page_reports_propose_page_failure( + store: KBStore, monkeypatch: pytest.MonkeyPatch, +) -> None: + """A propose_page failure (e.g. an invalid page kind) must degrade to a + reported skip reason rather than losing the already-successful + synthesis — the answer computation and the filing step fail independently.""" + from vouch import synthesize as synth_mod + from vouch.proposals import ProposalError + + def _boom(*args, **kwargs): + raise ProposalError("simulated propose_page failure") + + monkeypatch.setattr(synth_mod, "propose_page", _boom) + _auth_kb(store) + result = synth_mod.synthesize( + store, query="auth tokens", depth=5, + file_as_page=True, proposed_by="agent-x", + ) + assert result["answer"] != "" # synthesis itself still succeeded + assert result["page_proposal_id"] is None + assert result["page_proposal_skipped_reason"] == "simulated propose_page failure" + assert store.list_proposals() == [] + + def test_file_as_page_uses_custom_title(store: KBStore) -> None: _auth_kb(store) result = synthesize.synthesize(