Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,22 @@ All notable changes to vouch are documented here. Format follows
## [Unreleased]

### Added
- **`kb.synthesize --file-as-page` — file a good answer back as a page
proposal** (roadmap 1.4, #736): `kb.synthesize` already produces
citation-bearing prose traceable to approved claims, but turning a good
answer into a page meant manually copying the text and re-typing the
cited claim ids into a separate `kb.propose_page` call. `file_as_page`
wires the two together — the answer's own claim ids become the new
page's `claims`, filed via the existing `propose_page`, still fully
gated: a pending proposal, never auto-approved. Skipped, not an error,
when the answer is empty or cites nothing (an uncited "answer" is the KB
saying it doesn't know, not knowledge worth filing) or when filing fails
(e.g. a title collision) — either way reported via the new
`page_proposal_id` / `page_proposal_skipped_reason` result fields.
Registered on MCP, JSONL, and CLI (`vouch synthesize --file-as-page
[--page-title T] [--agent A]`). Opt-in and additive — `file_as_page`
defaults to `False`, so every existing caller's result shape is
unchanged.
- **bench: composite guards** (#616): `efficiency`, `consistency` and `canary`
as bounded multipliers over the composite, plus a `bench_version` stamp on
every report. Reported **beside** the composite, never folded into it —
Expand Down
18 changes: 17 additions & 1 deletion src/vouch/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -3837,12 +3837,28 @@ def context_hook() -> None:
help="Draft the answer with the configured compile.llm_cmd, grounded in "
"pages and approved claims (citations still verified mechanically).",
)
def synthesize(query: str, depth: int, max_chars: int, use_llm: bool) -> None:
@click.option(
"--file-as-page", is_flag=True,
help="File the answer back as a page proposal — still gated by review, "
"never auto-approved. Skipped, not an error, if the answer is empty "
"or cites nothing.",
)
@click.option("--page-title", default=None,
help="Title for --file-as-page (default: 'Answer: <query>').")
@click.option("--agent", "agent", default=None,
help="Proposer identity for --file-as-page (default: VOUCH_AGENT/whoami).")
def synthesize(
query: str, depth: int, max_chars: int, use_llm: bool,
file_as_page: bool, page_title: str | None, agent: str | None,
) -> None:
"""Answer a query from the KB, with inline citations."""
store = _load_store()
with _cli_errors():
result = synth.synthesize(
store, query=query, depth=depth, max_chars=max_chars, llm=use_llm,
file_as_page=file_as_page,
proposed_by=(agent or _whoami()) if file_as_page else None,
page_title=page_title,
)
_emit_json(result)

Expand Down
4 changes: 4 additions & 0 deletions src/vouch/jsonl_server.py
Original file line number Diff line number Diff line change
Expand Up @@ -263,12 +263,16 @@ def _h_context(p: dict) -> dict:


def _h_synthesize(p: dict) -> dict:
file_as_page = bool(p.get("file_as_page", False))
return synthesize(
_store(),
query=p["query"],
depth=int(p.get("depth", 3)),
max_chars=int(p.get("max_chars", 4000)),
llm=bool(p.get("llm", False)),
file_as_page=file_as_page,
proposed_by=_agent() if file_as_page else None,
page_title=p.get("page_title"),
)


Expand Down
10 changes: 10 additions & 0 deletions src/vouch/server.py
Original file line number Diff line number Diff line change
Expand Up @@ -375,6 +375,8 @@ def kb_synthesize(
depth: int = 3,
max_chars: int = 4000,
llm: bool = False,
file_as_page: bool = False,
page_title: str | None = None,
) -> dict[str, Any]:
"""Answer a query from the review-gated KB, with inline `[id]` citations,
an explicit gaps block, and a synthesis_confidence grade.
Expand All @@ -384,9 +386,17 @@ def kb_synthesize(
claims only); `llm=True` drafts the answer with the deployment-configured
LLM (compile.llm_cmd) grounded in pages and approved claims — citations
are still verified mechanically, and the call is synchronous.

`file_as_page=True` additionally files the answer back as a page
proposal — still gated by review, never auto-approved. Skipped (not an
error) when the answer is empty or cites nothing; see
`page_proposal_id` / `page_proposal_skipped_reason` in the result.
"""
return synthesize(
_store(), query=query, depth=depth, max_chars=max_chars, llm=llm,
file_as_page=file_as_page,
proposed_by=_agent() if file_as_page else None,
page_title=page_title,
)


Expand Down
81 changes: 80 additions & 1 deletion src/vouch/synthesize.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
from . import llm_draft
from .context import build_context_pack
from .models import Claim, ClaimStatus, Page, PageStatus
from .proposals import ProposalError, propose_page
from .storage import ArtifactNotFoundError, KBStore

Confidence = Literal["high", "medium", "low"]
Expand Down Expand Up @@ -263,6 +264,9 @@ def synthesize(
depth: int = 3,
max_chars: int = 4000,
llm: bool = False,
file_as_page: bool = False,
proposed_by: str | None = None,
page_title: str | None = None,
) -> dict[str, Any]:
"""Answer `query` from the review-gated KB, with inline citations.

Expand All @@ -272,12 +276,87 @@ def synthesize(
and `_meta.synthesis_confidence`. With `llm=True` the answer is drafted
by the deployment-configured LLM grounded in pages and approved claims;
every citation is still verified mechanically.

`file_as_page=True` additionally files the answer back as a `PAGE`
proposal (`proposed_by` required) — gated by review like every other
write, never auto-approved. Skipped, not an error, when the answer is
empty or cites nothing: an uncited "answer" is the KB saying it doesn't
know, not knowledge worth filing. The result gains `page_proposal_id`
(`None` when skipped or filing failed) and, in either case,
`page_proposal_skipped_reason`.
"""
if llm:
return _llm_synthesize(
result = _llm_synthesize(
store, query=query, depth=depth, max_chars=max_chars,
)
else:
result = _deterministic_synthesize(
store, query=query, depth=depth, max_chars=max_chars,
)
return _maybe_file_page_proposal(
store, result, query=query, file_as_page=file_as_page,
proposed_by=proposed_by, page_title=page_title,
)


def _maybe_file_page_proposal(
store: KBStore,
result: dict[str, Any],
*,
query: str,
file_as_page: bool,
proposed_by: str | None,
page_title: str | None,
) -> dict[str, Any]:
"""Optionally file `result["answer"]` back as a PAGE proposal.

Never approves anything — `propose_page` goes through the same review
gate as every other write. Filing is skipped (not an error) when the
answer is empty or cites no claims: an uncited "answer" is the KB saying
it doesn't know, not knowledge worth filing. A `propose_page` failure
(e.g. a title collision) degrades the same way — the synthesis itself
already succeeded, and losing that to a secondary, optional step would
be a worse failure than reporting why filing didn't happen.
"""
if not file_as_page:
return result
if proposed_by is None:
raise ValueError("file_as_page requires proposed_by")

answer = result["answer"]
cited_claims: list[str] = list(result["claims"])
if not answer or not cited_claims:
result["page_proposal_id"] = None
result["page_proposal_skipped_reason"] = "empty or uncited answer — nothing to file"
return result

title = (page_title or f"Answer: {query}").strip()
body_lines = [f"# {title}", "", answer]
if result["gaps"]:
body_lines += ["", "## Gaps", *(f"- {g}" for g in result["gaps"])]
body = "\n".join(body_lines) + "\n"

try:
pr = propose_page(
store, title=title, body=body, claim_ids=cited_claims,
proposed_by=proposed_by,
rationale=f"filed from kb.synthesize answer to: {query!r}",
)
except (ProposalError, ArtifactNotFoundError) as e:
result["page_proposal_id"] = None
result["page_proposal_skipped_reason"] = str(e)
return result

result["page_proposal_id"] = pr.id
result["page_proposal_skipped_reason"] = None
return result


def _deterministic_synthesize(
store: KBStore, *, query: str, depth: int, max_chars: int,
) -> dict[str, Any]:
"""The no-LLM path, extracted verbatim from `synthesize` so `file_as_page`
can wrap either backend's result the same way."""
pack = build_context_pack(store, query=query, limit=depth)
items = pack["items"] if isinstance(pack, dict) else pack.items

Expand Down
138 changes: 137 additions & 1 deletion tests/test_synthesize.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@

from vouch import capabilities, health, synthesize
from vouch.jsonl_server import HANDLERS, handle_request
from vouch.models import Claim, ClaimStatus, Page, PageStatus
from vouch.models import Claim, ClaimStatus, Page, PageStatus, ProposalStatus
from vouch.storage import KBStore

_CITE = re.compile(r"\[([^\[\]]+)\]")
Expand Down Expand Up @@ -206,3 +206,139 @@ def test_jsonl_synthesize_handler(store: KBStore, monkeypatch) -> None:
assert set(resp["result"]["claims"]) <= set(ids)
for cid in resp["result"]["claims"]:
assert f"[{cid}]" in resp["result"]["answer"]


# --- file_as_page ------------------------------------------------------------


def test_file_as_page_files_a_pending_proposal(store: KBStore) -> None:
ids = _auth_kb(store)
result = synthesize.synthesize(
store, query="auth tokens", depth=5,
file_as_page=True, proposed_by="agent-x",
)
assert result["page_proposal_id"] is not None
assert result["page_proposal_skipped_reason"] is None

pending = [p for p in store.list_proposals() if p.status == ProposalStatus.PENDING]
assert len(pending) == 1
proposal = pending[0]
assert proposal.id == result["page_proposal_id"]
assert proposal.proposed_by == "agent-x"
assert set(proposal.payload["claims"]) == set(result["claims"])
assert set(proposal.payload["claims"]) <= set(ids)
assert result["answer"] in proposal.payload["body"]
# never approved — still behind the review gate like every other write
assert proposal.status == ProposalStatus.PENDING


def test_file_as_page_body_includes_gaps_section(store: KBStore) -> None:
"""A partially-covered query (some cited claims, some uncovered terms)
must carry its gaps into the filed page body, not just the result dict."""
_auth_kb(store)
result = synthesize.synthesize(
store, query="auth billing invoices", depth=5,
file_as_page=True, proposed_by="agent-x",
)
assert result["answer"] != ""
assert result["gaps"]
proposal = store.get_proposal(result["page_proposal_id"])
assert "## Gaps" in proposal.payload["body"]
for gap in result["gaps"]:
assert f"- {gap}" in proposal.payload["body"]


def test_file_as_page_reports_propose_page_failure(
store: KBStore, monkeypatch: pytest.MonkeyPatch,
) -> None:
"""A propose_page failure (e.g. an invalid page kind) must degrade to a
reported skip reason rather than losing the already-successful
synthesis — the answer computation and the filing step fail independently."""
from vouch import synthesize as synth_mod
from vouch.proposals import ProposalError

def _boom(*args, **kwargs):
raise ProposalError("simulated propose_page failure")

monkeypatch.setattr(synth_mod, "propose_page", _boom)
_auth_kb(store)
result = synth_mod.synthesize(
store, query="auth tokens", depth=5,
file_as_page=True, proposed_by="agent-x",
)
assert result["answer"] != "" # synthesis itself still succeeded
assert result["page_proposal_id"] is None
assert result["page_proposal_skipped_reason"] == "simulated propose_page failure"
assert store.list_proposals() == []


def test_file_as_page_uses_custom_title(store: KBStore) -> None:
_auth_kb(store)
result = synthesize.synthesize(
store, query="auth tokens", depth=5,
file_as_page=True, proposed_by="agent-x", page_title="Auth Q&A",
)
proposal = store.get_proposal(result["page_proposal_id"])
assert proposal.payload["title"] == "Auth Q&A"


def test_file_as_page_skipped_when_answer_is_empty(store: KBStore) -> None:
_auth_kb(store)
result = synthesize.synthesize(
store, query="kubernetes networking topology",
file_as_page=True, proposed_by="agent-x",
)
assert result["answer"] == ""
assert result["page_proposal_id"] is None
assert "uncited" in result["page_proposal_skipped_reason"] or "empty" in (
result["page_proposal_skipped_reason"] or ""
)
assert store.list_proposals() == []


def test_file_as_page_without_proposed_by_raises(store: KBStore) -> None:
_auth_kb(store)
with pytest.raises(ValueError, match="proposed_by"):
synthesize.synthesize(store, query="auth", file_as_page=True)


def test_file_as_page_defaults_to_false(store: KBStore) -> None:
"""The existing no-arg call shape must keep working exactly as before —
file_as_page defaults off and the result carries no proposal keys."""
_auth_kb(store)
result = synthesize.synthesize(store, query="auth", depth=5)
assert "page_proposal_id" not in result
assert "page_proposal_skipped_reason" not in result
assert store.list_proposals() == []


def test_jsonl_synthesize_file_as_page(store: KBStore, monkeypatch) -> None:
_auth_kb(store)
monkeypatch.chdir(store.root)
resp = handle_request({
"id": "s3", "method": "kb.synthesize",
"params": {"query": "auth tokens", "depth": 5, "file_as_page": True},
})
assert resp["ok"]
assert resp["result"]["page_proposal_id"] is not None
pending = [p for p in store.list_proposals() if p.status == ProposalStatus.PENDING]
assert len(pending) == 1


def test_cli_synthesize_file_as_page(store: KBStore, monkeypatch, tmp_path: Path) -> None:
from click.testing import CliRunner

from vouch.cli import cli

_auth_kb(store)
monkeypatch.chdir(store.root)
runner = CliRunner()
result = runner.invoke(
cli, ["synthesize", "auth tokens", "--depth", "5", "--file-as-page",
"--agent", "cli-tester"],
)
assert result.exit_code == 0, result.output
payload = json.loads(result.output)
assert payload["page_proposal_id"] is not None
proposal = store.get_proposal(payload["page_proposal_id"])
assert proposal.proposed_by == "cli-tester"
Loading