diff --git a/CHANGELOG.md b/CHANGELOG.md index c582c62d..15e3ae5c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,45 @@ All notable changes to vouch are documented here. Format follows ## [Unreleased] ### Added +- **conversation and memory-export importers** (#431): `vouch import chat-json + ` normalises the common JSON chat shapes — openai's branching + `mapping` tree (delegated to the existing chatgpt importer), claude.ai's + `chat_messages`, and the generic `messages: [{role, content}]` almost + everything else emits, from a file, a `.jsonl`, or a `.zip` — into one + PENDING page per conversation, cited to a per-conversation source. + `vouch import memory-export ` reads a prior memory tool's dump (JSON + array, object of records, JSONL, or one memory per line) and files each + memory as a claim quoting its own source verbatim, so the receipt verifies + and the imported fact is citable rather than asserted. `markdown-vault` is + registered as an alias of the note-vault importer, so the three formats the + issue names are all reachable from one surface. Two guards keep an import + from becoming a reviewer's problem: `--max-proposals` caps a whole run and + the report says when the cap was hit (rerunning continues where it left + off), and candidates an approved claim or a pending proposal already cover + are dropped — lexically first, since a base install has no `[embeddings]` + extra, with the embedding hits (#147) folded in on top when available. + `--dry-run` reports without enqueuing, `--max-claims` files receipt-backed + claims from a conversation's answers, and nothing here calls `approve()`. +- **note-vault importers — arrive with years of notes already written** (#612): + `vouch import obsidian `, plus `joplin` (folder or `.jex`), `notes` + (apple notes html/txt export), `keep` (google takeout folder or `.zip`), and + `md` (a plain markdown folder). One PENDING page proposal per note, each + cited to a source registered from **the note's own bytes** — which is the + thing an embedding-based importer structurally cannot offer: claims extracted + from an imported note quote real offsets, so their receipts verify and the + knowledge is citable rather than paraphrased. Frontmatter is carried into + source metadata; wikilinks (and joplin's `:/id` links) become `references` + relation proposals, but only where the target resolves to a note that was + actually imported. Re-running is idempotent on a stable per-note identity + derived from the origin's own identifier — unchanged notes are no-ops, + changed ones refresh their pending proposal in place, decided ones stay + decided — so a large vault imports in `--limit` slices and resumes. Claims + are opt-in and bounded through the existing density-selection knob + (`--max-claims`, default 0 = pages only), so ten thousand notes cannot become + ten thousand pending claims. `--dry-run` previews. Nothing is approved: an + import is a proposal firehose, not a write. `vouch import chatgpt` now also + exists as an alias so one `vouch import ` surface covers every source; + the flat `vouch import-chatgpt` stays for back-compat. - **explicit pins — a working set that always enters the pack** (#615): `vouch pin ` / `vouch pins list` / `vouch unpin `. Pinned claims and pages lead every context pack instead of having to win the query each turn, diff --git a/src/vouch/cli.py b/src/vouch/cli.py index dfcaeb0f..e637521f 100644 --- a/src/vouch/cli.py +++ b/src/vouch/cli.py @@ -33,6 +33,7 @@ from . import codex_rollout as codex_rollout_mod from . import compile as compile_mod from . import contradictions as contradictions_mod +from . import conversation_import as conversation_import_mod from . import digest as digest_mod from . import fetch as fetch_mod from . import hub as hub_mod @@ -41,6 +42,7 @@ from . import lifecycle as life from . import metrics as metrics_mod from . import migrations as migrations_mod +from . import note_import as note_import_mod from . import notify as notify_mod from . import pins as pins_mod from . import pr_cache as prc_mod @@ -116,6 +118,8 @@ def _cli_errors() -> Iterator[None]: migrations_mod.MigrationError, chatgpt_import_mod.ChatGPTImportError, codex_rollout_mod.CodexRolloutError, + note_import_mod.NoteImportError, + conversation_import_mod.ConversationImportError, pins_mod.PinError, ) as e: raise click.ClickException(str(e)) from e @@ -4409,6 +4413,253 @@ def import_chatgpt_cmd( _echo("run `vouch review` to decide.") +# --- import: note vaults (issue #612) -------------------------------------- + + +@cli.group(name="import") +def import_group() -> None: + """Import an existing note vault as PENDING proposals. + + One page per note, cited to a source registered from the note's own bytes — + so claims extracted from it quote real offsets and their receipts verify. + Re-running is idempotent: unchanged notes are no-ops, changed ones refresh + their pending proposal, decided ones stay decided. Nothing is approved. + """ + + +def _import_vault_cmd(kind: str, path: Path, limit: int | None, max_claims: int, + dry_run: bool, as_json: bool) -> None: + store = _load_store() + with _cli_errors(): + report = note_import_mod.import_vault( + store, kind, path, limit=limit, max_claims=max_claims, + dry_run=dry_run, generated_at=datetime.now(UTC).isoformat(), + ) + if as_json: + _emit_json(report) + return + verb = "would import" if dry_run else "imported" + _echo( + f"{report['notes']} note(s) — {verb} {report['imported']} new, " + f"{report['updated']} updated, {report['skipped']} skipped" + ) + if report["claims"] or report["relations"]: + _echo( + f" + {report['claims']} claim(s), {report['relations']} link " + f"relation(s) proposed" + ) + for row in report["rows"]: + if row["action"] == "skipped": + continue + pid = row.get("proposal_id") or "(dry-run)" + _echo(f" • {pid} {row['title']}") + if not dry_run and (report["imported"] or report["updated"]): + _echo("run `vouch review` to decide.") + + +def _vault_options(fn: Any) -> Any: + """The four flags every `vouch import ` subcommand shares.""" + fn = click.option( + "--json", "as_json", is_flag=True, help="Machine-readable report." + )(fn) + fn = click.option( + "--dry-run", is_flag=True, help="Parse and report; file nothing." + )(fn) + fn = click.option( + "--max-claims", "max_claims", type=int, default=0, show_default=True, + help="File up to N receipt-backed claims per note (0 = pages only).", + )(fn) + fn = click.option( + "--limit", type=int, default=None, + help="Import at most N notes, in sorted order (resumable).", + )(fn) + return fn + + +@import_group.command("obsidian") +@click.argument("vault", type=click.Path(exists=True, path_type=Path)) +@_vault_options +def import_obsidian_cmd( + vault: Path, limit: int | None, max_claims: int, dry_run: bool, as_json: bool +) -> None: + """Import an Obsidian vault (frontmatter, wikilinks, folder structure).""" + _import_vault_cmd("obsidian", vault, limit, max_claims, dry_run, as_json) + + +@import_group.command("md") +@click.argument("folder", type=click.Path(exists=True, path_type=Path)) +@_vault_options +def import_md_cmd( + folder: Path, limit: int | None, max_claims: int, dry_run: bool, as_json: bool +) -> None: + """Import a plain markdown folder.""" + _import_vault_cmd("md", folder, limit, max_claims, dry_run, as_json) + + +@import_group.command("joplin") +@click.argument("export_path", type=click.Path(exists=True, path_type=Path)) +@_vault_options +def import_joplin_cmd( + export_path: Path, limit: int | None, max_claims: int, dry_run: bool, as_json: bool +) -> None: + """Import a Joplin export — a .jex archive or the folder it unpacks to.""" + _import_vault_cmd("joplin", export_path, limit, max_claims, dry_run, as_json) + + +@import_group.command("notes") +@click.argument("export_path", type=click.Path(exists=True, path_type=Path)) +@_vault_options +def import_apple_notes_cmd( + export_path: Path, limit: int | None, max_claims: int, dry_run: bool, as_json: bool +) -> None: + """Import an Apple Notes export (a folder of .html/.txt per note).""" + _import_vault_cmd("notes", export_path, limit, max_claims, dry_run, as_json) + + +@import_group.command("keep") +@click.argument("export_path", type=click.Path(exists=True, path_type=Path)) +@_vault_options +def import_keep_cmd( + export_path: Path, limit: int | None, max_claims: int, dry_run: bool, as_json: bool +) -> None: + """Import a Google Keep (Takeout) export — the folder or the .zip.""" + _import_vault_cmd("keep", export_path, limit, max_claims, dry_run, as_json) + + +def _export_options(fn: Any) -> Any: + """The flags every conversation/memory export subcommand shares.""" + fn = click.option( + "--json", "as_json", is_flag=True, help="Machine-readable report." + )(fn) + fn = click.option( + "--dry-run", is_flag=True, help="Parse and report; file nothing." + )(fn) + fn = click.option( + "--no-dedup", "no_dedup", is_flag=True, + help="File every candidate, even one the KB already covers.", + )(fn) + fn = click.option( + "--max-proposals", "max_proposals", type=int, default=None, + help="Cap the whole run; the report says when the cap was hit.", + )(fn) + fn = click.option( + "--limit", type=int, default=None, + help="Consider at most N entries, in export order.", + )(fn) + return fn + + +def _echo_import_report(report: dict[str, Any], unit: str, dry_run: bool) -> None: + verb = "would import" if dry_run else "imported" + _echo( + f"{report[unit]} {unit[:-1]}(s) — {verb} {report['imported']} new, " + f"{report.get('updated', 0)} updated, {report['skipped']} skipped" + ) + if report.get("claims"): + _echo(f" + {report['claims']} receipt-backed claim(s) proposed") + if report["capped"]: + _echo( + f" ! stopped at --max-proposals {report['max_proposals']} — rerun " + f"to continue where this left off" + ) + if not dry_run and (report["imported"] or report.get("updated")): + _echo("run `vouch review` to decide.") + + +@import_group.command("chat-json") +@click.argument( + "export_path", type=click.Path(exists=True, dir_okay=False, path_type=Path) +) +@click.option( + "--max-claims", "max_claims", type=int, default=0, show_default=True, + help="File up to N receipt-backed claims per conversation (0 = pages only).", +) +@_export_options +def import_chat_json_cmd( + export_path: Path, limit: int | None, max_proposals: int | None, + no_dedup: bool, dry_run: bool, as_json: bool, max_claims: int, +) -> None: + """Import a JSON chat export (claude.ai, gemini, perplexity, openai, …). + + One PENDING page per conversation, cited to a per-conversation source. + Re-importing is idempotent. Nothing is approved. + """ + store = _load_store() + with _cli_errors(): + report = conversation_import_mod.import_conversations( + store, export_path, limit=limit, max_proposals=max_proposals, + max_claims=max_claims, dry_run=dry_run, dedup=not no_dedup, + generated_at=datetime.now(UTC).isoformat(), + ) + if as_json: + _emit_json(report) + return + _echo_import_report(report, "conversations", dry_run) + + +@import_group.command("memory-export") +@click.argument( + "export_path", type=click.Path(exists=True, dir_okay=False, path_type=Path) +) +@_export_options +def import_memory_export_cmd( + export_path: Path, limit: int | None, max_proposals: int | None, + no_dedup: bool, dry_run: bool, as_json: bool, +) -> None: + """Import a prior memory tool's dump as PENDING claim proposals. + + Each memory becomes a claim quoting its own source verbatim, so the + receipt verifies and the imported fact is citable rather than asserted. + """ + store = _load_store() + with _cli_errors(): + report = conversation_import_mod.import_memories( + store, export_path, limit=limit, max_proposals=max_proposals, + dry_run=dry_run, dedup=not no_dedup, + ) + if as_json: + _emit_json(report) + return + _echo_import_report(report, "memories", dry_run) + + +@import_group.command("markdown-vault") +@click.argument("folder", type=click.Path(exists=True, path_type=Path)) +@_vault_options +def import_markdown_vault_cmd( + folder: Path, limit: int | None, max_claims: int, dry_run: bool, as_json: bool +) -> None: + """Import a markdown folder (alias of `vouch import md`).""" + _import_vault_cmd("md", folder, limit, max_claims, dry_run, as_json) + + +@import_group.command("chatgpt") +@click.argument( + "export_path", type=click.Path(exists=True, dir_okay=False, path_type=Path) +) +@click.option( + "--limit", type=int, default=None, + help="Import at most N conversations, in export order.", +) +@click.option("--dry-run", is_flag=True, help="Parse and report; file nothing.") +@click.option("--json", "as_json", is_flag=True, help="Machine-readable report.") +@click.pass_context +def import_chatgpt_sub_cmd( + ctx: click.Context, + export_path: Path, limit: int | None, dry_run: bool, as_json: bool +) -> None: + """Import a ChatGPT history export (alias of `vouch import-chatgpt`). + + The conversation importer predates this group; it lives here too so the + one `vouch import ` surface covers every source. The flat + `vouch import-chatgpt` stays for back-compat. + """ + ctx.invoke( + import_chatgpt_cmd, export_path=export_path, limit=limit, + dry_run=dry_run, as_json=as_json, + ) + + # --- auto-pr: open N mergeable PRs against any github repo ----------------- diff --git a/src/vouch/conversation_import.py b/src/vouch/conversation_import.py new file mode 100644 index 00000000..91a518cf --- /dev/null +++ b/src/vouch/conversation_import.py @@ -0,0 +1,820 @@ +"""Import conversation and memory exports as review-gated proposals (#431). + +Someone arriving with existing agent history — a claude.ai or gemini or +perplexity chat export, a memory dump from a prior memory tool — starts with an +empty KB and re-teaches everything by hand. That is the biggest friction on +adoption, and it is the gap ``chatgpt_import`` (one vendor) and the note-vault +importers (files, not conversations) leave open. + +Two readers, both tolerant: + +* ``chat-json`` normalises the common JSON conversation shapes — openai's + branching ``mapping`` tree (delegated to ``chatgpt_import``), claude.ai's + ``chat_messages``, and the generic ``messages: [{role, content}]`` an export + from almost anything else emits — into the same ``Conversation`` / + ``Exchange`` pair the chatgpt importer already uses. One PENDING page per + conversation, cited to a per-conversation source. +* ``memory-export`` reads a prior tool's memory dump — a JSON array, a JSON + object of records, JSONL, or one memory per line — and files each memory as a + receipt-backed claim that quotes its own source verbatim. + +``markdown-vault`` is the third format the issue names; it is the note-vault +importer, registered here as an alias rather than reimplemented. + +Three things keep an import from becoming a reviewer's problem: + +* **``--max-proposals``** caps a single run and says so in the report, so a + ten-year history cannot flood the queue in one shot. +* **Dedup** drops a candidate that an approved claim or an already-pending + proposal covers. Lexical first, because a base install has no ``[embeddings]`` + extra and dedup that silently stops working is how an unattended import + floods a queue anyway; the embedding hits (#147) fold in on top when + available. +* **``--dry-run``** reports what a real run would file and touches nothing. + +Never calls ``approve()``. Everything lands PENDING, and a human drains the +queue exactly as for any other write. +""" + +from __future__ import annotations + +import json +import os +import re +import zipfile +from dataclasses import dataclass, field +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +from .chatgpt_import import Conversation, Exchange, _parse_conversation +from .extract import extract_receipt_claims +from .models import Proposal, ProposalKind, ProposalStatus +from .proposals import default_scope, propose_page, propose_quoted_claim +from .storage import KBStore + +# Deliberately not one of admission's AUTO_CAPTURE_ACTORS: an import is a human +# choosing to file their own history, so admission verdicts stay advisory and +# the pages reach review instead of being auto-rejected as capture noise. +CONVERSATION_ACTOR = "conversation-import" + +PAGE_TYPE = "session" +FORMATS = ("chat-json", "memory-export", "markdown-vault") + +# Mirrors the ceiling in `chatgpt_import` and the note-vault importers. +_MAX_EXPORT_BYTES = 200 * 1024 * 1024 +_MAX_TURN_CHARS = 2_000 +_MAX_EXCHANGES_PER_PAGE = 50 +_MAX_TITLE_CHARS = 120 +_MAX_MEMORY_CHARS = 2_000 +_MIN_MEMORY_CHARS = 20 +_SLUG_RE = re.compile(r"[^a-z0-9]+") +_WORD_RE = re.compile(r"[a-z0-9']+") + +# Same shape as the stoplist in `contradictions` — words too common to say +# anything about whether two candidates are the same knowledge. +_STOPWORDS = frozenset({ + "the", "a", "an", "is", "are", "was", "were", "be", "been", "being", + "to", "of", "in", "on", "for", "and", "or", "but", "with", "that", + "this", "it", "as", "at", "by", "from", "into", "than", "then", "we", + "you", "i", "they", "he", "she", "our", "your", "my", +}) + +# Above this token overlap with an approved claim or a pending proposal, a +# candidate is the same knowledge and is dropped rather than filed again. +DEFAULT_DEDUP_THRESHOLD = 0.75 + +_ROLE_ALIASES = { + "user": "user", "human": "user", "prompter": "user", "me": "user", + "assistant": "assistant", "model": "assistant", "ai": "assistant", + "bot": "assistant", "gpt": "assistant", "claude": "assistant", +} + +_MEMORY_KEYS = ("memory", "text", "content", "fact", "value", "note", "body") + + +class ConversationImportError(RuntimeError): + """Raised when an export can't be read or doesn't parse as one. + + The CLI turns this into a clean ``Error: ...`` line via ``_cli_errors``; + nothing is written to the KB when it's raised. + """ + + +@dataclass +class Memory: + """One remembered fact from a prior tool's memory dump.""" + + text: str + key: str + created_at: str | None = None + tags: list[str] = field(default_factory=list) + + +# --- shared ----------------------------------------------------------------- + + +def _tokens(text: str) -> set[str]: + return { + w for w in _WORD_RE.findall(text.lower()) + if w not in _STOPWORDS and len(w) > 2 + } + + +def overlap(a: str, b: str) -> float: + """Jaccard overlap of two texts' significant tokens, 0.0 to 1.0.""" + ta, tb = _tokens(a), _tokens(b) + if not ta or not tb: + return 0.0 + return len(ta & tb) / len(ta | tb) + + +def _read_export_bytes(path: Path) -> bytes: + """The export's bytes, from a bare file or the first JSON entry in a ZIP. + + Size is checked before reading — for a ZIP against the entry's *declared* + uncompressed size, so a small archive cannot smuggle in a zip-bombed + payload. + """ + try: + is_zip = zipfile.is_zipfile(path) + except OSError as e: + raise ConversationImportError(f"cannot read export file {path}: {e}") from e + if is_zip: + with zipfile.ZipFile(path) as zf: + entries = [ + i for i in sorted(zf.infolist(), key=lambda i: i.filename) + if not i.is_dir() and i.filename.lower().endswith((".json", ".jsonl")) + ] + if not entries: + raise ConversationImportError( + f"{path.name} holds no .json export — pass the export file " + f"itself if the archive is laid out differently" + ) + info = entries[0] + if info.file_size > _MAX_EXPORT_BYTES: + raise ConversationImportError( + f"{info.filename} is too large to import " + f"({info.file_size} bytes > {_MAX_EXPORT_BYTES} byte limit)" + ) + return zf.read(info) + try: + if path.stat().st_size > _MAX_EXPORT_BYTES: + raise ConversationImportError( + f"{path.name} is too large to import " + f"(> {_MAX_EXPORT_BYTES} byte limit)" + ) + return path.read_bytes() + except OSError as e: + raise ConversationImportError(f"cannot read export file {path}: {e}") from e + + +def _load_json(path: Path) -> Any: + raw = _read_export_bytes(path).decode("utf-8", errors="replace") + try: + return json.loads(raw) + except json.JSONDecodeError: + # JSONL is the other thing every tool emits. One bad line is skipped, + # not fatal — an export is someone else's file. + rows = [] + for line in raw.splitlines(): + line = line.strip() + if not line: + continue + try: + rows.append(json.loads(line)) + except json.JSONDecodeError: + continue + if rows: + return rows + raise ConversationImportError( + f"{path.name} is neither JSON nor JSONL — this does not look like " + f"an export" + ) from None + + +def _clip(text: str, limit: int = _MAX_TURN_CHARS) -> str: + stripped = text.strip() + if len(stripped) <= limit: + return stripped + return stripped[: limit - 1].rstrip() + "…" + + +def _iso(value: Any) -> str | None: + """An export timestamp (epoch seconds, epoch millis, or ISO) as ISO-8601.""" + if isinstance(value, str) and value.strip(): + return value.strip() + if not isinstance(value, (int, float)) or isinstance(value, bool): + return None + seconds = float(value) + if seconds > 1e11: # milliseconds + seconds /= 1000.0 + try: + return datetime.fromtimestamp(seconds, tz=UTC).isoformat() + except (OverflowError, OSError, ValueError): + return None + + +# --- chat-json -------------------------------------------------------------- + + +def _block_text(content: Any) -> str: + """Message content as text, whether it is a string or typed blocks.""" + if isinstance(content, str): + return content.strip() + if isinstance(content, list): + parts = [ + str(block.get("text", "")).strip() + for block in content + if isinstance(block, dict) and str(block.get("text", "")).strip() + ] + return "\n\n".join(parts) + if isinstance(content, dict): + return _block_text(content.get("text") or content.get("parts") or "") + return "" + + +def _turns(raw_messages: Any) -> list[tuple[str, str]]: + """Normalise a flat message list into ``(role, text)`` pairs.""" + out: list[tuple[str, str]] = [] + if not isinstance(raw_messages, list): + return out + for message in raw_messages: + if not isinstance(message, dict): + continue + raw_role = message.get("role") or message.get("sender") or message.get("author") + if isinstance(raw_role, dict): + raw_role = raw_role.get("role") + role = _ROLE_ALIASES.get(str(raw_role or "").strip().lower()) + if role is None: + continue + text = _block_text( + message.get("content") + if message.get("content") is not None + else message.get("text") + ) + if text: + out.append((role, _clip(text))) + return out + + +def _pair(turns: list[tuple[str, str]]) -> list[Exchange]: + """Pair each user turn with the assistant turn that answered it.""" + exchanges: list[Exchange] = [] + pending: str | None = None + for role, text in turns: + if role == "user": + pending = text + elif pending is not None: + exchanges.append(Exchange(user=pending, assistant=text)) + pending = None + return exchanges + + +def _messages_of(raw: dict[str, Any]) -> Any: + for key in ("chat_messages", "messages", "turns", "conversation", "history"): + if isinstance(raw.get(key), list): + return raw[key] + return None + + +def _generic_conversation(raw: Any, index: int) -> Conversation | None: + if not isinstance(raw, dict): + return None + messages = _messages_of(raw) + if messages is None: + return None + conv_id = raw.get("uuid") or raw.get("id") or raw.get("conversation_id") + conv_id = str(conv_id).strip() if conv_id else f"conversation-{index}" + title = raw.get("name") or raw.get("title") + exchanges = _pair(_turns(messages)) + return Conversation( + conversation_id=conv_id, + title=str(title).strip()[:_MAX_TITLE_CHARS] if title else None, + created_at=_iso(raw.get("created_at") or raw.get("create_time")), + updated_at=_iso(raw.get("updated_at") or raw.get("update_time")), + exchanges=exchanges, + ) + + +def parse_chat_json(path: Path) -> list[Conversation]: + """Parse a JSON conversation export into normalised conversations. + + Handles, in order of specificity: openai's branching ``mapping`` tree (via + ``chatgpt_import``), claude.ai's ``chat_messages``, the generic + ``messages: [{role, content}]`` shape, and a bare list of messages with no + conversation wrapper at all. Entries that match none of them are skipped — + an export is someone else's file and its schema drifts. + """ + data = _load_json(path) + if isinstance(data, dict): + for key in ("conversations", "chats", "data", "items"): + if isinstance(data.get(key), list): + data = data[key] + break + if isinstance(data, dict): + data = [data] + if not isinstance(data, list): + raise ConversationImportError( + f"{path.name}: expected a JSON array of conversations (or an object " + f"wrapping one) — this does not look like a chat export" + ) + + # A bare list of messages, with no conversation wrapper. + if data and all( + isinstance(row, dict) and _messages_of(row) is None and "mapping" not in row + for row in data + ): + turns = _turns(data) + if turns: + return [Conversation( + conversation_id=path.stem or "conversation", + title=path.stem or None, + exchanges=_pair(turns), + )] + + out: list[Conversation] = [] + for index, raw in enumerate(data): + conv: Conversation | None = None + if isinstance(raw, dict) and isinstance(raw.get("mapping"), dict): + conv = _parse_conversation(raw) + if conv is None: + conv = _generic_conversation(raw, index) + if conv is not None: + out.append(conv) + if not out: + raise ConversationImportError( + f"{path.name}: no conversations found — every entry was missing a " + f"recognisable message list" + ) + return out + + +# --- memory-export ---------------------------------------------------------- + + +def _memory_of(raw: Any, index: int) -> Memory | None: + if isinstance(raw, str): + text = raw.strip() + return Memory(text=text, key=f"memory-{index}") if text else None + if not isinstance(raw, dict): + return None + text = "" + for key in _MEMORY_KEYS: + candidate = raw.get(key) + if isinstance(candidate, str) and candidate.strip(): + text = candidate.strip() + break + if not text: + return None + identifier = raw.get("id") or raw.get("uuid") or raw.get("key") + raw_tags = raw.get("tags") or raw.get("categories") or raw.get("labels") + tags = [ + str(t).strip() for t in raw_tags + if isinstance(raw_tags, list) and str(t).strip() + ][:20] if isinstance(raw_tags, list) else [] + return Memory( + text=_clip(text, _MAX_MEMORY_CHARS), + key=str(identifier).strip() if identifier else f"memory-{index}", + created_at=_iso(raw.get("created_at") or raw.get("timestamp")), + tags=tags, + ) + + +def parse_memory_export(path: Path) -> list[Memory]: + """Parse a prior tool's memory dump into normalised memories. + + Accepts a JSON array of strings or records, an object wrapping one under a + ``memories``/``facts``/``items`` key, an object *of* records, JSONL, and — + when the file is not JSON at all — one memory per non-empty line. Memories + shorter than ``_MIN_MEMORY_CHARS`` are dropped as acknowledgements rather + than knowledge. + """ + try: + data: Any = _load_json(path) + except ConversationImportError: + text = _read_export_bytes(path).decode("utf-8", errors="replace") + data = [line.strip() for line in text.splitlines() if line.strip()] + + if isinstance(data, dict): + for key in ("memories", "facts", "items", "data", "records"): + if isinstance(data.get(key), list): + data = data[key] + break + else: + data = [ + {"id": k, **v} if isinstance(v, dict) else {"id": k, "memory": v} + for k, v in data.items() + ] + if not isinstance(data, list): + raise ConversationImportError( + f"{path.name}: expected a list of memories — this does not look " + f"like a memory export" + ) + + out: list[Memory] = [] + seen: set[str] = set() + for index, raw in enumerate(data): + memory = _memory_of(raw, index) + if memory is None or len(memory.text) < _MIN_MEMORY_CHARS: + continue + if memory.text in seen: # the same fact twice in one dump + continue + seen.add(memory.text) + out.append(memory) + if not out: + raise ConversationImportError( + f"{path.name}: no memories found — entries were empty, too short, " + f"or carried no recognisable text field" + ) + return out + + +# --- dedup ------------------------------------------------------------------ + + +def already_known( + store: KBStore, text: str, *, threshold: float = DEFAULT_DEDUP_THRESHOLD +) -> str | None: + """The id of an approved claim or pending proposal that already says this. + + Lexical first on purpose: the embedding path (#147) needs the + ``[embeddings]`` extra, and dedup that silently stops working on a base + install is precisely how an unattended import floods a review queue. The + embedding hits fold in on top when the extra is present. + """ + stripped = text.strip() + if not stripped: + return None + for claim in store.list_claims(): + if overlap(stripped, claim.text) >= threshold: + return claim.id + for proposal in store.list_proposals(ProposalStatus.PENDING): + if proposal.kind is not ProposalKind.CLAIM: + continue + existing = str(proposal.payload.get("text", "")) + if overlap(stripped, existing) >= threshold: + return proposal.id + try: + from .embeddings.similarity import find_similar_on_propose + + for warning in find_similar_on_propose(store, stripped): + artifact_id = warning.get("artifact_id") + if isinstance(artifact_id, str): + return artifact_id + except ImportError: + pass + return None + + +# --- conversations -> proposals --------------------------------------------- + + +def session_key(conversation_id: str) -> str: + """The session id one conversation dedups on, across every re-import.""" + return f"chat-import-{conversation_id}" + + +def _page_slug(conversation_id: str) -> str: + slug = _SLUG_RE.sub("-", conversation_id.lower()).strip("-") or "conversation" + return f"chat-{slug}"[:80] + + +def build_page_title(conv: Conversation) -> str: + if conv.title: + return f"conversation: {conv.title}"[:_MAX_TITLE_CHARS] + return f"conversation: {conv.conversation_id}"[:_MAX_TITLE_CHARS] + + +def build_page_body(conv: Conversation, *, generated_at: str | None = None) -> str: + stamp = generated_at or datetime.now(UTC).isoformat() + lines = [ + f"# {build_page_title(conv)}", + "", + f"- imported: {stamp}", + f"- conversation: {conv.conversation_id}", + ] + if conv.created_at: + lines.append(f"- started: {conv.created_at}") + if conv.updated_at: + lines.append(f"- last-active: {conv.updated_at}") + lines.append(f"- exchanges: {len(conv.exchanges)}") + lines.extend(["", "## exchanges", ""]) + shown = conv.exchanges[:_MAX_EXCHANGES_PER_PAGE] + for exchange in shown: + lines.extend([f"**you:** {exchange.user}", "", f"**assistant:** {exchange.assistant}", ""]) + hidden = len(conv.exchanges) - len(shown) + if hidden > 0: + lines.append( + f"(… {hidden} more exchange(s) — full conversation in the cited source)" + ) + return "\n".join(lines).rstrip() + "\n" + + +def _source_content(conv: Conversation) -> bytes: + """The conversation's full text as deterministic JSON bytes. + + Deterministic serialization means an unchanged conversation hashes to the + same source id on every import — ``put_source`` dedups on content, so + re-imports never pile up copies. + """ + payload = { + "conversation_id": conv.conversation_id, + "title": conv.title, + "created_at": conv.created_at, + "updated_at": conv.updated_at, + "exchanges": [ + {"user": e.user, "assistant": e.assistant} for e in conv.exchanges + ], + } + return json.dumps( + payload, ensure_ascii=False, indent=2, sort_keys=True + ).encode("utf-8") + + +def _comparable_body(body: str) -> str: + return "\n".join( + line for line in body.splitlines() if not line.startswith("- imported:") + ) + + +def _find_existing_page(store: KBStore, sid: str) -> Proposal | None: + for proposal in store.list_proposals(None): + if proposal.kind == ProposalKind.PAGE and proposal.session_id == sid: + return proposal + return None + + +@dataclass +class _Budget: + """The `--max-proposals` cap, and whether a run actually hit it.""" + + limit: int | None + spent: int = 0 + hit: bool = False + + def take(self, n: int = 1) -> bool: + if self.limit is None: + self.spent += n + return True + if self.spent + n > self.limit: + self.hit = True + return False + self.spent += n + return True + + +def import_conversations( + store: KBStore, + path: Path, + *, + actor: str | None = None, + limit: int | None = None, + max_proposals: int | None = None, + max_claims: int = 0, + dry_run: bool = False, + dedup: bool = True, + generated_at: str | None = None, +) -> dict[str, Any]: + """Import a chat export into PENDING proposals. Never calls ``approve()``. + + One page per conversation, deduped on a stable per-conversation session id: + a decided proposal blocks re-import, a still-PENDING one refreshes in place + when the conversation grew, an unchanged one is a no-op. ``max_claims`` > 0 + additionally files that many receipt-backed claims per conversation from + the assistant's own answers. + + ``max_proposals`` caps the whole run — the report says whether the cap was + hit, so an operator can see the import was truncated rather than finished. + """ + conversations = parse_chat_json(path) + if limit is not None and limit >= 0: + conversations = conversations[:limit] + resolved_actor = actor or os.environ.get("VOUCH_AGENT") or CONVERSATION_ACTOR + budget = _Budget(max_proposals) + + rows: list[dict[str, Any]] = [] + counts = {"imported": 0, "updated": 0, "skipped": 0} + claims_filed = 0 + + for conv in conversations: + sid = session_key(conv.conversation_id) + row: dict[str, Any] = { + "conversation": conv.conversation_id, + "title": build_page_title(conv), + "session_id": sid, + } + if not conv.exchanges: + row.update(action="skipped", reason="no exchanges") + counts["skipped"] += 1 + rows.append(row) + continue + existing = _find_existing_page(store, sid) + if existing is not None and existing.status != ProposalStatus.PENDING: + row.update(action="skipped", reason="already-imported", proposal_id=existing.id) + counts["skipped"] += 1 + rows.append(row) + continue + body = build_page_body(conv, generated_at=generated_at) + if existing is not None and _comparable_body(body) == _comparable_body( + str(existing.payload.get("body", "")) + ): + row.update(action="skipped", reason="unchanged", proposal_id=existing.id) + counts["skipped"] += 1 + rows.append(row) + continue + if not budget.take(): + row.update(action="skipped", reason="max-proposals") + counts["skipped"] += 1 + rows.append(row) + continue + if dry_run: + action = "updated" if existing is not None else "imported" + row.update(action=action, dry_run=True) + counts[action] += 1 + rows.append(row) + continue + + source = store.put_source( + _source_content(conv), + title=row["title"], + locator=f"chat:{conv.conversation_id}", + media_type="application/json", + tags=["chat-import", "conversation-import"], + scope=default_scope(store), + ) + if existing is not None: + refreshed = existing.model_copy(deep=True) + refreshed.payload["title"] = row["title"] + refreshed.payload["body"] = body + refreshed.payload["sources"] = [source.id] + store.update_proposal(refreshed) + row.update(action="updated", proposal_id=existing.id) + counts["updated"] += 1 + else: + proposal = propose_page( + store, + title=row["title"], + body=body, + page_type=PAGE_TYPE, + source_ids=[source.id], + proposed_by=resolved_actor, + tags=["chat-import"], + session_id=sid, + slug_hint=_page_slug(conv.conversation_id), + rationale="imported conversation export", + ) + row.update(action="imported", proposal_id=proposal.id) + counts["imported"] += 1 + + if max_claims > 0: + filed = _claims_from_answers( + store, conv, actor=resolved_actor, max_claims=max_claims, + budget=budget, dedup=dedup, + ) + row["claims"] = filed + claims_filed += filed + rows.append(row) + + return { + "format": "chat-json", + "conversations": len(conversations), + "imported": counts["imported"], + "updated": counts["updated"], + "skipped": counts["skipped"], + "claims": claims_filed, + "proposals": budget.spent, + "max_proposals": max_proposals, + "capped": budget.hit, + "dry_run": dry_run, + "rows": rows, + } + + +def _claims_from_answers( + store: KBStore, + conv: Conversation, + *, + actor: str, + max_claims: int, + budget: _Budget, + dedup: bool, +) -> int: + """Receipt-backed claims from the assistant's answers in one conversation. + + The answer is registered as its own source and each quotable span quotes it + verbatim, so the receipt verifies by construction — the same path session + answer-memory uses. Claims are what a reader wants out of a chat history; + the page is the context they came from. + """ + filed = 0 + for exchange in conv.exchanges: + if filed >= max_claims: + break + answer = exchange.assistant.strip() + if len(answer) < _MIN_MEMORY_CHARS: + continue + if dedup and already_known(store, answer) is not None: + continue + if not budget.take(): + break + source = store.put_source( + answer.encode("utf-8"), + title=f"answer from {conv.conversation_id}", + locator=f"chat:{conv.conversation_id}#answer", + tags=["chat-import"], + scope=default_scope(store), + ) + results = extract_receipt_claims( + store, source.id, proposed_by=actor, + max_claims=max_claims - filed, limit=max_claims - filed, + ) + filed += len(results) + # `extract_receipt_claims` files its own proposals; charge the budget + # for what it actually filed rather than the one slot reserved above. + budget.spent += max(0, len(results) - 1) + return filed + + +# --- memories -> proposals -------------------------------------------------- + + +def import_memories( + store: KBStore, + path: Path, + *, + actor: str | None = None, + limit: int | None = None, + max_proposals: int | None = None, + dry_run: bool = False, + dedup: bool = True, +) -> dict[str, Any]: + """Import a memory dump into PENDING claim proposals. No ``approve()``. + + Each memory is registered as its own source and filed as a claim quoting + that source verbatim, so the receipt verifies by construction and the + imported fact is citable rather than asserted. A memory an approved claim + or a pending proposal already covers is dropped. + """ + memories = parse_memory_export(path) + if limit is not None and limit >= 0: + memories = memories[:limit] + resolved_actor = actor or os.environ.get("VOUCH_AGENT") or CONVERSATION_ACTOR + budget = _Budget(max_proposals) + + rows: list[dict[str, Any]] = [] + counts = {"imported": 0, "skipped": 0} + for memory in memories: + row: dict[str, Any] = {"memory": memory.key, "text": _clip(memory.text, 120)} + duplicate = already_known(store, memory.text) if dedup else None + if duplicate is not None: + row.update(action="skipped", reason="already-known", duplicate_of=duplicate) + counts["skipped"] += 1 + rows.append(row) + continue + if not budget.take(): + row.update(action="skipped", reason="max-proposals") + counts["skipped"] += 1 + rows.append(row) + continue + if dry_run: + row.update(action="imported", dry_run=True) + counts["imported"] += 1 + rows.append(row) + continue + source = store.put_source( + memory.text.encode("utf-8"), + title=f"memory {memory.key}", + locator=f"memory:{memory.key}", + tags=["memory-import", *memory.tags], + metadata={"memory_key": memory.key, "created_at": memory.created_at}, + scope=default_scope(store), + ) + result = propose_quoted_claim( + store, text=memory.text, source_id=source.id, quote=memory.text, + proposed_by=resolved_actor, + rationale="imported memory export", + ) + if result is None: + # The verbatim check failed (e.g. bytes mangled on decode) — the + # claim is dropped rather than filed without a working receipt. + row.update(action="skipped", reason="unquotable") + counts["skipped"] += 1 + budget.spent -= 1 + rows.append(row) + continue + row.update(action="imported", proposal_id=result.proposal.id) + counts["imported"] += 1 + rows.append(row) + + return { + "format": "memory-export", + "memories": len(memories), + "imported": counts["imported"], + "skipped": counts["skipped"], + "proposals": budget.spent, + "max_proposals": max_proposals, + "capped": budget.hit, + "dry_run": dry_run, + "rows": rows, + } diff --git a/src/vouch/note_import.py b/src/vouch/note_import.py new file mode 100644 index 00000000..f75da7cb --- /dev/null +++ b/src/vouch/note_import.py @@ -0,0 +1,843 @@ +"""Import a note vault into review-gated proposals (issue #612). + +People arrive with years of notes already written — obsidian vaults, joplin +archives, apple notes and google keep exports, plain markdown folders — and a +fresh KB has nothing to say to them. ``vouch import obsidian `` (and its +four siblings) files **one PENDING page proposal per note**, each cited to a +source registered from the note's own bytes. + +That last part is the point, and it is what an embedding-based importer +structurally cannot offer: the source content is the note verbatim, so every +claim extracted from it quotes real bytes at real offsets and its receipt +verifies. Imported knowledge is citable, not paraphrased. + +Re-importing the same vault is safe. Each note gets a stable identity derived +from the origin's own identifier — the vault-relative path, or joplin's/keep's +own note id where the format carries one — recorded on the source ``locator`` +and used as the proposal's session id. A note that changed refreshes its still +PENDING proposal in place; an unchanged one is a flat no-op; a decided proposal +is history and blocks re-import. Nothing duplicates, so a big vault can be +imported in ``--limit`` slices and resumed. + +Wikilinks become ``references`` relation proposals, but only where the target +resolves to a note that was actually imported — a link to a note that does not +exist yet is dropped rather than filed as a dangling edge. + +Claims are opt-in and bounded. ``--max-claims`` is the density selection knob +(``extract.select_spans``): a ten-thousand-note vault must not turn into ten +thousand pending claims, so the default files pages only. + +Never calls ``approve()``. An import is a proposal firehose, not a write — a +human reviews it with ``vouch review`` like everything else. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import tarfile +import zipfile +from dataclasses import dataclass, field +from datetime import UTC, datetime +from html.parser import HTMLParser +from pathlib import Path +from typing import Any + +import yaml + +from . import audit +from .extract import extract_receipt_claims +from .models import Proposal, ProposalKind, ProposalStatus +from .proposals import ProposalError, default_scope, propose_page, propose_relation +from .storage import KBStore + +# Deliberately NOT one of admission's AUTO_CAPTURE_ACTORS: an import is a human +# choosing to file their own notes, so admission verdicts stay advisory and the +# pages reach review instead of being auto-rejected as capture noise. Mirrors +# `chatgpt_import.CHATGPT_ACTOR`. +NOTE_IMPORT_ACTOR = "note-import" + +# Each page is a review surface over exactly one imported note, cited to it. +PAGE_TYPE = "source-summary" + +KINDS = ("obsidian", "joplin", "notes", "keep", "md") + +# Mirrors the ceilings elsewhere (fetch, codex_rollout, chatgpt_import): one +# oversized archive must not exhaust memory. +_MAX_VAULT_BYTES = 200 * 1024 * 1024 +_MAX_NOTE_BYTES = 2 * 1024 * 1024 +_MAX_BODY_CHARS = 4_000 +_MAX_LINKS_PER_NOTE = 50 +_MAX_TITLE_CHARS = 120 +_SLUG_RE = re.compile(r"[^a-z0-9]+") + +# `[[Target]]`, `[[Target|alias]]`, `[[Target#heading]]`, `[[Target^block]]`. +_WIKILINK_RE = re.compile(r"\[\[([^\]\[|#^]+)(?:[#^][^\]\[|]*)?(?:\|[^\]\[]*)?\]\]") +# Joplin's internal link form: `[title](:/32-hex-id)`. +_JOPLIN_LINK_RE = re.compile(r"\]\(:/([0-9a-fA-F]{32})\)") + +_MARKDOWN_SUFFIXES = frozenset({".md", ".markdown", ".mdown", ".txt"}) +_NOTES_SUFFIXES = frozenset({".html", ".htm", ".txt", ".md"}) +# Directories a vault carries that are machinery, not notes. +_SKIP_DIRS = frozenset( + {".obsidian", ".trash", ".git", ".stfolder", "_resources", "node_modules"} +) + + +class NoteImportError(RuntimeError): + """Raised when a vault can't be read or doesn't parse as one. + + The CLI translates this into a clean ``Error: ...`` line via + ``_cli_errors``; nothing is written to the KB when it's raised. + """ + + +@dataclass +class Note: + """One imported note, normalised across the five source formats.""" + + key: str + """Stable identity from the origin — relative path, or the vault's own id.""" + + title: str + body: str + raw: bytes + """The note's own bytes, registered verbatim so receipts verify.""" + + locator: str + frontmatter: dict[str, Any] = field(default_factory=dict) + links: list[str] = field(default_factory=list) + tags: list[str] = field(default_factory=list) + created_at: str | None = None + updated_at: str | None = None + media_type: str = "text/markdown" + + +# --- shared parsing -------------------------------------------------------- + + +def _clip(text: str, limit: int = _MAX_BODY_CHARS) -> str: + text = text.strip() + if len(text) <= limit: + return text + return text[: limit - 1].rstrip() + "…" + + +def _title_from(stem: str, frontmatter: dict[str, Any], body: str) -> str: + """Frontmatter title, else the first `# heading`, else the filename.""" + raw = frontmatter.get("title") + if isinstance(raw, str) and raw.strip(): + return raw.strip()[:_MAX_TITLE_CHARS] + for line in body.splitlines(): + stripped = line.strip() + if stripped.startswith("# "): + return stripped[2:].strip()[:_MAX_TITLE_CHARS] or stem + if stripped: + break + return stem[:_MAX_TITLE_CHARS] + + +def split_frontmatter(text: str) -> tuple[dict[str, Any], str]: + """Split a leading ``---`` YAML frontmatter block off a note body. + + Tolerant by design: a malformed or non-mapping block is left in the body + rather than raising, because a vault is other people's files and one bad + note must not fail an import of ten thousand. + """ + if not text.startswith("---"): + return {}, text + lines = text.splitlines(keepends=True) + if not lines or lines[0].strip() != "---": + return {}, text + for idx in range(1, len(lines)): + if lines[idx].strip() in {"---", "..."}: + block = "".join(lines[1:idx]) + try: + loaded = yaml.safe_load(block) + except yaml.YAMLError: + return {}, text + if not isinstance(loaded, dict): + return {}, text + return loaded, "".join(lines[idx + 1 :]) + return {}, text + + +def _tags_from_frontmatter(frontmatter: dict[str, Any]) -> list[str]: + raw = frontmatter.get("tags") or frontmatter.get("tag") + if isinstance(raw, str): + parts = [p.strip() for p in raw.replace(",", " ").split()] + elif isinstance(raw, list): + parts = [str(p).strip() for p in raw] + else: + return [] + return [p.lstrip("#") for p in parts if p and p.strip("#")][:20] + + +def _wikilinks(body: str) -> list[str]: + out: list[str] = [] + seen: set[str] = set() + for match in _WIKILINK_RE.finditer(body): + target = match.group(1).strip() + if not target or target in seen: + continue + seen.add(target) + out.append(target) + if len(out) >= _MAX_LINKS_PER_NOTE: + break + return out + + +def _read_text(path: Path) -> str: + raw = path.read_bytes()[:_MAX_NOTE_BYTES] + return raw.decode("utf-8", errors="replace") + + +def _walk_notes(root: Path, suffixes: frozenset[str]) -> list[Path]: + """Every note file under ``root``, skipping vault machinery, sorted. + + Sorted so an interrupted `--limit` run resumes deterministically instead of + re-walking in filesystem order and picking a different slice. + """ + out: list[Path] = [] + total = 0 + for path in sorted(root.rglob("*")): + # One guard around every stat-backed call: a file that vanishes mid-walk + # is skipped whichever of them notices first. `is_file()` reaches the + # filesystem through `Path.stat` on some python versions and `os.stat` + # on others, so catching only around the explicit `stat()` left the + # skip version-dependent. + try: + if not path.is_file() or path.suffix.lower() not in suffixes: + continue + parents = path.relative_to(root).parts[:-1] + if any(part in _SKIP_DIRS or part.startswith(".") for part in parents): + continue + if path.name.startswith("."): + continue + total += path.stat().st_size + except OSError: + continue + if total > _MAX_VAULT_BYTES: + raise NoteImportError( + f"vault exceeds the {_MAX_VAULT_BYTES // (1024 * 1024)} MiB import " + f"ceiling at {path} — import a subfolder at a time" + ) + out.append(path) + return out + + +# --- obsidian / plain markdown --------------------------------------------- + + +def load_markdown_vault(root: Path, *, kind: str = "obsidian") -> list[Note]: + """Obsidian vaults and plain markdown folders — the same shape. + + Obsidian's on-disk format *is* a folder of markdown with YAML frontmatter + and `[[wikilinks]]`; the only difference from a plain folder is the + `.obsidian/` config directory, which `_walk_notes` skips either way. One + loader, two front doors. + """ + if not root.is_dir(): + raise NoteImportError(f"not a directory: {root}") + notes: list[Note] = [] + for path in _walk_notes(root, _MARKDOWN_SUFFIXES): + text = _read_text(path) + frontmatter, body = split_frontmatter(text) + rel = path.relative_to(root).as_posix() + key = rel[: -len(path.suffix)] if path.suffix else rel + stat = path.stat() + notes.append( + Note( + key=key, + title=_title_from(path.stem, frontmatter, body), + body=body, + raw=path.read_bytes()[:_MAX_NOTE_BYTES], + locator=f"{kind}:{rel}", + frontmatter=frontmatter, + links=_wikilinks(body), + tags=_tags_from_frontmatter(frontmatter), + created_at=_iso(getattr(stat, "st_birthtime", None) or stat.st_mtime), + updated_at=_iso(stat.st_mtime), + ) + ) + if not notes: + raise NoteImportError(f"no markdown notes found under {root}") + return notes + + +def _iso(epoch: Any) -> str | None: + try: + return datetime.fromtimestamp(float(epoch), tz=UTC).isoformat() + except (TypeError, ValueError, OSError): + return None + + +def _iso_ms(millis: Any) -> str | None: + try: + return datetime.fromtimestamp(float(millis) / 1000.0, tz=UTC).isoformat() + except (TypeError, ValueError, OSError): + return None + + +# --- joplin ---------------------------------------------------------------- + + +def _parse_joplin_note(text: str, fallback_key: str) -> Note | None: + """One joplin RAW/JEX note: title line, body, then a `key: value` footer. + + Joplin appends its metadata as trailing `key: value` lines after the last + blank line. `type_: 1` marks a note; folders, tags and resources use other + type codes and are skipped — importing a folder record as a note is the + classic mistake with this format. + """ + lines = text.splitlines() + meta: dict[str, str] = {} + cut = len(lines) + for idx in range(len(lines) - 1, -1, -1): + line = lines[idx] + if not line.strip(): + cut = idx + break + if ":" not in line: + return None + key, _, value = line.partition(":") + if not key.strip() or " " in key.strip(): + return None + meta[key.strip()] = value.strip() + if meta.get("type_") not in {"1", None} or "type_" not in meta: + return None + head = lines[:cut] + title = head[0].strip() if head else "" + body = "\n".join(head[1:]).strip() + note_id = meta.get("id") or fallback_key + return Note( + key=note_id, + title=(title or note_id)[:_MAX_TITLE_CHARS], + body=body, + raw=text.encode("utf-8"), + locator=f"joplin:{note_id}", + frontmatter={ + k: v + for k, v in meta.items() + if k in {"id", "parent_id", "source_url", "author", "is_todo"} + }, + links=list(dict.fromkeys(_JOPLIN_LINK_RE.findall(body)))[:_MAX_LINKS_PER_NOTE], + created_at=meta.get("user_created_time") or meta.get("created_time"), + updated_at=meta.get("user_updated_time") or meta.get("updated_time"), + ) + + +def load_joplin(path: Path) -> list[Note]: + """A joplin export: a `.jex` tar archive, or the folder it unpacks to.""" + notes: list[Note] = [] + if path.is_file(): + if not tarfile.is_tarfile(path): + raise NoteImportError( + f"{path} is not a joplin .jex archive (expected a tar) — pass the " + f"exported folder instead" + ) + with tarfile.open(path) as tar: + total = 0 + for member in tar.getmembers(): + if not member.isfile() or not member.name.endswith(".md"): + continue + total += member.size + if total > _MAX_VAULT_BYTES: + raise NoteImportError( + f"{path} exceeds the " + f"{_MAX_VAULT_BYTES // (1024 * 1024)} MiB import ceiling" + ) + handle = tar.extractfile(member) + if handle is None: # pragma: no cover - isfile() guarantees a stream + continue + text = handle.read(_MAX_NOTE_BYTES).decode("utf-8", errors="replace") + note = _parse_joplin_note(text, Path(member.name).stem) + if note is not None: + notes.append(note) + elif path.is_dir(): + for note_path in _walk_notes(path, frozenset({".md"})): + note = _parse_joplin_note(_read_text(note_path), note_path.stem) + if note is not None: + notes.append(note) + else: + raise NoteImportError(f"no such joplin export: {path}") + if not notes: + raise NoteImportError( + f"no joplin notes found in {path} — a joplin export is a folder of .md " + f"files with a trailing metadata block, or the .jex tar of one" + ) + notes.sort(key=lambda n: n.key) + return notes + + +# --- apple notes ----------------------------------------------------------- + + +class _TextExtractor(HTMLParser): + """Minimal HTML-to-text: apple's exporter emits styled html, not markdown. + + No dependency is added for this — an import must not require a parser the + rest of vouch has no use for. Block-level tags become newlines; everything + else contributes its text. + """ + + _BLOCK = frozenset({"p", "div", "br", "li", "tr", "h1", "h2", "h3", "h4", "h5"}) + + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self.parts: list[str] = [] + self._skip = 0 + + def handle_starttag(self, tag: str, attrs: Any) -> None: + if tag in {"script", "style"}: + self._skip += 1 + elif tag in self._BLOCK: + self.parts.append("\n") + + def handle_endtag(self, tag: str) -> None: + if tag in {"script", "style"} and self._skip: + self._skip -= 1 + elif tag in self._BLOCK: + self.parts.append("\n") + + def handle_data(self, data: str) -> None: + if not self._skip: + self.parts.append(data) + + def text(self) -> str: + joined = "".join(self.parts) + return re.sub(r"\n{3,}", "\n\n", joined).strip() + + +def html_to_text(html: str) -> str: + parser = _TextExtractor() + try: + parser.feed(html) + parser.close() + except Exception: # a malformed note must not fail the import + return re.sub(r"<[^>]+>", " ", html).strip() + return parser.text() + + +def load_apple_notes(root: Path) -> list[Note]: + """An apple notes export: a folder of .html (or .txt/.md) per note.""" + if not root.is_dir(): + raise NoteImportError(f"not a directory: {root}") + notes: list[Note] = [] + for path in _walk_notes(root, _NOTES_SUFFIXES): + raw = path.read_bytes()[:_MAX_NOTE_BYTES] + text = raw.decode("utf-8", errors="replace") + body = html_to_text(text) if path.suffix.lower() in {".html", ".htm"} else text + rel = path.relative_to(root).as_posix() + stat = path.stat() + notes.append( + Note( + key=rel[: -len(path.suffix)] if path.suffix else rel, + title=_title_from(path.stem, {}, body), + body=body, + raw=body.encode("utf-8"), # the extracted text is what claims quote + locator=f"notes:{rel}", + links=_wikilinks(body), + created_at=_iso(getattr(stat, "st_birthtime", None) or stat.st_mtime), + updated_at=_iso(stat.st_mtime), + media_type="text/plain", + ) + ) + if not notes: + raise NoteImportError(f"no apple-notes files found under {root}") + return notes + + +# --- google keep ----------------------------------------------------------- + + +def _keep_note(payload: dict[str, Any], fallback_key: str) -> Note | None: + if payload.get("isTrashed"): + return None + title = str(payload.get("title") or "").strip() + text = str(payload.get("textContent") or "").strip() + checklist = payload.get("listContent") + if isinstance(checklist, list): + rows = [ + f"- [{'x' if item.get('isChecked') else ' '}] {item.get('text', '')}" + for item in checklist + if isinstance(item, dict) + ] + text = "\n".join(filter(None, [text, *rows])).strip() + if not title and not text: + return None + labels = payload.get("labels") + tags = [ + str(item["name"]) + for item in labels or [] + if isinstance(item, dict) and item.get("name") + ][:20] + body = text + return Note( + key=fallback_key, + title=(title or fallback_key)[:_MAX_TITLE_CHARS], + body=body, + raw=body.encode("utf-8"), + locator=f"keep:{fallback_key}", + frontmatter={ + k: payload[k] + for k in ("isPinned", "isArchived", "color") + if k in payload + }, + tags=tags, + created_at=_iso_ms(payload.get("createdTimestampUsec", 0) / 1000) + if isinstance(payload.get("createdTimestampUsec"), (int, float)) + else None, + updated_at=_iso_ms(payload.get("userEditedTimestampUsec", 0) / 1000) + if isinstance(payload.get("userEditedTimestampUsec"), (int, float)) + else None, + media_type="text/plain", + ) + + +def load_google_keep(path: Path) -> list[Note]: + """A google takeout keep export: `.json` per note, or the zip.""" + notes: list[Note] = [] + if path.is_file() and zipfile.is_zipfile(path): + with zipfile.ZipFile(path) as zf: + total = 0 + for info in sorted(zf.infolist(), key=lambda i: i.filename): + if info.is_dir() or not info.filename.lower().endswith(".json"): + continue + if "/Keep/" not in f"/{info.filename}" and "keep" not in info.filename.lower(): + continue + total += info.file_size + if total > _MAX_VAULT_BYTES: + raise NoteImportError( + f"{path} exceeds the " + f"{_MAX_VAULT_BYTES // (1024 * 1024)} MiB import ceiling" + ) + try: + payload = json.loads(zf.read(info).decode("utf-8", errors="replace")) + except (json.JSONDecodeError, OSError): + continue + if isinstance(payload, dict): + note = _keep_note(payload, Path(info.filename).stem) + if note is not None: + notes.append(note) + elif path.is_dir(): + for json_path in _walk_notes(path, frozenset({".json"})): + try: + payload = json.loads(_read_text(json_path)) + except json.JSONDecodeError: + continue + if isinstance(payload, dict): + note = _keep_note(payload, json_path.stem) + if note is not None: + notes.append(note) + else: + raise NoteImportError(f"no such keep export: {path}") + if not notes: + raise NoteImportError( + f"no google keep notes found in {path} — a takeout keep export is a " + f"folder of one .json per note (or the takeout .zip holding it)" + ) + notes.sort(key=lambda n: n.key) + return notes + + +# --- vault -> proposals ---------------------------------------------------- + + +_LOADERS = { + "obsidian": lambda p: load_markdown_vault(p, kind="obsidian"), + "md": lambda p: load_markdown_vault(p, kind="md"), + "joplin": load_joplin, + "notes": load_apple_notes, + "keep": load_google_keep, +} + + +def load_vault(kind: str, path: Path) -> list[Note]: + """Parse a vault of ``kind`` into normalised notes. No KB writes.""" + loader = _LOADERS.get(kind) + if loader is None: + raise NoteImportError( + f"unknown vault kind {kind!r} — one of {', '.join(KINDS)}" + ) + if not path.exists(): + raise NoteImportError(f"no such path: {path}") + return loader(path) + + +def session_key(kind: str, note_key: str) -> str: + """The session id one note dedups on, across every re-import. + + Derived from the origin's own identifier, hashed only to bound the length — + a deep vault path or a keep title can be arbitrarily long, and the session + id is an index key, not a display string. The readable identifier survives + on the source ``locator`` and in the page body. + """ + digest = hashlib.sha256(f"{kind}:{note_key}".encode()).hexdigest()[:16] + return f"note-{kind}-{digest}" + + +def _page_slug(kind: str, note_key: str) -> str: + slug = _SLUG_RE.sub("-", note_key.lower()).strip("-") or "note" + return f"{kind}-{slug}"[:80] + + +def build_page_body( + note: Note, *, kind: str, generated_at: str | None = None +) -> str: + """The review surface for one note: provenance, then the note itself.""" + stamp = generated_at or datetime.now(UTC).isoformat() + lines = [ + f"# {note.title}", + "", + f"- imported: {stamp}", + f"- from: {kind}", + f"- note: {note.key}", + ] + if note.created_at: + lines.append(f"- created: {note.created_at}") + if note.updated_at: + lines.append(f"- updated: {note.updated_at}") + if note.tags: + lines.append(f"- tags: {', '.join(note.tags)}") + if note.links: + lines.append(f"- links: {', '.join(note.links[:20])}") + extra = { + k: v for k, v in note.frontmatter.items() if k not in {"title", "tags", "tag"} + } + if extra: + lines.extend(["", "## frontmatter", ""]) + lines.extend(f"- {k}: {v}" for k, v in sorted(extra.items())[:30]) + lines.extend(["", "## note", "", _clip(note.body) or "(empty note)"]) + return "\n".join(lines).rstrip() + "\n" + + +def _comparable_body(body: str) -> str: + """The page body minus its import timestamp, so re-importing an unchanged + note compares equal across runs.""" + return "\n".join( + line for line in body.splitlines() if not line.startswith("- imported:") + ) + + +def _find_existing_page(store: KBStore, session_id: str) -> Proposal | None: + """The page proposal (any status) already filed for this note. + + Filtered to PAGE kind for the same reason `chatgpt_import` filters: an + approved import can spawn follow-on proposals under the same session id + (the wikilink relations, for one), and those must never be mistaken for the + page when a re-import looks for its dedup target. + """ + for proposal in store.list_proposals(None): + if proposal.kind == ProposalKind.PAGE and proposal.session_id == session_id: + return proposal + return None + + +def _resolve_link(target: str, by_key: dict[str, str], by_title: dict[str, str]) -> str | None: + """A wikilink target -> the imported note's key, or None if unresolvable. + + Obsidian resolves `[[Note]]` by basename anywhere in the vault, and by full + path when one is given; joplin links by id. Titles are matched + case-insensitively and only when unambiguous — a link that could mean two + notes is dropped rather than guessed at. + """ + if target in by_key: + return by_key[target] + stripped = target.removesuffix(".md") + if stripped in by_key: + return by_key[stripped] + return by_title.get(stripped.lower()) + + +def import_vault( + store: KBStore, + kind: str, + path: Path, + *, + actor: str | None = None, + limit: int | None = None, + max_claims: int = 0, + dry_run: bool = False, + generated_at: str | None = None, +) -> dict[str, Any]: + """Import one note vault into PENDING proposals. Never calls ``approve()``. + + One page proposal per note, deduped on a stable per-note session id: a + decided proposal blocks re-import, a still-PENDING one refreshes in place + when the note changed, and an unchanged note is a no-op. ``limit`` bounds + how many notes are considered, in sorted order, so a large vault imports in + resumable slices. + + ``max_claims`` > 0 additionally files that many receipt-backed claims per + note through the density-selection knob. It defaults to 0 — pages only — + because a ten-thousand-note vault must not become ten thousand pending + claims on someone's first command. + + With ``dry_run`` nothing is written; the report shows what a real run would + file. + """ + notes = load_vault(kind, path) + if limit is not None and limit >= 0: + notes = notes[:limit] + resolved_actor = actor or os.environ.get("VOUCH_AGENT") or NOTE_IMPORT_ACTOR + + rows: list[dict[str, Any]] = [] + counts = {"imported": 0, "updated": 0, "skipped": 0} + claims_filed = 0 + # note key -> registered source id, for the wikilink pass. Only notes that + # actually got a source land here, so a link into a `--limit`-truncated + # slice resolves to nothing and is dropped rather than filed dangling. + source_by_key: dict[str, str] = {} + by_key = {note.key: note.key for note in notes} + titles: dict[str, list[str]] = {} + for note in notes: + titles.setdefault(note.title.lower(), []).append(note.key) + by_title = {title: keys[0] for title, keys in titles.items() if len(keys) == 1} + + for note in notes: + sid = session_key(kind, note.key) + row: dict[str, Any] = {"note": note.key, "title": note.title, "session_id": sid} + existing = _find_existing_page(store, sid) + if existing is not None and existing.status != ProposalStatus.PENDING: + row.update(action="skipped", reason="already-imported", proposal_id=existing.id) + counts["skipped"] += 1 + rows.append(row) + continue + body = build_page_body(note, kind=kind, generated_at=generated_at) + if existing is not None and _comparable_body(body) == _comparable_body( + str(existing.payload.get("body", "")) + ): + row.update(action="skipped", reason="unchanged", proposal_id=existing.id) + counts["skipped"] += 1 + rows.append(row) + continue + if dry_run: + action = "updated" if existing is not None else "imported" + row.update(action=action, dry_run=True) + counts[action] += 1 + rows.append(row) + continue + + source = store.put_source( + note.raw, + title=note.title, + locator=note.locator, + media_type=note.media_type, + tags=[f"{kind}-import", "note-import"], + metadata={ + "note_key": note.key, + "import_kind": kind, + **({"frontmatter": note.frontmatter} if note.frontmatter else {}), + }, + scope=default_scope(store), + ) + source_by_key[note.key] = source.id + + if existing is not None: + refreshed = existing.model_copy(deep=True) + refreshed.payload["title"] = note.title + refreshed.payload["body"] = body + refreshed.payload["sources"] = [source.id] + store.update_proposal(refreshed) + audit.log_event( + store.kb_dir, + event="proposal.page.update", + actor=resolved_actor, + object_ids=[existing.id], + data={"reason": f"{kind} re-import", "note": note.key}, + ) + row.update(action="updated", proposal_id=existing.id) + counts["updated"] += 1 + else: + proposal = propose_page( + store, + title=note.title, + body=body, + page_type=PAGE_TYPE, + source_ids=[source.id], + proposed_by=resolved_actor, + tags=[f"{kind}-import", *note.tags], + session_id=sid, + slug_hint=_page_slug(kind, note.key), + rationale=f"imported {kind} note", + ) + row.update(action="imported", proposal_id=proposal.id) + counts["imported"] += 1 + + if max_claims > 0: + filed = extract_receipt_claims( + store, source.id, proposed_by=resolved_actor, max_claims=max_claims, + ) + row["claims"] = len(filed) + claims_filed += len(filed) + rows.append(row) + + relations = 0 + if not dry_run: + relations = _propose_links( + store, notes, source_by_key, by_key, by_title, actor=resolved_actor, kind=kind + ) + + return { + "kind": kind, + "notes": len(notes), + "imported": counts["imported"], + "updated": counts["updated"], + "skipped": counts["skipped"], + "claims": claims_filed, + "relations": relations, + "dry_run": dry_run, + "rows": rows, + } + + +def _propose_links( + store: KBStore, + notes: list[Note], + source_by_key: dict[str, str], + by_key: dict[str, str], + by_title: dict[str, str], + *, + actor: str, + kind: str, +) -> int: + """File a `references` relation per resolvable wikilink. Returns the count. + + Runs after every source is registered, so a link is judged against the + whole imported set rather than against whatever happened to come first in + the walk. Unresolvable targets are dropped: a vault is full of links to + notes that were never written, and a dangling edge is not knowledge. + """ + filed = 0 + seen: set[tuple[str, str]] = set() + for note in notes: + src_id = source_by_key.get(note.key) + if src_id is None: + continue + for target in note.links: + target_key = _resolve_link(target, by_key, by_title) + if target_key is None or target_key == note.key: + continue + target_id = source_by_key.get(target_key) + if target_id is None or (src_id, target_id) in seen: + continue + seen.add((src_id, target_id)) + try: + propose_relation( + store, + src=src_id, + relation="references", + target=target_id, + proposed_by=actor, + rationale=f"{kind} link: {note.key} -> {target_key}", + session_id=session_key(kind, note.key), + ) + except ProposalError: + # A duplicate or otherwise-rejected edge is not worth failing + # a ten-thousand-note import over. + continue + filed += 1 + return filed diff --git a/tests/test_conversation_import.py b/tests/test_conversation_import.py new file mode 100644 index 00000000..a635afbe --- /dev/null +++ b/tests/test_conversation_import.py @@ -0,0 +1,706 @@ +"""Conversation and memory export importers (#431). + +The load-bearing invariant is the one the issue names: an importer has no path +to `approve`. Everything else here is the two things that keep an unattended +import from becoming a reviewer's problem — the per-run cap and dedup — plus +the tolerance every reader needs, because an export is someone else's file. +""" + +from __future__ import annotations + +import json +import zipfile +from pathlib import Path + +import pytest +from click.testing import CliRunner + +from vouch import conversation_import as ci +from vouch.cli import cli +from vouch.conversation_import import ( + ConversationImportError, + import_conversations, + import_memories, + parse_chat_json, + parse_memory_export, +) +from vouch.models import ProposalKind, ProposalStatus +from vouch.storage import KBStore + +ANSWER_A = ( + "Deploys run every second Tuesday from the release branch, never from main." +) +ANSWER_B = ( + "The staging environment refreshes nightly at 02:00 UTC from a sanitised dump." +) + + +@pytest.fixture +def store(tmp_path: Path) -> KBStore: + return KBStore.init(tmp_path / "kb") + + +def _write(path: Path, payload: object) -> Path: + path.write_text(json.dumps(payload), encoding="utf-8") + return path + + +@pytest.fixture +def claude_export(tmp_path: Path) -> Path: + """A claude.ai-shaped export: `chat_messages` with `sender` and blocks.""" + return _write(tmp_path / "claude.json", [ + { + "uuid": "conv-1", + "name": "Deploy cadence", + "created_at": "2026-07-01T09:00:00Z", + "chat_messages": [ + {"sender": "human", "text": "when do we deploy?"}, + {"sender": "assistant", "content": [{"type": "text", "text": ANSWER_A}]}, + {"sender": "human", "text": "and staging?"}, + {"sender": "assistant", "text": ANSWER_B}, + ], + }, + { + "uuid": "conv-2", + "name": "Empty one", + "chat_messages": [{"sender": "human", "text": "hello?"}], + }, + ]) + + +# --- the invariant --------------------------------------------------------- + + +def test_everything_lands_pending_and_nothing_is_approved( + store: KBStore, claude_export: Path +) -> None: + report = import_conversations(store, claude_export, max_claims=2) + assert report["imported"] == 1 + assert all( + p.status is ProposalStatus.PENDING for p in store.list_proposals(None) + ) + assert store.list_pages() == [] + assert store.list_claims() == [] + + +def test_the_module_has_no_path_to_approve() -> None: + source = Path(ci.__file__).read_text(encoding="utf-8") + assert not hasattr(ci, "approve") + assert "approve" not in { + line.split()[-1] for line in source.splitlines() + if line.strip().startswith(("import ", "from ")) + } + + +# --- chat-json ------------------------------------------------------------- + + +def test_claude_shaped_export_pairs_turns(claude_export: Path) -> None: + conversations = parse_chat_json(claude_export) + assert [c.conversation_id for c in conversations] == ["conv-1", "conv-2"] + first = conversations[0] + assert first.title == "Deploy cadence" + assert [e.user for e in first.exchanges] == ["when do we deploy?", "and staging?"] + assert first.exchanges[0].assistant == ANSWER_A + assert conversations[1].exchanges == [] # a question with no answer + + +def test_a_conversation_with_no_exchanges_is_skipped( + store: KBStore, claude_export: Path +) -> None: + report = import_conversations(store, claude_export) + skipped = next(r for r in report["rows"] if r["conversation"] == "conv-2") + assert skipped["reason"] == "no exchanges" + + +def test_generic_role_content_shape(store: KBStore, tmp_path: Path) -> None: + export = _write(tmp_path / "gemini.json", {"conversations": [{ + "id": "g-1", "title": "Retries", + "messages": [ + {"role": "user", "content": "how many retries?"}, + {"role": "model", "content": "Five, with exponential backoff."}, + ], + }]}) + conversations = parse_chat_json(export) + assert conversations[0].exchanges[0].assistant == "Five, with exponential backoff." + + +def test_a_bare_message_list_becomes_one_conversation(tmp_path: Path) -> None: + export = _write(tmp_path / "session.json", [ + {"role": "user", "content": "what is the retry limit?"}, + {"role": "assistant", "content": "Five."}, + ]) + conversations = parse_chat_json(export) + assert len(conversations) == 1 + assert conversations[0].conversation_id == "session" + + +def test_an_openai_mapping_tree_is_delegated(tmp_path: Path) -> None: + """The branching export shape stays `chatgpt_import`'s job — this reader + recognises it and hands it over rather than parsing it a second way.""" + export = _write(tmp_path / "conversations.json", [{ + "conversation_id": "o-1", "title": "About deploys", + "mapping": { + "root": {"id": "root", "message": None, "parent": None, "children": ["u1"]}, + "u1": {"id": "u1", "parent": "root", "children": ["a1"], "message": { + "author": {"role": "user"}, + "content": {"content_type": "text", "parts": ["when?"]}, + }}, + "a1": {"id": "a1", "parent": "u1", "children": [], "message": { + "author": {"role": "assistant"}, + "content": {"content_type": "text", "parts": [ANSWER_A]}, + }}, + }, + }]) + conversations = parse_chat_json(export) + assert conversations[0].exchanges[0].assistant == ANSWER_A + + +def test_jsonl_and_zip_exports_are_read(tmp_path: Path) -> None: + jsonl = tmp_path / "chats.jsonl" + jsonl.write_text("\n".join([ + json.dumps({"id": "j-1", "messages": [ + {"role": "user", "content": "q"}, + {"role": "assistant", "content": ANSWER_A}, + ]}), + "{ not json — skipped", + ]), encoding="utf-8") + assert parse_chat_json(jsonl)[0].conversation_id == "j-1" + + archive = tmp_path / "export.zip" + with zipfile.ZipFile(archive, "w") as zf: + zf.writestr("export/chats.json", jsonl.read_text(encoding="utf-8").splitlines()[0]) + assert parse_chat_json(archive)[0].conversation_id == "j-1" + + +def test_an_unrecognisable_file_is_an_actionable_error(tmp_path: Path) -> None: + junk = tmp_path / "junk.json" + junk.write_text("this is not json at all", encoding="utf-8") + with pytest.raises(ConversationImportError, match="neither JSON nor JSONL"): + parse_chat_json(junk) + + scalar = _write(tmp_path / "scalar.json", 7) + with pytest.raises(ConversationImportError, match="does not look like a chat"): + parse_chat_json(scalar) + + shapeless = _write(tmp_path / "shapeless.json", [{"nothing": "useful"}]) + with pytest.raises(ConversationImportError, match="no conversations found"): + parse_chat_json(shapeless) + + +def test_the_page_cites_the_whole_conversation( + store: KBStore, claude_export: Path +) -> None: + import_conversations(store, claude_export) + page = next( + p for p in store.list_proposals(None) if p.kind == ProposalKind.PAGE + ) + (source_id,) = page.payload["sources"] + content = store.read_source_content(source_id).decode("utf-8") + assert ANSWER_A in content + assert ANSWER_B in content + assert store.get_source(source_id).locator == "chat:conv-1" + assert ANSWER_A in page.payload["body"] + + +def test_reimport_is_idempotent_and_refreshes_a_grown_conversation( + store: KBStore, tmp_path: Path +) -> None: + path = tmp_path / "claude.json" + _write(path, [{"uuid": "c-1", "name": "T", "chat_messages": [ + {"sender": "human", "text": "q1"}, {"sender": "assistant", "text": ANSWER_A}, + ]}]) + first = import_conversations(store, path, generated_at="2026-07-31T00:00:00Z") + assert first["imported"] == 1 + + unchanged = import_conversations(store, path, generated_at="2026-08-01T00:00:00Z") + assert unchanged["skipped"] == 1 + assert unchanged["rows"][0]["reason"] == "unchanged" + + _write(path, [{"uuid": "c-1", "name": "T", "chat_messages": [ + {"sender": "human", "text": "q1"}, {"sender": "assistant", "text": ANSWER_A}, + {"sender": "human", "text": "q2"}, {"sender": "assistant", "text": ANSWER_B}, + ]}]) + grown = import_conversations(store, path) + assert grown["updated"] == 1 + pages = [p for p in store.list_proposals(None) if p.kind == ProposalKind.PAGE] + assert len(pages) == 1 # refreshed in place, not re-filed + assert ANSWER_B in pages[0].payload["body"] + + +def test_a_decided_proposal_blocks_reimport( + store: KBStore, claude_export: Path +) -> None: + from vouch.proposals import reject + + import_conversations(store, claude_export) + page = next(p for p in store.list_proposals(None) if p.kind == ProposalKind.PAGE) + reject(store, page.id, rejected_by="reviewer-example", reason="not wanted") + report = import_conversations(store, claude_export) + row = next(r for r in report["rows"] if r["conversation"] == "conv-1") + assert row["reason"] == "already-imported" + + +def test_claims_are_receipt_backed_and_off_by_default( + store: KBStore, claude_export: Path +) -> None: + from vouch import receipts + + off = import_conversations(store, claude_export) + assert off["claims"] == 0 + assert not [p for p in store.list_proposals(None) if p.kind == ProposalKind.CLAIM] + + fresh = KBStore.init(claude_export.parent / "kb2") + on = import_conversations(fresh, claude_export, max_claims=2) + assert on["claims"] > 0 + claims = [p for p in fresh.list_proposals(None) if p.kind == ProposalKind.CLAIM] + assert claims + for proposal in claims: + evidence = fresh.get_evidence(proposal.payload["evidence"][0]) + result = receipts.verify_receipt( + evidence, fresh.read_source_content(evidence.source_id) + ) + assert result.status is receipts.ReceiptStatus.VERIFIED + + +# --- memory-export --------------------------------------------------------- + + +MEMORIES = [ + "The user prefers tabs over spaces in every language.", + "Deploys are cut from the release branch on alternate Tuesdays.", +] + + +def test_memory_array_of_records(store: KBStore, tmp_path: Path) -> None: + export = _write(tmp_path / "memories.json", [ + {"id": "m1", "memory": MEMORIES[0], "tags": ["prefs"]}, + {"id": "m2", "text": MEMORIES[1], "created_at": 1767225600}, + ]) + report = import_memories(store, export) + assert report["imported"] == 2 + claims = [p for p in store.list_proposals(None) if p.kind == ProposalKind.CLAIM] + assert {p.payload["text"] for p in claims} == set(MEMORIES) + assert all(p.status is ProposalStatus.PENDING for p in claims) + + +def test_memory_claims_quote_their_own_source(store: KBStore, tmp_path: Path) -> None: + from vouch import receipts + + export = _write(tmp_path / "memories.json", [MEMORIES[0]]) + import_memories(store, export) + proposal = next( + p for p in store.list_proposals(None) if p.kind == ProposalKind.CLAIM + ) + evidence = store.get_evidence(proposal.payload["evidence"][0]) + result = receipts.verify_receipt( + evidence, store.read_source_content(evidence.source_id) + ) + assert result.status is receipts.ReceiptStatus.VERIFIED + + +def test_memory_object_of_records_and_wrapper_key( + store: KBStore, tmp_path: Path +) -> None: + wrapped = _write(tmp_path / "wrapped.json", {"memories": [MEMORIES[0]]}) + assert [m.text for m in parse_memory_export(wrapped)] == [MEMORIES[0]] + + keyed = _write(tmp_path / "keyed.json", { + "m1": MEMORIES[0], + "m2": {"fact": MEMORIES[1]}, + }) + assert {m.text for m in parse_memory_export(keyed)} == set(MEMORIES) + + +def test_memory_plain_text_lines(store: KBStore, tmp_path: Path) -> None: + export = tmp_path / "memories.txt" + export.write_text("\n".join([*MEMORIES, "", "short"]), encoding="utf-8") + parsed = parse_memory_export(export) + assert [m.text for m in parsed] == MEMORIES # "short" is below the floor + + +def test_memory_export_drops_repeats_within_one_dump(tmp_path: Path) -> None: + export = _write(tmp_path / "dupes.json", [MEMORIES[0], MEMORIES[0]]) + assert len(parse_memory_export(export)) == 1 + + +def test_an_empty_memory_export_is_an_actionable_error(tmp_path: Path) -> None: + empty = _write(tmp_path / "empty.json", [{"unrelated": 1}, "tiny"]) + with pytest.raises(ConversationImportError, match="no memories found"): + parse_memory_export(empty) + scalar = _write(tmp_path / "scalar.json", 7) + with pytest.raises(ConversationImportError, match="expected a list of memories"): + parse_memory_export(scalar) + + +# --- the two guards -------------------------------------------------------- + + +def test_dedup_drops_a_memory_an_approved_claim_already_covers( + store: KBStore, tmp_path: Path +) -> None: + from vouch.proposals import approve, propose_claim + + src = store.put_source(MEMORIES[0].encode("utf-8")) + pr = propose_claim( + store, text=MEMORIES[0], evidence=[src.id], proposed_by="agent-a" + ) + approve(store, pr.proposal.id, approved_by="human-b") + + export = _write(tmp_path / "memories.json", MEMORIES) + report = import_memories(store, export) + assert report["imported"] == 1 + dropped = next(r for r in report["rows"] if r["action"] == "skipped") + assert dropped["reason"] == "already-known" + + +def test_dedup_drops_a_memory_a_pending_proposal_already_covers( + store: KBStore, tmp_path: Path +) -> None: + export = _write(tmp_path / "memories.json", [MEMORIES[0]]) + assert import_memories(store, export)["imported"] == 1 + assert import_memories(store, export)["skipped"] == 1 + + +def test_dedup_can_be_turned_off(store: KBStore, tmp_path: Path) -> None: + export = _write(tmp_path / "memories.json", [MEMORIES[0]]) + import_memories(store, export) + assert import_memories(store, export, dedup=False)["imported"] == 1 + + +def test_dedup_folds_in_embedding_hits_when_the_extra_is_present( + store: KBStore, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The embedding half of #147 is additive on top of the lexical guard, and + is stubbed here so the branch runs with or without the extra installed.""" + import sys + import types + + module = types.ModuleType("vouch.embeddings.similarity") + module.find_similar_on_propose = lambda store, text: [ # type: ignore[attr-defined] + {"artifact_id": None}, # ignored: not a string + {"artifact_id": "semantic-twin"}, + ] + monkeypatch.setitem(sys.modules, "vouch.embeddings.similarity", module) + export = _write(tmp_path / "memories.json", [MEMORIES[0]]) + report = import_memories(store, export) + assert report["imported"] == 0 + assert report["rows"][0]["duplicate_of"] == "semantic-twin" + + +def test_dedup_still_works_on_a_base_install( + store: KBStore, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Without the `[embeddings]` extra the import fails and the lexical guard + is all there is — which is why it runs first.""" + import sys + + monkeypatch.setitem(sys.modules, "vouch.embeddings.similarity", None) + export = _write(tmp_path / "memories.json", [MEMORIES[0]]) + assert import_memories(store, export)["imported"] == 1 + assert import_memories(store, export)["skipped"] == 1 + + +def test_max_proposals_caps_a_run_and_reports_it( + store: KBStore, tmp_path: Path +) -> None: + export = _write(tmp_path / "memories.json", MEMORIES) + report = import_memories(store, export, max_proposals=1) + assert report["imported"] == 1 + assert report["capped"] is True + assert report["skipped"] == 1 + assert report["rows"][1]["reason"] == "max-proposals" + assert len(store.list_proposals(ProposalStatus.PENDING)) == 1 + + # rerunning continues where it left off — the first is deduped, the + # second lands. that is what makes a large history importable at all. + second = import_memories(store, export) + assert second["imported"] == 1 + + +def test_max_proposals_caps_conversations_too( + store: KBStore, claude_export: Path +) -> None: + report = import_conversations(store, claude_export, max_proposals=0) + assert report["imported"] == 0 + assert report["capped"] is True + + +def test_an_uncapped_run_reports_no_cap(store: KBStore, tmp_path: Path) -> None: + export = _write(tmp_path / "memories.json", MEMORIES) + report = import_memories(store, export) + assert report["capped"] is False + assert report["max_proposals"] is None + assert report["proposals"] == 2 + + +def test_dry_run_reports_without_enqueuing( + store: KBStore, tmp_path: Path, claude_export: Path +) -> None: + export = _write(tmp_path / "memories.json", MEMORIES) + memories = import_memories(store, export, dry_run=True) + conversations = import_conversations(store, claude_export, dry_run=True) + assert memories["imported"] == 2 + assert conversations["imported"] == 1 + assert memories["dry_run"] is True + assert store.list_proposals(None) == [] + assert not list(store.list_sources()) + + +def test_limit_slices_the_export(store: KBStore, tmp_path: Path) -> None: + export = _write(tmp_path / "memories.json", MEMORIES) + assert import_memories(store, export, limit=1)["imported"] == 1 + + +def test_an_unquotable_memory_is_dropped_rather_than_filed( + store: KBStore, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A claim whose receipt cannot be located is not knowledge — the importer + drops it instead of filing a citation that would fail verification.""" + monkeypatch.setattr(ci, "propose_quoted_claim", lambda *a, **k: None) + export = _write(tmp_path / "memories.json", [MEMORIES[0]]) + report = import_memories(store, export) + assert report["imported"] == 0 + assert report["rows"][0]["reason"] == "unquotable" + assert report["proposals"] == 0 + + +def test_overlap_is_zero_without_shared_signal(store: KBStore) -> None: + # all stopwords / too-short tokens on one side -> nothing to score against + assert ci.overlap("we do it", MEMORIES[0]) == 0.0 + assert ci.overlap(MEMORIES[0], "") == 0.0 + assert ci.already_known(store, " ") is None + + +# --- cli ------------------------------------------------------------------- + + +def _run(store: KBStore, args: list[str]): + return CliRunner().invoke(cli, args, env={"VOUCH_KB_PATH": str(store.kb_dir)}) + + +def test_cli_chat_json(store: KBStore, claude_export: Path) -> None: + result = _run(store, ["import", "chat-json", str(claude_export), "--json"]) + assert result.exit_code == 0, result.output + report = json.loads(result.output) + assert report["format"] == "chat-json" + assert report["imported"] == 1 + + +def test_cli_chat_json_human_output(store: KBStore, claude_export: Path) -> None: + result = _run( + store, ["import", "chat-json", str(claude_export), "--max-claims", "2"] + ) + assert result.exit_code == 0, result.output + assert "imported 1 new" in result.output + assert "receipt-backed claim(s) proposed" in result.output + assert "run `vouch review` to decide." in result.output + + +def test_cli_memory_export_with_a_cap(store: KBStore, tmp_path: Path) -> None: + export = _write(tmp_path / "memories.json", MEMORIES) + result = _run( + store, ["import", "memory-export", str(export), "--max-proposals", "1"] + ) + assert result.exit_code == 0, result.output + assert "stopped at --max-proposals 1" in result.output + + +def test_cli_dry_run_and_no_dedup(store: KBStore, tmp_path: Path) -> None: + export = _write(tmp_path / "memories.json", MEMORIES) + dry = _run(store, ["import", "memory-export", str(export), "--dry-run"]) + assert "would import 2 new" in dry.output + assert store.list_proposals(None) == [] + + _run(store, ["import", "memory-export", str(export)]) + again = _run( + store, ["import", "memory-export", str(export), "--no-dedup", "--json"] + ) + assert json.loads(again.output)["imported"] == 2 + + +def test_cli_markdown_vault_alias(store: KBStore, tmp_path: Path) -> None: + folder = tmp_path / "vault" + folder.mkdir() + (folder / "note.md").write_text("# Note\n\nbody\n", encoding="utf-8") + result = _run(store, ["import", "markdown-vault", str(folder), "--json"]) + assert result.exit_code == 0, result.output + assert json.loads(result.output)["imported"] == 1 + + +def test_cli_reports_a_bad_export_cleanly(store: KBStore, tmp_path: Path) -> None: + junk = tmp_path / "junk.json" + junk.write_text("not json", encoding="utf-8") + result = _run(store, ["import", "chat-json", str(junk)]) + assert result.exit_code != 0 + assert "Traceback" not in result.output + assert "does not look like" in result.output + + +def test_cli_import_group_lists_every_format() -> None: + result = CliRunner().invoke(cli, ["import", "--help"]) + assert result.exit_code == 0 + for name in ("chat-json", "memory-export", "markdown-vault"): + assert name in result.output + + +# --- the tolerant paths ---------------------------------------------------- +# +# An export is someone else's file, written by a tool whose schema drifts. +# Every branch below is a "skip it and carry on" or an actionable refusal. + + +def test_an_unreadable_export_is_reported_not_raised( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + export = _write(tmp_path / "chats.json", []) + + def boom(_path: object) -> bool: + raise OSError("permission denied") + + monkeypatch.setattr(ci.zipfile, "is_zipfile", boom) + with pytest.raises(ConversationImportError, match="cannot read export file"): + parse_chat_json(export) + + monkeypatch.undo() + + def boom_bytes(self: Path) -> bytes: + raise OSError("disk gone") + + monkeypatch.setattr(Path, "read_bytes", boom_bytes) + with pytest.raises(ConversationImportError, match="cannot read export file"): + parse_chat_json(export) + + +def test_a_zip_with_no_json_entry_is_actionable(tmp_path: Path) -> None: + archive = tmp_path / "export.zip" + with zipfile.ZipFile(archive, "w") as zf: + zf.writestr("export/readme.txt", "nothing useful") + with pytest.raises(ConversationImportError, match=r"holds no \.json export"): + parse_chat_json(archive) + + +def test_exports_over_the_byte_ceiling_are_refused( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(ci, "_MAX_EXPORT_BYTES", 4) + bare = _write(tmp_path / "chats.json", [{"id": "well over four bytes"}]) + with pytest.raises(ConversationImportError, match="too large to import"): + parse_chat_json(bare) + + archive = tmp_path / "export.zip" + with zipfile.ZipFile(archive, "w") as zf: + zf.writestr("export/chats.json", "[] well over four bytes") + with pytest.raises(ConversationImportError, match="too large to import"): + parse_chat_json(archive) + + +def test_clip_truncates_long_text() -> None: + assert ci._clip("x" * 50, limit=10).endswith("…") + + +def test_timestamps_accept_seconds_millis_and_iso() -> None: + assert ci._iso("2026-07-31T00:00:00Z") == "2026-07-31T00:00:00Z" + assert ci._iso(1767225600) == ci._iso(1767225600000) # millis normalised + assert ci._iso(None) is None + assert ci._iso(True) is None # a bool is not a timestamp + assert ci._iso(1e30) is None # out of range degrades, never raises + + +def test_message_content_shapes_are_all_read() -> None: + assert ci._block_text({"text": "nested"}) == "nested" + assert ci._block_text({"parts": ["a", "b"]}) == "" # a dict of parts, no text + assert ci._block_text(7) == "" + assert ci._turns("not-a-list") == [] + assert ci._turns([7, {"role": "narrator", "content": "x"}]) == [] + # some exports nest the author as an object + assert ci._turns([{"author": {"role": "user"}, "content": "hi there"}]) == [ + ("user", "hi there") + ] + + +def test_entries_that_are_not_objects_are_skipped(tmp_path: Path) -> None: + export = _write(tmp_path / "chats.json", [7, {"id": "c", "messages": [ + {"role": "user", "content": "q"}, {"role": "assistant", "content": ANSWER_A}, + ]}]) + assert [c.conversation_id for c in parse_chat_json(export)] == ["c"] + + +def test_memory_entries_that_are_not_records_are_skipped(tmp_path: Path) -> None: + export = _write(tmp_path / "memories.json", [7, None, MEMORIES[0]]) + assert [m.text for m in parse_memory_export(export)] == [MEMORIES[0]] + + +def test_page_falls_back_to_the_conversation_id_for_a_title( + store: KBStore, tmp_path: Path +) -> None: + export = _write(tmp_path / "chats.json", [{"id": "c-42", "messages": [ + {"role": "user", "content": "q"}, {"role": "assistant", "content": ANSWER_A}, + ]}]) + import_conversations(store, export) + page = next(p for p in store.list_proposals(None) if p.kind == ProposalKind.PAGE) + assert page.payload["title"] == "conversation: c-42" + + +def test_a_long_conversation_says_how_many_exchanges_it_elided( + store: KBStore, tmp_path: Path +) -> None: + messages = [] + for i in range(ci._MAX_EXCHANGES_PER_PAGE + 5): + messages.append({"role": "user", "content": f"question {i}"}) + messages.append({"role": "assistant", "content": f"answer number {i}"}) + export = _write(tmp_path / "long.json", [{ + "id": "c-long", "title": "Long one", + "updated_at": "2026-07-31T00:00:00Z", "messages": messages, + }]) + import_conversations(store, export) + body = next( + p for p in store.list_proposals(None) if p.kind == ProposalKind.PAGE + ).payload["body"] + assert "- last-active: 2026-07-31T00:00:00Z" in body + assert "more exchange(s) — full conversation in the cited source" in body + + +def test_limit_slices_a_chat_export(store: KBStore, claude_export: Path) -> None: + report = import_conversations(store, claude_export, limit=1) + assert report["conversations"] == 1 + + +def test_claim_extraction_stops_at_each_of_its_three_bounds( + store: KBStore, tmp_path: Path +) -> None: + """max_claims, the too-short-answer floor, and the run-wide budget each + stop the per-conversation claim loop on their own.""" + export = _write(tmp_path / "chats.json", [{ + "id": "c-1", "title": "T", + "messages": [ + {"role": "user", "content": "q1"}, + {"role": "assistant", "content": "ok"}, # below the floor + {"role": "user", "content": "q2"}, + {"role": "assistant", "content": ANSWER_A}, + {"role": "user", "content": "q3"}, + {"role": "assistant", "content": ANSWER_B}, + ], + }]) + capped = import_conversations(store, export, max_claims=1) + assert capped["claims"] == 1 + + # dedup: a *different* conversation repeating an answer already filed adds + # nothing, even though its page is new + repeat = _write(tmp_path / "repeat.json", [{ + "id": "c-2", "title": "Asked again", + "messages": [ + {"role": "user", "content": "remind me?"}, + {"role": "assistant", "content": ANSWER_A}, + ], + }]) + again = import_conversations(store, repeat, max_claims=2) + assert again["imported"] == 1 + assert again["claims"] == 0 + + fresh = KBStore.init(tmp_path / "kb-budget") + # one proposal of budget goes to the page, leaving none for the claims + budgeted = import_conversations(fresh, export, max_claims=2, max_proposals=1) + assert budgeted["imported"] == 1 + assert budgeted["claims"] == 0 + assert budgeted["capped"] is True diff --git a/tests/test_note_import.py b/tests/test_note_import.py new file mode 100644 index 00000000..f844396a --- /dev/null +++ b/tests/test_note_import.py @@ -0,0 +1,758 @@ +"""Note-vault importers: obsidian, joplin, apple notes, keep, markdown (#612).""" + +from __future__ import annotations + +import json +import tarfile +import zipfile +from pathlib import Path + +import pytest +from click.testing import CliRunner + +from vouch import note_import +from vouch.cli import cli +from vouch.models import ProposalKind, ProposalStatus +from vouch.note_import import NoteImportError, import_vault, load_vault, session_key +from vouch.storage import KBStore + + +@pytest.fixture +def store(tmp_path: Path) -> KBStore: + return KBStore.init(tmp_path / "kb") + + +@pytest.fixture +def vault(tmp_path: Path) -> Path: + """A small obsidian vault: frontmatter, a wikilink, a subfolder, machinery.""" + root = tmp_path / "vault" + (root / "notes").mkdir(parents=True) + (root / ".obsidian").mkdir() + (root / ".obsidian" / "app.json").write_text("{}", encoding="utf-8") + (root / "Postgres.md").write_text( + "---\ntitle: Postgres tuning\ntags: [db, ops]\nowner: alice-example\n---\n" + "The connection pool must be sized to the worker count, never higher.\n" + "See [[Deploys]] for how this rolls out.\n", + encoding="utf-8", + ) + (root / "notes" / "Deploys.md").write_text( + "# Deploys\n\nDeploys run every second Tuesday from the release branch.\n", + encoding="utf-8", + ) + (root / "notes" / "Orphan.md").write_text( + "A note linking to [[Nothing At All]] which was never written.\n", + encoding="utf-8", + ) + return root + + +# --- parsing --------------------------------------------------------------- + + +def test_split_frontmatter_reads_a_mapping() -> None: + fm, body = note_import.split_frontmatter("---\na: 1\nb: two\n---\nbody here\n") + assert fm == {"a": 1, "b": "two"} + assert body.strip() == "body here" + + +def test_split_frontmatter_leaves_malformed_blocks_alone() -> None: + # A vault is other people's files: one bad note must not fail the import. + text = "---\n: : :\n - broken\n---\nbody\n" + fm, body = note_import.split_frontmatter(text) + assert fm == {} + assert body == text + + +def test_split_frontmatter_ignores_a_non_mapping_block() -> None: + fm, body = note_import.split_frontmatter("---\n- a\n- b\n---\nbody\n") + assert fm == {} + assert "- a" in body + + +def test_obsidian_loader_reads_titles_tags_and_links(vault: Path) -> None: + notes = {n.key: n for n in load_vault("obsidian", vault)} + assert set(notes) == {"Postgres", "notes/Deploys", "notes/Orphan"} + pg = notes["Postgres"] + assert pg.title == "Postgres tuning" # frontmatter wins + assert pg.tags == ["db", "ops"] + assert pg.links == ["Deploys"] + assert pg.locator == "obsidian:Postgres.md" + assert notes["notes/Deploys"].title == "Deploys" # falls back to the heading + # `.obsidian/` is machinery, not notes + assert not any(k.startswith(".obsidian") for k in notes) + + +def test_wikilink_variants_are_normalised() -> None: + links = note_import._wikilinks( + "[[Plain]] [[Aliased|shown as this]] [[Deep#heading]] [[Block^ref]] [[Plain]]" + ) + assert links == ["Plain", "Aliased", "Deep", "Block"] + + +def test_loader_rejects_an_unknown_kind(vault: Path) -> None: + with pytest.raises(NoteImportError, match="unknown vault kind"): + load_vault("evernote", vault) + + +def test_loader_rejects_a_missing_path(tmp_path: Path) -> None: + with pytest.raises(NoteImportError, match="no such path"): + load_vault("obsidian", tmp_path / "nope") + + +def test_empty_folder_is_an_actionable_error(tmp_path: Path) -> None: + empty = tmp_path / "empty" + empty.mkdir() + with pytest.raises(NoteImportError, match="no markdown notes"): + load_vault("md", empty) + + +# --- import ---------------------------------------------------------------- + + +def test_import_files_one_pending_page_per_note(store: KBStore, vault: Path) -> None: + report = import_vault(store, "obsidian", vault, generated_at="2026-07-31T00:00:00Z") + assert report["notes"] == 3 + assert report["imported"] == 3 + pages = [p for p in store.list_proposals(None) if p.kind == ProposalKind.PAGE] + assert len(pages) == 3 + assert all(p.status is ProposalStatus.PENDING for p in pages) + titles = {p.payload["title"] for p in pages} + assert "Postgres tuning" in titles + + +def test_page_cites_a_source_holding_the_note_verbatim( + store: KBStore, vault: Path +) -> None: + # The whole advantage over an embedding importer: the source is the note's + # own bytes, so a claim extracted from it quotes real offsets. + import_vault(store, "obsidian", vault) + page = next( + p for p in store.list_proposals(None) + if p.kind == ProposalKind.PAGE and p.payload["title"] == "Postgres tuning" + ) + (source_id,) = page.payload["sources"] + content = store.read_source_content(source_id).decode("utf-8") + assert "The connection pool must be sized to the worker count" in content + source = store.get_source(source_id) + assert source.locator == "obsidian:Postgres.md" + assert source.metadata["note_key"] == "Postgres" + assert source.metadata["frontmatter"]["owner"] == "alice-example" + + +def test_reimport_of_an_unchanged_vault_is_a_no_op(store: KBStore, vault: Path) -> None: + first = import_vault(store, "obsidian", vault, generated_at="2026-07-31T00:00:00Z") + assert first["imported"] == 3 + second = import_vault(store, "obsidian", vault, generated_at="2026-08-01T00:00:00Z") + assert second["imported"] == 0 + assert second["skipped"] == 3 + assert all(r["reason"] == "unchanged" for r in second["rows"]) + pages = [p for p in store.list_proposals(None) if p.kind == ProposalKind.PAGE] + assert len(pages) == 3 # nothing duplicated + + +def test_a_changed_note_refreshes_its_pending_proposal_in_place( + store: KBStore, vault: Path +) -> None: + import_vault(store, "obsidian", vault) + before = [p for p in store.list_proposals(None) if p.kind == ProposalKind.PAGE] + (vault / "notes" / "Deploys.md").write_text( + "# Deploys\n\nDeploys now run weekly, on Thursdays.\n", encoding="utf-8" + ) + report = import_vault(store, "obsidian", vault) + assert report["updated"] == 1 + assert report["skipped"] == 2 + after = [p for p in store.list_proposals(None) if p.kind == ProposalKind.PAGE] + assert len(after) == len(before) # refreshed, not re-filed + refreshed = next(p for p in after if p.payload["title"] == "Deploys") + assert "weekly, on Thursdays" in refreshed.payload["body"] + + +def test_a_decided_proposal_blocks_reimport(store: KBStore, vault: Path) -> None: + import_vault(store, "obsidian", vault) + page = next( + p for p in store.list_proposals(None) + if p.kind == ProposalKind.PAGE and p.payload["title"] == "Deploys" + ) + from vouch.proposals import reject + + reject(store, page.id, rejected_by="reviewer-example", reason="not wanted") + (vault / "notes" / "Deploys.md").write_text("# Deploys\n\nchanged\n", encoding="utf-8") + report = import_vault(store, "obsidian", vault) + row = next(r for r in report["rows"] if r["note"] == "notes/Deploys") + assert row["action"] == "skipped" + assert row["reason"] == "already-imported" + + +def test_limit_slices_the_vault_deterministically(store: KBStore, vault: Path) -> None: + first = import_vault(store, "obsidian", vault, limit=2) + assert first["imported"] == 2 + # The remaining note lands on the next run, and the first two are no-ops — + # which is what makes a big vault resumable. + second = import_vault(store, "obsidian", vault) + assert second["imported"] == 1 + assert second["skipped"] == 2 + + +def test_dry_run_writes_nothing(store: KBStore, vault: Path) -> None: + report = import_vault(store, "obsidian", vault, dry_run=True) + assert report["imported"] == 3 + assert report["dry_run"] is True + assert store.list_proposals(None) == [] + assert not list(store.list_sources()) + + +def test_wikilinks_become_relation_proposals_only_where_they_resolve( + store: KBStore, vault: Path +) -> None: + report = import_vault(store, "obsidian", vault) + assert report["relations"] == 1 # Postgres -> Deploys; the orphan link is dropped + rels = [p for p in store.list_proposals(None) if p.kind == ProposalKind.RELATION] + assert len(rels) == 1 + payload = rels[0].payload + src = store.get_source(payload["source"]) + target = store.get_source(payload["target"]) + assert src.locator == "obsidian:Postgres.md" + assert target.locator == "obsidian:notes/Deploys.md" + assert payload["relation"] == "references" + + +def test_claims_are_off_by_default_and_bounded_when_on( + store: KBStore, vault: Path +) -> None: + off = import_vault(store, "obsidian", vault) + assert off["claims"] == 0 + assert not [p for p in store.list_proposals(None) if p.kind == ProposalKind.CLAIM] + + fresh = KBStore.init(vault.parent / "kb2") + on = import_vault(fresh, "obsidian", vault, max_claims=1) + assert on["claims"] > 0 + claims = [p for p in fresh.list_proposals(None) if p.kind == ProposalKind.CLAIM] + assert claims + assert all(p.status is ProposalStatus.PENDING for p in claims) # gate intact + # `--max-claims` is a per-note ceiling: 3 notes, at most 1 claim each. + assert len(claims) <= 3 + + +def test_session_key_is_stable_and_kind_scoped() -> None: + assert session_key("obsidian", "a/b") == session_key("obsidian", "a/b") + assert session_key("obsidian", "a/b") != session_key("md", "a/b") + + +# --- the other four formats ------------------------------------------------ + + +def _joplin_note(note_id: str, title: str, body: str, extra: str = "") -> str: + return ( + f"{title}\n\n{body}\n\n" + f"id: {note_id}\n" + f"parent_id: 0123456789abcdef0123456789abcdef\n" + f"created_time: 2026-01-01T00:00:00.000Z\n" + f"{extra}" + f"type_: 1" + ) + + +def test_joplin_folder_import(store: KBStore, tmp_path: Path) -> None: + export = tmp_path / "joplin" + export.mkdir() + a_id = "a" * 32 + b_id = "b" * 32 + (export / f"{a_id}.md").write_text( + _joplin_note(a_id, "Runbook", f"Restart order matters. See [link](:/{b_id})."), + encoding="utf-8", + ) + (export / f"{b_id}.md").write_text( + _joplin_note(b_id, "Escalation", "Page the on-call after ten minutes."), + encoding="utf-8", + ) + # A folder record must not be imported as a note. + (export / "folder.md").write_text( + "Notebook\n\nid: c0ffee00000000000000000000000000\ntype_: 2", encoding="utf-8" + ) + report = import_vault(store, "joplin", export) + assert report["notes"] == 2 + assert report["imported"] == 2 + assert report["relations"] == 1 # the `:/id` link resolves + titles = { + p.payload["title"] + for p in store.list_proposals(None) + if p.kind == ProposalKind.PAGE + } + assert titles == {"Runbook", "Escalation"} + + +def test_joplin_jex_archive_import(store: KBStore, tmp_path: Path) -> None: + note_path = tmp_path / f"{'d' * 32}.md" + note_path.write_text(_joplin_note("d" * 32, "From a jex", "Body text."), "utf-8") + jex = tmp_path / "export.jex" + with tarfile.open(jex, "w") as tar: + tar.add(note_path, arcname=note_path.name) + report = import_vault(store, "joplin", jex) + assert report["imported"] == 1 + + +def test_joplin_rejects_a_non_archive_file(store: KBStore, tmp_path: Path) -> None: + bogus = tmp_path / "notes.jex" + bogus.write_text("not a tar", encoding="utf-8") + with pytest.raises(NoteImportError, match=r"not a joplin \.jex archive"): + import_vault(store, "joplin", bogus) + + +def test_apple_notes_html_import(store: KBStore, tmp_path: Path) -> None: + export = tmp_path / "notes-export" + export.mkdir() + (export / "Grocery.html").write_text( + "<html><head><style>b{}</style></head><body>" + "<h1>Grocery</h1><div>Oat milk</div><div>Coffee beans</div>" + "<script>ignored()</script></body></html>", + encoding="utf-8", + ) + report = import_vault(store, "notes", export) + assert report["imported"] == 1 + page = next( + p for p in store.list_proposals(None) if p.kind == ProposalKind.PAGE + ) + assert page.payload["title"] == "Grocery" + body = page.payload["body"] + assert "Oat milk" in body and "Coffee beans" in body + assert "ignored()" not in body # script/style are dropped, not read as text + + +def test_html_to_text_survives_malformed_markup() -> None: + assert "hello" in note_import.html_to_text("<p>hello<<</p") + + +def test_google_keep_folder_import(store: KBStore, tmp_path: Path) -> None: + export = tmp_path / "Keep" + export.mkdir() + (export / "Shopping.json").write_text(json.dumps({ + "title": "Shopping", + "textContent": "for the weekend", + "listContent": [ + {"text": "bread", "isChecked": False}, + {"text": "milk", "isChecked": True}, + ], + "labels": [{"name": "errands"}], + "userEditedTimestampUsec": 1767225600000000, + }), encoding="utf-8") + (export / "Trashed.json").write_text( + json.dumps({"title": "Old", "textContent": "x", "isTrashed": True}), + encoding="utf-8", + ) + report = import_vault(store, "keep", export) + assert report["imported"] == 1 # the trashed note is skipped + page = next(p for p in store.list_proposals(None) if p.kind == ProposalKind.PAGE) + assert page.payload["title"] == "Shopping" + assert "- [ ] bread" in page.payload["body"] + assert "- [x] milk" in page.payload["body"] + + +def test_google_keep_takeout_zip_import(store: KBStore, tmp_path: Path) -> None: + archive = tmp_path / "takeout.zip" + with zipfile.ZipFile(archive, "w") as zf: + zf.writestr( + "Takeout/Keep/Idea.json", + json.dumps({"title": "Idea", "textContent": "ship the importer"}), + ) + zf.writestr("Takeout/Mail/ignored.json", json.dumps({"title": "no"})) + report = import_vault(store, "keep", archive) + assert report["imported"] == 1 + + +def test_markdown_folder_import(store: KBStore, tmp_path: Path) -> None: + folder = tmp_path / "docs" + folder.mkdir() + (folder / "one.md").write_text("# One\n\nfirst\n", encoding="utf-8") + (folder / "two.txt").write_text("plain text note\n", encoding="utf-8") + report = import_vault(store, "md", folder) + assert report["notes"] == 2 + assert report["kind"] == "md" + + +# --- cli ------------------------------------------------------------------- + + +def _run(store: KBStore, args: list[str]): + return CliRunner().invoke(cli, args, env={"VOUCH_KB_PATH": str(store.kb_dir)}) + + +def test_cli_import_obsidian(store: KBStore, vault: Path) -> None: + result = _run(store, ["import", "obsidian", str(vault), "--json"]) + assert result.exit_code == 0, result.output + report = json.loads(result.output) + assert report["imported"] == 3 + assert report["kind"] == "obsidian" + + +def test_cli_import_dry_run_reports_without_writing( + store: KBStore, vault: Path +) -> None: + result = _run(store, ["import", "obsidian", str(vault), "--dry-run"]) + assert result.exit_code == 0, result.output + assert "would import 3 new" in result.output + assert store.list_proposals(None) == [] + + +def test_cli_import_max_claims_is_forwarded(store: KBStore, vault: Path) -> None: + result = _run( + store, ["import", "md", str(vault), "--max-claims", "1", "--json"] + ) + assert result.exit_code == 0, result.output + assert json.loads(result.output)["claims"] > 0 + + +def test_cli_import_group_lists_every_kind() -> None: + result = CliRunner().invoke(cli, ["import", "--help"]) + assert result.exit_code == 0 + for kind in ("obsidian", "joplin", "notes", "keep", "md", "chatgpt"): + assert kind in result.output + + +def test_cli_import_reports_a_bad_vault_cleanly(store: KBStore, tmp_path: Path) -> None: + empty = tmp_path / "empty" + empty.mkdir() + result = _run(store, ["import", "obsidian", str(empty)]) + assert result.exit_code != 0 + assert "no markdown notes" in result.output + assert "Traceback" not in result.output + + +def test_cli_human_output_reports_claims_and_links( + store: KBStore, vault: Path +) -> None: + result = _run(store, ["import", "obsidian", str(vault), "--max-claims", "1"]) + assert result.exit_code == 0, result.output + assert "claim(s)" in result.output + assert "link relation(s) proposed" in result.output + assert "run `vouch review` to decide." in result.output + # skipped rows are counted, not listed + again = _run(store, ["import", "obsidian", str(vault)]) + assert "3 skipped" in again.output + assert "•" not in again.output + + +def test_cli_each_kind_reaches_its_loader(store: KBStore, tmp_path: Path) -> None: + """One CLI test per subcommand — the group is the surface #612 asks for, + and a subcommand wired to the wrong kind would be invisible otherwise.""" + joplin = tmp_path / "joplin" + joplin.mkdir() + (joplin / f"{'e' * 32}.md").write_text( + _joplin_note("e" * 32, "Joplin note", "body"), encoding="utf-8" + ) + notes = tmp_path / "apple" + notes.mkdir() + (notes / "Note.txt").write_text("apple note body", encoding="utf-8") + keep = tmp_path / "Keep" + keep.mkdir() + (keep / "Keep note.json").write_text( + json.dumps({"title": "Keep note", "textContent": "keep body"}), encoding="utf-8" + ) + md = tmp_path / "docs" + md.mkdir() + (md / "doc.md").write_text("# Doc\n\nbody\n", encoding="utf-8") + + for kind, path in ( + ("joplin", joplin), ("notes", notes), ("keep", keep), ("md", md) + ): + result = _run(store, ["import", kind, str(path), "--json"]) + assert result.exit_code == 0, (kind, result.output) + report = json.loads(result.output) + assert report["kind"] == kind + assert report["imported"] == 1 + + +def test_cli_import_chatgpt_alias(store: KBStore, tmp_path: Path) -> None: + """`vouch import chatgpt` is the same importer as the flat command — one + `vouch import <kind>` surface over every source.""" + export = tmp_path / "conversations.json" + export.write_text(json.dumps([{ + "conversation_id": "conv-1", + "title": "About deploys", + "mapping": { + "root": {"id": "root", "message": None, "parent": None, "children": ["u1"]}, + "u1": { + "id": "u1", "parent": "root", "children": ["a1"], + "message": { + "author": {"role": "user"}, + "content": {"content_type": "text", "parts": ["when do we deploy?"]}, + }, + }, + "a1": { + "id": "a1", "parent": "u1", "children": [], + "message": { + "author": {"role": "assistant"}, + "content": { + "content_type": "text", + "parts": ["Every second Tuesday from the release branch."], + }, + }, + }, + }, + }]), encoding="utf-8") + result = _run(store, ["import", "chatgpt", str(export), "--json"]) + assert result.exit_code == 0, result.output + assert json.loads(result.output)["imported"] == 1 + + +# --- the tolerant paths ---------------------------------------------------- +# +# A vault is other people's files. Every branch below is a "skip it and carry +# on" that exists so one malformed note cannot fail an import of ten thousand. + + +def test_clip_truncates_a_long_body(store: KBStore, tmp_path: Path) -> None: + folder = tmp_path / "long" + folder.mkdir() + (folder / "big.md").write_text("x" * 10_000, encoding="utf-8") + import_vault(store, "md", folder) + page = next(p for p in store.list_proposals(None) if p.kind == ProposalKind.PAGE) + assert page.payload["body"].rstrip().endswith("…") + + +def test_frontmatter_needs_a_delimiter_line_and_a_terminator() -> None: + # `---title: x` starts with `---` but is not a fence + assert note_import.split_frontmatter("---title: x\nbody\n")[0] == {} + # opened and never closed + assert note_import.split_frontmatter("---\na: 1\nbody with no terminator\n")[0] == {} + + +def test_frontmatter_tags_accept_a_string_list() -> None: + fm, _ = note_import.split_frontmatter('---\ntags: "db, ops #infra"\n---\nbody\n') + assert note_import._tags_from_frontmatter(fm) == ["db", "ops", "infra"] + assert note_import._tags_from_frontmatter({"tags": 7}) == [] + + +def test_wikilinks_stop_at_the_per_note_ceiling() -> None: + body = " ".join(f"[[Note{i}]]" for i in range(note_import._MAX_LINKS_PER_NOTE + 10)) + assert len(note_import._wikilinks(body)) == note_import._MAX_LINKS_PER_NOTE + + +def test_walk_skips_machinery_and_dotfiles(store: KBStore, tmp_path: Path) -> None: + root = tmp_path / "vault2" + (root / "_resources").mkdir(parents=True) + (root / "_resources" / "attached.md").write_text("attachment", encoding="utf-8") + (root / ".hidden.md").write_text("hidden", encoding="utf-8") + (root / "Real.md").write_text("# Real\n\nkept\n", encoding="utf-8") + keys = {n.key for n in load_vault("md", root)} + assert keys == {"Real"} + + +def test_walk_skips_a_file_it_cannot_stat( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + root = tmp_path / "racy" + root.mkdir() + (root / "gone.md").write_text("body", encoding="utf-8") + (root / "kept.md").write_text("# Kept\n\nbody\n", encoding="utf-8") + real_stat = Path.stat + + def flaky(self: Path, *a: object, **kw: object): + if self.name == "gone.md": + raise OSError("vanished mid-walk") + return real_stat(self, *a, **kw) + + monkeypatch.setattr(Path, "stat", flaky) + assert {n.key for n in load_vault("md", root)} == {"kept"} + + +def test_vault_over_the_byte_ceiling_is_refused( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(note_import, "_MAX_VAULT_BYTES", 4) + root = tmp_path / "huge" + root.mkdir() + (root / "a.md").write_text("well over four bytes", encoding="utf-8") + with pytest.raises(NoteImportError, match="import ceiling"): + load_vault("md", root) + + +def test_markdown_loader_rejects_a_file(tmp_path: Path) -> None: + target = tmp_path / "one.md" + target.write_text("body", encoding="utf-8") + with pytest.raises(NoteImportError, match="not a directory"): + load_vault("md", target) + + +def test_timestamp_helpers_degrade_instead_of_raising() -> None: + assert note_import._iso("not-a-number") is None + assert note_import._iso_ms(object()) is None + + +def test_joplin_skips_notes_whose_footer_is_not_metadata(tmp_path: Path) -> None: + export = tmp_path / "joplin" + export.mkdir() + # trailing block with a line that carries no colon + (export / "a.md").write_text("Title\n\nbody\n\njust prose\n", encoding="utf-8") + # trailing block whose "key" has a space in it — prose, not metadata + (export / "b.md").write_text("Title\n\nbody\n\nnot a key: value\n", encoding="utf-8") + with pytest.raises(NoteImportError, match="no joplin notes"): + load_vault("joplin", export) + + +def test_joplin_missing_path_is_actionable(tmp_path: Path) -> None: + with pytest.raises(NoteImportError, match="no such path"): + load_vault("joplin", tmp_path / "nope") + # the loader guards for itself too — it is importable on its own + with pytest.raises(NoteImportError, match="no such joplin export"): + note_import.load_joplin(tmp_path / "nope") + + +def test_joplin_jex_skips_non_note_members(tmp_path: Path) -> None: + note = tmp_path / f"{'f' * 32}.md" + note.write_text(_joplin_note("f" * 32, "Kept", "body"), encoding="utf-8") + resource = tmp_path / "resource.bin" + resource.write_bytes(b"attachment") + inner = tmp_path / "subdir" + inner.mkdir() + jex = tmp_path / "mixed.jex" + with tarfile.open(jex, "w") as tar: + tar.add(note, arcname=note.name) + tar.add(resource, arcname=resource.name) + # a *directory* whose name ends in .md clears the suffix filter but has + # no readable stream — extractfile returns None + tar.add(inner, arcname="looks-like-a-note.md") + notes = load_vault("joplin", jex) + assert [n.title for n in notes] == ["Kept"] + + +def test_joplin_jex_over_the_ceiling_is_refused( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(note_import, "_MAX_VAULT_BYTES", 4) + note = tmp_path / f"{'g' * 32}.md" + note.write_text(_joplin_note("g" * 32, "Big", "body"), encoding="utf-8") + jex = tmp_path / "big.jex" + with tarfile.open(jex, "w") as tar: + tar.add(note, arcname=note.name) + with pytest.raises(NoteImportError, match="import ceiling"): + load_vault("joplin", jex) + + +def test_html_to_text_falls_back_when_the_parser_blows_up( + monkeypatch: pytest.MonkeyPatch +) -> None: + def boom(self: object, data: str) -> None: + raise RuntimeError("parser exploded") + + monkeypatch.setattr(note_import._TextExtractor, "feed", boom) + assert note_import.html_to_text("<p>still readable</p>") == "still readable" + + +def test_apple_notes_loader_rejects_a_file_and_an_empty_folder( + tmp_path: Path +) -> None: + target = tmp_path / "one.html" + target.write_text("<p>x</p>", encoding="utf-8") + with pytest.raises(NoteImportError, match="not a directory"): + load_vault("notes", target) + empty = tmp_path / "no-notes" + empty.mkdir() + with pytest.raises(NoteImportError, match="no apple-notes files"): + load_vault("notes", empty) + + +def test_keep_skips_an_entirely_empty_note(tmp_path: Path) -> None: + export = tmp_path / "Keep" + export.mkdir() + (export / "blank.json").write_text(json.dumps({"color": "WHITE"}), encoding="utf-8") + (export / "real.json").write_text( + json.dumps({"title": "Real", "textContent": "body"}), encoding="utf-8" + ) + assert [n.title for n in load_vault("keep", export)] == ["Real"] + + +def test_keep_skips_undecodable_json(tmp_path: Path) -> None: + export = tmp_path / "Keep" + export.mkdir() + (export / "broken.json").write_text("{not json", encoding="utf-8") + (export / "real.json").write_text( + json.dumps({"title": "Real", "textContent": "body"}), encoding="utf-8" + ) + assert [n.title for n in load_vault("keep", export)] == ["Real"] + + +def test_keep_zip_skips_undecodable_json_and_non_notes(tmp_path: Path) -> None: + archive = tmp_path / "takeout.zip" + with zipfile.ZipFile(archive, "w") as zf: + zf.writestr("Takeout/Keep/broken.json", "{not json") + zf.writestr("Takeout/Keep/attachment.png", b"\x89PNG") # not json at all + zf.writestr("Takeout/Keep/", b"") # a directory entry + zf.writestr( + "Takeout/Keep/real.json", json.dumps({"title": "Real", "textContent": "b"}) + ) + assert [n.title for n in load_vault("keep", archive)] == ["Real"] + + +def test_keep_zip_over_the_ceiling_is_refused( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(note_import, "_MAX_VAULT_BYTES", 4) + archive = tmp_path / "takeout.zip" + with zipfile.ZipFile(archive, "w") as zf: + zf.writestr("Takeout/Keep/a.json", json.dumps({"title": "A", "textContent": "b"})) + with pytest.raises(NoteImportError, match="import ceiling"): + load_vault("keep", archive) + + +def test_keep_missing_path_and_empty_export_are_actionable(tmp_path: Path) -> None: + with pytest.raises(NoteImportError, match="no such path"): + load_vault("keep", tmp_path / "nope") + with pytest.raises(NoteImportError, match="no such keep export"): + note_import.load_google_keep(tmp_path / "nope") + empty = tmp_path / "Keep" + empty.mkdir() + with pytest.raises(NoteImportError, match="no google keep notes"): + load_vault("keep", empty) + + +def test_a_link_written_with_its_extension_still_resolves( + store: KBStore, tmp_path: Path +) -> None: + root = tmp_path / "vault3" + root.mkdir() + (root / "A.md").write_text("links to [[B.md]]\n", encoding="utf-8") + (root / "B.md").write_text("target\n", encoding="utf-8") + assert import_vault(store, "obsidian", root)["relations"] == 1 + + +def test_links_are_not_refiled_for_notes_that_were_skipped( + store: KBStore, vault: Path +) -> None: + # Second run: every note is unchanged, so no source is registered and the + # link pass has nothing to file — the relation must not duplicate. + assert import_vault(store, "obsidian", vault)["relations"] == 1 + assert import_vault(store, "obsidian", vault)["relations"] == 0 + rels = [p for p in store.list_proposals(None) if p.kind == ProposalKind.RELATION] + assert len(rels) == 1 + + +def test_a_link_into_an_unchanged_note_is_not_refiled( + store: KBStore, vault: Path +) -> None: + """The partial case: the linking note changed, its target did not. The + target has no source this run, so the edge is left alone rather than + re-proposed against a stale id.""" + assert import_vault(store, "obsidian", vault)["relations"] == 1 + (vault / "Postgres.md").write_text( + "---\ntitle: Postgres tuning\n---\nrewritten. still see [[Deploys]].\n", + encoding="utf-8", + ) + report = import_vault(store, "obsidian", vault) + assert report["updated"] == 1 + assert report["relations"] == 0 + rels = [p for p in store.list_proposals(None) if p.kind == ProposalKind.RELATION] + assert len(rels) == 1 + + +def test_a_rejected_relation_does_not_fail_the_import( + store: KBStore, vault: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + def refuse(*args: object, **kwargs: object) -> None: + from vouch.proposals import ProposalError + + raise ProposalError("nope") + + monkeypatch.setattr(note_import, "propose_relation", refuse) + report = import_vault(store, "obsidian", vault) + assert report["imported"] == 3 # the pages still land + assert report["relations"] == 0