feat(import): conversation and memory-export importers

CHANGELOG.md

@@ -7,6 +7,45 @@ All notable changes to vouch are documented here. Format follows
 ## [Unreleased]
 
 ### Added
+- **conversation and memory-export importers** (#431): `vouch import chat-json
+  <export>` normalises the common JSON chat shapes — openai's branching
+  `mapping` tree (delegated to the existing chatgpt importer), claude.ai's
+  `chat_messages`, and the generic `messages: [{role, content}]` almost
+  everything else emits, from a file, a `.jsonl`, or a `.zip` — into one
+  PENDING page per conversation, cited to a per-conversation source.
+  `vouch import memory-export <dump>` reads a prior memory tool's dump (JSON
+  array, object of records, JSONL, or one memory per line) and files each
+  memory as a claim quoting its own source verbatim, so the receipt verifies
+  and the imported fact is citable rather than asserted. `markdown-vault` is
+  registered as an alias of the note-vault importer, so the three formats the
+  issue names are all reachable from one surface. Two guards keep an import
+  from becoming a reviewer's problem: `--max-proposals` caps a whole run and
+  the report says when the cap was hit (rerunning continues where it left
+  off), and candidates an approved claim or a pending proposal already cover
+  are dropped — lexically first, since a base install has no `[embeddings]`
+  extra, with the embedding hits (#147) folded in on top when available.
+  `--dry-run` reports without enqueuing, `--max-claims` files receipt-backed
+  claims from a conversation's answers, and nothing here calls `approve()`.
+- **note-vault importers — arrive with years of notes already written** (#612):
+  `vouch import obsidian <vault>`, plus `joplin` (folder or `.jex`), `notes`
+  (apple notes html/txt export), `keep` (google takeout folder or `.zip`), and
+  `md` (a plain markdown folder). One PENDING page proposal per note, each
+  cited to a source registered from **the note's own bytes** — which is the
+  thing an embedding-based importer structurally cannot offer: claims extracted
+  from an imported note quote real offsets, so their receipts verify and the
+  knowledge is citable rather than paraphrased. Frontmatter is carried into
+  source metadata; wikilinks (and joplin's `:/id` links) become `references`
+  relation proposals, but only where the target resolves to a note that was
+  actually imported. Re-running is idempotent on a stable per-note identity
+  derived from the origin's own identifier — unchanged notes are no-ops,
+  changed ones refresh their pending proposal in place, decided ones stay
+  decided — so a large vault imports in `--limit` slices and resumes. Claims
+  are opt-in and bounded through the existing density-selection knob
+  (`--max-claims`, default 0 = pages only), so ten thousand notes cannot become
+  ten thousand pending claims. `--dry-run` previews. Nothing is approved: an
+  import is a proposal firehose, not a write. `vouch import chatgpt` now also
+  exists as an alias so one `vouch import <kind>` surface covers every source;
+  the flat `vouch import-chatgpt` stays for back-compat.
 - **explicit pins — a working set that always enters the pack** (#615):
   `vouch pin <id>` / `vouch pins list` / `vouch unpin <id>`. Pinned claims and
   pages lead every context pack instead of having to win the query each turn,

src/vouch/cli.py

@@ -33,6 +33,7 @@
 from . import codex_rollout as codex_rollout_mod
 from . import compile as compile_mod
 from . import contradictions as contradictions_mod
+from . import conversation_import as conversation_import_mod
 from . import digest as digest_mod
 from . import fetch as fetch_mod
 from . import hub as hub_mod
@@ -41,6 +42,7 @@
 from . import lifecycle as life
 from . import metrics as metrics_mod
 from . import migrations as migrations_mod
+from . import note_import as note_import_mod
 from . import notify as notify_mod
 from . import pins as pins_mod
 from . import pr_cache as prc_mod
@@ -116,6 +118,8 @@ def _cli_errors() -> Iterator[None]:
         migrations_mod.MigrationError,
         chatgpt_import_mod.ChatGPTImportError,
         codex_rollout_mod.CodexRolloutError,
+        note_import_mod.NoteImportError,
+        conversation_import_mod.ConversationImportError,
         pins_mod.PinError,
     ) as e:
         raise click.ClickException(str(e)) from e
@@ -4409,6 +4413,253 @@ def import_chatgpt_cmd(
         _echo("run `vouch review` to decide.")
 
 
+# --- import: note vaults (issue #612) --------------------------------------
+
+
+@cli.group(name="import")
+def import_group() -> None:
+    """Import an existing note vault as PENDING proposals.
+
+    One page per note, cited to a source registered from the note's own bytes —
+    so claims extracted from it quote real offsets and their receipts verify.
+    Re-running is idempotent: unchanged notes are no-ops, changed ones refresh
+    their pending proposal, decided ones stay decided. Nothing is approved.
+    """
+
+
+def _import_vault_cmd(kind: str, path: Path, limit: int | None, max_claims: int,
+                      dry_run: bool, as_json: bool) -> None:
+    store = _load_store()
+    with _cli_errors():
+        report = note_import_mod.import_vault(
+            store, kind, path, limit=limit, max_claims=max_claims,
+            dry_run=dry_run, generated_at=datetime.now(UTC).isoformat(),
+        )
+    if as_json:
+        _emit_json(report)
+        return
+    verb = "would import" if dry_run else "imported"
+    _echo(
+        f"{report['notes']} note(s) — {verb} {report['imported']} new, "
+        f"{report['updated']} updated, {report['skipped']} skipped"
+    )
+    if report["claims"] or report["relations"]:
+        _echo(
+            f"  + {report['claims']} claim(s), {report['relations']} link "
+            f"relation(s) proposed"
+        )
+    for row in report["rows"]:
+        if row["action"] == "skipped":
+            continue
+        pid = row.get("proposal_id") or "(dry-run)"
+        _echo(f"  • {pid}  {row['title']}")
+    if not dry_run and (report["imported"] or report["updated"]):
+        _echo("run `vouch review` to decide.")
+
+
+def _vault_options(fn: Any) -> Any:
+    """The four flags every `vouch import <kind>` subcommand shares."""
+    fn = click.option(
+        "--json", "as_json", is_flag=True, help="Machine-readable report."
+    )(fn)
+    fn = click.option(
+        "--dry-run", is_flag=True, help="Parse and report; file nothing."
+    )(fn)
+    fn = click.option(
+        "--max-claims", "max_claims", type=int, default=0, show_default=True,
+        help="File up to N receipt-backed claims per note (0 = pages only).",
+    )(fn)
+    fn = click.option(
+        "--limit", type=int, default=None,
+        help="Import at most N notes, in sorted order (resumable).",
+    )(fn)
+    return fn
+
+
+@import_group.command("obsidian")
+@click.argument("vault", type=click.Path(exists=True, path_type=Path))
+@_vault_options
+def import_obsidian_cmd(
+    vault: Path, limit: int | None, max_claims: int, dry_run: bool, as_json: bool
+) -> None:
+    """Import an Obsidian vault (frontmatter, wikilinks, folder structure)."""
+    _import_vault_cmd("obsidian", vault, limit, max_claims, dry_run, as_json)
+
+
+@import_group.command("md")
+@click.argument("folder", type=click.Path(exists=True, path_type=Path))
+@_vault_options
+def import_md_cmd(
+    folder: Path, limit: int | None, max_claims: int, dry_run: bool, as_json: bool
+) -> None:
+    """Import a plain markdown folder."""
+    _import_vault_cmd("md", folder, limit, max_claims, dry_run, as_json)
+
+
+@import_group.command("joplin")
+@click.argument("export_path", type=click.Path(exists=True, path_type=Path))
+@_vault_options
+def import_joplin_cmd(
+    export_path: Path, limit: int | None, max_claims: int, dry_run: bool, as_json: bool
+) -> None:
+    """Import a Joplin export — a .jex archive or the folder it unpacks to."""
+    _import_vault_cmd("joplin", export_path, limit, max_claims, dry_run, as_json)
+
+
+@import_group.command("notes")
+@click.argument("export_path", type=click.Path(exists=True, path_type=Path))
+@_vault_options
+def import_apple_notes_cmd(
+    export_path: Path, limit: int | None, max_claims: int, dry_run: bool, as_json: bool
+) -> None:
+    """Import an Apple Notes export (a folder of .html/.txt per note)."""
+    _import_vault_cmd("notes", export_path, limit, max_claims, dry_run, as_json)
+
+
+@import_group.command("keep")
+@click.argument("export_path", type=click.Path(exists=True, path_type=Path))
+@_vault_options
+def import_keep_cmd(
+    export_path: Path, limit: int | None, max_claims: int, dry_run: bool, as_json: bool
+) -> None:
+    """Import a Google Keep (Takeout) export — the folder or the .zip."""
+    _import_vault_cmd("keep", export_path, limit, max_claims, dry_run, as_json)
+
+
+def _export_options(fn: Any) -> Any:
+    """The flags every conversation/memory export subcommand shares."""
+    fn = click.option(
+        "--json", "as_json", is_flag=True, help="Machine-readable report."
+    )(fn)
+    fn = click.option(
+        "--dry-run", is_flag=True, help="Parse and report; file nothing."
+    )(fn)
+    fn = click.option(
+        "--no-dedup", "no_dedup", is_flag=True,
+        help="File every candidate, even one the KB already covers.",
+    )(fn)
+    fn = click.option(
+        "--max-proposals", "max_proposals", type=int, default=None,
+        help="Cap the whole run; the report says when the cap was hit.",
+    )(fn)
+    fn = click.option(
+        "--limit", type=int, default=None,
+        help="Consider at most N entries, in export order.",
+    )(fn)
+    return fn
+
+
+def _echo_import_report(report: dict[str, Any], unit: str, dry_run: bool) -> None:
+    verb = "would import" if dry_run else "imported"
+    _echo(
+        f"{report[unit]} {unit[:-1]}(s) — {verb} {report['imported']} new, "
+        f"{report.get('updated', 0)} updated, {report['skipped']} skipped"
+    )
+    if report.get("claims"):
+        _echo(f"  + {report['claims']} receipt-backed claim(s) proposed")
+    if report["capped"]:
+        _echo(
+            f"  ! stopped at --max-proposals {report['max_proposals']} — rerun "
+            f"to continue where this left off"
+        )
+    if not dry_run and (report["imported"] or report.get("updated")):
+        _echo("run `vouch review` to decide.")
+
+
+@import_group.command("chat-json")
+@click.argument(
+    "export_path", type=click.Path(exists=True, dir_okay=False, path_type=Path)
+)
+@click.option(
+    "--max-claims", "max_claims", type=int, default=0, show_default=True,
+    help="File up to N receipt-backed claims per conversation (0 = pages only).",
+)
+@_export_options
+def import_chat_json_cmd(
+    export_path: Path, limit: int | None, max_proposals: int | None,
+    no_dedup: bool, dry_run: bool, as_json: bool, max_claims: int,
+) -> None:
+    """Import a JSON chat export (claude.ai, gemini, perplexity, openai, …).
+
+    One PENDING page per conversation, cited to a per-conversation source.
+    Re-importing is idempotent. Nothing is approved.
+    """
+    store = _load_store()
+    with _cli_errors():
+        report = conversation_import_mod.import_conversations(
+            store, export_path, limit=limit, max_proposals=max_proposals,
+            max_claims=max_claims, dry_run=dry_run, dedup=not no_dedup,
+            generated_at=datetime.now(UTC).isoformat(),
+        )
+    if as_json:
+        _emit_json(report)
+        return
+    _echo_import_report(report, "conversations", dry_run)
+
+
+@import_group.command("memory-export")
+@click.argument(
+    "export_path", type=click.Path(exists=True, dir_okay=False, path_type=Path)
+)
+@_export_options
+def import_memory_export_cmd(
+    export_path: Path, limit: int | None, max_proposals: int | None,
+    no_dedup: bool, dry_run: bool, as_json: bool,
+) -> None:
+    """Import a prior memory tool's dump as PENDING claim proposals.
+
+    Each memory becomes a claim quoting its own source verbatim, so the
+    receipt verifies and the imported fact is citable rather than asserted.
+    """
+    store = _load_store()
+    with _cli_errors():
+        report = conversation_import_mod.import_memories(
+            store, export_path, limit=limit, max_proposals=max_proposals,
+            dry_run=dry_run, dedup=not no_dedup,
+        )
+    if as_json:
+        _emit_json(report)
+        return
+    _echo_import_report(report, "memories", dry_run)
+
+
+@import_group.command("markdown-vault")
+@click.argument("folder", type=click.Path(exists=True, path_type=Path))
+@_vault_options
+def import_markdown_vault_cmd(
+    folder: Path, limit: int | None, max_claims: int, dry_run: bool, as_json: bool
+) -> None:
+    """Import a markdown folder (alias of `vouch import md`)."""
+    _import_vault_cmd("md", folder, limit, max_claims, dry_run, as_json)
+
+
+@import_group.command("chatgpt")
+@click.argument(
+    "export_path", type=click.Path(exists=True, dir_okay=False, path_type=Path)
+)
+@click.option(
+    "--limit", type=int, default=None,
+    help="Import at most N conversations, in export order.",
+)
+@click.option("--dry-run", is_flag=True, help="Parse and report; file nothing.")
+@click.option("--json", "as_json", is_flag=True, help="Machine-readable report.")
+@click.pass_context
+def import_chatgpt_sub_cmd(
+    ctx: click.Context,
+    export_path: Path, limit: int | None, dry_run: bool, as_json: bool
+) -> None:
+    """Import a ChatGPT history export (alias of `vouch import-chatgpt`).
+
+    The conversation importer predates this group; it lives here too so the
+    one `vouch import <kind>` surface covers every source. The flat
+    `vouch import-chatgpt` stays for back-compat.
+    """
+    ctx.invoke(
+        import_chatgpt_cmd, export_path=export_path, limit=limit,
+        dry_run=dry_run, as_json=as_json,
+    )
+
+
 # --- auto-pr: open N mergeable PRs against any github repo -----------------