diff --git a/CHANGELOG.md b/CHANGELOG.md index c582c62d..2742d912 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,26 @@ All notable changes to vouch are documented here. Format follows ## [Unreleased] ### Added +- **note-vault importers — arrive with years of notes already written** (#612): + `vouch import obsidian `, plus `joplin` (folder or `.jex`), `notes` + (apple notes html/txt export), `keep` (google takeout folder or `.zip`), and + `md` (a plain markdown folder). One PENDING page proposal per note, each + cited to a source registered from **the note's own bytes** — which is the + thing an embedding-based importer structurally cannot offer: claims extracted + from an imported note quote real offsets, so their receipts verify and the + knowledge is citable rather than paraphrased. Frontmatter is carried into + source metadata; wikilinks (and joplin's `:/id` links) become `references` + relation proposals, but only where the target resolves to a note that was + actually imported. Re-running is idempotent on a stable per-note identity + derived from the origin's own identifier — unchanged notes are no-ops, + changed ones refresh their pending proposal in place, decided ones stay + decided — so a large vault imports in `--limit` slices and resumes. Claims + are opt-in and bounded through the existing density-selection knob + (`--max-claims`, default 0 = pages only), so ten thousand notes cannot become + ten thousand pending claims. `--dry-run` previews. Nothing is approved: an + import is a proposal firehose, not a write. `vouch import chatgpt` now also + exists as an alias so one `vouch import ` surface covers every source; + the flat `vouch import-chatgpt` stays for back-compat. - **explicit pins — a working set that always enters the pack** (#615): `vouch pin ` / `vouch pins list` / `vouch unpin `. Pinned claims and pages lead every context pack instead of having to win the query each turn, diff --git a/src/vouch/cli.py b/src/vouch/cli.py index dfcaeb0f..3dcac7ac 100644 --- a/src/vouch/cli.py +++ b/src/vouch/cli.py @@ -41,6 +41,7 @@ from . import lifecycle as life from . import metrics as metrics_mod from . import migrations as migrations_mod +from . import note_import as note_import_mod from . import notify as notify_mod from . import pins as pins_mod from . import pr_cache as prc_mod @@ -116,6 +117,7 @@ def _cli_errors() -> Iterator[None]: migrations_mod.MigrationError, chatgpt_import_mod.ChatGPTImportError, codex_rollout_mod.CodexRolloutError, + note_import_mod.NoteImportError, pins_mod.PinError, ) as e: raise click.ClickException(str(e)) from e @@ -4409,6 +4411,146 @@ def import_chatgpt_cmd( _echo("run `vouch review` to decide.") +# --- import: note vaults (issue #612) -------------------------------------- + + +@cli.group(name="import") +def import_group() -> None: + """Import an existing note vault as PENDING proposals. + + One page per note, cited to a source registered from the note's own bytes — + so claims extracted from it quote real offsets and their receipts verify. + Re-running is idempotent: unchanged notes are no-ops, changed ones refresh + their pending proposal, decided ones stay decided. Nothing is approved. + """ + + +def _import_vault_cmd(kind: str, path: Path, limit: int | None, max_claims: int, + dry_run: bool, as_json: bool) -> None: + store = _load_store() + with _cli_errors(): + report = note_import_mod.import_vault( + store, kind, path, limit=limit, max_claims=max_claims, + dry_run=dry_run, generated_at=datetime.now(UTC).isoformat(), + ) + if as_json: + _emit_json(report) + return + verb = "would import" if dry_run else "imported" + _echo( + f"{report['notes']} note(s) — {verb} {report['imported']} new, " + f"{report['updated']} updated, {report['skipped']} skipped" + ) + if report["claims"] or report["relations"]: + _echo( + f" + {report['claims']} claim(s), {report['relations']} link " + f"relation(s) proposed" + ) + for row in report["rows"]: + if row["action"] == "skipped": + continue + pid = row.get("proposal_id") or "(dry-run)" + _echo(f" • {pid} {row['title']}") + if not dry_run and (report["imported"] or report["updated"]): + _echo("run `vouch review` to decide.") + + +def _vault_options(fn: Any) -> Any: + """The four flags every `vouch import ` subcommand shares.""" + fn = click.option( + "--json", "as_json", is_flag=True, help="Machine-readable report." + )(fn) + fn = click.option( + "--dry-run", is_flag=True, help="Parse and report; file nothing." + )(fn) + fn = click.option( + "--max-claims", "max_claims", type=int, default=0, show_default=True, + help="File up to N receipt-backed claims per note (0 = pages only).", + )(fn) + fn = click.option( + "--limit", type=int, default=None, + help="Import at most N notes, in sorted order (resumable).", + )(fn) + return fn + + +@import_group.command("obsidian") +@click.argument("vault", type=click.Path(exists=True, path_type=Path)) +@_vault_options +def import_obsidian_cmd( + vault: Path, limit: int | None, max_claims: int, dry_run: bool, as_json: bool +) -> None: + """Import an Obsidian vault (frontmatter, wikilinks, folder structure).""" + _import_vault_cmd("obsidian", vault, limit, max_claims, dry_run, as_json) + + +@import_group.command("md") +@click.argument("folder", type=click.Path(exists=True, path_type=Path)) +@_vault_options +def import_md_cmd( + folder: Path, limit: int | None, max_claims: int, dry_run: bool, as_json: bool +) -> None: + """Import a plain markdown folder.""" + _import_vault_cmd("md", folder, limit, max_claims, dry_run, as_json) + + +@import_group.command("joplin") +@click.argument("export_path", type=click.Path(exists=True, path_type=Path)) +@_vault_options +def import_joplin_cmd( + export_path: Path, limit: int | None, max_claims: int, dry_run: bool, as_json: bool +) -> None: + """Import a Joplin export — a .jex archive or the folder it unpacks to.""" + _import_vault_cmd("joplin", export_path, limit, max_claims, dry_run, as_json) + + +@import_group.command("notes") +@click.argument("export_path", type=click.Path(exists=True, path_type=Path)) +@_vault_options +def import_apple_notes_cmd( + export_path: Path, limit: int | None, max_claims: int, dry_run: bool, as_json: bool +) -> None: + """Import an Apple Notes export (a folder of .html/.txt per note).""" + _import_vault_cmd("notes", export_path, limit, max_claims, dry_run, as_json) + + +@import_group.command("keep") +@click.argument("export_path", type=click.Path(exists=True, path_type=Path)) +@_vault_options +def import_keep_cmd( + export_path: Path, limit: int | None, max_claims: int, dry_run: bool, as_json: bool +) -> None: + """Import a Google Keep (Takeout) export — the folder or the .zip.""" + _import_vault_cmd("keep", export_path, limit, max_claims, dry_run, as_json) + + +@import_group.command("chatgpt") +@click.argument( + "export_path", type=click.Path(exists=True, dir_okay=False, path_type=Path) +) +@click.option( + "--limit", type=int, default=None, + help="Import at most N conversations, in export order.", +) +@click.option("--dry-run", is_flag=True, help="Parse and report; file nothing.") +@click.option("--json", "as_json", is_flag=True, help="Machine-readable report.") +@click.pass_context +def import_chatgpt_sub_cmd( + ctx: click.Context, + export_path: Path, limit: int | None, dry_run: bool, as_json: bool +) -> None: + """Import a ChatGPT history export (alias of `vouch import-chatgpt`). + + The conversation importer predates this group; it lives here too so the + one `vouch import ` surface covers every source. The flat + `vouch import-chatgpt` stays for back-compat. + """ + ctx.invoke( + import_chatgpt_cmd, export_path=export_path, limit=limit, + dry_run=dry_run, as_json=as_json, + ) + + # --- auto-pr: open N mergeable PRs against any github repo ----------------- diff --git a/src/vouch/note_import.py b/src/vouch/note_import.py new file mode 100644 index 00000000..f75da7cb --- /dev/null +++ b/src/vouch/note_import.py @@ -0,0 +1,843 @@ +"""Import a note vault into review-gated proposals (issue #612). + +People arrive with years of notes already written — obsidian vaults, joplin +archives, apple notes and google keep exports, plain markdown folders — and a +fresh KB has nothing to say to them. ``vouch import obsidian `` (and its +four siblings) files **one PENDING page proposal per note**, each cited to a +source registered from the note's own bytes. + +That last part is the point, and it is what an embedding-based importer +structurally cannot offer: the source content is the note verbatim, so every +claim extracted from it quotes real bytes at real offsets and its receipt +verifies. Imported knowledge is citable, not paraphrased. + +Re-importing the same vault is safe. Each note gets a stable identity derived +from the origin's own identifier — the vault-relative path, or joplin's/keep's +own note id where the format carries one — recorded on the source ``locator`` +and used as the proposal's session id. A note that changed refreshes its still +PENDING proposal in place; an unchanged one is a flat no-op; a decided proposal +is history and blocks re-import. Nothing duplicates, so a big vault can be +imported in ``--limit`` slices and resumed. + +Wikilinks become ``references`` relation proposals, but only where the target +resolves to a note that was actually imported — a link to a note that does not +exist yet is dropped rather than filed as a dangling edge. + +Claims are opt-in and bounded. ``--max-claims`` is the density selection knob +(``extract.select_spans``): a ten-thousand-note vault must not turn into ten +thousand pending claims, so the default files pages only. + +Never calls ``approve()``. An import is a proposal firehose, not a write — a +human reviews it with ``vouch review`` like everything else. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import tarfile +import zipfile +from dataclasses import dataclass, field +from datetime import UTC, datetime +from html.parser import HTMLParser +from pathlib import Path +from typing import Any + +import yaml + +from . import audit +from .extract import extract_receipt_claims +from .models import Proposal, ProposalKind, ProposalStatus +from .proposals import ProposalError, default_scope, propose_page, propose_relation +from .storage import KBStore + +# Deliberately NOT one of admission's AUTO_CAPTURE_ACTORS: an import is a human +# choosing to file their own notes, so admission verdicts stay advisory and the +# pages reach review instead of being auto-rejected as capture noise. Mirrors +# `chatgpt_import.CHATGPT_ACTOR`. +NOTE_IMPORT_ACTOR = "note-import" + +# Each page is a review surface over exactly one imported note, cited to it. +PAGE_TYPE = "source-summary" + +KINDS = ("obsidian", "joplin", "notes", "keep", "md") + +# Mirrors the ceilings elsewhere (fetch, codex_rollout, chatgpt_import): one +# oversized archive must not exhaust memory. +_MAX_VAULT_BYTES = 200 * 1024 * 1024 +_MAX_NOTE_BYTES = 2 * 1024 * 1024 +_MAX_BODY_CHARS = 4_000 +_MAX_LINKS_PER_NOTE = 50 +_MAX_TITLE_CHARS = 120 +_SLUG_RE = re.compile(r"[^a-z0-9]+") + +# `[[Target]]`, `[[Target|alias]]`, `[[Target#heading]]`, `[[Target^block]]`. +_WIKILINK_RE = re.compile(r"\[\[([^\]\[|#^]+)(?:[#^][^\]\[|]*)?(?:\|[^\]\[]*)?\]\]") +# Joplin's internal link form: `[title](:/32-hex-id)`. +_JOPLIN_LINK_RE = re.compile(r"\]\(:/([0-9a-fA-F]{32})\)") + +_MARKDOWN_SUFFIXES = frozenset({".md", ".markdown", ".mdown", ".txt"}) +_NOTES_SUFFIXES = frozenset({".html", ".htm", ".txt", ".md"}) +# Directories a vault carries that are machinery, not notes. +_SKIP_DIRS = frozenset( + {".obsidian", ".trash", ".git", ".stfolder", "_resources", "node_modules"} +) + + +class NoteImportError(RuntimeError): + """Raised when a vault can't be read or doesn't parse as one. + + The CLI translates this into a clean ``Error: ...`` line via + ``_cli_errors``; nothing is written to the KB when it's raised. + """ + + +@dataclass +class Note: + """One imported note, normalised across the five source formats.""" + + key: str + """Stable identity from the origin — relative path, or the vault's own id.""" + + title: str + body: str + raw: bytes + """The note's own bytes, registered verbatim so receipts verify.""" + + locator: str + frontmatter: dict[str, Any] = field(default_factory=dict) + links: list[str] = field(default_factory=list) + tags: list[str] = field(default_factory=list) + created_at: str | None = None + updated_at: str | None = None + media_type: str = "text/markdown" + + +# --- shared parsing -------------------------------------------------------- + + +def _clip(text: str, limit: int = _MAX_BODY_CHARS) -> str: + text = text.strip() + if len(text) <= limit: + return text + return text[: limit - 1].rstrip() + "…" + + +def _title_from(stem: str, frontmatter: dict[str, Any], body: str) -> str: + """Frontmatter title, else the first `# heading`, else the filename.""" + raw = frontmatter.get("title") + if isinstance(raw, str) and raw.strip(): + return raw.strip()[:_MAX_TITLE_CHARS] + for line in body.splitlines(): + stripped = line.strip() + if stripped.startswith("# "): + return stripped[2:].strip()[:_MAX_TITLE_CHARS] or stem + if stripped: + break + return stem[:_MAX_TITLE_CHARS] + + +def split_frontmatter(text: str) -> tuple[dict[str, Any], str]: + """Split a leading ``---`` YAML frontmatter block off a note body. + + Tolerant by design: a malformed or non-mapping block is left in the body + rather than raising, because a vault is other people's files and one bad + note must not fail an import of ten thousand. + """ + if not text.startswith("---"): + return {}, text + lines = text.splitlines(keepends=True) + if not lines or lines[0].strip() != "---": + return {}, text + for idx in range(1, len(lines)): + if lines[idx].strip() in {"---", "..."}: + block = "".join(lines[1:idx]) + try: + loaded = yaml.safe_load(block) + except yaml.YAMLError: + return {}, text + if not isinstance(loaded, dict): + return {}, text + return loaded, "".join(lines[idx + 1 :]) + return {}, text + + +def _tags_from_frontmatter(frontmatter: dict[str, Any]) -> list[str]: + raw = frontmatter.get("tags") or frontmatter.get("tag") + if isinstance(raw, str): + parts = [p.strip() for p in raw.replace(",", " ").split()] + elif isinstance(raw, list): + parts = [str(p).strip() for p in raw] + else: + return [] + return [p.lstrip("#") for p in parts if p and p.strip("#")][:20] + + +def _wikilinks(body: str) -> list[str]: + out: list[str] = [] + seen: set[str] = set() + for match in _WIKILINK_RE.finditer(body): + target = match.group(1).strip() + if not target or target in seen: + continue + seen.add(target) + out.append(target) + if len(out) >= _MAX_LINKS_PER_NOTE: + break + return out + + +def _read_text(path: Path) -> str: + raw = path.read_bytes()[:_MAX_NOTE_BYTES] + return raw.decode("utf-8", errors="replace") + + +def _walk_notes(root: Path, suffixes: frozenset[str]) -> list[Path]: + """Every note file under ``root``, skipping vault machinery, sorted. + + Sorted so an interrupted `--limit` run resumes deterministically instead of + re-walking in filesystem order and picking a different slice. + """ + out: list[Path] = [] + total = 0 + for path in sorted(root.rglob("*")): + # One guard around every stat-backed call: a file that vanishes mid-walk + # is skipped whichever of them notices first. `is_file()` reaches the + # filesystem through `Path.stat` on some python versions and `os.stat` + # on others, so catching only around the explicit `stat()` left the + # skip version-dependent. + try: + if not path.is_file() or path.suffix.lower() not in suffixes: + continue + parents = path.relative_to(root).parts[:-1] + if any(part in _SKIP_DIRS or part.startswith(".") for part in parents): + continue + if path.name.startswith("."): + continue + total += path.stat().st_size + except OSError: + continue + if total > _MAX_VAULT_BYTES: + raise NoteImportError( + f"vault exceeds the {_MAX_VAULT_BYTES // (1024 * 1024)} MiB import " + f"ceiling at {path} — import a subfolder at a time" + ) + out.append(path) + return out + + +# --- obsidian / plain markdown --------------------------------------------- + + +def load_markdown_vault(root: Path, *, kind: str = "obsidian") -> list[Note]: + """Obsidian vaults and plain markdown folders — the same shape. + + Obsidian's on-disk format *is* a folder of markdown with YAML frontmatter + and `[[wikilinks]]`; the only difference from a plain folder is the + `.obsidian/` config directory, which `_walk_notes` skips either way. One + loader, two front doors. + """ + if not root.is_dir(): + raise NoteImportError(f"not a directory: {root}") + notes: list[Note] = [] + for path in _walk_notes(root, _MARKDOWN_SUFFIXES): + text = _read_text(path) + frontmatter, body = split_frontmatter(text) + rel = path.relative_to(root).as_posix() + key = rel[: -len(path.suffix)] if path.suffix else rel + stat = path.stat() + notes.append( + Note( + key=key, + title=_title_from(path.stem, frontmatter, body), + body=body, + raw=path.read_bytes()[:_MAX_NOTE_BYTES], + locator=f"{kind}:{rel}", + frontmatter=frontmatter, + links=_wikilinks(body), + tags=_tags_from_frontmatter(frontmatter), + created_at=_iso(getattr(stat, "st_birthtime", None) or stat.st_mtime), + updated_at=_iso(stat.st_mtime), + ) + ) + if not notes: + raise NoteImportError(f"no markdown notes found under {root}") + return notes + + +def _iso(epoch: Any) -> str | None: + try: + return datetime.fromtimestamp(float(epoch), tz=UTC).isoformat() + except (TypeError, ValueError, OSError): + return None + + +def _iso_ms(millis: Any) -> str | None: + try: + return datetime.fromtimestamp(float(millis) / 1000.0, tz=UTC).isoformat() + except (TypeError, ValueError, OSError): + return None + + +# --- joplin ---------------------------------------------------------------- + + +def _parse_joplin_note(text: str, fallback_key: str) -> Note | None: + """One joplin RAW/JEX note: title line, body, then a `key: value` footer. + + Joplin appends its metadata as trailing `key: value` lines after the last + blank line. `type_: 1` marks a note; folders, tags and resources use other + type codes and are skipped — importing a folder record as a note is the + classic mistake with this format. + """ + lines = text.splitlines() + meta: dict[str, str] = {} + cut = len(lines) + for idx in range(len(lines) - 1, -1, -1): + line = lines[idx] + if not line.strip(): + cut = idx + break + if ":" not in line: + return None + key, _, value = line.partition(":") + if not key.strip() or " " in key.strip(): + return None + meta[key.strip()] = value.strip() + if meta.get("type_") not in {"1", None} or "type_" not in meta: + return None + head = lines[:cut] + title = head[0].strip() if head else "" + body = "\n".join(head[1:]).strip() + note_id = meta.get("id") or fallback_key + return Note( + key=note_id, + title=(title or note_id)[:_MAX_TITLE_CHARS], + body=body, + raw=text.encode("utf-8"), + locator=f"joplin:{note_id}", + frontmatter={ + k: v + for k, v in meta.items() + if k in {"id", "parent_id", "source_url", "author", "is_todo"} + }, + links=list(dict.fromkeys(_JOPLIN_LINK_RE.findall(body)))[:_MAX_LINKS_PER_NOTE], + created_at=meta.get("user_created_time") or meta.get("created_time"), + updated_at=meta.get("user_updated_time") or meta.get("updated_time"), + ) + + +def load_joplin(path: Path) -> list[Note]: + """A joplin export: a `.jex` tar archive, or the folder it unpacks to.""" + notes: list[Note] = [] + if path.is_file(): + if not tarfile.is_tarfile(path): + raise NoteImportError( + f"{path} is not a joplin .jex archive (expected a tar) — pass the " + f"exported folder instead" + ) + with tarfile.open(path) as tar: + total = 0 + for member in tar.getmembers(): + if not member.isfile() or not member.name.endswith(".md"): + continue + total += member.size + if total > _MAX_VAULT_BYTES: + raise NoteImportError( + f"{path} exceeds the " + f"{_MAX_VAULT_BYTES // (1024 * 1024)} MiB import ceiling" + ) + handle = tar.extractfile(member) + if handle is None: # pragma: no cover - isfile() guarantees a stream + continue + text = handle.read(_MAX_NOTE_BYTES).decode("utf-8", errors="replace") + note = _parse_joplin_note(text, Path(member.name).stem) + if note is not None: + notes.append(note) + elif path.is_dir(): + for note_path in _walk_notes(path, frozenset({".md"})): + note = _parse_joplin_note(_read_text(note_path), note_path.stem) + if note is not None: + notes.append(note) + else: + raise NoteImportError(f"no such joplin export: {path}") + if not notes: + raise NoteImportError( + f"no joplin notes found in {path} — a joplin export is a folder of .md " + f"files with a trailing metadata block, or the .jex tar of one" + ) + notes.sort(key=lambda n: n.key) + return notes + + +# --- apple notes ----------------------------------------------------------- + + +class _TextExtractor(HTMLParser): + """Minimal HTML-to-text: apple's exporter emits styled html, not markdown. + + No dependency is added for this — an import must not require a parser the + rest of vouch has no use for. Block-level tags become newlines; everything + else contributes its text. + """ + + _BLOCK = frozenset({"p", "div", "br", "li", "tr", "h1", "h2", "h3", "h4", "h5"}) + + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self.parts: list[str] = [] + self._skip = 0 + + def handle_starttag(self, tag: str, attrs: Any) -> None: + if tag in {"script", "style"}: + self._skip += 1 + elif tag in self._BLOCK: + self.parts.append("\n") + + def handle_endtag(self, tag: str) -> None: + if tag in {"script", "style"} and self._skip: + self._skip -= 1 + elif tag in self._BLOCK: + self.parts.append("\n") + + def handle_data(self, data: str) -> None: + if not self._skip: + self.parts.append(data) + + def text(self) -> str: + joined = "".join(self.parts) + return re.sub(r"\n{3,}", "\n\n", joined).strip() + + +def html_to_text(html: str) -> str: + parser = _TextExtractor() + try: + parser.feed(html) + parser.close() + except Exception: # a malformed note must not fail the import + return re.sub(r"<[^>]+>", " ", html).strip() + return parser.text() + + +def load_apple_notes(root: Path) -> list[Note]: + """An apple notes export: a folder of .html (or .txt/.md) per note.""" + if not root.is_dir(): + raise NoteImportError(f"not a directory: {root}") + notes: list[Note] = [] + for path in _walk_notes(root, _NOTES_SUFFIXES): + raw = path.read_bytes()[:_MAX_NOTE_BYTES] + text = raw.decode("utf-8", errors="replace") + body = html_to_text(text) if path.suffix.lower() in {".html", ".htm"} else text + rel = path.relative_to(root).as_posix() + stat = path.stat() + notes.append( + Note( + key=rel[: -len(path.suffix)] if path.suffix else rel, + title=_title_from(path.stem, {}, body), + body=body, + raw=body.encode("utf-8"), # the extracted text is what claims quote + locator=f"notes:{rel}", + links=_wikilinks(body), + created_at=_iso(getattr(stat, "st_birthtime", None) or stat.st_mtime), + updated_at=_iso(stat.st_mtime), + media_type="text/plain", + ) + ) + if not notes: + raise NoteImportError(f"no apple-notes files found under {root}") + return notes + + +# --- google keep ----------------------------------------------------------- + + +def _keep_note(payload: dict[str, Any], fallback_key: str) -> Note | None: + if payload.get("isTrashed"): + return None + title = str(payload.get("title") or "").strip() + text = str(payload.get("textContent") or "").strip() + checklist = payload.get("listContent") + if isinstance(checklist, list): + rows = [ + f"- [{'x' if item.get('isChecked') else ' '}] {item.get('text', '')}" + for item in checklist + if isinstance(item, dict) + ] + text = "\n".join(filter(None, [text, *rows])).strip() + if not title and not text: + return None + labels = payload.get("labels") + tags = [ + str(item["name"]) + for item in labels or [] + if isinstance(item, dict) and item.get("name") + ][:20] + body = text + return Note( + key=fallback_key, + title=(title or fallback_key)[:_MAX_TITLE_CHARS], + body=body, + raw=body.encode("utf-8"), + locator=f"keep:{fallback_key}", + frontmatter={ + k: payload[k] + for k in ("isPinned", "isArchived", "color") + if k in payload + }, + tags=tags, + created_at=_iso_ms(payload.get("createdTimestampUsec", 0) / 1000) + if isinstance(payload.get("createdTimestampUsec"), (int, float)) + else None, + updated_at=_iso_ms(payload.get("userEditedTimestampUsec", 0) / 1000) + if isinstance(payload.get("userEditedTimestampUsec"), (int, float)) + else None, + media_type="text/plain", + ) + + +def load_google_keep(path: Path) -> list[Note]: + """A google takeout keep export: `.json` per note, or the zip.""" + notes: list[Note] = [] + if path.is_file() and zipfile.is_zipfile(path): + with zipfile.ZipFile(path) as zf: + total = 0 + for info in sorted(zf.infolist(), key=lambda i: i.filename): + if info.is_dir() or not info.filename.lower().endswith(".json"): + continue + if "/Keep/" not in f"/{info.filename}" and "keep" not in info.filename.lower(): + continue + total += info.file_size + if total > _MAX_VAULT_BYTES: + raise NoteImportError( + f"{path} exceeds the " + f"{_MAX_VAULT_BYTES // (1024 * 1024)} MiB import ceiling" + ) + try: + payload = json.loads(zf.read(info).decode("utf-8", errors="replace")) + except (json.JSONDecodeError, OSError): + continue + if isinstance(payload, dict): + note = _keep_note(payload, Path(info.filename).stem) + if note is not None: + notes.append(note) + elif path.is_dir(): + for json_path in _walk_notes(path, frozenset({".json"})): + try: + payload = json.loads(_read_text(json_path)) + except json.JSONDecodeError: + continue + if isinstance(payload, dict): + note = _keep_note(payload, json_path.stem) + if note is not None: + notes.append(note) + else: + raise NoteImportError(f"no such keep export: {path}") + if not notes: + raise NoteImportError( + f"no google keep notes found in {path} — a takeout keep export is a " + f"folder of one .json per note (or the takeout .zip holding it)" + ) + notes.sort(key=lambda n: n.key) + return notes + + +# --- vault -> proposals ---------------------------------------------------- + + +_LOADERS = { + "obsidian": lambda p: load_markdown_vault(p, kind="obsidian"), + "md": lambda p: load_markdown_vault(p, kind="md"), + "joplin": load_joplin, + "notes": load_apple_notes, + "keep": load_google_keep, +} + + +def load_vault(kind: str, path: Path) -> list[Note]: + """Parse a vault of ``kind`` into normalised notes. No KB writes.""" + loader = _LOADERS.get(kind) + if loader is None: + raise NoteImportError( + f"unknown vault kind {kind!r} — one of {', '.join(KINDS)}" + ) + if not path.exists(): + raise NoteImportError(f"no such path: {path}") + return loader(path) + + +def session_key(kind: str, note_key: str) -> str: + """The session id one note dedups on, across every re-import. + + Derived from the origin's own identifier, hashed only to bound the length — + a deep vault path or a keep title can be arbitrarily long, and the session + id is an index key, not a display string. The readable identifier survives + on the source ``locator`` and in the page body. + """ + digest = hashlib.sha256(f"{kind}:{note_key}".encode()).hexdigest()[:16] + return f"note-{kind}-{digest}" + + +def _page_slug(kind: str, note_key: str) -> str: + slug = _SLUG_RE.sub("-", note_key.lower()).strip("-") or "note" + return f"{kind}-{slug}"[:80] + + +def build_page_body( + note: Note, *, kind: str, generated_at: str | None = None +) -> str: + """The review surface for one note: provenance, then the note itself.""" + stamp = generated_at or datetime.now(UTC).isoformat() + lines = [ + f"# {note.title}", + "", + f"- imported: {stamp}", + f"- from: {kind}", + f"- note: {note.key}", + ] + if note.created_at: + lines.append(f"- created: {note.created_at}") + if note.updated_at: + lines.append(f"- updated: {note.updated_at}") + if note.tags: + lines.append(f"- tags: {', '.join(note.tags)}") + if note.links: + lines.append(f"- links: {', '.join(note.links[:20])}") + extra = { + k: v for k, v in note.frontmatter.items() if k not in {"title", "tags", "tag"} + } + if extra: + lines.extend(["", "## frontmatter", ""]) + lines.extend(f"- {k}: {v}" for k, v in sorted(extra.items())[:30]) + lines.extend(["", "## note", "", _clip(note.body) or "(empty note)"]) + return "\n".join(lines).rstrip() + "\n" + + +def _comparable_body(body: str) -> str: + """The page body minus its import timestamp, so re-importing an unchanged + note compares equal across runs.""" + return "\n".join( + line for line in body.splitlines() if not line.startswith("- imported:") + ) + + +def _find_existing_page(store: KBStore, session_id: str) -> Proposal | None: + """The page proposal (any status) already filed for this note. + + Filtered to PAGE kind for the same reason `chatgpt_import` filters: an + approved import can spawn follow-on proposals under the same session id + (the wikilink relations, for one), and those must never be mistaken for the + page when a re-import looks for its dedup target. + """ + for proposal in store.list_proposals(None): + if proposal.kind == ProposalKind.PAGE and proposal.session_id == session_id: + return proposal + return None + + +def _resolve_link(target: str, by_key: dict[str, str], by_title: dict[str, str]) -> str | None: + """A wikilink target -> the imported note's key, or None if unresolvable. + + Obsidian resolves `[[Note]]` by basename anywhere in the vault, and by full + path when one is given; joplin links by id. Titles are matched + case-insensitively and only when unambiguous — a link that could mean two + notes is dropped rather than guessed at. + """ + if target in by_key: + return by_key[target] + stripped = target.removesuffix(".md") + if stripped in by_key: + return by_key[stripped] + return by_title.get(stripped.lower()) + + +def import_vault( + store: KBStore, + kind: str, + path: Path, + *, + actor: str | None = None, + limit: int | None = None, + max_claims: int = 0, + dry_run: bool = False, + generated_at: str | None = None, +) -> dict[str, Any]: + """Import one note vault into PENDING proposals. Never calls ``approve()``. + + One page proposal per note, deduped on a stable per-note session id: a + decided proposal blocks re-import, a still-PENDING one refreshes in place + when the note changed, and an unchanged note is a no-op. ``limit`` bounds + how many notes are considered, in sorted order, so a large vault imports in + resumable slices. + + ``max_claims`` > 0 additionally files that many receipt-backed claims per + note through the density-selection knob. It defaults to 0 — pages only — + because a ten-thousand-note vault must not become ten thousand pending + claims on someone's first command. + + With ``dry_run`` nothing is written; the report shows what a real run would + file. + """ + notes = load_vault(kind, path) + if limit is not None and limit >= 0: + notes = notes[:limit] + resolved_actor = actor or os.environ.get("VOUCH_AGENT") or NOTE_IMPORT_ACTOR + + rows: list[dict[str, Any]] = [] + counts = {"imported": 0, "updated": 0, "skipped": 0} + claims_filed = 0 + # note key -> registered source id, for the wikilink pass. Only notes that + # actually got a source land here, so a link into a `--limit`-truncated + # slice resolves to nothing and is dropped rather than filed dangling. + source_by_key: dict[str, str] = {} + by_key = {note.key: note.key for note in notes} + titles: dict[str, list[str]] = {} + for note in notes: + titles.setdefault(note.title.lower(), []).append(note.key) + by_title = {title: keys[0] for title, keys in titles.items() if len(keys) == 1} + + for note in notes: + sid = session_key(kind, note.key) + row: dict[str, Any] = {"note": note.key, "title": note.title, "session_id": sid} + existing = _find_existing_page(store, sid) + if existing is not None and existing.status != ProposalStatus.PENDING: + row.update(action="skipped", reason="already-imported", proposal_id=existing.id) + counts["skipped"] += 1 + rows.append(row) + continue + body = build_page_body(note, kind=kind, generated_at=generated_at) + if existing is not None and _comparable_body(body) == _comparable_body( + str(existing.payload.get("body", "")) + ): + row.update(action="skipped", reason="unchanged", proposal_id=existing.id) + counts["skipped"] += 1 + rows.append(row) + continue + if dry_run: + action = "updated" if existing is not None else "imported" + row.update(action=action, dry_run=True) + counts[action] += 1 + rows.append(row) + continue + + source = store.put_source( + note.raw, + title=note.title, + locator=note.locator, + media_type=note.media_type, + tags=[f"{kind}-import", "note-import"], + metadata={ + "note_key": note.key, + "import_kind": kind, + **({"frontmatter": note.frontmatter} if note.frontmatter else {}), + }, + scope=default_scope(store), + ) + source_by_key[note.key] = source.id + + if existing is not None: + refreshed = existing.model_copy(deep=True) + refreshed.payload["title"] = note.title + refreshed.payload["body"] = body + refreshed.payload["sources"] = [source.id] + store.update_proposal(refreshed) + audit.log_event( + store.kb_dir, + event="proposal.page.update", + actor=resolved_actor, + object_ids=[existing.id], + data={"reason": f"{kind} re-import", "note": note.key}, + ) + row.update(action="updated", proposal_id=existing.id) + counts["updated"] += 1 + else: + proposal = propose_page( + store, + title=note.title, + body=body, + page_type=PAGE_TYPE, + source_ids=[source.id], + proposed_by=resolved_actor, + tags=[f"{kind}-import", *note.tags], + session_id=sid, + slug_hint=_page_slug(kind, note.key), + rationale=f"imported {kind} note", + ) + row.update(action="imported", proposal_id=proposal.id) + counts["imported"] += 1 + + if max_claims > 0: + filed = extract_receipt_claims( + store, source.id, proposed_by=resolved_actor, max_claims=max_claims, + ) + row["claims"] = len(filed) + claims_filed += len(filed) + rows.append(row) + + relations = 0 + if not dry_run: + relations = _propose_links( + store, notes, source_by_key, by_key, by_title, actor=resolved_actor, kind=kind + ) + + return { + "kind": kind, + "notes": len(notes), + "imported": counts["imported"], + "updated": counts["updated"], + "skipped": counts["skipped"], + "claims": claims_filed, + "relations": relations, + "dry_run": dry_run, + "rows": rows, + } + + +def _propose_links( + store: KBStore, + notes: list[Note], + source_by_key: dict[str, str], + by_key: dict[str, str], + by_title: dict[str, str], + *, + actor: str, + kind: str, +) -> int: + """File a `references` relation per resolvable wikilink. Returns the count. + + Runs after every source is registered, so a link is judged against the + whole imported set rather than against whatever happened to come first in + the walk. Unresolvable targets are dropped: a vault is full of links to + notes that were never written, and a dangling edge is not knowledge. + """ + filed = 0 + seen: set[tuple[str, str]] = set() + for note in notes: + src_id = source_by_key.get(note.key) + if src_id is None: + continue + for target in note.links: + target_key = _resolve_link(target, by_key, by_title) + if target_key is None or target_key == note.key: + continue + target_id = source_by_key.get(target_key) + if target_id is None or (src_id, target_id) in seen: + continue + seen.add((src_id, target_id)) + try: + propose_relation( + store, + src=src_id, + relation="references", + target=target_id, + proposed_by=actor, + rationale=f"{kind} link: {note.key} -> {target_key}", + session_id=session_key(kind, note.key), + ) + except ProposalError: + # A duplicate or otherwise-rejected edge is not worth failing + # a ten-thousand-note import over. + continue + filed += 1 + return filed diff --git a/tests/test_note_import.py b/tests/test_note_import.py new file mode 100644 index 00000000..f844396a --- /dev/null +++ b/tests/test_note_import.py @@ -0,0 +1,758 @@ +"""Note-vault importers: obsidian, joplin, apple notes, keep, markdown (#612).""" + +from __future__ import annotations + +import json +import tarfile +import zipfile +from pathlib import Path + +import pytest +from click.testing import CliRunner + +from vouch import note_import +from vouch.cli import cli +from vouch.models import ProposalKind, ProposalStatus +from vouch.note_import import NoteImportError, import_vault, load_vault, session_key +from vouch.storage import KBStore + + +@pytest.fixture +def store(tmp_path: Path) -> KBStore: + return KBStore.init(tmp_path / "kb") + + +@pytest.fixture +def vault(tmp_path: Path) -> Path: + """A small obsidian vault: frontmatter, a wikilink, a subfolder, machinery.""" + root = tmp_path / "vault" + (root / "notes").mkdir(parents=True) + (root / ".obsidian").mkdir() + (root / ".obsidian" / "app.json").write_text("{}", encoding="utf-8") + (root / "Postgres.md").write_text( + "---\ntitle: Postgres tuning\ntags: [db, ops]\nowner: alice-example\n---\n" + "The connection pool must be sized to the worker count, never higher.\n" + "See [[Deploys]] for how this rolls out.\n", + encoding="utf-8", + ) + (root / "notes" / "Deploys.md").write_text( + "# Deploys\n\nDeploys run every second Tuesday from the release branch.\n", + encoding="utf-8", + ) + (root / "notes" / "Orphan.md").write_text( + "A note linking to [[Nothing At All]] which was never written.\n", + encoding="utf-8", + ) + return root + + +# --- parsing --------------------------------------------------------------- + + +def test_split_frontmatter_reads_a_mapping() -> None: + fm, body = note_import.split_frontmatter("---\na: 1\nb: two\n---\nbody here\n") + assert fm == {"a": 1, "b": "two"} + assert body.strip() == "body here" + + +def test_split_frontmatter_leaves_malformed_blocks_alone() -> None: + # A vault is other people's files: one bad note must not fail the import. + text = "---\n: : :\n - broken\n---\nbody\n" + fm, body = note_import.split_frontmatter(text) + assert fm == {} + assert body == text + + +def test_split_frontmatter_ignores_a_non_mapping_block() -> None: + fm, body = note_import.split_frontmatter("---\n- a\n- b\n---\nbody\n") + assert fm == {} + assert "- a" in body + + +def test_obsidian_loader_reads_titles_tags_and_links(vault: Path) -> None: + notes = {n.key: n for n in load_vault("obsidian", vault)} + assert set(notes) == {"Postgres", "notes/Deploys", "notes/Orphan"} + pg = notes["Postgres"] + assert pg.title == "Postgres tuning" # frontmatter wins + assert pg.tags == ["db", "ops"] + assert pg.links == ["Deploys"] + assert pg.locator == "obsidian:Postgres.md" + assert notes["notes/Deploys"].title == "Deploys" # falls back to the heading + # `.obsidian/` is machinery, not notes + assert not any(k.startswith(".obsidian") for k in notes) + + +def test_wikilink_variants_are_normalised() -> None: + links = note_import._wikilinks( + "[[Plain]] [[Aliased|shown as this]] [[Deep#heading]] [[Block^ref]] [[Plain]]" + ) + assert links == ["Plain", "Aliased", "Deep", "Block"] + + +def test_loader_rejects_an_unknown_kind(vault: Path) -> None: + with pytest.raises(NoteImportError, match="unknown vault kind"): + load_vault("evernote", vault) + + +def test_loader_rejects_a_missing_path(tmp_path: Path) -> None: + with pytest.raises(NoteImportError, match="no such path"): + load_vault("obsidian", tmp_path / "nope") + + +def test_empty_folder_is_an_actionable_error(tmp_path: Path) -> None: + empty = tmp_path / "empty" + empty.mkdir() + with pytest.raises(NoteImportError, match="no markdown notes"): + load_vault("md", empty) + + +# --- import ---------------------------------------------------------------- + + +def test_import_files_one_pending_page_per_note(store: KBStore, vault: Path) -> None: + report = import_vault(store, "obsidian", vault, generated_at="2026-07-31T00:00:00Z") + assert report["notes"] == 3 + assert report["imported"] == 3 + pages = [p for p in store.list_proposals(None) if p.kind == ProposalKind.PAGE] + assert len(pages) == 3 + assert all(p.status is ProposalStatus.PENDING for p in pages) + titles = {p.payload["title"] for p in pages} + assert "Postgres tuning" in titles + + +def test_page_cites_a_source_holding_the_note_verbatim( + store: KBStore, vault: Path +) -> None: + # The whole advantage over an embedding importer: the source is the note's + # own bytes, so a claim extracted from it quotes real offsets. + import_vault(store, "obsidian", vault) + page = next( + p for p in store.list_proposals(None) + if p.kind == ProposalKind.PAGE and p.payload["title"] == "Postgres tuning" + ) + (source_id,) = page.payload["sources"] + content = store.read_source_content(source_id).decode("utf-8") + assert "The connection pool must be sized to the worker count" in content + source = store.get_source(source_id) + assert source.locator == "obsidian:Postgres.md" + assert source.metadata["note_key"] == "Postgres" + assert source.metadata["frontmatter"]["owner"] == "alice-example" + + +def test_reimport_of_an_unchanged_vault_is_a_no_op(store: KBStore, vault: Path) -> None: + first = import_vault(store, "obsidian", vault, generated_at="2026-07-31T00:00:00Z") + assert first["imported"] == 3 + second = import_vault(store, "obsidian", vault, generated_at="2026-08-01T00:00:00Z") + assert second["imported"] == 0 + assert second["skipped"] == 3 + assert all(r["reason"] == "unchanged" for r in second["rows"]) + pages = [p for p in store.list_proposals(None) if p.kind == ProposalKind.PAGE] + assert len(pages) == 3 # nothing duplicated + + +def test_a_changed_note_refreshes_its_pending_proposal_in_place( + store: KBStore, vault: Path +) -> None: + import_vault(store, "obsidian", vault) + before = [p for p in store.list_proposals(None) if p.kind == ProposalKind.PAGE] + (vault / "notes" / "Deploys.md").write_text( + "# Deploys\n\nDeploys now run weekly, on Thursdays.\n", encoding="utf-8" + ) + report = import_vault(store, "obsidian", vault) + assert report["updated"] == 1 + assert report["skipped"] == 2 + after = [p for p in store.list_proposals(None) if p.kind == ProposalKind.PAGE] + assert len(after) == len(before) # refreshed, not re-filed + refreshed = next(p for p in after if p.payload["title"] == "Deploys") + assert "weekly, on Thursdays" in refreshed.payload["body"] + + +def test_a_decided_proposal_blocks_reimport(store: KBStore, vault: Path) -> None: + import_vault(store, "obsidian", vault) + page = next( + p for p in store.list_proposals(None) + if p.kind == ProposalKind.PAGE and p.payload["title"] == "Deploys" + ) + from vouch.proposals import reject + + reject(store, page.id, rejected_by="reviewer-example", reason="not wanted") + (vault / "notes" / "Deploys.md").write_text("# Deploys\n\nchanged\n", encoding="utf-8") + report = import_vault(store, "obsidian", vault) + row = next(r for r in report["rows"] if r["note"] == "notes/Deploys") + assert row["action"] == "skipped" + assert row["reason"] == "already-imported" + + +def test_limit_slices_the_vault_deterministically(store: KBStore, vault: Path) -> None: + first = import_vault(store, "obsidian", vault, limit=2) + assert first["imported"] == 2 + # The remaining note lands on the next run, and the first two are no-ops — + # which is what makes a big vault resumable. + second = import_vault(store, "obsidian", vault) + assert second["imported"] == 1 + assert second["skipped"] == 2 + + +def test_dry_run_writes_nothing(store: KBStore, vault: Path) -> None: + report = import_vault(store, "obsidian", vault, dry_run=True) + assert report["imported"] == 3 + assert report["dry_run"] is True + assert store.list_proposals(None) == [] + assert not list(store.list_sources()) + + +def test_wikilinks_become_relation_proposals_only_where_they_resolve( + store: KBStore, vault: Path +) -> None: + report = import_vault(store, "obsidian", vault) + assert report["relations"] == 1 # Postgres -> Deploys; the orphan link is dropped + rels = [p for p in store.list_proposals(None) if p.kind == ProposalKind.RELATION] + assert len(rels) == 1 + payload = rels[0].payload + src = store.get_source(payload["source"]) + target = store.get_source(payload["target"]) + assert src.locator == "obsidian:Postgres.md" + assert target.locator == "obsidian:notes/Deploys.md" + assert payload["relation"] == "references" + + +def test_claims_are_off_by_default_and_bounded_when_on( + store: KBStore, vault: Path +) -> None: + off = import_vault(store, "obsidian", vault) + assert off["claims"] == 0 + assert not [p for p in store.list_proposals(None) if p.kind == ProposalKind.CLAIM] + + fresh = KBStore.init(vault.parent / "kb2") + on = import_vault(fresh, "obsidian", vault, max_claims=1) + assert on["claims"] > 0 + claims = [p for p in fresh.list_proposals(None) if p.kind == ProposalKind.CLAIM] + assert claims + assert all(p.status is ProposalStatus.PENDING for p in claims) # gate intact + # `--max-claims` is a per-note ceiling: 3 notes, at most 1 claim each. + assert len(claims) <= 3 + + +def test_session_key_is_stable_and_kind_scoped() -> None: + assert session_key("obsidian", "a/b") == session_key("obsidian", "a/b") + assert session_key("obsidian", "a/b") != session_key("md", "a/b") + + +# --- the other four formats ------------------------------------------------ + + +def _joplin_note(note_id: str, title: str, body: str, extra: str = "") -> str: + return ( + f"{title}\n\n{body}\n\n" + f"id: {note_id}\n" + f"parent_id: 0123456789abcdef0123456789abcdef\n" + f"created_time: 2026-01-01T00:00:00.000Z\n" + f"{extra}" + f"type_: 1" + ) + + +def test_joplin_folder_import(store: KBStore, tmp_path: Path) -> None: + export = tmp_path / "joplin" + export.mkdir() + a_id = "a" * 32 + b_id = "b" * 32 + (export / f"{a_id}.md").write_text( + _joplin_note(a_id, "Runbook", f"Restart order matters. See [link](:/{b_id})."), + encoding="utf-8", + ) + (export / f"{b_id}.md").write_text( + _joplin_note(b_id, "Escalation", "Page the on-call after ten minutes."), + encoding="utf-8", + ) + # A folder record must not be imported as a note. + (export / "folder.md").write_text( + "Notebook\n\nid: c0ffee00000000000000000000000000\ntype_: 2", encoding="utf-8" + ) + report = import_vault(store, "joplin", export) + assert report["notes"] == 2 + assert report["imported"] == 2 + assert report["relations"] == 1 # the `:/id` link resolves + titles = { + p.payload["title"] + for p in store.list_proposals(None) + if p.kind == ProposalKind.PAGE + } + assert titles == {"Runbook", "Escalation"} + + +def test_joplin_jex_archive_import(store: KBStore, tmp_path: Path) -> None: + note_path = tmp_path / f"{'d' * 32}.md" + note_path.write_text(_joplin_note("d" * 32, "From a jex", "Body text."), "utf-8") + jex = tmp_path / "export.jex" + with tarfile.open(jex, "w") as tar: + tar.add(note_path, arcname=note_path.name) + report = import_vault(store, "joplin", jex) + assert report["imported"] == 1 + + +def test_joplin_rejects_a_non_archive_file(store: KBStore, tmp_path: Path) -> None: + bogus = tmp_path / "notes.jex" + bogus.write_text("not a tar", encoding="utf-8") + with pytest.raises(NoteImportError, match=r"not a joplin \.jex archive"): + import_vault(store, "joplin", bogus) + + +def test_apple_notes_html_import(store: KBStore, tmp_path: Path) -> None: + export = tmp_path / "notes-export" + export.mkdir() + (export / "Grocery.html").write_text( + "<html><head><style>b{}</style></head><body>" + "<h1>Grocery</h1><div>Oat milk</div><div>Coffee beans</div>" + "<script>ignored()</script></body></html>", + encoding="utf-8", + ) + report = import_vault(store, "notes", export) + assert report["imported"] == 1 + page = next( + p for p in store.list_proposals(None) if p.kind == ProposalKind.PAGE + ) + assert page.payload["title"] == "Grocery" + body = page.payload["body"] + assert "Oat milk" in body and "Coffee beans" in body + assert "ignored()" not in body # script/style are dropped, not read as text + + +def test_html_to_text_survives_malformed_markup() -> None: + assert "hello" in note_import.html_to_text("<p>hello<<</p") + + +def test_google_keep_folder_import(store: KBStore, tmp_path: Path) -> None: + export = tmp_path / "Keep" + export.mkdir() + (export / "Shopping.json").write_text(json.dumps({ + "title": "Shopping", + "textContent": "for the weekend", + "listContent": [ + {"text": "bread", "isChecked": False}, + {"text": "milk", "isChecked": True}, + ], + "labels": [{"name": "errands"}], + "userEditedTimestampUsec": 1767225600000000, + }), encoding="utf-8") + (export / "Trashed.json").write_text( + json.dumps({"title": "Old", "textContent": "x", "isTrashed": True}), + encoding="utf-8", + ) + report = import_vault(store, "keep", export) + assert report["imported"] == 1 # the trashed note is skipped + page = next(p for p in store.list_proposals(None) if p.kind == ProposalKind.PAGE) + assert page.payload["title"] == "Shopping" + assert "- [ ] bread" in page.payload["body"] + assert "- [x] milk" in page.payload["body"] + + +def test_google_keep_takeout_zip_import(store: KBStore, tmp_path: Path) -> None: + archive = tmp_path / "takeout.zip" + with zipfile.ZipFile(archive, "w") as zf: + zf.writestr( + "Takeout/Keep/Idea.json", + json.dumps({"title": "Idea", "textContent": "ship the importer"}), + ) + zf.writestr("Takeout/Mail/ignored.json", json.dumps({"title": "no"})) + report = import_vault(store, "keep", archive) + assert report["imported"] == 1 + + +def test_markdown_folder_import(store: KBStore, tmp_path: Path) -> None: + folder = tmp_path / "docs" + folder.mkdir() + (folder / "one.md").write_text("# One\n\nfirst\n", encoding="utf-8") + (folder / "two.txt").write_text("plain text note\n", encoding="utf-8") + report = import_vault(store, "md", folder) + assert report["notes"] == 2 + assert report["kind"] == "md" + + +# --- cli ------------------------------------------------------------------- + + +def _run(store: KBStore, args: list[str]): + return CliRunner().invoke(cli, args, env={"VOUCH_KB_PATH": str(store.kb_dir)}) + + +def test_cli_import_obsidian(store: KBStore, vault: Path) -> None: + result = _run(store, ["import", "obsidian", str(vault), "--json"]) + assert result.exit_code == 0, result.output + report = json.loads(result.output) + assert report["imported"] == 3 + assert report["kind"] == "obsidian" + + +def test_cli_import_dry_run_reports_without_writing( + store: KBStore, vault: Path +) -> None: + result = _run(store, ["import", "obsidian", str(vault), "--dry-run"]) + assert result.exit_code == 0, result.output + assert "would import 3 new" in result.output + assert store.list_proposals(None) == [] + + +def test_cli_import_max_claims_is_forwarded(store: KBStore, vault: Path) -> None: + result = _run( + store, ["import", "md", str(vault), "--max-claims", "1", "--json"] + ) + assert result.exit_code == 0, result.output + assert json.loads(result.output)["claims"] > 0 + + +def test_cli_import_group_lists_every_kind() -> None: + result = CliRunner().invoke(cli, ["import", "--help"]) + assert result.exit_code == 0 + for kind in ("obsidian", "joplin", "notes", "keep", "md", "chatgpt"): + assert kind in result.output + + +def test_cli_import_reports_a_bad_vault_cleanly(store: KBStore, tmp_path: Path) -> None: + empty = tmp_path / "empty" + empty.mkdir() + result = _run(store, ["import", "obsidian", str(empty)]) + assert result.exit_code != 0 + assert "no markdown notes" in result.output + assert "Traceback" not in result.output + + +def test_cli_human_output_reports_claims_and_links( + store: KBStore, vault: Path +) -> None: + result = _run(store, ["import", "obsidian", str(vault), "--max-claims", "1"]) + assert result.exit_code == 0, result.output + assert "claim(s)" in result.output + assert "link relation(s) proposed" in result.output + assert "run `vouch review` to decide." in result.output + # skipped rows are counted, not listed + again = _run(store, ["import", "obsidian", str(vault)]) + assert "3 skipped" in again.output + assert "•" not in again.output + + +def test_cli_each_kind_reaches_its_loader(store: KBStore, tmp_path: Path) -> None: + """One CLI test per subcommand — the group is the surface #612 asks for, + and a subcommand wired to the wrong kind would be invisible otherwise.""" + joplin = tmp_path / "joplin" + joplin.mkdir() + (joplin / f"{'e' * 32}.md").write_text( + _joplin_note("e" * 32, "Joplin note", "body"), encoding="utf-8" + ) + notes = tmp_path / "apple" + notes.mkdir() + (notes / "Note.txt").write_text("apple note body", encoding="utf-8") + keep = tmp_path / "Keep" + keep.mkdir() + (keep / "Keep note.json").write_text( + json.dumps({"title": "Keep note", "textContent": "keep body"}), encoding="utf-8" + ) + md = tmp_path / "docs" + md.mkdir() + (md / "doc.md").write_text("# Doc\n\nbody\n", encoding="utf-8") + + for kind, path in ( + ("joplin", joplin), ("notes", notes), ("keep", keep), ("md", md) + ): + result = _run(store, ["import", kind, str(path), "--json"]) + assert result.exit_code == 0, (kind, result.output) + report = json.loads(result.output) + assert report["kind"] == kind + assert report["imported"] == 1 + + +def test_cli_import_chatgpt_alias(store: KBStore, tmp_path: Path) -> None: + """`vouch import chatgpt` is the same importer as the flat command — one + `vouch import <kind>` surface over every source.""" + export = tmp_path / "conversations.json" + export.write_text(json.dumps([{ + "conversation_id": "conv-1", + "title": "About deploys", + "mapping": { + "root": {"id": "root", "message": None, "parent": None, "children": ["u1"]}, + "u1": { + "id": "u1", "parent": "root", "children": ["a1"], + "message": { + "author": {"role": "user"}, + "content": {"content_type": "text", "parts": ["when do we deploy?"]}, + }, + }, + "a1": { + "id": "a1", "parent": "u1", "children": [], + "message": { + "author": {"role": "assistant"}, + "content": { + "content_type": "text", + "parts": ["Every second Tuesday from the release branch."], + }, + }, + }, + }, + }]), encoding="utf-8") + result = _run(store, ["import", "chatgpt", str(export), "--json"]) + assert result.exit_code == 0, result.output + assert json.loads(result.output)["imported"] == 1 + + +# --- the tolerant paths ---------------------------------------------------- +# +# A vault is other people's files. Every branch below is a "skip it and carry +# on" that exists so one malformed note cannot fail an import of ten thousand. + + +def test_clip_truncates_a_long_body(store: KBStore, tmp_path: Path) -> None: + folder = tmp_path / "long" + folder.mkdir() + (folder / "big.md").write_text("x" * 10_000, encoding="utf-8") + import_vault(store, "md", folder) + page = next(p for p in store.list_proposals(None) if p.kind == ProposalKind.PAGE) + assert page.payload["body"].rstrip().endswith("…") + + +def test_frontmatter_needs_a_delimiter_line_and_a_terminator() -> None: + # `---title: x` starts with `---` but is not a fence + assert note_import.split_frontmatter("---title: x\nbody\n")[0] == {} + # opened and never closed + assert note_import.split_frontmatter("---\na: 1\nbody with no terminator\n")[0] == {} + + +def test_frontmatter_tags_accept_a_string_list() -> None: + fm, _ = note_import.split_frontmatter('---\ntags: "db, ops #infra"\n---\nbody\n') + assert note_import._tags_from_frontmatter(fm) == ["db", "ops", "infra"] + assert note_import._tags_from_frontmatter({"tags": 7}) == [] + + +def test_wikilinks_stop_at_the_per_note_ceiling() -> None: + body = " ".join(f"[[Note{i}]]" for i in range(note_import._MAX_LINKS_PER_NOTE + 10)) + assert len(note_import._wikilinks(body)) == note_import._MAX_LINKS_PER_NOTE + + +def test_walk_skips_machinery_and_dotfiles(store: KBStore, tmp_path: Path) -> None: + root = tmp_path / "vault2" + (root / "_resources").mkdir(parents=True) + (root / "_resources" / "attached.md").write_text("attachment", encoding="utf-8") + (root / ".hidden.md").write_text("hidden", encoding="utf-8") + (root / "Real.md").write_text("# Real\n\nkept\n", encoding="utf-8") + keys = {n.key for n in load_vault("md", root)} + assert keys == {"Real"} + + +def test_walk_skips_a_file_it_cannot_stat( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + root = tmp_path / "racy" + root.mkdir() + (root / "gone.md").write_text("body", encoding="utf-8") + (root / "kept.md").write_text("# Kept\n\nbody\n", encoding="utf-8") + real_stat = Path.stat + + def flaky(self: Path, *a: object, **kw: object): + if self.name == "gone.md": + raise OSError("vanished mid-walk") + return real_stat(self, *a, **kw) + + monkeypatch.setattr(Path, "stat", flaky) + assert {n.key for n in load_vault("md", root)} == {"kept"} + + +def test_vault_over_the_byte_ceiling_is_refused( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(note_import, "_MAX_VAULT_BYTES", 4) + root = tmp_path / "huge" + root.mkdir() + (root / "a.md").write_text("well over four bytes", encoding="utf-8") + with pytest.raises(NoteImportError, match="import ceiling"): + load_vault("md", root) + + +def test_markdown_loader_rejects_a_file(tmp_path: Path) -> None: + target = tmp_path / "one.md" + target.write_text("body", encoding="utf-8") + with pytest.raises(NoteImportError, match="not a directory"): + load_vault("md", target) + + +def test_timestamp_helpers_degrade_instead_of_raising() -> None: + assert note_import._iso("not-a-number") is None + assert note_import._iso_ms(object()) is None + + +def test_joplin_skips_notes_whose_footer_is_not_metadata(tmp_path: Path) -> None: + export = tmp_path / "joplin" + export.mkdir() + # trailing block with a line that carries no colon + (export / "a.md").write_text("Title\n\nbody\n\njust prose\n", encoding="utf-8") + # trailing block whose "key" has a space in it — prose, not metadata + (export / "b.md").write_text("Title\n\nbody\n\nnot a key: value\n", encoding="utf-8") + with pytest.raises(NoteImportError, match="no joplin notes"): + load_vault("joplin", export) + + +def test_joplin_missing_path_is_actionable(tmp_path: Path) -> None: + with pytest.raises(NoteImportError, match="no such path"): + load_vault("joplin", tmp_path / "nope") + # the loader guards for itself too — it is importable on its own + with pytest.raises(NoteImportError, match="no such joplin export"): + note_import.load_joplin(tmp_path / "nope") + + +def test_joplin_jex_skips_non_note_members(tmp_path: Path) -> None: + note = tmp_path / f"{'f' * 32}.md" + note.write_text(_joplin_note("f" * 32, "Kept", "body"), encoding="utf-8") + resource = tmp_path / "resource.bin" + resource.write_bytes(b"attachment") + inner = tmp_path / "subdir" + inner.mkdir() + jex = tmp_path / "mixed.jex" + with tarfile.open(jex, "w") as tar: + tar.add(note, arcname=note.name) + tar.add(resource, arcname=resource.name) + # a *directory* whose name ends in .md clears the suffix filter but has + # no readable stream — extractfile returns None + tar.add(inner, arcname="looks-like-a-note.md") + notes = load_vault("joplin", jex) + assert [n.title for n in notes] == ["Kept"] + + +def test_joplin_jex_over_the_ceiling_is_refused( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(note_import, "_MAX_VAULT_BYTES", 4) + note = tmp_path / f"{'g' * 32}.md" + note.write_text(_joplin_note("g" * 32, "Big", "body"), encoding="utf-8") + jex = tmp_path / "big.jex" + with tarfile.open(jex, "w") as tar: + tar.add(note, arcname=note.name) + with pytest.raises(NoteImportError, match="import ceiling"): + load_vault("joplin", jex) + + +def test_html_to_text_falls_back_when_the_parser_blows_up( + monkeypatch: pytest.MonkeyPatch +) -> None: + def boom(self: object, data: str) -> None: + raise RuntimeError("parser exploded") + + monkeypatch.setattr(note_import._TextExtractor, "feed", boom) + assert note_import.html_to_text("<p>still readable</p>") == "still readable" + + +def test_apple_notes_loader_rejects_a_file_and_an_empty_folder( + tmp_path: Path +) -> None: + target = tmp_path / "one.html" + target.write_text("<p>x</p>", encoding="utf-8") + with pytest.raises(NoteImportError, match="not a directory"): + load_vault("notes", target) + empty = tmp_path / "no-notes" + empty.mkdir() + with pytest.raises(NoteImportError, match="no apple-notes files"): + load_vault("notes", empty) + + +def test_keep_skips_an_entirely_empty_note(tmp_path: Path) -> None: + export = tmp_path / "Keep" + export.mkdir() + (export / "blank.json").write_text(json.dumps({"color": "WHITE"}), encoding="utf-8") + (export / "real.json").write_text( + json.dumps({"title": "Real", "textContent": "body"}), encoding="utf-8" + ) + assert [n.title for n in load_vault("keep", export)] == ["Real"] + + +def test_keep_skips_undecodable_json(tmp_path: Path) -> None: + export = tmp_path / "Keep" + export.mkdir() + (export / "broken.json").write_text("{not json", encoding="utf-8") + (export / "real.json").write_text( + json.dumps({"title": "Real", "textContent": "body"}), encoding="utf-8" + ) + assert [n.title for n in load_vault("keep", export)] == ["Real"] + + +def test_keep_zip_skips_undecodable_json_and_non_notes(tmp_path: Path) -> None: + archive = tmp_path / "takeout.zip" + with zipfile.ZipFile(archive, "w") as zf: + zf.writestr("Takeout/Keep/broken.json", "{not json") + zf.writestr("Takeout/Keep/attachment.png", b"\x89PNG") # not json at all + zf.writestr("Takeout/Keep/", b"") # a directory entry + zf.writestr( + "Takeout/Keep/real.json", json.dumps({"title": "Real", "textContent": "b"}) + ) + assert [n.title for n in load_vault("keep", archive)] == ["Real"] + + +def test_keep_zip_over_the_ceiling_is_refused( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(note_import, "_MAX_VAULT_BYTES", 4) + archive = tmp_path / "takeout.zip" + with zipfile.ZipFile(archive, "w") as zf: + zf.writestr("Takeout/Keep/a.json", json.dumps({"title": "A", "textContent": "b"})) + with pytest.raises(NoteImportError, match="import ceiling"): + load_vault("keep", archive) + + +def test_keep_missing_path_and_empty_export_are_actionable(tmp_path: Path) -> None: + with pytest.raises(NoteImportError, match="no such path"): + load_vault("keep", tmp_path / "nope") + with pytest.raises(NoteImportError, match="no such keep export"): + note_import.load_google_keep(tmp_path / "nope") + empty = tmp_path / "Keep" + empty.mkdir() + with pytest.raises(NoteImportError, match="no google keep notes"): + load_vault("keep", empty) + + +def test_a_link_written_with_its_extension_still_resolves( + store: KBStore, tmp_path: Path +) -> None: + root = tmp_path / "vault3" + root.mkdir() + (root / "A.md").write_text("links to [[B.md]]\n", encoding="utf-8") + (root / "B.md").write_text("target\n", encoding="utf-8") + assert import_vault(store, "obsidian", root)["relations"] == 1 + + +def test_links_are_not_refiled_for_notes_that_were_skipped( + store: KBStore, vault: Path +) -> None: + # Second run: every note is unchanged, so no source is registered and the + # link pass has nothing to file — the relation must not duplicate. + assert import_vault(store, "obsidian", vault)["relations"] == 1 + assert import_vault(store, "obsidian", vault)["relations"] == 0 + rels = [p for p in store.list_proposals(None) if p.kind == ProposalKind.RELATION] + assert len(rels) == 1 + + +def test_a_link_into_an_unchanged_note_is_not_refiled( + store: KBStore, vault: Path +) -> None: + """The partial case: the linking note changed, its target did not. The + target has no source this run, so the edge is left alone rather than + re-proposed against a stale id.""" + assert import_vault(store, "obsidian", vault)["relations"] == 1 + (vault / "Postgres.md").write_text( + "---\ntitle: Postgres tuning\n---\nrewritten. still see [[Deploys]].\n", + encoding="utf-8", + ) + report = import_vault(store, "obsidian", vault) + assert report["updated"] == 1 + assert report["relations"] == 0 + rels = [p for p in store.list_proposals(None) if p.kind == ProposalKind.RELATION] + assert len(rels) == 1 + + +def test_a_rejected_relation_does_not_fail_the_import( + store: KBStore, vault: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + def refuse(*args: object, **kwargs: object) -> None: + from vouch.proposals import ProposalError + + raise ProposalError("nope") + + monkeypatch.setattr(note_import, "propose_relation", refuse) + report = import_vault(store, "obsidian", vault) + assert report["imported"] == 3 # the pages still land + assert report["relations"] == 0