feat(ingest): add Kobo native KoboReader.sqlite read-only source adapter

docs/ROADMAP-FUTURE.md

@@ -60,7 +60,12 @@ your real library" before adding features.
 
 1. **More read-only sources, same guardrails** — Readest progress, Kobo's native
    `KoboReader.sqlite`, Calibre-Web read-state, sideloaded EPUB/PDF. Each is a new
-   adapter behind the existing `unify` join.
+   adapter behind the existing `unify` join. **Kobo native (`ingest/kobo.py`)
+   done** — snapshot-first `KoboReader.sqlite` reader over the `content` table
+   (`ContentType = 6`, chapter-row dedup, schema-drift tolerant), merged through
+   the existing `unify` join with zero changes to `ingest/unify.py`; wired into
+   `ingest/config.py` (`[kobo]` / `STACKS_KOBO_DB`) and `ingest/refresh.py`.
+   Readest, Calibre-Web read-state, and sideloaded EPUB/PDF remain open.
 2. **Annotations & highlights** — surface a private, searchable "commonplace book"
    from KOReader highlights; never synced anywhere.
 3. **Series & TBR intelligence** — "next in a series you own," progress through a

ingest/config.py

@@ -1,4 +1,4 @@
-"""Runtime configuration — where the real Calibre/KOReader libraries live.
+"""Runtime configuration — where the real Calibre/KOReader/Kobo libraries live.
 
 Resolution order (later overrides earlier):
 
@@ -38,6 +38,7 @@
         "STACKS_DEMO",
         "STACKS_CALIBRE_DB",
         "STACKS_KOREADER_DB",
+        "STACKS_KOBO_DB",
         "STACKS_KOSYNC_HOST",
         "STACKS_KOSYNC_USER",
         "STACKS_KOSYNC_KEY",
@@ -67,6 +68,7 @@ class Config:
     kosync_user: Optional[str]
     kosync_key: Optional[str]  # md5 key, from the environment only
     demo: bool
+    kobo_db: Optional[Path] = None  # Kobo's native KoboReader.sqlite (read-only source)
     aperture_strength: float = 0.0  # boost-only discovery-widening lens (0 = off)
     embeddings_enabled: bool = False  # optional, strictly-local semantic signal
     dnf_signals: bool = False  # opt-in soft down-weighting of stalled themes
@@ -93,7 +95,9 @@ def kosync_configured(self) -> bool:
 
     @property
     def has_real_sources(self) -> bool:
-        return self.calibre_db is not None or self.koreader_db is not None
+        return (
+            self.calibre_db is not None or self.koreader_db is not None or self.kobo_db is not None
+        )
 
 
 def _read_toml(path: Path) -> dict[str, object]:
@@ -124,6 +128,7 @@ def load_config(
     toml = _read_toml(path)
     calibre = _section(toml, "calibre")
     koreader = _section(toml, "koreader")
+    kobo = _section(toml, "kobo")
     kosync = _section(toml, "kosync")
     storage = _section(toml, "storage")
 
@@ -163,6 +168,7 @@ def pick_int(env_key: str, key: str) -> int:
     return Config(
         calibre_db=_opt_path(pick("STACKS_CALIBRE_DB", calibre, "path")),
         koreader_db=_opt_path(pick("STACKS_KOREADER_DB", koreader, "path")),
+        kobo_db=_opt_path(pick("STACKS_KOBO_DB", kobo, "path")),
         data_dir=data_dir,
         kosync_host=pick("STACKS_KOSYNC_HOST", kosync, "host"),
         kosync_user=pick("STACKS_KOSYNC_USER", kosync, "user"),

ingest/kobo.py

@@ -0,0 +1,149 @@
+"""Read per-book reading statistics from Kobo's native ``KoboReader.sqlite``.
+
+Strictly read-only (see :mod:`ingest.snapshot`). Kobo stores per-book state in a
+single ``content`` table that mixes one row per book *and* one row per chapter
+of that book. Top-level "book" rows carry ``ContentType = 6``; chapter rows can
+also carry ``ContentType = 6`` on some firmware versions but point back at their
+parent via ``BookID``. :func:`_top_level_rows` keeps only the volume row per
+book — the one whose ``BookID`` is empty or equal to its own ``ContentID``.
+
+There is no per-session log in ``content`` (unlike KOReader's
+``page_stat_data``), so :attr:`~ingest.models.ReadingStat.sessions` is always
+``0`` here — an honest "unknown", not a guess.
+"""
+
+from __future__ import annotations
+
+import sqlite3
+from datetime import UTC, datetime
+from pathlib import Path
+
+from ingest.models import ReadingStat
+from ingest.snapshot import columns, open_snapshot, table_exists
+
+#: Kobo's ContentType for a top-level book/volume (chapters use other values,
+#: though some firmware reuses 6 for chapters too — see module docstring).
+BOOK_CONTENT_TYPE = 6
+
+#: Kobo's ReadStatus: 0 = unread, 1 = reading, 2 = finished.
+READ_STATUS_FINISHED = 2
+
+
+def _normalize_authors(raw: str) -> tuple[str, ...]:
+    """Kobo's ``Attribution`` is a single free-text, comma-separated author string."""
+    parts = [p.strip() for p in raw.split(",")]
+    return tuple(p for p in parts if p)
+
+
+def _parse_timestamp(raw: str) -> int:
+    """Parse Kobo's ISO-8601 ``DateLastRead`` into unix seconds; ``0`` if unparsable."""
+    text = raw.strip()
+    if not text:
+        return 0
+    if text.endswith("Z"):
+        text = text[:-1] + "+00:00"
+    try:
+        dt = datetime.fromisoformat(text)
+    except ValueError:
+        return 0
+    if dt.tzinfo is None:
+        dt = dt.replace(tzinfo=UTC)
+    return int(dt.timestamp())
+
+
+def _top_level_rows(conn: sqlite3.Connection, cols: frozenset[str]) -> list[sqlite3.Row]:
+    """Select ``ContentType = 6`` rows, deduplicated to one row per book volume.
+
+    Kobo's ``content`` table stores one row per chapter alongside the top-level
+    volume row for the same book. The volume row is identified by ``BookID``
+    being empty/NULL or equal to its own ``ContentID`` (a chapter row's
+    ``BookID`` points at its parent volume's ``ContentID`` instead).
+    """
+    rows = list(
+        conn.execute(
+            "SELECT * FROM content WHERE ContentType = ? ORDER BY rowid",
+            (BOOK_CONTENT_TYPE,),
+        ).fetchall()
+    )
+    if "MimeType" in cols:
+        rows = [r for r in rows if r["MimeType"]]
+    if "ContentID" not in cols:
+        return rows
+    grouped: dict[str, sqlite3.Row] = {}
+    for r in rows:
+        content_id = str(r["ContentID"])
+        book_id = r["BookID"] if "BookID" in cols else None
+        is_volume = not book_id or book_id == content_id
+        group_key = str(book_id) if book_id else content_id
+        if group_key not in grouped or is_volume:
+            grouped[group_key] = r
+    return [grouped[k] for k in sorted(grouped)]
+
+
+def _title_key(title: str, authors: tuple[str, ...]) -> str:
+    """Fallback join key, matching :func:`ingest.unify.normalize_key` (koreader.py's idiom)."""
+    from ingest.unify import normalize_key
+
+    return normalize_key(title, authors)
+
+
+def read_stats(conn: sqlite3.Connection) -> list[ReadingStat]:
+    """Read per-book stats from an open read-only Kobo connection."""
+    if not table_exists(conn, "content"):
+        return []
+    cols = columns(conn, "content")
+    stats: list[ReadingStat] = []
+    for r in _top_level_rows(conn, cols):
+        title = str(r["Title"]) if "Title" in cols and r["Title"] else ""
+        authors = (
+            _normalize_authors(str(r["Attribution"]))
+            if "Attribution" in cols and r["Attribution"]
+            else ()
+        )
+        total_pages = (
+            int(r["___NumPages"]) if "___NumPages" in cols and r["___NumPages"] is not None else 0
+        )
+        pct = (
+            int(r["___PercentRead"])
+            if "___PercentRead" in cols and r["___PercentRead"] is not None
+            else 0
+        )
+        read_status = (
+            int(r["ReadStatus"]) if "ReadStatus" in cols and r["ReadStatus"] is not None else 0
+        )
+        if read_status == READ_STATUS_FINISHED and total_pages > 0:
+            pages_read = total_pages
+        elif total_pages > 0:
+            pages_read = round(pct / 100 * total_pages)
+        else:
+            pages_read = 0
+        read_time = (
+            int(r["TimeSpentReading"])
+            if "TimeSpentReading" in cols and r["TimeSpentReading"] is not None
+            else 0
+        )
+        last_read = (
+            _parse_timestamp(str(r["DateLastRead"]))
+            if "DateLastRead" in cols and r["DateLastRead"]
+            else 0
+        )
+        stats.append(
+            ReadingStat(
+                key=_title_key(title, authors),
+                title=title,
+                authors=authors,
+                pages_read=pages_read,
+                total_pages=total_pages,
+                read_time_seconds=read_time,
+                last_read_ts=last_read,
+                sessions=0,  # Kobo's `content` table has no per-session log.
+                highlights=0,  # not tracked in `content`; annotations are a future source
+            )
+        )
+    return stats
+
+
+def load_stats(kobo_db: Path, snapshot_dir: Path) -> list[ReadingStat]:
+    """Snapshot the Kobo DB and return all stats — the read-only entry point."""
+    with open_snapshot(kobo_db, snapshot_dir) as conn:
+        return read_stats(conn)

ingest/refresh.py

@@ -28,6 +28,7 @@
 
 from ingest.calibre import load_library
 from ingest.config import KNOWN_STACKS_ENV, Config
+from ingest.kobo import load_stats as load_kobo_stats
 from ingest.koreader import load_daily_activity, load_stats
 from ingest.kosync import FixtureKosync, ProgressSource
 from ingest.models import DailyActivity, DeviceProgress, ReadingStat, ReadingState
@@ -174,7 +175,11 @@ class RefreshResult:
 def source_mtimes(config: Config) -> dict[str, int]:
     """Integer mtimes of the configured source files that currently exist."""
     out: dict[str, int] = {}
-    for name, path in (("calibre", config.calibre_db), ("koreader", config.koreader_db)):
+    for name, path in (
+        ("calibre", config.calibre_db),
+        ("koreader", config.koreader_db),
+        ("kobo", config.kobo_db),
+    ):
         if path is not None and path.is_file():
             out[name] = int(path.stat().st_mtime)
     return out
@@ -202,6 +207,7 @@ def _ingest_real(
     snap = config.snapshot_dir
     books = load_library(config.calibre_db, snap) if config.calibre_db else []
     stats = load_stats(config.koreader_db, snap) if config.koreader_db else []
+    stats = stats + (load_kobo_stats(config.kobo_db, snap) if config.kobo_db else [])
     activity = load_daily_activity(config.koreader_db, snap) if config.koreader_db else []
     progress_result = _resolve_progress(_kosync(config), stats, store, now)
     states = unify(books, stats, FixtureKosync(progress_result.progress))
@@ -322,6 +328,8 @@ def doctor(
     if not config.demo:
         checks.extend(_check_source("Calibre", config.calibre_db, "books"))
         checks.extend(_check_source("KOReader", config.koreader_db, "book"))
+        if config.kobo_db is not None:
+            checks.extend(_check_source("Kobo", config.kobo_db, "content"))
         checks.append(
             Check(
                 "kosync",

tests/test_config.py

@@ -12,6 +12,7 @@ def test_defaults_when_nothing_set(tmp_path: Path) -> None:
     cfg = load_config(env={}, config_path=tmp_path / "absent.toml")
     assert cfg.calibre_db is None
     assert cfg.koreader_db is None
+    assert cfg.kobo_db is None
     assert cfg.data_dir == DEFAULT_DATA_DIR
     assert cfg.demo is False
     assert cfg.kosync_configured is False
@@ -22,6 +23,7 @@ def test_env_overrides(tmp_path: Path) -> None:
     env = {
         "STACKS_CALIBRE_DB": "/lib/metadata.db",
         "STACKS_KOREADER_DB": "/lib/statistics.sqlite",
+        "STACKS_KOBO_DB": "/lib/KoboReader.sqlite",
         "STACKS_DATA_DIR": str(tmp_path / "state"),
         "STACKS_KOSYNC_HOST": "https://sync.example",
         "STACKS_KOSYNC_USER": "reader",
@@ -31,6 +33,7 @@ def test_env_overrides(tmp_path: Path) -> None:
     cfg = load_config(env=env, config_path=tmp_path / "absent.toml")
     assert cfg.calibre_db == Path("/lib/metadata.db")
     assert cfg.koreader_db == Path("/lib/statistics.sqlite")
+    assert cfg.kobo_db == Path("/lib/KoboReader.sqlite")
     assert cfg.data_dir == tmp_path / "state"
     assert cfg.demo is True
     assert cfg.kosync_configured is True
@@ -39,6 +42,16 @@ def test_env_overrides(tmp_path: Path) -> None:
     assert cfg.snapshot_dir == tmp_path / "state" / "snapshots"
 
 
+def test_kobo_only_counts_as_a_real_source(tmp_path: Path) -> None:
+    cfg = load_config(
+        env={"STACKS_KOBO_DB": "/lib/KoboReader.sqlite"}, config_path=tmp_path / "absent.toml"
+    )
+    assert cfg.calibre_db is None
+    assert cfg.koreader_db is None
+    assert cfg.kobo_db == Path("/lib/KoboReader.sqlite")
+    assert cfg.has_real_sources is True
+
+
 def test_toml_file_is_read(tmp_path: Path) -> None:
     toml = tmp_path / "stacks.toml"
     toml.write_text(
@@ -47,6 +60,8 @@ def test_toml_file_is_read(tmp_path: Path) -> None:
         path = "/books/metadata.db"
         [koreader]
         path = "/books/statistics.sqlite"
+        [kobo]
+        path = "/books/KoboReader.sqlite"
         [kosync]
         host = "https://sync.example"
         user = "me"
@@ -58,6 +73,7 @@ def test_toml_file_is_read(tmp_path: Path) -> None:
     cfg = load_config(env={}, config_path=toml)
     assert cfg.calibre_db == Path("/books/metadata.db")
     assert cfg.koreader_db == Path("/books/statistics.sqlite")
+    assert cfg.kobo_db == Path("/books/KoboReader.sqlite")
     assert cfg.kosync_host == "https://sync.example"
     assert cfg.kosync_user == "me"
     assert cfg.data_dir == Path("/srv/stacks-data")