Speed up cache freshness checks on large archives

claude_code_log/cache.py

@@ -10,7 +10,7 @@
 from contextlib import contextmanager
 from datetime import datetime, timezone
 from pathlib import Path
-from typing import Any, Dict, Generator, List, Optional
+from typing import Any, Callable, Dict, Generator, List, Optional
 
 from packaging import version
 from pydantic import BaseModel
@@ -39,7 +39,6 @@ class CachedFileInfo(BaseModel):
     source_mtime: float
     cached_mtime: float
     message_count: int
-    session_ids: list[str]
 
 
 class SessionCacheData(BaseModel):
@@ -260,6 +259,32 @@ def subagents_fingerprint(jsonl_path: Path) -> str:
     return f"{len(metas)}:{newest}"
 
 
+def _cache_row_is_fresh(
+    row: sqlite3.Row, source_mtime: float, current_fp: Callable[[], str]
+) -> bool:
+    """Decide whether a cached_files row is still fresh.
+
+    Single source of truth for the freshness rule shared by
+    is_file_cached() and get_modified_files(). current_fp is a callable
+    so callers only pay for the sidecar fingerprint scan when the mtime
+    check passes (and get_modified_files() can plug in its
+    scandir-optimized variant).
+
+    Cache is valid if modification times match (within 1 second
+    tolerance) and the sidecar inputs of spawn discovery (#213) match
+    too — new agent-*.meta.json files appear without touching the
+    source jsonl. Pre-007 rows carry NULL: accept those only when the
+    file has no sidecars today (nothing to miss), so legacy caches
+    don't mass-invalidate while sessions WITH sidecars reparse once.
+    """
+    if abs(source_mtime - row["source_mtime"]) >= 1.0:
+        return False
+    cached_fp = row["subagents_fingerprint"]
+    if cached_fp is None:
+        return current_fp() == ""
+    return cached_fp == current_fp()
+
+
 def get_cache_db_path(projects_dir: Path) -> Path:
     """Get cache database path, respecting CLAUDE_CODE_LOG_CACHE_PATH env var.
 
@@ -280,6 +305,16 @@ def get_cache_db_path(projects_dir: Path) -> Path:
 # ========== Cache Manager ==========
 
 
+# Process-level memo of cache DBs whose migrations have already been
+# checked. CacheManager is constructed per project (the TUI project
+# selector builds one per table row, on every resize), so without the
+# memo every instance re-pays the migration check: a connection plus
+# version-table and migration-directory reads, ~2ms each (issue #12).
+# The exists() guard in _init_database covers --clear-cache deleting
+# the DB file mid-process.
+_migrated_db_paths: set[str] = set()
+
+
 class CacheManager:
     """SQLite-based cache manager for Claude Code Log."""
 
@@ -404,8 +439,11 @@ def batch(self) -> Generator[None, None, None]:
 
     def _init_database(self) -> None:
         """Create schema if needed using migration runner."""
-        # Run any pending migrations
+        key = str(self.db_path)
+        if key in _migrated_db_paths and self.db_path.exists():
+            return
         run_migrations(self.db_path)
+        _migrated_db_paths.add(key)
 
     def _ensure_project_exists(self) -> None:
         """Ensure project record exists and get its ID."""
@@ -598,23 +636,11 @@ def is_file_cached(self, jsonl_path: Path) -> bool:
         if not row:
             return False
 
-        source_mtime = jsonl_path.stat().st_mtime
-        cached_mtime = row["source_mtime"]
-
-        # Cache is valid if modification times match (within 1 second tolerance)
-        if abs(source_mtime - cached_mtime) >= 1.0:
-            return False
-
-        # The sidecar inputs of spawn discovery (#213) must match too —
-        # new agent-*.meta.json files appear without touching the source
-        # jsonl. Pre-007 rows carry NULL: accept those only when the file
-        # has no sidecars today (nothing to miss), so legacy caches don't
-        # mass-invalidate while sessions WITH sidecars reparse once.
-        cached_fp = row["subagents_fingerprint"]
-        current_fp = subagents_fingerprint(jsonl_path)
-        if cached_fp is None:
-            return current_fp == ""
-        return cached_fp == current_fp
+        return _cache_row_is_fresh(
+            row,
+            jsonl_path.stat().st_mtime,
+            lambda: subagents_fingerprint(jsonl_path),
+        )
 
     def load_cached_entries(self, jsonl_path: Path) -> Optional[List[TranscriptEntry]]:
         """Load cached transcript entries for a JSONL file."""
@@ -886,21 +912,82 @@ def get_working_directories(self) -> List[str]:
         if self._project_id is None:
             return []
 
+        with self._get_connection() as conn:
+            return self._get_working_directories(conn)
+
+    def _get_working_directories(self, conn: sqlite3.Connection) -> List[str]:
+        """get_working_directories() on an already-open connection."""
+        rows = conn.execute(
+            "SELECT DISTINCT cwd FROM sessions WHERE project_id = ? AND cwd IS NOT NULL",
+            (self._project_id,),
+        ).fetchall()
+        return [row["cwd"] for row in rows]
+
+    def get_modified_files(self, jsonl_files: List[Path]) -> List[Path]:
+        """Get list of JSONL files that need to be reprocessed.
+
+        Batched equivalent of calling is_file_cached() per file: one SQL
+        query fetches every cached row (one connection open instead of
+        one per file), and one scandir per parent directory rules out
+        subagent-sidecar fingerprints, replacing the per-file query +
+        directory probes that dominated freshness-check time on large
+        archives (issue #12).
+        """
+        if not jsonl_files:
+            return []
+        if self._project_id is None:
+            return list(jsonl_files)
+
         with self._get_connection() as conn:
             rows = conn.execute(
-                "SELECT DISTINCT cwd FROM sessions WHERE project_id = ? AND cwd IS NOT NULL",
+                "SELECT file_name, source_mtime, subagents_fingerprint"
+                " FROM cached_files WHERE project_id = ?",
                 (self._project_id,),
             ).fetchall()
+        cached = {row["file_name"]: row for row in rows}
+
+        # A session file's sidecars live under <parent>/<stem>/subagents,
+        # so one scandir of the parent tells us which files can have a
+        # non-empty fingerprint without any per-file stat probes.
+        subdir_names: Dict[Path, set[str]] = {}
+
+        def current_fp(jsonl_file: Path) -> str:
+            parent = jsonl_file.parent
+            if parent.name == "subagents":
+                # Agent file in a subagents dir: its sidecars sit next to
+                # it, so the full fingerprint scan is required.
+                return subagents_fingerprint(jsonl_file)
+            names = subdir_names.get(parent)
+            if names is None:
+                try:
+                    with os.scandir(parent) as entries:
+                        names = {e.name for e in entries if e.is_dir()}
+                except OSError:
+                    names = set[str]()
+                subdir_names[parent] = names
+            return subagents_fingerprint(jsonl_file) if jsonl_file.stem in names else ""
+
+        modified: List[Path] = []
+        for jsonl_file in jsonl_files:
+            row = cached.get(jsonl_file.name)
+            if row is None:
+                modified.append(jsonl_file)
+                continue
+
+            try:
+                source_mtime = jsonl_file.stat().st_mtime
+            except OSError:
+                # Missing file: same outcome as is_file_cached()'s
+                # exists() check returning False.
+                modified.append(jsonl_file)
+                continue
+
+            if not _cache_row_is_fresh(
+                row, source_mtime, lambda file=jsonl_file: current_fp(file)
+            ):
+                modified.append(jsonl_file)
 
-        return [row["cwd"] for row in rows]
-
-    def get_modified_files(self, jsonl_files: List[Path]) -> List[Path]:
-        """Get list of JSONL files that need to be reprocessed."""
-        return [
-            jsonl_file
-            for jsonl_file in jsonl_files
-            if not self.is_file_cached(jsonl_file)
-        ]
+        return modified
 
     def get_cached_project_data(self) -> Optional[ProjectCache]:
         """Get the cached project data if available."""
@@ -923,19 +1010,11 @@ def get_cached_project_data(self) -> Optional[ProjectCache]:
 
             cached_files: Dict[str, CachedFileInfo] = {}
             for row in file_rows:
-                # Get session IDs for this file from messages
-                session_rows = conn.execute(
-                    "SELECT DISTINCT session_id FROM messages WHERE file_id = ? AND session_id IS NOT NULL",
-                    (row["id"],),
-                ).fetchall()
-                session_ids = [r["session_id"] for r in session_rows]
-
                 cached_files[row["file_name"]] = CachedFileInfo(
                     file_path=row["file_path"],
                     source_mtime=row["source_mtime"],
                     cached_mtime=row["cached_mtime"],
                     message_count=row["message_count"],
-                    session_ids=session_ids,
                 )
 
             # Get sessions
@@ -961,6 +1040,10 @@ def get_cached_project_data(self) -> Optional[ProjectCache]:
                     team_name=row["team_name"] if "team_name" in row.keys() else None,
                 )
 
+            # On the same connection: calling the public method here
+            # would open a second connection per call.
+            working_directories = self._get_working_directories(conn)
+
         return ProjectCache(
             version=project_row["version"],
             cache_created=project_row["cache_created"],
@@ -973,7 +1056,7 @@ def get_cached_project_data(self) -> Optional[ProjectCache]:
             total_cache_creation_tokens=project_row["total_cache_creation_tokens"],
             total_cache_read_tokens=project_row["total_cache_read_tokens"],
             sessions=sessions,
-            working_directories=self.get_working_directories(),
+            working_directories=working_directories,
             earliest_timestamp=project_row["earliest_timestamp"],
             latest_timestamp=project_row["latest_timestamp"],
         )

dev-docs/application_model.md

@@ -135,6 +135,13 @@ per-file load loop, per-session generation) in `CacheManager.batch()`:
 one shared connection reused for the scope and closed on exit (including
 on exception). `batch()` nesting is a no-op reuse, so the wraps compose.
 
+Freshness checks are batched too (issue #12): `get_modified_files()`
+fetches every cached row for the project in one query (one connection
+open, or zero extra inside a `batch()` scope) and rules out
+subagent-sidecar fingerprints with one scandir per parent directory,
+instead of a per-file query + `is_dir()` probes. This is what makes
+TUI startup near-instant on multi-thousand-file archives.
+
 For the operations / recovery side (archived sessions, manual
 deletion, `cleanupPeriodDays`), see
 [`docs/restoring-archived-sessions.md`](../docs/restoring-archived-sessions.md).
@@ -145,8 +152,9 @@ deletion, `cleanupPeriodDays`), see
 small migration system. Each migration is a `NNN_description.sql` file
 applied in numeric order by `migrations/runner.py`. The schema-version
 table tracks which migrations have run; `cache.py` invokes the runner
-on every connection open, so a fresh checkout running against an old
-cache DB transparently upgrades.
+the first time a given cache DB is opened in a process (memoised per
+DB path, re-checked if the file disappears), so a fresh checkout
+running against an old cache DB transparently upgrades.
 
 Current migrations: