diff --git a/claude_code_log/cache.py b/claude_code_log/cache.py index c865d922..82766a17 100644 --- a/claude_code_log/cache.py +++ b/claude_code_log/cache.py @@ -10,7 +10,7 @@ from contextlib import contextmanager from datetime import datetime, timezone from pathlib import Path -from typing import Any, Dict, Generator, List, Optional +from typing import Any, Callable, Dict, Generator, List, Optional from packaging import version from pydantic import BaseModel @@ -39,7 +39,6 @@ class CachedFileInfo(BaseModel): source_mtime: float cached_mtime: float message_count: int - session_ids: list[str] class SessionCacheData(BaseModel): @@ -260,6 +259,32 @@ def subagents_fingerprint(jsonl_path: Path) -> str: return f"{len(metas)}:{newest}" +def _cache_row_is_fresh( + row: sqlite3.Row, source_mtime: float, current_fp: Callable[[], str] +) -> bool: + """Decide whether a cached_files row is still fresh. + + Single source of truth for the freshness rule shared by + is_file_cached() and get_modified_files(). current_fp is a callable + so callers only pay for the sidecar fingerprint scan when the mtime + check passes (and get_modified_files() can plug in its + scandir-optimized variant). + + Cache is valid if modification times match (within 1 second + tolerance) and the sidecar inputs of spawn discovery (#213) match + too — new agent-*.meta.json files appear without touching the + source jsonl. Pre-007 rows carry NULL: accept those only when the + file has no sidecars today (nothing to miss), so legacy caches + don't mass-invalidate while sessions WITH sidecars reparse once. + """ + if abs(source_mtime - row["source_mtime"]) >= 1.0: + return False + cached_fp = row["subagents_fingerprint"] + if cached_fp is None: + return current_fp() == "" + return cached_fp == current_fp() + + def get_cache_db_path(projects_dir: Path) -> Path: """Get cache database path, respecting CLAUDE_CODE_LOG_CACHE_PATH env var. @@ -280,6 +305,16 @@ def get_cache_db_path(projects_dir: Path) -> Path: # ========== Cache Manager ========== +# Process-level memo of cache DBs whose migrations have already been +# checked. CacheManager is constructed per project (the TUI project +# selector builds one per table row, on every resize), so without the +# memo every instance re-pays the migration check: a connection plus +# version-table and migration-directory reads, ~2ms each (issue #12). +# The exists() guard in _init_database covers --clear-cache deleting +# the DB file mid-process. +_migrated_db_paths: set[str] = set() + + class CacheManager: """SQLite-based cache manager for Claude Code Log.""" @@ -404,8 +439,11 @@ def batch(self) -> Generator[None, None, None]: def _init_database(self) -> None: """Create schema if needed using migration runner.""" - # Run any pending migrations + key = str(self.db_path) + if key in _migrated_db_paths and self.db_path.exists(): + return run_migrations(self.db_path) + _migrated_db_paths.add(key) def _ensure_project_exists(self) -> None: """Ensure project record exists and get its ID.""" @@ -598,23 +636,11 @@ def is_file_cached(self, jsonl_path: Path) -> bool: if not row: return False - source_mtime = jsonl_path.stat().st_mtime - cached_mtime = row["source_mtime"] - - # Cache is valid if modification times match (within 1 second tolerance) - if abs(source_mtime - cached_mtime) >= 1.0: - return False - - # The sidecar inputs of spawn discovery (#213) must match too — - # new agent-*.meta.json files appear without touching the source - # jsonl. Pre-007 rows carry NULL: accept those only when the file - # has no sidecars today (nothing to miss), so legacy caches don't - # mass-invalidate while sessions WITH sidecars reparse once. - cached_fp = row["subagents_fingerprint"] - current_fp = subagents_fingerprint(jsonl_path) - if cached_fp is None: - return current_fp == "" - return cached_fp == current_fp + return _cache_row_is_fresh( + row, + jsonl_path.stat().st_mtime, + lambda: subagents_fingerprint(jsonl_path), + ) def load_cached_entries(self, jsonl_path: Path) -> Optional[List[TranscriptEntry]]: """Load cached transcript entries for a JSONL file.""" @@ -886,21 +912,82 @@ def get_working_directories(self) -> List[str]: if self._project_id is None: return [] + with self._get_connection() as conn: + return self._get_working_directories(conn) + + def _get_working_directories(self, conn: sqlite3.Connection) -> List[str]: + """get_working_directories() on an already-open connection.""" + rows = conn.execute( + "SELECT DISTINCT cwd FROM sessions WHERE project_id = ? AND cwd IS NOT NULL", + (self._project_id,), + ).fetchall() + return [row["cwd"] for row in rows] + + def get_modified_files(self, jsonl_files: List[Path]) -> List[Path]: + """Get list of JSONL files that need to be reprocessed. + + Batched equivalent of calling is_file_cached() per file: one SQL + query fetches every cached row (one connection open instead of + one per file), and one scandir per parent directory rules out + subagent-sidecar fingerprints, replacing the per-file query + + directory probes that dominated freshness-check time on large + archives (issue #12). + """ + if not jsonl_files: + return [] + if self._project_id is None: + return list(jsonl_files) + with self._get_connection() as conn: rows = conn.execute( - "SELECT DISTINCT cwd FROM sessions WHERE project_id = ? AND cwd IS NOT NULL", + "SELECT file_name, source_mtime, subagents_fingerprint" + " FROM cached_files WHERE project_id = ?", (self._project_id,), ).fetchall() + cached = {row["file_name"]: row for row in rows} + + # A session file's sidecars live under //subagents, + # so one scandir of the parent tells us which files can have a + # non-empty fingerprint without any per-file stat probes. + subdir_names: Dict[Path, set[str]] = {} + + def current_fp(jsonl_file: Path) -> str: + parent = jsonl_file.parent + if parent.name == "subagents": + # Agent file in a subagents dir: its sidecars sit next to + # it, so the full fingerprint scan is required. + return subagents_fingerprint(jsonl_file) + names = subdir_names.get(parent) + if names is None: + try: + with os.scandir(parent) as entries: + names = {e.name for e in entries if e.is_dir()} + except OSError: + names = set[str]() + subdir_names[parent] = names + return subagents_fingerprint(jsonl_file) if jsonl_file.stem in names else "" + + modified: List[Path] = [] + for jsonl_file in jsonl_files: + row = cached.get(jsonl_file.name) + if row is None: + modified.append(jsonl_file) + continue + + try: + source_mtime = jsonl_file.stat().st_mtime + except OSError: + # Missing file: same outcome as is_file_cached()'s + # exists() check returning False. + modified.append(jsonl_file) + continue + + if not _cache_row_is_fresh( + row, source_mtime, lambda file=jsonl_file: current_fp(file) + ): + modified.append(jsonl_file) - return [row["cwd"] for row in rows] - - def get_modified_files(self, jsonl_files: List[Path]) -> List[Path]: - """Get list of JSONL files that need to be reprocessed.""" - return [ - jsonl_file - for jsonl_file in jsonl_files - if not self.is_file_cached(jsonl_file) - ] + return modified def get_cached_project_data(self) -> Optional[ProjectCache]: """Get the cached project data if available.""" @@ -923,19 +1010,11 @@ def get_cached_project_data(self) -> Optional[ProjectCache]: cached_files: Dict[str, CachedFileInfo] = {} for row in file_rows: - # Get session IDs for this file from messages - session_rows = conn.execute( - "SELECT DISTINCT session_id FROM messages WHERE file_id = ? AND session_id IS NOT NULL", - (row["id"],), - ).fetchall() - session_ids = [r["session_id"] for r in session_rows] - cached_files[row["file_name"]] = CachedFileInfo( file_path=row["file_path"], source_mtime=row["source_mtime"], cached_mtime=row["cached_mtime"], message_count=row["message_count"], - session_ids=session_ids, ) # Get sessions @@ -961,6 +1040,10 @@ def get_cached_project_data(self) -> Optional[ProjectCache]: team_name=row["team_name"] if "team_name" in row.keys() else None, ) + # On the same connection: calling the public method here + # would open a second connection per call. + working_directories = self._get_working_directories(conn) + return ProjectCache( version=project_row["version"], cache_created=project_row["cache_created"], @@ -973,7 +1056,7 @@ def get_cached_project_data(self) -> Optional[ProjectCache]: total_cache_creation_tokens=project_row["total_cache_creation_tokens"], total_cache_read_tokens=project_row["total_cache_read_tokens"], sessions=sessions, - working_directories=self.get_working_directories(), + working_directories=working_directories, earliest_timestamp=project_row["earliest_timestamp"], latest_timestamp=project_row["latest_timestamp"], ) diff --git a/dev-docs/application_model.md b/dev-docs/application_model.md index da5292a1..7e48085b 100644 --- a/dev-docs/application_model.md +++ b/dev-docs/application_model.md @@ -135,6 +135,13 @@ per-file load loop, per-session generation) in `CacheManager.batch()`: one shared connection reused for the scope and closed on exit (including on exception). `batch()` nesting is a no-op reuse, so the wraps compose. +Freshness checks are batched too (issue #12): `get_modified_files()` +fetches every cached row for the project in one query (one connection +open, or zero extra inside a `batch()` scope) and rules out +subagent-sidecar fingerprints with one scandir per parent directory, +instead of a per-file query + `is_dir()` probes. This is what makes +TUI startup near-instant on multi-thousand-file archives. + For the operations / recovery side (archived sessions, manual deletion, `cleanupPeriodDays`), see [`docs/restoring-archived-sessions.md`](../docs/restoring-archived-sessions.md). @@ -145,8 +152,9 @@ deletion, `cleanupPeriodDays`), see small migration system. Each migration is a `NNN_description.sql` file applied in numeric order by `migrations/runner.py`. The schema-version table tracks which migrations have run; `cache.py` invokes the runner -on every connection open, so a fresh checkout running against an old -cache DB transparently upgrades. +the first time a given cache DB is opened in a process (memoised per +DB path, re-checked if the file disappears), so a fresh checkout +running against an old cache DB transparently upgrades. Current migrations: