From d46f5abd2caff7a7147d9570919664066f4b5c18 Mon Sep 17 00:00:00 2001 From: Daniel Demmel Date: Mon, 6 Jul 2026 23:40:21 +0000 Subject: [PATCH 1/3] Speed up cache freshness checks 13-54x on large archives (#12) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Even with a fully warm cache, launching the TUI on a multi-thousand- file archive spent seconds re-verifying freshness. Profiling against a real 2GB / 5,381-item ~/.claude/projects copy showed the file stats were never the problem (~5ms for 1,830 files); the cost was per-file SQLite connections, per-file sidecar dir probes, and a dead query. Four changes, all in cache.py: - get_modified_files(): batched equivalent of per-file is_file_cached() — one query fetches every cached row for the project (one connection open instead of one per file, composing with batch() from #251), and one scandir per parent directory rules out subagent-sidecar fingerprints (#213) so the fingerprint scan only runs for files that actually have a / dir (~154 of 1,830 in the test archive). Semantics verified identical to the per-file path on real data. - get_cached_project_data(): drop the per-file "SELECT DISTINCT session_id FROM messages" loop feeding CachedFileInfo.session_ids — the field was written but never read anywhere (production, tests, or docs), and cost N queries over the largest table on every call. Field removed from the model. - get_cached_project_data(): fetch working_directories on the same connection instead of opening a second one per call. - _init_database(): memoise the migration check per DB path per process — the TUI project selector constructs a CacheManager per row on every resize, each re-paying ~2ms of migration checks. An exists() guard keeps --clear-cache's DB deletion re-migrating. Measured on the 2GB archive (63 projects, warm cache, best of 3), before -> after on current main: TUI launch, one project (453 files): 646ms -> 12ms (54x) TUI project selector (63 projects): 1190ms -> 276ms (4x) Full-archive freshness scan: 4652ms -> 358ms (13x) Approaches considered and rejected: directory-mtime fast path (unsound — appends don't touch dir mtime; and stats were never the bottleneck), hash/probabilistic checks (same reason), parallelism (unnecessary at these timings). Co-Authored-By: Claude Fable 5 --- claude_code_log/cache.py | 123 ++++++++++++++++++++++++++++------ dev-docs/application_model.md | 12 +++- 2 files changed, 113 insertions(+), 22 deletions(-) diff --git a/claude_code_log/cache.py b/claude_code_log/cache.py index c865d922..f55303f9 100644 --- a/claude_code_log/cache.py +++ b/claude_code_log/cache.py @@ -39,7 +39,6 @@ class CachedFileInfo(BaseModel): source_mtime: float cached_mtime: float message_count: int - session_ids: list[str] class SessionCacheData(BaseModel): @@ -280,6 +279,16 @@ def get_cache_db_path(projects_dir: Path) -> Path: # ========== Cache Manager ========== +# Process-level memo of cache DBs whose migrations have already been +# checked. CacheManager is constructed per project (the TUI project +# selector builds one per table row, on every resize), so without the +# memo every instance re-pays the migration check: a connection plus +# version-table and migration-directory reads, ~2ms each (issue #12). +# The exists() guard in _init_database covers --clear-cache deleting +# the DB file mid-process. +_migrated_db_paths: set[str] = set() + + class CacheManager: """SQLite-based cache manager for Claude Code Log.""" @@ -404,8 +413,11 @@ def batch(self) -> Generator[None, None, None]: def _init_database(self) -> None: """Create schema if needed using migration runner.""" - # Run any pending migrations + key = str(self.db_path) + if key in _migrated_db_paths and self.db_path.exists(): + return run_migrations(self.db_path) + _migrated_db_paths.add(key) def _ensure_project_exists(self) -> None: """Ensure project record exists and get its ID.""" @@ -886,21 +898,96 @@ def get_working_directories(self) -> List[str]: if self._project_id is None: return [] + with self._get_connection() as conn: + return self._get_working_directories(conn) + + def _get_working_directories(self, conn: sqlite3.Connection) -> List[str]: + """get_working_directories() on an already-open connection.""" + rows = conn.execute( + "SELECT DISTINCT cwd FROM sessions WHERE project_id = ? AND cwd IS NOT NULL", + (self._project_id,), + ).fetchall() + return [row["cwd"] for row in rows] + + def get_modified_files(self, jsonl_files: List[Path]) -> List[Path]: + """Get list of JSONL files that need to be reprocessed. + + Batched equivalent of calling is_file_cached() per file: one SQL + query fetches every cached row (one connection open instead of + one per file), and one scandir per parent directory rules out + subagent-sidecar fingerprints, replacing the per-file query + + directory probes that dominated freshness-check time on large + archives (issue #12). + """ + if not jsonl_files: + return [] + if self._project_id is None: + return list(jsonl_files) + with self._get_connection() as conn: rows = conn.execute( - "SELECT DISTINCT cwd FROM sessions WHERE project_id = ? AND cwd IS NOT NULL", + "SELECT file_name, source_mtime, subagents_fingerprint" + " FROM cached_files WHERE project_id = ?", (self._project_id,), ).fetchall() + cached = {row["file_name"]: row for row in rows} + + # A session file's sidecars live under //subagents, + # so one scandir of the parent tells us which files can have a + # non-empty fingerprint without any per-file stat probes. + subdir_names: Dict[Path, set[str]] = {} + + modified: List[Path] = [] + for jsonl_file in jsonl_files: + row = cached.get(jsonl_file.name) + if row is None: + modified.append(jsonl_file) + continue + + try: + source_mtime = jsonl_file.stat().st_mtime + except OSError: + # Missing file: same outcome as is_file_cached()'s + # exists() check returning False. + modified.append(jsonl_file) + continue + + # Cache is valid if modification times match (within 1 second + # tolerance) + if abs(source_mtime - row["source_mtime"]) >= 1.0: + modified.append(jsonl_file) + continue + + parent = jsonl_file.parent + if parent.name == "subagents": + # Agent file in a subagents dir: its sidecars sit next to + # it, so the full fingerprint scan is required. + current_fp = subagents_fingerprint(jsonl_file) + else: + names = subdir_names.get(parent) + if names is None: + try: + with os.scandir(parent) as entries: + names = {e.name for e in entries if e.is_dir()} + except OSError: + names = set() + subdir_names[parent] = names + current_fp = ( + subagents_fingerprint(jsonl_file) + if jsonl_file.stem in names + else "" + ) - return [row["cwd"] for row in rows] + # Same NULL-fingerprint rule as is_file_cached(): pre-007 + # rows are accepted only while the file has no sidecars. + cached_fp = row["subagents_fingerprint"] + if cached_fp is None: + if current_fp != "": + modified.append(jsonl_file) + elif cached_fp != current_fp: + modified.append(jsonl_file) - def get_modified_files(self, jsonl_files: List[Path]) -> List[Path]: - """Get list of JSONL files that need to be reprocessed.""" - return [ - jsonl_file - for jsonl_file in jsonl_files - if not self.is_file_cached(jsonl_file) - ] + return modified def get_cached_project_data(self) -> Optional[ProjectCache]: """Get the cached project data if available.""" @@ -923,19 +1010,11 @@ def get_cached_project_data(self) -> Optional[ProjectCache]: cached_files: Dict[str, CachedFileInfo] = {} for row in file_rows: - # Get session IDs for this file from messages - session_rows = conn.execute( - "SELECT DISTINCT session_id FROM messages WHERE file_id = ? AND session_id IS NOT NULL", - (row["id"],), - ).fetchall() - session_ids = [r["session_id"] for r in session_rows] - cached_files[row["file_name"]] = CachedFileInfo( file_path=row["file_path"], source_mtime=row["source_mtime"], cached_mtime=row["cached_mtime"], message_count=row["message_count"], - session_ids=session_ids, ) # Get sessions @@ -961,6 +1040,10 @@ def get_cached_project_data(self) -> Optional[ProjectCache]: team_name=row["team_name"] if "team_name" in row.keys() else None, ) + # On the same connection: calling the public method here + # would open a second connection per call. + working_directories = self._get_working_directories(conn) + return ProjectCache( version=project_row["version"], cache_created=project_row["cache_created"], @@ -973,7 +1056,7 @@ def get_cached_project_data(self) -> Optional[ProjectCache]: total_cache_creation_tokens=project_row["total_cache_creation_tokens"], total_cache_read_tokens=project_row["total_cache_read_tokens"], sessions=sessions, - working_directories=self.get_working_directories(), + working_directories=working_directories, earliest_timestamp=project_row["earliest_timestamp"], latest_timestamp=project_row["latest_timestamp"], ) diff --git a/dev-docs/application_model.md b/dev-docs/application_model.md index da5292a1..7e48085b 100644 --- a/dev-docs/application_model.md +++ b/dev-docs/application_model.md @@ -135,6 +135,13 @@ per-file load loop, per-session generation) in `CacheManager.batch()`: one shared connection reused for the scope and closed on exit (including on exception). `batch()` nesting is a no-op reuse, so the wraps compose. +Freshness checks are batched too (issue #12): `get_modified_files()` +fetches every cached row for the project in one query (one connection +open, or zero extra inside a `batch()` scope) and rules out +subagent-sidecar fingerprints with one scandir per parent directory, +instead of a per-file query + `is_dir()` probes. This is what makes +TUI startup near-instant on multi-thousand-file archives. + For the operations / recovery side (archived sessions, manual deletion, `cleanupPeriodDays`), see [`docs/restoring-archived-sessions.md`](../docs/restoring-archived-sessions.md). @@ -145,8 +152,9 @@ deletion, `cleanupPeriodDays`), see small migration system. Each migration is a `NNN_description.sql` file applied in numeric order by `migrations/runner.py`. The schema-version table tracks which migrations have run; `cache.py` invokes the runner -on every connection open, so a fresh checkout running against an old -cache DB transparently upgrades. +the first time a given cache DB is opened in a process (memoised per +DB path, re-checked if the file disappears), so a fresh checkout +running against an old cache DB transparently upgrades. Current migrations: From 39c1c21408339eb46c7dbb46cdf08bcf0cdd5b66 Mon Sep 17 00:00:00 2001 From: Daniel Demmel Date: Tue, 7 Jul 2026 01:16:32 +0100 Subject: [PATCH 2/3] Fix lint issue --- claude_code_log/cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/claude_code_log/cache.py b/claude_code_log/cache.py index f55303f9..7e2c1856 100644 --- a/claude_code_log/cache.py +++ b/claude_code_log/cache.py @@ -970,7 +970,7 @@ def get_modified_files(self, jsonl_files: List[Path]) -> List[Path]: with os.scandir(parent) as entries: names = {e.name for e in entries if e.is_dir()} except OSError: - names = set() + names = set[str]() subdir_names[parent] = names current_fp = ( subagents_fingerprint(jsonl_file) From 96871c4c5cd72052ee992313aae22d7eac2b0564 Mon Sep 17 00:00:00 2001 From: Daniel Demmel Date: Tue, 7 Jul 2026 08:27:31 +0100 Subject: [PATCH 3/3] PR feedback --- claude_code_log/cache.py | 102 +++++++++++++++++++-------------------- 1 file changed, 51 insertions(+), 51 deletions(-) diff --git a/claude_code_log/cache.py b/claude_code_log/cache.py index 7e2c1856..82766a17 100644 --- a/claude_code_log/cache.py +++ b/claude_code_log/cache.py @@ -10,7 +10,7 @@ from contextlib import contextmanager from datetime import datetime, timezone from pathlib import Path -from typing import Any, Dict, Generator, List, Optional +from typing import Any, Callable, Dict, Generator, List, Optional from packaging import version from pydantic import BaseModel @@ -259,6 +259,32 @@ def subagents_fingerprint(jsonl_path: Path) -> str: return f"{len(metas)}:{newest}" +def _cache_row_is_fresh( + row: sqlite3.Row, source_mtime: float, current_fp: Callable[[], str] +) -> bool: + """Decide whether a cached_files row is still fresh. + + Single source of truth for the freshness rule shared by + is_file_cached() and get_modified_files(). current_fp is a callable + so callers only pay for the sidecar fingerprint scan when the mtime + check passes (and get_modified_files() can plug in its + scandir-optimized variant). + + Cache is valid if modification times match (within 1 second + tolerance) and the sidecar inputs of spawn discovery (#213) match + too — new agent-*.meta.json files appear without touching the + source jsonl. Pre-007 rows carry NULL: accept those only when the + file has no sidecars today (nothing to miss), so legacy caches + don't mass-invalidate while sessions WITH sidecars reparse once. + """ + if abs(source_mtime - row["source_mtime"]) >= 1.0: + return False + cached_fp = row["subagents_fingerprint"] + if cached_fp is None: + return current_fp() == "" + return cached_fp == current_fp() + + def get_cache_db_path(projects_dir: Path) -> Path: """Get cache database path, respecting CLAUDE_CODE_LOG_CACHE_PATH env var. @@ -610,23 +636,11 @@ def is_file_cached(self, jsonl_path: Path) -> bool: if not row: return False - source_mtime = jsonl_path.stat().st_mtime - cached_mtime = row["source_mtime"] - - # Cache is valid if modification times match (within 1 second tolerance) - if abs(source_mtime - cached_mtime) >= 1.0: - return False - - # The sidecar inputs of spawn discovery (#213) must match too — - # new agent-*.meta.json files appear without touching the source - # jsonl. Pre-007 rows carry NULL: accept those only when the file - # has no sidecars today (nothing to miss), so legacy caches don't - # mass-invalidate while sessions WITH sidecars reparse once. - cached_fp = row["subagents_fingerprint"] - current_fp = subagents_fingerprint(jsonl_path) - if cached_fp is None: - return current_fp == "" - return cached_fp == current_fp + return _cache_row_is_fresh( + row, + jsonl_path.stat().st_mtime, + lambda: subagents_fingerprint(jsonl_path), + ) def load_cached_entries(self, jsonl_path: Path) -> Optional[List[TranscriptEntry]]: """Load cached transcript entries for a JSONL file.""" @@ -937,6 +951,22 @@ def get_modified_files(self, jsonl_files: List[Path]) -> List[Path]: # non-empty fingerprint without any per-file stat probes. subdir_names: Dict[Path, set[str]] = {} + def current_fp(jsonl_file: Path) -> str: + parent = jsonl_file.parent + if parent.name == "subagents": + # Agent file in a subagents dir: its sidecars sit next to + # it, so the full fingerprint scan is required. + return subagents_fingerprint(jsonl_file) + names = subdir_names.get(parent) + if names is None: + try: + with os.scandir(parent) as entries: + names = {e.name for e in entries if e.is_dir()} + except OSError: + names = set[str]() + subdir_names[parent] = names + return subagents_fingerprint(jsonl_file) if jsonl_file.stem in names else "" + modified: List[Path] = [] for jsonl_file in jsonl_files: row = cached.get(jsonl_file.name) @@ -952,39 +982,9 @@ def get_modified_files(self, jsonl_files: List[Path]) -> List[Path]: modified.append(jsonl_file) continue - # Cache is valid if modification times match (within 1 second - # tolerance) - if abs(source_mtime - row["source_mtime"]) >= 1.0: - modified.append(jsonl_file) - continue - - parent = jsonl_file.parent - if parent.name == "subagents": - # Agent file in a subagents dir: its sidecars sit next to - # it, so the full fingerprint scan is required. - current_fp = subagents_fingerprint(jsonl_file) - else: - names = subdir_names.get(parent) - if names is None: - try: - with os.scandir(parent) as entries: - names = {e.name for e in entries if e.is_dir()} - except OSError: - names = set[str]() - subdir_names[parent] = names - current_fp = ( - subagents_fingerprint(jsonl_file) - if jsonl_file.stem in names - else "" - ) - - # Same NULL-fingerprint rule as is_file_cached(): pre-007 - # rows are accepted only while the file has no sidecars. - cached_fp = row["subagents_fingerprint"] - if cached_fp is None: - if current_fp != "": - modified.append(jsonl_file) - elif cached_fp != current_fp: + if not _cache_row_is_fresh( + row, source_mtime, lambda file=jsonl_file: current_fp(file) + ): modified.append(jsonl_file) return modified