From f303a5f1f56046358a3740472479fd3fa52ce7f4 Mon Sep 17 00:00:00 2001 From: Daniel Demmel Date: Tue, 7 Jul 2026 00:49:54 +0000 Subject: [PATCH 1/2] Fix perpetual re-regeneration of projects with up-to-date caches MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Running the full generation twice in a row kept "regenerating" the same projects every run even though nothing changed. Reproduced on a real 2GB archive copy: 7 projects re-rendered their combined pages and 2 projects re-flagged one session on every single run, adding ~70s per run. Two root causes, both self-inconsistent staleness bookkeeping: 1. Paginated combined HTML (reason: message_count_changed). _generate_paginated_html cached len(page_messages) — the raw rendered entry count, which includes Summary/AiTitle/Attachment entries — but is_page_stale() compares that against SUM(sessions.message_count), computed by compute_session_data, which skips those types. Any project where the two counting rules diverge was permanently stale. Fix: cache the sessions-table sum for the page (same source the check reads), exactly as last_timestamp already did. The rendered count still feeds the on-page stats display. 2. Ghost sessions (reason: not_cached). A fork session whose own messages all deduplicate into the original session survives only through its agent sidechains ({trunk}#agent-{id} sessionIds). compute_session_data coalesces those to the trunk and writes a sessions-table row, but _generate_individual_session_files dropped agent ids from its render set instead of coalescing — so the session was never rendered, its html_cache row never written, and get_stale_sessions flagged it forever. Fix: coalesce with get_parent_session_id in the render set (generate_session already accepts {sid}#agent-* entries), and likewise in the pagination session filter so such sessions claim a page slot instead of their messages silently vanishing from the combined output. Both fixes self-heal: the first run after upgrading regenerates the affected projects once, writes consistent bookkeeping, and every subsequent run is fully cached. Verified on the archive copy: run 1 regenerated the 9 known offenders, run 2 reported all 63 projects cached (8.4s total, previously ~80s with 9 spurious regenerations). Co-Authored-By: Claude Fable 5 --- claude_code_log/converter.py | 45 +++++++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/claude_code_log/converter.py b/claude_code_log/converter.py index 3ca79363..a5319341 100644 --- a/claude_code_log/converter.py +++ b/claude_code_log/converter.py @@ -1689,10 +1689,19 @@ def _generate_paginated_html( total_output_tokens = 0 total_cache_creation_tokens = 0 total_cache_read_tokens = 0 + # The cached count must be the sessions-table sum, because + # is_page_stale() compares it against SUM(sessions.message_count) + # on the next run. len(page_messages) counts what's rendered + # (including Summary/AiTitle/Attachment entries that + # compute_session_data skips), so caching it left any page where + # the two rules diverge permanently "message_count_changed" — + # regenerating on every single run. + page_sessions_message_count = 0 for session_id in page_session_ids: if session_id in session_data: s = session_data[session_id] + page_sessions_message_count += s.message_count if s.first_timestamp and ( first_timestamp is None or s.first_timestamp < first_timestamp ): @@ -1778,7 +1787,7 @@ def _generate_paginated_html( html_path=html_path, page_size_config=page_size, session_ids=page_session_ids, - message_count=page_message_count, + message_count=page_sessions_message_count, first_timestamp=first_timestamp, last_timestamp=last_timestamp, total_input_tokens=total_input_tokens, @@ -2047,14 +2056,17 @@ def convert_jsonl_to( # Use cached session data if available, otherwise build from messages if cached_data is not None: warmup_session_ids = get_warmup_session_ids(messages) + # Coalesce agent sessionIds to their trunk (same rule as + # compute_session_data and _generate_individual_session_files): + # a fork session surviving only through its agent sidechains + # must still claim its page slot, or its messages are grouped + # under a session no page owns and vanish from the output. current_session_ids: set[str] = set() for message in messages: session_id = getattr(message, "sessionId", "") - if ( - session_id - and session_id not in warmup_session_ids - and not is_agent_session(session_id) - ): + if session_id: + session_id = get_parent_session_id(session_id) + if session_id and session_id not in warmup_session_ids: current_session_ids.add(session_id) session_data = { session_id: session_cache @@ -2401,16 +2413,23 @@ def _generate_individual_session_files( # Pre-compute warmup sessions to exclude them warmup_session_ids = get_warmup_session_ids(messages) - # Find all unique session IDs (excluding warmup and agent sessions) + # Find all unique session IDs (excluding warmup sessions). Agent + # sessionIds ({trunk}#agent-{id}) coalesce to their trunk rather + # than being dropped: compute_session_data() coalesces the same + # way when it writes the sessions table, and a fork session whose + # own messages were all deduplicated into the original session can + # survive ONLY through its agent sidechains. Dropping those left + # such a session in the sessions table but never rendered, so + # get_stale_sessions() flagged it "not_cached" on every run — + # regenerating the project forever without ever writing the file. session_ids: set[str] = set() for message in messages: if hasattr(message, "sessionId"): - session_id: str = getattr(message, "sessionId") - if ( - session_id - and session_id not in warmup_session_ids - and not is_agent_session(session_id) - ): + raw_session_id: str = getattr(message, "sessionId") + if not raw_session_id: + continue + session_id = get_parent_session_id(raw_session_id) + if session_id and session_id not in warmup_session_ids: session_ids.add(session_id) # Get session data from cache for better titles From 1921c8f1beea83a095ac9e136a97df107fb96ac2 Mon Sep 17 00:00:00 2001 From: Daniel Demmel Date: Tue, 7 Jul 2026 09:28:25 +0100 Subject: [PATCH 2/2] PR review --- claude_code_log/converter.py | 55 +++++++++++------------------------- claude_code_log/utils.py | 39 +++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 39 deletions(-) diff --git a/claude_code_log/converter.py b/claude_code_log/converter.py index a5319341..ea122d05 100644 --- a/claude_code_log/converter.py +++ b/claude_code_log/converter.py @@ -18,6 +18,8 @@ from .cache import CacheManager from .utils import ( + coalesce_trunk_session_id, + collect_trunk_session_ids, format_timestamp_range, get_parent_session_id, get_project_display_name, @@ -1431,8 +1433,8 @@ def compute_session_data( ): continue - session_id = get_parent_session_id(getattr(message, "sessionId", "")) - if not session_id or session_id in warmup: + session_id = coalesce_trunk_session_id(message, warmup) + if not session_id: continue if session_id not in result: @@ -2055,19 +2057,9 @@ def convert_jsonl_to( assert cache_manager is not None # Ensured by use_pagination condition # Use cached session data if available, otherwise build from messages if cached_data is not None: - warmup_session_ids = get_warmup_session_ids(messages) - # Coalesce agent sessionIds to their trunk (same rule as - # compute_session_data and _generate_individual_session_files): - # a fork session surviving only through its agent sidechains - # must still claim its page slot, or its messages are grouped - # under a session no page owns and vanish from the output. - current_session_ids: set[str] = set() - for message in messages: - session_id = getattr(message, "sessionId", "") - if session_id: - session_id = get_parent_session_id(session_id) - if session_id and session_id not in warmup_session_ids: - current_session_ids.add(session_id) + current_session_ids = collect_trunk_session_ids( + messages, get_warmup_session_ids(messages) + ) session_data = { session_id: session_cache for session_id, session_cache in cached_data.sessions.items() @@ -2410,27 +2402,14 @@ def _generate_individual_session_files( ext = get_file_extension(format) suffix = _variant_suffix(detail, compact, format, no_timestamps, no_recaps) - # Pre-compute warmup sessions to exclude them - warmup_session_ids = get_warmup_session_ids(messages) - - # Find all unique session IDs (excluding warmup sessions). Agent - # sessionIds ({trunk}#agent-{id}) coalesce to their trunk rather - # than being dropped: compute_session_data() coalesces the same - # way when it writes the sessions table, and a fork session whose - # own messages were all deduplicated into the original session can - # survive ONLY through its agent sidechains. Dropping those left - # such a session in the sessions table but never rendered, so - # get_stale_sessions() flagged it "not_cached" on every run — + # Find all unique session IDs, excluding warmup sessions and + # coalescing agent sessionIds to their trunk — same rule as + # compute_session_data() when it writes the sessions table. + # Dropping (rather than coalescing) agent ids left agent-sidechain- + # only sessions in the sessions table but never rendered, so + # get_stale_sessions() flagged them "not_cached" on every run — # regenerating the project forever without ever writing the file. - session_ids: set[str] = set() - for message in messages: - if hasattr(message, "sessionId"): - raw_session_id: str = getattr(message, "sessionId") - if not raw_session_id: - continue - session_id = get_parent_session_id(raw_session_id) - if session_id and session_id not in warmup_session_ids: - session_ids.add(session_id) + session_ids = collect_trunk_session_ids(messages, get_warmup_session_ids(messages)) # Get session data from cache for better titles session_data: dict[str, Any] = {} @@ -3173,10 +3152,8 @@ def _rel_to_index(p: Path) -> str: ), ): continue - if not hasattr(_msg, "sessionId"): - continue - _sid = get_parent_session_id(getattr(_msg, "sessionId", "")) - if not _sid or _sid in warmup_for_teams: + _sid = coalesce_trunk_session_id(_msg, warmup_for_teams) + if not _sid: continue _tn = getattr(_msg, "teamName", None) if _tn and _sid not in team_name_per_session: diff --git a/claude_code_log/utils.py b/claude_code_log/utils.py index 91d60450..b676ab08 100644 --- a/claude_code_log/utils.py +++ b/claude_code_log/utils.py @@ -692,6 +692,45 @@ def get_warmup_session_ids(messages: list[TranscriptEntry]) -> set[str]: return warmup_sessions +def coalesce_trunk_session_id( + message: TranscriptEntry, warmup_session_ids: set[str] +) -> str: + """Return ``message``'s sessionId coalesced to its trunk, or ``""``. + + The shared normalization every session-collection site must agree + on: agent synthetic sessionIds (``{trunk}#agent-{id}``) coalesce to + their trunk via ``get_parent_session_id``, and warmup-only sessions + are excluded. Returns ``""`` for entries without a sessionId (e.g. + ``SummaryTranscriptEntry``) or whose trunk is a warmup session. + """ + session_id = getattr(message, "sessionId", "") or "" + if session_id: + session_id = get_parent_session_id(session_id) + if session_id in warmup_session_ids: + return "" + return session_id + + +def collect_trunk_session_ids( + messages: list[TranscriptEntry], warmup_session_ids: set[str] +) -> set[str]: + """Collect the distinct trunk sessionIds present in ``messages``. + + Coalescing (rather than dropping) agent sessionIds matters: a fork + session whose own messages were all deduplicated into the original + session can survive ONLY through its agent sidechains, and it must + still claim its page/file slot or its messages are grouped under a + session no output owns and vanish (regenerating forever — see + ``_generate_individual_session_files``). + """ + session_ids: set[str] = set() + for message in messages: + session_id = coalesce_trunk_session_id(message, warmup_session_ids) + if session_id: + session_ids.add(session_id) + return session_ids + + # Artifact favicons are 1-2 emoji (ZWJ sequences can span ~a dozen code # points); anything longer is malformed input not worth echoing inline. # Shared by the HTML and Markdown renderers so the cap can't drift.