Reuse cached conversation as prefix for dialog-summary helper

copilotj/multiagent/leader_multiagent.py

@@ -5,7 +5,7 @@
 import asyncio
 import json
 import os
-from collections.abc import Awaitable, Callable
+from collections.abc import Awaitable, Callable, Sequence
 from dataclasses import replace
 from typing import Annotated, Any
 
@@ -62,6 +62,7 @@
     build_observation_message,
     build_tool_prompt,
     make_summary_prompt,
+    make_summary_tail,
 )
 from copilotj.multiagent.tools import system_info
 from copilotj.plugin import ClientPluginAPI
@@ -115,6 +116,10 @@ def __init__(
 
         # Per-dialog conversation state. Populated by begin_dialog() and
         # extended by continue_dialog(). Reset at the start of each dialog.
+        # INVARIANT: append-only within a dialog — existing entries are never
+        # mutated in place. Helper calls snapshot this (see dialog_messages /
+        # LeaderDriven._leader_dialog_snapshot) to reuse it as a cache prefix,
+        # which relies on entries being stable once written.
         self._dialog_messages: list[TextMessage] = []
 
         self._apis = apis
@@ -150,6 +155,18 @@ def __init__(
         ]
         self.agents = agents if agents else {}
 
+    @property
+    def dialog_messages(self) -> Sequence[TextMessage]:
+        """Read-only view of the cached conversation turns.
+
+        The underlying ``_dialog_messages`` is append-only within a dialog and
+        reset by ``begin_dialog``; existing entries are never mutated in place.
+        Returns the live reference typed as a read-only ``Sequence`` — callers
+        must not mutate it; for a mutable copy use
+        ``LeaderDriven._leader_dialog_snapshot``.
+        """
+        return self._dialog_messages
+
     def set_save_workflow_handler(
         self,
         handler: SaveWorkflowHandler,
@@ -669,15 +686,37 @@ async def _persist_dialog_to_kb(
         except Exception as kb_e:
             self.log_error(f"[WARNING] KB build failed for dialog {dialog_id}: {kb_e}")
 
+    def _leader_dialog_snapshot(self) -> list[TextMessage]:
+        """Shallow copy of the leader's cached conversation, for reuse as a
+        prompt-cache prefix by helper calls.
+
+        Returns a new list; the ``TextMessage`` entries are shared by reference,
+        which is safe because ``_dialog_messages`` is append-only. Empty when no
+        dialog has started (``begin_dialog`` not yet called) — callers fall back
+        to a standalone prompt in that case.
+        """
+        return list(self.leader_agent.dialog_messages)
+
     async def _generate_dialog_summary(self, dialog_context: dict) -> str | None:
         steps = dialog_context["steps"]
         self.log_info(f"[SUMMARY] Generating dialog summary from {len(steps)} dialog steps...")
 
-        steps_text = json.dumps(steps, indent=2, ensure_ascii=False)
-        summary_prompt = make_summary_prompt(dialog_context["task"], steps_text)
+        # Reuse the leader's cached conversation as the prefix (cache hit on the
+        # whole dialog) and append a short summary instruction; only the tail is
+        # fresh. Falls back to the standalone re-serialized-steps prompt when no
+        # dialog has been cached yet (e.g. pre-begin_dialog edge case).
+        snapshot = self._leader_dialog_snapshot()
+        if snapshot:
+            messages = [
+                *snapshot,
+                TextMessage(role="user", text=make_summary_tail(dialog_context["task"])),
+            ]
+        else:
+            steps_text = json.dumps(steps, indent=2, ensure_ascii=False)
+            messages = [TextMessage(role="user", text=make_summary_prompt(dialog_context["task"], steps_text))]
 
         try:
-            response = await self.model_client.create([TextMessage(role="user", text=summary_prompt)])
+            response = await self.model_client.create(messages)
         except Exception as e:
             self.log_error(f"[ERROR] Error generating dialog context summary: {e}")
             return None

copilotj/multiagent/leader_prompts.py

@@ -22,6 +22,7 @@
     "build_initial_user_message",
     "build_observation_message",
     "make_summary_prompt",
+    "make_summary_tail",
     "make_workflow_definition_prompt",
     "build_tool_prompt",
     "build_available_specialized_agents_prompt",
@@ -500,15 +501,7 @@
 """
 
 
-def make_summary_prompt(task: str, steps_text: str) -> str:
-    return f"""
-You are an expert at compressing a dialog execution trace into concise context for future conversation.
-User ask: {task}
-Execution Steps to Summarize:
-{steps_text}
-Please summarize the ImageJ task execution steps as reusable conversation context.
-Do not generate workflow JSON, workflow steps, or template variables. Workflow saving has a separate prompt.
-
+_SUMMARY_FORMAT_BLOCK = """\
 ## Required Summary Format
 The summary must be comprehensive like:
 <Example>
@@ -526,6 +519,35 @@ def make_summary_prompt(task: str, steps_text: str) -> str:
 """
 
 
+def make_summary_prompt(task: str, steps_text: str) -> str:
+    return f"""
+You are an expert at compressing a dialog execution trace into concise context for future conversation.
+User ask: {task}
+Execution Steps to Summarize:
+{steps_text}
+Please summarize the ImageJ task execution steps as reusable conversation context.
+Do not generate workflow JSON, workflow steps, or template variables. Workflow saving has a separate prompt.
+
+{_SUMMARY_FORMAT_BLOCK}"""
+
+
+def make_summary_tail(task: str) -> str:
+    """Short instruction appended to a leader-conversation snapshot.
+
+    Asks for a reusable summary of the conversation ABOVE (which already holds
+    the full execution trace as assistant/user turns), so no ``steps_text`` is
+    re-serialized. Paired with ``LeaderDriven._leader_dialog_snapshot`` so the
+    prior conversation is the cached prefix and only this tail is fresh.
+    """
+    return (
+        "Summarize the CopilotJ dialog above as reusable conversation context for future turns. "
+        f"The user's original request was: {task}\n"
+        "Do not generate workflow JSON, workflow steps, or template variables. "
+        "Workflow saving has a separate prompt.\n\n"
+        f"{_SUMMARY_FORMAT_BLOCK}"
+    )
+
+
 def build_tool_prompt(tools: list[Tool]) -> str:
     """Build a formatted prompt string for available tools."""
     prompt = "**Available Tools**:\n"

copilotj/test/core/model_client/test_anthropic.py

@@ -148,3 +148,60 @@ def test_anthropic_format_tools_empty():
     """Empty tool list returns NOT_GIVEN."""
     result = AnthropicChatCompletionClient._format_tools([])
     assert result is anthropic.NOT_GIVEN
+
+
+def test_anthropic_format_snapshot_plus_tail_cache_compatible():
+    """Appending a ``user`` tail to a ``user``-ended snapshot keeps every
+    earlier content block identical and relocates BP2 onto the tail.
+
+    This is the unit-testable proxy for the helper-call cache trick: the
+    leader's last request cached ``[system, ..., last_user_turn]`` (BP2 on that
+    last turn). A helper sending ``[*snapshot, user:tail]`` must keep all prior
+    blocks content-identical so the prefix is a cache READ, with only the tail
+    fresh. The consecutive-``user`` merge coalesces the last turn + tail into
+    one message; that must not alter any earlier block's bytes.
+    """
+    snapshot = [
+        TextMessage(role="system", text="Be helpful"),
+        TextMessage(role="user", text="count cells"),
+        TextMessage(role="assistant", text="Thought: ..."),
+        TextMessage(role="user", text="Observation: 42 cells"),  # dialog ends in user
+    ]
+    tail = TextMessage(role="user", text="Summarize the dialog above.")
+
+    snap_system, snap_msgs = AnthropicChatCompletionClient._format_messages(snapshot)
+    help_system, help_msgs = AnthropicChatCompletionClient._format_messages([*snapshot, tail])
+
+    def content_blocks(msgs):
+        """Flatten (type, payload) per content block, ignoring cache_control."""
+        out = []
+        for m in msgs:
+            for b in m["content"]:
+                payload = b.get("text") if b["type"] == "text" else b.get("source")
+                out.append((b["type"], payload))
+        return out
+
+    # BP1 (system) identical -> the system breakpoint still reads.
+    assert help_system == snap_system
+
+    snap_blocks = content_blocks(snap_msgs)
+    help_blocks = content_blocks(help_msgs)
+
+    # Helper = snapshot blocks + exactly one appended tail block; no earlier block changed.
+    assert help_blocks[: len(snap_blocks)] == snap_blocks
+    assert help_blocks[len(snap_blocks)] == ("text", "Summarize the dialog above.")
+    assert len(help_blocks) == len(snap_blocks) + 1
+
+    # BP2 sits on the tail (last content block of the last message); the merged
+    # last message carries the prior user turn + the tail, in order.
+    assert help_msgs[-1]["content"][-1]["text"] == "Summarize the dialog above."
+    assert help_msgs[-1]["content"][0]["text"] == "Observation: 42 cells"
+    assert "cache_control" in help_msgs[-1]["content"][-1]
+    assert "cache_control" not in help_msgs[-1]["content"][0]
+    # Guard Anthropic's 4-breakpoint budget: across all message blocks, exactly
+    # one cache_control (the tail). With the system breakpoint that's 2/4 total,
+    # so the request stays in budget and a stale breakpoint on an earlier block
+    # would fail this assertion instead of silently inflating cost.
+    cached_blocks = [b for m in help_msgs for b in m["content"] if "cache_control" in b]
+    assert len(cached_blocks) == 1
+    assert cached_blocks[0]["text"] == "Summarize the dialog above."

copilotj/test/multiagent/test_helper_cache_prefix.py

@@ -0,0 +1,122 @@
+# SPDX-FileCopyrightText: Copyright contributors to the CopilotJ project.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+"""Tests for reusing the leader's cached conversation as a prompt-cache prefix.
+
+Covers the snapshot primitive and the ``_generate_dialog_summary`` rewrite (#2):
+when a dialog is cached, the summary call appends a short tail to a snapshot of
+``_dialog_messages`` (so the conversation prefix is a cache hit) instead of
+sending a brand-new standalone prompt. When no dialog is cached yet, it falls
+back to the legacy re-serialized-steps prompt.
+"""
+
+import asyncio
+from types import SimpleNamespace
+
+from copilotj.core.message import TextMessage
+from copilotj.core.model_client._types import ModelResponse
+from copilotj.multiagent.leader_multiagent import LeaderDriven
+
+
+def _stub_leaderdriven(dialog_messages, model_response_content="summary text"):
+    """Build a ``LeaderDriven`` with only the attributes that
+    ``_leader_dialog_snapshot`` / ``_generate_dialog_summary`` touch.
+
+    Bypasses the heavy ``__init__`` (which loads agent configs, model client,
+    plugin APIs, etc.) — we only need ``leader_agent``, ``model_client``, and
+    the two log helpers.
+    """
+    ld = LeaderDriven.__new__(LeaderDriven)
+    ld.leader_agent = SimpleNamespace(dialog_messages=list(dialog_messages))
+
+    captured: dict = {}
+
+    async def fake_create(messages, **_kwargs):
+        captured["messages"] = list(messages)
+        return ModelResponse(
+            reasoning_content=None,
+            content=model_response_content,
+            tool_calls=None,
+            finish_reason="stop",
+        )
+
+    ld.model_client = SimpleNamespace(create=fake_create)
+    ld.log_info = lambda *_a, **_k: None
+    ld.log_error = lambda *_a, **_k: None
+    return ld, captured
+
+
+# --------------------------------------------------------------------------- #
+# _leader_dialog_snapshot
+# --------------------------------------------------------------------------- #
+
+
+def test_snapshot_is_a_shallow_copy_not_an_alias():
+    ld, _ = _stub_leaderdriven([TextMessage(role="user", text="hi")])
+    live = ld.leader_agent.dialog_messages
+    snap = ld._leader_dialog_snapshot()
+    assert snap == live
+    assert snap is not live  # new list, so callers can't mutate the leader's state
+    snap.append(TextMessage(role="user", text="mutate"))
+    assert ld.leader_agent.dialog_messages == [TextMessage(role="user", text="hi")]
+
+
+def test_snapshot_empty_before_any_dialog():
+    ld, _ = _stub_leaderdriven([])
+    assert ld._leader_dialog_snapshot() == []
+
+
+# --------------------------------------------------------------------------- #
+# _generate_dialog_summary (#2)
+# --------------------------------------------------------------------------- #
+
+
+def _user_ended_dialog() -> list[TextMessage]:
+    """A dialog that ends in a user turn, like a real ReAct trace."""
+    return [
+        TextMessage(role="system", text="sys"),
+        TextMessage(role="user", text="count cells"),
+        TextMessage(role="assistant", text="Thought: ..."),
+        TextMessage(role="user", text="Observation: 42 cells"),
+    ]
+
+
+def test_summary_uses_snapshot_prefix_when_dialog_cached():
+    dialog = _user_ended_dialog()
+    ld, captured = _stub_leaderdriven(dialog)
+    steps = [{"thought": "do thing", "name": "run_macro", "response": "ok"}]
+
+    result = asyncio.run(ld._generate_dialog_summary({"task": "count cells", "steps": steps}))
+
+    msgs = captured["messages"]
+    # The cached conversation is reused verbatim as the prefix ...
+    assert msgs[: len(dialog)] == dialog
+    # ... followed by exactly one appended user tail.
+    assert len(msgs) == len(dialog) + 1
+    assert msgs[-1].role == "user"
+    assert "count cells" in msgs[-1].text
+    # The tail must NOT re-serialize the steps JSON (that's the whole point).
+    assert "run_macro" not in msgs[-1].text
+    assert '"thought"' not in msgs[-1].text
+    assert result == "summary text"
+
+
+def test_summary_falls_back_to_standalone_prompt_when_snapshot_empty():
+    ld, captured = _stub_leaderdriven([])
+    steps = [{"thought": "do thing", "name": "run_macro", "response": "ok"}]
+
+    asyncio.run(ld._generate_dialog_summary({"task": "count cells", "steps": steps}))
+
+    msgs = captured["messages"]
+    # Legacy path: a single user message whose body embeds the steps JSON.
+    assert len(msgs) == 1
+    assert msgs[0].role == "user"
+    assert "count cells" in msgs[0].text
+    assert "run_macro" in msgs[0].text  # steps_text present in fallback
+
+
+def test_summary_returns_none_on_empty_model_response():
+    ld, _ = _stub_leaderdriven(_user_ended_dialog(), model_response_content=None)
+    result = asyncio.run(ld._generate_dialog_summary({"task": "t", "steps": []}))
+    assert result is None